From 20a5afa84e8d08a1cf586ff799baae65639c2c1a Mon Sep 17 00:00:00 2001 From: Egor Duplensky Date: Fri, 10 Dec 2021 13:53:23 +0300 Subject: [PATCH] [CPU] Fix mixing VEX and non-VEX instructions (#8658) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Quote: The Skylake microarchitecture implements a different state machine than prior generations to manage the YMM state transition associated with mixing SSE and AVX instructions. It no longer saves the entire upper YMM state when executing an SSE instruction when in “Modified and Unsaved” state, but saves the upper bits of individual register. As a result, mixing SSE and AVX instructions will experience a penalty associated with partial register dependency of the destination registers being used and additional blend operation on the upper bits of the destination registers. Such type of penalties have a huge impact on openvino's and oneDNN's kernels. Basically the mixing of VEX and non-VEX instructions should be avoided. --- .../emitters/jit_eltwise_emitters.cpp | 6 +- .../emitters/jit_load_store_emitters.cpp | 41 ++++++------ .../nodes/common/permute_kernel.cpp | 20 +++--- .../nodes/mkldnn_bin_conv_node.cpp | 10 +-- .../nodes/mkldnn_def_conv_node.cpp | 66 +++++++++---------- .../nodes/mkldnn_eltwise_node.cpp | 8 +-- .../nodes/mkldnn_fake_quantize_node.cpp | 34 +++++----- .../mkldnn_plugin/nodes/mkldnn_input_node.cpp | 14 ++-- .../nodes/mkldnn_interpolate_node.cpp | 16 ++--- .../mkldnn_plugin/nodes/mkldnn_mvn_node.cpp | 20 +++--- .../nodes/mkldnn_normalize_node.cpp | 30 ++++----- .../nodes/mkldnn_region_yolo_node.cpp | 8 +-- .../nodes/mkldnn_roi_pooling_node.cpp | 4 +- inference-engine/thirdparty/mkl-dnn | 2 +- 14 files changed, 138 insertions(+), 141 deletions(-) diff --git a/inference-engine/src/mkldnn_plugin/emitters/jit_eltwise_emitters.cpp b/inference-engine/src/mkldnn_plugin/emitters/jit_eltwise_emitters.cpp index ae759a456fb..03eb2fdca44 100644 --- a/inference-engine/src/mkldnn_plugin/emitters/jit_eltwise_emitters.cpp +++ b/inference-engine/src/mkldnn_plugin/emitters/jit_eltwise_emitters.cpp @@ -85,8 +85,8 @@ void jit_mul_add_emitter::emit_isa(const std::vector &in_vec_idxs, const if (isa == cpu::x64::sse41) { h->uni_vmovups(vmm_dst, vmm_src0); - h->mulps(vmm_dst, vmm_src1); - h->addps(vmm_dst, vmm_src2); + h->uni_vmulps(vmm_dst, vmm_dst, vmm_src1); + h->uni_vaddps(vmm_dst, vmm_dst, vmm_src2); } else { Vmm vmm_mul0; if (vmm_dst.getIdx() == vmm_src0.getIdx()) { @@ -656,7 +656,7 @@ void jit_equal_emitter::emit_isa(const std::vector &in_vec_idxs, const s } else if (isa == cpu::x64::avx2) { h->vcmpeqps(vmm_aux0, vmm_src0, vmm_src1); h->uni_vmovups(vmm_dst, table_val("zero")); - h->vblendvps(vmm_dst, vmm_dst, table_val("one"), vmm_aux0); + h->uni_vblendvps(vmm_dst, vmm_dst, table_val("one"), vmm_aux0); } else { h->vcmpps(k_mask, vmm_src0, vmm_src1, _cmp_eq_oq); h->uni_vmovups(vmm_dst, table_val("zero")); diff --git a/inference-engine/src/mkldnn_plugin/emitters/jit_load_store_emitters.cpp b/inference-engine/src/mkldnn_plugin/emitters/jit_load_store_emitters.cpp index 4d1e3819394..f6ad72a3e36 100644 --- a/inference-engine/src/mkldnn_plugin/emitters/jit_load_store_emitters.cpp +++ b/inference-engine/src/mkldnn_plugin/emitters/jit_load_store_emitters.cpp @@ -190,7 +190,7 @@ void jit_load_emitter::load_bytes(const Vmm &vmm, const Xbyak::Reg64 ®, int o } if (bytes_to_load >= 8 && bytes_to_load < 16) - h->pinsrq(xmm, addr(start_bytes), 0); + h->uni_vpinsrq(xmm, xmm, addr(start_bytes), 0); else if (bytes_to_load == 16) h->uni_vmovdqu(xmm, addr(start_bytes)); @@ -202,17 +202,17 @@ void jit_load_emitter::load_bytes(const Vmm &vmm, const Xbyak::Reg64 ®, int o h->uni_vpinsrw(xmm, xmm, addr(start_bytes), 0); h->uni_vpinsrb(xmm, xmm, addr(start_bytes + 2), 2); break; - case 4: h->pinsrd(xmm, addr(start_bytes), 0); break; + case 4: h->uni_vpinsrd(xmm, xmm, addr(start_bytes), 0); break; case 5: - h->pinsrd(xmm, addr(start_bytes), 0); + h->uni_vpinsrd(xmm, xmm, addr(start_bytes), 0); h->uni_vpinsrb(xmm, xmm, addr(start_bytes + 4), 4); break; case 6: - h->pinsrd(xmm, addr(start_bytes), 0); + h->uni_vpinsrd(xmm, xmm, addr(start_bytes), 0); h->uni_vpinsrw(xmm, xmm, addr(start_bytes + 4), 2); break; case 7: - h->pinsrd(xmm, addr(start_bytes), 0); + h->uni_vpinsrd(xmm, xmm, addr(start_bytes), 0); h->uni_vpinsrw(xmm, xmm, addr(start_bytes + 4), 2); h->uni_vpinsrb(xmm, xmm, addr(start_bytes + 6), 6); break; @@ -223,17 +223,17 @@ void jit_load_emitter::load_bytes(const Vmm &vmm, const Xbyak::Reg64 ®, int o h->uni_vpinsrw(xmm, xmm, addr(start_bytes + 8), 4); h->uni_vpinsrb(xmm, xmm, addr(start_bytes + 10), 10); break; - case 12: h->pinsrd(xmm, addr(start_bytes + 8), 2); break; + case 12: h->uni_vpinsrd(xmm, xmm, addr(start_bytes + 8), 2); break; case 13: - h->pinsrd(xmm, addr(start_bytes + 8), 2); + h->uni_vpinsrd(xmm, xmm, addr(start_bytes + 8), 2); h->uni_vpinsrb(xmm, xmm, addr(start_bytes + 12), 12); break; case 14: - h->pinsrd(xmm, addr(start_bytes + 8), 2); + h->uni_vpinsrd(xmm, xmm, addr(start_bytes + 8), 2); h->uni_vpinsrw(xmm, xmm, addr(start_bytes + 12), 6); break; case 15: - h->pinsrd(xmm, addr(start_bytes + 8), 2); + h->uni_vpinsrd(xmm, xmm, addr(start_bytes + 8), 2); h->uni_vpinsrw(xmm, xmm, addr(start_bytes + 12), 6); h->uni_vpinsrb(xmm, xmm, addr(start_bytes + 14), 14); break; @@ -465,10 +465,7 @@ template if (is_xmm || is_ymm) { uint8 imm = 1; imm = ~((imm << load_num) - imm); // shift load_num bit - if (is_xmm) - h->blendps(vmm, table_val(fill_value), imm); - else - h->vblendps(vmm, vmm, table_val(fill_value), imm); + h->uni_vblendps(vmm, vmm, table_val(fill_value), imm); } else if (is_zmm) { uint64_t tail_mask = 1; tail_mask = ~((tail_mask << load_num) - tail_mask); @@ -673,7 +670,7 @@ template } if (bytes_to_store >= 8 && bytes_to_store < 16) - h->pextrq(addr(start_bytes), xmm, 0); + h->uni_vpextrq(addr(start_bytes), xmm, 0); else if (bytes_to_store == 16) h->uni_vmovdqu(addr(start_bytes), xmm); @@ -687,17 +684,17 @@ template h->uni_vpextrw(addr(start_bytes), xmm, 0); h->uni_vpextrb(addr(start_bytes + 2), xmm, 2); break; - case 4: h->pextrd(addr(start_bytes), xmm, 0); break; + case 4: h->uni_vpextrd(addr(start_bytes), xmm, 0); break; case 5: - h->pextrd(addr(start_bytes), xmm, 0); + h->uni_vpextrd(addr(start_bytes), xmm, 0); h->uni_vpextrb(addr(start_bytes + 4), xmm, 4); break; case 6: - h->pextrd(addr(start_bytes), xmm, 0); + h->uni_vpextrd(addr(start_bytes), xmm, 0); h->uni_vpextrw(addr(start_bytes + 4), xmm, 2); break; case 7: - h->pextrd(addr(start_bytes), xmm, 0); + h->uni_vpextrd(addr(start_bytes), xmm, 0); h->uni_vpextrw(addr(start_bytes + 4), xmm, 2); h->uni_vpextrb(addr(start_bytes + 6), xmm, 6); break; @@ -708,17 +705,17 @@ template h->uni_vpextrw(addr(start_bytes + 8), xmm, 4); h->uni_vpextrb(addr(start_bytes + 10), xmm, 10); break; - case 12: h->pextrd(addr(start_bytes + 8), xmm, 2); break; + case 12: h->uni_vpextrd(addr(start_bytes + 8), xmm, 2); break; case 13: - h->pextrd(addr(start_bytes + 8), xmm, 2); + h->uni_vpextrd(addr(start_bytes + 8), xmm, 2); h->uni_vpextrb(addr(start_bytes + 12), xmm, 12); break; case 14: - h->pextrd(addr(start_bytes + 8), xmm, 2); + h->uni_vpextrd(addr(start_bytes + 8), xmm, 2); h->uni_vpextrw(addr(start_bytes + 12), xmm, 6); break; case 15: - h->pextrd(addr(start_bytes + 8), xmm, 2); + h->uni_vpextrd(addr(start_bytes + 8), xmm, 2); h->uni_vpextrw(addr(start_bytes + 12), xmm, 6); h->uni_vpextrb(addr(start_bytes + 14), xmm, 14); break; diff --git a/inference-engine/src/mkldnn_plugin/nodes/common/permute_kernel.cpp b/inference-engine/src/mkldnn_plugin/nodes/common/permute_kernel.cpp index 4babccb5186..e5a079dfb92 100644 --- a/inference-engine/src/mkldnn_plugin/nodes/common/permute_kernel.cpp +++ b/inference-engine/src/mkldnn_plugin/nodes/common/permute_kernel.cpp @@ -47,21 +47,21 @@ struct jit_uni_permute_kernel_f32 : public jit_uni_permute_kernel, public jit_ge void load(const Xbyak::Xmm &xmm, const Xbyak::Address &addr) { switch (jcp.data_size) { - case 16: movups(xmm, addr); break; - case 8: movsd(xmm, addr); break; - case 4: movss(xmm, addr); break; - case 2: pinsrw(xmm, addr, 0x0); break; - case 1: pinsrb(xmm, addr, 0x0); break; + case 16: uni_vmovups(xmm, addr); break; + case 8: uni_vmovsd(xmm, addr); break; + case 4: uni_vmovss(xmm, addr); break; + case 2: uni_vpinsrw(xmm, xmm, addr, 0x0); break; + case 1: uni_vpinsrb(xmm, xmm, addr, 0x0); break; } } void store(const Xbyak::Address &addr, const Xbyak::Xmm &xmm) { switch (jcp.data_size) { - case 16: movups(addr, xmm); break; - case 8: movsd(addr, xmm); break; - case 4: movss(addr, xmm); break; - case 2: pextrw(addr, xmm, 0x0); break; - case 1: pextrb(addr, xmm, 0x0); break; + case 16: uni_vmovups(addr, xmm); break; + case 8: uni_vmovsd(addr, xmm); break; + case 4: uni_vmovss(addr, xmm); break; + case 2: uni_vpextrw(addr, xmm, 0x0); break; + case 1: uni_vpextrb(addr, xmm, 0x0); break; } } diff --git a/inference-engine/src/mkldnn_plugin/nodes/mkldnn_bin_conv_node.cpp b/inference-engine/src/mkldnn_plugin/nodes/mkldnn_bin_conv_node.cpp index b44b830495f..079421635ec 100644 --- a/inference-engine/src/mkldnn_plugin/nodes/mkldnn_bin_conv_node.cpp +++ b/inference-engine/src/mkldnn_plugin/nodes/mkldnn_bin_conv_node.cpp @@ -216,7 +216,7 @@ private: case memory::data_type::s32: if (scalar_load) { mov(reg_tmp_32, op); - movq(xmm_in, reg_tmp_64); + uni_vmovq(xmm_in, reg_tmp_64); } else { uni_vmovups(vmm_in, op); } @@ -224,7 +224,7 @@ private: case memory::data_type::s8: if (scalar_load) { movsx(reg_tmp_32, op); - movq(xmm_in, reg_tmp_64); + uni_vmovq(xmm_in, reg_tmp_64); } else { uni_vpmovsxbd(vmm_in, op); } @@ -232,7 +232,7 @@ private: case memory::data_type::u8: if (scalar_load) { movzx(reg_tmp_32, op); - movq(xmm_in, reg_tmp_64); + uni_vmovq(xmm_in, reg_tmp_64); } else { uni_vpmovzxbd(vmm_in, op); } @@ -542,7 +542,7 @@ private: if (jcp_.exclude_pad) { mov(reg_shift, kw_padding[jj]); imul(reg_shift, reg_tmp_32); - movq(Xmm(vmm_shift.getIdx()), reg_shift); + uni_vmovq(Xmm(vmm_shift.getIdx()), reg_shift); uni_vbroadcastss(vmm_shift, Xmm(vmm_shift.getIdx())); uni_vcvtdq2ps(vmm_shift, vmm_shift); } @@ -613,7 +613,7 @@ private: } else { Ymm ymm_prev_dst = Ymm(vmm_sum.getIdx()); vperm2i128(ymm_prev_dst, ymm_prev_dst, ymm_prev_dst, 0x01); - vpslldq(vmm_sum, vmm_sum, (oc - jcp_.oc_block / 2) * sizeof(float)); + uni_vpslldq(vmm_sum, vmm_sum, (oc - jcp_.oc_block / 2) * sizeof(float)); } uni_vaddps(vmm_dst, vmm_dst, vmm_sum); diff --git a/inference-engine/src/mkldnn_plugin/nodes/mkldnn_def_conv_node.cpp b/inference-engine/src/mkldnn_plugin/nodes/mkldnn_def_conv_node.cpp index 7890ac0428d..fc0722d6ff6 100644 --- a/inference-engine/src/mkldnn_plugin/nodes/mkldnn_def_conv_node.cpp +++ b/inference-engine/src/mkldnn_plugin/nodes/mkldnn_def_conv_node.cpp @@ -318,10 +318,10 @@ private: size_t ind_off_lh = ind_off_hl + 1; size_t ind_off_ll = ind_off_lh + 1; - movq(xmm_v1_off, qword[aux_reg_sampled_offs + ind_off_ll * jcp_.typesize_sampled_offsets]); - movq(xmm_v2_off, qword[aux_reg_sampled_offs + ind_off_hl * jcp_.typesize_sampled_offsets]); - movq(xmm_v3_off, qword[aux_reg_sampled_offs + ind_off_lh * jcp_.typesize_sampled_offsets]); - movq(xmm_v4_off, qword[aux_reg_sampled_offs + ind_off_hh * jcp_.typesize_sampled_offsets]); + uni_vmovq(xmm_v1_off, qword[aux_reg_sampled_offs + ind_off_ll * jcp_.typesize_sampled_offsets]); + uni_vmovq(xmm_v2_off, qword[aux_reg_sampled_offs + ind_off_hl * jcp_.typesize_sampled_offsets]); + uni_vmovq(xmm_v3_off, qword[aux_reg_sampled_offs + ind_off_lh * jcp_.typesize_sampled_offsets]); + uni_vmovq(xmm_v4_off, qword[aux_reg_sampled_offs + ind_off_hh * jcp_.typesize_sampled_offsets]); // w's computation uni_vbroadcastss(vmm_w1, dword[aux_reg_sampled_wei + ind_off_ll * jcp_.typesize_sampled_wei]); @@ -339,29 +339,29 @@ private: size_t input_buffer_off = (size_t) kh * jcp_.kw * jcp_.ic + kw * jcp_.ic; - pmovsxdq(xmm_v1_off, xmm_v1_off); - movq(reg_tmp_64, xmm_v1_off); + uni_vpmovsxdq(xmm_v1_off, xmm_v1_off); + uni_vmovq(reg_tmp_64, xmm_v1_off); imul(reg_tmp_64, reg_tmp_64, jcp_.ic * jcp_.typesize_in); add(reg_tmp_64, aux2_reg_input); uni_vmovups(vmm_v1, ptr[reg_tmp_64]); uni_vmulps(vmm_v1, vmm_v1, vmm_w1); - pmovsxdq(xmm_v2_off, xmm_v2_off); - movq(reg_tmp_64, xmm_v2_off); + uni_vpmovsxdq(xmm_v2_off, xmm_v2_off); + uni_vmovq(reg_tmp_64, xmm_v2_off); imul(reg_tmp_64, reg_tmp_64, jcp_.ic * jcp_.typesize_in); add(reg_tmp_64, aux2_reg_input); uni_vmovups(vmm_v2, ptr[reg_tmp_64]); uni_vmulps(vmm_v2, vmm_v2, vmm_w2); - pmovsxdq(xmm_v3_off, xmm_v3_off); - movq(reg_tmp_64, xmm_v3_off); + uni_vpmovsxdq(xmm_v3_off, xmm_v3_off); + uni_vmovq(reg_tmp_64, xmm_v3_off); imul(reg_tmp_64, reg_tmp_64, jcp_.ic * jcp_.typesize_in); add(reg_tmp_64, aux2_reg_input); uni_vmovups(vmm_v3, ptr[reg_tmp_64]); uni_vmulps(vmm_v3, vmm_v3, vmm_w3); - pmovsxdq(xmm_v4_off, xmm_v4_off); - movq(reg_tmp_64, xmm_v4_off); + uni_vpmovsxdq(xmm_v4_off, xmm_v4_off); + uni_vmovq(reg_tmp_64, xmm_v4_off); imul(reg_tmp_64, reg_tmp_64, jcp_.ic * jcp_.typesize_in); add(reg_tmp_64, aux2_reg_input); uni_vmovups(vmm_v4, ptr[reg_tmp_64]); @@ -384,38 +384,38 @@ private: jl(loop_end, T_NEAR); size_t input_buffer_off = (size_t) kh * jcp_.kw * jcp_.ic + kw * jcp_.ic; - pmovsxdq(xmm_v1_off, xmm_v1_off); - movq(reg_tmp_64, xmm_v1_off); + uni_vpmovsxdq(xmm_v1_off, xmm_v1_off); + uni_vmovq(reg_tmp_64, xmm_v1_off); imul(reg_tmp_64, reg_tmp_64, jcp_.ic * jcp_.typesize_in); add(reg_tmp_64, aux2_reg_input); - movss(xmm_v1, ptr[reg_tmp_64]); - mulss(xmm_v1, xmm_w1); + uni_vmovss(xmm_v1, ptr[reg_tmp_64]); + uni_vmulss(xmm_v1, xmm_v1, xmm_w1); - pmovsxdq(xmm_v2_off, xmm_v2_off); - movq(reg_tmp_64, xmm_v2_off); + uni_vpmovsxdq(xmm_v2_off, xmm_v2_off); + uni_vmovq(reg_tmp_64, xmm_v2_off); imul(reg_tmp_64, reg_tmp_64, jcp_.ic * jcp_.typesize_in); add(reg_tmp_64, aux2_reg_input); - movss(xmm_v2, ptr[reg_tmp_64]); - mulss(xmm_v2, xmm_w2); + uni_vmovss(xmm_v2, ptr[reg_tmp_64]); + uni_vmulss(xmm_v2, xmm_v2, xmm_w2); - pmovsxdq(xmm_v3_off, xmm_v3_off); - movq(reg_tmp_64, xmm_v3_off); + uni_vpmovsxdq(xmm_v3_off, xmm_v3_off); + uni_vmovq(reg_tmp_64, xmm_v3_off); imul(reg_tmp_64, reg_tmp_64, jcp_.ic * jcp_.typesize_in); add(reg_tmp_64, aux2_reg_input); - movss(xmm_v3, ptr[reg_tmp_64]); - mulss(xmm_v3, xmm_w3); + uni_vmovss(xmm_v3, ptr[reg_tmp_64]); + uni_vmulss(xmm_v3, xmm_v3, xmm_w3); - pmovsxdq(xmm_v4_off, xmm_v4_off); - movq(reg_tmp_64, xmm_v4_off); + uni_vpmovsxdq(xmm_v4_off, xmm_v4_off); + uni_vmovq(reg_tmp_64, xmm_v4_off); imul(reg_tmp_64, reg_tmp_64, jcp_.ic * jcp_.typesize_in); add(reg_tmp_64, aux2_reg_input); - movss(xmm_v4, ptr[reg_tmp_64]); - mulss(xmm_v4, xmm_w4); + uni_vmovss(xmm_v4, ptr[reg_tmp_64]); + uni_vmulss(xmm_v4, xmm_v4, xmm_w4); - addss(xmm_v1, xmm_v2); - addss(xmm_v1, xmm_v3); - addss(xmm_v1, xmm_v4); - movss(ptr[aux3_reg_input_buffer + input_buffer_off * jcp_.typesize_in], xmm_v1); + uni_vaddss(xmm_v1, xmm_v1, xmm_v2); + uni_vaddss(xmm_v1, xmm_v1, xmm_v3); + uni_vaddss(xmm_v1, xmm_v1, xmm_v4); + uni_vmovss(ptr[aux3_reg_input_buffer + input_buffer_off * jcp_.typesize_in], xmm_v1); add(aux2_reg_input, jcp_.typesize_in); add(aux3_reg_input_buffer, jcp_.typesize_in); @@ -477,7 +477,7 @@ private: } else { for (int oc = 0; oc < tail_size; oc++) { size_t out_off = (size_t) ow * jcp_.oc + oc + r * (jcp_.oc_block / 2); - movq(reg_tmp_64, xmm_dst); + uni_vmovq(reg_tmp_64, xmm_dst); mov(ptr[aux_reg_output + out_off * jcp_.typesize_out], reg_tmp_32); if (isa == cpu::x64::sse41) { diff --git a/inference-engine/src/mkldnn_plugin/nodes/mkldnn_eltwise_node.cpp b/inference-engine/src/mkldnn_plugin/nodes/mkldnn_eltwise_node.cpp index c32932b7d62..353a022faf9 100644 --- a/inference-engine/src/mkldnn_plugin/nodes/mkldnn_eltwise_node.cpp +++ b/inference-engine/src/mkldnn_plugin/nodes/mkldnn_eltwise_node.cpp @@ -610,7 +610,7 @@ private: switch (src_prc) { case Precision::FP32: case Precision::I32: - movss(xmm_src, op); + uni_vmovss(xmm_src, op); break; case Precision::BF16: uni_vpinsrw(xmm_src, xmm_src, op, 0); @@ -626,11 +626,11 @@ private: break; case Precision::I8: movsx(reg_tmp_32, op); - movq(xmm_src, reg_tmp_64); + uni_vmovq(xmm_src, reg_tmp_64); break; case Precision::U8: movzx(reg_tmp_32, op); - movq(xmm_src, reg_tmp_64); + uni_vmovq(xmm_src, reg_tmp_64); break; default: assert(!"unknown src_prc"); @@ -757,7 +757,7 @@ private: switch (dst_prc) { case Precision::FP32: case Precision::I32: - movss(op, xmm_dst); + uni_vmovss(op, xmm_dst); break; case Precision::BF16: uni_vpsrld(xmm_dst, xmm_dst, 16); diff --git a/inference-engine/src/mkldnn_plugin/nodes/mkldnn_fake_quantize_node.cpp b/inference-engine/src/mkldnn_plugin/nodes/mkldnn_fake_quantize_node.cpp index f8ee928c64f..319a41528e6 100644 --- a/inference-engine/src/mkldnn_plugin/nodes/mkldnn_fake_quantize_node.cpp +++ b/inference-engine/src/mkldnn_plugin/nodes/mkldnn_fake_quantize_node.cpp @@ -157,9 +157,9 @@ struct jit_uni_binarization_kernel : public jit_uni_quantize_kernel, public jit_ uni_vpxor(xmm_wei(0), xmm_wei(0), xmm_wei(0)); uni_vpxor(xmm_mask(0), xmm_mask(0), xmm_mask(0)); - movss(xmm_src(0), ptr[reg_from + c * sizeof(float)]); - movss(xmm_wei(0), ptr[reg_thresholds + c * sizeof(float)]); - movss(xmm_mask(0), ptr[reg_output_mask + c * sizeof(float)]); + uni_vmovss(xmm_src(0), ptr[reg_from + c * sizeof(float)]); + uni_vmovss(xmm_wei(0), ptr[reg_thresholds + c * sizeof(float)]); + uni_vmovss(xmm_mask(0), ptr[reg_output_mask + c * sizeof(float)]); uni_vcmpgtps(xmm_src(0), xmm_src(0), xmm_wei(0)); uni_vpcmpeqd(xmm_src(0), xmm_src(0), xmm_mask(0)); uni_vmovmskps(reg_src_32, xmm_src(0)); @@ -598,13 +598,13 @@ private: jle(exit_label, T_NEAR); for (int i = 0; i < jqp_.c % tail4_simd_w; i++) { - movss(xmm_crop_low(0), ptr[reg_crop_low + i * wei_type_size]); - movss(xmm_crop_high(0), ptr[reg_crop_high + i * wei_type_size]); - movss(xmm_input_scale(0), ptr[reg_input_scale + i * wei_type_size]); - movss(xmm_input_shift(0), ptr[reg_input_shift + i * wei_type_size]); + uni_vmovss(xmm_crop_low(0), ptr[reg_crop_low + i * wei_type_size]); + uni_vmovss(xmm_crop_high(0), ptr[reg_crop_high + i * wei_type_size]); + uni_vmovss(xmm_input_scale(0), ptr[reg_input_scale + i * wei_type_size]); + uni_vmovss(xmm_input_shift(0), ptr[reg_input_shift + i * wei_type_size]); if (do_dequantization) { - movss(xmm_output_scale(0), ptr[reg_output_scale + i * wei_type_size]); - movss(xmm_output_shift(0), ptr[reg_output_shift + i * wei_type_size]); + uni_vmovss(xmm_output_scale(0), ptr[reg_output_scale + i * wei_type_size]); + uni_vmovss(xmm_output_shift(0), ptr[reg_output_shift + i * wei_type_size]); } load_scalar(xmm_val(0), ptr[aux_reg_from + i * src_type_size], jqp_.src_prc); @@ -695,15 +695,15 @@ private: switch (src_prc) { case Precision::FP32: case Precision::I32: - movss(xmm_src, op); + uni_vmovss(xmm_src, op); break; case Precision::I8: movsx(reg_tmp_32, op); - movq(xmm_src, reg_tmp_64); + uni_vmovq(xmm_src, reg_tmp_64); break; case Precision::U8: movzx(reg_tmp_32, op); - movq(xmm_src, reg_tmp_64); + uni_vmovq(xmm_src, reg_tmp_64); break; default: assert(!"unknown src_prc"); @@ -784,12 +784,12 @@ private: case Precision::I8: uni_vpackssdw(xmm_dst, xmm_dst, xmm_dst); uni_vpacksswb(xmm_dst, xmm_dst, xmm_dst); - movd(op, xmm_dst); + uni_vmovd(op, xmm_dst); break; case Precision::U8: uni_vpackusdw(xmm_dst, xmm_dst, xmm_dst); uni_vpackuswb(xmm_dst, xmm_dst, xmm_dst); - movd(op, xmm_dst); + uni_vmovd(op, xmm_dst); break; default: assert(!"unknown dst_prc"); @@ -804,18 +804,18 @@ private: switch (dst_prc) { case Precision::FP32: case Precision::I32: - movss(op, xmm_dst); + uni_vmovss(op, xmm_dst); break; case Precision::I8: uni_vpackssdw(xmm_dst, xmm_dst, xmm_dst); uni_vpacksswb(xmm_dst, xmm_dst, xmm_dst); - movq(reg_tmp_64, xmm_dst); + uni_vmovq(reg_tmp_64, xmm_dst); mov(op, reg_tmp_8); break; case Precision::U8: uni_vpackusdw(xmm_dst, xmm_dst, xmm_dst); uni_vpackuswb(xmm_dst, xmm_dst, xmm_dst); - movq(reg_tmp_64, xmm_dst); + uni_vmovq(reg_tmp_64, xmm_dst); mov(op, reg_tmp_8); break; default: diff --git a/inference-engine/src/mkldnn_plugin/nodes/mkldnn_input_node.cpp b/inference-engine/src/mkldnn_plugin/nodes/mkldnn_input_node.cpp index 982b5e8ea8d..e5f8e5691eb 100644 --- a/inference-engine/src/mkldnn_plugin/nodes/mkldnn_input_node.cpp +++ b/inference-engine/src/mkldnn_plugin/nodes/mkldnn_input_node.cpp @@ -105,13 +105,13 @@ protected: auto b = xmm2; auto c = xmm3; - movdqu(a, xword[src]); // load 4 floats - movdqu(b, a); // b = a - movdqu(c, a); // c = a - pcmpeqd(b, zero); // if (a == 0) b = 1 else b = 0 - pand(c, mask); // c = a & 01111111100000000000000000000000 - pcmpeqd(c, zero); // if (c == 0) c = 1 else c = 0 - ptest(b, c); // if ((!b & c) == 0) CF = 1 else CF = 0 + uni_vmovdqu(a, xword[src]); // load 4 floats + uni_vmovdqu(b, a); // b = a + uni_vmovdqu(c, a); // c = a + uni_vpcmpeqd(b, b, zero); // if (a == 0) b = 1 else b = 0 + uni_vpand(c, mask); // c = a & 01111111100000000000000000000000 + uni_vpcmpeqd(c, c, zero); // if (c == 0) c = 1 else c = 0 + uni_vtestps(b, c); // if ((!b & c) == 0) CF = 1 else CF = 0 } template diff --git a/inference-engine/src/mkldnn_plugin/nodes/mkldnn_interpolate_node.cpp b/inference-engine/src/mkldnn_plugin/nodes/mkldnn_interpolate_node.cpp index d58adc21d81..4f5cae00350 100644 --- a/inference-engine/src/mkldnn_plugin/nodes/mkldnn_interpolate_node.cpp +++ b/inference-engine/src/mkldnn_plugin/nodes/mkldnn_interpolate_node.cpp @@ -1203,10 +1203,10 @@ private: jl(tail_loop_end_label, T_NEAR); // get idx for input - movss(Xmm(vmm_tbl_y.getIdx()), ptr[reg_tbl_y]); + uni_vmovss(Xmm(vmm_tbl_y.getIdx()), ptr[reg_tbl_y]); gather_i32_indices(vmm_index_in_y, reg_index_y, 0, vmm_tbl_y, 1, memory::data_type::s32, true); - movss(Xmm(vmm_val.getIdx()), ptr[reg_tbl_x]); + uni_vmovss(Xmm(vmm_val.getIdx()), ptr[reg_tbl_x]); gather_i32_indices(vmm_index_in_x, reg_index, 0, vmm_val, 1, memory::data_type::s32, true); // gather weightX by input idx, used in y0-y3 gather_i32_indices(vmm_weightX0, reg_weight_x, 0, vmm_val, grid_len, memory::data_type::f32, true); @@ -1434,18 +1434,18 @@ private: switch (src_dt) { case memory::data_type::f32: case memory::data_type::s32: - movss(xmm_src, op); + uni_vmovss(xmm_src, op); break; case memory::data_type::s8: movsx(reg_tmp_32, op); - movq(xmm_src, reg_tmp_64); + uni_vmovq(xmm_src, reg_tmp_64); break; case memory::data_type::u8: movzx(reg_tmp_32, op); - movq(xmm_src, reg_tmp_64); + uni_vmovq(xmm_src, reg_tmp_64); break; case memory::data_type::bf16: - pinsrw(xmm_src, op, 0x0); + uni_vpinsrw(xmm_src, xmm_src, op, 0x0); uni_vpslld(xmm_src, xmm_src, 16); break; default: @@ -1540,7 +1540,7 @@ private: switch (dst_dt) { case memory::data_type::f32: case memory::data_type::s32: - movss(op, xmm_dst); + uni_vmovss(op, xmm_dst); break; case memory::data_type::s8: uni_vpackssdw(xmm_dst, xmm_dst, xmm_dst); @@ -1556,7 +1556,7 @@ private: break; case memory::data_type::bf16: uni_vpsrld(xmm_dst, xmm_dst, 16); - pextrw(op, xmm_dst, 0x0); + uni_vpextrw(op, xmm_dst, 0x0); break; default: assert(!"unknown dst_dt"); diff --git a/inference-engine/src/mkldnn_plugin/nodes/mkldnn_mvn_node.cpp b/inference-engine/src/mkldnn_plugin/nodes/mkldnn_mvn_node.cpp index 88f538e4603..e8e913ecb9a 100644 --- a/inference-engine/src/mkldnn_plugin/nodes/mkldnn_mvn_node.cpp +++ b/inference-engine/src/mkldnn_plugin/nodes/mkldnn_mvn_node.cpp @@ -104,17 +104,17 @@ struct jit_uni_mvn_mean_variance_kernel_f32 : public jit_uni_mvn_mean_variance_k Xbyak::Ymm ymm_sum = Xbyak::Ymm(vmm_dst.getIdx()); vextractf128(xmm_aux1, ymm_sum, 0); vextractf128(xmm_aux2, ymm_sum, 1); - addps(xmm_aux1, xmm_aux2); + uni_vaddps(xmm_aux1, xmm_aux1, xmm_aux2); hsum_store(xmm_aux1); } else { Xbyak::Zmm zmm_sum = Xbyak::Zmm(vmm_dst.getIdx()); vextractf32x4(xmm_aux1, zmm_sum, 0); vextractf32x4(xmm_aux2, zmm_sum, 1); - addps(xmm_aux1, xmm_aux2); + uni_vaddps(xmm_aux1, xmm_aux1, xmm_aux2); vextractf32x4(xmm_aux2, zmm_sum, 2); vextractf32x4(xmm_aux3, zmm_sum, 3); - addps(xmm_aux2, xmm_aux3); - addps(xmm_aux1, xmm_aux2); + uni_vaddps(xmm_aux2, xmm_aux2, xmm_aux3); + uni_vaddps(xmm_aux1, xmm_aux1, xmm_aux2); hsum_store(xmm_aux1); } } else { @@ -344,14 +344,14 @@ private: } inline void hsum_store(Xbyak::Xmm xmm_sum) { - movshdup(xmm_aux3, xmm_sum); // sum:1,2,3,4; aux3:2,2,4,4 - addps(xmm_sum, xmm_aux3); // sum:1+2,2+2,3+4,4+4 - movhlps(xmm_aux3, xmm_sum); // aux3:3+4,4+4,4,4 - addps(xmm_sum, xmm_aux3); // sum:1+2+3+4,... + uni_vmovshdup(xmm_aux3, xmm_sum); // sum:1,2,3,4; aux3:2,2,4,4 + uni_vaddps(xmm_sum, xmm_sum, xmm_aux3); // sum:1+2,2+2,3+4,4+4 + uni_vmovhlps(xmm_aux3, xmm_aux3, xmm_sum); // aux3:3+4,4+4,4,4 + uni_vaddps(xmm_sum, xmm_sum, xmm_aux3); // sum:1+2+3+4,... if (jcp_.normalize_variance) { - movss(ptr[reg_variance], xmm_sum); + uni_vmovss(ptr[reg_variance], xmm_sum); } else { - movss(ptr[reg_sum], xmm_sum); + uni_vmovss(ptr[reg_sum], xmm_sum); } } }; diff --git a/inference-engine/src/mkldnn_plugin/nodes/mkldnn_normalize_node.cpp b/inference-engine/src/mkldnn_plugin/nodes/mkldnn_normalize_node.cpp index da08a4768cf..90ff314da8a 100644 --- a/inference-engine/src/mkldnn_plugin/nodes/mkldnn_normalize_node.cpp +++ b/inference-engine/src/mkldnn_plugin/nodes/mkldnn_normalize_node.cpp @@ -92,17 +92,17 @@ struct jit_uni_normalize_modulo_kernel_f32 : public jit_uni_normalize_modulo_ker Xbyak::Ymm ymm_sqr_sum = Xbyak::Ymm(vmm_sqr_sum.getIdx()); vextractf128(xmm_aux1, ymm_sqr_sum, 0); vextractf128(xmm_aux2, ymm_sqr_sum, 1); - addps(xmm_aux1, xmm_aux2); + uni_vaddps(xmm_aux1, xmm_aux1, xmm_aux2); hsum_store(xmm_aux1); } else { Xbyak::Zmm zmm_sqr_sum = Xbyak::Zmm(vmm_sqr_sum.getIdx()); vextractf32x4(xmm_aux1, zmm_sqr_sum, 0); vextractf32x4(xmm_aux2, zmm_sqr_sum, 1); - addps(xmm_aux1, xmm_aux2); + uni_vaddps(xmm_aux1, xmm_aux1, xmm_aux2); vextractf32x4(xmm_aux2, zmm_sqr_sum, 2); vextractf32x4(xmm_aux3, zmm_sqr_sum, 3); - addps(xmm_aux2, xmm_aux3); - addps(xmm_aux1, xmm_aux2); + uni_vaddps(xmm_aux2, xmm_aux2, xmm_aux3); + uni_vaddps(xmm_aux1, xmm_aux1, xmm_aux2); hsum_store(xmm_aux1); } } @@ -128,11 +128,11 @@ private: Xbyak::Xmm xmm_aux3 = Xbyak::Xmm(4); inline void hsum_store(Xbyak::Xmm xmm_sqr_sum) { - movshdup(xmm_aux3, xmm_sqr_sum); // sqrt_sum:1,2,3,4; aux3:2,2,4,4 - addps(xmm_sqr_sum, xmm_aux3); // sqrt_sum:1+2,2+2,3+4,4+4 - movhlps(xmm_aux3, xmm_sqr_sum); // aux3:3+4,4+4,4,4 - addps(xmm_sqr_sum, xmm_aux3); // sqrt_sum:1+2+3+4,... - movss(ptr[reg_modulo], xmm_sqr_sum); + uni_vmovshdup(xmm_aux3, xmm_sqr_sum); // sqrt_sum:1,2,3,4; aux3:2,2,4,4 + uni_vaddps(xmm_sqr_sum, xmm_sqr_sum, xmm_aux3); // sqrt_sum:1+2,2+2,3+4,4+4 + uni_vmovhlps(xmm_aux3, xmm_aux3, xmm_sqr_sum); // aux3:3+4,4+4,4,4 + uni_vaddps(xmm_sqr_sum, xmm_sqr_sum, xmm_aux3); // sqrt_sum:1+2+3+4,... + uni_vmovss(ptr[reg_modulo], xmm_sqr_sum); } inline void load_vector(Vmm vmm_src, const Xbyak::Address &op, memory::data_type src_dt) { @@ -497,19 +497,19 @@ private: switch (src_dt) { case memory::data_type::f32: case memory::data_type::s32: - movss(xmm_src, op); + uni_vmovss(xmm_src, op); break; case memory::data_type::bf16: - pinsrw(xmm_src, op, 0x0); + uni_vpinsrw(xmm_src, xmm_src, op, 0x0); uni_vpslld(xmm_src, xmm_src, 16); break; case memory::data_type::s8: movsx(reg_tmp_32, op); - movq(xmm_src, reg_tmp_64); + uni_vmovq(xmm_src, reg_tmp_64); break; case memory::data_type::u8: movzx(reg_tmp_32, op); - movq(xmm_src, reg_tmp_64); + uni_vmovq(xmm_src, reg_tmp_64); break; default: assert(!"unknown dst_dt"); @@ -572,11 +572,11 @@ private: switch (dst_dt) { case memory::data_type::f32: case memory::data_type::s32: - movss(op, xmm_dst); + uni_vmovss(op, xmm_dst); break; case memory::data_type::bf16: uni_vpsrld(xmm_dst, xmm_dst, 16); - pextrw(op, xmm_dst, 0x0); + uni_vpextrw(op, xmm_dst, 0x0); break; case memory::data_type::s8: uni_vpackssdw(xmm_dst, xmm_dst, xmm_dst); diff --git a/inference-engine/src/mkldnn_plugin/nodes/mkldnn_region_yolo_node.cpp b/inference-engine/src/mkldnn_plugin/nodes/mkldnn_region_yolo_node.cpp index 300f0c7ce3b..84666f3ce2e 100644 --- a/inference-engine/src/mkldnn_plugin/nodes/mkldnn_region_yolo_node.cpp +++ b/inference-engine/src/mkldnn_plugin/nodes/mkldnn_region_yolo_node.cpp @@ -202,10 +202,10 @@ private: inline void load_scalar(Xbyak::Xmm xmm_src, const Xbyak::Address &op, InferenceEngine::Precision src_dt) { switch (src_dt) { case InferenceEngine::Precision::FP32: - movss(xmm_src, op); + uni_vmovss(xmm_src, op); break; case InferenceEngine::Precision::BF16: - pinsrw(xmm_src, op, 0x0); + uni_vpinsrw(xmm_src, xmm_src, op, 0x0); uni_vpslld(xmm_src, xmm_src, 16); break; default: @@ -215,11 +215,11 @@ private: inline void store_scalar(const Xbyak::Address &op, Xbyak::Xmm xmm_dst, InferenceEngine::Precision dst_dt) { switch (dst_dt) { case InferenceEngine::Precision::FP32: - movss(op, xmm_dst); + uni_vmovss(op, xmm_dst); break; case InferenceEngine::Precision::BF16: uni_vpsrld(xmm_dst, xmm_dst, 16); - pextrw(op, xmm_dst, 0x0); + uni_vpextrw(op, xmm_dst, 0x0); break; default: assert(!"unknown dst_dt"); diff --git a/inference-engine/src/mkldnn_plugin/nodes/mkldnn_roi_pooling_node.cpp b/inference-engine/src/mkldnn_plugin/nodes/mkldnn_roi_pooling_node.cpp index 44880db3c30..6f314411c9c 100644 --- a/inference-engine/src/mkldnn_plugin/nodes/mkldnn_roi_pooling_node.cpp +++ b/inference-engine/src/mkldnn_plugin/nodes/mkldnn_roi_pooling_node.cpp @@ -209,9 +209,9 @@ private: } void roi_pool_bilinear(int c_blocks) { - movq(xmm_yf, reg_yf); + uni_vmovq(xmm_yf, reg_yf); uni_vbroadcastss(vmm_yf, xmm_yf); - movq(xmm_xf, reg_xf); + uni_vmovq(xmm_xf, reg_xf); uni_vbroadcastss(vmm_xf, xmm_xf); Vmm vmm_src00 = get_src_reg(0); diff --git a/inference-engine/thirdparty/mkl-dnn b/inference-engine/thirdparty/mkl-dnn index d5d0552538a..5adbcb757c7 160000 --- a/inference-engine/thirdparty/mkl-dnn +++ b/inference-engine/thirdparty/mkl-dnn @@ -1 +1 @@ -Subproject commit d5d0552538a0962bd83161ca060220c82f8c4b49 +Subproject commit 5adbcb757c77f1bf0cd11ad58dd92e93ea2e3561