IntenseWebs/openvino
3
0 Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[CPU] Fix mixing VEX and non-VEX instructions (#8658)

Browse Source 
Quote: The Skylake microarchitecture implements a different state
machine than prior generations to manage the YMM state transition
associated with mixing SSE and AVX instructions.
It no longer saves the entire upper YMM state when executing
an SSE instruction when in “Modified and Unsaved” state,
but saves the upper bits of individual register.
As a result, mixing SSE and AVX instructions will experience
a penalty associated with partial register dependency of
the destination registers being used and additional blend
operation on the upper bits of the destination registers.

Such type of penalties have a huge impact on openvino's and oneDNN's kernels.
Basically the mixing of VEX and non-VEX instructions should be
avoided.
This commit is contained in:
Egor Duplensky authored and GitHub committed 2021-12-10 13:53:23 +03:00
1 parent 96fd5dce0b
commit 20a5afa84e
14 files changed
+138 -141
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

inference-engine/src/mkldnn_plugin/emitters/jit_eltwise_emitters.cpp
+3 -3
View File
@@ -85,8 +85,8 @@ void jit_mul_add_emitter::emit_isa(const std::vector<size_t> &in_vec_idxs, const
if (isa == cpu::x64::sse41) {
h->uni_vmovups(vmm_dst, vmm_src0);
h->mulps(vmm_dst, vmm_src1);
h->addps(vmm_dst, vmm_src2);
h->uni_vmulps(vmm_dst, vmm_dst, vmm_src1);
h->uni_vaddps(vmm_dst, vmm_dst, vmm_src2);
} else {
Vmm vmm_mul0;
if (vmm_dst.getIdx() == vmm_src0.getIdx()) {
@@ -656,7 +656,7 @@ void jit_equal_emitter::emit_isa(const std::vector<size_t> &in_vec_idxs, const s
} else if (isa == cpu::x64::avx2) {
h->vcmpeqps(vmm_aux0, vmm_src0, vmm_src1);
h->uni_vmovups(vmm_dst, table_val("zero"));
h->vblendvps(vmm_dst, vmm_dst, table_val("one"), vmm_aux0);
h->uni_vblendvps(vmm_dst, vmm_dst, table_val("one"), vmm_aux0);
} else {
h->vcmpps(k_mask, vmm_src0, vmm_src1, _cmp_eq_oq);
h->uni_vmovups(vmm_dst, table_val("zero"));
inference-engine/src/mkldnn_plugin/emitters/jit_load_store_emitters.cpp
+19 -22
View File
@@ -190,7 +190,7 @@ void jit_load_emitter::load_bytes(const Vmm &vmm, const Xbyak::Reg64 &reg, int o
}
if (bytes_to_load >= 8 && bytes_to_load < 16)
h->pinsrq(xmm, addr(start_bytes), 0);
h->uni_vpinsrq(xmm, xmm, addr(start_bytes), 0);
else if (bytes_to_load == 16)
h->uni_vmovdqu(xmm, addr(start_bytes));
@@ -202,17 +202,17 @@ void jit_load_emitter::load_bytes(const Vmm &vmm, const Xbyak::Reg64 &reg, int o
h->uni_vpinsrw(xmm, xmm, addr(start_bytes), 0);
h->uni_vpinsrb(xmm, xmm, addr(start_bytes + 2), 2);
break;
case 4: h->pinsrd(xmm, addr(start_bytes), 0); break;
case 4: h->uni_vpinsrd(xmm, xmm, addr(start_bytes), 0); break;
case 5:
h->pinsrd(xmm, addr(start_bytes), 0);
h->uni_vpinsrd(xmm, xmm, addr(start_bytes), 0);
h->uni_vpinsrb(xmm, xmm, addr(start_bytes + 4), 4);
break;
case 6:
h->pinsrd(xmm, addr(start_bytes), 0);
h->uni_vpinsrd(xmm, xmm, addr(start_bytes), 0);
h->uni_vpinsrw(xmm, xmm, addr(start_bytes + 4), 2);
break;
case 7:
h->pinsrd(xmm, addr(start_bytes), 0);
h->uni_vpinsrd(xmm, xmm, addr(start_bytes), 0);
h->uni_vpinsrw(xmm, xmm, addr(start_bytes + 4), 2);
h->uni_vpinsrb(xmm, xmm, addr(start_bytes + 6), 6);
break;
@@ -223,17 +223,17 @@ void jit_load_emitter::load_bytes(const Vmm &vmm, const Xbyak::Reg64 &reg, int o
h->uni_vpinsrw(xmm, xmm, addr(start_bytes + 8), 4);
h->uni_vpinsrb(xmm, xmm, addr(start_bytes + 10), 10);
break;
case 12: h->pinsrd(xmm, addr(start_bytes + 8), 2); break;
case 12: h->uni_vpinsrd(xmm, xmm, addr(start_bytes + 8), 2); break;
case 13:
h->pinsrd(xmm, addr(start_bytes + 8), 2);
h->uni_vpinsrd(xmm, xmm, addr(start_bytes + 8), 2);
h->uni_vpinsrb(xmm, xmm, addr(start_bytes + 12), 12);
break;
case 14:
h->pinsrd(xmm, addr(start_bytes + 8), 2);
h->uni_vpinsrd(xmm, xmm, addr(start_bytes + 8), 2);
h->uni_vpinsrw(xmm, xmm, addr(start_bytes + 12), 6);
break;
case 15:
h->pinsrd(xmm, addr(start_bytes + 8), 2);
h->uni_vpinsrd(xmm, xmm, addr(start_bytes + 8), 2);
h->uni_vpinsrw(xmm, xmm, addr(start_bytes + 12), 6);
h->uni_vpinsrb(xmm, xmm, addr(start_bytes + 14), 14);
break;
@@ -465,10 +465,7 @@ template <typename Vmm>
if (is_xmm || is_ymm) {
uint8 imm = 1;
imm = ~((imm << load_num) - imm); // shift load_num bit
if (is_xmm)
h->blendps(vmm, table_val(fill_value), imm);
else
h->vblendps(vmm, vmm, table_val(fill_value), imm);
h->uni_vblendps(vmm, vmm, table_val(fill_value), imm);
} else if (is_zmm) {
uint64_t tail_mask = 1;
tail_mask = ~((tail_mask << load_num) - tail_mask);
@@ -673,7 +670,7 @@ template <typename Vmm>
}
if (bytes_to_store >= 8 && bytes_to_store < 16)
h->pextrq(addr(start_bytes), xmm, 0);
h->uni_vpextrq(addr(start_bytes), xmm, 0);
else if (bytes_to_store == 16)
h->uni_vmovdqu(addr(start_bytes), xmm);
@@ -687,17 +684,17 @@ template <typename Vmm>
h->uni_vpextrw(addr(start_bytes), xmm, 0);
h->uni_vpextrb(addr(start_bytes + 2), xmm, 2);
break;
case 4: h->pextrd(addr(start_bytes), xmm, 0); break;
case 4: h->uni_vpextrd(addr(start_bytes), xmm, 0); break;
case 5:
h->pextrd(addr(start_bytes), xmm, 0);
h->uni_vpextrd(addr(start_bytes), xmm, 0);
h->uni_vpextrb(addr(start_bytes + 4), xmm, 4);
break;
case 6:
h->pextrd(addr(start_bytes), xmm, 0);
h->uni_vpextrd(addr(start_bytes), xmm, 0);
h->uni_vpextrw(addr(start_bytes + 4), xmm, 2);
break;
case 7:
h->pextrd(addr(start_bytes), xmm, 0);
h->uni_vpextrd(addr(start_bytes), xmm, 0);
h->uni_vpextrw(addr(start_bytes + 4), xmm, 2);
h->uni_vpextrb(addr(start_bytes + 6), xmm, 6);
break;
@@ -708,17 +705,17 @@ template <typename Vmm>
h->uni_vpextrw(addr(start_bytes + 8), xmm, 4);
h->uni_vpextrb(addr(start_bytes + 10), xmm, 10);
break;
case 12: h->pextrd(addr(start_bytes + 8), xmm, 2); break;
case 12: h->uni_vpextrd(addr(start_bytes + 8), xmm, 2); break;
case 13:
h->pextrd(addr(start_bytes + 8), xmm, 2);
h->uni_vpextrd(addr(start_bytes + 8), xmm, 2);
h->uni_vpextrb(addr(start_bytes + 12), xmm, 12);
break;
case 14:
h->pextrd(addr(start_bytes + 8), xmm, 2);
h->uni_vpextrd(addr(start_bytes + 8), xmm, 2);
h->uni_vpextrw(addr(start_bytes + 12), xmm, 6);
break;
case 15:
h->pextrd(addr(start_bytes + 8), xmm, 2);
h->uni_vpextrd(addr(start_bytes + 8), xmm, 2);
h->uni_vpextrw(addr(start_bytes + 12), xmm, 6);
h->uni_vpextrb(addr(start_bytes + 14), xmm, 14);
break;
inference-engine/src/mkldnn_plugin/nodes/common/permute_kernel.cpp
+10 -10
View File
@@ -47,21 +47,21 @@ struct jit_uni_permute_kernel_f32 : public jit_uni_permute_kernel, public jit_ge
void load(const Xbyak::Xmm &xmm, const Xbyak::Address &addr) {
switch (jcp.data_size) {
case 16: movups(xmm, addr); break;
case 8: movsd(xmm, addr); break;
case 4: movss(xmm, addr); break;
case 2: pinsrw(xmm, addr, 0x0); break;
case 1: pinsrb(xmm, addr, 0x0); break;
case 16: uni_vmovups(xmm, addr); break;
case 8: uni_vmovsd(xmm, addr); break;
case 4: uni_vmovss(xmm, addr); break;
case 2: uni_vpinsrw(xmm, xmm, addr, 0x0); break;
case 1: uni_vpinsrb(xmm, xmm, addr, 0x0); break;
}
}
void store(const Xbyak::Address &addr, const Xbyak::Xmm &xmm) {
switch (jcp.data_size) {
case 16: movups(addr, xmm); break;
case 8: movsd(addr, xmm); break;
case 4: movss(addr, xmm); break;
case 2: pextrw(addr, xmm, 0x0); break;
case 1: pextrb(addr, xmm, 0x0); break;
case 16: uni_vmovups(addr, xmm); break;
case 8: uni_vmovsd(addr, xmm); break;
case 4: uni_vmovss(addr, xmm); break;
case 2: uni_vpextrw(addr, xmm, 0x0); break;
case 1: uni_vpextrb(addr, xmm, 0x0); break;
}
}
inference-engine/src/mkldnn_plugin/nodes/mkldnn_bin_conv_node.cpp
+5 -5
View File
@@ -216,7 +216,7 @@ private:
case memory::data_type::s32:
if (scalar_load) {
mov(reg_tmp_32, op);
movq(xmm_in, reg_tmp_64);
uni_vmovq(xmm_in, reg_tmp_64);
} else {
uni_vmovups(vmm_in, op);
}
@@ -224,7 +224,7 @@ private:
case memory::data_type::s8:
if (scalar_load) {
movsx(reg_tmp_32, op);
movq(xmm_in, reg_tmp_64);
uni_vmovq(xmm_in, reg_tmp_64);
} else {
uni_vpmovsxbd(vmm_in, op);
}
@@ -232,7 +232,7 @@ private:
case memory::data_type::u8:
if (scalar_load) {
movzx(reg_tmp_32, op);
movq(xmm_in, reg_tmp_64);
uni_vmovq(xmm_in, reg_tmp_64);
} else {
uni_vpmovzxbd(vmm_in, op);
}
@@ -542,7 +542,7 @@ private:
if (jcp_.exclude_pad) {
mov(reg_shift, kw_padding[jj]);
imul(reg_shift, reg_tmp_32);
movq(Xmm(vmm_shift.getIdx()), reg_shift);
uni_vmovq(Xmm(vmm_shift.getIdx()), reg_shift);
uni_vbroadcastss(vmm_shift, Xmm(vmm_shift.getIdx()));
uni_vcvtdq2ps(vmm_shift, vmm_shift);
}
@@ -613,7 +613,7 @@ private:
} else {
Ymm ymm_prev_dst = Ymm(vmm_sum.getIdx());
vperm2i128(ymm_prev_dst, ymm_prev_dst, ymm_prev_dst, 0x01);
vpslldq(vmm_sum, vmm_sum, (oc - jcp_.oc_block / 2) * sizeof(float));
uni_vpslldq(vmm_sum, vmm_sum, (oc - jcp_.oc_block / 2) * sizeof(float));
}
uni_vaddps(vmm_dst, vmm_dst, vmm_sum);
inference-engine/src/mkldnn_plugin/nodes/mkldnn_def_conv_node.cpp
+33 -33
View File
@@ -318,10 +318,10 @@ private:
size_t ind_off_lh = ind_off_hl + 1;
size_t ind_off_ll = ind_off_lh + 1;
movq(xmm_v1_off, qword[aux_reg_sampled_offs + ind_off_ll * jcp_.typesize_sampled_offsets]);
movq(xmm_v2_off, qword[aux_reg_sampled_offs + ind_off_hl * jcp_.typesize_sampled_offsets]);
movq(xmm_v3_off, qword[aux_reg_sampled_offs + ind_off_lh * jcp_.typesize_sampled_offsets]);
movq(xmm_v4_off, qword[aux_reg_sampled_offs + ind_off_hh * jcp_.typesize_sampled_offsets]);
uni_vmovq(xmm_v1_off, qword[aux_reg_sampled_offs + ind_off_ll * jcp_.typesize_sampled_offsets]);
uni_vmovq(xmm_v2_off, qword[aux_reg_sampled_offs + ind_off_hl * jcp_.typesize_sampled_offsets]);
uni_vmovq(xmm_v3_off, qword[aux_reg_sampled_offs + ind_off_lh * jcp_.typesize_sampled_offsets]);
uni_vmovq(xmm_v4_off, qword[aux_reg_sampled_offs + ind_off_hh * jcp_.typesize_sampled_offsets]);
// w's computation
uni_vbroadcastss(vmm_w1, dword[aux_reg_sampled_wei + ind_off_ll * jcp_.typesize_sampled_wei]);
@@ -339,29 +339,29 @@ private:
size_t input_buffer_off = (size_t) kh * jcp_.kw * jcp_.ic + kw * jcp_.ic;
pmovsxdq(xmm_v1_off, xmm_v1_off);
movq(reg_tmp_64, xmm_v1_off);
uni_vpmovsxdq(xmm_v1_off, xmm_v1_off);
uni_vmovq(reg_tmp_64, xmm_v1_off);
imul(reg_tmp_64, reg_tmp_64, jcp_.ic * jcp_.typesize_in);
add(reg_tmp_64, aux2_reg_input);
uni_vmovups(vmm_v1, ptr[reg_tmp_64]);
uni_vmulps(vmm_v1, vmm_v1, vmm_w1);
pmovsxdq(xmm_v2_off, xmm_v2_off);
movq(reg_tmp_64, xmm_v2_off);
uni_vpmovsxdq(xmm_v2_off, xmm_v2_off);
uni_vmovq(reg_tmp_64, xmm_v2_off);
imul(reg_tmp_64, reg_tmp_64, jcp_.ic * jcp_.typesize_in);
add(reg_tmp_64, aux2_reg_input);
uni_vmovups(vmm_v2, ptr[reg_tmp_64]);
uni_vmulps(vmm_v2, vmm_v2, vmm_w2);
pmovsxdq(xmm_v3_off, xmm_v3_off);
movq(reg_tmp_64, xmm_v3_off);
uni_vpmovsxdq(xmm_v3_off, xmm_v3_off);
uni_vmovq(reg_tmp_64, xmm_v3_off);
imul(reg_tmp_64, reg_tmp_64, jcp_.ic * jcp_.typesize_in);
add(reg_tmp_64, aux2_reg_input);
uni_vmovups(vmm_v3, ptr[reg_tmp_64]);
uni_vmulps(vmm_v3, vmm_v3, vmm_w3);
pmovsxdq(xmm_v4_off, xmm_v4_off);
movq(reg_tmp_64, xmm_v4_off);
uni_vpmovsxdq(xmm_v4_off, xmm_v4_off);
uni_vmovq(reg_tmp_64, xmm_v4_off);
imul(reg_tmp_64, reg_tmp_64, jcp_.ic * jcp_.typesize_in);
add(reg_tmp_64, aux2_reg_input);
uni_vmovups(vmm_v4, ptr[reg_tmp_64]);
@@ -384,38 +384,38 @@ private:
jl(loop_end, T_NEAR);
size_t input_buffer_off = (size_t) kh * jcp_.kw * jcp_.ic + kw * jcp_.ic;
pmovsxdq(xmm_v1_off, xmm_v1_off);
movq(reg_tmp_64, xmm_v1_off);
uni_vpmovsxdq(xmm_v1_off, xmm_v1_off);
uni_vmovq(reg_tmp_64, xmm_v1_off);
imul(reg_tmp_64, reg_tmp_64, jcp_.ic * jcp_.typesize_in);
add(reg_tmp_64, aux2_reg_input);
movss(xmm_v1, ptr[reg_tmp_64]);
mulss(xmm_v1, xmm_w1);
uni_vmovss(xmm_v1, ptr[reg_tmp_64]);
uni_vmulss(xmm_v1, xmm_v1, xmm_w1);
pmovsxdq(xmm_v2_off, xmm_v2_off);
movq(reg_tmp_64, xmm_v2_off);
uni_vpmovsxdq(xmm_v2_off, xmm_v2_off);
uni_vmovq(reg_tmp_64, xmm_v2_off);
imul(reg_tmp_64, reg_tmp_64, jcp_.ic * jcp_.typesize_in);
add(reg_tmp_64, aux2_reg_input);
movss(xmm_v2, ptr[reg_tmp_64]);
mulss(xmm_v2, xmm_w2);
uni_vmovss(xmm_v2, ptr[reg_tmp_64]);
uni_vmulss(xmm_v2, xmm_v2, xmm_w2);
pmovsxdq(xmm_v3_off, xmm_v3_off);
movq(reg_tmp_64, xmm_v3_off);
uni_vpmovsxdq(xmm_v3_off, xmm_v3_off);
uni_vmovq(reg_tmp_64, xmm_v3_off);
imul(reg_tmp_64, reg_tmp_64, jcp_.ic * jcp_.typesize_in);
add(reg_tmp_64, aux2_reg_input);
movss(xmm_v3, ptr[reg_tmp_64]);
mulss(xmm_v3, xmm_w3);
uni_vmovss(xmm_v3, ptr[reg_tmp_64]);
uni_vmulss(xmm_v3, xmm_v3, xmm_w3);
pmovsxdq(xmm_v4_off, xmm_v4_off);
movq(reg_tmp_64, xmm_v4_off);
uni_vpmovsxdq(xmm_v4_off, xmm_v4_off);
uni_vmovq(reg_tmp_64, xmm_v4_off);
imul(reg_tmp_64, reg_tmp_64, jcp_.ic * jcp_.typesize_in);
add(reg_tmp_64, aux2_reg_input);
movss(xmm_v4, ptr[reg_tmp_64]);
mulss(xmm_v4, xmm_w4);
uni_vmovss(xmm_v4, ptr[reg_tmp_64]);
uni_vmulss(xmm_v4, xmm_v4, xmm_w4);
addss(xmm_v1, xmm_v2);
addss(xmm_v1, xmm_v3);
addss(xmm_v1, xmm_v4);
movss(ptr[aux3_reg_input_buffer + input_buffer_off * jcp_.typesize_in], xmm_v1);
uni_vaddss(xmm_v1, xmm_v1, xmm_v2);
uni_vaddss(xmm_v1, xmm_v1, xmm_v3);
uni_vaddss(xmm_v1, xmm_v1, xmm_v4);
uni_vmovss(ptr[aux3_reg_input_buffer + input_buffer_off * jcp_.typesize_in], xmm_v1);
add(aux2_reg_input, jcp_.typesize_in);
add(aux3_reg_input_buffer, jcp_.typesize_in);
@@ -477,7 +477,7 @@ private:
} else {
for (int oc = 0; oc < tail_size; oc++) {
size_t out_off = (size_t) ow * jcp_.oc + oc + r * (jcp_.oc_block / 2);
movq(reg_tmp_64, xmm_dst);
uni_vmovq(reg_tmp_64, xmm_dst);
mov(ptr[aux_reg_output + out_off * jcp_.typesize_out], reg_tmp_32);
if (isa == cpu::x64::sse41) {
inference-engine/src/mkldnn_plugin/nodes/mkldnn_eltwise_node.cpp
+4 -4
View File
@@ -610,7 +610,7 @@ private:
switch (src_prc) {
case Precision::FP32:
case Precision::I32:
movss(xmm_src, op);
uni_vmovss(xmm_src, op);
break;
case Precision::BF16:
uni_vpinsrw(xmm_src, xmm_src, op, 0);
@@ -626,11 +626,11 @@ private:
break;
case Precision::I8:
movsx(reg_tmp_32, op);
movq(xmm_src, reg_tmp_64);
uni_vmovq(xmm_src, reg_tmp_64);
break;
case Precision::U8:
movzx(reg_tmp_32, op);
movq(xmm_src, reg_tmp_64);
uni_vmovq(xmm_src, reg_tmp_64);
break;
default:
assert(!"unknown src_prc");
@@ -757,7 +757,7 @@ private:
switch (dst_prc) {
case Precision::FP32:
case Precision::I32:
movss(op, xmm_dst);
uni_vmovss(op, xmm_dst);
break;
case Precision::BF16:
uni_vpsrld(xmm_dst, xmm_dst, 16);
inference-engine/src/mkldnn_plugin/nodes/mkldnn_fake_quantize_node.cpp
+17 -17
View File
@@ -157,9 +157,9 @@ struct jit_uni_binarization_kernel : public jit_uni_quantize_kernel, public jit_
uni_vpxor(xmm_wei(0), xmm_wei(0), xmm_wei(0));
uni_vpxor(xmm_mask(0), xmm_mask(0), xmm_mask(0));
movss(xmm_src(0), ptr[reg_from + c * sizeof(float)]);
movss(xmm_wei(0), ptr[reg_thresholds + c * sizeof(float)]);
movss(xmm_mask(0), ptr[reg_output_mask + c * sizeof(float)]);
uni_vmovss(xmm_src(0), ptr[reg_from + c * sizeof(float)]);
uni_vmovss(xmm_wei(0), ptr[reg_thresholds + c * sizeof(float)]);
uni_vmovss(xmm_mask(0), ptr[reg_output_mask + c * sizeof(float)]);
uni_vcmpgtps(xmm_src(0), xmm_src(0), xmm_wei(0));
uni_vpcmpeqd(xmm_src(0), xmm_src(0), xmm_mask(0));
uni_vmovmskps(reg_src_32, xmm_src(0));
@@ -598,13 +598,13 @@ private:
jle(exit_label, T_NEAR);
for (int i = 0; i < jqp_.c % tail4_simd_w; i++) {
movss(xmm_crop_low(0), ptr[reg_crop_low + i * wei_type_size]);
movss(xmm_crop_high(0), ptr[reg_crop_high + i * wei_type_size]);
movss(xmm_input_scale(0), ptr[reg_input_scale + i * wei_type_size]);
movss(xmm_input_shift(0), ptr[reg_input_shift + i * wei_type_size]);
uni_vmovss(xmm_crop_low(0), ptr[reg_crop_low + i * wei_type_size]);
uni_vmovss(xmm_crop_high(0), ptr[reg_crop_high + i * wei_type_size]);
uni_vmovss(xmm_input_scale(0), ptr[reg_input_scale + i * wei_type_size]);
uni_vmovss(xmm_input_shift(0), ptr[reg_input_shift + i * wei_type_size]);
if (do_dequantization) {
movss(xmm_output_scale(0), ptr[reg_output_scale + i * wei_type_size]);
movss(xmm_output_shift(0), ptr[reg_output_shift + i * wei_type_size]);
uni_vmovss(xmm_output_scale(0), ptr[reg_output_scale + i * wei_type_size]);
uni_vmovss(xmm_output_shift(0), ptr[reg_output_shift + i * wei_type_size]);
}
load_scalar(xmm_val(0), ptr[aux_reg_from + i * src_type_size], jqp_.src_prc);
@@ -695,15 +695,15 @@ private:
switch (src_prc) {
case Precision::FP32:
case Precision::I32:
movss(xmm_src, op);
uni_vmovss(xmm_src, op);
break;
case Precision::I8:
movsx(reg_tmp_32, op);
movq(xmm_src, reg_tmp_64);
uni_vmovq(xmm_src, reg_tmp_64);
break;
case Precision::U8:
movzx(reg_tmp_32, op);
movq(xmm_src, reg_tmp_64);
uni_vmovq(xmm_src, reg_tmp_64);
break;
default:
assert(!"unknown src_prc");
@@ -784,12 +784,12 @@ private:
case Precision::I8:
uni_vpackssdw(xmm_dst, xmm_dst, xmm_dst);
uni_vpacksswb(xmm_dst, xmm_dst, xmm_dst);
movd(op, xmm_dst);
uni_vmovd(op, xmm_dst);
break;
case Precision::U8:
uni_vpackusdw(xmm_dst, xmm_dst, xmm_dst);
uni_vpackuswb(xmm_dst, xmm_dst, xmm_dst);
movd(op, xmm_dst);
uni_vmovd(op, xmm_dst);
break;
default:
assert(!"unknown dst_prc");
@@ -804,18 +804,18 @@ private:
switch (dst_prc) {
case Precision::FP32:
case Precision::I32:
movss(op, xmm_dst);
uni_vmovss(op, xmm_dst);
break;
case Precision::I8:
uni_vpackssdw(xmm_dst, xmm_dst, xmm_dst);
uni_vpacksswb(xmm_dst, xmm_dst, xmm_dst);
movq(reg_tmp_64, xmm_dst);
uni_vmovq(reg_tmp_64, xmm_dst);
mov(op, reg_tmp_8);
break;
case Precision::U8:
uni_vpackusdw(xmm_dst, xmm_dst, xmm_dst);
uni_vpackuswb(xmm_dst, xmm_dst, xmm_dst);
movq(reg_tmp_64, xmm_dst);
uni_vmovq(reg_tmp_64, xmm_dst);
mov(op, reg_tmp_8);
break;
default:
inference-engine/src/mkldnn_plugin/nodes/mkldnn_input_node.cpp
+7 -7
View File
@@ -105,13 +105,13 @@ protected:
auto b = xmm2;
auto c = xmm3;
movdqu(a, xword[src]); // load 4 floats
movdqu(b, a); // b = a
movdqu(c, a); // c = a
pcmpeqd(b, zero); // if (a == 0) b = 1 else b = 0
pand(c, mask); // c = a & 01111111100000000000000000000000
pcmpeqd(c, zero); // if (c == 0) c = 1 else c = 0
ptest(b, c); // if ((!b & c) == 0) CF = 1 else CF = 0
uni_vmovdqu(a, xword[src]); // load 4 floats
uni_vmovdqu(b, a); // b = a
uni_vmovdqu(c, a); // c = a
uni_vpcmpeqd(b, b, zero); // if (a == 0) b = 1 else b = 0
uni_vpand(c, mask); // c = a & 01111111100000000000000000000000
uni_vpcmpeqd(c, c, zero); // if (c == 0) c = 1 else c = 0
uni_vtestps(b, c); // if ((!b & c) == 0) CF = 1 else CF = 0
}
template<cpu_isa_t isa>
inference-engine/src/mkldnn_plugin/nodes/mkldnn_interpolate_node.cpp
+8 -8
View File
@@ -1203,10 +1203,10 @@ private:
jl(tail_loop_end_label, T_NEAR);
// get idx for input
movss(Xmm(vmm_tbl_y.getIdx()), ptr[reg_tbl_y]);
uni_vmovss(Xmm(vmm_tbl_y.getIdx()), ptr[reg_tbl_y]);
gather_i32_indices(vmm_index_in_y, reg_index_y, 0, vmm_tbl_y, 1, memory::data_type::s32, true);
movss(Xmm(vmm_val.getIdx()), ptr[reg_tbl_x]);
uni_vmovss(Xmm(vmm_val.getIdx()), ptr[reg_tbl_x]);
gather_i32_indices(vmm_index_in_x, reg_index, 0, vmm_val, 1, memory::data_type::s32, true);
// gather weightX by input idx, used in y0-y3
gather_i32_indices(vmm_weightX0, reg_weight_x, 0, vmm_val, grid_len, memory::data_type::f32, true);
@@ -1434,18 +1434,18 @@ private:
switch (src_dt) {
case memory::data_type::f32:
case memory::data_type::s32:
movss(xmm_src, op);
uni_vmovss(xmm_src, op);
break;
case memory::data_type::s8:
movsx(reg_tmp_32, op);
movq(xmm_src, reg_tmp_64);
uni_vmovq(xmm_src, reg_tmp_64);
break;
case memory::data_type::u8:
movzx(reg_tmp_32, op);
movq(xmm_src, reg_tmp_64);
uni_vmovq(xmm_src, reg_tmp_64);
break;
case memory::data_type::bf16:
pinsrw(xmm_src, op, 0x0);
uni_vpinsrw(xmm_src, xmm_src, op, 0x0);
uni_vpslld(xmm_src, xmm_src, 16);
break;
default:
@@ -1540,7 +1540,7 @@ private:
switch (dst_dt) {
case memory::data_type::f32:
case memory::data_type::s32:
movss(op, xmm_dst);
uni_vmovss(op, xmm_dst);
break;
case memory::data_type::s8:
uni_vpackssdw(xmm_dst, xmm_dst, xmm_dst);
@@ -1556,7 +1556,7 @@ private:
break;
case memory::data_type::bf16:
uni_vpsrld(xmm_dst, xmm_dst, 16);
pextrw(op, xmm_dst, 0x0);
uni_vpextrw(op, xmm_dst, 0x0);
break;
default:
assert(!"unknown dst_dt");
inference-engine/src/mkldnn_plugin/nodes/mkldnn_mvn_node.cpp
+10 -10
View File
@@ -104,17 +104,17 @@ struct jit_uni_mvn_mean_variance_kernel_f32 : public jit_uni_mvn_mean_variance_k
Xbyak::Ymm ymm_sum = Xbyak::Ymm(vmm_dst.getIdx());
vextractf128(xmm_aux1, ymm_sum, 0);
vextractf128(xmm_aux2, ymm_sum, 1);
addps(xmm_aux1, xmm_aux2);
uni_vaddps(xmm_aux1, xmm_aux1, xmm_aux2);
hsum_store(xmm_aux1);
} else {
Xbyak::Zmm zmm_sum = Xbyak::Zmm(vmm_dst.getIdx());
vextractf32x4(xmm_aux1, zmm_sum, 0);
vextractf32x4(xmm_aux2, zmm_sum, 1);
addps(xmm_aux1, xmm_aux2);
uni_vaddps(xmm_aux1, xmm_aux1, xmm_aux2);
vextractf32x4(xmm_aux2, zmm_sum, 2);
vextractf32x4(xmm_aux3, zmm_sum, 3);
addps(xmm_aux2, xmm_aux3);
addps(xmm_aux1, xmm_aux2);
uni_vaddps(xmm_aux2, xmm_aux2, xmm_aux3);
uni_vaddps(xmm_aux1, xmm_aux1, xmm_aux2);
hsum_store(xmm_aux1);
}
} else {
@@ -344,14 +344,14 @@ private:
}
inline void hsum_store(Xbyak::Xmm xmm_sum) {
movshdup(xmm_aux3, xmm_sum); // sum:1,2,3,4; aux3:2,2,4,4
addps(xmm_sum, xmm_aux3); // sum:1+2,2+2,3+4,4+4
movhlps(xmm_aux3, xmm_sum); // aux3:3+4,4+4,4,4
addps(xmm_sum, xmm_aux3); // sum:1+2+3+4,...
uni_vmovshdup(xmm_aux3, xmm_sum); // sum:1,2,3,4; aux3:2,2,4,4
uni_vaddps(xmm_sum, xmm_sum, xmm_aux3); // sum:1+2,2+2,3+4,4+4
uni_vmovhlps(xmm_aux3, xmm_aux3, xmm_sum); // aux3:3+4,4+4,4,4
uni_vaddps(xmm_sum, xmm_sum, xmm_aux3); // sum:1+2+3+4,...
if (jcp_.normalize_variance) {
movss(ptr[reg_variance], xmm_sum);
uni_vmovss(ptr[reg_variance], xmm_sum);
} else {
movss(ptr[reg_sum], xmm_sum);
uni_vmovss(ptr[reg_sum], xmm_sum);
}
}
};
inference-engine/src/mkldnn_plugin/nodes/mkldnn_normalize_node.cpp
+15 -15
View File
@@ -92,17 +92,17 @@ struct jit_uni_normalize_modulo_kernel_f32 : public jit_uni_normalize_modulo_ker
Xbyak::Ymm ymm_sqr_sum = Xbyak::Ymm(vmm_sqr_sum.getIdx());
vextractf128(xmm_aux1, ymm_sqr_sum, 0);
vextractf128(xmm_aux2, ymm_sqr_sum, 1);
addps(xmm_aux1, xmm_aux2);
uni_vaddps(xmm_aux1, xmm_aux1, xmm_aux2);
hsum_store(xmm_aux1);
} else {
Xbyak::Zmm zmm_sqr_sum = Xbyak::Zmm(vmm_sqr_sum.getIdx());
vextractf32x4(xmm_aux1, zmm_sqr_sum, 0);
vextractf32x4(xmm_aux2, zmm_sqr_sum, 1);
addps(xmm_aux1, xmm_aux2);
uni_vaddps(xmm_aux1, xmm_aux1, xmm_aux2);
vextractf32x4(xmm_aux2, zmm_sqr_sum, 2);
vextractf32x4(xmm_aux3, zmm_sqr_sum, 3);
addps(xmm_aux2, xmm_aux3);
addps(xmm_aux1, xmm_aux2);
uni_vaddps(xmm_aux2, xmm_aux2, xmm_aux3);
uni_vaddps(xmm_aux1, xmm_aux1, xmm_aux2);
hsum_store(xmm_aux1);
}
}
@@ -128,11 +128,11 @@ private:
Xbyak::Xmm xmm_aux3 = Xbyak::Xmm(4);
inline void hsum_store(Xbyak::Xmm xmm_sqr_sum) {
movshdup(xmm_aux3, xmm_sqr_sum); // sqrt_sum:1,2,3,4; aux3:2,2,4,4
addps(xmm_sqr_sum, xmm_aux3); // sqrt_sum:1+2,2+2,3+4,4+4
movhlps(xmm_aux3, xmm_sqr_sum); // aux3:3+4,4+4,4,4
addps(xmm_sqr_sum, xmm_aux3); // sqrt_sum:1+2+3+4,...
movss(ptr[reg_modulo], xmm_sqr_sum);
uni_vmovshdup(xmm_aux3, xmm_sqr_sum); // sqrt_sum:1,2,3,4; aux3:2,2,4,4
uni_vaddps(xmm_sqr_sum, xmm_sqr_sum, xmm_aux3); // sqrt_sum:1+2,2+2,3+4,4+4
uni_vmovhlps(xmm_aux3, xmm_aux3, xmm_sqr_sum); // aux3:3+4,4+4,4,4
uni_vaddps(xmm_sqr_sum, xmm_sqr_sum, xmm_aux3); // sqrt_sum:1+2+3+4,...
uni_vmovss(ptr[reg_modulo], xmm_sqr_sum);
}
inline void load_vector(Vmm vmm_src, const Xbyak::Address &op, memory::data_type src_dt) {
@@ -497,19 +497,19 @@ private:
switch (src_dt) {
case memory::data_type::f32:
case memory::data_type::s32:
movss(xmm_src, op);
uni_vmovss(xmm_src, op);
break;
case memory::data_type::bf16:
pinsrw(xmm_src, op, 0x0);
uni_vpinsrw(xmm_src, xmm_src, op, 0x0);
uni_vpslld(xmm_src, xmm_src, 16);
break;
case memory::data_type::s8:
movsx(reg_tmp_32, op);
movq(xmm_src, reg_tmp_64);
uni_vmovq(xmm_src, reg_tmp_64);
break;
case memory::data_type::u8:
movzx(reg_tmp_32, op);
movq(xmm_src, reg_tmp_64);
uni_vmovq(xmm_src, reg_tmp_64);
break;
default:
assert(!"unknown dst_dt");
@@ -572,11 +572,11 @@ private:
switch (dst_dt) {
case memory::data_type::f32:
case memory::data_type::s32:
movss(op, xmm_dst);
uni_vmovss(op, xmm_dst);
break;
case memory::data_type::bf16:
uni_vpsrld(xmm_dst, xmm_dst, 16);
pextrw(op, xmm_dst, 0x0);
uni_vpextrw(op, xmm_dst, 0x0);
break;
case memory::data_type::s8:
uni_vpackssdw(xmm_dst, xmm_dst, xmm_dst);
inference-engine/src/mkldnn_plugin/nodes/mkldnn_region_yolo_node.cpp
+4 -4
View File
@@ -202,10 +202,10 @@ private:
inline void load_scalar(Xbyak::Xmm xmm_src, const Xbyak::Address &op, InferenceEngine::Precision src_dt) {
switch (src_dt) {
case InferenceEngine::Precision::FP32:
movss(xmm_src, op);
uni_vmovss(xmm_src, op);
break;
case InferenceEngine::Precision::BF16:
pinsrw(xmm_src, op, 0x0);
uni_vpinsrw(xmm_src, xmm_src, op, 0x0);
uni_vpslld(xmm_src, xmm_src, 16);
break;
default:
@@ -215,11 +215,11 @@ private:
inline void store_scalar(const Xbyak::Address &op, Xbyak::Xmm xmm_dst, InferenceEngine::Precision dst_dt) {
switch (dst_dt) {
case InferenceEngine::Precision::FP32:
movss(op, xmm_dst);
uni_vmovss(op, xmm_dst);
break;
case InferenceEngine::Precision::BF16:
uni_vpsrld(xmm_dst, xmm_dst, 16);
pextrw(op, xmm_dst, 0x0);
uni_vpextrw(op, xmm_dst, 0x0);
break;
default:
assert(!"unknown dst_dt");
inference-engine/src/mkldnn_plugin/nodes/mkldnn_roi_pooling_node.cpp
+2 -2
View File
@@ -209,9 +209,9 @@ private:
}
void roi_pool_bilinear(int c_blocks) {
movq(xmm_yf, reg_yf);
uni_vmovq(xmm_yf, reg_yf);
uni_vbroadcastss(vmm_yf, xmm_yf);
movq(xmm_xf, reg_xf);
uni_vmovq(xmm_xf, reg_xf);
uni_vbroadcastss(vmm_xf, xmm_xf);
Vmm vmm_src00 = get_src_reg(0);
inference-engine/thirdparty/mkl-dnn
Vendored
+1 -1
Submodule inference-engine/thirdparty/mkl-dnn updated: d5d0552538...5adbcb757c.