IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[CPU] Added support AVX2 and SSE for jit_emu_vcvtneps2bf16 (#12718)

Browse Source 
* [CPU] Added support AVX2 and SSE for jit_emu_vcvtneps2bf16

* Upped thresholds

* check condition

* Removed thresholds

* Added kernel test

* Fixed AVX2 and SSE impl: removed AVX512 instructions

* come back

* Fixed jit_emu_vcvtneps2bf16 sse in=out

* update store_emitter for sse

* Added I32 test and updated tests

* united avx2 and sse impls

* Fixed ymm/xmm for avx512

* United with avx512_bf16

* fixed unused variable warning

* unified float2bfloat in convert_emitter
This commit is contained in:
Alexandra Sidorova
2022-09-20 20:19:39 +04:00
committed by GitHub
parent a34b59d897
commit e109bd1a32
12 changed files with 200 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files

74
src/plugins/intel_cpu/src/emitters/jit_bf16_emitters.hpp
View File

@@ -9,11 +9,12 @@
namespace ov {
namespace intel_cpu {
class jit_emu_vcvtneps2bf16 : public jit_emitter {
class jit_uni_vcvtneps2bf16 : public jit_emitter {
public:
jit_emu_vcvtneps2bf16(dnnl::impl::cpu::x64::jit_generator* host, dnnl::impl::cpu::x64::cpu_isa_t host_isa,
jit_uni_vcvtneps2bf16(dnnl::impl::cpu::x64::jit_generator* host, dnnl::impl::cpu::x64::cpu_isa_t host_isa,
InferenceEngine::Precision exec_prc = InferenceEngine::Precision::BF16) : jit_emitter(host, host_isa, exec_prc) {
prepare_table();
if (!dnnl::impl::cpu::x64::mayiuse(dnnl::impl::cpu::x64::avx512_core_bf16))
prepare_table();
}
size_t get_inputs_num() const override { return 1; }
@@ -22,11 +23,31 @@ private:
void emit_impl(const std::vector<size_t>& in_vec_idxs, const std::vector<size_t>& out_vec_idxs,
const std::vector<size_t>& pool_vec_idxs, const std::vector<size_t>& pool_gpr_idxs,
const emitter_context *emit_context) const override {
if (host_isa_ == dnnl::impl::cpu::x64::cpu_isa_t::avx512_core) {
Xbyak::Zmm in = Xbyak::Zmm(in_vec_idxs[0]);
Xbyak::Ymm out = Xbyak::Ymm(out_vec_idxs[0]);
Xbyak::Zmm aux = Xbyak::Zmm(aux_vec_idxs[0]);
Xbyak::Zmm aux1 = Xbyak::Zmm(aux_vec_idxs[1]);
if (host_isa_ == dnnl::impl::cpu::x64::avx512_core) {
emit_isa<dnnl::impl::cpu::x64::avx512_core>(in_vec_idxs, out_vec_idxs);
} else if (host_isa_ == dnnl::impl::cpu::x64::avx2) {
emit_isa<dnnl::impl::cpu::x64::avx2>(in_vec_idxs, out_vec_idxs);
} else if (host_isa_ == dnnl::impl::cpu::x64::sse41) {
emit_isa<dnnl::impl::cpu::x64::sse41>(in_vec_idxs, out_vec_idxs);
} else {
assert(!"unsupported isa");
}
}
template <dnnl::impl::cpu::x64::cpu_isa_t isa>
void emit_isa(const std::vector<size_t>& in_vec_idxs, const std::vector<size_t>& out_vec_idxs) const {
using namespace Xbyak;
using Vmm = typename dnnl::impl::utils::conditional3<isa == dnnl::impl::cpu::x64::sse41, Xmm, isa == dnnl::impl::cpu::x64::avx2, Ymm, Zmm>::type;
Vmm in = Vmm(in_vec_idxs[0]);
if (dnnl::impl::cpu::x64::mayiuse(dnnl::impl::cpu::x64::avx512_core_bf16)) {
Ymm out = Ymm(out_vec_idxs[0]);
h->vcvtneps2bf16(out, in);
} else if (host_isa_ == dnnl::impl::cpu::x64::cpu_isa_t::avx512_core) {
Zmm aux = Zmm(aux_vec_idxs[0]);
Zmm aux1 = Zmm(aux_vec_idxs[1]);
Ymm out = Ymm(out_vec_idxs[0]);
h->uni_vpsrld(aux, in, 16);
h->vpandd(aux, aux, table_val("one"));
@@ -36,11 +57,32 @@ private:
h->vfixupimmps(aux, in, table_val("selector"), 0);
h->vpsrad(aux, aux, 16);
h->vpmovdw(out, aux);
} else {
assert(!"unsupported isa");
}
};
} else { // round_to_nearest_even emulation
Vmm aux = Vmm(aux_vec_idxs[0]);
Xmm out = Xmm(out_vec_idxs[0]);
if (host_isa_ == dnnl::impl::cpu::x64::cpu_isa_t::avx2) {
h->uni_vandps(aux, in, table_val("rounding"));
} else {
h->uni_vmovups(aux, in);
h->uni_vandps(aux, aux, table_val("rounding"));
}
h->uni_vpsrld(aux, aux, 1);
h->uni_vpaddd(aux, aux, in);
h->uni_vpsrld(aux, aux, 16);
// dword to word using truncation
h->uni_vandps(aux, aux, table_val("mask_truncation_word"));
h->uni_vpackusdw(aux, aux, aux);
if (host_isa_ == dnnl::impl::cpu::x64::cpu_isa_t::avx2) {
h->vextracti128(out, Ymm(aux.getIdx()), 0);
} else {
h->uni_vmovups(out, aux);
}
}
}
inline int encode_fixup_selector(int input, int output) {
return ((output) << (4 * (input)));
@@ -66,10 +108,16 @@ private:
encode_fixup_selector(fixup_input_code_pinf_, fixup_output_code_copy_input_);
push_arg_entry_of("one", 0x00000001, true);
push_arg_entry_of("even", 0x00007fff, true);
push_arg_entry_of("rounding", 0x00010000, true);
push_arg_entry_of("selector", selector_int32, true);
push_arg_entry_of("mask_truncation_word", 0x0000ffff, true);
}
size_t aux_vecs_count() const override { return 2; }
size_t aux_vecs_count() const override {
if (dnnl::impl::cpu::x64::mayiuse(dnnl::impl::cpu::x64::avx512_core_bf16))
return 0;
return host_isa_ == dnnl::impl::cpu::x64::avx512_core ? 2 : 1;
}
};
} // namespace intel_cpu

33
src/plugins/intel_cpu/src/emitters/jit_conversion_emitters.cpp
View File

@@ -22,8 +22,8 @@ jit_convert_emitter::jit_convert_emitter(jit_generator *host, cpu_isa_t host_isa
input_type = node->get_input_element_type(0);
output_type = node->get_output_element_type(0);
if (!mayiuse(avx512_core_bf16) && mayiuse(avx512_core))
emu_vcvtneps2bf16.reset(new jit_emu_vcvtneps2bf16(host, host_isa));
if (output_type == ov::element::bf16)
uni_vcvtneps2bf16.reset(new jit_uni_vcvtneps2bf16(host, host_isa));
}
void jit_convert_emitter::validate_types() const {
@@ -42,22 +42,19 @@ size_t jit_convert_emitter::get_inputs_num() const { return 1; }
void jit_convert_emitter::emit_data() const {
jit_emitter::emit_data();
if (emu_vcvtneps2bf16)
emu_vcvtneps2bf16->emit_data();
if (uni_vcvtneps2bf16)
uni_vcvtneps2bf16->emit_data();
}
template <dnnl::impl::cpu::x64::cpu_isa_t isa>
void jit_convert_emitter::float2bfloat(const std::vector<size_t> &in_vec_idxs, const std::vector<size_t> &out_vec_idxs) const {
Zmm zmm_src = Zmm(in_vec_idxs[0]);
Zmm zmm_dst = Zmm(out_vec_idxs[0]);
using Vmm = typename conditional3<isa == cpu::x64::sse41, Xmm, isa == cpu::x64::avx2, Ymm, Zmm>::type;
Vmm vmm_src = Vmm(in_vec_idxs[0]);
Vmm vmm_dst = Vmm(out_vec_idxs[0]);
if (!uni_vcvtneps2bf16)
IE_THROW() << "Converter from float to bf16 isn't initialized!";
if (mayiuse(avx512_core_bf16)) {
h->vcvtneps2bf16(zmm_dst, zmm_src);
} else {
if (!emu_vcvtneps2bf16)
IE_THROW() << "Converter from float to bf16 isn't initialized!";
emu_vcvtneps2bf16->emit_code({static_cast<size_t>(zmm_src.getIdx())}, {static_cast<size_t>(zmm_dst.getIdx())});
}
uni_vcvtneps2bf16->emit_code({static_cast<size_t>(vmm_src.getIdx())}, {static_cast<size_t>(vmm_dst.getIdx())});
}
jit_convert_truncation_emitter::jit_convert_truncation_emitter(jit_generator *host, cpu_isa_t host_isa,
@@ -135,12 +132,12 @@ void jit_convert_truncation_emitter::emit_isa(const std::vector<size_t> &in_vec_
break;
case ov::element::bf16:
if (input_type == ov::element::f32) {
float2bfloat({static_cast<size_t>(vmm_src.getIdx())}, {static_cast<size_t>(vmm_dst.getIdx())});
float2bfloat<isa>({static_cast<size_t>(vmm_src.getIdx())}, {static_cast<size_t>(vmm_dst.getIdx())});
} else {
if (one_of(input_type, ov::element::i8, ov::element::u8)) {
h->uni_vcvtdq2ps(vmm_dst, vmm_dst);
}
float2bfloat({static_cast<size_t>(vmm_dst.getIdx())}, {static_cast<size_t>(vmm_dst.getIdx())});
float2bfloat<isa>({static_cast<size_t>(vmm_dst.getIdx())}, {static_cast<size_t>(vmm_dst.getIdx())});
}
break;
case ov::element::i8:
@@ -249,12 +246,12 @@ void jit_convert_saturation_emitter::emit_isa(const std::vector<size_t> &in_vec_
break;
case ov::element::bf16:
if (input_type == ov::element::f32) {
float2bfloat({static_cast<size_t>(vmm_src.getIdx())}, {static_cast<size_t>(vmm_dst.getIdx())});
float2bfloat<isa>({static_cast<size_t>(vmm_src.getIdx())}, {static_cast<size_t>(vmm_dst.getIdx())});
} else {
if (one_of(input_type, ov::element::i8, ov::element::u8)) {
h->uni_vcvtdq2ps(vmm_dst, vmm_dst);
}
float2bfloat({static_cast<size_t>(vmm_dst.getIdx())}, {static_cast<size_t>(vmm_dst.getIdx())});
float2bfloat<isa>({static_cast<size_t>(vmm_dst.getIdx())}, {static_cast<size_t>(vmm_dst.getIdx())});
}
break;
case ov::element::i8:

3
src/plugins/intel_cpu/src/emitters/jit_conversion_emitters.hpp
View File

@@ -22,6 +22,7 @@ protected:
void emit_data() const override;
void validate_types() const;
template <dnnl::impl::cpu::x64::cpu_isa_t isa>
void float2bfloat(const std::vector<size_t> &in_vec_idxs, const std::vector<size_t> &out_vec_idxs) const;
ov::element::Type input_type;
@@ -35,7 +36,7 @@ protected:
ov::element::u8
};
std::shared_ptr<jit_emu_vcvtneps2bf16> emu_vcvtneps2bf16 = nullptr;
std::shared_ptr<jit_uni_vcvtneps2bf16> uni_vcvtneps2bf16 = nullptr;
};
// This emitter is covered by specification of "Convert" operation. The implementation uses a "warp-around" conversion.

52
src/plugins/intel_cpu/src/emitters/jit_load_store_emitters.cpp
View File

@@ -553,9 +553,7 @@ jit_store_emitter::jit_store_emitter(dnnl::impl::cpu::x64::jit_generator *host,
prepare_table();
v_len_elt_ = get_vec_length() / exec_prc.size();
store_size_ = store_num * dst_prc.size();
if (!mayiuse(cpu::x64::avx512_core_bf16) && mayiuse(cpu::x64::avx512_core)) {
emu_vcvtneps2bf16_.reset(new jit_emu_vcvtneps2bf16(host, host_isa));
}
uni_vcvtneps2bf16_.reset(new jit_uni_vcvtneps2bf16(host, host_isa));
}
inline bool jit_store_emitter::is_saturation() const {
@@ -588,6 +586,10 @@ size_t jit_store_emitter::aux_vecs_count() const {
(src_prc_ == Precision::FP32 && dst_prc_ == Precision::BF16))
count++;
// for data swapping to avoid using Xmm(0) as I/O xmm for jit_uni_vcvtneps2bf16
if ((host_isa_ == cpu::x64::sse41) && (src_prc_ == Precision::FP32 && dst_prc_ == Precision::BF16))
count++;
// zero value, zeroed and passed from caller from performance standpoint(zeroed one time and not need preserve and restore status)
if (mayiuse(cpu::x64::avx512_core) && one_of(dst_prc_, Precision::U8, Precision::U16))
count++;
@@ -599,8 +601,8 @@ size_t jit_store_emitter::get_inputs_num() const { return 1; }
void jit_store_emitter::emit_data() const {
jit_emitter::emit_data();
if (emu_vcvtneps2bf16_)
emu_vcvtneps2bf16_->emit_data();
if (uni_vcvtneps2bf16_)
uni_vcvtneps2bf16_->emit_data();
}
void jit_store_emitter::emit_impl(const std::vector<size_t> &in_idxs, const std::vector<size_t> &out_idxs,
@@ -1014,19 +1016,35 @@ void jit_store_emitter::store_dword_to_word_extension(const Vmm &vmm, const Xbya
};
if (is_bf16) {
// to avoid src vmm pollution
if (src_prc_ == Precision::FP32) {
ymm = Ymm(aux_vec_idxs[0]);
}
if (mayiuse(cpu::x64::avx512_core_bf16)) {
h->vcvtneps2bf16(ymm, zmm);
if (mayiuse(cpu::x64::avx512_core)) {
// to avoid src vmm pollution
if (src_prc_ == Precision::FP32) {
ymm = Ymm(aux_vec_idxs[0]);
}
uni_vcvtneps2bf16_->emit_code({static_cast<size_t>(zmm.getIdx())}, {static_cast<size_t>(ymm.getIdx())});
if (store_num == 16) {
h->vmovdqu16(ptr[reg + offset], ymm);
} else {
store_bytes(ymm, reg, offset, store_num * 2);
}
} else {
emu_vcvtneps2bf16_->emit_code({static_cast<size_t>(vmm.getIdx())}, {static_cast<size_t>(ymm.getIdx())});
}
if (store_num == 16) {
h->vmovdqu16(ptr[reg + offset], ymm);
} else {
store_bytes(ymm, reg, offset, store_num * 2);
// to avoid src vmm pollution
if (src_prc_ == Precision::FP32) {
xmm = Xmm(aux_vec_idxs[0]);
}
// For sse41 mask register has to be Xmm(0) so we cannot use Xmm(0) as I/O vmm in uni_vcvtneps2bf16_
if (host_isa_ == cpu::x64::sse41 && src_prc_ == Precision::FP32) {
auto xmm_aux1 = Xmm(aux_vec_idxs[1]);
h->uni_vmovups(xmm_aux1, vmm);
uni_vcvtneps2bf16_->emit_code({static_cast<size_t>(vmm.getIdx())}, {static_cast<size_t>(vmm.getIdx())},
{static_cast<size_t>(xmm.getIdx())});
h->uni_vmovups(xmm, vmm);
h->uni_vmovups(vmm, xmm_aux1); // return original data to src vmm
} else {
uni_vcvtneps2bf16_->emit_code({static_cast<size_t>(vmm.getIdx())}, {static_cast<size_t>(xmm.getIdx())});
}
store_bytes(xmm, reg, offset, store_num * 2);
}
} else {
switch (store_num) {

6
src/plugins/intel_cpu/src/emitters/jit_load_store_emitters.hpp
View File

@@ -133,8 +133,8 @@ public:
void emit_data() const override;
std::shared_ptr<jit_emu_vcvtneps2bf16> get_emu_vcvtneps2bf16() const {
return emu_vcvtneps2bf16_;
std::shared_ptr<jit_uni_vcvtneps2bf16> get_uni_vcvtneps2bf16() const {
return uni_vcvtneps2bf16_;
}
private:
@@ -165,7 +165,7 @@ private:
Precision src_prc_;
Precision dst_prc_;
arithmetic_mode mode_ = arithmetic_mode::saturation;
std::shared_ptr<jit_emu_vcvtneps2bf16> emu_vcvtneps2bf16_;
std::shared_ptr<jit_uni_vcvtneps2bf16> uni_vcvtneps2bf16_;
};
} // namespace intel_cpu

15
src/plugins/intel_cpu/src/nodes/common/softmax.cpp
View File

@@ -65,8 +65,8 @@ struct jit_uni_softmax_kernel_f32 : public jit_uni_softmax_kernel, public jit_ge
void generate() override {
exp_injector.reset(new jit_uni_eltwise_injector_f32<isa>(this, dnnl::impl::alg_kind::eltwise_exp, 0.f, 0.f, 1.0f));
if (!mayiuse(avx512_core_bf16) && mayiuse(avx512_core))
emu_vcvtneps2bf16.reset(new jit_emu_vcvtneps2bf16(this, isa));
if (mayiuse(avx512_core))
uni_vcvtneps2bf16.reset(new jit_uni_vcvtneps2bf16(this, isa));
this->preamble();
@@ -164,8 +164,8 @@ struct jit_uni_softmax_kernel_f32 : public jit_uni_softmax_kernel, public jit_ge
this->postamble();
if (!mayiuse(avx512_core_bf16) && mayiuse(avx512_core))
emu_vcvtneps2bf16->emit_data();
if (uni_vcvtneps2bf16)
uni_vcvtneps2bf16->emit_data();
exp_injector->prepare_table();
}
@@ -191,7 +191,7 @@ private:
const Xbyak::Opmask k_mask = Xbyak::Opmask(1);
std::unique_ptr<jit_emu_vcvtneps2bf16> emu_vcvtneps2bf16;
std::unique_ptr<jit_uni_vcvtneps2bf16> uni_vcvtneps2bf16;
std::shared_ptr<jit_uni_eltwise_injector_f32<isa>> exp_injector;
@@ -218,10 +218,7 @@ private:
uni_vmovups(op, vmm_dst);
break;
case Precision::BF16:
if (mayiuse(avx512_core_bf16))
vcvtneps2bf16(ymm_dst, vmm_dst);
else
emu_vcvtneps2bf16->emit_code({static_cast<size_t>(vmm_dst.getIdx())}, {static_cast<size_t>(ymm_dst.getIdx())});
uni_vcvtneps2bf16->emit_code({static_cast<size_t>(vmm_dst.getIdx())}, {static_cast<size_t>(ymm_dst.getIdx())});
vmovdqu16(op, ymm_dst);
break;
default:

15
src/plugins/intel_cpu/src/nodes/eltwise.cpp
View File

@@ -167,8 +167,8 @@ struct jit_uni_eltwise_generic : public jit_uni_eltwise_kernel, public jit_gener
this, p->entry_[i], vmm_d_weights, vmm_d_bias, reg_d_weights, reg_d_bias));
}
if (!mayiuse(avx512_core_bf16) && mayiuse(avx512_core))
emu_vcvtneps2bf16.reset(new jit_emu_vcvtneps2bf16(this, isa));
if (mayiuse(avx512_core))
uni_vcvtneps2bf16.reset(new jit_uni_vcvtneps2bf16(this, isa));
const auto &jep = jep_;
@@ -359,8 +359,8 @@ struct jit_uni_eltwise_generic : public jit_uni_eltwise_kernel, public jit_gener
this->postamble();
if (!mayiuse(avx512_core_bf16) && mayiuse(avx512_core))
emu_vcvtneps2bf16->emit_data();
if (uni_vcvtneps2bf16)
uni_vcvtneps2bf16->emit_data();
eltwise_emitter->emit_data();
for (int i = 0; i < post_op_emitters.size(); i++) {
@@ -409,7 +409,7 @@ private:
Vmm vmm_d_bias = Vmm(13);
Vmm vmm_zero = Vmm(15);
std::shared_ptr<jit_emu_vcvtneps2bf16> emu_vcvtneps2bf16;
std::shared_ptr<jit_uni_vcvtneps2bf16> uni_vcvtneps2bf16;
std::shared_ptr<jit_emitter> eltwise_emitter = nullptr;
std::vector<std::shared_ptr<jit_emitter>> post_op_emitters = {};
@@ -703,10 +703,7 @@ private:
uni_vmovups(op, vmm_dst);
break;
case Precision::BF16:
if (mayiuse(avx512_core_bf16))
vcvtneps2bf16(ymm_dst, vmm_dst);
else
emu_vcvtneps2bf16->emit_code({static_cast<size_t>(vmm_dst.getIdx())}, {static_cast<size_t>(ymm_dst.getIdx())});
uni_vcvtneps2bf16->emit_code({static_cast<size_t>(vmm_dst.getIdx())}, {static_cast<size_t>(ymm_dst.getIdx())});
vmovdqu16(op, ymm_dst);
break;
case Precision::I16:

15
src/plugins/intel_cpu/src/nodes/normalize.cpp
View File

@@ -229,8 +229,8 @@ struct jit_uni_normalize_kernel_f32 : public jit_uni_normalize_kernel, public ji
}
}
if (!mayiuse(avx512_core_bf16) && mayiuse(avx512_core))
emu_vcvtneps2bf16.reset(new jit_emu_vcvtneps2bf16(this, isa));
if (mayiuse(avx512_core))
uni_vcvtneps2bf16.reset(new jit_uni_vcvtneps2bf16(this, isa));
this->preamble();
@@ -255,8 +255,8 @@ struct jit_uni_normalize_kernel_f32 : public jit_uni_normalize_kernel, public ji
this->postamble();
if (!mayiuse(avx512_core_bf16) && mayiuse(avx512_core) && emu_vcvtneps2bf16 != nullptr)
emu_vcvtneps2bf16->emit_data();
if (uni_vcvtneps2bf16)
uni_vcvtneps2bf16->emit_data();
for (auto& inj : eltwise_injectors)
inj->prepare_table();
}
@@ -296,7 +296,7 @@ private:
Vmm vmm_d_bias = Vmm(6);
Vmm vmm_zero = Vmm(7);
std::unique_ptr<jit_emu_vcvtneps2bf16> emu_vcvtneps2bf16 = nullptr;
std::unique_ptr<jit_uni_vcvtneps2bf16> uni_vcvtneps2bf16 = nullptr;
std::vector<std::shared_ptr<jit_uni_eltwise_injector_f32<isa>>> eltwise_injectors;
std::vector<std::shared_ptr<jit_uni_depthwise_injector_f32<isa>>> depthwise_injectors;
@@ -571,10 +571,7 @@ private:
if (dst_dt == memory::data_type::f32) {
uni_vmovups(op, vmm_dst);
} else if (dst_dt == memory::data_type::bf16) {
if (mayiuse(avx512_core_bf16))
vcvtneps2bf16(ymm_dst, vmm_dst);
else
emu_vcvtneps2bf16->emit_code({static_cast<size_t>(vmm_dst.getIdx())}, {static_cast<size_t>(ymm_dst.getIdx())});
uni_vcvtneps2bf16->emit_code({static_cast<size_t>(vmm_dst.getIdx())}, {static_cast<size_t>(ymm_dst.getIdx())});
vmovdqu16(op, ymm_dst);
} else if (dst_dt == memory::data_type::u8) {
uni_vcvtps2dq(vmm_dst, vmm_dst);

30
src/plugins/intel_cpu/src/nodes/reduce.cpp
View File

@@ -124,8 +124,8 @@ struct jit_uni_reduce_kernel_f32 : public jit_uni_reduce_kernel, public jit_gene
exp_injector = std::make_shared<jit_uni_eltwise_injector_f32<isa>>(this, alg_kind::eltwise_exp, 0.f, 0.f, 1);
}
if (!mayiuse(avx512_core_bf16) && mayiuse(avx512_core))
emu_vcvtneps2bf16 = std::make_shared<jit_emu_vcvtneps2bf16>(this, isa);
if (mayiuse(avx512_core))
uni_vcvtneps2bf16 = std::make_shared<jit_uni_vcvtneps2bf16>(this, isa);
this->preamble();
@@ -155,8 +155,8 @@ struct jit_uni_reduce_kernel_f32 : public jit_uni_reduce_kernel, public jit_gene
this->postamble();
if (!mayiuse(avx512_core_bf16) && mayiuse(avx512_core))
emu_vcvtneps2bf16->emit_data();
if (mayiuse(avx512_core))
uni_vcvtneps2bf16->emit_data();
if (jcp_.reduce_mode == Algorithm::ReduceAnd || jcp_.reduce_mode == Algorithm::ReduceL1 || jcp_.reduce_mode == Algorithm::ReduceMax ||
jcp_.reduce_mode == Algorithm::ReduceMin || jcp_.reduce_mode == Algorithm::ReduceProd || jcp_.reduce_mode == Algorithm::ReduceOr) {
@@ -210,7 +210,7 @@ private:
Xbyak::Label l_table;
std::shared_ptr<jit_emu_vcvtneps2bf16> emu_vcvtneps2bf16;
std::shared_ptr<jit_uni_vcvtneps2bf16> uni_vcvtneps2bf16;
std::shared_ptr<jit_uni_eltwise_injector_f32<isa>> exp_injector;
inline void reduce_main() {
@@ -913,10 +913,7 @@ private:
uni_vmovups(op, vmm_dst);
break;
case memory::data_type::bf16:
if (mayiuse(avx512_core_bf16))
vcvtneps2bf16(ymm_dst, vmm_dst);
else
emu_vcvtneps2bf16->emit_code({static_cast<size_t>(vmm_dst.getIdx())}, {static_cast<size_t>(ymm_dst.getIdx())});
uni_vcvtneps2bf16->emit_code({static_cast<size_t>(vmm_dst.getIdx())}, {static_cast<size_t>(ymm_dst.getIdx())});
vmovdqu16(op, ymm_dst);
break;
case memory::data_type::s8:
@@ -1109,8 +1106,8 @@ struct jit_uni_reduce_post_kernel_f32 : public jit_uni_reduce_post_kernel, publi
log_injector = std::make_shared<jit_uni_eltwise_injector_f32<isa>>(this, alg_kind::eltwise_log, 0.f, 0.f, 1.f);
}
if (!mayiuse(avx512_core_bf16) && mayiuse(avx512_core))
emu_vcvtneps2bf16 = std::make_shared<jit_emu_vcvtneps2bf16>(this, isa);
if (mayiuse(avx512_core))
uni_vcvtneps2bf16 = std::make_shared<jit_uni_vcvtneps2bf16>(this, isa);
this->preamble();
@@ -1158,8 +1155,8 @@ struct jit_uni_reduce_post_kernel_f32 : public jit_uni_reduce_post_kernel, publi
this->postamble();
if (!mayiuse(avx512_core_bf16) && mayiuse(avx512_core))
emu_vcvtneps2bf16->emit_data();
if (mayiuse(avx512_core))
uni_vcvtneps2bf16->emit_data();
if (jcp_.reduce_mode == Algorithm::ReduceLogSum || jcp_.reduce_mode == Algorithm::ReduceLogSumExp) {
log_injector->prepare_table();
@@ -1205,7 +1202,7 @@ private:
Vmm vmm_d_weights = Vmm(7);
Vmm vmm_d_bias = Vmm(8);
std::shared_ptr<jit_emu_vcvtneps2bf16> emu_vcvtneps2bf16;
std::shared_ptr<jit_uni_vcvtneps2bf16> uni_vcvtneps2bf16;
std::shared_ptr<jit_uni_eltwise_injector_f32<isa>> log_injector;
std::vector<std::shared_ptr<jit_uni_eltwise_injector_f32<isa>>> eltwise_injectors;
@@ -1532,10 +1529,7 @@ private:
uni_vmovups(op, vmm_dst);
break;
case memory::data_type::bf16:
if (mayiuse(avx512_core_bf16))
vcvtneps2bf16(ymm_dst, vmm_dst);
else
emu_vcvtneps2bf16->emit_code({static_cast<size_t>(vmm_dst.getIdx())}, {static_cast<size_t>(ymm_dst.getIdx())});
uni_vcvtneps2bf16->emit_code({static_cast<size_t>(vmm_dst.getIdx())}, {static_cast<size_t>(ymm_dst.getIdx())});
vmovdqu16(op, ymm_dst);
break;
case memory::data_type::s8:

15
src/plugins/intel_cpu/src/nodes/region_yolo.cpp
View File

@@ -41,8 +41,8 @@ struct jit_uni_logistic_kernel_f32 : public jit_uni_logistic_kernel, public jit_
void generate() override {
exp_injector.reset(new jit_uni_eltwise_injector_f32<isa>(this, dnnl::impl::alg_kind::eltwise_exp, 0.f, 0.f, 1.f));
if (!mayiuse(avx512_core_bf16) && mayiuse(avx512_core))
emu_vcvtneps2bf16.reset(new jit_emu_vcvtneps2bf16(this, isa));
if (mayiuse(avx512_core))
uni_vcvtneps2bf16.reset(new jit_uni_vcvtneps2bf16(this, isa));
this->preamble();
@@ -91,8 +91,8 @@ struct jit_uni_logistic_kernel_f32 : public jit_uni_logistic_kernel, public jit_
this->postamble();
if (!mayiuse(avx512_core_bf16) && mayiuse(avx512_core))
emu_vcvtneps2bf16->emit_data();
if (uni_vcvtneps2bf16)
uni_vcvtneps2bf16->emit_data();
exp_injector->prepare_table();
@@ -119,7 +119,7 @@ private:
const Xbyak::Opmask k_mask = Xbyak::Opmask(1);
std::unique_ptr<jit_emu_vcvtneps2bf16> emu_vcvtneps2bf16;
std::unique_ptr<jit_uni_vcvtneps2bf16> uni_vcvtneps2bf16;
Xbyak::Label l_table;
@@ -192,10 +192,7 @@ private:
uni_vmovups(op, vmm_dst);
break;
case InferenceEngine::Precision::BF16:
if (mayiuse(avx512_core_bf16))
vcvtneps2bf16(ymm_dst, vmm_dst);
else
emu_vcvtneps2bf16->emit_code({static_cast<size_t>(vmm_dst.getIdx())}, {static_cast<size_t>(ymm_dst.getIdx())});
uni_vcvtneps2bf16->emit_code({static_cast<size_t>(vmm_dst.getIdx())}, {static_cast<size_t>(ymm_dst.getIdx())});
vmovdqu16(op, ymm_dst);
break;
default:

10
src/plugins/intel_cpu/src/utils/jit_kernel.cpp
View File

@@ -128,6 +128,16 @@ InferenceEngine::Precision type2precision<float>() {
return InferenceEngine::Precision::FP32;
}
template<>
InferenceEngine::Precision type2precision<int32_t>() {
return InferenceEngine::Precision::I32;
}
template<>
InferenceEngine::Precision type2precision<bfloat16_t>() {
return InferenceEngine::Precision::BF16;
}
template<>
InferenceEngine::Precision type2precision<uint8_t>() {
return InferenceEngine::Precision::U8;

50
src/plugins/intel_cpu/tests/unit/jit_kernel_test.cpp
View File

@@ -268,9 +268,9 @@ struct jit_variable_load_store_test_kernel {
size_t size;
};
template<size_t N>
template<size_t N, bool is_src>
void test() {
kernel_impl<N> kernel;
kernel_impl<N, is_src> kernel;
kernel.init();
const size_t size = 3;
@@ -297,7 +297,7 @@ struct jit_variable_load_store_test_kernel {
}
private:
template<size_t N>
template<size_t N, bool is_src>
class kernel_impl : public jit_test_kernel<Params> {
public:
void generate() override {
@@ -307,10 +307,10 @@ private:
auto dst_ptr = jit_kernel::arg(&Params::dst);
auto size = jit_kernel::arg(&Params::size);
auto dst = jit_kernel::var<DstT[N]>();
auto interm = jit_kernel::var<typename std::conditional<is_src, SrcT[N], DstT[N]>::type>();
jit_kernel::load(dst, src_ptr, size);
jit_kernel::store(dst_ptr, dst, size);
jit_kernel::load(interm, src_ptr, size);
jit_kernel::store(dst_ptr, interm, size);
jit_kernel::postamble();
}
@@ -321,26 +321,52 @@ TEST(JitKernel, variable_load_and_store) {
{
jit_variable_load_store_test_kernel<uint8_t, float> kernel;
if (mayiuse(cpu_isa_t::avx512_core)) {
kernel.test<16>();
kernel.test<16, false>();
}
if (mayiuse(cpu_isa_t::avx2)) {
kernel.test<8>();
kernel.test<8, false>();
}
if (mayiuse(cpu_isa_t::sse41)) {
kernel.test<4>();
kernel.test<4, false>();
}
}
{
jit_variable_load_store_test_kernel<int8_t, int8_t> kernel;
if (mayiuse(cpu_isa_t::avx512_core)) {
kernel.test<16>();
kernel.test<16, false>();
}
if (mayiuse(cpu_isa_t::avx2)) {
kernel.test<8>();
kernel.test<8, false>();
}
if (mayiuse(cpu_isa_t::sse41)) {
kernel.test<4>();
kernel.test<4, false>();
}
}
{
jit_variable_load_store_test_kernel<float, bfloat16_t> kernel;
if (mayiuse(cpu_isa_t::avx512_core)) {
kernel.test<16, true>();
}
if (mayiuse(cpu_isa_t::avx2)) {
kernel.test<8, true>();
}
if (mayiuse(cpu_isa_t::sse41)) {
kernel.test<4, true>();
}
}
{
jit_variable_load_store_test_kernel<int32_t, bfloat16_t> kernel;
if (mayiuse(cpu_isa_t::avx512_core)) {
kernel.test<16, true>();
}
if (mayiuse(cpu_isa_t::avx2)) {
kernel.test<8, true>();
}
if (mayiuse(cpu_isa_t::sse41)) {
kernel.test<4, true>();
}
}
}