IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[CPU] RandomUniform-8 implementation. (#20171)

Browse Source
This commit is contained in:
Nikolay Shchegolev
2023-10-18 12:50:46 +04:00
committed by GitHub
parent 5f336a6870
commit 4eab5b4635
29 changed files with 2320 additions and 159 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/core/src/op/random_uniform.cpp
+2
View File
@@ -92,6 +92,8 @@ bool RandomUniform::evaluate(TensorVector& outputs, const TensorVector& inputs)
const auto& t_out = get_out_type();
OPENVINO_ASSERT(validate::out_et(t_out), "Unsupported type of RandomUniform: " + t_out.get_type_name());
outputs[0].set_shape(out_shape);
auto state = ov::reference::random_uniform(out_dims.data(),
static_cast<const char*>(inputs[1].data()),
static_cast<const char*>(inputs[2].data()),
src/core/tests/copy.cpp
+2 -2
View File
@@ -447,12 +447,12 @@ TEST(copy, random_uniform) {
const auto min_val_param = make_shared<ov::op::v0::Parameter>(element::f32, Shape{1});
const auto max_val_param = make_shared<ov::op::v0::Parameter>(element::f32, Shape{1});
auto out_shape = make_shared<ov::op::v0::Constant>(element::i64, Shape{3}, std::vector<int64_t>{1, 2, 3});
auto out_shape = make_shared<ov::op::v0::Constant>(element::i64, Shape{3}, shape);
auto ru =
std::make_shared<ov::op::v8::RandomUniform>(out_shape, min_val_param, max_val_param, element::f32, 150, 10);
// Call `evaluate` to update m_state
auto outputs = ov::TensorVector{{element::i64, out_shape->get_shape(), shape.data()}};
auto outputs = ov::TensorVector{{element::i64, {1lu, 2lu, 3lu}}};
ru->evaluate(outputs,
ov::TensorVector{{element::i64, out_shape->get_shape(), shape.data()},
{element::f32, min_val_param->get_shape(), &min},
src/frontends/onnx/frontend/src/op/random_uniform.cpp
+1
View File
@@ -29,6 +29,7 @@ OutputVector random_uniform(const Node& node) {
const auto target_type = common::get_ngraph_element_type(dtype);
const uint64_t global_seed = 0;
// TODO: This multiplication leads to a mismatch in accuracy. Issue: 123003
const auto seed_uint64 = static_cast<uint64_t>(seed * 1000);
return {std::make_shared<ngraph::opset8::RandomUniform>(target_shape_const,
src/plugins/intel_cpu/src/cpu_types.cpp
+2
View File
@@ -123,6 +123,7 @@ static const TypeToNameMap& get_type_to_name_tbl() {
{ "ScatterElementsUpdate", Type::ScatterElementsUpdate},
{ "ScatterNDUpdate", Type::ScatterNDUpdate},
{ "Interpolate", Type::Interpolate},
{ "RandomUniform", Type::RandomUniform},
{ "ReduceL1", Type::Reduce},
{ "ReduceL2", Type::Reduce},
{ "ReduceLogicalAnd", Type::Reduce},
@@ -317,6 +318,7 @@ std::string NameFromType(const Type type) {
CASE(PriorBox);
CASE(PriorBoxClustered)
CASE(MHA);
CASE(RandomUniform);
CASE(Unique);
CASE(Ngram);
CASE(Unknown);
src/plugins/intel_cpu/src/cpu_types.h
+1
View File
@@ -110,6 +110,7 @@ enum class Type {
PriorBoxClustered,
Interaction,
MHA,
RandomUniform,
Unique,
Ngram
};
src/plugins/intel_cpu/src/node.cpp
+2 -2
View File
@@ -95,8 +95,6 @@ Node::Node(const std::shared_ptr<ngraph::Node>& op,
typeStr(op->get_type_name()),
type(TypeFromName(op->get_type_name())),
profiling(op->get_friendly_name()) {
const std::string errorPrefix = "Ngraph operation " + std::string(op->get_type_name()) + " with name " + op->get_friendly_name();
for (size_t i = 0; i < op->get_input_size(); i++) {
const auto &shape = op->get_input_partial_shape(i);
if (shape.rank().is_dynamic()) {
@@ -480,6 +478,8 @@ std::string Node::getPrimitiveDescriptorType() const {
SEARCH_TYPE(_dw);
SEARCH_TYPE(_1x1);
#undef SEARCH_TYPE
if (type == impl_desc_type::unknown)
str_type = "unknown";
else if (str_type.empty())
src/plugins/intel_cpu/src/node.h
+3 -1
View File
@@ -41,6 +41,8 @@
#include "nodes/executors/mvn_list.hpp"
#include "nodes/executors/executor.hpp"
#define THROW_CPU_NODE_ERR(...) OPENVINO_THROW(getTypeStr(), " node with name '", getName(), "' ", __VA_ARGS__)
namespace ov {
namespace intel_cpu {
@@ -353,7 +355,7 @@ public:
inplace = InPlaceType::Unknown;
}
std::string getPrimitiveDescriptorType() const;
virtual std::string getPrimitiveDescriptorType() const;
PerfCount &PerfCounter() { return perfCounter; }
src/plugins/intel_cpu/src/nodes/grid_sample.cpp
+16 -8
View File
@@ -10,9 +10,11 @@
#include <ngraph/opsets/opset1.hpp>
using namespace InferenceEngine;
using namespace dnnl::impl::cpu;
using namespace ov::intel_cpu;
using namespace ov::intel_cpu::node;
#if defined(OPENVINO_ARCH_X86_64)
using namespace dnnl::impl::cpu;
#endif // OPENVINO_ARCH_X86_64
#define THROW_ERROR IE_THROW() << getTypeStr() << " node with name '" << getName() << "' "
@@ -23,10 +25,14 @@ bool GridSample::isSupportedOperation(const std::shared_ptr<const ov::Node>& op,
errorMessage = "Not supported GridSample operation version. CPU plug-in supports only 9th version.";
return false;
}
#if defined(OPENVINO_ARCH_X86_64)
if (!x64::mayiuse(x64::sse41)) {
errorMessage = "Not supported CPU instructions set.";
return false;
}
#else
return false;
#endif // OPENVINO_ARCH_X86_64
} catch (...) {
return false;
}
@@ -34,6 +40,8 @@ bool GridSample::isSupportedOperation(const std::shared_ptr<const ov::Node>& op,
return true;
}
#if defined(OPENVINO_ARCH_X86_64)
GridSample::GridSample(const std::shared_ptr<ov::Node>& op, const GraphContext::CPtr context)
: Node(op, context, NgraphShapeInferFactory(op, PortMask(1))) {
std::string errorMessage;
@@ -110,7 +118,7 @@ void GridSample::initSupportedPrimitiveDescriptors() {
}
void GridSample::createPrimitive() {
GridSampleKernelConfParams jcp;
kernel::GridSampleKernelConfParams jcp;
jcp.inDataPrc = dataPrecision;
jcp.gridPrc = gridPrecision;
@@ -133,15 +141,13 @@ void GridSample::createPrimitive() {
jcp.cannelNum = jcp.dynamicChannel ? 1lu : srcDataDims[1];
}
#if defined(OPENVINO_ARCH_X86_64)
if (x64::mayiuse(x64::avx512_core)) {
jitKernel.reset(new GridSampleKernel<x64::avx512_core>(jcp));
jitKernel.reset(new kernel::GridSampleKernel<x64::avx512_core>(jcp));
} else if (x64::mayiuse(x64::avx2)) {
jitKernel.reset(new GridSampleKernel<x64::avx2>(jcp));
jitKernel.reset(new kernel::GridSampleKernel<x64::avx2>(jcp));
} else if (x64::mayiuse(x64::sse41)) {
jitKernel.reset(new GridSampleKernel<x64::sse41>(jcp));
jitKernel.reset(new kernel::GridSampleKernel<x64::sse41>(jcp));
}
#endif // OPENVINO_ARCH_X86_64
if (!jitKernel) {
THROW_ERROR << " could not create JIT kernel.";
}
@@ -268,7 +274,7 @@ void GridSample::execute(dnnl::stream strm) {
auto threadBody = [&](const int ithr, const int nthr) {
const auto& p = execParamsPerThread[ithr];
auto arg = GridSamplesKernelExecArgs();
auto arg = kernel::GridSamplesKernelExecArgs();
if (p.workAmount == 0lu) {
return;
}
@@ -311,3 +317,5 @@ void GridSample::executeDynamicImpl(dnnl::stream strm) {
bool GridSample::created() const {
return getType() == Type::GridSample;
}
#endif // OPENVINO_ARCH_X86_64
src/plugins/intel_cpu/src/nodes/grid_sample.hpp
+1 -1
View File
@@ -72,7 +72,7 @@ private:
static constexpr size_t IN_DATA = 0;
static constexpr size_t IN_GRID = 1;
std::shared_ptr<GridSampleKernelBase> jitKernel;
std::shared_ptr<kernel::GridSampleKernelBase> jitKernel;
};
} // namespace node
src/plugins/intel_cpu/src/nodes/kernels/x64/grid_sample.cpp
+4 -2
View File
@@ -1,4 +1,4 @@
// Copyright (C) 2022 Intel Corporation
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
@@ -8,12 +8,13 @@ using namespace dnnl::impl::cpu;
namespace ov {
namespace intel_cpu {
namespace kernel {
#define GET_OFF(field) offsetof(GridSamplesKernelExecArgs, field)
template <x64::cpu_isa_t isa>
GridSampleKernel<isa>::GridSampleKernel(const GridSampleKernelConfParams& jcp) :
GridSampleKernelBase(jit_name(), jcp) {
GridSampleKernelBase(jit_name(), jcp, isa) {
vlen = x64::cpu_isa_traits<isa>::vlen;
dataTypeSize = jcp.inDataPrc.size();
gridTypeSize = jcp.gridPrc.size();
@@ -2085,5 +2086,6 @@ template class GridSampleKernel<x64::avx512_core>;
template class GridSampleKernel<x64::avx2>;
template class GridSampleKernel<x64::sse41>;
} // namespace kernel
} // namespace intel_cpu
} // namespace ov
src/plugins/intel_cpu/src/nodes/kernels/x64/grid_sample.hpp
+12 -2
View File
@@ -1,4 +1,4 @@
// Copyright (C) 2022 Intel Corporation
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
@@ -14,6 +14,12 @@ namespace intel_cpu {
enum class GridSampleInterpolationMode { BILINEAR, BICUBIC, NEAREST };
enum class GridSamplePaddingMode { ZEROS, BORDER, REFLECTION };
namespace kernel {
class GridSampleKernelBase;
#if defined(OPENVINO_ARCH_X86_64)
struct GridSampleKernelConfParams {
bool dynamicShapes = false;
bool dynamicBatch = false;
@@ -66,7 +72,8 @@ public:
assert(ker_);
ker_(args);
}
explicit GridSampleKernelBase(const char* name, const GridSampleKernelConfParams& jcp) : JitKernelBase(name), ker_(nullptr), jcp(jcp) {}
explicit GridSampleKernelBase(const char* name, const GridSampleKernelConfParams& jcp, dnnl::impl::cpu::x64::cpu_isa_t isa)
: JitKernelBase(name, isa), ker_(nullptr), jcp(jcp) {}
virtual void create_ker() = 0;
uint64_t getVecLen() {
@@ -173,5 +180,8 @@ private:
void hwShiftPs2dq(const Vmm& vDst, const Vmm& vHCoord, const Vmm& vWCoord, const Vmm& vWidth);
};
#endif // OPENVINO_ARCH_X86_64
} // namespace kernel
} // namespace intel_cpu
} // namespace ov
src/plugins/intel_cpu/src/nodes/kernels/x64/jit_kernel_base.cpp
+236 -101
View File
@@ -1,172 +1,243 @@
// Copyright (C) 2022 Intel Corporation
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "jit_kernel_base.hpp"
using namespace ov;
using namespace intel_cpu;
using namespace dnnl::impl::cpu;
namespace ov {
namespace intel_cpu {
namespace kernel {
void JitKernelBase::uni_vfmsub132ps(const Xbyak::Xmm& vDst,
const Xbyak::Xmm& vSrc,
JitKernelBase::JitKernelBase(const char* name, x64::cpu_isa_t isa)
: x64::jit_generator(name, nullptr, x64::MAX_CODE_SIZE, true, isa), m_isa(isa) {
vlen = x64::isa_max_vlen(isa);
}
void JitKernelBase::uni_vfmsub132ps(const Xbyak::Xmm& v_dst,
const Xbyak::Xmm& v_src,
const Xbyak::Operand& op) {
if (isValidIsa(x64::avx2)) {
vfmsub132ps(vDst, vSrc, op);
vfmsub132ps(v_dst, v_src, op);
} else if (isValidIsa(x64::avx)) {
assert(vDst.getIdx() != vSrc.getIdx());
vmulps(vDst, vDst, op);
vsubps(vDst, vDst, vSrc);
assert(v_dst.getIdx() != v_src.getIdx());
vmulps(v_dst, v_dst, op);
vsubps(v_dst, v_dst, v_src);
} else {
assert(vDst.getIdx() != vSrc.getIdx());
mulps(vDst, op);
subps(vDst, vSrc);
assert(v_dst.getIdx() != v_src.getIdx());
mulps(v_dst, op);
subps(v_dst, v_src);
}
}
void JitKernelBase::uni_vfnmadd132ps(const Xbyak::Xmm& vDst,
const Xbyak::Xmm& vSrc,
void JitKernelBase::uni_vfnmadd132ps(const Xbyak::Xmm& v_dst,
const Xbyak::Xmm& v_src,
const Xbyak::Operand& op) {
if (isValidIsa(x64::avx2)) {
vfnmadd132ps(vDst, vSrc, op);
vfnmadd132ps(v_dst, v_src, op);
} else if (isValidIsa(x64::avx)) {
assert(vDst.getIdx() != vSrc.getIdx());
vmulps(vDst, vDst, op);
vsubps(vDst, vSrc, vDst);
assert(v_dst.getIdx() != v_src.getIdx());
vmulps(v_dst, v_dst, op);
vsubps(v_dst, v_src, v_dst);
} else {
assert(vDst.getIdx() != vSrc.getIdx());
mulps(vDst, op);
subps(vSrc, vDst);
movups(vDst, vSrc);
assert(v_dst.getIdx() != v_src.getIdx());
mulps(v_dst, op);
subps(v_src, v_dst);
movups(v_dst, v_src);
}
}
void JitKernelBase::uni_vfmsub231ps(const Xbyak::Xmm& vDst,
const Xbyak::Xmm& vSrc,
void JitKernelBase::uni_vfmsub231ps(const Xbyak::Xmm& v_dst,
const Xbyak::Xmm& v_src,
const Xbyak::Operand& op) {
if (isValidIsa(x64::avx2)) {
vfmsub231ps(vDst, vSrc, op);
vfmsub231ps(v_dst, v_src, op);
} else if (isValidIsa(x64::avx)) {
assert(!vDst.isEqualIfNotInherited(op));
vmulps(vSrc, vSrc, op);
vsubps(vDst, vSrc, vDst);
assert(!v_dst.isEqualIfNotInherited(op));
vmulps(v_src, v_src, op);
vsubps(v_dst, v_src, v_dst);
} else {
assert(!vDst.isEqualIfNotInherited(op));
mulps(vSrc, op);
subps(vSrc, vDst);
movups(vDst, vSrc);
assert(!v_dst.isEqualIfNotInherited(op));
mulps(v_src, op);
subps(v_src, v_dst);
movups(v_dst, v_src);
}
}
void JitKernelBase::uni_vpaddd(const Xbyak::Ymm& vDst,
const Xbyak::Ymm& vSrc,
void JitKernelBase::uni_vpaddd(const Xbyak::Ymm& v_dst,
const Xbyak::Ymm& v_src,
const Xbyak::Operand& op) {
if (isValidIsa(x64::avx2)) {
vpaddd(vDst, vSrc, op);
vpaddd(v_dst, v_src, op);
} else if (isValidIsa(x64::avx)) {
Xbyak::Xmm xmmDst(vDst.getIdx());
vmovups(vDst, vSrc);
Xbyak::Xmm xmmDst(v_dst.getIdx());
vmovups(v_dst, v_src);
if (op.isYMM()) {
Xbyak::Ymm ymmOp(op.getIdx());
Xbyak::Xmm xmmOp(op.getIdx());
paddd(xmmDst, xmmOp);
vperm2f128(vDst, vDst, vDst, 0x1);
vperm2f128(v_dst, v_dst, v_dst, 0x1);
vperm2f128(ymmOp, ymmOp, ymmOp, 0x1);
paddd(xmmDst, xmmOp);
vperm2f128(vDst, vDst, vDst, 0x1);
vperm2f128(v_dst, v_dst, v_dst, 0x1);
vperm2f128(ymmOp, ymmOp, ymmOp, 0x1);
} else if (op.isMEM()) {
const int vlen = x64::cpu_isa_traits<x64::sse41>::vlen;
paddd(xmmDst, op.getAddress());
vperm2f128(vDst, vDst, vDst, 0x1);
vperm2f128(v_dst, v_dst, v_dst, 0x1);
paddd(xmmDst, ptr[op.getAddress().getRegExp() + vlen]);
vperm2f128(vDst, vDst, vDst, 0x1);
vperm2f128(v_dst, v_dst, v_dst, 0x1);
} else {
IE_THROW() << "Not supported operand type.";
}
} else if (isValidIsa(x64::sse41)) {
assert(vDst.getIdx() != vSrc.getIdx());
paddd(vDst, op);
assert(v_dst.getIdx() != v_src.getIdx());
paddd(v_dst, op);
} else {
IE_THROW() << "Not defined behavior for instruction 'vpaddd' in current instructions set.";
}
}
void JitKernelBase::uni_vpsubd(const Xbyak::Ymm& vDst,
const Xbyak::Ymm& vSrc,
void JitKernelBase::uni_vpaddq(const Xbyak::Xmm& v_dst,
const Xbyak::Xmm& v_src,
const Xbyak::Operand& op) {
if (isValidIsa(x64::avx2)) {
vpsubd(vDst, vSrc, op);
vpaddq(v_dst, v_src, op);
} else {
if (v_dst.getIdx() != v_src.getIdx()) {
movups(v_dst, v_src);
}
paddq(v_dst, op);
}
}
void JitKernelBase::uni_vpsubd(const Xbyak::Ymm& v_dst,
const Xbyak::Ymm& v_src,
const Xbyak::Operand& op) {
if (isValidIsa(x64::avx2)) {
vpsubd(v_dst, v_src, op);
} else if (isValidIsa(x64::avx)) {
Xbyak::Xmm xmmDst(vDst.getIdx());
vmovups(vDst, vSrc);
Xbyak::Xmm xmmDst(v_dst.getIdx());
vmovups(v_dst, v_src);
if (op.isYMM()) {
Xbyak::Ymm ymmOp(op.getIdx());
Xbyak::Xmm xmmOp(op.getIdx());
psubd(xmmDst, xmmOp);
vperm2f128(vDst, vDst, vDst, 0x1);
vperm2f128(v_dst, v_dst, v_dst, 0x1);
vperm2f128(ymmOp, ymmOp, ymmOp, 0x1);
psubd(xmmDst, xmmOp);
vperm2f128(vDst, vDst, vDst, 0x1);
vperm2f128(v_dst, v_dst, v_dst, 0x1);
vperm2f128(ymmOp, ymmOp, ymmOp, 0x1);
} else if (op.isMEM()) {
const int vlen = x64::cpu_isa_traits<x64::sse41>::vlen;
psubd(xmmDst, op.getAddress());
vperm2f128(vDst, vDst, vDst, 0x1);
vperm2f128(v_dst, v_dst, v_dst, 0x1);
psubd(xmmDst, ptr[op.getAddress().getRegExp() + vlen]);
vperm2f128(vDst, vDst, vDst, 0x1);
vperm2f128(v_dst, v_dst, v_dst, 0x1);
} else {
IE_THROW() << "Not supported operand type.";
}
} else if (isValidIsa(x64::sse41)) {
assert(vDst.getIdx() != vSrc.getIdx());
psubd(vDst, op);
assert(v_dst.getIdx() != v_src.getIdx());
psubd(v_dst, op);
} else {
IE_THROW() << "Not defined behavior for instruction 'vpsubd' in current instructions set.";
}
}
void JitKernelBase::uni_vdivps(const Xbyak::Xmm& vDst,
void JitKernelBase::uni_vsubpd(const Xbyak::Xmm& v_dst,
const Xbyak::Xmm& v_src,
const Xbyak::Operand& op) {
if (isValidIsa(x64::avx)) {
vsubpd(v_dst, v_src, op);
} else {
if (v_dst.getIdx() != v_src.getIdx()) {
movups(v_dst, v_src);
}
subpd(v_dst, op);
}
}
void JitKernelBase::uni_vmulpd(const Xbyak::Xmm& v_dst,
const Xbyak::Xmm& v_src,
const Xbyak::Operand& op) {
if (isValidIsa(x64::avx)) {
vmulpd(v_dst, v_src, op);
} else {
if (v_dst.getIdx() != v_src.getIdx()) {
movups(v_dst, v_src);
}
mulpd(v_dst, op);
}
}
void JitKernelBase::uni_vpmuludq(const Xbyak::Xmm& v_dst,
const Xbyak::Xmm& v_src,
const Xbyak::Operand& op) {
if (isValidIsa(x64::avx2)) {
vpmuludq(v_dst, v_src, op);
} else {
if (v_dst.getIdx() != v_src.getIdx()) {
movups(v_dst, v_src);
}
pmuludq(v_dst, op);
}
}
void JitKernelBase::uni_vdivps(const Xbyak::Xmm& v_dst,
const Xbyak::Operand& op1,
const Xbyak::Operand& op2) {
if (isValidIsa(x64::avx)) {
vdivps(vDst, op1, op2);
vdivps(v_dst, op1, op2);
} else {
if (!vDst.isEqualIfNotInherited(op1)) {
movups(vDst, op1);
if (!v_dst.isEqualIfNotInherited(op1)) {
movups(v_dst, op1);
}
divps(vDst, op2);
divps(v_dst, op2);
}
}
void JitKernelBase::uni_vandps(const Xbyak::Xmm& vDst,
void JitKernelBase::uni_vdivpd(const Xbyak::Xmm& v_dst,
const Xbyak::Xmm& v_src,
const Xbyak::Operand& op) {
if (isValidIsa(x64::avx)) {
vdivpd(v_dst, v_src, op);
} else {
if (v_dst.getIdx() != v_src.getIdx()) {
movups(v_dst, v_src);
}
divpd(v_dst, op);
}
}
void JitKernelBase::uni_vandps(const Xbyak::Xmm& v_dst,
const Xbyak::Xmm& vSrs,
const Xbyak::Operand &op) {
if (isValidIsa(x64::avx)) {
vandps(vDst, vSrs, op);
vandps(v_dst, vSrs, op);
} else {
if (!vDst.isEqualIfNotInherited(vSrs)) {
movups(vDst, vSrs);
if (!v_dst.isEqualIfNotInherited(vSrs)) {
movups(v_dst, vSrs);
}
andps(vDst, op);
andps(v_dst, op);
}
}
void JitKernelBase::uni_vandnps(const Xbyak::Xmm& vDst,
void JitKernelBase::uni_vandnps(const Xbyak::Xmm& v_dst,
const Xbyak::Xmm& vSrs,
const Xbyak::Operand &op) {
if (isValidIsa(x64::avx)) {
vandnps(vDst, vSrs, op);
vandnps(v_dst, vSrs, op);
} else {
if (!vDst.isEqualIfNotInherited(vSrs)) {
movups(vDst, vSrs);
if (!v_dst.isEqualIfNotInherited(vSrs)) {
movups(v_dst, vSrs);
}
andnps(vDst, op);
andnps(v_dst, op);
}
}
void JitKernelBase::gatherdd(const Xbyak::Xmm& vDst,
void JitKernelBase::gatherdd(const Xbyak::Xmm& v_dst,
const Xbyak::Reg64& rSrcPtr,
const Xbyak::Xmm& vSrcShift,
const Xbyak::Opmask& kReadMask,
@@ -178,28 +249,28 @@ void JitKernelBase::gatherdd(const Xbyak::Xmm& vDst,
if (!useMask)
kxnord(kReadMask, kReadMask, kReadMask);
if (zeroFill)
uni_vpxor(vDst, vDst, vDst);
uni_vpxor(v_dst, v_dst, v_dst);
vpgatherdd(vDst | kReadMask, ptr[rSrcPtr + vSrcShift]);
vpgatherdd(v_dst | kReadMask, ptr[rSrcPtr + vSrcShift]);
}
void JitKernelBase::gatherdd(const Xbyak::Xmm& vDst,
void JitKernelBase::gatherdd(const Xbyak::Xmm& v_dst,
const Xbyak::Reg64& rSrcPtr,
const Xbyak::Xmm& vSrcShift,
const Xbyak::Xmm& vReadMask,
const bool useMask,
const bool zeroFill) {
if (vDst.getIdx() == vSrcShift.getIdx() || vDst.getIdx() == vReadMask.getIdx() || vSrcShift.getIdx() == vReadMask.getIdx()) {
if (v_dst.getIdx() == vSrcShift.getIdx() || v_dst.getIdx() == vReadMask.getIdx() || vSrcShift.getIdx() == vReadMask.getIdx()) {
IE_THROW() << "Any pair of the index, mask, or destination registers cannot be the same.";
}
if (zeroFill)
pxor(vDst, vDst); // Don't use vpxor. It zeros the rest of the YMM register.
pxor(v_dst, v_dst); // Don't use vpxor. It zeros the rest of the YMM register.
if (isValidIsa(x64::avx2)) {
if (!useMask)
uni_vpcmpeqd(vReadMask, vReadMask, vReadMask);
vpgatherdd(vDst, ptr[rSrcPtr + vSrcShift], vReadMask);
vpgatherdd(v_dst, ptr[rSrcPtr + vSrcShift], vReadMask);
} else {
auto rAux = getReg64();
Xbyak::Reg32 r32Aux = Xbyak::Reg32(rAux.getIdx());
@@ -213,7 +284,7 @@ void JitKernelBase::gatherdd(const Xbyak::Xmm& vDst,
je(lLoopNext, T_NEAR);
}
uni_vpextrd(r32Aux, vSrcShift, i);
pinsrd(vDst, ptr[rSrcPtr + rAux], i);
pinsrd(v_dst, ptr[rSrcPtr + rAux], i);
if (useMask)
L(lLoopNext);
@@ -221,30 +292,30 @@ void JitKernelBase::gatherdd(const Xbyak::Xmm& vDst,
}
}
void JitKernelBase::gatherdd(const Xbyak::Ymm& vDst,
void JitKernelBase::gatherdd(const Xbyak::Ymm& v_dst,
const Xbyak::Reg64& rSrcPtr,
const Xbyak::Ymm& vSrcShift,
const Xbyak::Ymm& vReadMask,
const bool useMask,
const bool zeroFill) {
if (vDst.getIdx() == vSrcShift.getIdx() || vDst.getIdx() == vReadMask.getIdx() || vSrcShift.getIdx() == vReadMask.getIdx()) {
if (v_dst.getIdx() == vSrcShift.getIdx() || v_dst.getIdx() == vReadMask.getIdx() || vSrcShift.getIdx() == vReadMask.getIdx()) {
IE_THROW() << "Any pair of the index, mask, or destination registers cannot be the same.";
}
if (isValidIsa(x64::avx2)) {
if (!useMask)
uni_vpcmpeqd(vReadMask, vReadMask, vReadMask);
if (zeroFill)
uni_vpxor(vDst, vDst, vDst);
uni_vpxor(v_dst, v_dst, v_dst);
vpgatherdd(vDst, ptr[rSrcPtr + vSrcShift], vReadMask);
vpgatherdd(v_dst, ptr[rSrcPtr + vSrcShift], vReadMask);
} else {
Xbyak::Xmm xmmDst = Xbyak::Xmm(vDst.getIdx()),
Xbyak::Xmm xmmDst = Xbyak::Xmm(v_dst.getIdx()),
xmmSrcShft = Xbyak::Xmm(vSrcShift.getIdx()),
xmmReadMask = Xbyak::Xmm(vReadMask.getIdx());
for (uint8_t i = 0; i < 2; i++) {
gatherdd(xmmDst, rSrcPtr, xmmSrcShft, xmmReadMask, useMask, zeroFill);
vperm2f128(vDst, vDst, vDst, 0x1);
vperm2f128(v_dst, v_dst, v_dst, 0x1);
vperm2f128(vSrcShift, vSrcShift, vSrcShift, 0x1);
if (useMask)
vperm2f128(vReadMask, vReadMask, vReadMask, 0x1);
@@ -252,6 +323,15 @@ void JitKernelBase::gatherdd(const Xbyak::Ymm& vDst,
}
}
void JitKernelBase::uni_vpbroadcastq(const Xbyak::Xmm &x, const Xbyak::Operand &op) {
if (isValidIsa(x64::avx2)) {
vpbroadcastq(x, op);
} else {
movsd(x, op);
shufpd(x, x, 0x0);
}
}
void JitKernelBase::uni_vpbroadcastd(const Xbyak::Xmm &x, const Xbyak::Operand &op) {
if (isValidIsa(x64::avx2)) {
vpbroadcastd(x, op);
@@ -285,6 +365,57 @@ void JitKernelBase::uni_vpbroadcastd(const Xbyak::Ymm &x, const Xbyak::Operand &
}
}
void JitKernelBase::uni_vroundpd(const Xbyak::Xmm& v_dst, const Xbyak::Operand& op, const uint8_t imm) {
if (isValidIsa(x64::avx512_core)) {
vrndscalepd(v_dst, op, imm & 0x3);
} else if (isValidIsa(x64::avx)) {
vroundpd(v_dst, op, imm);
} else {
roundpd(v_dst, op, imm);
}
}
void JitKernelBase::uni_vcvtdq2pd(const Xbyak::Xmm& v_dst,
const Xbyak::Operand& op) {
if (isValidIsa(x64::avx)) {
vcvtdq2pd(v_dst, op);
} else {
cvtdq2pd(v_dst, op);
}
}
void JitKernelBase::uni_vcvtpd2dq(const Xbyak::Xmm& v_dst,
const Xbyak::Operand& op) {
if (isValidIsa(x64::avx)) {
vcvtpd2dq(v_dst, op);
} else {
cvtpd2dq(v_dst, op);
}
}
void JitKernelBase::uni_vpmovzxdq(const Xbyak::Xmm& v_dst,
const Xbyak::Operand& op) {
if (isValidIsa(x64::avx2)) {
vpmovzxdq(v_dst, op);
} else {
pmovzxdq(v_dst, op);
}
}
void JitKernelBase::uni_vshufpd(const Xbyak::Xmm& v_dst,
const Xbyak::Xmm& v_src,
const Xbyak::Operand& op,
uint8_t imm) {
if (isValidIsa(x64::avx)) {
vshufpd(v_dst, v_src, op, imm);
} else {
if (v_dst.getIdx() != v_src.getIdx()) {
movups(v_dst, v_src);
}
shufpd(v_dst, op, imm);
}
}
void JitKernelBase::fillRestWorkMask(const Xbyak::Opmask& dstMask,
const Xbyak::Reg64& rWorkRest) {
auto rOnes = getReg64();
@@ -362,7 +493,7 @@ void JitKernelBase::fillRestWorkMask(const Xbyak::Ymm& ymmDstMask,
L(lEnd);
}
void JitKernelBase::load(const Xbyak::Xmm& vDst,
void JitKernelBase::load(const Xbyak::Xmm& v_dst,
const Xbyak::Address& srcAddr,
const Xbyak::Reg64& rLoadNum,
const size_t typeSize,
@@ -373,7 +504,7 @@ void JitKernelBase::load(const Xbyak::Xmm& vDst,
const uint8_t elPerVec = x64::cpu_isa_traits<x64::sse41>::vlen / typeSize;
Xbyak::Label lEnd;
if (zeroFilling)
pxor(vDst, vDst);
pxor(v_dst, v_dst);
for (uint8_t i = 0; i < elPerVec; i++) {
cmp(rLoadNum, i);
@@ -381,18 +512,18 @@ void JitKernelBase::load(const Xbyak::Xmm& vDst,
const size_t offset = i * typeSize;
if (typeSize == 1)
pinsrb(vDst, ptr[srcAddr.getRegExp() + offset], i);
pinsrb(v_dst, ptr[srcAddr.getRegExp() + offset], i);
else if (typeSize == 2)
pinsrw(vDst, ptr[srcAddr.getRegExp() + offset], i);
pinsrw(v_dst, ptr[srcAddr.getRegExp() + offset], i);
else if (typeSize == 4)
pinsrd(vDst, ptr[srcAddr.getRegExp() + offset], i);
pinsrd(v_dst, ptr[srcAddr.getRegExp() + offset], i);
else if (typeSize == 8)
pinsrq(vDst, ptr[srcAddr.getRegExp() + offset], i);
pinsrq(v_dst, ptr[srcAddr.getRegExp() + offset], i);
}
L(lEnd);
}
void JitKernelBase::load(const Xbyak::Ymm& vDst,
void JitKernelBase::load(const Xbyak::Ymm& v_dst,
const Xbyak::Address& srcAddr,
const Xbyak::Reg64& rLoadNum,
const size_t typeSize,
@@ -403,8 +534,8 @@ void JitKernelBase::load(const Xbyak::Ymm& vDst,
const size_t elPerXmm = x64::cpu_isa_traits<x64::sse41>::vlen / typeSize;
Xbyak::Label lEnd;
if (zeroFilling)
uni_vpxor(vDst, vDst, vDst);
Xbyak::Xmm xmmDst(vDst.getIdx());
uni_vpxor(v_dst, v_dst, v_dst);
Xbyak::Xmm xmmDst(v_dst.getIdx());
for (size_t i = 0lu; i < 2lu; i++) {
Xbyak::Label lPerm;
@@ -427,13 +558,13 @@ void JitKernelBase::load(const Xbyak::Ymm& vDst,
}
L(lPerm);
vperm2f128(vDst, vDst, vDst, 0x1);
vperm2f128(v_dst, v_dst, v_dst, 0x1);
}
L(lEnd);
}
void JitKernelBase::store(const Xbyak::Address& dstAddr,
const Xbyak::Xmm& vSrc,
const Xbyak::Xmm& v_src,
const Xbyak::Reg64& rToStoreNum,
const size_t typeSize) {
if (!one_of(typeSize, 1u, 2u, 4u, 8u)) {
@@ -448,27 +579,27 @@ void JitKernelBase::store(const Xbyak::Address& dstAddr,
const size_t offset = i * typeSize;
if (typeSize == 1) {
uni_vpextrb(ptr[dstAddr.getRegExp() + offset], vSrc, i);
uni_vpextrb(ptr[dstAddr.getRegExp() + offset], v_src, i);
} else if (typeSize == 2) {
uni_vpextrw(ptr[dstAddr.getRegExp() + offset], vSrc, i);
uni_vpextrw(ptr[dstAddr.getRegExp() + offset], v_src, i);
} else if (typeSize == 4) {
uni_vpextrd(ptr[dstAddr.getRegExp() + offset], vSrc, i);
uni_vpextrd(ptr[dstAddr.getRegExp() + offset], v_src, i);
} else if (typeSize == 8) {
uni_vpextrq(ptr[dstAddr.getRegExp() + offset], vSrc, i);
uni_vpextrq(ptr[dstAddr.getRegExp() + offset], v_src, i);
}
}
L(lEnd);
}
void JitKernelBase::store(const Xbyak::Address& dstAddr,
const Xbyak::Ymm& vSrc,
const Xbyak::Ymm& v_src,
const Xbyak::Reg64& rToStoreNum,
const size_t typeSize) {
if (!one_of(typeSize, 1u, 2u, 4u, 8u)) {
IE_THROW() << "Could not store data with type size " << typeSize;
}
Xbyak::Label lEnd;
Xbyak::Xmm xmmSrc(vSrc.getIdx());
Xbyak::Xmm xmmSrc(v_src.getIdx());
const size_t elPerXmm = x64::cpu_isa_traits<x64::sse41>::vlen / typeSize;
for (int i = 0; i < 2; i++) {
@@ -493,7 +624,7 @@ void JitKernelBase::store(const Xbyak::Address& dstAddr,
}
L(lPerm);
vperm2f128(vSrc, vSrc, vSrc, 0x1);
vperm2f128(v_src, v_src, v_src, 0x1);
}
L(lEnd);
}
@@ -575,3 +706,7 @@ void JitKernelBase::memMovDD(const Xbyak::Reg64& rDst,
}
L(lEnd);
}
} // namespace kernel
} // namespace intel_cpu
} // namespace ov
src/plugins/intel_cpu/src/nodes/kernels/x64/jit_kernel_base.hpp
+104 -2
View File
@@ -1,14 +1,23 @@
// Copyright (C) 2022 Intel Corporation
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include "openvino/core/visibility.hpp"
#if defined(OPENVINO_ARCH_X86_64)
#include "cpu/x64/jit_generator.hpp"
#include "registers_pool.hpp"
#endif // OPENVINO_ARCH_X86_64
namespace ov {
namespace intel_cpu {
namespace kernel {
class JitKernelBase;
#if defined(OPENVINO_ARCH_X86_64)
#define getReg64() RegistersPool::Reg<Xbyak::Reg64>(registersPool)
#define getReg32() RegistersPool::Reg<Xbyak::Reg32>(registersPool)
@@ -17,7 +26,11 @@ namespace intel_cpu {
class JitKernelBase: public dnnl::impl::cpu::x64::jit_generator {
public:
JitKernelBase(const char* name) : dnnl::impl::cpu::x64::jit_generator(name) {}
JitKernelBase(const char* name, dnnl::impl::cpu::x64::cpu_isa_t max_cpu_isa);
dnnl::impl::cpu::x64::cpu_isa_t getIsa() { return m_isa; }
size_t getVectorLen() { return vlen; }
void uni_vfmsub132ps(const Xbyak::Xmm& vDst, const Xbyak::Xmm& vSrc, const Xbyak::Operand& op);
@@ -31,14 +44,24 @@ public:
void uni_vpaddd(const Xbyak::Ymm& vDst, const Xbyak::Ymm& vSrc, const Xbyak::Operand& op);
void uni_vpaddq(const Xbyak::Xmm& vDst, const Xbyak::Xmm& vSrc, const Xbyak::Operand& op);
void uni_vpsubd(const Xbyak::Xmm& vDst, const Xbyak::Xmm& vSrc, const Xbyak::Operand& op) {
jit_generator::uni_vpsubd(vDst, vSrc, op);
}
void uni_vpsubd(const Xbyak::Ymm& vDst, const Xbyak::Ymm& vSrc, const Xbyak::Operand& op);
void uni_vsubpd(const Xbyak::Xmm& v_dst, const Xbyak::Xmm& v_src, const Xbyak::Operand& op);
void uni_vmulpd(const Xbyak::Xmm& v_dst, const Xbyak::Xmm& v_src, const Xbyak::Operand& op);
void uni_vpmuludq(const Xbyak::Xmm& v_dst, const Xbyak::Xmm& op_1, const Xbyak::Operand& op_2);
void uni_vdivps(const Xbyak::Xmm& vDst, const Xbyak::Operand& op1, const Xbyak::Operand& op2);
void uni_vdivpd(const Xbyak::Xmm& v_dst, const Xbyak::Xmm& v_src, const Xbyak::Operand& op2);
void uni_vandps(const Xbyak::Xmm& vDst, const Xbyak::Xmm& vSrs, const Xbyak::Operand &op);
void uni_vandnps(const Xbyak::Xmm& vDst, const Xbyak::Xmm& vSrs, const Xbyak::Operand &op);
@@ -63,6 +86,18 @@ public:
void uni_vpbroadcastd(const Xbyak::Ymm &x, const Xbyak::Operand &op);
void uni_vpbroadcastq(const Xbyak::Xmm &x, const Xbyak::Operand &op);
void uni_vroundpd(const Xbyak::Xmm& v_dst, const Xbyak::Operand& op, const uint8_t imm);
void uni_vcvtdq2pd(const Xbyak::Xmm& v_dst, const Xbyak::Operand& op);
void uni_vcvtpd2dq(const Xbyak::Xmm& v_dst, const Xbyak::Operand& op);
void uni_vpmovzxdq(const Xbyak::Xmm& v_dst, const Xbyak::Operand& op);
void uni_vshufpd(const Xbyak::Xmm& v_dst, const Xbyak::Xmm& v_srs, const Xbyak::Operand& op, uint8_t imm);
void gatherdd(const Xbyak::Xmm& vDst,
const Xbyak::Reg64& rSrcPtr,
const Xbyak::Xmm& vSrcShift,
@@ -140,7 +175,9 @@ protected:
return dnnl::impl::cpu::x64::mayiuse(isa);
}
const dnnl::impl::cpu::x64::cpu_isa_t m_isa;
RegistersPool::Ptr registersPool;
size_t vlen;
enum {
// Comparison predicate operand (immediate byte) for single-precision floating-point values.
@@ -155,5 +192,70 @@ protected:
};
};
template<typename CompileParams, typename CallArgs>
class JitKernel : public JitKernelBase {
public:
using KernelFunc = void (*)(const CallArgs *);
explicit JitKernel(const char* name, const CompileParams& jcp, dnnl::impl::cpu::x64::cpu_isa_t max_cpu_isa)
: JitKernelBase{name, max_cpu_isa}, m_jcp{jcp}, m_func{nullptr} {}
~JitKernel() override = default;
dnnl::impl::status_t create_kernel() override {
const dnnl::impl::status_t code = jit_generator::create_kernel();
if (code != dnnl::impl::status::success) {
OPENVINO_THROW("Could not create kernel. Error code: ", std::to_string(code), ". ",
"Xbyak error code: ", Xbyak::ConvertErrorToString(Xbyak::GetError()));
}
m_func = (decltype(m_func))jit_ker();
return code;
}
void operator()(const CallArgs* args) const {
assert(m_func);
m_func(args);
}
void operator()(const CallArgs& args) const {
this->operator()(&args);
}
template <template<dnnl::impl::cpu::x64::cpu_isa_t isa> class KernelT>
static std::shared_ptr<JitKernel<CompileParams, CallArgs>> createInstance(const CompileParams& jcp) {
std::shared_ptr<JitKernel<CompileParams, CallArgs>> res;
try {
#define IF_ISA_CASE(ISA) \
if (dnnl::impl::cpu::x64::mayiuse(ISA)) \
res.reset(new KernelT<ISA>(jcp)); \
else
IF_ISA_CASE(dnnl::impl::cpu::x64::avx512_core)
IF_ISA_CASE(dnnl::impl::cpu::x64::avx2)
IF_ISA_CASE(dnnl::impl::cpu::x64::sse41);
#undef IF_ISA_CASE
if (res) {
res->create_kernel();
}
} catch (...) {
return nullptr;
}
return res;
}
protected:
CompileParams m_jcp;
private:
KernelFunc m_func;
};
#endif // OPENVINO_ARCH_X86_64
} // namespace kernel
} // namespace intel_cpu
} // namespace ov
src/plugins/intel_cpu/src/nodes/kernels/x64/random_uniform.cpp
+635
View File
@@ -0,0 +1,635 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "random_uniform.hpp"
using namespace dnnl::impl::cpu;
namespace ov {
namespace intel_cpu {
namespace kernel {
#define GET_OFF(field) offsetof(RandomUniformCallArgs, field)
template <x64::cpu_isa_t isa>
RandomUniform<isa>::RandomUniform(const RandomUniformCompileParams& jcp) :
JitKernel(jit_name(), jcp, isa) {
}
template <x64::cpu_isa_t isa>
void RandomUniform<isa>::generate() {
this->preamble();
registersPool = RegistersPool::create(isa, {rax, rcx, rsp, rdi, k0});
r64_dst = getReg64();
r64_work_amount = getReg64();
mov(r64_work_amount, ptr[r64_params + GET_OFF(work_amount)]);
mov(r64_dst, ptr[r64_params + GET_OFF(dst_ptr)]);
initVectors();
process();
registersPool.reset();
this->postamble();
}
template <>
void RandomUniform<x64::avx512_core>::initVectors() {
const auto r64_aux = getReg64();
const auto r32_aux = Xbyak::Reg32(r64_aux.getIdx());
const auto r16_aux = Xbyak::Reg16(r64_aux.getIdx());
v_max_mul_n_64 = getVmm();
v_max_mul_c_64 = getVmm();
v_add_low_k = getVmm();
v_add_up_k = getVmm();
v_n_inc = getVmm();
v_range = getVmm();
v_min = getVmm();
v_key_64 = getVmm();
v_counter_64 = getVmm();
v_n_64 = getVmm();
v_res_perm = getVmm();
if (m_jcp.out_data_type.is_real()) {
v_convert_0 = getVmm();
v_convert_1 = getVmm();
}
// Initialize constants.
#define BROADCAST_R(F, V, R, C) \
mov(R, C); \
F(V, R);
#define BROADCAST_P(F, V, R, C) \
mov(R, ptr[r64_params + GET_OFF(C)]); \
F(V, ptr[R]);
BROADCAST_R(vpbroadcastq, v_max_mul_n_64, r64_aux, STATISTIC_MAXIMIZING_MULTIPLIER_N)
BROADCAST_R(vpbroadcastq, v_max_mul_c_64, r64_aux, STATISTIC_MAXIMIZING_MULTIPLIER_COUNTER)
BROADCAST_R(vpbroadcastd, v_add_low_k, r32_aux, CRUSH_RESISTANCE_CONST_LOWER_VALUE)
BROADCAST_R(vpbroadcastd, v_add_up_k, r32_aux, CRUSH_RESISTANCE_CONST_UPPER_VALUE)
BROADCAST_R(vpbroadcastq, v_n_inc, r64_aux, 0x00000008)
if (m_jcp.out_data_type == element::f32) {
BROADCAST_R(vpbroadcastd, v_convert_0, r32_aux, 0x3f800000)
BROADCAST_R(vpbroadcastd, v_convert_1, r32_aux, 0x007fffff)
BROADCAST_P(vpbroadcastd, v_range, r64_aux, range_ptr)
BROADCAST_P(vpbroadcastd, v_min, r64_aux, min_ptr)
} else if (m_jcp.out_data_type == element::f16 && x64::mayiuse(x64::avx512_core_fp16)) {
BROADCAST_R(vpbroadcastw, v_convert_0, r16_aux, 0x3c00)
BROADCAST_R(vpbroadcastw, v_convert_1, r16_aux, 0x03ff)
BROADCAST_P(vpbroadcastw, v_range, r64_aux, range_ptr)
BROADCAST_P(vpbroadcastw, v_min, r64_aux, min_ptr)
} else if (m_jcp.out_data_type == element::bf16 && x64::mayiuse(x64::avx512_core_bf16)) {
v_convert_2 = getVmm();
const auto ymm_min = Xbyak::Ymm(v_min.getIdx());
const auto ymm_range = Xbyak::Ymm(v_range.getIdx());
BROADCAST_R(vpbroadcastw, v_convert_0, r16_aux, 0x3f80)
BROADCAST_R(vpbroadcastw, v_convert_1, r16_aux, 0x007f)
BROADCAST_R(vpbroadcastd, v_convert_2, r32_aux, 0x3f800000)
BROADCAST_P(vpbroadcastw, v_range, r64_aux, range_ptr)
vpmovzxwd(v_range, ymm_range);
uni_vpslld(v_range, v_range, 16);
BROADCAST_P(vpbroadcastw, v_min, r64_aux, min_ptr)
vpmovzxwd(v_min, ymm_min);
uni_vpslld(v_min, v_min, 16);
} else if (m_jcp.out_data_type == element::i32) {
const auto ymm_range = Xbyak::Ymm(v_range.getIdx());
BROADCAST_P(vpbroadcastd, v_range, r64_aux, range_ptr)
BROADCAST_P(vpbroadcastd, v_min, r64_aux, min_ptr)
uni_vcvtdq2pd(v_range, ymm_range);
} else {
OPENVINO_THROW("RandomUniform kernel does not support precision ", m_jcp.out_data_type, " for ", x64::get_isa_info());
}
// Initialize inputs.
BROADCAST_P(vpbroadcastq, v_key_64, r64_aux, key_ptr)
BROADCAST_P(vpbroadcastq, v_counter_64, r64_aux, counter_ptr)
BROADCAST_P(vpbroadcastq, v_n_64, r64_aux, n_ptr)
if (m_jcp.out_data_type.size() <= 4) {
static const uint64_t n_inc_arr[8] = { 0, 1, 2, 3, 4, 5, 6, 7 };
mov(r64_aux, reinterpret_cast<uintptr_t>(n_inc_arr));
} else {
static const uint64_t n_inc_arr[8] = { 0, 1, 2, 3, 4, 5, 6, 7 }; // TODO: i64
mov(r64_aux, reinterpret_cast<uintptr_t>(n_inc_arr));
}
uni_vpaddq(v_n_64, v_n_64, ptr[r64_aux]);
// Initialize auxiliary vectors.
static const uint32_t res_perm_mask[16] = { 0b00000000, 0b00010000, 0b00001000, 0b00011000, 0b00000010, 0b00010010, 0b00001010, 0b00011010,
0b00000100, 0b00010100, 0b00001100, 0b00011100, 0b00000110, 0b00010110, 0b00001110, 0b00011110 };
mov(r64_aux, reinterpret_cast<uintptr_t>(res_perm_mask));
uni_vmovups(v_res_perm, ptr[r64_aux]);
if (m_jcp.out_data_type == element::f16 && x64::mayiuse(x64::avx512_core_fp16)) {
v_perm_16 = getVmm();
static const uint16_t perm_16[32] = { 0b00000000, 0b00000010, 0b00000100, 0b00000110, 0b00001000, 0b00001010, 0b00001100, 0b00001110,
0b00010000, 0b00010010, 0b00010100, 0b00010110, 0b00011000, 0b00011010, 0b00011100, 0b00011110,
0b00100000, 0b00100010, 0b00100100, 0b00100110, 0b00101000, 0b00101010, 0b00101100, 0b00101110,
0b00110000, 0b00110010, 0b00110100, 0b00110110, 0b00111000, 0b00111010, 0b00111100, 0b00111110 };
mov(r64_aux, reinterpret_cast<uintptr_t>(perm_16));
uni_vmovups(v_perm_16, ptr[r64_aux]);
}
#undef BROADCAST_R
#undef BROADCAST_P
}
template <x64::cpu_isa_t isa> // Works for AVX2, SSE41
void RandomUniform<isa>::initVectors() {
const auto r64_aux = getReg64();
v_max_mul_n_64 = getVmm();
v_max_mul_c_64 = getVmm();
v_add_low_k = getVmm();
v_add_up_k = getVmm();
v_range = getVmm();
v_key_64 = getVmm();
v_counter_64 = getVmm();
v_n_64 = getVmm();
r64_n_inc = getReg64();
r64_min = getReg64();
#define INIT_ARR(A, V, R, T) \
static const T A[8] = { V, V, V, V, V, V, V, V }; \
if (isa == x64::avx2) { \
mov(R, reinterpret_cast<uintptr_t>(A)); \
} else { \
static const T* A##_aligned = A + (reinterpret_cast<int64_t>(A) % 16) / sizeof(T); \
mov(R, reinterpret_cast<uintptr_t>(A##_aligned)); \
}
// Initialize constants.
INIT_ARR(max_mul_n_64, STATISTIC_MAXIMIZING_MULTIPLIER_N, r64_aux, uint64_t);
uni_vmovups(v_max_mul_n_64, ptr[r64_aux]);
INIT_ARR(max_mul_c_64, STATISTIC_MAXIMIZING_MULTIPLIER_COUNTER, r64_aux, uint64_t);
uni_vmovups(v_max_mul_c_64, ptr[r64_aux]);
INIT_ARR(add_low_k, CRUSH_RESISTANCE_CONST_LOWER_VALUE, r64_aux, uint32_t);
uni_vmovups(v_add_low_k, ptr[r64_aux]);
INIT_ARR(add_up_k, CRUSH_RESISTANCE_CONST_UPPER_VALUE, r64_aux, uint32_t);
uni_vmovups(v_add_up_k, ptr[r64_aux]);
INIT_ARR(n_inc_step, isa == x64::avx2 ? 4 : 2, r64_n_inc, uint64_t);
if (m_jcp.out_data_type == element::f32) {
r64_convert_0 = getReg64();
r64_convert_1 = getReg64();
INIT_ARR(convert_0, 0x3f800000, r64_convert_0, uint32_t);
INIT_ARR(convert_1, 0x007fffff, r64_convert_1, uint32_t);
mov(r64_aux, ptr[r64_params + GET_OFF(range_ptr)]);
uni_vpbroadcastd(v_range, ptr[r64_aux]);
auto v_aux = getVmm();
mov(r64_aux, ptr[r64_params + GET_OFF(min_ptr)]);
uni_vpbroadcastd(v_aux, ptr[r64_aux]);
static uint32_t min_arr[8];
mov(r64_min, reinterpret_cast<uintptr_t>(min_arr));
uni_vmovups(ptr[r64_min], v_aux);
} else if (m_jcp.out_data_type == element::i32) {
r64_f64_pow_52 = getReg64();
const auto v_aux = getVmm();
const auto xmm_range = Xbyak::Xmm(v_range.getIdx());
INIT_ARR(f64_pow_52, 0x4330000000000000, r64_f64_pow_52, uint64_t);
mov(r64_aux, ptr[r64_params + GET_OFF(range_ptr)]);
uni_vpbroadcastd(v_range, ptr[r64_aux]);
mov(r64_aux, ptr[r64_params + GET_OFF(min_ptr)]);
uni_vpbroadcastd(v_aux, ptr[r64_aux]);
static uint32_t min_arr[8];
mov(r64_min, reinterpret_cast<uintptr_t>(min_arr));
uni_vmovups(ptr[r64_min], v_aux);
uni_vcvtdq2pd(v_range, xmm_range);
} else {
OPENVINO_THROW("RandomUniform kernel does not support precision ", m_jcp.out_data_type, " for ", x64::get_isa_info());
}
// Initialize inputs.
mov(r64_aux, ptr[r64_params + GET_OFF(key_ptr)]);
uni_vpbroadcastq(v_key_64, ptr[r64_aux]);
mov(r64_aux, ptr[r64_params + GET_OFF(counter_ptr)]);
uni_vpbroadcastq(v_counter_64, ptr[r64_aux]);
mov(r64_aux, ptr[r64_params + GET_OFF(n_ptr)]);
uni_vpbroadcastq(v_n_64, ptr[r64_aux]);
if (m_jcp.out_data_type.size() <= 4) {
if (isa == x64::avx2) {
static const uint64_t n_inc_arr[4] = { 0, 1, 2, 3 };
mov(r64_aux, reinterpret_cast<uintptr_t>(n_inc_arr));
} else {
static uint64_t n_inc_arr[4];
static uint64_t* n_inc_arr_aligned = n_inc_arr + (reinterpret_cast<int64_t>(n_inc_arr) % 16) / sizeof(uint64_t);
n_inc_arr_aligned[0] = 0;
n_inc_arr_aligned[1] = 1;
mov(r64_aux, reinterpret_cast<uintptr_t>(n_inc_arr_aligned));
}
} else {
static const uint64_t n_inc_arr[4] = { 0, 1, 2, 3 }; // TODO: i64
mov(r64_aux, reinterpret_cast<uintptr_t>(n_inc_arr));
}
uni_vpaddq(v_n_64, v_n_64, ptr[r64_aux]);
#undef INIT_ARR
}
template <x64::cpu_isa_t isa>
void RandomUniform<isa>::process() {
auto v_dst_0 = getVmm();
auto v_dst_1 = getVmm();
std::vector<Vmm> v_res{ v_dst_0, v_dst_1 };
auto step = vlen;
if (one_of(m_jcp.out_data_type.size(), 2lu, 4lu)) {
step = vlen * 2 / sizeof(uint32_t);
} else if (m_jcp.out_data_type.size() == 8) {
step = vlen / sizeof(uint32_t);
}
Xbyak::Label l_loop, l_tail;
L(l_loop); {
cmp(r64_work_amount, step);
jl(l_tail, T_NEAR);
runPhilox(v_res, v_key_64, v_counter_64, v_n_64);
convert(v_res, v_res);
uni_vmovups(ptr[r64_dst], v_dst_0);
add(r64_dst, vlen);
if (one_of(m_jcp.out_data_type.size(), 4lu, 8lu)) {
uni_vmovups(ptr[r64_dst], v_dst_1);
add(r64_dst, vlen);
}
if (isa == x64::avx512_core) {
uni_vpaddd(v_n_64, v_n_64, v_n_inc);
} else {
uni_vpaddd(v_n_64, v_n_64, ptr[r64_n_inc]);
}
sub(r64_work_amount, step);
jmp(l_loop, T_NEAR);
}
L(l_tail);
tail(v_res);
}
template <x64::cpu_isa_t isa>
void RandomUniform<isa>::calculateRound(const Vmm& vmm_k_0, const Vmm& vmm_k_1, const Vmm& vmm_c_0, const Vmm& vmm_c_1,
const Vmm& vmm_n_0, const Vmm& vmm_n_1, const Vmm& vmm_aux_0, const Vmm& vmm_aux_1) {
uni_vpmuludq(vmm_aux_0, vmm_n_0, v_max_mul_n_64); // {p0,p1,p0,p1} = {n0,_,n0,_} * {m0,_,m0,_}
uni_vpmuludq(vmm_aux_1, vmm_c_0, v_max_mul_c_64); // {r0,r1,r0,r1} = {c0,_,c0,_} * {m0,_,m0,_}
uni_vpshufd(vmm_c_0, vmm_aux_0, 0b10110001); // {p1,p0,p1,p0} = shuf {p0,p1,p0,p1}
uni_vxorps(vmm_c_0, vmm_c_0, vmm_c_1); // {c0,_,c0,_} = {p1,_,p1,_} ^ {c1,_,c1,_}
uni_vxorps(vmm_c_0, vmm_c_0, vmm_k_1); // {c0,_,c0,_} = {c0,_,c0,_} ^ {k1,_,k1,_}
uni_vpshufd(vmm_n_0, vmm_aux_1, 0b10110001); // {r1,r0,r1,r0} = shuf {r0,r1,r0,r1}
uni_vxorps(vmm_n_0, vmm_n_0, vmm_n_1); // {n0,_,n0,_} = {r1,_,r1,_} ^ {n1,_,n1,_}
uni_vxorps(vmm_n_0, vmm_n_0, vmm_k_0); // {n0,_,n0,_} = {n0,_,n0,_} ^ {k0,_,k0,_}
}
template <x64::cpu_isa_t isa>
void RandomUniform<isa>::runPhilox(const std::vector<Vmm>& vmm_dst, const Vmm& vmm_key, const Vmm& vmm_counter, const Vmm& vmm_n) {
auto vmm_k_0 = getVmm();
auto vmm_k_1 = getVmm();
auto vmm_n_0 = getVmm();
auto vmm_n_1 = vmm_dst[0];
auto vmm_c_0 = getVmm();
auto vmm_c_1 = getVmm();
auto vmm_aux_0 = getVmm();
auto vmm_aux_1 = vmm_dst[1];
uni_vmovups(vmm_k_0, vmm_key); // {k0,k1,k0,k1} -> {k0,_,k0,_}
uni_vpshufd(vmm_k_1, vmm_key, 0b10110001); // {k0,k1,k0,k1} -> {k1,_,k1,_}
uni_vpmuludq(vmm_aux_0, vmm_n, v_max_mul_n_64); // {p0,p1,p0,p1} = {n0,_,n0,_} * {m0,_,m0,_}
uni_vpmuludq(vmm_aux_1, vmm_counter, v_max_mul_c_64); // {r0,r1,r0,r1} = {c0,_,c0,_} * {m0,_,m0,_}
uni_vxorps(vmm_c_0, vmm_aux_0, vmm_counter); // {_,c0,_,c0} = {_,p1,_,p1} ^ {_,c1,_,c1}
uni_vxorps(vmm_c_0, vmm_c_0, vmm_key); // {_,c0,_,c0} = {_,c0,_,c0} ^ {_,k1,_,k1}
uni_vpshufd(vmm_c_0, vmm_c_0, 0b10110001); // {_,c0,_,c0} -> {c0,_,c0,_}
uni_vxorps(vmm_n_0, vmm_aux_1, vmm_n); // {_,n0,_,n0} = {_,r1,_,r1} ^ {_,n1,_,n1}
uni_vpshufd(vmm_n_0, vmm_n_0, 0b10110001); // {_,n0,_,n0} -> {n0,_,n0,_}
uni_vxorps(vmm_n_0, vmm_n_0, vmm_key); // {n0,_,n0,_} = {n0,_,n0,_} ^ {k0,_,k0,_}
for (size_t i = 0lu; i < ROUNDS_NUMBER - 1; i++) {
raiseKey(vmm_k_0, vmm_k_1);
std::swap(vmm_c_1, vmm_aux_0);
std::swap(vmm_n_1, vmm_aux_1);
calculateRound(vmm_k_0, vmm_k_1, vmm_c_0, vmm_c_1, vmm_n_0, vmm_n_1, vmm_aux_0, vmm_aux_1);
}
std::swap(vmm_c_1, vmm_aux_0);
std::swap(vmm_n_1, vmm_aux_1);
if (isa == x64::avx512_core) {
vpermt2d(vmm_n_0, v_res_perm, vmm_n_1); // {n0,n1,n0,n1} = perm {n0,_,n0,_} {n1,_,n1,_}
vpermt2d(vmm_c_0, v_res_perm, vmm_c_1); // {c0,c1,c0,c1} = perm {c0,_,c0,_} {c1,_,c1,_}
vshufpd(vmm_dst[0], vmm_n_0, vmm_c_0, 0b00000000); // {n0,n1,c0,c1} = shuf {n0,n1,n0,n1} {c0,c1,c0,c1}
vshufpd(vmm_dst[1], vmm_n_0, vmm_c_0, 0b11111111); // {n0,n1,c0,c1} = shuf {n0,n1,n0,n1} {c0,c1,c0,c1}
} else if (isa == x64::avx2) {
auto ymm_dst_0 = Xbyak::Ymm(vmm_dst[0].getIdx());
auto ymm_dst_1 = Xbyak::Ymm(vmm_dst[1].getIdx());
auto ymm_c_0 = Xbyak::Ymm(vmm_c_0.getIdx());
uni_vshufps(vmm_n_0, vmm_n_0, vmm_n_1, 0b10001000); // {n0,n0,n1,n1} = shuf {n0,_,n0,_} {n1,_,n1,_}
uni_vshufps(vmm_c_0, vmm_c_0, vmm_c_1, 0b10001000); // {c0,c0,c1,c1} = shuf {c0,_,c0,_} {c1,_,c1,_}
uni_vshufps(ymm_dst_1, vmm_n_0, vmm_c_0, 0b10001000); // {n0,n1,c0,c1} = shuf {n0,n0,n1,n1} {c0,c0,c1,c1}
uni_vshufps(vmm_c_0, vmm_n_0, vmm_c_0, 0b11011101); // {n0,n1,c0,c1} = shuf {n0,n0,n1,n1} {c0,c0,c1,c1}
vperm2f128(ymm_dst_0, ymm_dst_1, ymm_c_0, 0b00100000);
vperm2f128(ymm_dst_1, ymm_dst_1, ymm_c_0, 0b00110001);
} else {
uni_vshufps(vmm_n_0, vmm_n_0, vmm_n_1, 0b10001000);
uni_vshufps(vmm_c_0, vmm_c_0, vmm_c_1, 0b10001000);
uni_vshufps(vmm_dst[0], vmm_n_0, vmm_c_0, 0b10001000);
uni_vshufps(vmm_dst[1], vmm_n_0, vmm_c_0, 0b11011101);
}
}
template <x64::cpu_isa_t isa>
void RandomUniform<isa>::raiseKey(const Vmm& vmm_k_0, const Vmm& vmm_k_1) {
uni_vpaddd(vmm_k_0, vmm_k_0, v_add_low_k); // {k0,_,k0,_} + {l0,_,l0,_}
uni_vpaddd(vmm_k_1, vmm_k_1, v_add_up_k); // {k1,_,k1,_} + {u0,_,u0,_}
}
template <>
void RandomUniform<x64::avx512_core>::convert(const std::vector<Vmm>& v_dst, const std::vector<Vmm>& v_src) {
if (m_jcp.out_data_type.size() == 4) {
for (size_t i = 0lu; i < v_src.size(); i++) {
const auto& vmm_src = v_src[i];
const auto& vmm_dst = v_dst[i];
if (m_jcp.out_data_type == element::f32) {
uni_vandps(vmm_dst, vmm_src, v_convert_1);
uni_vorps(vmm_dst, vmm_dst, v_convert_0);
uni_vsubps(vmm_dst, vmm_dst, v_convert_0);
vfmadd132ps(vmm_dst, v_min, v_range);
} else if (m_jcp.out_data_type == element::i32) {
// x % (max - min) + min
const auto v_aux_0 = getVmm();
const auto v_aux_1 = getVmm();
const auto ymm_src = Xbyak::Ymm(vmm_src.getIdx());
const auto ymm_dst = Xbyak::Ymm(vmm_dst.getIdx());
const auto ymm_aux_1 = Xbyak::Ymm(v_aux_1.getIdx());
// Divide in the f64 due to the f32 loses accuracy here.
vcvtudq2pd(v_aux_0, ymm_src);
uni_vdivpd(v_aux_1, v_aux_0, v_range);
uni_vroundpd(v_aux_1, v_aux_1, 3);
vfnmadd132pd(v_aux_1, v_aux_0, v_range);
vextractf64x4(ymm_dst, vmm_src, 1);
vcvtudq2pd(v_aux_0, ymm_dst);
uni_vcvtpd2dq(ymm_dst, v_aux_1);
uni_vdivpd(v_aux_1, v_aux_0, v_range);
uni_vroundpd(v_aux_1, v_aux_1, 3);
vfnmadd132pd(v_aux_1, v_aux_0, v_range);
uni_vcvtpd2dq(ymm_aux_1, v_aux_1);
vshuff64x2(vmm_dst, vmm_dst, v_aux_1, 0b01000100);
uni_vpaddd(vmm_dst, vmm_dst, v_min);
} else {
OPENVINO_THROW("RandomUniform kernel does not support precision ", m_jcp.out_data_type, " for ", x64::get_isa_info());
}
}
} else if (m_jcp.out_data_type.size() == 2) {
if (m_jcp.out_data_type == element::f16 && x64::mayiuse(x64::avx512_core_fp16)) {
const auto& vmm_dst = v_dst[0];
if (v_src[0].getIdx() != vmm_dst.getIdx()) {
uni_vmovups(vmm_dst, v_src[0]);
}
vpermt2w(vmm_dst, v_perm_16, v_src[1]);
uni_vandps(vmm_dst, vmm_dst, v_convert_1);
uni_vorps(vmm_dst, vmm_dst, v_convert_0);
vsubph(vmm_dst, vmm_dst, v_convert_0);
vfmadd132ph(vmm_dst, v_min, v_range);
} else if (m_jcp.out_data_type == element::bf16 && x64::mayiuse(x64::avx512_core_bf16)) {
for (size_t i = 0lu; i < v_src.size(); i++) {
const auto& vmm_dst = v_dst[i];
uni_vandps(vmm_dst, v_src[i], v_convert_1);
uni_vorps(vmm_dst, vmm_dst, v_convert_0);
uni_vpslld(vmm_dst, vmm_dst, 16);
uni_vsubps(vmm_dst, vmm_dst, v_convert_2);
vfmadd132ps(vmm_dst, v_min, v_range);
}
vcvtne2ps2bf16(v_dst[0], v_dst[1], v_dst[0]);
} else {
OPENVINO_THROW("RandomUniform kernel does not support precision ", m_jcp.out_data_type, " for ", x64::get_isa_info());
}
} else if (m_jcp.out_data_type.size() == 8) {
if (m_jcp.out_data_type == element::i64) {
// TODO: in scope of i64 enabling.
}
OPENVINO_THROW("RandomUniform kernel does not support precision ", m_jcp.out_data_type, " for ", x64::get_isa_info());
} else {
OPENVINO_THROW("RandomUniform kernel does not support precision ", m_jcp.out_data_type, " for ", x64::get_isa_info());
}
}
template <x64::cpu_isa_t isa> // Works for AVX2, SSE41
void RandomUniform<isa>::convert(const std::vector<Vmm>& v_dst, const std::vector<Vmm>& v_src) {
if (m_jcp.out_data_type.size() == 4) {
for (size_t i = 0lu; i < v_src.size(); i++) {
auto vmm_src = v_src[i];
auto vmm_dst = v_dst[i];
if (m_jcp.out_data_type == element::f32) {
uni_vandps(vmm_dst, vmm_src, ptr[r64_convert_1]);
uni_vorps(vmm_dst, vmm_dst, ptr[r64_convert_0]);
uni_vsubps(vmm_dst, vmm_dst, ptr[r64_convert_0]);
if (isa == x64::avx2) {
vfmadd213ps(vmm_dst, v_range, ptr[r64_min]);
} else {
uni_vmulps(vmm_dst, vmm_dst, v_range);
uni_vaddps(vmm_dst, vmm_dst, ptr[r64_min]);
}
} else if (m_jcp.out_data_type == element::i32) {
// x % (max - min) + min
const auto v_aux_0 = getVmm();
const auto v_aux_1 = getVmm();
const auto xmm_dst = Xbyak::Xmm(vmm_dst.getIdx());
const auto ymm_dst = Xbyak::Ymm(vmm_dst.getIdx());
const auto xmm_aux_1 = Xbyak::Xmm(v_aux_1.getIdx());
// Convert u32->f64. TODO: move to convert emitter after i64 enabling.
uni_vpmovzxdq(v_aux_0, xmm_dst);
uni_vorps(v_aux_0, v_aux_0, ptr[r64_f64_pow_52]);
uni_vsubpd(v_aux_0, v_aux_0, ptr[r64_f64_pow_52]);
// Divide in the f64 due to the f32 loses accuracy here.
uni_vdivpd(v_aux_1, v_aux_0, v_range);
uni_vroundpd(v_aux_1, v_aux_1, 3);
if (isa == x64::avx2) {
vfnmadd132pd(v_aux_1, v_aux_0, v_range);
} else {
uni_vmulpd(v_aux_1, v_aux_1, v_range);
uni_vsubpd(v_aux_0, v_aux_0, v_aux_1);
uni_vmovups(v_aux_1, v_aux_0);
}
if (isa == x64::avx2) {
vperm2f128(ymm_dst, ymm_dst, ymm_dst, 0b00000001);
} else {
uni_vshufpd(vmm_dst, vmm_dst, vmm_dst, 0b00000001);
}
// Convert u32->f64. TODO: move to convert emitter after i64 enabling.
uni_vpmovzxdq(v_aux_0, xmm_dst);
uni_vorps(v_aux_0, v_aux_0, ptr[r64_f64_pow_52]);
uni_vsubpd(v_aux_0, v_aux_0, ptr[r64_f64_pow_52]);
uni_vcvtpd2dq(xmm_dst, v_aux_1);
uni_vdivpd(v_aux_1, v_aux_0, v_range);
uni_vroundpd(v_aux_1, v_aux_1, 3);
if (isa == x64::avx2) {
vfnmadd132pd(v_aux_1, v_aux_0, v_range);
} else {
uni_vmulpd(v_aux_1, v_aux_1, v_range);
uni_vsubpd(v_aux_0, v_aux_0, v_aux_1);
uni_vmovups(v_aux_1, v_aux_0);
}
uni_vcvtpd2dq(xmm_aux_1, v_aux_1);
if (isa == x64::avx2) {
vperm2f128(ymm_dst, ymm_dst, v_aux_1, 0b00100000);
} else {
uni_vshufpd(vmm_dst, vmm_dst, v_aux_1, 0b00000000);
}
uni_vpaddd(vmm_dst, vmm_dst, ptr[r64_min]);
} else {
OPENVINO_THROW("RandomUniform kernel does not support precision ", m_jcp.out_data_type, " for ", x64::get_isa_info());
}
}
} else if (m_jcp.out_data_type.size() == 8) {
if (m_jcp.out_data_type == element::i64) {
// TODO: in scope of i64 enabling.
}
OPENVINO_THROW("RandomUniform kernel does not support precision ", m_jcp.out_data_type, " for ", x64::get_isa_info());
} else {
OPENVINO_THROW("RandomUniform kernel does not support precision ", m_jcp.out_data_type, " for ", x64::get_isa_info());
}
}
template <>
void RandomUniform<x64::avx512_core>::tail(const std::vector<Vmm>& vmm_dst) {
Xbyak::Label l_end;
const auto k_rest_mask = getMask();
cmp(r64_work_amount, 0);
jle(l_end, T_NEAR);
runPhilox(vmm_dst, v_key_64, v_counter_64, v_n_64);
convert(vmm_dst, vmm_dst);
if (m_jcp.out_data_type.size() == 4) {
Xbyak::Label l_0;
const auto step = vlen / sizeof(uint32_t);
cmp(r64_work_amount, step);
jl(l_0, T_NEAR);
uni_vmovups(ptr[r64_dst], vmm_dst[0]);
add(r64_dst, vlen);
sub(r64_work_amount, step);
fillRestWorkMask(k_rest_mask, r64_work_amount);
uni_vmovups(ptr[r64_dst] | k_rest_mask, vmm_dst[1]);
jmp(l_end, T_NEAR);
L(l_0);
fillRestWorkMask(k_rest_mask, r64_work_amount);
uni_vmovups(ptr[r64_dst] | k_rest_mask, vmm_dst[0]);
} else if (m_jcp.out_data_type.size() == 2) {
fillRestWorkMask(k_rest_mask, r64_work_amount);
vmovdqu16(ptr[r64_dst] | k_rest_mask, vmm_dst[0]);
}
L(l_end);
}
template <>
void RandomUniform<x64::avx2>::tail(const std::vector<Vmm>& vmm_dst) {
Xbyak::Label l_0, l_end;
const auto step = vlen / sizeof(uint32_t);
cmp(r64_work_amount, 0);
jle(l_end, T_NEAR);
runPhilox(vmm_dst, v_key_64, v_counter_64, v_n_64);
convert(vmm_dst, vmm_dst);
const auto v_rest_mask = getVmm();
cmp(r64_work_amount, step);
jl(l_0, T_NEAR);
uni_vmovups(ptr[r64_dst], vmm_dst[0]);
add(r64_dst, vlen);
sub(r64_work_amount, step);
fillRestWorkMask(v_rest_mask, r64_work_amount, m_jcp.out_data_type.size());
vmaskmovps(ptr[r64_dst], v_rest_mask, vmm_dst[1]);
jmp(l_end, T_NEAR);
L(l_0);
fillRestWorkMask(v_rest_mask, r64_work_amount, m_jcp.out_data_type.size());
vmaskmovps(ptr[r64_dst], v_rest_mask, vmm_dst[0]);
L(l_end);
}
template <x64::cpu_isa_t isa>
void RandomUniform<isa>::tail(const std::vector<Vmm>& vmm_dst) {
Xbyak::Label l_0, l_end;
const auto step = vlen / sizeof(uint32_t);
cmp(r64_work_amount, 0);
jle(l_end, T_NEAR);
runPhilox(vmm_dst, v_key_64, v_counter_64, v_n_64);
convert(vmm_dst, vmm_dst);
cmp(r64_work_amount, step);
jl(l_0, T_NEAR);
uni_vmovups(ptr[r64_dst], vmm_dst[0]);
add(r64_dst, vlen);
sub(r64_work_amount, step);
store(ptr[r64_dst], vmm_dst[1], r64_work_amount, m_jcp.out_data_type.size());
jmp(l_end, T_NEAR);
L(l_0);
store(ptr[r64_dst], vmm_dst[0], r64_work_amount, m_jcp.out_data_type.size());
L(l_end);
}
template class RandomUniform<x64::avx512_core>;
template class RandomUniform<x64::avx2>;
template class RandomUniform<x64::sse41>;
} // namespace kernel
} // namespace intel_cpu
} // namespace ov
src/plugins/intel_cpu/src/nodes/kernels/x64/random_uniform.hpp
+99
View File
@@ -0,0 +1,99 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include "jit_kernel_base.hpp"
#if defined(OPENVINO_ARCH_X86_64)
namespace ov {
namespace intel_cpu {
namespace kernel {
struct RandomUniformCompileParams {
element::Type out_data_type = element::f32;
};
struct RandomUniformCallArgs {
void* dst_ptr;
const void* key_ptr;
const void* counter_ptr;
const void* n_ptr;
const void* min_ptr;
const void* range_ptr;
uint64_t work_amount = 0lu;
};
template <dnnl::impl::cpu::x64::cpu_isa_t isa>
class RandomUniform : public JitKernel<RandomUniformCompileParams, RandomUniformCallArgs> {
public:
DECLARE_CPU_JIT_AUX_FUNCTIONS(RandomUniform)
explicit RandomUniform(const RandomUniformCompileParams& jcp);
void generate() override;
private:
using Vmm = typename dnnl::impl::utils::conditional3<isa == dnnl::impl::cpu::x64::avx512_core, Xbyak::Zmm,
isa == dnnl::impl::cpu::x64::sse41, Xbyak::Xmm,
Xbyak::Ymm>::type;
using Vmask = typename dnnl::impl::utils::conditional3<isa == dnnl::impl::cpu::x64::avx512_core, Xbyak::Opmask,
isa == dnnl::impl::cpu::x64::sse41, Xbyak::Xmm,
Xbyak::Ymm>::type;
RegistersPool::Reg<Xbyak::Reg64> r64_dst;
RegistersPool::Reg<Xbyak::Reg64> r64_work_amount;
RegistersPool::Reg<Xbyak::Reg64> r64_n_inc;
RegistersPool::Reg<Xbyak::Reg64> r64_convert_0;
RegistersPool::Reg<Xbyak::Reg64> r64_convert_1;
RegistersPool::Reg<Xbyak::Reg64> r64_min;
RegistersPool::Reg<Xbyak::Reg64> r64_f64_pow_52;
const Xbyak::Reg64 r64_params = Xbyak::Reg64(dnnl::impl::cpu::x64::abi_param_regs[0]);
// Vector registers.
RegistersPool::Reg<Vmm> v_max_mul_n_64;
RegistersPool::Reg<Vmm> v_max_mul_c_64;
RegistersPool::Reg<Vmm> v_add_low_k;
RegistersPool::Reg<Vmm> v_add_up_k;
RegistersPool::Reg<Vmm> v_convert_0;
RegistersPool::Reg<Vmm> v_convert_1;
RegistersPool::Reg<Vmm> v_convert_2;
RegistersPool::Reg<Vmm> v_n_inc;
RegistersPool::Reg<Vmm> v_key_64;
RegistersPool::Reg<Vmm> v_counter_64;
RegistersPool::Reg<Vmm> v_n_64;
RegistersPool::Reg<Vmm> v_min;
RegistersPool::Reg<Vmm> v_range;
RegistersPool::Reg<Vmm> v_res_perm;
RegistersPool::Reg<Vmm> v_perm_16;
void initVectors();
void process();
void runPhilox(const std::vector<Vmm>& vmm_res, const Vmm& vmm_key, const Vmm& vmm_counter, const Vmm& vmm_n);
void calculateRound(const Vmm& vmm_k_0, const Vmm& vmm_k_1, const Vmm& vmm_c_0, const Vmm& vmm_c_1,
const Vmm& vmm_n_0, const Vmm& vmm_n_1, const Vmm& vmm_aux_0, const Vmm& vmm_aux_1);
void raiseKey(const Vmm& vmm_k_0, const Vmm& vmm_k_1);
void convert(const std::vector<Vmm>& vmm_dst, const std::vector<Vmm>& vmm_src);
void tail(const std::vector<Vmm>& vmm_dst);
static constexpr uint64_t ROUNDS_NUMBER = 10lu;
static constexpr uint32_t CRUSH_RESISTANCE_CONST_LOWER_VALUE = 0x9E3779B9;
static constexpr uint32_t CRUSH_RESISTANCE_CONST_UPPER_VALUE = 0xBB67AE85;
static constexpr uint64_t STATISTIC_MAXIMIZING_MULTIPLIER_N = 0xD2511F53;
static constexpr uint64_t STATISTIC_MAXIMIZING_MULTIPLIER_COUNTER = 0xCD9E8D57;
};
} // namespace kernel
} // namespace intel_cpu
} // namespace ov
#endif // OPENVINO_ARCH_X86_64
src/plugins/intel_cpu/src/nodes/random_uniform.cpp
+532
View File
@@ -0,0 +1,532 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "random_uniform.hpp"
#include "ie_parallel.hpp"
#include "ie_ngraph_utils.hpp"
#include <openvino/op/constant.hpp>
#include <openvino/op/random_uniform.hpp>
#include "shape_inference/custom/random_uniform.hpp"
namespace ov {
namespace intel_cpu {
namespace node {
bool RandomUniform::isSupportedOperation(const std::shared_ptr<const ov::Node>& op, std::string& errorMessage) noexcept {
try {
if (op->get_type_info() != op::v8::RandomUniform::get_type_info_static()) {
errorMessage = "Only RandomUniform operation from the opset8 is supported by the CPU plugin.";
return false;
}
} catch (...) {
return false;
}
return true;
}
RandomUniform::RandomUniform(const std::shared_ptr<ov::Node>& op, const GraphContext::CPtr& context)
: Node(op, context, RandomUniformShapeInferFactory(op)) {
std::string errorMessage;
if (!isSupportedOperation(op, errorMessage)) {
THROW_CPU_NODE_ERR(errorMessage);
}
// RandomUniform should generate new sequence each run even if all inputs are constants. So that method Node::IsConstant()
// doesn't return 'True' for RandomUniform with all constant inputs and the node generates new values for each inference,
// we set 'NoConst' value for 'ConstantType' in ctor.
constant = ConstantType::NoConst;
auto rnd_op = as_type_ptr<op::v8::RandomUniform>(op);
m_global_seed = rnd_op->get_global_seed();
m_op_seed = rnd_op->get_op_seed();
m_output_prc = op->get_output_element_type(0);
for (size_t i = 0lu; i < op->get_input_size(); i++) {
if (is_type<op::v0::Constant>(op->get_input_node_ptr(i))) {
m_const_inputs[i] = true;
}
}
if (m_algo == STL) {
m_generator = std::default_random_engine{static_cast<uint32_t>(m_op_seed)};
}
}
void RandomUniform::getSupportedDescriptors() {
if (getParentEdges().size() != 3) {
THROW_CPU_NODE_ERR("has incorrect number of input edges.");
}
if (getChildEdges().empty()) {
THROW_CPU_NODE_ERR("has incorrect number of output edges.");
}
}
void RandomUniform::initSupportedPrimitiveDescriptors() {
auto shape_prc = getOriginalInputPrecisionAtPort(SHAPE);
if (!one_of(shape_prc, InferenceEngine::Precision::I32, InferenceEngine::Precision::I64)) {
shape_prc = InferenceEngine::Precision::I32;
}
auto out_prc = getOriginalOutputPrecisionAtPort(0);
if (out_prc.is_float() && ((m_algo == PHILOX &&
!one_of(out_prc, InferenceEngine::Precision::FP32, InferenceEngine::Precision::FP16, InferenceEngine::Precision::BF16)) ||
(m_algo == STL && !one_of(out_prc, InferenceEngine::Precision::FP32)))) {
out_prc = InferenceEngine::Precision::FP32;
}
if (!out_prc.is_float() && !one_of(out_prc, InferenceEngine::Precision::I32, InferenceEngine::Precision::I64)) {
out_prc = InferenceEngine::Precision::I32;
}
m_output_prc = InferenceEngine::details::convertPrecision(out_prc);
addSupportedPrimDesc({{LayoutType::ncsp, shape_prc, m_const_inputs[SHAPE]},
{LayoutType::ncsp, out_prc, m_const_inputs[MIN_VAL]},
{LayoutType::ncsp, out_prc, m_const_inputs[MAX_VAL]}},
{{LayoutType::ncsp, out_prc}},
ref_any);
}
void RandomUniform::createPrimitive() {
if (m_const_inputs[MIN_VAL]) {
initEdgeValues(m_min_val, getParentEdgeAt(MIN_VAL)->getMemoryPtr()->getData(), m_output_prc);
}
if (m_const_inputs[MAX_VAL]) {
initEdgeValues(m_max_val, getParentEdgeAt(MAX_VAL)->getMemoryPtr()->getData(), m_output_prc);
evalRange();
}
if (m_algo == PHILOX) {
#if defined(OPENVINO_ARCH_X86_64)
kernel::RandomUniformCompileParams jcp;
jcp.out_data_type = m_output_prc;
m_jit_kernel = kernel::JitKernel<kernel::RandomUniformCompileParams, kernel::RandomUniformCallArgs>::createInstance<kernel::RandomUniform>(jcp);
if (m_jit_kernel) {
if (auto selected_pd = getSelectedPrimitiveDescriptor()) {
using namespace dnnl::impl::cpu;
if (m_jit_kernel->getIsa() == x64::avx512_core) {
selected_pd->setImplementationType(jit_avx512);
} else if (m_jit_kernel->getIsa() == x64::avx2) {
selected_pd->setImplementationType(jit_avx2);
} else if (m_jit_kernel->getIsa() == x64::sse41) {
selected_pd->setImplementationType(jit_sse42);
}
}
}
#endif // OPENVINO_ARCH_X86_64
}
if (m_const_inputs[SHAPE]) {
Node::createPrimitive();
}
}
bool RandomUniform::needPrepareParams() const {
if (m_out_shape != getChildEdgeAt(0)->getMemoryPtr()->getShape().getStaticDims()) {
return true;
}
return false;
}
void RandomUniform::prepareParams() {
m_out_shape = getChildEdgeAt(0)->getMemoryPtr()->getShape().getStaticDims();
m_out_el_num = std::accumulate(m_out_shape.begin(), m_out_shape.end(), 1lu, std::multiplies<Dim>());
if (m_algo == PHILOX) {
m_skip_count = m_out_el_num * SKIP_CONST;
if (m_out_el_num < PHILOX_PARALLEL_EXECUTION_THRESHOLD) {
m_threads_num = 1;
} else {
m_threads_num = parallel_get_max_threads();
}
m_thread_params.resize(m_threads_num);
parallel_nt(m_threads_num, [&](const int ithr, const int nthr) {
auto& p = m_thread_params[ithr];
uint64_t start = 0lu, end = 0lu;
if (m_jit_kernel) {
#if defined(OPENVINO_ARCH_X86_64)
const auto block_size = (m_jit_kernel->getVectorLen() / m_output_prc.size()) * 2;
const auto blocks_num = (m_out_el_num + block_size - 1) / block_size;
const auto blocks_per_thr = (blocks_num + nthr - 1) / nthr;
start = ithr * blocks_per_thr * block_size;
end = (ithr + 1) * blocks_per_thr * block_size;
#endif // OPENVINO_ARCH_X86_64
} else {
const auto groups_num = (m_out_el_num + PHILOX_GROUP_SIZE - 1) / PHILOX_GROUP_SIZE;
const auto groups_per_thr = (groups_num + nthr - 1) / nthr;
start = ithr * groups_per_thr * PHILOX_GROUP_SIZE;
end = (ithr + 1) * groups_per_thr * PHILOX_GROUP_SIZE;
p.step = m_output_prc.size() > 4 ? 2 : 4;
}
if (end > m_out_el_num) {
end = m_out_el_num;
}
if (start > end) {
start = end;
}
p.work_amount = end - start;
p.n_shift = start / PHILOX_GROUP_SIZE;
p.dst_shift = start * m_output_prc.size();
});
}
}
void RandomUniform::execute(dnnl::stream strm) {
if (!m_const_inputs[MIN_VAL]) {
initEdgeValues(m_min_val, getParentEdgeAt(MIN_VAL)->getMemoryPtr()->getData(), m_output_prc);
if (m_const_inputs[MAX_VAL]) {
evalRange();
}
}
if (!m_const_inputs[MAX_VAL]) {
initEdgeValues(m_max_val, getParentEdgeAt(MAX_VAL)->getMemoryPtr()->getData(), m_output_prc);
evalRange();
}
auto data = getChildEdgeAt(0)->getMemoryPtr()->getData();
if (m_algo == PHILOX) {
m_state = computePhilox(data, m_out_el_num, m_state);
} else if (m_algo == STL) {
computeStl(data, m_out_el_num);
} else {
THROW_CPU_NODE_ERR("unsupported algorithm.");
}
}
void RandomUniform::executeDynamicImpl(dnnl::stream strm) {
execute(strm);
}
////////////// PHILOX algo ///////////////
namespace {
// Following const values are taken from the original paper:
// https://www.thesalmons.org/john/random123/papers/random123sc11.pdf
constexpr uint32_t CRUSH_RESISTANCE_CONST_LOWER_VALUE = 0x9E3779B9;
constexpr uint32_t CRUSH_RESISTANCE_CONST_UPPER_VALUE = 0xBB67AE85;
constexpr uint64_t STATISTIC_MAXIMIZING_MULTIPLIER_N = 0xD2511F53;
constexpr uint64_t STATISTIC_MAXIMIZING_MULTIPLIER_COUNTER = 0xCD9E8D57;
constexpr uint64_t ROUNDS_NUMBER = 10llu;
inline void calculateRound(const uint32_t* key, uint32_t* counter, uint32_t* n) {
uint64_t prod_0 = STATISTIC_MAXIMIZING_MULTIPLIER_N * n[0];
uint64_t prod_1 = STATISTIC_MAXIMIZING_MULTIPLIER_COUNTER * counter[0];
n[0] = static_cast<uint32_t>(prod_1 >> 32) ^ n[1] ^ key[0];
n[1] = static_cast<uint32_t>(prod_1);
counter[0] = static_cast<uint32_t>(prod_0 >> 32) ^ counter[1] ^ key[1];
counter[1] = static_cast<uint32_t>(prod_0);
}
inline void raiseKey(uint32_t* key) {
key[0] += CRUSH_RESISTANCE_CONST_LOWER_VALUE;
key[1] += CRUSH_RESISTANCE_CONST_UPPER_VALUE;
}
inline void runPhilox(uint64_t key, uint64_t counter, uint64_t n, uint32_t* res) {
uint32_t* key_32 = reinterpret_cast<uint32_t*>(&key);
uint32_t* counter_32 = reinterpret_cast<uint32_t*>(&counter);
uint32_t* n_32 = reinterpret_cast<uint32_t*>(&n);
for (size_t i = 0lu; i < ROUNDS_NUMBER; i++) {
calculateRound(key_32, counter_32, n_32);
if (i < ROUNDS_NUMBER - 1)
raiseKey(key_32);
}
res[0] = n_32[0];
res[1] = n_32[1];
res[2] = counter_32[0];
res[3] = counter_32[1];
}
inline void convertToOutputType(const uint32_t* in,
float min,
float range,
float* out,
size_t el_to_copy) {
RandomUniform::OutputType out_val;
for (size_t i = 0lu; i < el_to_copy; i++) {
out_val.u32 = 0x3f800000 | (in[i] & 0x7fffffu);
out[i] = (out_val.f32 - 1.f) * range + min;
}
}
inline void convertToOutputType(const uint32_t* in,
float16 min,
float16 range,
float16* out,
size_t el_to_copy) {
RandomUniform::OutputType out_val;
for (size_t i = 0lu; i < el_to_copy; i++) {
uint16_t x_uint16 = static_cast<uint16_t>(in[i]);
out_val.u16 = 0x3c00 | (x_uint16 & 0x03ffu);
out[i] = (out_val.f16 - static_cast<float16>(1)) * range + min;
}
}
inline void convertToOutputType(const uint32_t* in,
bfloat16 min,
bfloat16 range,
bfloat16* out,
size_t el_to_copy) {
RandomUniform::OutputType out_val;
for (size_t i = 0lu; i < el_to_copy; i++) {
uint16_t x_uint16 = static_cast<uint16_t>(in[i]);
out_val.u16 = 0x3f80 | (x_uint16 & 0x7fu);
out[i] = (out_val.bf16 - static_cast<bfloat16>(1)) * range + min;
}
}
inline void convertToOutputType(const uint32_t* in,
int32_t min,
int32_t range,
int32_t* out,
size_t el_to_copy) {
for (size_t i = 0lu; i < el_to_copy; i++) {
out[i] = static_cast<int32_t>(in[i] % range + min);
}
}
inline void convertToOutputType(const uint32_t* in,
int64_t min,
int64_t range,
int64_t* out,
size_t el_to_copy) {
for (size_t i = 0lu; i < el_to_copy; i++) {
out[i] = static_cast<int64_t>(((static_cast<uint64_t>(in[i * 2]) << 32) + in[i * 2 + 1]) % range + min);
}
}
} // namespace
std::pair<uint64_t, uint64_t> RandomUniform::computePhilox(void* out, size_t out_el_num, const std::pair<uint64_t, uint64_t>& prev_state) {
// When both seed values are equal to zero RandomUniform should generate non-deterministic sequence.
if (m_global_seed == 0lu && m_op_seed == 0lu) {
std::srand(static_cast<unsigned int>(std::time(nullptr)));
m_global_seed = std::rand();
}
uint64_t n_state = prev_state.first;
uint64_t counter_state = prev_state.second;
uint64_t counter = counter_state > 0 ? counter_state : m_op_seed;
auto out_u8 = reinterpret_cast<uint8_t*>(out);
if (m_jit_kernel) {
#if defined(OPENVINO_ARCH_X86_64)
parallel_nt(m_threads_num, [&](const int ithr, const int nthr) {
auto& p = m_thread_params[ithr];
if (p.work_amount == 0lu) {
return;
}
auto n = n_state + p.n_shift;
kernel::RandomUniformCallArgs args;
args.dst_ptr = (out_u8 + p.dst_shift);
args.key_ptr = &m_global_seed;
args.counter_ptr = &counter;
args.n_ptr = &n;
args.min_ptr = &m_min_val;
args.range_ptr = &m_range_val;
args.work_amount = p.work_amount;
(*m_jit_kernel)(&args);
});
#endif // OPENVINO_ARCH_X86_64
} else {
auto threadBody = [&](const int ithr, const int nthr) {
auto& p = m_thread_params[ithr];
if (p.work_amount == 0lu) {
return;
}
auto n = n_state + p.n_shift;
auto out_cur = out_u8 + p.dst_shift;
auto work_rest = static_cast<int64_t>(p.work_amount);
uint32_t res[4];
#define EXEC_CASE(P) \
case element::P: { \
auto out_t = reinterpret_cast<element_type_traits<element::P>::value_type *>(out_cur); \
for (; work_rest > 0l; work_rest -= p.step, out_t += p.step) { \
runPhilox(m_global_seed, counter, n, res); \
auto el_to_copy = std::min(p.step, static_cast<uint64_t>(work_rest)); \
convertToOutputType(res, m_min_val.P, m_range_val.P, out_t, el_to_copy); \
if (++n == 0) { \
counter++; \
} \
} \
} break;
switch (m_output_prc) {
EXEC_CASE(f32)
EXEC_CASE(f16)
EXEC_CASE(bf16)
EXEC_CASE(i32)
EXEC_CASE(i64)
default: THROW_CPU_NODE_ERR("Unsupported type of RandomUniform: ", m_output_prc.to_string());
}
#undef EXEC_CASE
};
parallel_nt(m_threads_num, threadBody);
}
// Calculate counter values for next RandomUniform run.
n_state += m_skip_count;
if (n_state < m_skip_count) {
counter_state++;
}
return { n_state, counter_state };
}
////////////// STL algo ///////////////
void RandomUniform::computeStl(void* out, size_t work_amount) {
switch (m_output_prc) {
case element::f32: {
generateData<float, std::uniform_real_distribution<float>>(
std::uniform_real_distribution<float>{m_min_val.f32, m_max_val.f32}, out, work_amount);
} break;
case element::i32: {
generateData<int32_t, std::uniform_int_distribution<int32_t>>(
std::uniform_int_distribution<int32_t>{m_min_val.i32, m_max_val.i32}, out, work_amount);
} break;
case element::i64: {
generateData<int64_t, std::uniform_int_distribution<int64_t>>(
std::uniform_int_distribution<int64_t>{m_min_val.i64, m_max_val.i64}, out, work_amount);
} break;
default:
THROW_CPU_NODE_ERR("has unsupported output type: ", m_output_prc);
}
}
template <typename T, typename DISTR_TYPE>
void RandomUniform::generateData(DISTR_TYPE distribution, void* out, size_t work_amount) {
auto dst = reinterpret_cast<T*>(out);
for (size_t i = 0; i < work_amount; i++) {
*dst = distribution(m_generator);
dst++;
}
}
//////////////////////////////////
void RandomUniform::initEdgeValues(OutputType& dst, const void* src, const element::Type& output_type) {
#define EL_CASE(E) \
case element::E: \
dst.E = *reinterpret_cast<const element_type_traits<element::E>::value_type *>(src); \
break;
switch (output_type) {
EL_CASE(f32)
EL_CASE(f16)
EL_CASE(bf16)
EL_CASE(i32)
EL_CASE(i64)
EL_CASE(f64)
default:
THROW_CPU_NODE_ERR("has unsupported output precision: ", output_type);
}
#undef EL_CASE
}
void RandomUniform::evalRange() {
#define EL_CASE(E) \
case element::E: \
m_range_val.E = m_max_val.E - m_min_val.E; \
break;
switch (m_output_prc) {
EL_CASE(f32)
EL_CASE(f16)
EL_CASE(bf16)
EL_CASE(i32)
EL_CASE(i64)
EL_CASE(f64)
default:
THROW_CPU_NODE_ERR("has unsupported output precision: ", m_output_prc);
}
#undef EL_CASE
}
std::string RandomUniform::getPrimitiveDescriptorType() const {
auto selectedPrimitiveDesc = getSelectedPrimitiveDescriptor();
impl_desc_type type = impl_desc_type::undef;
if (selectedPrimitiveDesc) {
type = selectedPrimitiveDesc->getImplementationType();
}
std::string str_type;
auto add_type = [&](std::string t) {
if (!str_type.empty() && t.c_str()[0] != '_')
str_type += "_";
str_type += t;
};
#define SEARCH_TYPE(_type) \
if ((type & impl_desc_type::_type) == impl_desc_type::_type) \
add_type(#_type)
SEARCH_TYPE(undef);
SEARCH_TYPE(jit);
SEARCH_TYPE(ref);
SEARCH_TYPE(avx512);
SEARCH_TYPE(avx2);
SEARCH_TYPE(sse42);
SEARCH_TYPE(any);
#undef SEARCH_TYPE
if (type == impl_desc_type::unknown)
str_type = "unknown";
else if (str_type.empty())
str_type = "undef";
if (selectedPrimitiveDesc) {
if (selectedPrimitiveDesc->getConfig().outConfs[0].getMemDesc()->getPrecision() != InferenceEngine::Precision::U8) {
str_type += "_" + std::string(selectedPrimitiveDesc->getConfig().outConfs[0].getMemDesc()->getPrecision().name());
} else {
str_type += "_I8";
}
}
return str_type;
}
bool RandomUniform::needShapeInfer() const {
return !m_const_inputs[SHAPE];
}
bool RandomUniform::isExecutable() const {
return !isInputTensorAtPortEmpty(SHAPE);
}
bool RandomUniform::created() const {
return getType() == Type::RandomUniform;
}
} // namespace node
} // namespace intel_cpu
} // namespace ov
src/plugins/intel_cpu/src/nodes/random_uniform.hpp
+120
View File
@@ -0,0 +1,120 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include <node.h>
#include <random>
#include "kernels/x64/random_uniform.hpp"
namespace ov {
namespace intel_cpu {
namespace node {
class RandomUniform : public Node {
public:
union OutputType {
float f32;
float16 f16;
bfloat16 bf16;
double f64;
int32_t i32;
uint32_t u32;
uint16_t u16;
int64_t i64;
};
RandomUniform(const std::shared_ptr<ov::Node>& op, const GraphContext::CPtr& context);
void getSupportedDescriptors() override;
void initSupportedPrimitiveDescriptors() override;
bool needPrepareParams() const override;
void prepareParams() override;
void execute(dnnl::stream strm) override;
void executeDynamicImpl(dnnl::stream strm) override;
bool isExecutable() const override;
void createPrimitive() override;
bool created() const override;
bool canBeInPlace() const override { return false; }
static bool isSupportedOperation(const std::shared_ptr<const ov::Node>& op, std::string& errorMessage) noexcept;
std::string getPrimitiveDescriptorType() const override;
protected:
bool needShapeInfer() const override;
private:
void computeStl(void* out, size_t work_amount);
std::pair<uint64_t, uint64_t> computePhilox(void* out, size_t work_amount, const std::pair<uint64_t, uint64_t>& prev_state);
template <typename T, typename DISTR_TYPE>
void generateData(DISTR_TYPE distribution, void* out, size_t work_amount);
void initOutShape(VectorDims& dst, const void* src, const element::Type& shape_type, size_t len);
void initEdgeValues(OutputType& dst, const void* src, const element::Type& output_type);
void evalRange();
enum { SHAPE = 0, MIN_VAL, MAX_VAL };
enum AlgoType { STL, PHILOX };
bool m_const_inputs[3] = {false, false, false};
ov::element::Type m_output_prc;
uint64_t m_global_seed = 0lu;
uint64_t m_op_seed = 0lu;
std::pair<uint64_t, uint64_t> m_state {0lu, 0lu};
VectorDims m_out_shape = {};
uint64_t m_out_el_num = 1lu;
OutputType m_min_val;
OutputType m_max_val;
OutputType m_range_val;
AlgoType m_algo = PHILOX;
std::default_random_engine m_generator;
struct ThreadParams {
uint64_t work_amount = 0lu;
uint64_t dst_shift = 0lu;
uint64_t n_shift = 0lu;
uint64_t step = 0lu;
};
uint64_t m_threads_num = 0lu;
std::vector<ThreadParams> m_thread_params;
///// PHILOX constants /////
// Determines how many sequence elements of RNG sequence are skipped between runs.
// Can be any positive value, 256 is chosen for parity with Tensorflow.
static constexpr uint64_t SKIP_CONST = 256lu;
// Philox algorithm returns 4 elements of RNG sequence per each invocation
static constexpr uint64_t PHILOX_GROUP_SIZE = 4lu;
// Output elements number threshold to execute on one thread.
static constexpr uint64_t PHILOX_PARALLEL_EXECUTION_THRESHOLD = 1000lu;
uint64_t m_skip_count = 0lu;
/////////////////////////////////////////////////////////////////////////////////
std::shared_ptr<kernel::JitKernelBase> m_jit_kernel;
};
} // namespace node
} // namespace intel_cpu
} // namespace ov
src/plugins/intel_cpu/src/nodes/reference.cpp
+21 -36
View File
@@ -2,18 +2,12 @@
// SPDX-License-Identifier: Apache-2.0
//
#include <dnnl_extension_utils.h>
#include <ie_ngraph_utils.hpp>
#include <ngraph/opsets/opset1.hpp>
#include "common/blocked_desc_creator.h"
#include "common/cpu_memcpy.h"
#include "openvino/core/shape_util.hpp"
#include "openvino/runtime/tensor.hpp"
#include "reference.h"
using namespace dnnl;
#include "common/cpu_memcpy.h"
#include <ie_ngraph_utils.hpp>
#include "openvino/core/shape_util.hpp"
using namespace InferenceEngine;
using namespace InferenceEngine::details;
@@ -21,21 +15,15 @@ namespace ov {
namespace intel_cpu {
namespace node {
Reference::Reference(const std::shared_ptr<ngraph::Node>& op, const GraphContext::CPtr context,
Reference::Reference(const std::shared_ptr<ov::Node>& op, const GraphContext::CPtr& context,
const std::string& errorMessage) :
Node(op, context, NgraphShapeInferFactory(op, FULL_PORT_MASK)), ngraphOp(op), additionalErrorMessage(errorMessage) {
Node(op, context, NgraphShapeInferFactory(op, FULL_PORT_MASK)), ovCoreNode(op), additionalErrorMessage(errorMessage) {
if (!op->has_evaluate()) {
IE_THROW(NotImplemented) << "Cannot fallback on ngraph reference implementation (Ngraph::Node::evaluate() is not implemented)";
}
setType(Type::Reference);
setTypeStr("Reference");
// RandomUniform should generate new sequence each run even if all inputs are constants. So that method Node::IsConstant()
// doesn't return 'True' for RandomUniform with all constant inputs and the node generates new values for each inference,
// we set 'NoConst' value for 'ConstantType' in ctor
if (ov::is_type<ngraph::op::v8::RandomUniform>(ngraphOp)) {
constant = ConstantType::NoConst;
}
}
void Reference::getSupportedDescriptors() {}
@@ -47,13 +35,13 @@ void Reference::initSupportedPrimitiveDescriptors() {
std::vector<PortConfigurator> inputConfigurators;
inputConfigurators.reserve(inputShapes.size());
for (size_t i = 0; i < inputShapes.size(); i++) {
inputConfigurators.emplace_back(LayoutType::ncsp, convertPrecision(ngraphOp->get_input_element_type(i)), inputShapes[i]);
inputConfigurators.emplace_back(LayoutType::ncsp, convertPrecision(ovCoreNode->get_input_element_type(i)), inputShapes[i]);
}
std::vector<PortConfigurator> outputConfigurators;
outputConfigurators.reserve(inputShapes.size());
for (size_t i = 0; i < outputShapes.size(); i++) {
outputConfigurators.emplace_back(LayoutType::ncsp, convertPrecision(ngraphOp->get_output_element_type(i)), outputShapes[i]);
outputConfigurators.emplace_back(LayoutType::ncsp, convertPrecision(ovCoreNode->get_output_element_type(i)), outputShapes[i]);
}
addSupportedPrimDesc(inputConfigurators, outputConfigurators, impl_desc_type::ref);
@@ -64,8 +52,8 @@ void Reference::createPrimitive() {}
void Reference::execute(dnnl::stream strm) {
auto inputs = prepareInputs();
auto outputs = prepareOutputs();
if (!ngraphOp->evaluate(outputs, inputs)) {
IE_THROW() << "Evaluation failed on node of type: " << std::string(ngraphOp->get_type_name()) << " name: " << getName();
if (!ovCoreNode->evaluate(outputs, inputs)) {
THROW_CPU_NODE_ERR("evaluation failed for core operation: ", std::string(ovCoreNode->get_type_name()));
}
}
@@ -81,18 +69,16 @@ void Reference::executeDynamicImpl(dnnl::stream strm) {
for (size_t i = 0; i < outputShapes.size(); ++i) {
auto mem_desc = getBaseMemDescAtOutputPort(i);
if (mem_desc->isDefined()) {
outputs.emplace_back(ngraphOp->get_output_element_type(i), mem_desc->getShape().getStaticDims());
outputs.emplace_back(ovCoreNode->get_output_element_type(i), mem_desc->getShape().getStaticDims());
} else {
outputs.emplace_back(ngraphOp->get_output_element_type(i), ov::util::make_dynamic_shape());
outputs.emplace_back(ovCoreNode->get_output_element_type(i), ov::util::make_dynamic_shape());
}
}
} else {
IE_THROW(Unexpected) <<
"Unexpected shape infer result status during the inference of a node with type " <<
getTypeStr() << " and name " << getName();
THROW_CPU_NODE_ERR("got unexpected shape infer result status during the inference.");
}
if (!ngraphOp->evaluate(outputs, inputs)) {
IE_THROW() << "Evaluation failed on node of type: " << std::string(ngraphOp->get_type_name()) << " name: " << getName();
if (!ovCoreNode->evaluate(outputs, inputs)) {
THROW_CPU_NODE_ERR("evaluation failed for core operation: ", std::string(ovCoreNode->get_type_name()));
}
if (ShapeInferStatus::skip == result.status) {
std::vector<VectorDims> newOutputDims;
@@ -105,8 +91,7 @@ void Reference::executeDynamicImpl(dnnl::stream strm) {
auto memory = getChildEdgesAtPort(i)[0]->getMemoryPtr();
auto& tensor = outputs[i];
if (memory->getSize() != tensor.get_byte_size()) {
IE_THROW(Unexpected) << "Output tensor data size mismatch occurred during the inference of a node with type " <<
getTypeStr() << " and name " << getName() << " on output port number " << i;
THROW_CPU_NODE_ERR("output tensor data size mismatch occurred during the inference on output port number ", i);
}
cpu_memcpy(memory->getData(), tensor.data(), tensor.get_byte_size());
}
@@ -125,9 +110,9 @@ ov::TensorVector Reference::prepareInputs() const {
ov::TensorVector inputs;
for (size_t i = 0; i < inputShapes.size(); i++) {
void *srcDataPtr = getParentEdgesAtPort(i)[0]->getMemory().getData();
ov::Shape shape = ngraphOp->get_input_partial_shape(i).rank().get_length() == 0 ?
ov::Shape shape = ovCoreNode->get_input_partial_shape(i).rank().get_length() == 0 ?
ov::Shape{} : getParentEdgesAtPort(i)[0]->getMemory().getStaticDims();
inputs.push_back(ov::Tensor(ngraphOp->get_input_element_type(i), shape, srcDataPtr));
inputs.push_back(ov::Tensor(ovCoreNode->get_input_element_type(i), shape, srcDataPtr));
}
return inputs;
}
@@ -136,9 +121,9 @@ ov::TensorVector Reference::prepareOutputs() const {
ov::TensorVector outputs;
for (size_t i = 0; i < outputShapes.size(); i++) {
void *dstDataPtr = getChildEdgesAtPort(i)[0]->getMemory().getData();
ov::Shape shape = ngraphOp->get_output_partial_shape(i).rank().get_length() == 0 ?
ov::Shape shape = ovCoreNode->get_output_partial_shape(i).rank().get_length() == 0 ?
ov::Shape{} : getChildEdgesAtPort(i)[0]->getMemory().getStaticDims();
outputs.push_back(ov::Tensor(ngraphOp->get_output_element_type(i), shape, dstDataPtr));
outputs.push_back(ov::Tensor(ovCoreNode->get_output_element_type(i), shape, dstDataPtr));
}
return outputs;
}
src/plugins/intel_cpu/src/nodes/reference.h
+2 -2
View File
@@ -12,7 +12,7 @@ namespace node {
class Reference : public Node {
public:
Reference(const std::shared_ptr<ngraph::Node>& op, const GraphContext::CPtr context, const std::string& errorMessage);
Reference(const std::shared_ptr<ov::Node>& op, const GraphContext::CPtr& context, const std::string& errorMessage);
void getSupportedDescriptors() override;
void initSupportedPrimitiveDescriptors() override;
@@ -29,7 +29,7 @@ private:
ov::TensorVector prepareOutputs() const;
private:
const std::shared_ptr<ngraph::Node> ngraphOp;
const std::shared_ptr<ov::Node> ovCoreNode;
const std::string additionalErrorMessage;
};
src/plugins/intel_cpu/src/nodes_factory.cpp
+2
View File
@@ -79,6 +79,7 @@
#include "nodes/experimental_detectron_generate_proposals_single_image.h"
#include "nodes/generate_proposals.h"
#include "nodes/embedding_bag_packed_sum.h"
#include "nodes/random_uniform.hpp"
#include "nodes/reduce.h"
#include "nodes/if.h"
#include "nodes/ctc_greedy_decoder.h"
@@ -180,6 +181,7 @@ Node::NodesFactory::NodesFactory()
INTEL_CPU_NODE(Unique, Type::Unique);
INTEL_CPU_NODE(Ngram, Type::Ngram);
INTEL_CPU_NODE(Interpolate, Type::Interpolate);
INTEL_CPU_NODE(RandomUniform, Type::RandomUniform);
INTEL_CPU_NODE(Reduce, Type::Reduce);
INTEL_CPU_NODE(Gather, Type::Gather);
INTEL_CPU_NODE(NonMaxSuppression, Type::NonMaxSuppression);
src/plugins/intel_cpu/src/shape_inference/custom/random_uniform.cpp
+47
View File
@@ -0,0 +1,47 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "random_uniform.hpp"
#include <openvino/op/random_uniform.hpp>
namespace ov {
namespace intel_cpu {
namespace node {
// TODO: remove after fixing the issue 123011
IShapeInfer::Result RandomUniformShapeInfer::infer(
const std::vector<std::reference_wrapper<const VectorDims>>& input_shapes,
const std::unordered_map<size_t, MemoryPtr>& data_dependency) {
VectorDims dims;
const auto& mem = data_dependency.at(0);
const auto rank = mem->getShape().getElementsCount();
auto shape_prc = mem->getDesc().getPrecision();
switch (shape_prc) {
case InferenceEngine::Precision::I32: {
auto data = reinterpret_cast<const int32_t*>(mem->getData());
dims.assign(data, data + rank);
} break;
case InferenceEngine::Precision::I64: {
auto data = reinterpret_cast<const int64_t*>(mem->getData());
dims.assign(data, data + rank);
} break;
default:
OPENVINO_THROW("Unexpected Shape input precision: ", shape_prc);
}
return {{dims}, ShapeInferStatus::success};
}
RandomUniformShapeInferFactory::RandomUniformShapeInferFactory(const std::shared_ptr<ov::Node>& op) : m_op(op) {
OPENVINO_ASSERT(ov::is_type<const op::v8::RandomUniform>(m_op),
"Unexpected op type in RandomUniform shape inference factory: ", m_op->get_type_name());
}
ShapeInferPtr RandomUniformShapeInferFactory::makeShapeInfer() const {
return std::make_shared<RandomUniformShapeInfer>();
}
} // namespace node
} // namespace intel_cpu
} // namespace ov
src/plugins/intel_cpu/src/shape_inference/custom/random_uniform.hpp
+37
View File
@@ -0,0 +1,37 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "shape_inference/shape_inference_cpu.hpp"
#include <node.h>
#pragma once
namespace ov {
namespace intel_cpu {
namespace node {
class RandomUniformShapeInfer : public ShapeInferEmptyPads {
public:
explicit RandomUniformShapeInfer() {}
IShapeInfer::Result infer(
const std::vector<std::reference_wrapper<const VectorDims>>& input_shapes,
const std::unordered_map<size_t, MemoryPtr>& data_dependency) override;
port_mask_t get_port_mask() const override {
return PortMask(0);
}
};
class RandomUniformShapeInferFactory : public ShapeInferFactory {
public:
explicit RandomUniformShapeInferFactory(const std::shared_ptr<ov::Node>& op);
ShapeInferPtr makeShapeInfer() const override;
private:
std::shared_ptr<ov::Node> m_op;
};
} // namespace node
} // namespace intel_cpu
} // namespace ov
src/plugins/intel_cpu/tests/functional/shared_tests_instances/skip_tests_config.cpp
+2
View File
@@ -189,6 +189,8 @@ std::vector<std::string> disabledTestPatterns() {
R"(smoke_LSTMSequenceCommon.*LSTMSequenceTest.Inference.*CONVERT_TO_TI.*)",
// Issue: 122094
R"(smoke_Interpolate_Basic_Down_Sample_Tail/InterpolateLayerTest.Inference.*(asymmetric|align_corners).*f16.*)",
// Need to generate sequence exactly in the i64 data type. Enable in scope of i64 enabling.
R"(.*RandomUniformLayerTestCPU.*OutPrc=i64.*)",
};
#if defined(OPENVINO_ARCH_X86)
src/plugins/intel_cpu/tests/functional/single_layer_tests/classes/random_uniform.cpp
+265
View File
@@ -0,0 +1,265 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "random_uniform.hpp"
#include "ov_models/builders.hpp"
using namespace CPUTestUtils;
using namespace ov::test;
namespace CPULayerTestsDefinitions {
std::string RandomUniformLayerTestCPU::getTestCaseName(const testing::TestParamInfo<RandomUniformLayerTestCPUParamSet>& obj) {
const auto& out_shape = std::get<0>(obj.param);
const auto& min_max = std::get<1>(obj.param);
std::ostringstream result;
result << "IS={" << out_shape.size();
result << "}_OS=" << out_shape;
result << "_Min=" << std::get<0>(min_max);
result << "_Max=" << std::get<1>(min_max);
result << "_ShapePrc=" << std::get<2>(obj.param);
result << "_OutPrc=" << std::get<3>(obj.param);
result << "_GlobalSeed=" << std::get<4>(obj.param);
result << "_OperationalSeed=" << std::get<5>(obj.param);
result << "_ConstIn={" << utils::bool2str(std::get<6>(obj.param)) << ","
<< utils::bool2str(std::get<7>(obj.param)) << ","
<< utils::bool2str(std::get<8>(obj.param)) << "}";
result << CPUTestsBase::getTestCaseName(std::get<9>(obj.param));
const auto& config = std::get<10>(obj.param);
if (!config.empty()) {
result << "_PluginConf={";
for (const auto& conf_item : config) {
result << "_" << conf_item.first << "=";
conf_item.second.print(result);
}
result << "}";
}
return result.str();
}
void RandomUniformLayerTestCPU::SetUp() {
targetDevice = utils::DEVICE_CPU;
const auto& params = this->GetParam();
m_output_shape = std::get<0>(params);
const auto& min_max = std::get<1>(params);
const auto& shape_prc = std::get<2>(params);
const auto& output_prc = std::get<3>(params);
m_global_seed = std::get<4>(params);
m_operational_seed = std::get<5>(params);
const auto& const_in_1 = std::get<6>(params);
const auto& const_in_2 = std::get<7>(params);
const auto& const_in_3 = std::get<8>(params);
const auto& cpu_params = std::get<9>(params);
configuration = std::get<10>(params);
m_min_val = std::get<0>(min_max);
m_max_val = std::get<1>(min_max);
std::tie(inFmts, outFmts, priority, selectedType) = cpu_params;
#if defined(OV_CPU_WITH_ACL)
updateSelectedType("ref_any", output_prc, configuration);
#else
if (output_prc == ElementType::i64) {
updateSelectedType(getPrimitiveType(), ElementType::i32, configuration);
} else if (output_prc == ElementType::f64) {
updateSelectedType(getPrimitiveType(), ElementType::f32, configuration);
} else if (output_prc == ElementType::f16) {
if (InferenceEngine::with_cpu_x86_avx512_core_fp16()) {
updateSelectedType(getPrimitiveType(), ElementType::f16, configuration);
} else {
updateSelectedType(getPrimitiveType(), ElementType::f32, configuration);
}
} else if (output_prc == ElementType::bf16) {
if (InferenceEngine::with_cpu_x86_bfloat16()) {
updateSelectedType(getPrimitiveType(), ElementType::bf16, configuration);
} else {
updateSelectedType("ref_any", ElementType::bf16, configuration);
}
} else {
updateSelectedType(getPrimitiveType(), output_prc, configuration);
}
#endif
std::vector<InputShape> in_shapes;
ov::ParameterVector in_params;
std::vector<std::shared_ptr<ov::Node>> inputs;
if (!const_in_1) {
in_shapes.push_back({{}, {{m_output_shape.size()}}});
in_params.push_back(std::make_shared<ov::op::v0::Parameter>(shape_prc, ov::PartialShape{static_cast<int64_t>(m_output_shape.size())}));
in_params.back()->set_friendly_name("shape");
inputs.push_back(in_params.back());
} else {
inputs.push_back(ngraph::builder::makeConstant(shape_prc, {m_output_shape.size()}, m_output_shape));
}
if (!const_in_2) {
in_shapes.push_back({{}, {{1}}});
in_params.push_back(std::make_shared<ov::op::v0::Parameter>(output_prc, ov::PartialShape{1}));
in_params.back()->set_friendly_name("minval");
inputs.push_back(in_params.back());
} else {
inputs.push_back(ngraph::builder::makeConstant(output_prc, {1}, std::vector<double>{m_min_val}));
}
if (!const_in_3) {
in_shapes.push_back({{}, {{1}}});
in_params.push_back(std::make_shared<ov::op::v0::Parameter>(output_prc, ov::PartialShape{1}));
in_params.back()->set_friendly_name("maxval");
inputs.push_back(in_params.back());
} else {
inputs.push_back(ngraph::builder::makeConstant(output_prc, {1}, std::vector<double>{m_max_val}));
}
init_input_shapes(in_shapes);
const auto rnd_op = std::make_shared<ov::op::v8::RandomUniform>(inputs[0], inputs[1], inputs[2], output_prc, m_global_seed, m_operational_seed);
const ov::ResultVector results{std::make_shared<ov::op::v0::Result>(rnd_op)};
function = std::make_shared<ov::Model>(results, in_params, "RandomUniformLayerTestCPU");
}
template<typename TD, typename TS>
void fill_data(TD* dst, const TS* src, size_t len) {
for (size_t i = 0llu; i < len; i++) {
dst[i] = static_cast<TD>(src[i]);
}
}
void RandomUniformLayerTestCPU::generate_inputs(const std::vector<ov::Shape>& targetInputStaticShapes) {
inputs.clear();
const auto& func_inputs = function->inputs();
for (size_t i = 0llu; i < func_inputs.size(); ++i) {
const auto& func_input = func_inputs[i];
const auto& name = func_input.get_node()->get_friendly_name();
const auto& in_prc = func_input.get_element_type();
auto tensor = ov::Tensor(in_prc, targetInputStaticShapes[i]);
#define CASE(P, S, L) \
case P : \
fill_data(tensor.data<ov::element_type_traits<P>::value_type>(), S, L); break;
if (name == "shape") {
switch (in_prc) {
CASE(ElementType::i32, m_output_shape.data(), m_output_shape.size())
CASE(ElementType::i64, m_output_shape.data(), m_output_shape.size())
default:
OPENVINO_THROW("RandomUniform does not support precision ", in_prc, " for the Shape input.");
}
} else if (name == "minval") {
switch (in_prc) {
CASE(ElementType::f32, &m_min_val, 1)
CASE(ElementType::f16, &m_min_val, 1)
CASE(ElementType::bf16, &m_min_val, 1)
CASE(ElementType::i32, &m_min_val, 1)
CASE(ElementType::i64, &m_min_val, 1)
CASE(ElementType::f64, &m_min_val, 1)
default:
OPENVINO_THROW("RandomUniform does not support precision ", in_prc, " for the Minval input.");
}
} else if (name == "maxval") {
switch (in_prc) {
CASE(ElementType::f32, &m_max_val, 1)
CASE(ElementType::f16, &m_max_val, 1)
CASE(ElementType::bf16, &m_max_val, 1)
CASE(ElementType::i32, &m_max_val, 1)
CASE(ElementType::i64, &m_max_val, 1)
CASE(ElementType::f64, &m_max_val, 1)
default:
OPENVINO_THROW("RandomUniform does not support precision ", in_prc, " for the Maxval input.");
}
}
#undef CASE
inputs.insert({func_input.get_node_shared_ptr(), tensor});
}
}
void RandomUniformLayerTestCPU::compare(const std::vector<ov::Tensor>& expected, const std::vector<ov::Tensor>& actual) {
if (m_global_seed != 0lu || m_operational_seed != 0lu) {
SubgraphBaseTest::compare(expected, actual);
return;
}
// When both seed values are equal to zero, RandomUniform should generate non-deterministic sequence.
// In this case will use Mean and Variance metrics.
#define CASE(X) case X : rndUCompare<ov::element_type_traits<X>::value_type>(expected[0], actual[0]); break;
switch (expected[0].get_element_type()) {
CASE(ElementType::f32)
CASE(ElementType::i32)
CASE(ElementType::f16)
CASE(ElementType::bf16)
CASE(ElementType::i64)
CASE(ElementType::f64)
default: OPENVINO_THROW("Unsupported element type: ", expected[0].get_element_type());
}
#undef CASE
}
precisions_map RandomUniformLayerTestCPU::get_ref_precisions_convert_map() {
precisions_map precisions;
if (!InferenceEngine::with_cpu_x86_avx512_core()) {
precisions.insert({ ov::element::bf16, ov::element::f32 });
}
if (!InferenceEngine::with_cpu_x86_avx512_core_fp16()) {
precisions.insert({ ov::element::f16, ov::element::f32 });
}
return precisions;
}
inline double less_or_equal(double a, double b) {
return (b - a) >= (std::fmax(std::fabs(a), std::fabs(b)) * std::numeric_limits<double>::epsilon());
}
template<typename T>
void RandomUniformLayerTestCPU::rndUCompare(const ov::Tensor& expected, const ov::Tensor& actual) {
auto actual_data = actual.data<T>();
size_t shape_size_cnt = ov::shape_size(expected.get_shape());
double act_mean = 0.0;
double act_variance = 0.0;
const double exp_mean = (m_max_val + m_min_val) / 2.0;
const double exp_variance = std::pow(m_max_val - m_min_val, 2) / 12.0;
for (size_t i = 0; i < shape_size_cnt; ++i) {
auto actual_value = static_cast<double>(actual_data[i]);
if (std::isnan(actual_value)) {
std::ostringstream out_stream;
out_stream << "Actual value is NAN on coordinate: " << i;
throw std::runtime_error(out_stream.str());
}
act_mean += actual_value;
act_variance += std::pow(actual_value - exp_mean, 2);
}
act_mean /= shape_size_cnt;
act_variance /= shape_size_cnt;
auto rel_mean = (exp_mean - act_mean) / (m_max_val - m_min_val);
auto rel_variance = (exp_variance - act_variance) / std::pow(m_max_val - m_min_val, 2);
if (!(less_or_equal(rel_mean, m_mean_threshold) && less_or_equal(rel_variance, m_variance_threshold))) {
std::ostringstream out_stream;
out_stream << "rel_mean < m_mean_threshold && rel_variance < m_variance_threshold" <<
"\n\t rel_mean: " << rel_mean <<
"\n\t rel_variance: " << rel_variance;
throw std::runtime_error(out_stream.str());
}
}
TEST_P(RandomUniformLayerTestCPU, CompareWithRefs) {
run();
CheckPluginRelatedResults(compiledModel, "RandomUniform");
}
} // namespace CPULayerTestsDefinitions
src/plugins/intel_cpu/tests/functional/single_layer_tests/classes/random_uniform.hpp
+53
View File
@@ -0,0 +1,53 @@
// Copyright (C) 2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include "shared_test_classes/base/ov_subgraph.hpp"
#include "test_utils/cpu_test_utils.hpp"
namespace CPULayerTestsDefinitions {
typedef std::tuple<
ov::Shape, // Output shapes
std::tuple<double, double>, // Min and Max values
ov::test::ElementType, // Shape precision
ov::test::ElementType, // Output precision
uint64_t, // Global seed
uint64_t, // Operational seed
bool, // Is 1st input constant
bool, // Is 2nd input constant
bool, // Is 3rd input constant
CPUTestUtils::CPUSpecificParams, // CPU specific params
ov::AnyMap // Additional plugin configuration
> RandomUniformLayerTestCPUParamSet;
class RandomUniformLayerTestCPU : public testing::WithParamInterface<RandomUniformLayerTestCPUParamSet>,
public ov::test::SubgraphBaseTest, public CPUTestUtils::CPUTestsBase {
public:
static std::string getTestCaseName(const testing::TestParamInfo<RandomUniformLayerTestCPUParamSet>& obj);
protected:
void SetUp() override;
void generate_inputs(const std::vector<ov::Shape>& target_shapes) override;
void compare(const std::vector<ov::Tensor>& expected, const std::vector<ov::Tensor>& actual) override;
precisions_map get_ref_precisions_convert_map() override;
template<typename T>
void rndUCompare(const ov::Tensor& expected, const ov::Tensor& actual);
private:
ov::Shape m_output_shape;
uint64_t m_global_seed;
uint64_t m_operational_seed;
double m_min_val;
double m_max_val;
static constexpr double m_mean_threshold = 0.05;
static constexpr double m_variance_threshold = 0.1;
};
} // namespace CPULayerTestsDefinitions
src/plugins/intel_cpu/tests/functional/single_layer_tests/instances/common/random_uniform.cpp
+68
View File
@@ -0,0 +1,68 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "single_layer_tests/classes/random_uniform.hpp"
using namespace CPUTestUtils;
using namespace ov::test;
namespace CPULayerTestsDefinitions {
namespace RandomUniform {
static const std::vector<ElementType> shape_prc = {
ElementType::i32,
ElementType::i64
};
static const std::vector<ov::Shape> output_shapes = {
{500},
{4, 3, 210}
};
static const std::vector<uint64_t> global_seed = {
0, 8
};
static const std::vector<uint64_t> operational_seed = {
0, 3, 5
};
static const std::vector<std::tuple<double, double>> min_max = {
{0, 50},
{-50, 50},
{-50, 0}
};
INSTANTIATE_TEST_SUITE_P(smoke_Param, RandomUniformLayerTestCPU,
::testing::Combine(
::testing::ValuesIn(output_shapes),
::testing::ValuesIn(min_max),
::testing::ValuesIn(shape_prc),
::testing::Values(ElementType::f32, ElementType::i32),
::testing::ValuesIn(global_seed),
::testing::ValuesIn(operational_seed),
::testing::Values(false),
::testing::Values(false),
::testing::Values(false),
::testing::Values(emptyCPUSpec),
::testing::Values(empty_plugin_config)),
RandomUniformLayerTestCPU::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_ParamConst, RandomUniformLayerTestCPU,
::testing::Combine(
::testing::Values(output_shapes[0]),
::testing::Values(min_max[0]),
::testing::Values(ElementType::i32),
::testing::Values(ElementType::f32),
::testing::Values(1),
::testing::Values(0),
::testing::Values(true, false),
::testing::Values(true, false),
::testing::Values(true, false),
::testing::Values(emptyCPUSpec),
::testing::Values(empty_plugin_config)),
RandomUniformLayerTestCPU::getTestCaseName);
} // namespace RandomUniform
} // namespace CPULayerTestsDefinitions
src/plugins/intel_cpu/tests/functional/single_layer_tests/instances/x64/random_uniform.cpp
+46
View File
@@ -0,0 +1,46 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "single_layer_tests/classes/random_uniform.hpp"
using namespace CPUTestUtils;
using namespace ov::test;
namespace CPULayerTestsDefinitions {
namespace RandomUniform {
static const std::vector<ElementType> output_prc_nightly = {
ElementType::f32,
ElementType::f16,
ElementType::bf16,
ElementType::i32,
ElementType::i64
};
// Need to validate the Kernel corner cases.
static const std::vector<ov::Shape> output_shapes_nightly = {
{1}, {2}, {3}, {2, 2}, {5}, {2, 3}, {7}, {2, 2, 2}, {3, 3}, {2, 5}, {11}, {2, 3, 2}, {13}, {2, 7}, {3, 5},
{4, 4}, {1, 17}, {2, 9}, {19}, {4, 5}, {21}, {11, 2}, {23, 1}, {4, 2, 3}, {5, 5}, {26}, {1, 27}, {14, 2},
{29}, {10, 3}, {31}, {2, 8, 2}, {33}, {17, 2}, {5, 7}, {2, 3, 2, 3}, {37}, {2, 19}, {2, 20}, {41}, {42},
{43}, {22, 2}, {3, 5, 3}, {5, 2, 5}, {1, 3, 1, 17, 1}, {26, 2}, {53}, {54}, {55}, {56}, {57}, {58}, {59},
{2, 32}, {99}, {127}, {128}, {129}, {199}, {255}, {499}, {997}, {1753}, {2899}
};
INSTANTIATE_TEST_SUITE_P(nightly_Param, RandomUniformLayerTestCPU,
::testing::Combine(
::testing::ValuesIn(output_shapes_nightly),
::testing::Values(std::tuple<double, double>{-31, 17}),
::testing::Values(ElementType::i32),
::testing::ValuesIn(output_prc_nightly),
::testing::Values(3),
::testing::Values(1),
::testing::Values(true, false),
::testing::Values(true, false),
::testing::Values(true, false),
::testing::Values(emptyCPUSpec),
::testing::Values(empty_plugin_config)),
RandomUniformLayerTestCPU::getTestCaseName);
} // namespace RandomUniform
} // namespace CPULayerTestsDefinitions
src/plugins/intel_cpu/tests/functional/test_utils/cpu_test_utils.hpp
+1
View File
@@ -170,6 +170,7 @@ protected:
// common parameters
const auto emptyCPUSpec = CPUSpecificParams{{}, {}, {}, {}};
const std::map<std::string, std::string> cpuEmptyPluginConfig;
const ov::AnyMap empty_plugin_config{};
const std::map<std::string, std::string> cpuFP32PluginConfig =
{ { InferenceEngine::PluginConfigParams::KEY_ENFORCE_BF16, InferenceEngine::PluginConfigParams::NO } };
const std::map<std::string, std::string> cpuBF16PluginConfig =
src/tests/test_utils/common_test_utils/include/common_test_utils/common_utils.hpp
+4
View File
@@ -115,6 +115,10 @@ inline std::string set2str(const std::set<vecElementType>& set) {
return std::string("()");
}
inline std::string bool2str(const bool val) {
return val ? "True" : "False";
}
template <typename master, typename slave>
std::vector<std::pair<master, slave>> combineParams(const std::map<master, std::vector<slave>>& keyValueSets) {
std::vector<std::pair<master, slave>> resVec;