IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[CPU] Native dynamic shapes support in the Softmax node (#7852)

Browse Source
This commit is contained in:
Maksim Kutakov 2021-10-25 14:11:59 +03:00 committed by GitHub
parent 86208efd56
commit e6bc8c59a9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 408 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
inference-engine/src/mkldnn_plugin/memory_desc/cpu_memory_desc_utils.cpp
View File

@ -107,4 +107,14 @@ std::string MemoryDescUtils::dims2str(const VectorDims& dims) {
return output.str();
}
std::shared_ptr<MemoryDesc> MemoryDescUtils::makeDummyDesc(const MemoryDesc &desc, Dim dummyVal) {
const auto& maxDims = desc.getShape().getMaxDims();
const auto& dims = desc.getShape().getDims();
VectorDims dummyDims(dims.size());
for (size_t i = 0; i < dims.size(); ++i) {
dummyDims[i] = dims[i] == Shape::UNDEFINED_DIM ? std::min(maxDims[i], dummyVal) : dims[i];
}
return desc.cloneWithNewDims(dummyDims);
}
} // namespace MKLDNNPlugin

8
inference-engine/src/mkldnn_plugin/memory_desc/cpu_memory_desc_utils.h
View File

@ -72,6 +72,14 @@ public:
*/
static InferenceEngine::TensorDesc convertToTensorDesc(const MemoryDesc& desc);
/**
* @brief Makes a dummy descriptor where all undefined values are replaced with the smallest value between the parameter and the upper bound dim
* @param desc MemoryDesc from which the new descriptor is generated
* @param dummyVal Dim value to replace undefined dimensions
* @return a new MemoryDesc with dummy values instead of undefined dims
*/
static std::shared_ptr<MemoryDesc> makeDummyDesc(const MemoryDesc& desc, Dim dummyVal = 64);
/**
* @brief Converts dim to string, undefined dim represented as ?
* @param dim Dim to be converted

80
inference-engine/src/mkldnn_plugin/memory_desc/dnnl_blocked_memory_desc.cpp
View File

@ -244,22 +244,14 @@ bool DnnlBlockedMemoryDesc::isCompatible(const DnnlBlockedMemoryDesc& rhs) const
thisExtra.scale_adjust == rhsExtra.scale_adjust) && wrappedThis.similar_to(wrappedRhs, true, true, 0, stride_start, true, true);
}
DnnlBlockedMemoryDesc::DnnlBlockedMemoryDesc(const mkldnn::memory::desc& mdesc) :
MemoryDesc(MKLDNNExtensionUtils::convertToVectorDims(mdesc.dims()), DnnlBlocked) {
desc = mdesc;
if (desc.data.format_kind == dnnl::impl::format_kind::any)
IE_THROW(Unexpected) << "Memory format any is prohibited!";
static VectorDims extractOrder(const mkldnn::memory::desc& desc) {
const auto dims = desc.dims();
mkldnn::impl::memory_desc_wrapper descWrapped(desc.data);
if (!descWrapped.is_blocking_desc())
IE_THROW(Unexpected) << "Can't create DnnlBlockedMemoryDesc from not blocking desc";
if (descWrapped.has_runtime_dims_or_strides()) {
IE_THROW(Unexpected) << "Cannot calculate order from undefined dims or strides";
}
const auto dims = desc.dims();
const auto &blk_desc = descWrapped.blocking_desc();
const size_t outer_ndims = dims.size();
@ -285,14 +277,25 @@ DnnlBlockedMemoryDesc::DnnlBlockedMemoryDesc(const mkldnn::memory::desc& mdesc)
(blk_desc.strides[ind_l] == blk_desc.strides[ind_r] && outer_block_dims[ind_l] > outer_block_dims[ind_r]);
});
// blocked order
// [new_outer_order] U [inner_idxs]
SizeVector blk_order(total_ndims, 0);
std::copy(outer_order.begin(), outer_order.end(), blk_order.begin());
std::copy(blk_desc.inner_idxs, blk_desc.inner_idxs + blk_desc.inner_nblks, blk_order.begin() + dims.size());
order.swap(blk_order);
return blk_order;
}
DnnlBlockedMemoryDesc::DnnlBlockedMemoryDesc(const mkldnn::memory::desc& mdesc) :
MemoryDesc(MKLDNNExtensionUtils::convertToVectorDims(mdesc.dims()), DnnlBlocked) {
desc = mdesc;
if (desc.data.format_kind == dnnl::impl::format_kind::any)
IE_THROW(Unexpected) << "Memory format any is prohibited!";
mkldnn::impl::memory_desc_wrapper descWrapped(desc.data);
if (!descWrapped.is_blocking_desc())
IE_THROW(Unexpected) << "Can't create DnnlBlockedMemoryDesc from not blocking desc";
order = extractOrder(desc);
initBlockedParams();
}
@ -362,6 +365,23 @@ bool DnnlBlockedMemoryDesc::isTailCFormat() const {
return true;
}
static mkldnn::memory::desc cloneDescWithNewDims(const mkldnn::memory::desc& desc, const VectorDims& dims, const VectorDims& order) {
using namespace dnnl::impl::utils;
auto mklDims = MKLDNNExtensionUtils::convertToDnnlDims(dims);
mkldnn::memory::desc newMklDesc = desc;
array_copy(newMklDesc.data.dims, mklDims.data(), mklDims.size());
std::vector<int> perm(order.begin(), order.begin() + mklDims.size());
auto& blockingDesc = newMklDesc.data.format_desc.blocking;
auto numInnerBlks = blockingDesc.inner_nblks;
std::vector<int> innerBlks(std::begin(blockingDesc.inner_blks), std::begin(blockingDesc.inner_blks) + numInnerBlks);
std::vector<int> innerIdxs(std::begin(blockingDesc.inner_idxs), std::begin(blockingDesc.inner_idxs) + numInnerBlks);
auto retCode = dnnl::impl::fill_blocked(newMklDesc.data, perm, innerBlks, innerIdxs);
if (retCode != dnnl::impl::status::success) {
IE_THROW() << "Can not clone DnnlBlockedMemoryDesc with dims: " << MemoryDescUtils::dims2str(dims);
}
return newMklDesc;
}
MemoryDescPtr DnnlBlockedMemoryDesc::cloneWithNewDimsImp(const VectorDims &dims) const {
if (std::any_of(dims.begin(), dims.end(), [](size_t x){ return Shape::UNDEFINED_DIM == x; })) {
IE_THROW() << "Can't clone desc if new dims are undefined";
@ -378,20 +398,7 @@ MemoryDescPtr DnnlBlockedMemoryDesc::cloneWithNewDimsImp(const VectorDims &dims)
IE_THROW(NotImplemented) << "Can't clone desc with new dims for not dense tensor";
}
using namespace dnnl::impl::utils;
auto mklDims = MKLDNNExtensionUtils::convertToDnnlDims(dims);
mkldnn::memory::desc newMklDesc = desc;
array_copy(newMklDesc.data.dims, mklDims.data(), mklDims.size());
std::vector<int> perm(order.begin(), order.begin() + mklDims.size());
auto& blockingDesc = newMklDesc.data.format_desc.blocking;
auto numInnerBlks = blockingDesc.inner_nblks;
std::vector<int> innerBlks(std::begin(blockingDesc.inner_blks), std::begin(blockingDesc.inner_blks) + numInnerBlks);
std::vector<int> innerIdxs(std::begin(blockingDesc.inner_idxs), std::begin(blockingDesc.inner_idxs) + numInnerBlks);
auto retCode = dnnl::impl::fill_blocked(newMklDesc.data, perm, innerBlks, innerIdxs);
if (retCode != dnnl::impl::status::success) {
IE_THROW() << "Can not clone DnnlBlockedMemoryDesc with dims: " << MemoryDescUtils::dims2str(dims);
}
return DnnlBlockedMemoryDescPtr(new DnnlBlockedMemoryDesc(newMklDesc));
return DnnlBlockedMemoryDescPtr(new DnnlBlockedMemoryDesc(cloneDescWithNewDims(desc, dims, order)));
}
static const std::map<int, std::vector<mkldnn::memory::format_tag>> form_tags_by_ndims {
@ -828,3 +835,24 @@ void DnnlBlockedMemoryDesc::recomputeDefaultStrides() {
}
}
}
DnnlBlockedMemoryDesc::DnnlBlockedMemoryDesc(const mkldnn::memory::desc& mdesc, const Shape& shape) :
MemoryDesc(shape, DnnlBlocked) {
if (mdesc.data.format_kind == dnnl::impl::format_kind::any)
IE_THROW(Unexpected) << "Memory format any is prohibited!";
mkldnn::impl::memory_desc_wrapper descWrapped(mdesc.data);
if (!descWrapped.is_blocking_desc())
IE_THROW(Unexpected) << "Can't create DnnlBlockedMemoryDesc from not blocking desc";
if (!shape.isCompatible(MKLDNNExtensionUtils::convertToVectorDims(mdesc.dims()))) {
IE_THROW(ParameterMismatch) << "Can not create DnnlBlockedMemoryDesc. memory::desc dims: " << vec2str(mdesc.dims()) <<
" are incompatible with provided shape: " << shape.toString() << ".";
}
order = extractOrder(mdesc);
desc = cloneDescWithNewDims(mdesc, shape.getDims(), order);
initBlockedParams();
}

6
inference-engine/src/mkldnn_plugin/memory_desc/dnnl_blocked_memory_desc.h
View File

@ -70,6 +70,11 @@ private:
explicit DnnlBlockedMemoryDesc(const mkldnn::memory::desc& mdesc);
// Creates DnnlBlockedMemoryDesc using the shape parameter as a true shape but all other params (layout, blocks, etc.) are used from the mdesc, but
// the mdesc own shape is ignored. The main purpose of this constructor is making dynamic descriptor form some dummy mdesc, which stores info about
// layout, blocking, strides, etc., and the provided dynamic shape.
DnnlBlockedMemoryDesc(const mkldnn::memory::desc& mdesc, const Shape& shape);
MemoryDescPtr cloneWithNewDimsImp(const VectorDims& dims) const override;
bool isPlainFormat() const;
@ -98,6 +103,7 @@ private:
mkldnn::memory::format_tag getFormat() const;
friend DnnlMemoryDescPtr MKLDNNExtensionUtils::makeDescriptor(const mkldnn::memory::desc &desc);
friend std::shared_ptr<DnnlBlockedMemoryDesc> MKLDNNExtensionUtils::makeUndefinedDesc(const mkldnn::memory::desc &desc, const Shape& shape);
friend class MemoryDescUtils;
};

8
inference-engine/src/mkldnn_plugin/mkldnn_extension_utils.cpp
View File

@ -136,3 +136,11 @@ size_t MKLDNNExtensionUtils::getMemSizeForDnnlDesc(mkldnn::memory::desc desc) {
size += offset0 * sizeOfDataType(desc.data_type());
return size;
}
std::shared_ptr<DnnlBlockedMemoryDesc> MKLDNNExtensionUtils::makeUndefinedDesc(const memory::desc &desc, const Shape &shape) {
if (desc.data.format_kind == dnnl_blocked) {
return std::shared_ptr<DnnlBlockedMemoryDesc>(new DnnlBlockedMemoryDesc(desc, shape));
} else {
IE_THROW(Unexpected) << "Cannot make undefined descriptor. Only dnnl_blocked type is allowed.";
}
}

11
inference-engine/src/mkldnn_plugin/mkldnn_extension_utils.h
View File

@ -34,6 +34,17 @@ public:
* @return pointer to DnnlBlockedMemoryDesc or DnnlMemoryDesc
*/
static std::shared_ptr<DnnlMemoryDesc> makeDescriptor(const mkldnn::memory::desc &desc);
/**
* @brief Helper function that creates DnnlBlockedMemoryDesc from defined mkldnn::memory::desc and undefined shape.
* It uses desc as an basis for the new undefined one. Specifically, type, layout, precision, blocks, extra data will be preserved.
* @param desc mkldnn::memory::desc dnnl desc which will be used as a basis of the new descriptor
* @param shape a new undefined shape
* @return pointer to the created DnnlBlockedMemoryDesc
* @note Obly blocked descriptors are allowed at the moment
*/
static std::shared_ptr<DnnlBlockedMemoryDesc> makeUndefinedDesc(const mkldnn::memory::desc &desc, const Shape& shape);
static size_t getMemSizeForDnnlDesc(mkldnn::memory::desc desc);
};

41
inference-engine/src/mkldnn_plugin/mkldnn_node.cpp
View File

@ -282,6 +282,11 @@ void MKLDNNNode::selectPreferPrimitiveDescriptor(const std::vector<impl_desc_typ
}
bool MKLDNNNode::canBeInPlace() const {
// TODO [DS]: enable inPlace for dynamic shapes
if (isDynamicNode()) {
return false;
}
if (getParentEdges().size() != 1 || getParentEdgeAt(0)->getParent()->getChildEdges().size() != 1 ||
(getParentEdgeAt(0)->getParent()->isConstant() && !getParentEdgeAt(0)->getChild()->isConstant()))
return false;
@ -494,8 +499,11 @@ void MKLDNNNode::execute(mkldnn::stream strm) {
void MKLDNNNode::executeDynamic(mkldnn::stream strm) {
if (needShapeInfer())
redefineOutputMemory(shapeInfer());
if (needPrepareParams())
if (needPrepareParams()) {
IE_ASSERT(inputShapesDefined()) << "Can't prepare params for " << getTypeStr() << " node with name: " << getName() <<
" since the input shapes are not defined.";
prepareParams();
}
executeDynamicImpl(strm);
updateLastInputDims();
}
@ -942,10 +950,16 @@ bool MKLDNNNode::isConfigDefined(const NodeConfig &config) const {
}
MemoryDescPtr MKLDNNNode::getSrcMemDesc(mkldnn::primitive_desc_iterator &primitive_desc_it, size_t idx) {
if (isDynamicNode()) {
return MKLDNNExtensionUtils::makeUndefinedDesc(primitive_desc_it.src_desc(idx), getInputShapeAtPort(idx));
}
return MKLDNNExtensionUtils::makeDescriptor(primitive_desc_it.src_desc(idx));
}
MemoryDescPtr MKLDNNNode::getDstMemDesc(mkldnn::primitive_desc_iterator &primitive_desc_it, size_t idx) {
if (isDynamicNode()) {
return MKLDNNExtensionUtils::makeUndefinedDesc(primitive_desc_it.dst_desc(idx), getOutputShapeAtPort(idx));
}
return MKLDNNExtensionUtils::makeDescriptor(primitive_desc_it.dst_desc(idx));
}
@ -1224,18 +1238,33 @@ bool MKLDNNNode::needShapeInfer() const {
}
std::vector<VectorDims> MKLDNNNode::shapeInfer() const {
std::vector<Shape> shapes;
for (size_t i = 0; i < inputShapes.size(); i++) {
shapes.push_back(getParentEdgesAtPort(i)[0]->getMemory().getDesc().getShape());
}
auto newOutputShapes = shapeInferGeneric(shapes);
IE_ASSERT(newOutputShapes.size() == outputShapes.size());
return newOutputShapes;
}
std::vector<VectorDims> MKLDNNNode::shapeInferGeneric(const std::vector<Shape>& shapes) const {
if (shapes.size() < opToShapeInfer->get_input_size()) {
IE_THROW(Unexpected) << "MKLDNNNode::shapeInferGeneric input shapes vector size is " << shapes.size() << ", but " << opToShapeInfer->get_input_size() <<
" required for node with name: " << getName();
}
for (size_t i = 0; i < opToShapeInfer->get_input_size(); i++) {
if (!dynamic_cast<ngraph::opset1::Constant *>(opToShapeInfer->get_input_node_ptr(i))) {
opToShapeInfer->get_input_tensor(i).set_partial_shape(
getParentEdgesAtPort(i)[0]->getMemory().getDesc().getShape().toPartialShape());
opToShapeInfer->get_input_tensor(i).set_partial_shape(shapes[i].toPartialShape());
}
}
opToShapeInfer->validate_and_infer_types();
IE_ASSERT(opToShapeInfer->get_output_size() == outputShapes.size());
std::vector<VectorDims> newOutputShapes(outputShapes.size());
std::vector<VectorDims> newOutputShapes(opToShapeInfer->get_output_size());
for (size_t i = 0; i < newOutputShapes.size(); i++) {
const auto &partShape = opToShapeInfer->get_output_partial_shape(i);
if (partShape.is_dynamic())

1
inference-engine/src/mkldnn_plugin/mkldnn_node.h
View File

@ -698,6 +698,7 @@ protected:
bool inputShapesModified() const;
virtual bool needShapeInfer() const;
std::vector<VectorDims> shapeInferGeneric(const std::vector<Shape>& inputDims) const;
virtual std::vector<VectorDims> shapeInfer() const;
// TODO [DS] : make pure after all nodes will be support dynamic shapes
virtual void executeDynamicImpl(mkldnn::stream strm) {

86
inference-engine/src/mkldnn_plugin/nodes/mkldnn_softmax_node.cpp
View File

@ -17,10 +17,6 @@ using namespace InferenceEngine;
bool MKLDNNSoftMaxNode::isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept {
try {
if (isDynamicNgraphNode(op)) {
errorMessage = "Doesn't support op with dynamic shapes";
return false;
}
if (!std::dynamic_pointer_cast<const ngraph::opset1::Softmax>(op)) {
errorMessage = "Only opset1 Softmax operation is supported";
return false;
@ -71,38 +67,11 @@ void MKLDNNSoftMaxNode::getSupportedDescriptors() {
}
void MKLDNNSoftMaxNode::createPrimitive() {
if (prim)
return;
auto in_candidate = getParentEdgeAt(0)->getMemory().GetDescWithType<DnnlMemoryDesc>()->getDnnlDesc();
MKLDNNDescriptor desc(std::shared_ptr<softmax_forward::desc>(
new softmax_forward::desc(prop_kind::forward_scoring, in_candidate, axis)));
descs[0] = desc;
std::shared_ptr<softmax_forward::desc> selected_desc_ptr = descs[0];
const NodeDesc *selected_pd = getSelectedPrimitiveDescriptor();
if (selected_pd == nullptr)
IE_THROW() << "Preferable primitive descriptor is not set for node " << getName() << ".";
auto prim_desc = softmax_forward::primitive_desc(*selected_desc_ptr, getEngine());
primitive_desc_iterator itpd = descs[0].createPrimitiveDescriptorIterator(getEngine());
while (itpd) {
impl_desc_type impl_type = parse_impl_name(itpd.impl_info_str());
auto primitiveDescriptor = getSelectedPrimitiveDescriptor();
if ((primitiveDescriptor != nullptr) && (impl_type == primitiveDescriptor->getImplementationType())) {
prim_desc = itpd.get();
break;
}
if (!itpd.next_impl())
break;
if (inputShapesDefined()) {
if (needPrepareParams())
prepareParams();
updateLastInputDims();
}
prim.reset(new softmax_forward(prim_desc));
auto src = getParentEdgesAtPort(0)[0]->getMemoryPtr()->GetPrimitive();
auto dst = getChildEdgesAtPort(0)[0]->getMemoryPtr()->GetPrimitive();
primArgs = {{DNNL_ARG_SRC, src}, {DNNL_ARG_DST, dst}};
}
bool MKLDNNSoftMaxNode::created() const {
@ -136,10 +105,55 @@ void MKLDNNSoftMaxNode::initOptimalPrimitiveDescriptor() {
void MKLDNNSoftMaxNode::createDescriptor(const std::vector<MemoryDescPtr> &inputDesc,
const std::vector<MemoryDescPtr> &outputDesc) {
auto in_candidate = MemoryDescUtils::convertToDnnlMemoryDesc(inputDesc[0])->getDnnlDesc();
auto inpDesc = inputDesc[0]->isDefined() ? inputDesc[0] : MemoryDescUtils::makeDummyDesc(*inputDesc[0]);
DnnlMemoryDescPtr definedInpMemDesc = MemoryDescUtils::convertToDnnlMemoryDesc(inpDesc);
auto in_candidate = definedInpMemDesc->getDnnlDesc();
MKLDNNDescriptor desc(std::shared_ptr<softmax_forward::desc>(
new softmax_forward::desc(prop_kind::forward_scoring, in_candidate, axis)));
descs.push_back(desc);
}
void MKLDNNSoftMaxNode::prepareParams() {
auto inpDesc = getParentEdgeAt(0)->getMemory().GetDescWithType<DnnlMemoryDesc>();
const auto& in_candidate = inpDesc->getDnnlDesc();
MKLDNNDescriptor desc(std::shared_ptr<softmax_forward::desc>(
new softmax_forward::desc(prop_kind::forward_scoring, in_candidate, axis)));
const NodeDesc *selected_pd = getSelectedPrimitiveDescriptor();
if (selected_pd == nullptr)
IE_THROW() << "Preferable primitive descriptor is not set for node " << getName() << ".";
softmax_forward::primitive_desc prim_desc;
primitive_desc_iterator itpd = desc.createPrimitiveDescriptorIterator(getEngine());
while (itpd) {
impl_desc_type impl_type = parse_impl_name(itpd.impl_info_str());
if (impl_type == selected_pd->getImplementationType() ||
// At least for oneDNN v2.4 the softmax primitive is optimized for the cases where the dimension of the softmax axis is physically dense.
// There could be situations where it is not possible to detect the optimized case in advance in case of dynamic shapes, but
// in runtime the shape could be suitable for the optimized implementation, so we have to select the optimized one.
(ref_any == selected_pd->getImplementationType() && (impl_type & jit))) {
prim_desc = itpd.get();
break;
}
if (!itpd.next_impl())
IE_THROW() << "Primitive descriptor was not found for node " << getName() << ".";
}
prim.reset(new softmax_forward(prim_desc));
auto src = getParentEdgesAtPort(0)[0]->getMemoryPtr()->GetPrimitive();
auto dst = getChildEdgesAtPort(0)[0]->getMemoryPtr()->GetPrimitive();
primArgs = {{DNNL_ARG_SRC, src}, {DNNL_ARG_DST, dst}};
}
void MKLDNNSoftMaxNode::executeDynamicImpl(dnnl::stream strm) {
MKLDNNNode::execute(strm);
}
std::vector<VectorDims> MKLDNNSoftMaxNode::shapeInfer() const {
return {getParentEdgesAtPort(0).front()->getMemory().getStaticDims()};
}
REG_MKLDNN_PRIM_FOR(MKLDNNSoftMaxNode, Softmax);

4
inference-engine/src/mkldnn_plugin/nodes/mkldnn_softmax_node.h
View File

@ -25,6 +25,10 @@ public:
static bool isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept;
void prepareParams() override;
void executeDynamicImpl(mkldnn::stream strm) override;
std::vector<VectorDims> shapeInfer() const override;
private:
size_t axis = 0;
};

143
inference-engine/tests/functional/plugin/cpu/single_layer_tests/softmax.cpp
View File

@ -9,8 +9,10 @@ using namespace InferenceEngine;
using namespace CPUTestUtils;
namespace CPULayerTestsDefinitions {
using ShapesDefenition = std::pair<std::vector<ngraph::PartialShape>, std::vector<std::vector<ngraph::Shape>>>;
struct SoftMaxConfig {
InferenceEngine::SizeVector inputShape;
ShapesDefenition inputShapes;
size_t axis;
};
@ -33,7 +35,20 @@ public:
std::ostringstream result;
result << "netPRC=" << netPrecision.name() << "_";
result << "IS=" << CommonTestUtils::vec2str(config.inputShape) << "_";
if (!config.inputShapes.first.empty()) {
result << "IS=" << CommonTestUtils::partialShape2str(config.inputShapes.first) << "_";
}
result << "TS=";
for (const auto& shape : config.inputShapes.second) {
result << "(";
if (!shape.empty()) {
auto itr = shape.begin();
do {
result << CommonTestUtils::vec2str(*itr);
} while (++itr != shape.end() && result << "_");
}
result << ")_";
}
result << "axis=" << config.axis << "_";
result << "trgDev=" << targetDevice;
result << CPUTestsBase::getTestCaseName(cpuParams);
@ -59,7 +74,12 @@ protected:
const auto ngPrc = FuncTestUtils::PrecisionUtils::convertIE2nGraphPrc(netPrecision);
auto params = ngraph::builder::makeParams(ngPrc, {config.inputShape});
targetStaticShapes = config.inputShapes.second;
inputDynamicShapes = config.inputShapes.first;
auto inputShape = targetStaticShapes.front().front();
auto params = ngraph::builder::makeParams(ngPrc, {inputShape});
const auto paramOuts =
ngraph::helpers::convert2OutputVector(ngraph::helpers::castOps2Nodes<ngraph::op::Parameter>(params));
@ -81,44 +101,97 @@ namespace {
//not optimized cpu spec
const auto notOptimizedCPUSpec = CPUSpecificParams{{}, {}, {}, "ref_any"};
const std::vector<SoftMaxConfig> optimizedConfigsFP32 {
{InferenceEngine::SizeVector{1, 100}, 1},
{InferenceEngine::SizeVector{10, 10}, 1},
{InferenceEngine::SizeVector{100, 1}, 0},
{InferenceEngine::SizeVector{100, 1}, 1},
{InferenceEngine::SizeVector{5, 5, 1}, 1},
{InferenceEngine::SizeVector{5, 5, 5}, 2},
{InferenceEngine::SizeVector{5, 5, 5, 5}, 0},
{InferenceEngine::SizeVector{5, 5, 1, 1}, 1},
{InferenceEngine::SizeVector{5, 5, 5, 5}, 1},
{InferenceEngine::SizeVector{5, 5, 5, 1}, 2},
{InferenceEngine::SizeVector{5, 5, 5, 5}, 2},
{InferenceEngine::SizeVector{5, 5, 5, 5}, 3},
{InferenceEngine::SizeVector{5, 5, 5, 5, 5}, 0},
{InferenceEngine::SizeVector{5, 5, 1, 1, 1}, 1},
{InferenceEngine::SizeVector{5, 5, 5, 5, 5}, 1},
{InferenceEngine::SizeVector{5, 5, 5, 1, 1}, 2},
{InferenceEngine::SizeVector{5, 5, 5, 5, 5}, 2},
{InferenceEngine::SizeVector{5, 5, 5, 1, 1}, 3},
{InferenceEngine::SizeVector{5, 5, 5, 5, 5}, 3},
{InferenceEngine::SizeVector{5, 5, 5, 5, 1}, 4},
{InferenceEngine::SizeVector{5, 5, 5, 5, 5}, 4},
const std::vector<SoftMaxConfig> optimizedConfigsFP32 = {
//Static shapes
{ShapesDefenition{{}, {{{1, 100}}}}, 1},
{ShapesDefenition{{}, {{{10, 10}}}}, 1},
{ShapesDefenition{{}, {{{100, 1}}}}, 0},
{ShapesDefenition{{}, {{{100, 1}}}}, 1},
{ShapesDefenition{{}, {{{5, 5, 1}}}}, 1},
{ShapesDefenition{{}, {{{5, 5, 5}}}}, 2},
{ShapesDefenition{{}, {{{5, 5, 5, 5}}}}, 0},
{ShapesDefenition{{}, {{{5, 5, 1, 1}}}}, 1},
{ShapesDefenition{{}, {{{5, 5, 5, 5}}}}, 1},
{ShapesDefenition{{}, {{{5, 5, 5, 1}}}}, 2},
{ShapesDefenition{{}, {{{5, 5, 5, 5}}}}, 2},
{ShapesDefenition{{}, {{{5, 5, 5, 5}}}}, 3},
{ShapesDefenition{{}, {{{5, 5, 5, 5, 5}}}}, 0},
{ShapesDefenition{{}, {{{5, 5, 1, 1, 1}}}}, 1},
{ShapesDefenition{{}, {{{5, 5, 5, 5, 5}}}}, 1},
{ShapesDefenition{{}, {{{5, 5, 5, 1, 1}}}}, 2},
{ShapesDefenition{{}, {{{5, 5, 5, 5, 5}}}}, 2},
{ShapesDefenition{{}, {{{5, 5, 5, 1, 1}}}}, 3},
{ShapesDefenition{{}, {{{5, 5, 5, 5, 5}}}}, 3},
{ShapesDefenition{{}, {{{5, 5, 5, 5, 1}}}}, 4},
{ShapesDefenition{{}, {{{5, 5, 5, 5, 5}}}}, 4},
//Dynamic shapes
{ShapesDefenition{
{ //dynamic shape
{-1, -1}
},
{ //target static shapes
{{10, 10}},
{{15, 15}},
{{10, 5}}
}}, 1},
{ShapesDefenition{
{ //dynamic shape
{{1, 100}, {1, 100}}
},
{ //target static shapes
{{10, 10}},
{{15, 15}},
{{10, 5}}
}}, 1},
{ShapesDefenition{
{ //dynamic shape
{-1, -1, 1, 1, 1}
},
{ //target static shapes
{{5, 5, 1, 1, 1}},
{{10, 7, 1, 1, 1}}
}}, 1},
};
const std::vector<SoftMaxConfig> notOptimizedConfigsFP32 {
{InferenceEngine::SizeVector{1, 100}, 0},
{InferenceEngine::SizeVector{10, 10}, 0},
{InferenceEngine::SizeVector{10, 10, 10}, 0},
{InferenceEngine::SizeVector{10, 10, 10}, 1},
//Static shapes
{ShapesDefenition{{}, {{{1, 100}}}}, 0},
{ShapesDefenition{{}, {{{10, 10}}}}, 0},
{ShapesDefenition{{}, {{{10, 10, 10}}}}, 0},
{ShapesDefenition{{}, {{{10, 10, 10}}}}, 1},
//Dynamic shapes
{ShapesDefenition{
{ //dynamic shape
{-1, -1}
},
{ //target static shapes
{{10, 1}}, {{15, 15}}, {{10, 5}}
}}, 0},
{ShapesDefenition{
{ //dynamic shape
{{1, 100}, {1, 100}, -1}
},
{ //target static shapes
{{10, 10, 10}}, {{10, 10, 1}}, {{10, 5, 10}}
}}, 1},
};
const std::vector<SoftMaxConfig> unsupportedConfigsFP32 {
{InferenceEngine::SizeVector{5, 5, 5, 5, 5, 5}, 0},
{InferenceEngine::SizeVector{5, 5, 5, 5, 5, 5}, 1},
{InferenceEngine::SizeVector{5, 5, 5, 5, 5, 5}, 2},
{InferenceEngine::SizeVector{5, 5, 5, 5, 5, 5}, 3},
{InferenceEngine::SizeVector{5, 5, 5, 5, 5, 5}, 4},
{InferenceEngine::SizeVector{5, 5, 5, 5, 5, 5}, 5},
//Static shapes
{ShapesDefenition{{}, {{{5, 5, 5, 5, 5, 5}}}}, 0},
{ShapesDefenition{{}, {{{5, 5, 5, 5, 5, 5}}}}, 1},
{ShapesDefenition{{}, {{{5, 5, 5, 5, 5, 5}}}}, 2},
{ShapesDefenition{{}, {{{5, 5, 5, 5, 5, 5}}}}, 3},
{ShapesDefenition{{}, {{{5, 5, 5, 5, 5, 5}}}}, 4},
{ShapesDefenition{{}, {{{5, 5, 5, 5, 5, 5}}}}, 5},
//Dynamic shapes
{ShapesDefenition{
{ //dynamic shape
{-1, -1, -1, -1, -1, -1}
},
{ //target static shapes
{{5, 5, 5, 5, 5, 5}}, {{7, 7, 7, 7, 7, 7}}
}}, 4},
};
const auto OptimizedParams = testing::Combine(

6
inference-engine/tests/ie_test_utils/common_test_utils/common_utils.hpp
View File

@ -50,9 +50,11 @@ inline std::string vec2str(const std::vector<vecElementType> &vec) {
inline std::string partialShape2str(const std::vector<ngraph::PartialShape>& partialShapes) {
std::ostringstream result;
for (const auto& partialShape : partialShapes) {
result << vec2str(partialShape.get_min_shape()) << "_" << vec2str(partialShape.get_max_shape());
result << partialShape;
}
return result.str();
auto retStr = result.str();
std::replace(retStr.begin(), retStr.end(), ',', '.');
return retStr;
}
inline std::string pair2str(const std::pair<size_t, size_t>& p) {

109
inference-engine/tests/unit/cpu/mkldnn_memory_desc_test.cpp
View File

@ -341,6 +341,115 @@ TEST(MemDescTest, MemSize) {
ASSERT_EQ(blockedDescDefined.getCurrentMemSize(), memDescDefined.getCurrentMemSize());
}
TEST(MakeUndefinedDnnlDesc, wrongType) {
GTEST_SKIP();
}
TEST(MakeUndefinedDnnlDesc, checkRank) {
using mkldnn::memory;
const memory::data_type dataType = memory::data_type::u8;
const memory::desc origin({10, 20, 15, 7}, dataType, memory::format_tag::nChw16c);
MKLDNNPlugin::Shape pluginShapeWrongRank(ngraph::PartialShape{{-1, -1}, {-1, -1}, {-1, -1}});
ASSERT_THROW(MKLDNNExtensionUtils::makeUndefinedDesc(origin, pluginShapeWrongRank), InferenceEngine::ParameterMismatch);
MKLDNNPlugin::Shape pluginShapeRightRank(ngraph::PartialShape{{-1, -1}, {-1, -1}, {-1, -1}, {-1, -1}});
MemoryDescPtr memDesc;
ASSERT_NO_THROW(memDesc = MKLDNNExtensionUtils::makeUndefinedDesc(origin, pluginShapeRightRank));
ASSERT_FALSE(memDesc->isDefined());
}
TEST(MakeUndefinedDnnlDesc, checkDims) {
using mkldnn::memory;
const memory::data_type dataType = memory::data_type::u8;
const memory::desc origin({10, 20, 15, 7}, dataType, memory::format_tag::nChw16c);
ngraph::PartialShape fullyUndef({{-1, -1}, {-1, -1}, {-1, -1}, {-1, -1}});
for (size_t i = 0; i < fullyUndef.size(); ++i) {
auto partialShape = fullyUndef;
partialShape[i] = {3}; // just a number which is not equal to any origin dims
ASSERT_THROW(MKLDNNExtensionUtils::makeUndefinedDesc(origin, MKLDNNPlugin::Shape(partialShape)), InferenceEngine::ParameterMismatch);
}
for (size_t i = 0; i < origin.dims().size(); ++i) {
auto partialShape = fullyUndef;
partialShape[i] = {origin.dims()[i]};
MemoryDescPtr memDesc;
ASSERT_NO_THROW(memDesc = MKLDNNExtensionUtils::makeUndefinedDesc(origin, MKLDNNPlugin::Shape(fullyUndef)));
ASSERT_FALSE(memDesc->isDefined());
}
}
TEST(MakeUndefinedDnnlDesc, checkLayout) {
using mkldnn::memory;
using payloadArgs = std::tuple<memory::format_tag, memory::dims, std::string>;
const memory::data_type dataType = memory::data_type::u8;
payloadArgs payload[] {
payloadArgs{ memory::format_tag::nChw16c, {1, 1, 10, 10}, "aBcd16b" }, // auto blocked
payloadArgs{ memory::format_tag::nhwc, {4, 2, 10, 7 }, "acdb" }, // permuted
payloadArgs{ memory::format_tag::nchw, {4, 2, 10, 7 }, "abcd" }, // plain
payloadArgs{ memory::format_tag::NChw16n16c, {4, 2, 10, 7 }, "ABcd16a16b" }, // blocked for 2 dims
payloadArgs{ memory::format_tag::Acdb16a, {96, 1, 7, 7 }, "Acdb16a" }, // same strides but not default order
// TODO [DS]: uncomment when serializeFormat() properly handles the permutation
//payloadArgs{ memory::format_tag::BAcd16a16b, {17, 2, 10, 7 }, "BAcd16a16b" }, // blocked and permuted outer dims
};
ngraph::PartialShape fullyUndef({{-1, -1}, {-1, -1}, {-1, -1}, {-1, -1}});
for (const auto& item : payload) {
dnnl::memory::format_tag fmt;
dnnl::memory::dims dims;
std::string strFormat;
std::tie(fmt, dims, strFormat) = item;
const memory::desc origin(dims, dataType, fmt);
auto undefDesc = MKLDNNExtensionUtils::makeUndefinedDesc(origin, MKLDNNPlugin::Shape(fullyUndef));
ASSERT_FALSE(undefDesc->isDefined());
MKLDNNPlugin::DnnlBlockedMemoryDesc referenceDesc(MKLDNNPlugin::Shape(fullyUndef), dataType, fmt);
ASSERT_TRUE(undefDesc->isCompatible(referenceDesc));
ASSERT_EQ(undefDesc->serializeFormat(), strFormat);
auto defDesc = undefDesc->cloneWithNewDims(MKLDNNExtensionUtils::convertToVectorDims(dims));
ASSERT_TRUE(defDesc->isDefined());
ASSERT_EQ(origin, defDesc->as<DnnlBlockedMemoryDesc>()->getDnnlDesc());
}
}
TEST(MakeUndefinedDnnlDesc, extraData) {
using mkldnn::memory;
using payloadArgs = std::tuple<memory::format_tag, memory::dims>;
const memory::data_type dataType = memory::data_type::u8;
payloadArgs payload[] {
payloadArgs{ memory::format_tag::nChw16c, {1, 1, 10, 10} }, // auto blocked
payloadArgs{ memory::format_tag::nhwc, {4, 2, 10, 7 } }, // permuted
payloadArgs{ memory::format_tag::nchw, {4, 2, 10, 7 } }, // plain
payloadArgs{ memory::format_tag::NChw16n16c, {4, 2, 10, 7 } }, // blocked for 2 dims
payloadArgs{ memory::format_tag::Acdb16a, {96, 1, 7, 7 } }, // same strides but not default order
payloadArgs{ memory::format_tag::BAcd16a16b, {17, 2, 10, 7 } }, // blocked and permuted outer dims
};
ngraph::PartialShape fullyUndef({{-1, -1}, {-1, -1}, {-1, -1}, {-1, -1}});
for (const auto& item : payload) {
dnnl::memory::format_tag fmt;
dnnl::memory::dims dims;
std::tie(fmt, dims) = item;
memory::desc origin(dims, dataType, fmt);
origin.data.extra.flags = dnnl_memory_extra_flag_compensation_conv_s8s8;
origin.data.extra.compensation_mask = 1;
origin.data.extra.scale_adjust = 2.0f;
auto undefDesc = MKLDNNExtensionUtils::makeUndefinedDesc(origin, MKLDNNPlugin::Shape(fullyUndef));
ASSERT_FALSE(undefDesc->isDefined());
auto defDesc = undefDesc->cloneWithNewDims(MKLDNNExtensionUtils::convertToVectorDims(dims));
ASSERT_TRUE(defDesc->isDefined());
auto referenceDesc = MKLDNNExtensionUtils::makeDescriptor(origin);
ASSERT_TRUE(defDesc->isCompatible(*referenceDesc));
ASSERT_EQ(origin, defDesc->as<DnnlBlockedMemoryDesc>()->getDnnlDesc());
}
}
TEST(isSameMethodTest, CheckTensorWithSameStrides) {
auto isSameDataFormat = [] (dnnl::memory::format_tag fmt, dnnl::memory::dims dims) {