IntenseWebs/openvino
3
0 Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[CPU] Dynamic shapes support using fallback on reference (#6882)

Browse Source
This commit is contained in:
Maksim Kutakov authored and GitHub committed 2021-09-08 10:33:20 +03:00
1 parent 4547818fb1
commit 5096fe19f1
197 files changed
+5538 -3628
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

docs/template_plugin/tests/functional/shared_tests_instances/behavior/infer_request_dynamic.cpp
+3 -2
View File
@@ -21,10 +21,11 @@ const std::vector<std::map<std::string, std::string>> configs = {
INSTANTIATE_TEST_CASE_P(smoke_BehaviorTests, InferRequestDynamicTests,
::testing::Combine(
::testing::ValuesIn(netPrecisions),
::testing::Values(ngraph::builder::subgraph::makeSplitConvConcat()),
::testing::Values(std::vector<std::pair<std::vector<size_t>, std::vector<size_t>>>{{{1, 4, 20, 20}, {1, 10, 18, 18}},
{{2, 4, 20, 20}, {2, 10, 18, 18}}}),
::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
::testing::ValuesIn(configs)),
InferRequestDynamicTests::getTestCaseName);
} // namespace
inference-engine/src/cldnn_engine/cldnn_engine.cpp
-12
View File
@@ -308,18 +308,6 @@ InferenceEngine::CNNNetwork clDNNEngine::CloneAndTransformNetwork(const Inferenc
return true;
});
pass_config->set_callback<ngraph::pass::ConvertNMS1ToNMS5,
ngraph::pass::ConvertNMS3ToNMS5,
ngraph::pass::ConvertNMS4ToNMS5,
ngraph::pass::ConvertNMSToNMSIEInternal>(
[](const_node_ptr &node) -> bool {
return node->input_value(0).get_shape().back() == 4lu &&
node->input_value(0).get_shape().front() == node->input_value(1).get_shape().front() &&
node->input_value(0).get_shape()[1] == node->input_value(1).get_shape().back() &&
node->input_value(0).get_shape().size() == 3lu &&
node->input_value(1).get_shape().size() == 3lu;
});
pass_config->set_callback<ngraph::pass::MVN6Decomposition>(
[](const_node_ptr &node) -> bool {
const auto mvn = std::dynamic_pointer_cast<const ngraph::op::v6::MVN>(node);
inference-engine/src/mkldnn_plugin/cpu_memory_desc_utils.cpp
-395
View File
@@ -1,395 +0,0 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "cpu_memory_desc.h"
#include "cpu_memory_desc_utils.h"
#include "mkldnn_memory.h"
#include "utils/general_utils.h"
#include "utils/cpu_utils.hpp"
#include <limits>
#include <vector>
#include <numeric>
#include <blob_factory.hpp>
using namespace mkldnn;
using namespace MKLDNNPlugin;
using namespace InferenceEngine;
namespace MKLDNNPlugin {
/**
* Convert to BlockedDescriptor
*
* mkl: IOhw_4i16o4i dims {32, 64, 128, 128}
* strides // the order of outer dims is encoded here
* inner_blks 4 16 4
* inner_idxs 1 0 1
*
* IE tensor desc has more expressive ability. Any oneDNN blocked tensor can be covreted.
* How to convert into IE representation:
* 0. Detect a new_outer_order of outer_dims via descending strides.
* 1. IE strides : concatenate strides in new_outer_order and inner strides.
* 2. IE dims : concatenate outer dims in new_outer_order with auto padding and inner blocks
* 3. IE order : concatenate new_outer_order and inner_idxs
*/
BlockedMemoryDesc MemoryDescUtils::convertToBlockedDescriptor(const MKLDNNMemoryDesc& inpDesc) {
mkldnn::memory::desc desc = inpDesc;
const auto dims = desc.dims();
if (desc.data.format_kind != dnnl_blocked)
IE_THROW() << "Conversion is not possible";
const auto &blk_desc = desc.data.format_desc.blocking;
const size_t outer_ndims = dims.size();
const size_t inner_ndims = blk_desc.inner_nblks;
const size_t total_ndims = outer_ndims + inner_ndims;
// strides of inner dims. In case of 4i16o4i will be {64, 4, 1}
std::vector<size_t> inner_strides(inner_ndims, 1);
for (size_t i = 1; i < blk_desc.inner_nblks; i++) {
inner_strides[blk_desc.inner_nblks - 1 - i] = inner_strides[blk_desc.inner_nblks - i] * blk_desc.inner_blks[blk_desc.inner_nblks - i];
}
// total inner block size. in case of 4i16o4i will be {16, 16, 1, 1}
std::vector<size_t> total_block_per_dim(outer_ndims, 1);
for (int i = 0; i < inner_ndims; i++) {
total_block_per_dim[blk_desc.inner_idxs[i]] *= blk_desc.inner_blks[i];
}
std::vector<size_t> outer_block_dims(std::begin(dims), std::begin(dims) + outer_ndims);
for (size_t i = 0; i < outer_block_dims.size(); i++) {
outer_block_dims[i] = div_up(outer_block_dims[i], total_block_per_dim[i]);
}
// order of outer dims. In case of IOhw_ will be {1, 0, 2, 3}
std::vector<size_t> outer_order(outer_ndims);
std::iota(outer_order.begin(), outer_order.end(), 0);
std::sort(outer_order.begin(), outer_order.end(),
[&blk_desc, &outer_block_dims] (size_t ind_l, size_t ind_r) {
return (blk_desc.strides[ind_l] > blk_desc.strides[ind_r]) ||
(blk_desc.strides[ind_l] == blk_desc.strides[ind_r] && outer_block_dims[ind_l] > outer_block_dims[ind_r]);
});
// IE blocked order
// [new_outer_order] U [inner_idxs]
SizeVector ie_blk_order(total_ndims, 0);
std::copy(outer_order.begin(), outer_order.end(), ie_blk_order.begin());
std::copy(blk_desc.inner_idxs, blk_desc.inner_idxs + blk_desc.inner_nblks, ie_blk_order.begin() + dims.size());
// IE blocked strides
// [outer_strides via new_outer_order] U [inner_strides]
SizeVector ie_blk_strides(total_ndims, 0);
std::copy(inner_strides.rbegin(), inner_strides.rend(), ie_blk_strides.rbegin());
std::transform(outer_order.begin(), outer_order.end(), ie_blk_strides.begin(),
[&] (size_t i) { return blk_desc.strides[i]; });
// IE blocked dims
// [dims via new_outer_order with auto pad] U [inner_blk_dims]
SizeVector ie_blk_dims(total_ndims, 0);
std::copy(blk_desc.inner_blks, blk_desc.inner_blks + blk_desc.inner_nblks,
ie_blk_dims.end() - blk_desc.inner_nblks);
std::transform(outer_order.begin(), outer_order.end(), ie_blk_dims.begin(),
[&] (size_t i) { return outer_block_dims[i]; });
// IE offset padded to data. Same as for oneDNN
SizeVector ie_blk_offset_to_data {desc.data.padded_offsets, desc.data.padded_offsets + desc.data.ndims};
size_t ie_blk_offset0 = desc.data.offset0;
// TODO: The tensor desc implementation allow to specify offset_to_data for inner blocked dims.
// Which is not obvious behavior. It required offset_to_data.size == total_ndims, so will
// fill it with zero.
ie_blk_offset_to_data.insert(ie_blk_offset_to_data.end(), inner_ndims, 0);
BlockedMemoryDesc res(MKLDNNMemory::convertToIePrec(desc.data_type()), SizeVector {begin(dims), end(dims)}, ie_blk_dims,
ie_blk_order, ie_blk_offset0, ie_blk_offset_to_data, ie_blk_strides);
return res;
}
InferenceEngine::TensorDesc MemoryDescUtils::convertToTensorDesc(const MemoryDesc& desc) {
if (auto blockingDesc = dynamic_cast<const BlockedMemoryDesc*>(&desc)) {
return InferenceEngine::TensorDesc(blockingDesc->getPrecision(), blockingDesc->getShape().getStaticDims(),
{blockingDesc->getBlockDims(), blockingDesc->getOrder(), blockingDesc->getOffsetPadding(),
blockingDesc->getOffsetPaddingToData(), blockingDesc->getStrides()});
} else if (auto mkldnnDesc = dynamic_cast<const MKLDNNMemoryDesc*>(&desc)) {
auto blockingDesc = convertToBlockedDescriptor(*mkldnnDesc);
return InferenceEngine::TensorDesc(blockingDesc.getPrecision(), blockingDesc.getShape().getStaticDims(),
{blockingDesc.getBlockDims(), blockingDesc.getOrder(), blockingDesc.getOffsetPadding(),
blockingDesc.getOffsetPaddingToData(), blockingDesc.getStrides()});
}
IE_THROW() << "Cannot convert MemoryDesc to InferenceEngine::TensorDesc";
return InferenceEngine::TensorDesc();
}
MKLDNNMemoryDesc MemoryDescUtils::convertToMKLDNNMemoryDesc(const MemoryDesc& desc) {
if (MemoryDescType::Blocked == desc.getType()) {
return convertToMKLDNNMemoryDesc(*(desc.as<BlockedMemoryDesc>()));
} else if (MemoryDescType::Mkldnn == desc.getType()) {
return *(desc.as<MKLDNNMemoryDesc>());
} else {
IE_THROW() << "Cannot convert MemoryDesc to MKLDNNMemoryDesc";
}
}
MKLDNNMemoryDesc MemoryDescUtils::convertToMKLDNNMemoryDesc(const BlockedMemoryDesc& desc) {
dnnl_memory_desc_t mkldnnDesc;
// scalar case
if (desc.getShape().getRank() == 0) {
mkldnn::memory::desc convertedDesc;
convertedDesc.data.format_kind = dnnl_blocked;
convertedDesc.data.data_type = memory::convert_to_c(MKLDNNMemory::convertToDataType(desc.getPrecision()));
convertedDesc.data.ndims = 1;
convertedDesc.data.dims[0] = 1;
convertedDesc.data.padded_dims[0] = 1;
convertedDesc.data.format_desc.blocking.strides[0] = 1;
convertedDesc.data.padded_offsets[0] = 0;
convertedDesc.data.offset0 = desc.getOffsetPadding();
return MKLDNNMemoryDesc(convertedDesc);
}
auto dims = desc.getShape().getStaticDims();
auto ie_blkdDims = desc.getBlockDims();
auto ie_order = desc.getOrder();
auto ie_offsetsToData = desc.getOffsetPaddingToData();
auto ie_strides = desc.getStrides();
size_t outer_ndims = dims.size();
size_t inner_ndims = ie_order.size() - dims.size();
bool is_descending_strides = true;
for (int i = 1; i < ie_strides.size(); i++) {
is_descending_strides &= (ie_strides[i-1] >= ie_strides[i]);
}
// TODO: That's strong constrains and can be mitigated. IE::TensorDesc allow to transpose blocked dims
// and may be we can achieve correct "descending strides" form which allow conversion.
if (!is_descending_strides)
IE_THROW() << "Unsupported case for conversion";
std::vector<size_t> outer_order(outer_ndims, outer_ndims + 1); // outer_order[i] is index of stride for i-th dimension
for (size_t i = 0; i < outer_ndims; i++) {
outer_order[ie_order[i]] = i;
}
bool outer_is_correct_permutation_of_n =
std::find(outer_order.begin(), outer_order.end(), outer_ndims + 1) == outer_order.end();
if (!outer_is_correct_permutation_of_n)
IE_THROW() << "Unsupported case for conversion";
bool inner_block_are_dense = one_of(ie_strides.back(), 0, 1); // stride 1 - is dense case, 0 - broad casted
for (int i = outer_ndims; i < ie_strides.size() - 1; i++) {
inner_block_are_dense &= (ie_strides[i] == ie_strides[i+1] * ie_blkdDims[i+1]);
}
if (!inner_block_are_dense)
IE_THROW() << "Unsupported case for conversion";
bool inner_pad_offsets_is_zero = std::all_of(ie_offsetsToData.begin() + outer_ndims, ie_offsetsToData.end(),
[](size_t pad) { return pad == 0; });
if (!inner_pad_offsets_is_zero)
IE_THROW() << "Unsupported case for conversion";
// Fill general memory desc fields
mkldnnDesc.format_kind = dnnl_blocked;
mkldnnDesc.extra.flags = 0;
mkldnnDesc.data_type = memory::convert_to_c(MKLDNNMemory::convertToDataType(desc.getPrecision()));
mkldnnDesc.ndims = dims.size();
mkldnnDesc.offset0 = desc.getOffsetPadding();
std::copy(dims.begin(), dims.end(), mkldnnDesc.dims);
std::copy(ie_offsetsToData.begin(), ie_offsetsToData.begin() + outer_ndims, mkldnnDesc.padded_offsets);
std::fill(mkldnnDesc.padded_dims, mkldnnDesc.padded_dims + outer_ndims, 1);
for (size_t i = 0; i < ie_order.size(); i++) {
auto idx = ie_order[i];
mkldnnDesc.padded_dims[idx] *= ie_blkdDims[i];
}
// Fill blocking desc
auto &dnn_blk_desc = mkldnnDesc.format_desc.blocking;
dnn_blk_desc.inner_nblks = inner_ndims;
std::copy(ie_blkdDims.end() - inner_ndims, ie_blkdDims.end(), dnn_blk_desc.inner_blks);
std::copy(ie_order.end() - inner_ndims, ie_order.end(), dnn_blk_desc.inner_idxs);
for (size_t i = 0; i < outer_ndims; i++) {
dnn_blk_desc.strides[i] = ie_strides[outer_order[i]];
}
return MKLDNNMemoryDesc(mkldnnDesc);
}
/**
* Construct from IE::TensorDesc
* @param tDesc
*
* IE IOhw_4i16o4i dims(N) = {32, 64, 128, 128}
* blockedDims {4, 2, 128, 128, 4, 16, 4} // total dims(inner, outermost, auto blocked/padded). Generally sorted by strides.
* strides {8388608, 4194304, 32768, 256, 64, 4, 1} // strides for blockedDims, growing sequence
* order {1, 0, 2, 3, 1, 0, 1} // matching to original dims
*
* All vectors blockedDims/strides/order have same size equals total num of internal blocked dims(inner_dims + outer_dims)
*
* Tensor descriptor filing is not deterministic. It allows any permutation of index which keeps order of
* real dims spliting.
* for {1, 0, 2, 3, 1, 0, 1} we can swap elements [1] <=> [4]
* but not [0]<=>[4] because it breacke spliting original dims into internal blocked dims
* Normalization of representation: Make strides growing but keep layout same as original. Not all
* layout allow us to meet normalize form of tensor desc.
*
* Limitation of conversion first N elements of order should be permutation of [0,1,2 ... N]
*/
MKLDNNMemoryDesc MemoryDescUtils::convertToMKLDNNMemoryDesc(const InferenceEngine::TensorDesc& tDesc) {
mkldnn::memory::desc mkldnnDesc({}, mkldnn::memory::data_type::undef, mkldnn::memory::format_tag::undef);
auto dims = tDesc.getDims();
// TODO: implicit conversion of dims is no good...
if (tDesc.getLayout() == Layout::SCALAR) {
mkldnnDesc.data.format_kind = dnnl_blocked;
mkldnnDesc.data.data_type = memory::convert_to_c(MKLDNNMemory::convertToDataType(tDesc.getPrecision()));
mkldnnDesc.data.ndims = 1;
mkldnnDesc.data.dims[0] = 1;
mkldnnDesc.data.padded_dims[0] = 1;
mkldnnDesc.data.format_desc.blocking.strides[0] = 1;
mkldnnDesc.data.padded_offsets[0] = 0;
mkldnnDesc.data.offset0 = tDesc.getBlockingDesc().getOffsetPadding();
return MKLDNNMemoryDesc(mkldnnDesc);
}
if (tDesc.getLayout() == Layout::ANY) {
mkldnnDesc.data.format_kind = dnnl_format_kind_any;
mkldnnDesc.data.data_type = memory::convert_to_c(MKLDNNMemory::convertToDataType(tDesc.getPrecision()));
mkldnnDesc.data.ndims = dims.size();
std::copy(dims.begin(), dims.end(), mkldnnDesc.data.dims);
std::copy(dims.begin(), dims.end(), mkldnnDesc.data.padded_dims);
mkldnnDesc.data.offset0 = tDesc.getBlockingDesc().getOffsetPadding();
std::fill(mkldnnDesc.data.padded_offsets, mkldnnDesc.data.padded_offsets + dims.size(), 0);
return MKLDNNMemoryDesc(mkldnnDesc);
}
auto ie_blkdDims = tDesc.getBlockingDesc().getBlockDims();
auto ie_order = tDesc.getBlockingDesc().getOrder();
auto ie_offsetsToData = tDesc.getBlockingDesc().getOffsetPaddingToData();
auto ie_strides = tDesc.getBlockingDesc().getStrides();
size_t outer_ndims = dims.size();
size_t inner_ndims = ie_order.size() - dims.size();
bool is_descending_strides = true;
for (int i = 1; i < ie_strides.size(); i++) {
is_descending_strides &= (ie_strides[i-1] >= ie_strides[i]);
}
// TODO: That's strong constrains and can be mitigated. IE::TensorDesc allow to transpose blocked dims
// and may be we can achieve correct "descending strides" form which allow conversion.
if (!is_descending_strides)
IE_THROW() << "Unsupported case for conversion";
std::vector<size_t> outer_order(outer_ndims, outer_ndims + 1); // outer_order[i] is index of stride for i-th dimension
for (size_t i = 0; i < outer_ndims; i++) {
outer_order[ie_order[i]] = i;
}
bool outer_is_correct_permutation_of_n =
std::find(outer_order.begin(), outer_order.end(), outer_ndims + 1) == outer_order.end();
if (!outer_is_correct_permutation_of_n)
IE_THROW() << "Unsupported case for conversion";
bool inner_block_are_dense = one_of(ie_strides.back(), 0, 1); // stride 1 - is dense case, 0 - broad casted
for (int i = outer_ndims; i < ie_strides.size() - 1; i++) {
inner_block_are_dense &= (ie_strides[i] == ie_strides[i+1] * ie_blkdDims[i+1]);
}
if (!inner_block_are_dense)
IE_THROW() << "Unsupported case for conversion";
bool inner_pad_offsets_is_zero = std::all_of(ie_offsetsToData.begin() + outer_ndims, ie_offsetsToData.end(),
[](size_t pad) { return pad == 0; });
if (!inner_pad_offsets_is_zero)
IE_THROW() << "Unsupported case for conversion";
// Fill general memory desc fields
mkldnnDesc.data.format_kind = dnnl_blocked;
mkldnnDesc.data.data_type = memory::convert_to_c(MKLDNNMemory::convertToDataType(tDesc.getPrecision()));
mkldnnDesc.data.ndims = dims.size();
mkldnnDesc.data.offset0 = tDesc.getBlockingDesc().getOffsetPadding();
std::copy(dims.begin(), dims.end(), mkldnnDesc.data.dims);
std::copy(ie_offsetsToData.begin(), ie_offsetsToData.begin() + outer_ndims, mkldnnDesc.data.padded_offsets);
std::fill(mkldnnDesc.data.padded_dims, mkldnnDesc.data.padded_dims + outer_ndims, 1);
for (size_t i = 0; i < ie_order.size(); i++) {
auto idx = ie_order[i];
mkldnnDesc.data.padded_dims[idx] *= ie_blkdDims[i];
}
// Fill blocking desc
auto &dnn_blk_desc = mkldnnDesc.data.format_desc.blocking;
dnn_blk_desc.inner_nblks = inner_ndims;
std::copy(ie_blkdDims.end() - inner_ndims, ie_blkdDims.end(), dnn_blk_desc.inner_blks);
std::copy(ie_order.end() - inner_ndims, ie_order.end(), dnn_blk_desc.inner_idxs);
for (size_t i = 0; i < outer_ndims; i++) {
dnn_blk_desc.strides[i] = ie_strides[outer_order[i]];
}
return MKLDNNMemoryDesc(mkldnnDesc);
}
BlockedMemoryDesc MemoryDescUtils::convertToBlockedDescriptor(const MemoryDesc &desc) {
if (desc.getType() == MemoryDescType::Blocked) {
return *(desc.as<BlockedMemoryDesc>());
} else if (desc.getType() == MemoryDescType::Mkldnn) {
return MemoryDescUtils::convertToBlockedDescriptor(*(desc.as<MKLDNNMemoryDesc>()));
} else {
IE_THROW() << "Cannot convert to blocked memory descriptor. Unsupported memory desc type";
}
}
MemoryDescPtr MemoryDescUtils::applyUndefinedOffset(const MKLDNNMemoryDesc& desc) {
if (desc.getFormatKind() != dnnl_format_kind_t::dnnl_blocked)
IE_THROW() << "applyUndefinedOffset doesn't support not dnnl_blocked MKLDNNMemoryDesc";
mkldnn::memory::desc retDesc = desc;
retDesc.data.offset0 = Shape::UNDEFINED_DIM;
return MKLDNNPlugin::make_unique<MKLDNNMemoryDesc>(retDesc);
}
MemoryDescPtr MemoryDescUtils::applyUndefinedOffset(const BlockedMemoryDesc &desc) {
std::vector<size_t> strides;
std::vector<size_t> offsetPaddingToData;
strides.resize(desc.getBlockDims().size(), Shape::UNDEFINED_DIM);
offsetPaddingToData.resize(desc.getBlockDims().size(), 0);
size_t offsetPadding = Shape::UNDEFINED_DIM;
return MKLDNNPlugin::make_unique<BlockedMemoryDesc>(desc.getPrecision(), desc.getShape().getDims(), desc.getBlockDims(),
desc.getOrder(), offsetPadding, offsetPaddingToData, strides);
}
MemoryDescPtr MemoryDescUtils::resetOffset(const MemoryDesc* desc) {
if (MemoryDescType::Blocked == desc->getType()) {
auto blockedDesc = desc->as<BlockedMemoryDesc>();
return MKLDNNPlugin::make_unique<BlockedMemoryDesc>(blockedDesc->getPrecision(), blockedDesc->getShape().getDims(),
blockedDesc->getBlockDims(), blockedDesc->getOrder());
} else if (MemoryDescType::Mkldnn == desc->getType()) {
auto mkldnnDesc = desc->as<MKLDNNMemoryDesc>();
mkldnn::memory::desc retDesc = *mkldnnDesc;
retDesc.data.offset0 = 0;
return MKLDNNPlugin::make_unique<MKLDNNMemoryDesc>(retDesc);
} else {
IE_THROW() << "resetOffset support Blocked and Mkldnn descpriptors only";
}
}
InferenceEngine::Blob::Ptr MemoryDescUtils::interpretAsBlob(const MKLDNNMemory &mem) {
// TODO [DS]: Rewrite when IE is moved to the new TensorDescriptor
auto& memDesc = mem.GetDesc();
InferenceEngine::TensorDesc desc = convertToTensorDesc(memDesc);
desc = InferenceEngine::TensorDesc(desc.getPrecision(), memDesc.getShape().getStaticDims(), desc.getBlockingDesc());
return MKLDNNPlugin::isEmptyTensorDesc(desc) ? make_blob_with_precision(desc) : make_blob_with_precision(desc, mem.GetData());
}
} // namespace MKLDNNPlugin
inference-engine/src/mkldnn_plugin/cpu_memory_desc_utils.h
-88
View File
@@ -1,88 +0,0 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include <ie_layouts.h>
#include <ie_blob.h>
namespace MKLDNNPlugin {
class MKLDNNMemoryDesc;
class BlockedMemoryDesc;
class MKLDNNMemory;
class MemoryDescUtils {
public:
/**
* @brief Converts MemoryDesc to InferenceEngine::TensorDesc
* @param desc MemoryDesc to be converted
* @return converted InferenceEngine::TensorDesc
*/
static InferenceEngine::TensorDesc convertToTensorDesc(const MemoryDesc& desc);
/**
* @brief Converts MemoryDesc to MKLDNNMemoryDesc
* @param desc MemoryDesc to be converted
* @return converted MKLDNNMemoryDesc
*/
static MKLDNNMemoryDesc convertToMKLDNNMemoryDesc(const MemoryDesc& desc);
/**
* @brief Converts BlockedMemoryDesc to MKLDNNMemoryDesc
* @param desc BlockedMemoryDesc to be converted
* @return converted MKLDNNMemoryDesc
*/
static MKLDNNMemoryDesc convertToMKLDNNMemoryDesc(const BlockedMemoryDesc& desc);
/**
* @brief Converts InferenceEngine::TensorDesc to MKLDNNMemoryDesc
* @param desc InferenceEngine::TensorDesc to be converted
* @return converted MKLDNNMemoryDesc
*/
static MKLDNNMemoryDesc convertToMKLDNNMemoryDesc(const InferenceEngine::TensorDesc& desc);
/**
* @brief Converts MemoryDesc to BlockedMemoryDesc
* @param desc MemoryDesc to be converted
* @return converted BlockedMemoryDesc
*/
static BlockedMemoryDesc convertToBlockedDescriptor(const MemoryDesc& desc);
/**
* @brief Converts MKLDNNMemoryDesc to BlockedMemoryDesc
* @param desc MKLDNNMemoryDesc to be converted
* @return converted BlockedMemoryDesc
*/
static BlockedMemoryDesc convertToBlockedDescriptor(const MKLDNNMemoryDesc& inpDesc);
/**
* @brief Creates MKLDNNMemoryDesc with offset0 of UNDEFINED_DIM size
* @param desc modifiable MKLDNNMemoryDesc
* @return pointer to MKLDNNMemoryDesc
*/
static MemoryDescPtr applyUndefinedOffset(const MKLDNNMemoryDesc& desc);
/**
* @brief Creates BlockedMemoryDesc with offsetPadding, strides of UNDEFINED_DIM size and offsetPaddingToData of 0 size
* @param desc modifiable BlockedMemoryDesc
* @return pointer to BlockedMemoryDesc
*/
static MemoryDescPtr applyUndefinedOffset(const BlockedMemoryDesc& desc);
/**
* @brief Creates MemoryDesc with offsetPadding of 0 size
* @param desc modifiable MemoryDesc
* @return pointer to MemoryDesc
*/
static MemoryDescPtr resetOffset(const MemoryDesc* desc);
/**
* @brief Creates InferenceEngine::Blob from MKLDNNMemory
* @param desc MKLDNNMemory from which will be created InferenceEngine::Blob
* @return pointer to InferenceEngine::Blob
*/
static InferenceEngine::Blob::Ptr interpretAsBlob(const MKLDNNMemory& mem);
};
} // namespace MKLDNNPlugin
inference-engine/src/mkldnn_plugin/cpu_shape.cpp
+49
View File
@@ -0,0 +1,49 @@
// Copyright (C) 2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "cpu_shape.h"
#include "utils/general_utils.h"
#include "memory_desc/cpu_memory_desc_utils.h"
using namespace MKLDNNPlugin;
bool Shape::isCompatible(const VectorDims &vecDims) const {
if (getRank() != vecDims.size()) {
return false;
}
auto comparator = [](Dim lhs, Dim rhs) {
return (lhs == rhs) || (lhs == Shape::UNDEFINED_DIM);
};
if (!std::equal(getDims().begin(), getDims().end(), vecDims.begin(), comparator)) {
return false;
}
if (!std::equal(getMaxDims().begin(), getMaxDims().end(), vecDims.begin(), [](Dim lhs, Dim rhs) { return lhs >= rhs; })) {
return false;
}
if (!std::equal(getMinDims().begin(), getMinDims().end(), vecDims.begin(), [](Dim lhs, Dim rhs) { return lhs <= rhs; })) {
return false;
}
return true;
}
std::string Shape::toString() const {
std::stringstream output;
output << "{";
size_t i = 0;
do {
if (dims[i] == Shape::UNDEFINED_DIM) {
output << MemoryDescUtils::dim2str(minDims[i]) << " - " << MemoryDescUtils::dim2str(maxDims[i]);
} else {
output << dims[i];
}
} while (++i < dims.size() && output << ", ");
output << "}";
return output.str();
}
inference-engine/src/mkldnn_plugin/cpu_shape.h
+33 -16
View File
@@ -9,7 +9,7 @@
#include <utility>
#include <ie_common.h>
#include <ngraph/partial_shape.hpp>
#include "mkldnn_dims.h"
#include "cpu_types.h"
namespace MKLDNNPlugin {
@@ -19,7 +19,9 @@ public:
explicit Shape(const ngraph::PartialShape& shape) {
minDims = shape.get_min_shape();
std::transform(minDims.begin(), minDims.end(), minDims.begin(), [](Dim x){ return ngraph::Interval::s_max == x ? UNDEFINED_DIM : x;});
maxDims = shape.get_max_shape();
std::transform(maxDims.begin(), maxDims.end(), maxDims.begin(), [](Dim x){ return ngraph::Interval::s_max == x ? UNDEFINED_DIM : x;});
type = shape.is_static() ? ShapeType::Static : ShapeType::Dynamic;
initDims();
@@ -34,7 +36,7 @@ public:
}
/**
* @brief
* @brief
* for static shape
* maxDims = [2, 3, 4, 5]
* minDims = [2, 3, 4, 5]
@@ -46,12 +48,12 @@ public:
* dims = [UNDEFINED_DIM, UNDEFINED_DIM, UNDEFINED_DIM, UNDEFINED_DIM]
* @return return lower bound of shape = [1, 1, 1, 1]
*/
const std::vector<size_t>& getMinDims() const {
const VectorDims& getMinDims() const {
return minDims;
}
/**
* @brief
* @brief
* for static shape
* maxDims = [2, 3, 4, 5]
* minDims = [2, 3, 4, 5]
@@ -63,15 +65,15 @@ public:
* dims = [UNDEFINED_DIM, UNDEFINED_DIM, UNDEFINED_DIM, UNDEFINED_DIM]
* @return return upper bound of shape = [6, 6, 6, 6]
*/
const std::vector<size_t>& getMaxDims() const {
const VectorDims& getMaxDims() const {
return maxDims;
}
/**
* @brief return defined shape or throw exception for dynamic case
* @brief return defined shape or throw exception for dynamic case
* @return return shape
*/
const std::vector<size_t>& getStaticDims() const {
const VectorDims& getStaticDims() const {
if (type != ShapeType::Static) {
IE_THROW() << "Cannot get dims for non static shape";
}
@@ -80,7 +82,7 @@ public:
}
/**
* @brief
* @brief
* for static shape
* maxDims = [2, 3, 4, 5]
* minDims = [2, 3, 4, 5]
@@ -92,13 +94,18 @@ public:
* dims = [2, 3, UNDEFINED_DIM, UNDEFINED_DIM]
* @return return shape with defined and undefined dims = [2, 3, UNDEFINED_DIM, UNDEFINED_DIM]
*/
const std::vector<size_t>& getDims() const {
const VectorDims& getDims() const {
return dims;
}
bool isStatic() const {
return type == ShapeType::Static;
}
bool isDynamic() const {
return type == ShapeType::Dynamic;
}
size_t getRank() const {
return minDims.size();
}
@@ -118,14 +125,21 @@ public:
}
ngraph::PartialShape toPartialShape() const {
std::vector<ngraph::Dimension> nGraphDims;
using ngraph::Dimension;
std::vector<Dimension> nGraphDims;
nGraphDims.reserve(minDims.size());
for (int i = 0; i < minDims.size(); i++) {
nGraphDims.emplace_back(minDims[i], maxDims[i]);
Dimension::value_type minDim = Shape::UNDEFINED_DIM == minDims[i] ? -1 : minDims[i];
Dimension::value_type maxDim = Shape::UNDEFINED_DIM == maxDims[i] ? -1 : maxDims[i];
nGraphDims.emplace_back(minDim, maxDim);
}
return ngraph::PartialShape(nGraphDims);
}
bool isCompatible(const VectorDims& vecDims) const;
std::string toString() const;
bool operator == (const Shape& rhs) const {
return minDims == rhs.minDims && maxDims == rhs.maxDims;
}
@@ -134,7 +148,11 @@ public:
return !(*this == rhs);
}
enum : size_t {
bool hasDefinedUpperBounds() const {
return std::all_of(maxDims.begin(), maxDims.end(), [](Dim dim){ return dim != UNDEFINED_DIM; });
}
enum : Dim {
UNDEFINED_DIM = 0xffffffffffffffff
};
@@ -151,9 +169,8 @@ private:
Dynamic
} type {ShapeType::Static};
std::vector<size_t> minDims;
std::vector<size_t> maxDims;
std::vector<size_t> dims;
VectorDims minDims;
VectorDims maxDims;
VectorDims dims;
};
} // namespace MKLDNNPlugin
inference-engine/src/mkldnn_plugin/cpu_types.h
+5
View File
@@ -4,8 +4,13 @@
#pragma once
#include <vector>
namespace MKLDNNPlugin {
using Dim = std::size_t;
using VectorDims = std::vector<Dim>;
enum Type {
Unknown,
Generic,
inference-engine/src/mkldnn_plugin/memory_desc/blocked_memory_desc.cpp
+34
View File
@@ -0,0 +1,34 @@
// Copyright (C) 2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "blocked_memory_desc.h"
#include "utils/general_utils.h"
using namespace MKLDNNPlugin;
bool BlockedMemoryDesc::isCompatible(const BlockedMemoryDesc &rhs) const {
if (this->getShape() != rhs.getShape() || this->getPrecision() != rhs.getPrecision())
return false;
if (!dimsEqualWeak(this->getBlockDims(), rhs.getBlockDims())) {
return false;
}
if (!dimsEqualWeak(this->getOffsetPaddingToData(), rhs.getOffsetPaddingToData())) {
return false;
}
// this check needed to avoid inserting unnecessary reorders if the memory is used in place and the batch size is equal to 1
size_t skipAxis = this->getShape().getRank() > 0 && this->getShape().getDims().front() == 1 ? 0 :
Shape::UNDEFINED_DIM; //ignore batch axis if batch size == 1
if (!dimsEqualWeak(this->getStrides(), rhs.getStrides(), skipAxis)) {
return false;
}
if (!dimsEqualWeak(this->getOrder(), rhs.getOrder())) {
return false;
}
return dimsEqualWeak(this->getOffsetPadding(), rhs.getOffsetPadding());
}
inference-engine/src/mkldnn_plugin/memory_desc/blocked_memory_desc.h
+83
View File
@@ -0,0 +1,83 @@
// Copyright (C) 2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include "cpu_memory_desc.h"
namespace MKLDNNPlugin {
class BlockedMemoryDesc : public virtual MemoryDesc {
public:
BlockedMemoryDesc() {}
/**
* @brief Returns the blocked dimensions
*
* @return blocked dimensions
*/
virtual const VectorDims& getBlockDims() const = 0;
/**
* @brief Returns the vector of order
*
* @return order
*/
virtual const VectorDims& getOrder() const = 0;
/**
* @brief Returns the per-dimension offset vector
*
* @return offsets
*/
virtual const VectorDims& getOffsetPaddingToData() const = 0;
/**
* @brief Returns the offset to the current memory block
*
* @return offset
*/
virtual size_t getOffsetPadding() const = 0;
/**
* @brief Returns strides for each dimension
*
* @return strides
*/
virtual const VectorDims& getStrides() const = 0;
/**
* @brief Check that desc has padded dims
*
* @return true if exist padded dims, otherwise false
*/
virtual bool blocksExtended() const = 0;
/**
* @brief Compute number of elements taking into account padded dims
*
* @return number of elements taking into account padded dims
*/
virtual size_t getPaddedElementsCount() const = 0;
protected:
/**
* @brief Check descs on compatibility
* WARNING: Check only BlockedMemoryDesc specific attributes like: strides, order etc.
* Doesn't perform type check for descs
* Doesn't perform descs specific attributes check
* @return true if compatible, otherwise false
*/
bool isCompatible(const BlockedMemoryDesc &rhs) const;
mutable VectorDims blockedDims;
mutable VectorDims strides;
mutable VectorDims order;
mutable VectorDims offsetPaddingToData;
};
using BlockedMemoryDescPtr = std::shared_ptr<BlockedMemoryDesc>;
using BlockedMemoryDescCPtr = std::shared_ptr<const BlockedMemoryDesc>;
} // namespace MKLDNNPlugin
inference-engine/src/mkldnn_plugin/cpu_blocked_memory_desc.cpp → inference-engine/src/mkldnn_plugin/memory_desc/cpu_blocked_memory_desc.cpp
+101 -47
View File
@@ -4,11 +4,12 @@
#include "cpu_blocked_memory_desc.h"
#include "mkldnn_memory.h"
#include "utils/cpu_utils.hpp"
#include "dnnl_blocked_memory_desc.h"
using namespace MKLDNNPlugin;
BlockedMemoryDesc::BlockedMemoryDesc(InferenceEngine::Precision prc, const std::vector<size_t>& dims) : MemoryDesc(dims, Blocked) , precision(prc) {
CpuBlockedMemoryDesc::CpuBlockedMemoryDesc(InferenceEngine::Precision prc, const Shape& shape) : MemoryDesc(shape, Blocked), precision(prc) {
auto& dims = shape.getDims();
order.resize(dims.size());
std::iota(order.begin(), order.end(), 0);
blockedDims = dims;
@@ -21,15 +22,15 @@ BlockedMemoryDesc::BlockedMemoryDesc(InferenceEngine::Precision prc, const std::
}
}
BlockedMemoryDesc::BlockedMemoryDesc(InferenceEngine::Precision prc, const std::vector<size_t>& dims, const std::vector<size_t>& blockedDims,
const std::vector<size_t>& order, size_t offsetPadding, const std::vector<size_t>& offsetPaddingToData,
const std::vector<size_t>& strides) : MemoryDesc(dims, Blocked), precision(prc) {
CpuBlockedMemoryDesc::CpuBlockedMemoryDesc(InferenceEngine::Precision prc, const Shape& shape, const VectorDims& blockedDims,
const VectorDims& order, size_t offsetPadding, const VectorDims& offsetPaddingToData,
const VectorDims& strides) : MemoryDesc(shape, Blocked), precision(prc) {
if (std::any_of(order.begin(), order.end(), [](size_t val) { return val == Shape::UNDEFINED_DIM; })) {
IE_THROW() << "BlockedMemoryDesc do not support undefined order.";
IE_THROW() << "CpuBlockedMemoryDesc do not support undefined order.";
}
if (std::any_of(blockedDims.begin() + dims.size(), blockedDims.end(), [](size_t val) { return val == Shape::UNDEFINED_DIM; })) {
IE_THROW() << "BlockedMemoryDesc doesn't support undefined blockedDims.";
if (std::any_of(blockedDims.begin() + shape.getRank(), blockedDims.end(), [](size_t val) { return val == Shape::UNDEFINED_DIM; })) {
IE_THROW() << "CpuBlockedMemoryDesc doesn't support undefined blockedDims.";
}
this->order = order;
@@ -65,7 +66,7 @@ BlockedMemoryDesc::BlockedMemoryDesc(InferenceEngine::Precision prc, const std::
}
}
bool BlockedMemoryDesc::isDefined() const {
bool CpuBlockedMemoryDesc::isDefinedImp() const {
bool defined = true;
defined = defined && std::none_of(blockedDims.cbegin(), blockedDims.cend(), [](size_t val) { return val == Shape::UNDEFINED_DIM; });
defined = defined && std::none_of(strides.cbegin(), strides.cend(), [](size_t val) { return val == Shape::UNDEFINED_DIM; });
@@ -76,48 +77,26 @@ bool BlockedMemoryDesc::isDefined() const {
return defined;
}
bool BlockedMemoryDesc::isCompatible(const MemoryDesc& rhs) const {
bool CpuBlockedMemoryDesc::isCompatible(const MemoryDesc& rhs) const {
const MemoryDesc* pRhs = &rhs;
if (auto blockingDesc = dynamic_cast<const BlockedMemoryDesc*>(pRhs)) {
return isCompatible(*blockingDesc);
} else if (auto mkldnnDesc = dynamic_cast<const MKLDNNMemoryDesc*>(pRhs)) {
return mkldnnDesc->isCompatible(*this);
if (auto cpuBlkDesc = dynamic_cast<const CpuBlockedMemoryDesc*>(pRhs)) {
return isCompatible(*cpuBlkDesc);
} else if (auto dnnlBlkDesc = dynamic_cast<const DnnlBlockedMemoryDesc*>(pRhs)) {
return isCompatible(*dnnlBlkDesc);
} else {
return false;
}
}
bool BlockedMemoryDesc::isCompatible(const BlockedMemoryDesc& rhs) const {
if (this->getShape() != rhs.getShape() || this->getPrecision() != rhs.getPrecision())
return false;
if (!dimsEqualWeak(this->getBlockDims(), rhs.getBlockDims())) {
return false;
}
if (!dimsEqualWeak(this->getOffsetPaddingToData(), rhs.getOffsetPaddingToData())) {
return false;
}
// this check needed to avoid inserting unnecessary reorders if the memory is used in place and the batch size is equal to 1
size_t skipAxis = this->getShape().getRank() > 0 && this->getShape().getDims().front() == 1 ? 0 :
Shape::UNDEFINED_DIM; //ignore batch axis if batch size == 1
if (!dimsEqualWeak(this->getStrides(), rhs.getStrides(), skipAxis)) {
return false;
}
if (!dimsEqualWeak(this->getOrder(), rhs.getOrder())) {
return false;
}
return dimsEqualWeak(this->getOffsetPadding(), rhs.getOffsetPadding());
bool CpuBlockedMemoryDesc::isCompatible(const CpuBlockedMemoryDesc &rhs) const {
return BlockedMemoryDesc::isCompatible(rhs);
}
bool BlockedMemoryDesc::isCompatible(const MKLDNNMemoryDesc& rhs) const {
bool CpuBlockedMemoryDesc::isCompatible(const DnnlBlockedMemoryDesc &rhs) const {
return rhs.isCompatible(*this);
}
size_t BlockedMemoryDesc::getMemSizeImp() const {
size_t CpuBlockedMemoryDesc::getCurrentMemSizeImp() const {
int64_t e_size = getOffsetPadding() + 1; // size in bytes (from begin of data to last element)
for (int j = 0; j < getBlockDims().size(); j++)
e_size += (getBlockDims()[j] - 1) * getStrides()[j];
@@ -128,7 +107,21 @@ size_t BlockedMemoryDesc::getMemSizeImp() const {
return e_size;
}
size_t BlockedMemoryDesc::getOffset(const InferenceEngine::SizeVector& v) const {
size_t CpuBlockedMemoryDesc::getMaxMemSize() const {
if (shape.isStatic()) {
return getCurrentMemSize();
}
auto& maxDims = shape.getMaxDims();
if (std::any_of(maxDims.begin(), maxDims.end(), [](size_t x){ return Shape::UNDEFINED_DIM == x; })) {
return UNDEFINED_SIZE;
}
auto maxDimsDesc = cloneWithNewDims(maxDims);
return maxDimsDesc->getCurrentMemSize();
}
size_t CpuBlockedMemoryDesc::getOffset(const InferenceEngine::SizeVector& v) const {
InferenceEngine::SizeVector off_v = v;
size_t n_blocked_dims = order.size();
@@ -148,7 +141,7 @@ size_t BlockedMemoryDesc::getOffset(const InferenceEngine::SizeVector& v) const
return offset;
}
size_t BlockedMemoryDesc::getElementOffset(size_t elemNumber) const {
size_t CpuBlockedMemoryDesc::getElementOffset(size_t elemNumber) const {
// TODO [DS]: rewrite to support dynamic shapes
auto& dims = shape.getStaticDims();
size_t n_dims = dims.size();
@@ -162,7 +155,7 @@ size_t BlockedMemoryDesc::getElementOffset(size_t elemNumber) const {
return getOffset(pos);
}
bool BlockedMemoryDesc::hasLayoutType(LayoutType layoutType) const {
bool CpuBlockedMemoryDesc::hasLayoutType(LayoutType layoutType) const {
switch (layoutType) {
case LayoutType::ncsp:
return isPlainFormat();
@@ -177,7 +170,7 @@ bool BlockedMemoryDesc::hasLayoutType(LayoutType layoutType) const {
}
}
bool BlockedMemoryDesc::isPlainFormat() const {
bool CpuBlockedMemoryDesc::isPlainFormat() const {
if (shape.getRank() != order.size()) {
return false;
}
@@ -189,7 +182,7 @@ bool BlockedMemoryDesc::isPlainFormat() const {
return true;
}
bool BlockedMemoryDesc::isBlockedCFormat(size_t blk_size) const {
bool CpuBlockedMemoryDesc::isBlockedCFormat(size_t blk_size) const {
if ((order.size() - shape.getRank()) != 1) {
return false;
}
@@ -207,7 +200,7 @@ bool BlockedMemoryDesc::isBlockedCFormat(size_t blk_size) const {
return true;
}
bool BlockedMemoryDesc::isTailCFormat() const {
bool CpuBlockedMemoryDesc::isTailCFormat() const {
if (shape.getRank() < 3) {
return false;
}
@@ -223,7 +216,7 @@ bool BlockedMemoryDesc::isTailCFormat() const {
return true;
}
std::string BlockedMemoryDesc::serializeFormat() const {
std::string CpuBlockedMemoryDesc::serializeFormat() const {
std::stringstream result;
char startLetter = 'a';
std::unordered_map<size_t, size_t> mapAxisBlockSize;
@@ -245,3 +238,64 @@ std::string BlockedMemoryDesc::serializeFormat() const {
return result.str();
}
MemoryDescPtr CpuBlockedMemoryDesc::cloneWithNewDimsImp(const VectorDims &dims) const {
if (std::any_of(dims.begin(), dims.end(), [](size_t x){ return Shape::UNDEFINED_DIM == x; })) {
IE_THROW() << "Can't clone desc if new dims are undefined";
}
// TODO [DS]: add stride recalculation for strided blobs
for (int i = strides.size() - 2; i >= 0 ; i--) {
if (strides[i] == Shape::UNDEFINED_DIM)
break;
if (strides[i] != strides[i + 1] * blockedDims[i + 1])
IE_THROW(NotImplemented) << "Can't clone desc with new dims for not dense tensor";
}
VectorDims newBlockedDims(order.size());
for (size_t i = 0; i < dims.size(); ++i) {
newBlockedDims[order[i]] = dims[i];
}
for (size_t i = dims.size(); i < order.size(); ++i) {
if (newBlockedDims[order[i]] != Shape::UNDEFINED_DIM) {
newBlockedDims[order[i]] = div_up(newBlockedDims[order[i]], blockedDims[i]);
newBlockedDims[i] = blockedDims[i];
}
}
VectorDims newOffsetPaddingToData;
if (std::none_of(offsetPaddingToData.begin(), offsetPaddingToData.end(), [](size_t x){ return x == Shape::UNDEFINED_DIM;})) {
newOffsetPaddingToData = offsetPaddingToData;
}
return std::make_shared<CpuBlockedMemoryDesc>(precision, Shape(dims), newBlockedDims, order, offsetPadding, newOffsetPaddingToData);
}
bool CpuBlockedMemoryDesc::blocksExtended() const {
const size_t rank = shape.getRank();
for (size_t i = rank; i < order.size(); i++) {
size_t idx = order[i];
Dim paddedDim = 1;
for (size_t j = rank; j < order.size(); j++) {
if (order[j] == idx)
paddedDim *= blockedDims[j];
}
if (blockedDims[idx] == Shape::UNDEFINED_DIM) {
paddedDim = Shape::UNDEFINED_DIM;
} else {
paddedDim *= blockedDims[idx];
}
if (paddedDim != shape.getDims()[idx])
return true;
}
return false;
}
size_t CpuBlockedMemoryDesc::getPaddedElementsCount() const {
if (std::any_of(blockedDims.begin(), blockedDims.end(), [](Dim dim) { return dim == Shape::UNDEFINED_DIM; }))
IE_THROW() << "Can't compute padded elements count for non undefined blocked dims";
return std::accumulate(blockedDims.begin(), blockedDims.end(), size_t{1}, std::multiplies<size_t>());
}
inference-engine/src/mkldnn_plugin/cpu_blocked_memory_desc.h → inference-engine/src/mkldnn_plugin/memory_desc/cpu_blocked_memory_desc.h
+33 -29
View File
@@ -4,41 +4,32 @@
#pragma once
#include "cpu_memory_desc.h"
#include "blocked_memory_desc.h"
#include "utils/general_utils.h"
namespace MKLDNNPlugin {
class MKLDNNMemoryDesc;
class BlockedMemoryDesc : public MemoryDesc {
class CpuBlockedMemoryDesc : public BlockedMemoryDesc {
public:
BlockedMemoryDesc(InferenceEngine::Precision prc, const std::vector<size_t>& dims);
CpuBlockedMemoryDesc(InferenceEngine::Precision prc, const Shape& shape);
BlockedMemoryDesc(InferenceEngine::Precision prc, const std::vector<size_t>& dims, const std::vector<size_t>& blockedDims,
const std::vector<size_t>& order, size_t offsetPadding = 0, const std::vector<size_t>& offsetPaddingToData = {},
const std::vector<size_t>& strides = {});
CpuBlockedMemoryDesc(InferenceEngine::Precision prc, const Shape& shape, const VectorDims& blockedDims,
const VectorDims& order, size_t offsetPadding = 0, const VectorDims& offsetPaddingToData = {},
const VectorDims& strides = {});
MemoryDescPtr clone() const override {
return MKLDNNPlugin::make_unique<BlockedMemoryDesc>(*this);
return std::make_shared<CpuBlockedMemoryDesc>(*this);
}
bool isDefined() const override;
bool isCompatible(const MemoryDesc& rhs) const override;
bool isCompatible(const BlockedMemoryDesc& rhs) const;
bool isCompatible(const MKLDNNMemoryDesc& rhs) const;
bool isCompatible(const CpuBlockedMemoryDesc &rhs) const;
bool isCompatible(const DnnlBlockedMemoryDesc &rhs) const;
InferenceEngine::Precision getPrecision() const override {
return precision;
}
void setPrecision(InferenceEngine::Precision prc) override {
precision = std::move(prc);
}
const std::vector<size_t>& getBlockDims() const {
const VectorDims& getBlockDims() const override {
return blockedDims;
}
@@ -47,7 +38,7 @@ public:
*
* @return order
*/
const std::vector<size_t>& getOrder() const {
const VectorDims& getOrder() const override {
return order;
}
@@ -56,7 +47,7 @@ public:
*
* @return offsets
*/
const std::vector<size_t>& getOffsetPaddingToData() const {
const VectorDims& getOffsetPaddingToData() const override {
return offsetPaddingToData;
}
/**
@@ -64,7 +55,7 @@ public:
*
* @return offset
*/
size_t getOffsetPadding() const {
size_t getOffsetPadding() const override {
return offsetPadding;
}
@@ -73,28 +64,41 @@ public:
*
* @return strides
*/
const std::vector<size_t>& getStrides() const {
const VectorDims& getStrides() const override {
return strides;
}
bool blocksExtended() const override;
bool hasLayoutType(LayoutType layoutType) const override;
std::string serializeFormat() const override;
size_t getMaxMemSize() const override;
size_t getPaddedElementsCount() const override;
private:
size_t getElementOffset(size_t elemNumber) const override;
size_t getMemSizeImp() const override;
size_t getCurrentMemSizeImp() const override;
size_t getOffset(const InferenceEngine::SizeVector& v) const;
bool isPlainFormat() const;
bool isBlockedCFormat(size_t blk_size) const;
bool isTailCFormat() const;
bool isDefinedImp() const override;
MemoryDescPtr cloneWithNewDimsImp(const VectorDims& dims) const override;
void setPrecision(InferenceEngine::Precision prc) override {
precision = std::move(prc);
}
private:
InferenceEngine::Precision precision;
std::vector<size_t> blockedDims;
std::vector<size_t> strides;
std::vector<size_t> order;
std::vector<size_t> offsetPaddingToData;
size_t offsetPadding;
mutable VectorDims paddedDims;
};
using CpuBlockedMemoryDescPtr = std::shared_ptr<CpuBlockedMemoryDesc>;
using CpuBlockedMemoryDescCPtr = std::shared_ptr<const CpuBlockedMemoryDesc>;
} // namespace MKLDNNPlugin
inference-engine/src/mkldnn_plugin/cpu_memory_desc.h → inference-engine/src/mkldnn_plugin/memory_desc/cpu_memory_desc.h
+65 -15
View File
@@ -7,13 +7,32 @@
#include <ie_common.h>
#include <ie_precision.hpp>
#include "cpu_shape.h"
#include "utils/general_utils.h"
#include "cpu_types.h"
#include "memory_desc/cpu_memory_desc_utils.h"
/**
* @brief
*
* MemoryDesc - the descriptor of tensor representation in memory. Describes all required information
* for proper allocation and handling tensor in some buffer. The real memory is not present, just description.
* This object answers on question how and where data with logical index [x1, x2, .. xN] placed in real buffer.
* In the simplest case it describe a mapping between "logical offset" and "real offset".
*
*/
namespace MKLDNNPlugin {
class MemoryDesc;
using MemoryDescPtr = std::shared_ptr<MemoryDesc>;
using MemoryDescCPtr = std::shared_ptr<const MemoryDesc>;
enum MemoryDescType {
Blocked,
Mkldnn
Undef = 0,
Blocked = 1,
Mkldnn = 1 << 1,
DnnlBlocked = Blocked | Mkldnn
};
enum class LayoutType : unsigned {
@@ -37,31 +56,51 @@ public:
virtual InferenceEngine::Precision getPrecision() const = 0;
virtual void setPrecision(InferenceEngine::Precision prc) = 0;
virtual MemoryDescPtr clone() const = 0;
virtual std::unique_ptr<MemoryDesc> clone() const = 0;
// clone descriptor with new dims. Throws an exception if some of the new dims conflicts with the internal shape (i.e. its defined dims ,rank, upper bounds)
MemoryDescPtr cloneWithNewDims(const VectorDims& dims) const {
if (!getShape().isCompatible(dims)) {
IE_THROW(ParameterMismatch) << "Can not clone with new dims. Descriptor's shape: " << getShape().toString() <<
" is incompatible with provided dimensions: " << MemoryDescUtils::dims2str(dims) << ".";
}
return cloneWithNewDimsImp(dims);
}
virtual bool isCompatible(const MemoryDesc& rhs) const = 0;
// Checks that all dimensions, offsets, strides, etc are defined (!= UNDEFINED_DIM)
virtual bool isDefined() const = 0;
bool isDefined() const {
if (descStatus::Unknown == status) {
status = isDefinedImp() ? descStatus::Defined : descStatus::Undefined;
}
return descStatus::Defined == status;
}
virtual bool hasLayoutType(LayoutType layoutType) const = 0;
virtual std::string serializeFormat() const = 0;
// Get memory upper bound if possible. Can be undefined
virtual size_t getMaxMemSize() const = 0;
/**
* @brief Get minimal required memory size in bytes.
* @return return minimal required memory size in bytes or UNDEFINED_SIZE in case undefined descriptor
*/
size_t getCurrentSize() const {
size_t getCurrentMemSize() const {
size_t retVal = UNDEFINED_SIZE;
if (isDefined()) {
retVal = getMemSizeImp();
retVal = getCurrentMemSizeImp();
}
return retVal;
}
bool hasDefinedMaxSize() const {
return getMaxMemSize() != MemoryDesc::UNDEFINED_SIZE;
}
template <typename T,
typename std::enable_if<!std::is_pointer<T>::value && !std::is_reference<T>::value, int>::type = 0,
typename std::enable_if<std::is_base_of<MemoryDesc, T>::value, int>::type = 0>
@@ -85,26 +124,37 @@ public:
static constexpr size_t UNDEFINED_SIZE = std::numeric_limits<size_t>::max();
protected:
MemoryDesc(const Shape& shape, MemoryDescType type)
: shape(shape), type(type) {}
MemoryDesc() : type(MemoryDescType::Undef) {}
MemoryDesc(Shape shape, MemoryDescType type)
: shape(std::move(shape)), type(type) {}
MemoryDesc(const std::vector<size_t>& dims, MemoryDescType type)
MemoryDesc(const VectorDims& dims, MemoryDescType type)
: shape(dims), type(type) {}
virtual size_t getMemSizeImp() const = 0;
virtual void setPrecision(InferenceEngine::Precision prc) = 0;
virtual size_t getCurrentMemSizeImp() const = 0;
// Get offset to the n'th element. Returns physical index of the element by the logical one considering padding, layout, blocking etc.
virtual size_t getElementOffset(size_t elemNumber) const = 0;
virtual bool isDefinedImp() const = 0;
virtual MemoryDescPtr cloneWithNewDimsImp(const VectorDims& dims) const = 0;
MemoryDescType type;
Shape shape;
mutable enum class descStatus : uint8_t {
Unknown,
Defined,
Undefined,
} status = descStatus::Unknown;
friend class BlobDumper;
// WA: optimizedNspc2Ncsp used getElementOffset inside implementation
friend class MKLDNNSplitNode;
friend MemoryDescPtr MemoryDescUtils::cloneWithNewPrecision(const MemoryDesc& desc, const InferenceEngine::Precision prec);
};
using MemoryDescPtr = std::unique_ptr<MemoryDesc>;
using MemoryDescConstPtr = std::unique_ptr<const MemoryDesc>;
} // namespace MKLDNNPlugin
inference-engine/src/mkldnn_plugin/memory_desc/cpu_memory_desc_utils.cpp
+153
View File
@@ -0,0 +1,153 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "cpu_memory_desc.h"
#include "memory_desc/cpu_memory_desc_utils.h"
#include "mkldnn_memory.h"
#include "memory_desc/dnnl_blocked_memory_desc.h"
#include "utils/general_utils.h"
#include "utils/cpu_utils.hpp"
#include <limits>
#include <vector>
#include <numeric>
#include <blob_factory.hpp>
#include <dnnl_types.h>
using namespace mkldnn;
using namespace MKLDNNPlugin;
using namespace InferenceEngine;
namespace MKLDNNPlugin {
DnnlMemoryDescPtr MemoryDescUtils::convertToDnnlMemoryDesc(const MemoryDescPtr &desc) {
if (MemoryDescType::Blocked == desc->getType()) {
const auto cpuDesc = desc->as<CpuBlockedMemoryDesc>();
return std::shared_ptr<DnnlBlockedMemoryDesc>(new DnnlBlockedMemoryDesc(cpuDesc->getPrecision(), cpuDesc->getShape(), cpuDesc->getBlockDims(),
cpuDesc->getOrder(), cpuDesc->getOffsetPadding(),
cpuDesc->getOffsetPaddingToData(), cpuDesc->getStrides()));
} else if (MemoryDescType::Mkldnn & desc->getType()) {
return std::dynamic_pointer_cast<DnnlMemoryDesc>(desc);
} else {
IE_THROW() << "Cannot convert MemoryDesc to DnnlMemoryDesc";
}
}
DnnlBlockedMemoryDesc MemoryDescUtils::convertToDnnlBlockedMemoryDesc(const MemoryDesc& desc) {
if (MemoryDescType::DnnlBlocked == desc.getType()) {
return DnnlBlockedMemoryDesc(*desc.as<DnnlBlockedMemoryDesc>());
} else if (MemoryDescType::Blocked == desc.getType()) {
const auto cpuDesc = desc.as<CpuBlockedMemoryDesc>();
return DnnlBlockedMemoryDesc(cpuDesc->getPrecision(), cpuDesc->getShape(), cpuDesc->getBlockDims(), cpuDesc->getOrder(), cpuDesc->getOffsetPadding(),
cpuDesc->getOffsetPaddingToData(), cpuDesc->getStrides());
} else {
IE_THROW() << "Cannot convert MemoryDesc to DnnlMemoryDesc";
}
}
CpuBlockedMemoryDesc MemoryDescUtils::convertToCpuBlockedMemoryDesc(const InferenceEngine::TensorDesc& desc) {
if (desc.getLayout() == InferenceEngine::Layout::ANY)
IE_THROW() << "Cannot convert InferenceEngine::TensorDesc with ANY layout to CpuBlockedMemoryDesc";
const auto &blkDesc = desc.getBlockingDesc();
return CpuBlockedMemoryDesc(desc.getPrecision(), Shape(desc.getDims()), blkDesc.getBlockDims(), blkDesc.getOrder(), blkDesc.getOffsetPadding(),
blkDesc.getOffsetPaddingToData(), blkDesc.getStrides());
}
DnnlBlockedMemoryDesc MemoryDescUtils::convertToDnnlBlockedMemoryDesc(const InferenceEngine::TensorDesc& desc) {
const auto &blkDesc = desc.getBlockingDesc();
if (desc.getLayout() == InferenceEngine::Layout::ANY)
IE_THROW() << "Cannot convert InferenceEngine::TensorDesc with ANY layout to DnnlBlockedMemoryDesc";
return DnnlBlockedMemoryDesc(desc.getPrecision(), Shape(desc.getDims()), blkDesc.getBlockDims(), blkDesc.getOrder(), blkDesc.getOffsetPadding(),
blkDesc.getOffsetPaddingToData(), blkDesc.getStrides());
}
BlockedMemoryDescPtr MemoryDescUtils::convertToBlockedMemoryDesc(const MemoryDescPtr &desc) {
if (desc->getType() & MemoryDescType::Blocked) {
return std::dynamic_pointer_cast<BlockedMemoryDesc>(desc);
} else {
IE_THROW() << "Can not convert unsupported memory descriptor";
}
}
MemoryDescPtr MemoryDescUtils::cloneWithUndefStridesAndOffset(const MemoryDesc& desc) {
if (desc.getType() == MemoryDescType::Mkldnn) {
IE_THROW() << "Can't apply undefined offset for mkldnn memory desc";
}
const auto blkMemDesc = desc.as<BlockedMemoryDesc>();
VectorDims strides;
VectorDims offsetPaddingToData;
strides.resize(blkMemDesc->getBlockDims().size(), Shape::UNDEFINED_DIM);
offsetPaddingToData.resize(blkMemDesc->getBlockDims().size(), 0);
size_t offsetPadding = Shape::UNDEFINED_DIM;
if (blkMemDesc->getType() == MemoryDescType::Blocked) {
return std::make_shared<CpuBlockedMemoryDesc>(blkMemDesc->getPrecision(), blkMemDesc->getShape(), blkMemDesc->getBlockDims(),
blkMemDesc->getOrder(), offsetPadding, offsetPaddingToData, strides);
} else if (blkMemDesc->getType() == MemoryDescType::DnnlBlocked) {
return DnnlBlockedMemoryDescPtr(new DnnlBlockedMemoryDesc(blkMemDesc->getPrecision(), blkMemDesc->getShape(),
blkMemDesc->getBlockDims(), blkMemDesc->getOrder(),
offsetPadding, offsetPaddingToData, strides));
} else {
IE_THROW() << "Cannot apply undefined offset. Unsupported memory desc type";
}
}
MemoryDescPtr MemoryDescUtils::cloneWithDefaultStridesAndOffset(const MemoryDesc& desc) {
const auto blkMemDesc = desc.as<BlockedMemoryDesc>();
if (MemoryDescType::Blocked == desc.getType()) {
return std::make_shared<CpuBlockedMemoryDesc>(blkMemDesc->getPrecision(), blkMemDesc->getShape(),
blkMemDesc->getBlockDims(), blkMemDesc->getOrder());
} else if (MemoryDescType::DnnlBlocked == desc.getType()) {
return DnnlBlockedMemoryDescPtr(new DnnlBlockedMemoryDesc(blkMemDesc->getPrecision(), blkMemDesc->getShape(),
blkMemDesc->getBlockDims(), blkMemDesc->getOrder()));
} else {
IE_THROW() << "cloneWithDefaultStridesAndOffset supports Blocked descriptors only";
}
}
MemoryDescPtr MemoryDescUtils::cloneWithNewPrecision(const MemoryDesc& desc, const InferenceEngine::Precision prec) {
MemoryDescPtr newDesc = desc.clone();
newDesc->setPrecision(prec);
return newDesc;
}
InferenceEngine::Blob::Ptr MemoryDescUtils::interpretAsBlob(const MKLDNNMemory &mem) {
// TODO [DS]: Rewrite when IE is moved to the new TensorDescriptor
auto& memDesc = mem.getDesc();
InferenceEngine::TensorDesc desc = convertToTensorDesc(memDesc);
desc = InferenceEngine::TensorDesc(desc.getPrecision(), memDesc.getShape().getStaticDims(), desc.getBlockingDesc());
return make_blob_with_precision(desc, mem.GetData());
}
InferenceEngine::TensorDesc MemoryDescUtils::convertToTensorDesc(const MemoryDesc& desc) {
if (auto blockingDesc = dynamic_cast<const BlockedMemoryDesc*>(&desc)) {
return InferenceEngine::TensorDesc(blockingDesc->getPrecision(), blockingDesc->getShape().getStaticDims(),
{blockingDesc->getBlockDims(), blockingDesc->getOrder(), blockingDesc->getOffsetPadding(),
blockingDesc->getOffsetPaddingToData(), blockingDesc->getStrides()});
} else {
IE_THROW() << "Cannot convert MemoryDesc to InferenceEngine::TensorDesc";
}
}
std::string MemoryDescUtils::dim2str(Dim dim) {
return dim == Shape::UNDEFINED_DIM ? "?" : std::to_string(dim);
}
std::string MemoryDescUtils::dims2str(const VectorDims& dims) {
std::stringstream output;
output << "{";
auto itr = dims.begin();
do {
output << dim2str(*itr);
} while (++itr != dims.end() && output << ", ");
output << "}";
return output.str();
}
} // namespace MKLDNNPlugin
inference-engine/src/mkldnn_plugin/memory_desc/cpu_memory_desc_utils.h
+107
View File
@@ -0,0 +1,107 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include <ie_layouts.h>
#include <ie_blob.h>
#include "mkldnn/ie_mkldnn.h"
namespace MKLDNNPlugin {
class MemoryDesc;
class DnnlMemoryDesc;
class BlockedMemoryDesc;
class DnnlBlockedMemoryDesc;
class CpuBlockedMemoryDesc;
class MKLDNNMemory;
class MemoryDescUtils {
public:
/**
* @brief Converts MemoryDesc to DnnlMemoryDesc
* @param desc MemoryDesc to be converted
* @return converted DnnlMemoryDesc
*/
static std::shared_ptr<DnnlMemoryDesc> convertToDnnlMemoryDesc(const std::shared_ptr<MemoryDesc> &desc);
/**
* @brief Converts MemoryDesc to DnnlBlockedMemoryDesc
* @param desc MemoryDesc to be converted
* @return converted DnnlBockedMemoryDesc
*/
static DnnlBlockedMemoryDesc convertToDnnlBlockedMemoryDesc(const MemoryDesc& desc);
/**
* @brief Converts InferenceEngine::TensorDesc to CpuBlockedMemoryDesc
* @param desc InferenceEngine::TensorDesc to be converted
* @return converted CpuBlockedMemoryDesc
*/
static CpuBlockedMemoryDesc convertToCpuBlockedMemoryDesc(const InferenceEngine::TensorDesc& desc);
/**
* @brief Converts InferenceEngine::TensorDesc to DnnlBlockedMemoryDesc
* @param desc InferenceEngine::TensorDesc to be converted
* @return converted DnnlBlockedMemoryDesc
*/
static DnnlBlockedMemoryDesc convertToDnnlBlockedMemoryDesc(const InferenceEngine::TensorDesc& desc);
/**
* @brief Converts MemoryDesc to BlockedMemoryDesc
* @param desc MemoryDesc to be converted
* @return converted BlockedMemoryDesc
*/
static std::shared_ptr<BlockedMemoryDesc> convertToBlockedMemoryDesc(const std::shared_ptr<MemoryDesc> &desc);
/**
* @brief Creates BlockedMemoryDesc with offsetPadding and strides of UNDEFINED_DIM size
* @param desc is the MemoryDesc to be cloned
* @return pointer to the new MemoryDesc
*/
static std::shared_ptr<MemoryDesc> cloneWithUndefStridesAndOffset(const MemoryDesc& desc);
/**
* @brief Creates MemoryDesc with offsetPadding of 0 size and default strides
* @param desc is the MemoryDesc to be cloned
* @return pointer to the new MemoryDesc
*/
static std::shared_ptr<MemoryDesc> cloneWithDefaultStridesAndOffset(const MemoryDesc& desc);
/**
* @brief Creates MemoryDesc with specified precision
* @param desc is the MemoryDesc to be cloned
* @return pointer to the new MemoryDesc
*/
static std::shared_ptr<MemoryDesc> cloneWithNewPrecision(const MemoryDesc& desc, const InferenceEngine::Precision prec);
/**
* @brief Creates InferenceEngine::Blob from MKLDNNMemory with the memory reuse
* @param desc MKLDNNMemory from which will be created InferenceEngine::Blob
* @return pointer to InferenceEngine::Blob
*/
static InferenceEngine::Blob::Ptr interpretAsBlob(const MKLDNNMemory& mem);
/**
* @brief Converts MemoryDesc to InferenceEngine::TensorDesc
* @param desc MemoryDesc to be converted
* @return converted InferenceEngine::TensorDesc
*/
static InferenceEngine::TensorDesc convertToTensorDesc(const MemoryDesc& desc);
/**
* @brief Converts dim to string, undefined dim represented as ?
* @param dim Dim to be converted
* @return dim as string
*/
static std::string dim2str(Dim dim);
/**
* @brief Converts dims to string, undefined dim represented as ?
* @param dim Dims to be converted
* @return dims as string
*/
static std::string dims2str(const VectorDims& dims);
};
} // namespace MKLDNNPlugin
inference-engine/src/mkldnn_plugin/memory_desc/dnnl_blocked_memory_desc.cpp
+796
View File
@@ -0,0 +1,796 @@
// Copyright (C) 2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "memory_desc/dnnl_blocked_memory_desc.h"
#include <dnnl_types.h>
#include <common/memory_desc_wrapper.hpp>
using namespace MKLDNNPlugin;
using namespace InferenceEngine;
DnnlBlockedMemoryDesc::DnnlBlockedMemoryDesc(InferenceEngine::Precision prc, const Shape& shape) : MemoryDesc(shape, DnnlBlocked) {
const auto ndims = shape.getRank();
const auto &dims = shape.getDims();
mkldnn::memory::dims plain_strides;
if (std::any_of(dims.begin(), dims.end(), [](size_t val) { return val == Shape::UNDEFINED_DIM; })) {
plain_strides.resize(ndims, DNNL_RUNTIME_DIM_VAL);
} else {
plain_strides.resize(ndims, 1);
for (size_t i = 1; i < ndims; i++) {
plain_strides[ndims - i -1] = plain_strides[ndims - i] * dims[ndims - i];
}
}
desc = {MKLDNNExtensionUtils::convertToDnnlDims(dims), MKLDNNExtensionUtils::IEPrecisionToDataType(prc), plain_strides};
order.resize(ndims);
std::iota(order.begin(), order.end(), 0);
initBlockedParams();
}
/**
* Construct from blocked parameters
*
* IE IOhw_4i16o4i dims(N) = {32, 64, 128, 128}
* blockedDims {4, 2, 128, 128, 4, 16, 4} // total dims(inner, outermost, auto blocked/padded). Generally sorted by strides.
* strides {8388608, 4194304, 32768, 256, 64, 4, 1} // strides for blockedDims, growing sequence
* order {1, 0, 2, 3, 1, 0, 1} // matching to original dims
*
* All vectors blockedDims/strides/order have same size equals total num of internal blocked dims(inner_dims + outer_dims)
*
* Tensor descriptor filing is not deterministic. It allows any permutation of index which keeps order of
* real dims spliting.
* for {1, 0, 2, 3, 1, 0, 1} we can swap elements [1] <=> [4]
* but not [0]<=>[4] because it break splitting original dims into internal blocked dims
* Normalization of representation: Make strides growing but keep layout same as original. Not all
* layout allow us to meet normalize form of tensor desc.
*
* Limitation of conversion first N elements of order should be permutation of [0,1,2 ... N]
*/
DnnlBlockedMemoryDesc::DnnlBlockedMemoryDesc(InferenceEngine::Precision prc, const Shape& shape, const VectorDims& blockedDims,
const VectorDims& order, size_t offsetPadding, const VectorDims& offsetPaddingToData,
const VectorDims& strides) : MemoryDesc(shape, DnnlBlocked) {
using namespace mkldnn;
// scalar case
if (shape.getRank() == 0) {
desc.data.format_kind = dnnl_blocked;
desc.data.data_type = memory::convert_to_c(MKLDNNExtensionUtils::IEPrecisionToDataType(prc));
desc.data.ndims = 1;
desc.data.dims[0] = 1;
desc.data.padded_dims[0] = 1;
desc.data.format_desc.blocking.strides[0] = 1;
desc.data.padded_offsets[0] = 0;
desc.data.offset0 = MKLDNNExtensionUtils::convertToDnnlDim(offsetPadding);
return;
}
if (order.size() != blockedDims.size()) {
IE_THROW() << "Can not construct DnnlBlockedMemoryDesc, order and blocked dims must have equals size";
}
if (!offsetPaddingToData.empty() && offsetPaddingToData.size() != order.size()) {
IE_THROW() << "Can not construct DnnlBlockedMemoryDesc, offsetPaddingToData must have equal size with order and blocked dims";
}
if (!strides.empty() && strides.size() != order.size()) {
IE_THROW() << "Can not construct DnnlBlockedMemoryDesc, strides must have equal size with order and blocked dims";
}
if (std::any_of(order.begin(), order.end(), [](size_t val) { return val == Shape::UNDEFINED_DIM; })) {
IE_THROW() << "DnnlBlockedMemoryDesc doesn't support undefined order.";
}
if (std::any_of(blockedDims.begin() + shape.getRank(), blockedDims.end(), [](size_t val) { return val == Shape::UNDEFINED_DIM; })) {
IE_THROW() << "DnnlBlockedMemoryDesc doesn't support undefined blockedDims.";
}
auto dims = MKLDNNExtensionUtils::convertToDnnlDims(shape.getDims());
size_t outer_ndims = dims.size();
size_t inner_ndims = order.size() - dims.size();
if (!strides.empty()) {
bool is_descending_strides = true;
for (int i = 1; i < strides.size(); i++) {
is_descending_strides &= (strides[i - 1] >= strides[i]);
}
// TODO: That's strong constrains and can be mitigated. IE::TensorDesc allow to transpose blocked dims
// and may be we can achieve correct "descending strides" form which allow conversion.
if (!is_descending_strides)
IE_THROW() << "Can not construct DnnlBlockedMemoryDesc from strides: " << vec2str(strides);
}
VectorDims outer_order(outer_ndims, outer_ndims + 1); // outer_order[i] is index of stride for i-th dimension
for (size_t i = 0; i < outer_ndims; i++) {
outer_order[order[i]] = i;
}
bool outer_is_correct_permutation_of_n =
std::find(outer_order.begin(), outer_order.end(), outer_ndims + 1) == outer_order.end();
if (!outer_is_correct_permutation_of_n)
IE_THROW() << "Can not construct DnnlBlockedMemoryDesc because of incorrect order: " << vec2str(order);
if (!strides.empty() && std::none_of(strides.begin(), strides.end(), [](size_t x) { return Shape::UNDEFINED_DIM == x; })) {
bool inner_block_are_dense = one_of(strides.back(), 0, 1); // stride 1 - is dense case, 0 - broad casted
for (int i = outer_ndims; i < strides.size() - 1; i++) {
inner_block_are_dense &= (strides[i] == strides[i + 1] * blockedDims[i + 1]);
}
if (!inner_block_are_dense)
IE_THROW() << "Can not construct DnnlBlockedMemoryDesc from strides: " << vec2str(strides) << " inner blocks are not dense.";
}
// Fill general memory desc fields
desc.data.format_kind = dnnl_blocked;
desc.data.extra.flags = 0;
desc.data.data_type = memory::convert_to_c(MKLDNNExtensionUtils::IEPrecisionToDataType(prc));
desc.data.ndims = dims.size();
desc.data.offset0 = MKLDNNExtensionUtils::convertToDnnlDim(offsetPadding);
std::copy(dims.begin(), dims.end(), desc.data.dims);
if (!offsetPaddingToData.empty()) {
bool inner_pad_offsets_is_zero = std::all_of(offsetPaddingToData.begin() + outer_ndims, offsetPaddingToData.end(),
[](size_t pad) { return pad == 0; });
if (!inner_pad_offsets_is_zero)
IE_THROW() << "Can not construct DnnlBlockedMemoryDesc, inner pad offsets is not zero: " << vec2str(offsetPaddingToData);
auto dnnlPaddedOffsets = MKLDNNExtensionUtils::convertToDnnlDims(offsetPaddingToData);
std::copy(dnnlPaddedOffsets.begin(), dnnlPaddedOffsets.begin() + outer_ndims, desc.data.padded_offsets);
} else {
std::fill(std::begin(desc.data.padded_offsets), std::begin(desc.data.padded_offsets) + outer_ndims, 0);
}
std::fill(desc.data.padded_dims, desc.data.padded_dims + outer_ndims, 1);
auto dnnlBlkDims = MKLDNNExtensionUtils::convertToDnnlDims(blockedDims);
for (size_t i = 0; i < order.size(); i++) {
auto idx = order[i];
if (desc.data.padded_dims[idx] != DNNL_RUNTIME_DIM_VAL && dnnlBlkDims[i] != DNNL_RUNTIME_DIM_VAL) {
desc.data.padded_dims[idx] *= dnnlBlkDims[i];
} else {
desc.data.padded_dims[idx] = DNNL_RUNTIME_DIM_VAL;
}
}
// Fill blocking desc
auto &dnn_blk_desc = desc.data.format_desc.blocking;
dnn_blk_desc.inner_nblks = inner_ndims;
std::copy(dnnlBlkDims.end() - inner_ndims, dnnlBlkDims.end(), dnn_blk_desc.inner_blks);
std::copy(order.end() - inner_ndims, order.end(), dnn_blk_desc.inner_idxs);
if (strides.empty()) {
if (std::any_of(dnnlBlkDims.begin(), dnnlBlkDims.end(), [](memory::dim val) { return val == DNNL_RUNTIME_DIM_VAL; })) {
std::fill(std::begin(dnn_blk_desc.strides), std::begin(dnn_blk_desc.strides) + outer_ndims, DNNL_RUNTIME_DIM_VAL);
} else {
//TODO [DS]: phase 2: refactor
std::vector<memory::dim> tmpStrides(order.size());
tmpStrides[order.size() - 1] = 1;
for (size_t i = 2; i <= order.size(); i++) {
tmpStrides[order.size() - i] = tmpStrides[order.size() - (i - 1)] * dnnlBlkDims[blockedDims.size() - (i - 1)];
}
for (size_t i = 0; i < outer_ndims; i++) {
dnn_blk_desc.strides[i] = tmpStrides[outer_order[i]];
}
}
} else {
for (size_t i = 0; i < outer_ndims; i++) {
auto dnnlStrides = MKLDNNExtensionUtils::convertToDnnlDims(strides);
dnn_blk_desc.strides[i] = dnnlStrides[outer_order[i]];
}
}
this->order = order;
initBlockedParams();
}
DnnlBlockedMemoryDesc::DnnlBlockedMemoryDesc(const Shape& shape, mkldnn::memory::data_type dataType, mkldnn::memory::format_tag format) :
MemoryDesc(shape, DnnlBlocked) {
using namespace mkldnn;
if (format == memory::format_tag::any || format == memory::format_tag::undef)
IE_THROW(Unexpected) << "Can't create mkldnn::desc with any or undef format";
const auto dims = shape.getDims();
if (format == memory::format_tag::x && shape.getRank() == 0) {
desc = mkldnn::memory::desc(mkldnn::memory::dims(1, 1), dataType, format);
} else {
desc = mkldnn::memory::desc(MKLDNNExtensionUtils::convertToDnnlDims(dims), dataType, format);
}
VectorDims perm;
VectorDims inner_blks;
VectorDims inner_idxs;
mkldnn::impl::memory_desc_wrapper::compute_blocking(mkldnn::memory::convert_to_c(format), perm, inner_blks, inner_idxs);
order.swap(perm);
order.insert(order.end(), inner_idxs.begin(), inner_idxs.end());
initBlockedParams();
}
bool DnnlBlockedMemoryDesc::isCompatible(const MemoryDesc& rhs) const {
if (auto desc = dynamic_cast<const DnnlBlockedMemoryDesc*>(&rhs)) {
return isCompatible(*desc);
} else if (auto desc = dynamic_cast<const CpuBlockedMemoryDesc*>(&rhs)) {
return isCompatible(*desc);
} else {
return false;
}
}
bool DnnlBlockedMemoryDesc::isCompatible(const CpuBlockedMemoryDesc& rhs) const {
return this->desc.data.extra.flags == dnnl_memory_extra_flag_none && BlockedMemoryDesc::isCompatible(rhs);
}
bool DnnlBlockedMemoryDesc::isCompatible(const DnnlBlockedMemoryDesc& rhs) const {
using namespace dnnl;
using namespace impl;
using namespace impl::utils;
if (this->getShape() != rhs.getShape() || this->getPrecision() != rhs.getPrecision()) {
return false;
}
if (this->desc == rhs.desc) {
return true;
}
memory_desc_wrapper wrappedThis(this->desc.data);
memory_desc_wrapper wrappedRhs(rhs.desc.data);
if (one_of(wrappedThis.format_kind(), format_kind::undef, format_kind::any))
return false;
int stride_start = wrappedThis.ndims() > 0 && wrappedThis.dims()[0] == 1 ? 1 : 0; // ignore batch axis stride if batch size == 1
const auto thisExtra = this->desc.data.extra;
const auto rhsExtra = rhs.desc.data.extra;
return this->getOrder() == rhs.getOrder() && (thisExtra.flags == rhsExtra.flags && thisExtra.compensation_mask == rhsExtra.compensation_mask &&
thisExtra.scale_adjust == rhsExtra.scale_adjust) && wrappedThis.similar_to(wrappedRhs, true, true, 0, stride_start, true, true);
}
DnnlBlockedMemoryDesc::DnnlBlockedMemoryDesc(const mkldnn::memory::desc& mdesc) :
MemoryDesc(MKLDNNExtensionUtils::convertToVectorDims(mdesc.dims()), DnnlBlocked) {
desc = mdesc;
if (desc.data.format_kind == dnnl::impl::format_kind::any)
IE_THROW(Unexpected) << "Memory format any is prohibited!";
mkldnn::impl::memory_desc_wrapper descWrapped(desc.data);
if (!descWrapped.is_blocking_desc())
IE_THROW(Unexpected) << "Can't create DnnlBlockedMemoryDesc from not blocking desc";
if (descWrapped.has_runtime_dims_or_strides()) {
IE_THROW(Unexpected) << "Cannot calculate order from undefined dims or strides";
}
const auto dims = desc.dims();
const auto &blk_desc = descWrapped.blocking_desc();
const size_t outer_ndims = dims.size();
const size_t inner_ndims = blk_desc.inner_nblks;
const size_t total_ndims = outer_ndims + inner_ndims;
// strides of inner dims. In case of 4i16o4i will be {64, 4, 1}
VectorDims inner_strides(inner_ndims, 1);
for (size_t i = 1; i < blk_desc.inner_nblks; i++) {
inner_strides[blk_desc.inner_nblks - 1 - i] = inner_strides[blk_desc.inner_nblks - i] * blk_desc.inner_blks[blk_desc.inner_nblks - i];
}
// total inner block size. in case of 4i16o4i will be {16, 16, 1, 1}
VectorDims total_block_per_dim(outer_ndims, 1);
for (int i = 0; i < inner_ndims; i++) {
total_block_per_dim[blk_desc.inner_idxs[i]] *= blk_desc.inner_blks[i];
}
VectorDims outer_block_dims(std::begin(dims), std::begin(dims) + outer_ndims);
for (size_t i = 0; i < outer_block_dims.size(); i++) {
outer_block_dims[i] = div_up(outer_block_dims[i], total_block_per_dim[i]);
}
// order of outer dims. In case of IOhw_ will be {1, 0, 2, 3}
VectorDims outer_order(outer_ndims);
std::iota(outer_order.begin(), outer_order.end(), 0);
std::sort(outer_order.begin(), outer_order.end(),
[&blk_desc, &outer_block_dims](size_t ind_l, size_t ind_r) {
return (blk_desc.strides[ind_l] > blk_desc.strides[ind_r]) ||
(blk_desc.strides[ind_l] == blk_desc.strides[ind_r] && outer_block_dims[ind_l] > outer_block_dims[ind_r]);
});
// blocked order
// [new_outer_order] U [inner_idxs]
SizeVector blk_order(total_ndims, 0);
std::copy(outer_order.begin(), outer_order.end(), blk_order.begin());
std::copy(blk_desc.inner_idxs, blk_desc.inner_idxs + blk_desc.inner_nblks, blk_order.begin() + dims.size());
order.swap(blk_order);
initBlockedParams();
}
bool DnnlBlockedMemoryDesc::hasLayoutType(LayoutType layoutType) const {
switch (layoutType) {
case LayoutType::ncsp:
return isPlainFormat();
case LayoutType::nspc:
return isTailCFormat();
case LayoutType::nCsp8c:
return isBlockedCFormat(8);
case LayoutType::nCsp16c:
return isBlockedCFormat(16);
default:
return false;
}
}
bool DnnlBlockedMemoryDesc::isPlainFormat() const {
if (shape.getRank() != order.size()) {
return false;
}
for (size_t i = 0; i < order.size(); ++i) {
if (order[i] != i) {
return false;
}
}
return true;
}
bool DnnlBlockedMemoryDesc::isBlockedCFormat(size_t blk_size) const {
const auto &blocking = desc.data.format_desc.blocking;
if (desc.data.format_kind !=dnnl_blocked ||
blocking.inner_nblks != 1 ||
blocking.inner_idxs[0] != 1)
return false;
if ((order.size() - shape.getRank()) != 1) {
return false;
}
for (size_t i = 0; i < order.size() - 1; ++i) {
if (order[i] != i) {
return false;
}
}
if (blk_size != UNREACHABLE_DIM && blk_size != blocking.inner_blks[0]) {
return false;
}
return true;
}
bool DnnlBlockedMemoryDesc::isTailCFormat() const {
if (shape.getRank() < 3) {
return false;
}
if (shape.getRank() != order.size()) {
return false;
}
if (!std::is_sorted(order.begin(), --order.end())) {
return false;
}
if (order.back() != 1) {
return false;
}
return true;
}
MemoryDescPtr DnnlBlockedMemoryDesc::cloneWithNewDimsImp(const VectorDims &dims) const {
if (std::any_of(dims.begin(), dims.end(), [](size_t x){ return Shape::UNDEFINED_DIM == x; })) {
IE_THROW() << "Can't clone desc if new dims are undefined";
}
// TODO [DS]: add stride recalculation for strided blobs
getStrides();
getBlockDims();
for (int i = strides.size() - 2; i >= 0 ; i--) {
if (strides[i] == Shape::UNDEFINED_DIM)
break;
if (strides[i] != strides[i + 1] * blockedDims[i + 1])
IE_THROW(NotImplemented) << "Can't clone desc with new dims for not dense tensor";
}
using namespace dnnl::impl::utils;
auto mklDims = MKLDNNExtensionUtils::convertToDnnlDims(dims);
mkldnn::memory::desc newMklDesc = desc;
array_copy(newMklDesc.data.dims, mklDims.data(), mklDims.size());
std::vector<int> perm(order.begin(), order.begin() + mklDims.size());
auto& blockingDesc = newMklDesc.data.format_desc.blocking;
auto numInnerBlks = blockingDesc.inner_nblks;
std::vector<int> innerBlks(std::begin(blockingDesc.inner_blks), std::begin(blockingDesc.inner_blks) + numInnerBlks);
std::vector<int> innerIdxs(std::begin(blockingDesc.inner_idxs), std::begin(blockingDesc.inner_idxs) + numInnerBlks);
auto retCode = dnnl::impl::fill_blocked(newMklDesc.data, perm, innerBlks, innerIdxs);
if (retCode != dnnl::impl::status::success) {
IE_THROW() << "Can not clone DnnlBlockedMemoryDesc with dims: " << MemoryDescUtils::dims2str(dims);
}
return DnnlBlockedMemoryDescPtr(new DnnlBlockedMemoryDesc(newMklDesc));
}
static const std::map<int, std::vector<mkldnn::memory::format_tag>> form_tags_by_ndims {
{0, {
mkldnn::memory::format_tag::a // TODO :: really 1d layout for scalar??
}}, {1, {
mkldnn::memory::format_tag::a
}}, {2, {
mkldnn::memory::format_tag::ab,
mkldnn::memory::format_tag::ba
}}, {3, {
mkldnn::memory::format_tag::abc,
mkldnn::memory::format_tag::acb,
mkldnn::memory::format_tag::bac,
mkldnn::memory::format_tag::bca,
mkldnn::memory::format_tag::cba,
mkldnn::memory::format_tag::Abc16a,
mkldnn::memory::format_tag::ABc16a16b,
mkldnn::memory::format_tag::ABc4a4b,
mkldnn::memory::format_tag::aBc16b,
mkldnn::memory::format_tag::aBc32b,
mkldnn::memory::format_tag::ABc16b16a,
mkldnn::memory::format_tag::Abc4a,
mkldnn::memory::format_tag::aBc4b,
mkldnn::memory::format_tag::ABc4b16a4b,
mkldnn::memory::format_tag::ABc2b8a4b,
mkldnn::memory::format_tag::ABc16b16a4b,
mkldnn::memory::format_tag::ABc16b16a2b,
mkldnn::memory::format_tag::ABc4b4a,
mkldnn::memory::format_tag::ABc8a16b2a,
mkldnn::memory::format_tag::ABc8a8b,
mkldnn::memory::format_tag::ABc8a4b,
mkldnn::memory::format_tag::aBc8b,
mkldnn::memory::format_tag::ABc8b16a2b,
mkldnn::memory::format_tag::ABc8b8a,
mkldnn::memory::format_tag::Acb16a,
mkldnn::memory::format_tag::Acb4a,
mkldnn::memory::format_tag::Acb8a,
mkldnn::memory::format_tag::BAc16a16b,
mkldnn::memory::format_tag::BAc16b16a,
}}, {4, { // Popular
mkldnn::memory::format_tag::abcd, // plain
mkldnn::memory::format_tag::acdb, // tail_c
mkldnn::memory::format_tag::aBcd8b, // blocked 8c
mkldnn::memory::format_tag::aBcd16b, // blocked 16c
mkldnn::memory::format_tag::abdc,
mkldnn::memory::format_tag::bacd,
mkldnn::memory::format_tag::bcda,
mkldnn::memory::format_tag::cdba,
mkldnn::memory::format_tag::dcab,
mkldnn::memory::format_tag::Abcd8a,
mkldnn::memory::format_tag::Abcd16a,
mkldnn::memory::format_tag::Abcd32a,
mkldnn::memory::format_tag::ABcd16a16b,
mkldnn::memory::format_tag::aBcd32b,
mkldnn::memory::format_tag::ABcd16b16a,
mkldnn::memory::format_tag::aBCd16b16c,
mkldnn::memory::format_tag::aBCd16c16b,
mkldnn::memory::format_tag::Abcd4a,
mkldnn::memory::format_tag::aBcd4b,
mkldnn::memory::format_tag::ABcd4b16a4b,
mkldnn::memory::format_tag::ABcd2b8a4b,
mkldnn::memory::format_tag::ABcd4b4a,
mkldnn::memory::format_tag::ABcd4a4b,
mkldnn::memory::format_tag::aBCd4c16b4c,
mkldnn::memory::format_tag::aBCd2c8b4c,
mkldnn::memory::format_tag::ABcd16b16a4b,
mkldnn::memory::format_tag::ABcd16b16a2b,
mkldnn::memory::format_tag::aBCd16c16b4c,
mkldnn::memory::format_tag::aBCd16c16b2c,
mkldnn::memory::format_tag::aBCd4c4b,
mkldnn::memory::format_tag::aBCd4b4c,
mkldnn::memory::format_tag::ABcd8a16b2a,
mkldnn::memory::format_tag::ABcd8a8b,
mkldnn::memory::format_tag::ABcd8a32b,
mkldnn::memory::format_tag::ABcd32a32b,
mkldnn::memory::format_tag::ABcd8a4b,
mkldnn::memory::format_tag::ABcd8b16a2b,
mkldnn::memory::format_tag::aBCd8b16c2b,
mkldnn::memory::format_tag::ABcd8b8a,
mkldnn::memory::format_tag::aBCd8b8c,
mkldnn::memory::format_tag::aBCd8b4c,
mkldnn::memory::format_tag::aBCd8c16b2c,
mkldnn::memory::format_tag::aBCd8c8b,
mkldnn::memory::format_tag::ABcd4a8b8a4b,
mkldnn::memory::format_tag::ABcd2a8b8a2b,
mkldnn::memory::format_tag::aBdc16b,
mkldnn::memory::format_tag::aBdc4b,
mkldnn::memory::format_tag::aBdc8b,
mkldnn::memory::format_tag::aCBd16b16c,
mkldnn::memory::format_tag::aCBd16c16b,
mkldnn::memory::format_tag::Acdb16a,
mkldnn::memory::format_tag::Acdb4a,
mkldnn::memory::format_tag::Acdb8a,
mkldnn::memory::format_tag::BAcd16a16b,
mkldnn::memory::format_tag::BAcd16b16a,
mkldnn::memory::format_tag::ABcd32a32b,
mkldnn::memory::format_tag::Acdb32a,
mkldnn::memory::format_tag::aBCd2b4c2b,
mkldnn::memory::format_tag::aBCd2c4b2c,
mkldnn::memory::format_tag::aBCd4b8c2b,
mkldnn::memory::format_tag::aBCd4c8b2c,
}}, {5, { // Popular
mkldnn::memory::format_tag::abcde, // plain
mkldnn::memory::format_tag::acdeb, // tail_c
mkldnn::memory::format_tag::aBcde8b, // blocked 8c
mkldnn::memory::format_tag::aBcde16b, // blocked 16c
mkldnn::memory::format_tag::abdec,
mkldnn::memory::format_tag::acbde,
mkldnn::memory::format_tag::bacde,
mkldnn::memory::format_tag::bcdea,
mkldnn::memory::format_tag::cdeba,
mkldnn::memory::format_tag::decab,
mkldnn::memory::format_tag::Abcde16a,
mkldnn::memory::format_tag::Abcde32a,
mkldnn::memory::format_tag::ABcde16a16b,
mkldnn::memory::format_tag::aBcde32b,
mkldnn::memory::format_tag::ABcde16b16a,
mkldnn::memory::format_tag::aBCde16b16c,
mkldnn::memory::format_tag::aBCde16c16b,
mkldnn::memory::format_tag::aBCde2c8b4c,
mkldnn::memory::format_tag::Abcde4a,
mkldnn::memory::format_tag::aBcde4b,
mkldnn::memory::format_tag::ABcde4b4a,
mkldnn::memory::format_tag::ABcde4a4b,
mkldnn::memory::format_tag::aBCde4b4c,
mkldnn::memory::format_tag::aBCde4c16b4c,
mkldnn::memory::format_tag::aBCde16c16b4c,
mkldnn::memory::format_tag::aBCde16c16b2c,
mkldnn::memory::format_tag::aBCde4c4b,
mkldnn::memory::format_tag::Abcde8a,
mkldnn::memory::format_tag::ABcde8a8b,
mkldnn::memory::format_tag::ABcde8a4b,
mkldnn::memory::format_tag::ABcde8b16a2b,
mkldnn::memory::format_tag::ABcde4b16a4b,
mkldnn::memory::format_tag::ABcde2b8a4b,
mkldnn::memory::format_tag::aBCde8b16c2b,
mkldnn::memory::format_tag::ABcde8b8a,
mkldnn::memory::format_tag::aBCde8b8c,
mkldnn::memory::format_tag::aBCde8b4c,
mkldnn::memory::format_tag::aBCde4b8c8b4c,
mkldnn::memory::format_tag::aBCde2b8c8b2c,
mkldnn::memory::format_tag::aBCde8c16b2c,
mkldnn::memory::format_tag::aBCde8c8b,
mkldnn::memory::format_tag::aBdec16b,
mkldnn::memory::format_tag::aBdec4b,
mkldnn::memory::format_tag::aBdec8b,
mkldnn::memory::format_tag::aCBde16b16c,
mkldnn::memory::format_tag::aCBde16c16b,
mkldnn::memory::format_tag::Acdeb16a,
mkldnn::memory::format_tag::Acdeb4a,
mkldnn::memory::format_tag::Acdeb8a,
mkldnn::memory::format_tag::BAcde16b16a,
mkldnn::memory::format_tag::BAcde16a16b,
mkldnn::memory::format_tag::aBdec32b,
mkldnn::memory::format_tag::aBCde2b4c2b,
mkldnn::memory::format_tag::aBCde2c4b2c,
mkldnn::memory::format_tag::aBCde4b8c2b,
mkldnn::memory::format_tag::aBCde4c8b2c,
}}, {6, { // Popular
mkldnn::memory::format_tag::abcdef, // plain
mkldnn::memory::format_tag::acbdef, // permute
mkldnn::memory::format_tag::defcab, // permute
mkldnn::memory::format_tag::aBcdef16b, // blocked 16c
mkldnn::memory::format_tag::aBCdef16b16c,
mkldnn::memory::format_tag::aBCdef16c16b,
mkldnn::memory::format_tag::aBcdef4b,
mkldnn::memory::format_tag::aBCdef2c8b4c,
mkldnn::memory::format_tag::aBCdef4c4b,
mkldnn::memory::format_tag::aBCdef4b4c,
mkldnn::memory::format_tag::aBCdef8b8c,
mkldnn::memory::format_tag::aBCdef8b4c,
mkldnn::memory::format_tag::aBCdef8c16b2c,
mkldnn::memory::format_tag::aBCdef4c16b4c,
mkldnn::memory::format_tag::aBCdef8c8b,
mkldnn::memory::format_tag::aBdefc16b,
mkldnn::memory::format_tag::aCBdef16c16b,
mkldnn::memory::format_tag::aCBdef16b16c,
mkldnn::memory::format_tag::aBdefc4b,
mkldnn::memory::format_tag::aBdefc8b,
mkldnn::memory::format_tag::Abcdef4a,
mkldnn::memory::format_tag::Abcdef8a,
mkldnn::memory::format_tag::Abcdef16a,
mkldnn::memory::format_tag::Abcdef32a,
mkldnn::memory::format_tag::aBCdef2b4c2b,
mkldnn::memory::format_tag::aBCdef2c4b2c,
mkldnn::memory::format_tag::aBCdef4b8c2b,
mkldnn::memory::format_tag::aBCdef4c8b2c,
}}
};
bool DnnlBlockedMemoryDesc::isSame(mkldnn::memory::format_tag fmt) const {
mkldnn::memory::desc refDesc(desc.dims(), desc.data_type(), fmt);
if (desc.data.ndims != refDesc.data.ndims)
return false;
if (desc.data.format_kind != dnnl_blocked || refDesc.data.format_kind != dnnl_blocked)
IE_THROW() << "DnnlMemoryDesc::isSame is not implemented for non blocked memory format";
auto actualBlkDesc = desc.data.format_desc.blocking;
auto refBlkDesc = refDesc.data.format_desc.blocking;
if (actualBlkDesc.inner_nblks != refBlkDesc.inner_nblks)
return false;
for (size_t i = 0; i < actualBlkDesc.inner_nblks; ++i)
if (actualBlkDesc.inner_blks[i] != refBlkDesc.inner_blks[i])
return false;
for (size_t i = 0; i < actualBlkDesc.inner_nblks; ++i)
if (actualBlkDesc.inner_idxs[i] != refBlkDesc.inner_idxs[i])
return false;
auto actualStrides = desc.data.format_desc.blocking.strides;
auto refStrides = refDesc.data.format_desc.blocking.strides;
VectorDims actualOrder(desc.data.ndims);
{
const auto dims = desc.dims();
VectorDims total_block_per_dim(dims.size(), 1);
const auto &blk_desc = desc.data.format_desc.blocking;
for (int i = 0; i < blk_desc.inner_nblks; i++) {
total_block_per_dim[blk_desc.inner_idxs[i]] *= blk_desc.inner_blks[i];
}
VectorDims outer_block_dims(std::begin(dims), std::begin(dims) + dims.size());
for (size_t i = 0; i < outer_block_dims.size(); i++) {
outer_block_dims[i] = div_up(outer_block_dims[i], total_block_per_dim[i]);
}
std::iota(actualOrder.begin(), actualOrder.end(), 0);
std::sort(actualOrder.begin(), actualOrder.end(),
[&actualStrides, &outer_block_dims] (size_t ind_l, size_t ind_r) {
return (actualStrides[ind_l] > actualStrides[ind_r]) ||
(actualStrides[ind_l] == actualStrides[ind_r] && outer_block_dims[ind_l] > outer_block_dims[ind_r]);
});
}
VectorDims refOrder(refDesc.data.ndims);
{
const auto dims = refDesc.dims();
VectorDims total_block_per_dim(dims.size(), 1);
const auto &blk_desc = refDesc.data.format_desc.blocking;
for (int i = 0; i < blk_desc.inner_nblks; i++) {
total_block_per_dim[blk_desc.inner_idxs[i]] *= blk_desc.inner_blks[i];
}
VectorDims outer_block_dims(std::begin(dims), std::begin(dims) + dims.size());
for (size_t i = 0; i < outer_block_dims.size(); i++) {
outer_block_dims[i] = div_up(outer_block_dims[i], total_block_per_dim[i]);
}
std::iota(refOrder.begin(), refOrder.end(), 0);
std::sort(refOrder.begin(), refOrder.end(),
[&refStrides, &outer_block_dims] (size_t ind_l, size_t ind_r) {
return (refStrides[ind_l] > refStrides[ind_r]) ||
(refStrides[ind_l] == refStrides[ind_r] && outer_block_dims[ind_l] > outer_block_dims[ind_r]);
});
}
if (actualOrder != refOrder) {
return false;
}
return true;
}
mkldnn::memory::format_tag DnnlBlockedMemoryDesc::getFormat() const {
// TODO [OneDNN]: Previously it was a field of tdesc, but now the brute
// force search here. Please avoid of using this method.
const auto ndims = desc.dims().size();
// There are no suitable format_tag for this
if (ndims == 0 || ndims > 6)
return mkldnn::memory::format_tag::undef;
for (const auto fmt : form_tags_by_ndims.at(ndims)) {
if (this->isSame(fmt))
return fmt;
}
return mkldnn::memory::format_tag::undef;
}
std::string DnnlBlockedMemoryDesc::serializeFormat() const {
auto fmt = getFormat();
return mkldnn::utils::fmt2str(fmt);
}
size_t DnnlBlockedMemoryDesc::getMaxMemSize() const {
if (shape.isStatic()) {
return getCurrentMemSize();
}
auto& maxDims = shape.getMaxDims();
if (std::any_of(maxDims.begin(), maxDims.end(), [](size_t x){ return Shape::UNDEFINED_DIM == x; })) {
return UNDEFINED_SIZE;
}
auto maxDimsDesc = cloneWithNewDims(maxDims);
return maxDimsDesc->getCurrentMemSize();
}
size_t DnnlBlockedMemoryDesc::getPaddedElementsCount() const {
return std::accumulate(std::begin(desc.data.padded_dims), std::begin(desc.data.padded_dims) + desc.data.ndims, size_t{1},
std::multiplies<int64_t>());
}
bool DnnlBlockedMemoryDesc::blocksExtended() const {
for (int i = 0; i < desc.data.ndims; i++) {
if (desc.data.dims[i] != desc.data.padded_dims[i])
return true;
}
return false;
}
void DnnlBlockedMemoryDesc::initBlockDims() {
const auto dims = desc.dims();
const auto &blk_desc = desc.data.format_desc.blocking;
const size_t outer_ndims = dims.size();
const size_t inner_ndims = blk_desc.inner_nblks;
const size_t total_ndims = outer_ndims + inner_ndims;
// total inner block size. in case of 4i16o4i will be {16, 16, 1, 1}
VectorDims total_block_per_dim(outer_ndims, 1);
for (int i = 0; i < inner_ndims; i++) {
total_block_per_dim[blk_desc.inner_idxs[i]] *= blk_desc.inner_blks[i];
}
// blocked dims
// [dims via new_outer_order with auto pad] U [inner_blk_dims]
VectorDims outer_block_dims = MKLDNNExtensionUtils::convertToVectorDims(dims);
for (size_t i = 0; i < outer_block_dims.size(); i++) {
if (outer_block_dims[i] != Shape::UNDEFINED_DIM) {
outer_block_dims[i] = div_up(outer_block_dims[i], total_block_per_dim[i]);
}
}
// order of outer dims. In case of IOhw_ will be {1, 0, 2, 3}
VectorDims outer_order(outer_ndims);
std::copy(order.begin(), order.begin() + outer_ndims, outer_order.begin());
blockedDims.resize(total_ndims, 0);
std::copy(blk_desc.inner_blks, blk_desc.inner_blks + blk_desc.inner_nblks,
blockedDims.end() - blk_desc.inner_nblks);
std::transform(outer_order.begin(), outer_order.end(), blockedDims.begin(),
[&] (size_t i) { return outer_block_dims[i]; });
}
void DnnlBlockedMemoryDesc::initStrides() {
const auto dims = desc.dims();
const auto &blk_desc = desc.data.format_desc.blocking;
const size_t outer_ndims = dims.size();
const size_t inner_ndims = blk_desc.inner_nblks;
const size_t total_ndims = outer_ndims + inner_ndims;
// strides of inner dims. In case of 4i16o4i will be {64, 4, 1}
VectorDims inner_strides(inner_ndims, 1);
for (size_t i = 1; i < blk_desc.inner_nblks; i++) {
inner_strides[blk_desc.inner_nblks - 1 - i] = inner_strides[blk_desc.inner_nblks - i] * blk_desc.inner_blks[blk_desc.inner_nblks - i];
}
// order of outer dims. In case of IOhw_ will be {1, 0, 2, 3}
VectorDims outer_order(outer_ndims);
std::copy(order.begin(), order.begin() + outer_ndims, outer_order.begin());
// blocked strides
// [outer_strides via new_outer_order] U [inner_strides]
strides.resize(total_ndims, 0);
std::copy(inner_strides.rbegin(), inner_strides.rend(), strides.rbegin());
std::transform(outer_order.begin(), outer_order.end(), strides.begin(),
[&](size_t i) { return blk_desc.strides[i] == DNNL_RUNTIME_DIM_VAL ? Shape::UNDEFINED_DIM : blk_desc.strides[i]; });
}
void DnnlBlockedMemoryDesc::initOffsetPadding() {
offsetPaddingToData = VectorDims(std::begin(desc.data.padded_offsets), std::begin(desc.data.padded_offsets) + getOrder().size());
}
inference-engine/src/mkldnn_plugin/memory_desc/dnnl_blocked_memory_desc.h
+99
View File
@@ -0,0 +1,99 @@
// Copyright (C) 2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include "blocked_memory_desc.h"
#include "mkldnn_memory.h"
#include "mkldnn_extension_utils.h"
namespace MKLDNNPlugin {
class DnnlBlockedMemoryDesc : public BlockedMemoryDesc, public DnnlMemoryDesc {
public:
// Creates planar DnnlBlockedMemoryDesc
DnnlBlockedMemoryDesc(InferenceEngine::Precision prc, const Shape& shape);
DnnlBlockedMemoryDesc(const Shape& shape, mkldnn::memory::data_type dataType, mkldnn::memory::format_tag format);
MemoryDescPtr clone() const override {
return std::make_shared<DnnlBlockedMemoryDesc>(*this);
}
bool isCompatible(const MemoryDesc& rhs) const override;
bool isCompatible(const DnnlBlockedMemoryDesc& rhs) const;
bool isCompatible(const CpuBlockedMemoryDesc& rhs) const;
const VectorDims& getBlockDims() const override {
return blockedDims;
}
const VectorDims& getOrder() const override {
return order;
}
const VectorDims& getOffsetPaddingToData() const override {
return offsetPaddingToData;
}
size_t getOffsetPadding() const override {
return MKLDNNExtensionUtils::convertToDim(desc.data.offset0);
}
const VectorDims& getStrides() const override {
return strides;
}
bool hasLayoutType(LayoutType layoutType) const override;
bool isSame(mkldnn::memory::format_tag fmt) const override;
std::string serializeFormat() const override;
size_t getMaxMemSize() const override;
bool blocksExtended() const override;
size_t getPaddedElementsCount() const override;
private:
DnnlBlockedMemoryDesc(InferenceEngine::Precision prc, const Shape& shape, const VectorDims& blockedDims,
const VectorDims& order, size_t offsetPadding = 0, const VectorDims& offsetPaddingToData = {},
const VectorDims& strides = {});
DnnlBlockedMemoryDesc(const mkldnn::memory::desc& mdesc);
MemoryDescPtr cloneWithNewDimsImp(const VectorDims& dims) const override;
bool isPlainFormat() const;
bool isBlockedCFormat(size_t blk_size = UNREACHABLE_DIM) const;
bool isTailCFormat() const;
// WA: we need to initialize blocked params into ctor to avoid bugs when we calculate these params in throughput mode
// TODO [DS]: should be reimplemented to avoid useless calculation
void initBlockedParams() {
initBlockDims();
initStrides();
initOffsetPadding();
}
void initBlockDims();
void initStrides();
void initOffsetPadding();
/**
* Try to define original format tag use on creation
*
* @return format tag if was able to define it
*/
mkldnn::memory::format_tag getFormat() const;
friend DnnlMemoryDescPtr MKLDNNExtensionUtils::makeDescriptor(const mkldnn::memory::desc &desc);
friend class MemoryDescUtils;
};
using DnnlBlockedMemoryDescPtr = std::shared_ptr<DnnlBlockedMemoryDesc>;
using DnnlBlockedMemoryDescCPtr = std::shared_ptr<const DnnlBlockedMemoryDesc>;
} // namespace MKLDNNPlugin
inference-engine/src/mkldnn_plugin/memory_desc/dnnl_memory_desc.cpp
+75
View File
@@ -0,0 +1,75 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "dnnl_memory_desc.h"
#include "mkldnn_extension_utils.h"
#include <common/memory_desc_wrapper.hpp>
#include "mkldnn/ie_mkldnn.h"
namespace MKLDNNPlugin {
DnnlMemoryDesc::DnnlMemoryDesc(const mkldnn::memory::desc& desc) :
MemoryDesc(Shape(MKLDNNExtensionUtils::convertToVectorDims(desc.dims())), Mkldnn), desc(desc) {
if (desc.data.format_kind == dnnl::impl::format_kind::any)
IE_THROW(Unexpected) << "Memory format any is prohibited!";
}
size_t DnnlMemoryDesc::getCurrentMemSizeImp() const {
return MKLDNNExtensionUtils::getMemSizeForDnnlDesc(desc);
}
size_t DnnlMemoryDesc::getElementOffset(size_t elemNumber) const {
mkldnn::impl::memory_desc_wrapper wrapped(desc.data);
return wrapped.off_l(elemNumber);
}
bool DnnlMemoryDesc::isCompatible(const MemoryDesc &rhs) const {
if (MemoryDescType::Mkldnn == rhs.getType()) {
return this->desc == rhs.as<DnnlMemoryDesc>()->desc;
} else {
return false;
}
}
// TODO: add serialization for packed format
std::string DnnlMemoryDesc::serializeFormat() const {
if (desc.data.format_kind == dnnl_format_kind_wino) {
switch (desc.data.format_desc.wino_desc.wino_format) {
case dnnl_wino_memory_format_t::dnnl_wino_wei_aaOIoi: return "wino_aaOIoi";
case dnnl_wino_memory_format_t::dnnl_wino_wei_aaOio: return "wino_aaOio";
case dnnl_wino_memory_format_t::dnnl_wino_wei_aaOBiOo: return "wino_aaOBiOo";
case dnnl_wino_memory_format_t::dnnl_wino_wei_OBaaIBOIio: return "wino_OBaaIBOIio";
default: return "wino_undef";
}
}
return "undef";
}
bool DnnlMemoryDesc::isDefinedImp() const {
mkldnn::impl::memory_desc_wrapper wrappedThis(desc.data);
if (wrappedThis.has_runtime_dims_or_strides()) {
return false;
}
return wrappedThis.offset0() != DNNL_RUNTIME_DIM_VAL;
}
InferenceEngine::Precision DnnlMemoryDesc::getPrecision() const {
return MKLDNNExtensionUtils::DataTypeToIEPrecision(desc.data_type());
}
MemoryDescPtr DnnlMemoryDesc::cloneWithNewDimsImp(const VectorDims &dims) const {
IE_THROW(Unexpected) << "Cannot clone non blocked oneDNN desc with new dims";
}
size_t DnnlMemoryDesc::getMaxMemSize() const {
if (shape.isDynamic()) {
IE_THROW() << "Can't compute max mem size for DnnlMemoryDesc with dynaimc shape";
}
return getCurrentMemSize();
}
} // namespace MKLDNNPlugin
inference-engine/src/mkldnn_plugin/memory_desc/dnnl_memory_desc.h
+71
View File
@@ -0,0 +1,71 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include "cpu_blocked_memory_desc.h"
#include "mkldnn_extension_utils.h"
namespace MKLDNNPlugin {
class DnnlMemoryDesc;
using DnnlMemoryDescPtr = std::shared_ptr<DnnlMemoryDesc>;
using DnnlMemoryDescCPtr = std::shared_ptr<const DnnlMemoryDesc>;
class DnnlMemoryDesc : public virtual MemoryDesc {
public:
mkldnn::memory::data_type getDataType() const {
return static_cast<mkldnn::memory::data_type>(desc.data.data_type);
}
dnnl_format_kind_t getFormatKind() const {
return desc.data.format_kind;
}
MemoryDescPtr clone() const override {
return std::make_shared<DnnlMemoryDesc>(*this);
}
std::string serializeFormat() const override;
InferenceEngine::Precision getPrecision() const override;
bool isCompatible(const MemoryDesc& rhs) const override;
size_t getMaxMemSize() const override;
const mkldnn::memory::desc& getDnnlDesc() const {
return desc;
}
bool hasLayoutType(LayoutType layoutType) const override { return false; }
virtual bool isSame(mkldnn::memory::format_tag fmt) const { return false; }
bool hasEmptyExtraData() const { return desc.data.extra.flags == dnnl_memory_extra_flag_none; }
protected:
DnnlMemoryDesc() {}
static constexpr size_t UNREACHABLE_DIM = std::numeric_limits<size_t>::max();
mkldnn::memory::desc desc;
void setPrecision(InferenceEngine::Precision prc) override {
desc.data.data_type = static_cast<dnnl_data_type_t>(MKLDNNExtensionUtils::IEPrecisionToDataType(prc));
}
private:
explicit DnnlMemoryDesc(const mkldnn::memory::desc& desc);
size_t getElementOffset(size_t elemNumber) const override;
size_t getCurrentMemSizeImp() const override;
bool isDefinedImp() const override;
MemoryDescPtr cloneWithNewDimsImp(const VectorDims& dims) const override;
friend DnnlMemoryDescPtr MKLDNNExtensionUtils::makeDescriptor(const mkldnn::memory::desc &desc);
};
} // namespace MKLDNNPlugin
inference-engine/src/mkldnn_plugin/mkldnn_dims.h
-95
View File
@@ -1,95 +0,0 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include "perf_count.h"
#include <vector>
#include <utility>
#include <mkldnn_types.h>
#include <ie_common.h>
#include <mkldnn.hpp>
namespace MKLDNNPlugin {
class MKLDNNDims {
public:
MKLDNNDims() = default;
explicit MKLDNNDims(const InferenceEngine::SizeVector& size) {
dims = std::vector<ptrdiff_t>(size.begin(), size.end());
}
explicit MKLDNNDims(const std::vector<ptrdiff_t>& dim) {
dims = dim;
}
MKLDNNDims(const mkldnn_dims_t dnn_dims, int dnn_ndims) {
dims = std::vector<ptrdiff_t>(dnn_dims, dnn_dims + dnn_ndims);
}
explicit MKLDNNDims(std::initializer_list<ptrdiff_t> ilist) : dims(ilist) {}
explicit MKLDNNDims(std::initializer_list<size_t > ilist) : dims(ilist.begin(), ilist.end()) {}
InferenceEngine::SizeVector ToSizeVector() const {
InferenceEngine::SizeVector size;
for (auto i : dims) {
size.push_back(i);
}
return size;
}
int ndims() const {
return dims.size();
}
ptrdiff_t size() const {
return size(0);
}
ptrdiff_t size(int start) const {
ptrdiff_t size = 1;
for (int i = start; i < dims.size(); i++) {
size *= dims[i];
}
return size;
}
void push_back(int val) {
dims.push_back(val);
}
operator mkldnn::memory::dims() const {
// TODO: it will convert each time.. not good
return mkldnn::memory::dims(dims.begin(), dims.end());
}
bool operator == (const MKLDNNDims& rhs) const {
if (dims.size() != rhs.dims.size()) {
return false;
}
return std::equal(rhs.dims.begin(), rhs.dims.end(), dims.begin());
}
bool operator != (const MKLDNNDims& rhs) const {
return !(*this == rhs);
}
ptrdiff_t& operator[](int idx) {
return dims[idx];
}
ptrdiff_t operator[](int idx) const {
return dims[idx];
}
private:
std::vector<ptrdiff_t> dims;
};
} // namespace MKLDNNPlugin
inference-engine/src/mkldnn_plugin/mkldnn_edge.cpp
+3 -63
View File
@@ -166,8 +166,6 @@ void MKLDNNEdge::allocate(const void* mem_ptr) {
auto& inputDesc = getInputDesc();
auto& outputDesc = getOutputDesc();
if (!inputDesc.isDefined() || !outputDesc.isDefined())
IE_THROW() << "Cannot allocate memory for undefined descriptors.";
if (!inputDesc.isCompatible(outputDesc))
IE_THROW() << "Cannot allocate memory for incompatible descriptors.";
@@ -222,57 +220,6 @@ void MKLDNNEdge::changeStatus(MKLDNNEdge::Status state) {
status = state;
}
// TODO [DS]: remove while DynamicShapes migration
// TODO [DS]: How should we validate shape compatibility?
// TODO [DS]: Why do we allow uninitialized shape?
const Shape& MKLDNNEdge::getShape() {
if (!shape.getRank()) {
Shape inShape;
Shape outShape;
auto childPtr = getChild();
auto parentPtr = getParent();
int inNum = getOutputNum();
if (inNum < 0) {
IE_THROW() << "Error cannot find input data for " << child.lock()->getName()
<< " from " << parent.lock()->getName();
}
if (inNum < childPtr->inputShapes.size()) {
outShape = childPtr->inputShapes[inNum];
}
int outNum = getInputNum();
if (outNum < 0) {
IE_THROW() << "Error cannot find output data for " << parent.lock()->getName()
<< " to " << child.lock()->getName();
}
if (outNum >= parentPtr->outputShapes.size())
outNum = 0;
if (outNum < parentPtr->outputShapes.size()) {
inShape = parentPtr->outputShapes[outNum];
}
if (inShape.getRank() && outShape.getRank() && inShape.getRank() != outShape.getRank() && inShape.getElementsCount() != outShape.getElementsCount())
IE_THROW() << "Nodes " << getParent()->getName() << " and " << getChild()->getName()
<< " have incompatible dimensions!";
if (outShape.getRank() != 0) {
shape = outShape;
} else if (inShape.getRank() != 0) {
shape = inShape;
} else {
shape = Shape(InferenceEngine::SizeVector({1}));
}
if (!(outShape.getRank() == 0 && inShape.getRank() == 0) && !shape.getRank())
IE_THROW() << "Cannot detect right dims for nodes " << getParent()->getName()
<< " and " << getChild()->getName();
}
return shape;
}
const MemoryDesc& MKLDNNEdge::getInputDesc() const {
auto parentPtr = getParent();
if (parentPtr->getSelectedPrimitiveDescriptor() == nullptr)
@@ -321,20 +268,14 @@ const MemoryDesc& MKLDNNEdge::getDesc() const {
}
const MKLDNNMemory &MKLDNNEdge::getMemory() {
if (status == Status::NotAllocated) {
memoryPtr.reset(new MKLDNNMemory(getParent()->getEngine()));
memoryPtr->Create(getDesc(), getSharedEdge()->getMemoryPtr()->GetData());
memoryFromEdge.reset();
changeStatus(Status::Allocated);
}
return *memoryPtr;
return *getMemoryPtr();
}
MKLDNNMemoryPtr &MKLDNNEdge::getMemoryPtr() {
if (status == Status::NotAllocated) {
memoryPtr.reset(new MKLDNNMemory(getParent()->getEngine()));
memoryPtr->Create(getDesc(), getSharedEdge()->getMemoryPtr()->GetData());
const auto &desc = getDesc();
memoryPtr->Create(desc, desc.isDefined() ? getSharedEdge()->getMemoryPtr()->GetData() : nullptr);
memoryFromEdge.reset();
changeStatus(Status::Allocated);
}
@@ -353,7 +294,6 @@ void MKLDNNEdge::validate() {
getMemory();
getParent();
getChild();
getShape();
if (status != Status::Allocated) {
IE_THROW() << "Error memory is not allocated!";
inference-engine/src/mkldnn_plugin/mkldnn_edge.h
+5 -3
View File
@@ -6,7 +6,7 @@
#include <ie_blob.h>
#include "cpu_shape.h"
#include "cpu_memory_desc.h"
#include "memory_desc/cpu_memory_desc.h"
#include "mkldnn_weights_cache.hpp"
#include <map>
@@ -51,7 +51,6 @@ public:
const std::shared_ptr<MKLDNNNode> getParent() const;
const std::shared_ptr<MKLDNNNode> getChild() const;
const Shape &getShape();
const MKLDNNMemory& getMemory();
MKLDNNMemoryPtr& getMemoryPtr();
@@ -68,6 +67,10 @@ public:
MKLDNNEdgePtr getSharedEdge() const;
MKLDNNEdgePtr getSharedEdge(std::nothrow_t) const;
bool hasDefinedMaxSize() const {
return getDesc().hasDefinedMaxSize();
}
private:
std::string name() const;
@@ -78,7 +81,6 @@ private:
bool useExternalMemory = false;
MKLDNNEdgeWeakPtr memoryFromEdge;
Shape shape;
MKLDNNMemoryPtr memoryPtr;
Status status = Status::Uninitialized;
inference-engine/src/mkldnn_plugin/mkldnn_extension_utils.cpp
+58 -6
View File
@@ -4,9 +4,8 @@
#include "mkldnn_extension_utils.h"
#include "utils/general_utils.h"
#include <limits>
#include <vector>
#include <numeric>
#include "memory_desc/dnnl_blocked_memory_desc.h"
using namespace mkldnn;
using namespace MKLDNNPlugin;
@@ -77,10 +76,63 @@ InferenceEngine::Precision MKLDNNExtensionUtils::DataTypeToIEPrecision(memory::d
}
}
InferenceEngine::SizeVector MKLDNNExtensionUtils::convertToSizeVector(const mkldnn::memory::dims& dims) {
return InferenceEngine::SizeVector(dims.begin(), dims.end());
Dim MKLDNNExtensionUtils::convertToDim(const dnnl::memory::dim &dim) {
return dim == DNNL_RUNTIME_DIM_VAL ? Shape::UNDEFINED_DIM : static_cast<size_t>(dim);
}
dnnl::memory::dim MKLDNNExtensionUtils::convertToDnnlDim(const Dim &dim) {
return dim == Shape::UNDEFINED_DIM ? DNNL_RUNTIME_DIM_VAL : static_cast<mkldnn::memory::dim>(dim);
}
std::vector<dnnl::memory::dim> MKLDNNExtensionUtils::convertToDnnlDims(const InferenceEngine::SizeVector& dims) {
return std::vector<dnnl::memory::dim>(dims.begin(), dims.end());;
VectorDims MKLDNNExtensionUtils::convertToVectorDims(const memory::dims& dims) {
std::vector<size_t> vecResult;
vecResult.reserve(dims.size());
std::back_insert_iterator<std::vector<size_t>> itr(vecResult);
std::transform(dims.begin(), dims.end(), itr, convertToDim);
return vecResult;
}
memory::dims MKLDNNExtensionUtils::convertToDnnlDims(const VectorDims& dims) {
memory::dims vecResult;
vecResult.reserve(dims.size());
std::back_insert_iterator<memory::dims> itr(vecResult);
std::transform(dims.begin(), dims.end(), itr, convertToDnnlDim);
return vecResult;
}
memory::format_tag MKLDNNExtensionUtils::GetPlainFormatByRank(size_t rank) {
switch (rank) {
case 0:
case 1:
return memory::format_tag::a;
case 2:
return memory::format_tag::ab;
case 3:
return memory::format_tag::abc;
case 4:
return memory::format_tag::abcd;
case 5:
return memory::format_tag::abcde;
case 6:
return memory::format_tag::abcdef;
default:
return memory::format_tag::undef;
}
}
DnnlMemoryDescPtr MKLDNNExtensionUtils::makeDescriptor(const mkldnn::memory::desc &desc) {
if (desc.data.format_kind == dnnl_blocked) {
return std::shared_ptr<DnnlBlockedMemoryDesc>(new DnnlBlockedMemoryDesc(desc));
} else {
return std::shared_ptr<DnnlMemoryDesc>(new DnnlMemoryDesc(desc));
}
}
size_t MKLDNNExtensionUtils::getMemSizeForDnnlDesc(mkldnn::memory::desc desc) {
const auto offset0 = desc.data.offset0;
desc.data.offset0 = 0;
size_t size = desc.get_size();
if (size == DNNL_RUNTIME_SIZE_VAL)
return MemoryDesc::UNDEFINED_SIZE;
size += offset0 * sizeOfDataType(desc.data_type());
return size;
}
inference-engine/src/mkldnn_plugin/mkldnn_extension_utils.h
+16 -3
View File
@@ -11,17 +11,30 @@
#include <string>
#include "mkldnn.hpp"
#include "cpu_memory_desc.h"
#include "memory_desc/cpu_memory_desc.h"
namespace MKLDNNPlugin {
class DnnlMemoryDesc;
class MKLDNNExtensionUtils {
public:
static uint8_t sizeOfDataType(mkldnn::memory::data_type dataType);
static mkldnn::memory::data_type IEPrecisionToDataType(const InferenceEngine::Precision& prec);
static InferenceEngine::Precision DataTypeToIEPrecision(mkldnn::memory::data_type dataType);
static InferenceEngine::SizeVector convertToSizeVector(const mkldnn::memory::dims& dims);
static std::vector<dnnl::memory::dim> convertToDnnlDims(const InferenceEngine::SizeVector& dims);
static Dim convertToDim(const dnnl::memory::dim &dim);
static dnnl::memory::dim convertToDnnlDim(const Dim &dim);
static VectorDims convertToVectorDims(const mkldnn::memory::dims& dims);
static std::vector<dnnl::memory::dim> convertToDnnlDims(const VectorDims& dims);
static mkldnn::memory::format_tag GetPlainFormatByRank(size_t rank);
/**
* @brief Creates DnnlBlockedMemoryDesc if desc is blocked, otherwise DnnlMemoryDesc
* @param desc mkldnn::memory::desc from which one of the descriptors will be created
* @return pointer to DnnlBlockedMemoryDesc or DnnlMemoryDesc
*/
static std::shared_ptr<DnnlMemoryDesc> makeDescriptor(const mkldnn::memory::desc &desc);
static size_t getMemSizeForDnnlDesc(mkldnn::memory::desc desc);
};
} // namespace MKLDNNPlugin
inference-engine/src/mkldnn_plugin/mkldnn_graph.cpp
+71 -45
View File
@@ -39,7 +39,7 @@
#include "utils/node_dumper.h"
#include "utils/ngraph_utils.hpp"
#include "utils/cpu_utils.hpp"
#include "cpu_memory_desc_utils.h"
#include "memory_desc/cpu_memory_desc_utils.h"
#include <ngraph/node.hpp>
#include <ngraph/function.hpp>
@@ -47,6 +47,7 @@
#include <ngraph/ops.hpp>
#include <transformations/utils/utils.hpp>
#include <low_precision/low_precision.hpp>
#include "memory_desc/dnnl_blocked_memory_desc.h"
using namespace mkldnn;
using namespace MKLDNNPlugin;
@@ -214,8 +215,11 @@ void MKLDNNGraph::Replicate(const CNNNetwork &network, const MKLDNNExtensionMana
graphNodes.push_back(node);
if (op->get_type_info() == ngraph::op::v0::Parameter::type_info) {
if (inputsInfo.count(node->getName()) != 0) {
const auto inInfo = inputsInfo.find(node->getName());
if (inInfo != inputsInfo.end()) {
inputNodesMap[node->getName()] = node;
if (inInfo->second->getInputData()->isDynamic())
graphHasDynamicInput = true;
}
}
@@ -439,8 +443,8 @@ void MKLDNNGraph::ExecuteConstantNodesOnly() {
}
static bool isReorderAvailable(const MemoryDesc& parentDesc, const MemoryDesc& childDesc, const mkldnn::engine& eng) {
memory::desc dstMemDesc = MemoryDescUtils::convertToMKLDNNMemoryDesc(childDesc);
memory::desc srcMemDesc = MemoryDescUtils::convertToMKLDNNMemoryDesc(parentDesc);;
memory::desc dstMemDesc = MemoryDescUtils::convertToDnnlMemoryDesc(childDesc.clone())->getDnnlDesc();
memory::desc srcMemDesc = MemoryDescUtils::convertToDnnlMemoryDesc(parentDesc.clone())->getDnnlDesc();
mkldnn::primitive_attr attr;
dnnl_primitive_desc_t result = nullptr;
@@ -522,6 +526,9 @@ static edge_clusters_t findEdgeClusters(const std::vector<MKLDNNEdgePtr> & graph
edge_cluster_idx_map_t edge_cluster_indices;
for (auto &edge : graphEdges) {
if (!edge->hasDefinedMaxSize())
continue;
auto edge_it = edge_cluster_indices.find(edge);
if (edge_it != edge_cluster_indices.end())
@@ -602,11 +609,11 @@ void MKLDNNGraph::AllocateWithReuse() {
int e_start = edge->getParent()->execIndex;
int e_finish = edge->getChild()->execIndex;
int64_t e_size = edge->getDesc().getCurrentSize(); // size in bytes (from the beginning of data to the last element)
if (e_size == MemoryDesc::UNDEFINED_SIZE) {
if (!edge->hasDefinedMaxSize()) {
IE_THROW() << "Can not allocate memory since the size is undefined.";
}
int64_t e_size = edge->getDesc().getMaxMemSize(); // size in bytes (from the beginning of data to the last element)
box.start = std::min(e_start, box.start);
box.finish = std::max(e_finish, box.finish);
box.size = std::max(e_size, box.size);
@@ -639,7 +646,7 @@ void MKLDNNGraph::AllocateWithReuse() {
size_t total_size = static_cast<size_t>(memSolver.solve()) * alignment;
memWorkspace = std::make_shared<MKLDNNMemory>(eng);
memWorkspace->Create(MKLDNNMemoryDesc({total_size}, mkldnn::memory::data_type::s8));
memWorkspace->Create(DnnlBlockedMemoryDesc(InferenceEngine::Precision::I8, Shape(InferenceEngine::SizeVector{total_size})));
if (edge_clusters.empty())
return;
@@ -658,7 +665,7 @@ void MKLDNNGraph::AllocateWithReuse() {
// TODO: WA for some test (like strided_slice_test) which use tensors with
// shapes {0}. And it is implisitly converted into {1} tensor.
// Zeroing of input data allow pass tests.
if (edge->getParent()->type == Input)
if (edge->getParent()->type == Input && edge->hasDefinedMaxSize())
edge->getMemoryPtr()->FillZero();
count++;
@@ -679,8 +686,11 @@ void MKLDNNGraph::Allocate() {
// Allocate memory space for all edges marked with NeedAllocation
AllocateWithReuse();
// Resolve all other edges with status NotAllocated or in-place
for (auto& node : graphNodes) node->resolveNotAllocatedEdges();
// Resolve all other edges with status NotAllocated and in-place
for (auto& node : graphNodes) node->resolveInPlaceEdges();
// Create dummy memory with undefined desc for edges that are not allocated on the previous stages (memory solver and inPlace resolving)
for (auto& edge : graphEdges) edge->allocate();
// Check all getters. Should work.
for (auto& edge : graphEdges) edge->validate();
@@ -703,7 +713,7 @@ void MKLDNNGraph::PushInputData(const std::string& name, const InferenceEngine::
void *inter_data_ptr = input->second->getChildEdgeAt(0)->getMemory().GetData();
if (ext_data_ptr != inter_data_ptr) {
auto ext_tdesc = MemoryDescUtils::convertToMKLDNNMemoryDesc(in->getTensorDesc());
auto ext_tdesc = MemoryDescUtils::convertToDnnlBlockedMemoryDesc(in->getTensorDesc());
auto ext_mem = MKLDNNMemory(eng);
ext_mem.Create(ext_tdesc, ext_data_ptr, false);
@@ -714,7 +724,7 @@ void MKLDNNGraph::PushInputData(const std::string& name, const InferenceEngine::
// todo: make sure 'name' exists in this map...
if (_normalizePreprocMap.find(name) != _normalizePreprocMap.end()) {
if (in->getTensorDesc().getPrecision() == InferenceEngine::Precision::FP32) {
_normalizePreprocMap[name].NormalizeImage(input->second->getChildEdgeAt(0)->getShape(),
_normalizePreprocMap[name].NormalizeImage(input->second->getOutputShapeAtPort(0),
reinterpret_cast<float *>(inter_data_ptr),
in->getTensorDesc().getLayout());
} else {
@@ -726,7 +736,7 @@ void MKLDNNGraph::PushInputData(const std::string& name, const InferenceEngine::
}
}
void MKLDNNGraph::PullOutputData(const BlobMap &out) {
void MKLDNNGraph::PullOutputData(BlobMap &out) {
if (!IsReady())
IE_THROW() << "Wrong state. Topology not ready.";
@@ -735,50 +745,60 @@ void MKLDNNGraph::PullOutputData(const BlobMap &out) {
auto node = outputMap.second;
const MKLDNNMemory& intr_blob = node->getParentEdgeAt(0)->getMemory();
if (!out.count(name)) {
const auto ext_blob = out.find(name);
if (ext_blob == out.end()) {
IE_THROW(Unexpected) << "The network outputs do not contain mkldnn graph output node name: \"" << name << "\"";
}
const Blob::Ptr &ext_blob = out.at(name);
auto srcPrec = MKLDNNExtensionUtils::DataTypeToIEPrecision(intr_blob.GetDataType());
auto dstPrec = ext_blob->getTensorDesc().getPrecision();
if (srcPrec == dstPrec && ext_blob->byteSize() != intr_blob.GetSize())
IE_THROW() << "Output blob byte size is not equal network output byte size ("
<< ext_blob->byteSize() << "!=" << intr_blob.GetSize() << ").";
if (ext_blob->size() != intr_blob.GetElementsCount())
IE_THROW() << "Output blob number of elements is not equal network output number of elements ("
<< ext_blob->size() << "!=" << intr_blob.GetElementsCount() << ").";
void *ext_blob_ptr = ext_blob->buffer();
void *intr_blob_ptr = intr_blob.GetData();
// That is the same memory. No need to copy
if (ext_blob_ptr == intr_blob_ptr) continue;
int MB = intr_blob.GetDims()[0];
int MB_to_process = node->batchToProcess();
// TODO: Should we support InferenceEngine::PluginConfigParams::KEY_DYN_BATCH_LIMIT???
if (config.batchLimit)
MB_to_process = std::min<int>(config.batchLimit, MB_to_process);
size_t size_to_copy = intr_blob.GetElementsCount() * MB_to_process / MB;
const auto actualDesc = MemoryDescUtils::convertToTensorDesc(node->getParentEdgeAt(0)->getDesc());
const auto expectedDesc = ext_blob->getTensorDesc();
const auto actualDesc = MemoryDescUtils::convertToTensorDesc(intr_blob.getDesc());
const auto &expectedDesc = ext_blob->second->getTensorDesc();
// TODO [NM]: need to create universal reorder which will be detect cases when we really need to use it
// WA: for cases when output shape after transformation will be 1x1x1x1 but model output is scalar
bool isScalarOutput = false;
if (actualDesc.getLayout() == SCALAR) {
isScalarOutput = expectedDesc.getLayout() == SCALAR ||
std::accumulate(expectedDesc.getDims().begin(), expectedDesc.getDims().end(), (size_t)1, std::multiplies<size_t>()) == 1;
(!expectedDesc.getDims().empty() &&
std::accumulate(expectedDesc.getDims().begin(), expectedDesc.getDims().end(), (size_t)1, std::multiplies<size_t>()) == 1);
} else if (expectedDesc.getLayout() == SCALAR) {
isScalarOutput = actualDesc.getLayout() == SCALAR ||
std::accumulate(actualDesc.getDims().begin(), actualDesc.getDims().end(), (size_t)1, std::multiplies<size_t>()) == 1;
(!actualDesc.getDims().empty() &&
std::accumulate(actualDesc.getDims().begin(), actualDesc.getDims().end(), (size_t)1, std::multiplies<size_t>()) == 1);
}
if (out[name]->getTensorDesc().getDims() != intr_blob.getStaticDims() && !isScalarOutput) {
if (!node->isDynamicNode())
IE_THROW() << "Output blob and node dims mismatch for node with name: \"" << name << "\"";
out[name]->setShape(intr_blob.getStaticDims());
}
auto srcPrec = actualDesc.getPrecision();
auto dstPrec = expectedDesc.getPrecision();
if (srcPrec == dstPrec && ext_blob->second->byteSize() != intr_blob.GetSize())
IE_THROW() << "Output blob byte size is not equal network output byte size ("
<< ext_blob->second->byteSize() << "!=" << intr_blob.GetSize() << ").";
void *ext_blob_ptr = ext_blob->second->buffer();
void *intr_blob_ptr = intr_blob.GetData();
// That is the same memory. No need to copy
if (ext_blob_ptr == intr_blob_ptr) continue;
const auto &outDims = intr_blob.getStaticDims();
size_t size_to_copy = intr_blob.GetDescWithType<BlockedMemoryDesc>()->getPaddedElementsCount();
// TODO: Should we support InferenceEngine::PluginConfigParams::KEY_DYN_BATCH_LIMIT???
// TODO [DS]: phase 2: should we support this behaviour? Looks obsolete in the dynamic shapes paradigm
if (config.batchLimit) {
if (node->isDynamicNode()) {
IE_THROW(NotImplemented) << "[DS] not implemented dynamic batch for node with dynamic shape";
}
int MB_to_process = node->batchToProcess();
size_to_copy = std::accumulate(outDims.begin() + 1, outDims.end(), (size_t)1, std::multiplies<size_t>()) * MB_to_process;
}
if (actualDesc.getBlockingDesc() != expectedDesc.getBlockingDesc() && !isScalarOutput) {
auto outBlobDesc = MemoryDescUtils::convertToMKLDNNMemoryDesc(expectedDesc);
auto outBlobDesc = MemoryDescUtils::convertToDnnlBlockedMemoryDesc(expectedDesc);
auto outBloMem = MKLDNNMemory(eng);
outBloMem.Create(outBlobDesc, ext_blob_ptr, false);
@@ -816,7 +836,11 @@ void MKLDNNGraph::Infer(MKLDNNInferRequest* request, int batch) {
ENABLE_CPU_DEBUG_CAP(nd.dumpInputBlobs(node));
OV_ITT_SCOPED_TASK(itt::domains::MKLDNNPlugin, node->profiling.execute);
node->execute(stream);
if (node->isDynamicNode()) {
node->executeDynamic(stream);
} else {
node->execute(stream);
}
ENABLE_CPU_DEBUG_CAP(nd.dumpOutputBlobs(node));
}
@@ -1070,6 +1094,7 @@ void MKLDNNGraph::DropDWConvNode(const MKLDNNNodePtr &node) {
if (!parent) continue;
MKLDNNEdgePtr &remEdge = p_edge;
const auto portCandidate = remEdge->getOutputNum();
int inNum = 0;
if (remEdge) {
inNum = remEdge->getInputNum();
@@ -1081,8 +1106,9 @@ void MKLDNNGraph::DropDWConvNode(const MKLDNNNodePtr &node) {
MKLDNNEdgePtr newEdge(new MKLDNNEdge(parent, parentConv, inNum, outNum));
graphEdges.push_back(newEdge);
parent->addEdge(newEdge);
parentConv->inputShapes.push_back(Shape(newEdge->getShape()));
parentConv->inputShapes.push_back(node->getInputShapeAtPort(portCandidate));
}
parentConv->outputShapes[0] = node->getOutputShapeAtPort(0);
}
void MKLDNNGraph::RemoveDroppedNodes() {
inference-engine/src/mkldnn_plugin/mkldnn_graph.h
+23 -3
View File
@@ -57,7 +57,7 @@ public:
}
void PushInputData(const std::string& name, const InferenceEngine::Blob::Ptr &in);
void PullOutputData(const InferenceEngine::BlobMap &out);
void PullOutputData(InferenceEngine::BlobMap &out);
void Infer(MKLDNNInferRequest* request = nullptr, int batch = -1);
@@ -85,6 +85,20 @@ public:
return outputNodesMap;
}
MKLDNNNodePtr getInputNodeByName(const std::string &name) {
auto input = inputNodesMap.find(name);
if (input == inputNodesMap.end())
IE_THROW() << "CPU execution graph doesn't contain input node with name: " << name;
return input->second;
}
MKLDNNNodePtr getOutputNodeByName(const std::string &name) {
auto output = outputNodesMap.find(name);
if (output == outputNodesMap.end())
IE_THROW() << "CPU execution graph doesn't contain output node with name: " << name;
return output->second;
}
bool hasInputWithName(const std::string& name) const {
return inputNodesMap.count(name);
}
@@ -172,6 +186,10 @@ public:
return isQuantizedFlag;
}
bool hasDynamicInput() const {
return graphHasDynamicInput;
}
protected:
void VisitNode(MKLDNNNodePtr node, std::vector<MKLDNNNodePtr>& sortedNodes);
@@ -196,8 +214,6 @@ protected:
MKLDNNMemoryPtr memWorkspace;
std::map<std::string, MKLDNNNodePtr> inputNodesMap;
std::map<std::string, MKLDNNNodePtr> outputNodesMap;
std::vector<MKLDNNNodePtr> graphNodes;
std::vector<MKLDNNEdgePtr> graphEdges;
@@ -205,6 +221,7 @@ protected:
std::string _name;
bool isQuantizedFlag = false;
bool graphHasDynamicInput = false;
static mkldnn::engine eng;
@@ -226,6 +243,9 @@ protected:
friend std::shared_ptr<ngraph::Function> dump_graph_as_ie_ngraph_net(const MKLDNNGraph &graph);
private:
// TODO: change std::map to std::unordered_map
std::map<std::string, MKLDNNNodePtr> inputNodesMap;
std::map<std::string, MKLDNNNodePtr> outputNodesMap;
// these node pointers (from graphNodes) are to avoid regular checking for
// constant node in ExecuteConstantNodesOnly and Infer methods
std::vector<MKLDNNNodePtr> constantGraphNodes;
inference-engine/src/mkldnn_plugin/mkldnn_graph_dumper.cpp
+6 -6
View File
@@ -49,11 +49,11 @@ std::map<std::string, std::string> extract_node_metadata(const MKLDNNNodePtr &no
std::string outputPrecisionsStr;
if (!node->getChildEdges().empty()) {
outputPrecisionsStr = node->getChildEdgeAt(0)->getMemory().GetDesc().getPrecision().name();
outputPrecisionsStr = node->getChildEdgeAt(0)->getMemory().getDesc().getPrecision().name();
bool isAllEqual = true;
for (size_t i = 1; i < node->getChildEdges().size(); i++) {
if (node->getChildEdgeAt(i - 1)->getMemory().GetDesc().getPrecision() != node->getChildEdgeAt(i)->getMemory().GetDesc().getPrecision()) {
if (node->getChildEdgeAt(i - 1)->getMemory().getDesc().getPrecision() != node->getChildEdgeAt(i)->getMemory().getDesc().getPrecision()) {
isAllEqual = false;
break;
}
@@ -62,12 +62,12 @@ std::map<std::string, std::string> extract_node_metadata(const MKLDNNNodePtr &no
// If all output precisions are the same, we store the name only once
if (!isAllEqual) {
for (size_t i = 1; i < node->getChildEdges().size(); i++)
outputPrecisionsStr += "," + std::string(node->getChildEdgeAt(i)->getMemory().GetDesc().getPrecision().name());
outputPrecisionsStr += "," + std::string(node->getChildEdgeAt(i)->getMemory().getDesc().getPrecision().name());
}
} else {
// Branch to correctly handle output nodes
if (!node->getParentEdges().empty()) {
outputPrecisionsStr = node->getParentEdgeAt(0)->getMemory().GetDesc().getPrecision().name();
outputPrecisionsStr = node->getParentEdgeAt(0)->getMemory().getDesc().getPrecision().name();
}
}
serialization_info[ExecGraphInfoSerialization::OUTPUT_PRECISIONS] = outputPrecisionsStr;
@@ -164,7 +164,7 @@ std::shared_ptr<ngraph::Function> dump_graph_as_ie_ngraph_net(const MKLDNNGraph
auto meta_data = extract_node_metadata(node);
std::shared_ptr<ngraph::Node> return_node;
if (is_input) {
auto& desc = node->getChildEdgeAt(0)->getMemory().GetDesc();
auto& desc = node->getChildEdgeAt(0)->getMemory().getDesc();
auto param = std::make_shared<ngraph::op::Parameter>(details::convertPrecision(desc.getPrecision()), desc.getShape().toPartialShape());
return_node = param;
params.push_back(param);
@@ -176,7 +176,7 @@ std::shared_ptr<ngraph::Function> dump_graph_as_ie_ngraph_net(const MKLDNNGraph
get_inputs(node), node->getSelectedPrimitiveDescriptor()->getConfig().outConfs.size());
for (size_t port = 0; port < return_node->get_output_size(); ++port) {
auto& desc = node->getChildEdgeAt(port)->getMemory().GetDesc();
auto& desc = node->getChildEdgeAt(port)->getMemory().getDesc();
return_node->set_output_type(port, details::convertPrecision(desc.getPrecision()), desc.getShape().toPartialShape());
}
}
inference-engine/src/mkldnn_plugin/mkldnn_graph_optimizer.cpp
+102 -104
View File
@@ -44,7 +44,7 @@
#include <algorithm>
#include "mkldnn_itt.h"
#include "cpu_memory_desc_utils.h"
#include "memory_desc/cpu_memory_desc_utils.h"
using namespace mkldnn;
using namespace MKLDNNPlugin;
@@ -156,14 +156,14 @@ void MKLDNNGraphOptimizer::ApplyImplSpecificGraphOptimizations(MKLDNNGraph &grap
void MKLDNNGraphOptimizer::FuseConvolutionAndBias(MKLDNNGraph &graph) {
auto& graphNodes = graph.GetNodes();
auto isSutableParentNode = [](MKLDNNNodePtr node) {
auto isSuitableParentNode = [](MKLDNNNodePtr node) {
return node->getType() == Convolution &&
node->getChildEdges().size() == 1 &&
node->getParentEdges().size() == 2 &&
node->getFusedWith().empty();
};
auto isSutableChildNode = [&](MKLDNNNodePtr parentNode, MKLDNNNodePtr childNode) {
auto isSuitableChildNode = [&](MKLDNNNodePtr parentNode, MKLDNNNodePtr childNode) {
if (childNode->getAlgorithm() != EltwiseAdd || !childNode->getFusedWith().empty() || childNode->getParentEdges().size() != 2)
return false;
@@ -171,8 +171,8 @@ void MKLDNNGraphOptimizer::FuseConvolutionAndBias(MKLDNNGraph &graph) {
if (biasNode->getType() != Input || !biasNode->isConstant() || biasNode->getChildEdges().size() != 1)
return false;
auto convOutDims = parentNode->getChildEdgesAtPort(0)[0]->getShape().getDims();
auto biasDims = getNormalizedDimsBySize(biasNode->getChildEdgesAtPort(0)[0]->getShape().getDims(),
auto convOutDims = parentNode->getOutputShapeAtPort(0).getDims();
auto biasDims = getNormalizedDimsBySize(biasNode->getOutputShapeAtPort(0).getDims(),
convOutDims.size());
// TODO [NM]: Legacy ConvBias fusion transformation supports both per-tensor (via explicit broadcasing) and per-channel cases.
// Most of the real models contain per-channel bias, so we need to reavaluate the need to support per-tensor variant.
@@ -193,13 +193,13 @@ void MKLDNNGraphOptimizer::FuseConvolutionAndBias(MKLDNNGraph &graph) {
auto parent = graphNodes.begin();
while (parent != graphNodes.end()) {
auto parentNode = *parent;
if (!isSutableParentNode(parentNode)) {
if (!isSuitableParentNode(parentNode)) {
parent++;
continue;
}
auto childNode = parentNode->getChildEdgeAt(0)->getChild();
if (!isSutableChildNode(parentNode, childNode)) {
if (!isSuitableChildNode(parentNode, childNode)) {
parent++;
continue;
}
@@ -255,7 +255,7 @@ void MKLDNNGraphOptimizer::FuseConvolutionAndBias(MKLDNNGraph &graph) {
graphEdges.push_back(newEdge);
parent->addEdge(newEdge);
parent->outputShapes[inNum] = Shape(SizeVector{parentEltwise->outputShapes[0].getStaticDims()[1]});
parent->outputShapes[inNum] = Shape(VectorDims{parentEltwise->outputShapes[0].getStaticDims()[1]});
parentEltwise->inputShapes.push_back(parent->outputShapes[0]);
}
}
@@ -305,7 +305,7 @@ void MKLDNNGraphOptimizer::FuseDeconvolutionAndSimpleOperation(MKLDNNGraph &grap
void MKLDNNGraphOptimizer::FuseMultiplyAndAdd(MKLDNNGraph &graph) {
auto& graphNodes = graph.GetNodes();
auto isSutableSecondInput = [](MKLDNNNodePtr node, SizeVector dataDims) {
auto isSuitableSecondInput = [](MKLDNNNodePtr node, VectorDims dataDims) {
if (node->getType() != Input || !node->isConstant())
return false;
auto secondInputDims = node->outputShapes[0].getDims();
@@ -323,32 +323,32 @@ void MKLDNNGraphOptimizer::FuseMultiplyAndAdd(MKLDNNGraph &graph) {
return true;
};
auto isSutableParentNode = [&](MKLDNNNodePtr node) {
auto isSuitableParentNode = [&](MKLDNNNodePtr node) {
if (node->getAlgorithm() != EltwiseMultiply || !node->getFusedWith().empty() ||
node->getParentEdges().size() != 2 || node->getChildEdges().size() != 1)
return false;
return isSutableSecondInput(node->getParentEdgesAtPort(1)[0]->getParent(), node->getParentEdgesAtPort(0)[0]->getShape().getDims());
return isSuitableSecondInput(node->getParentEdgesAtPort(1)[0]->getParent(), node->getInputShapeAtPort(0).getDims());
};
auto isSutableChildNode = [&](MKLDNNNodePtr parentNode, MKLDNNNodePtr childNode) {
auto isSuitableChildNode = [&](MKLDNNNodePtr parentNode, MKLDNNNodePtr childNode) {
if (childNode->getAlgorithm() != EltwiseAdd || !childNode->getFusedWith().empty() || childNode->getParentEdges().size() != 2)
return false;
return isSutableSecondInput(childNode->getParentEdgesAtPort(1)[0]->getParent(), childNode->getParentEdgesAtPort(0)[0]->getShape().getDims()) &&
parentNode->canFuse(childNode);
return isSuitableSecondInput(childNode->getParentEdgesAtPort(1)[0]->getParent(), childNode->getInputShapeAtPort(0).getStaticDims()) &&
parentNode->canFuse(childNode);
};
auto parent = graphNodes.begin();
while (parent != graphNodes.end()) {
auto parentNode = *parent;
if (!isSutableParentNode(parentNode)) {
if (!isSuitableParentNode(parentNode)) {
parent++;
continue;
}
auto childNode = parentNode->getChildEdgeAt(0)->getChild();
if (!isSutableChildNode(parentNode, childNode)) {
if (!isSuitableChildNode(parentNode, childNode)) {
parent++;
continue;
}
@@ -419,11 +419,11 @@ void MKLDNNGraphOptimizer::FuseMultiplyAndAdd(MKLDNNGraph &graph) {
void MKLDNNGraphOptimizer::FuseConvolutionAndZeroPoints(MKLDNNGraph &graph) {
auto& graphNodes = graph.GetNodes();
auto isSutableConvNode = [](MKLDNNNodePtr node) {
auto isSuitableConvNode = [](MKLDNNNodePtr node) {
bool retVal = false;
if (node->getType() == Convolution) {
if (auto convNode = std::dynamic_pointer_cast<MKLDNNConvolutionNode>(node)) {
auto rank = convNode->getParentEdgeAt(0)->getShape().getRank();
auto rank = convNode->getInputShapeAtPort(0).getRank();
// int8 depthwise convolution does not support fusing zero points in 3D case
if (implication(convNode->isDepthWise(), rank == 4)) {
retVal = true;
@@ -438,8 +438,8 @@ void MKLDNNGraphOptimizer::FuseConvolutionAndZeroPoints(MKLDNNGraph &graph) {
if (convNode == nullptr)
IE_THROW() << "Cannot get convolution node " << node->getName();
int IC = node->getParentEdgesAtPort(0)[0]->getShape().getDims()[1];
int OC = node->getChildEdgesAtPort(0)[0]->getShape().getDims()[1];
int IC = node->getInputShapeAtPort(0).getDims()[1];
int OC = node->getOutputShapeAtPort(0).getDims()[1];
if (Shape::UNDEFINED_DIM == IC || Shape::UNDEFINED_DIM == OC) {
return false;
@@ -467,11 +467,11 @@ void MKLDNNGraphOptimizer::FuseConvolutionAndZeroPoints(MKLDNNGraph &graph) {
if (arg0->getOriginalOutputPrecisionAtPort(0) != Precision::U8)
return false;
if (parent0->getParentEdgesAtPort(1)[0]->getShape().getRank() < 2) {
if (parent0->getInputShapeAtPort(1).getRank() < 2) {
return false;
}
auto zpDims = parent0->getParentEdgesAtPort(1)[0]->getShape().getDims();
auto zpDims = parent0->getInputShapeAtPort(1).getDims();
if (zpDims[0] != 1 || !dimsEqualStrong(zpDims[1], IC))
return false;
@@ -496,7 +496,7 @@ void MKLDNNGraphOptimizer::FuseConvolutionAndZeroPoints(MKLDNNGraph &graph) {
if (zeroPointsData == nullptr)
IE_THROW() << "zeroPointsBlob has not allocated buffer";
auto zeroPointDataSize = parent0->getParentEdgesAtPort(1)[0]->getShape().getDims()[1];
auto zeroPointDataSize = parent0->getInputShapeAtPort(1).getDims()[1];
if (Shape::UNDEFINED_DIM == zeroPointDataSize) {
return false;
}
@@ -580,7 +580,7 @@ void MKLDNNGraphOptimizer::FuseConvolutionAndZeroPoints(MKLDNNGraph &graph) {
for (int i = 0; i < graphNodes.size(); i++) {
auto conv = graphNodes[i];
if (!isSutableConvNode(conv)) continue;
if (!isSuitableConvNode(conv)) continue;
auto dataEltwise = conv->getParentEdgesAtPort(0)[0]->getParent();
auto weightsEltwise = conv->getParentEdgesAtPort(1)[0]->getParent();
@@ -605,14 +605,14 @@ static bool BF16QuantizeNodeFusing(MKLDNNNodePtr parentNode, MKLDNNNodePtr child
void MKLDNNGraphOptimizer::FuseFullyConnectedAndSimpleOperation(MKLDNNGraph &graph) {
auto& graphNodes = graph.GetNodes();
auto isSutableParentNode = [](MKLDNNNodePtr node) {
return node->getType() == FullyConnected && node->getChildEdges().size() == 1 && node->getParentEdgeAt(0)->getShape().getRank() != 3;
auto isSuitableParentNode = [](MKLDNNNodePtr node) {
return node->getType() == FullyConnected && node->getChildEdges().size() == 1 && node->getInputShapeAtPort(0).getRank() != 3;
};
auto parent = graphNodes.begin();
while (parent != graphNodes.end()) {
auto parentNode = *parent;
if (!isSutableParentNode(parentNode)) {
if (!isSuitableParentNode(parentNode)) {
parent++;
continue;
}
@@ -658,7 +658,7 @@ void MKLDNNGraphOptimizer::FuseConvolutionAndDWConvolution(MKLDNNGraph &graph) {
return conv->getWeightDims()[weightRank - 1] == 1 && conv->getWeightDims()[weightRank - 2] == 1;
};
auto isSutableParentConvolution = [&](MKLDNNNodePtr node) {
auto isSuitableParentConvolution = [&](MKLDNNNodePtr node) {
if (node->isDropped())
return false;
@@ -671,8 +671,8 @@ void MKLDNNGraphOptimizer::FuseConvolutionAndDWConvolution(MKLDNNGraph &graph) {
const auto &strides = conv->getStride();
const auto &paddings = conv->getPaddingL();
const auto &inDims = node->getParentEdgeAt(0)->getShape().getDims();
const auto &outDims = node->getChildEdgeAt(0)->getShape().getDims();
const auto &inDims = node->getInputShapeAtPort(0).getDims();
const auto &outDims = node->getOutputShapeAtPort(0).getDims();
bool isSupportedParams = conv->getGroupNum() == 1 &&
inDims.size() == 4 &&
dimsEqualStrong(inDims[inDims.size() - 1], outDims[outDims.size() - 1]) &&
@@ -686,7 +686,7 @@ void MKLDNNGraphOptimizer::FuseConvolutionAndDWConvolution(MKLDNNGraph &graph) {
return node->getChildEdges().size() == 1 && isConvolutionNode(node->getChildEdgeAt(0)->getChild());
};
auto isSutableChildConvolution = [&](const MKLDNNNodePtr &parentNode, const MKLDNNNodePtr &childNode) {
auto isSuitableChildConvolution = [&](const MKLDNNNodePtr &parentNode, const MKLDNNNodePtr &childNode) {
if (parentNode->isDropped() || childNode->isDropped())
return false;
@@ -729,7 +729,7 @@ void MKLDNNGraphOptimizer::FuseConvolutionAndDWConvolution(MKLDNNGraph &graph) {
convChild->getStride()[stridesSize - 1] == convChild->getStride()[stridesSize - 2] &&
withBias &&
one_of(convChild->getStride()[stridesSize - 1], 1, 2) &&
childNode->getChildEdgeAt(0)->getShape().getRank() == 4;
childNode->getOutputShapeAtPort(0).getRank() == 4;
return isSupportedParams;
};
@@ -761,10 +761,10 @@ void MKLDNNGraphOptimizer::FuseConvolutionAndDWConvolution(MKLDNNGraph &graph) {
if (!isConvolutionNode(graphNodes[i])) continue;
auto parentConvNode = graphNodes[i];
if (!isSutableParentConvolution(parentConvNode)) continue;
if (!isSuitableParentConvolution(parentConvNode)) continue;
auto childConvNode = parentConvNode->getChildEdgeAt(0)->getChild();
if (!isSutableChildConvolution(parentConvNode, childConvNode)) continue;
if (!isSuitableChildConvolution(parentConvNode, childConvNode)) continue;
if (!isFusingWorthwhile(parentConvNode, childConvNode)) continue;
@@ -783,7 +783,7 @@ void MKLDNNGraphOptimizer::FuseConvolutionAndDWConvolution(MKLDNNGraph &graph) {
void MKLDNNGraphOptimizer::FuseConvolutionAndSimpleOperationThroughMaxPool(MKLDNNGraph &graph) {
auto& graphNodes = graph.GetNodes();
auto isSutableParentNode = [](MKLDNNNodePtr node) {
auto isSuitableParentNode = [](MKLDNNNodePtr node) {
return (node->getType() == Convolution || node->getType() == BinaryConvolution) && node->getChildEdges().size() == 1 &&
node->getOriginalOutputPrecisionAtPort(0) == Precision::FP32;
};
@@ -791,7 +791,7 @@ void MKLDNNGraphOptimizer::FuseConvolutionAndSimpleOperationThroughMaxPool(MKLDN
auto parent = graphNodes.begin();
while (parent != graphNodes.end()) {
auto parentNode = *parent;
if (!isSutableParentNode(parentNode)) {
if (!isSuitableParentNode(parentNode)) {
parent++;
continue;
}
@@ -831,14 +831,14 @@ void MKLDNNGraphOptimizer::FuseConvolutionAndSimpleOperationThroughMaxPool(MKLDN
void MKLDNNGraphOptimizer::FuseConvolutionAndSimpleOperation(MKLDNNGraph &graph) {
auto& graphNodes = graph.GetNodes();
auto isSutableParentNode = [](MKLDNNNodePtr node) {
auto isSuitableParentNode = [](MKLDNNNodePtr node) {
return (node->getType() == Convolution || node->getType() == BinaryConvolution) && node->getChildEdges().size() == 1;
};
auto parent = graphNodes.begin();
while (parent != graphNodes.end()) {
auto parentNode = *parent;
if (!isSutableParentNode(parentNode)) {
if (!isSuitableParentNode(parentNode)) {
parent++;
continue;
}
@@ -876,7 +876,7 @@ void MKLDNNGraphOptimizer::FuseConvolutionAndSimpleOperation(MKLDNNGraph &graph)
void MKLDNNGraphOptimizer::FusePoolingAndFakeQuantize(MKLDNNGraph &graph) {
auto& graphNodes = graph.GetNodes();
auto isSutableParentNode = [](MKLDNNNodePtr node) {
auto isSuitableParentNode = [](MKLDNNNodePtr node) {
if (node->getType() == Pooling) {
if (!one_of(node->getOriginalInputPrecisionAtPort(0), Precision::U8, Precision::I8))
return false;
@@ -885,16 +885,16 @@ void MKLDNNGraphOptimizer::FusePoolingAndFakeQuantize(MKLDNNGraph &graph) {
return false;
};
auto isSutableChildNode = [](MKLDNNNodePtr node) {
auto isSuitableChildNode = [](MKLDNNNodePtr node) {
return node->getType() == FakeQuantize && node->getAlgorithm() != Algorithm::FQBinarization;
};
for (int i = 0; i < graphNodes.size(); i++) {
auto parent = graphNodes[i];
if (!isSutableParentNode(parent)) continue;
if (!isSuitableParentNode(parent)) continue;
auto child = parent->getChildEdgeAt(0)->getChild();
if (!isSutableChildNode(child)) continue;
if (!isSuitableChildNode(child)) continue;
child->fuseInto(parent);
@@ -994,15 +994,15 @@ void MKLDNNGraphOptimizer::FuseConvolutionSumAndConvolutionSumActivation(MKLDNNG
if (std::dynamic_pointer_cast<MKLDNNEltwiseNode>(graphNode)->isWithBroadcast()) continue;
// TODO: Enlarge to several inputs
bool isSutableNode = graphNode->getParentEdges().size() == 2;
if (!isSutableNode)
bool isSuitableNode = graphNode->getParentEdges().size() == 2;
if (!isSuitableNode)
continue;
auto parent1 = graphNode->getParentEdgeAt(0)->getParent();
auto parent2 = graphNode->getParentEdgeAt(1)->getParent();
auto parent1 = graphNode->getParentEdgesAtPort(0)[0]->getParent();
auto parent2 = graphNode->getParentEdgesAtPort(1)[0]->getParent();
bool isSutableParent1 = parent1->getType() == Convolution || parent1->getType() == BinaryConvolution;
bool isSutableParent2 = parent2->getType() == Convolution || parent2->getType() == BinaryConvolution;
bool isSuitableParent1 = parent1->getType() == Convolution || parent1->getType() == BinaryConvolution;
bool isSuitableParent2 = parent2->getType() == Convolution || parent2->getType() == BinaryConvolution;
auto canFuseSum = [](MKLDNNBinaryConvolutionNode *binConv, MKLDNNNodePtr fuseCandidate) {
if (binConv->getImplType() == impl_desc_type::ref)
@@ -1025,34 +1025,34 @@ void MKLDNNGraphOptimizer::FuseConvolutionSumAndConvolutionSumActivation(MKLDNNG
auto* binConvNode1 = dynamic_cast<MKLDNNBinaryConvolutionNode *>(parent1.get());
if (binConvNode1) {
isSutableParent1 = isSutableParent1 && canFuseSum(binConvNode1, graphNode);
isSuitableParent1 = isSuitableParent1 && canFuseSum(binConvNode1, graphNode);
}
auto* binConvNode2 = dynamic_cast<MKLDNNBinaryConvolutionNode *>(parent2.get());
if (binConvNode2) {
isSutableParent2 = isSutableParent2 && canFuseSum(binConvNode2, graphNode);
isSuitableParent2 = isSuitableParent2 && canFuseSum(binConvNode2, graphNode);
}
auto* convNode1 = dynamic_cast<MKLDNNConvolutionNode *>(parent1.get());
if (convNode1) {
if (!convNode1->canBeExecutedInInt8()) {
isSutableParent1 = isSutableParent1 && convNode1->getFusedWith().empty();
isSuitableParent1 = isSuitableParent1 && convNode1->getFusedWith().empty();
}
}
auto* convNode2 = dynamic_cast<MKLDNNConvolutionNode *>(parent2.get());
if (convNode2) {
if (!convNode2->canBeExecutedInInt8()) {
isSutableParent2 = isSutableParent2 && convNode2->getFusedWith().empty();
isSuitableParent2 = isSuitableParent2 && convNode2->getFusedWith().empty();
}
}
if (!isSutableParent1 && !isSutableParent2)
if (!isSuitableParent1 && !isSuitableParent2)
continue;
auto mergedConv = isSutableParent1 ? parent1 : parent2;
auto peerNode = isSutableParent1 ? parent2 : parent1;
if (isSutableParent1 && isSutableParent2) {
auto mergedConv = isSuitableParent1 ? parent1 : parent2;
auto peerNode = isSuitableParent1 ? parent2 : parent1;
if (isSuitableParent1 && isSuitableParent2) {
if ((peerNode->getType() == Convolution || peerNode->getType() == BinaryConvolution) &&
mergedConv->getChildEdges().size() != 1) {
mergedConv = parent2;
@@ -1070,7 +1070,7 @@ void MKLDNNGraphOptimizer::FuseConvolutionSumAndConvolutionSumActivation(MKLDNNG
bool fuse_allowed = mergedConv->getChildEdges().size() == 1;
for (size_t j = 0; fuse_allowed && j < mergedConv->getParentEdges().size(); j++)
if (mergedConv->getParentEdgeAt(j)->getParent() == peerNode)
if (mergedConv->getParentEdgesAtPort(j)[0]->getParent() == peerNode)
fuse_allowed = false;
// Fused Conv+Sum prim will be used inplace. That's mean that input blob will
@@ -1154,14 +1154,14 @@ void MKLDNNGraphOptimizer::FuseConvolutionSumAndConvolutionSumActivation(MKLDNNG
void MKLDNNGraphOptimizer::FuseMVNAndSimpleOperation(MKLDNNGraph &graph) {
auto& graphNodes = graph.GetNodes();
auto isSutableParentNode = [](MKLDNNNodePtr node) {
auto isSuitableParentNode = [](MKLDNNNodePtr node) {
return (node->getType() == MVN) && (node->getChildEdges().size() == 1);
};
auto parent = graphNodes.begin();
while (parent != graphNodes.end()) {
auto parentNode = *parent;
if (!isSutableParentNode(parentNode)) {
if (!isSuitableParentNode(parentNode)) {
parent++;
continue;
}
@@ -1196,7 +1196,7 @@ void MKLDNNGraphOptimizer::FuseInterpolateAndSimpleOperation(MKLDNNGraph &graph)
return node->getType() == Interpolate && node->getChildEdges().size() == 1;
};
auto isSutableChildNode = [&](MKLDNNNodePtr parentNode, MKLDNNNodePtr childNode) {
auto isSuitableChildNode = [&](MKLDNNNodePtr parentNode, MKLDNNNodePtr childNode) {
// Avoid cycle dependencies
for (auto &childParentEdge : childNode->getParentEdges()) {
for (auto &parentParentEdge : parentNode->getParentEdges()) {
@@ -1219,7 +1219,7 @@ void MKLDNNGraphOptimizer::FuseInterpolateAndSimpleOperation(MKLDNNGraph &graph)
}
auto childNode = parentNode->getChildEdgeAt(0)->getChild();
if (!isSutableChildNode(parentNode, childNode)) {
if (!isSuitableChildNode(parentNode, childNode)) {
parent++;
continue;
}
@@ -1244,14 +1244,14 @@ void MKLDNNGraphOptimizer::FuseInterpolateAndSimpleOperation(MKLDNNGraph &graph)
void MKLDNNGraphOptimizer::FuseNormalizeL2AndSimpleOperation(MKLDNNGraph &graph) {
auto& graphNodes = graph.GetNodes();
auto isSutableParentNode = [](MKLDNNNodePtr node) {
auto isSuitableParentNode = [](MKLDNNNodePtr node) {
return node->getType() == NormalizeL2 && node->getChildEdges().size() == 1;
};
auto parent = graphNodes.begin();
while (parent != graphNodes.end()) {
auto parentNode = *parent;
if (!isSutableParentNode(parentNode)) {
if (!isSuitableParentNode(parentNode)) {
parent++;
continue;
}
@@ -1282,11 +1282,11 @@ void MKLDNNGraphOptimizer::FuseNormalizeL2AndSimpleOperation(MKLDNNGraph &graph)
void MKLDNNGraphOptimizer::FuseEltwiseAndSimple(MKLDNNGraph &graph) {
auto& graphNodes = graph.GetNodes();
auto isSutableParentNode = [](MKLDNNNodePtr node) {
auto isSuitableParentNode = [](MKLDNNNodePtr node) {
return node->getType() == Eltwise && node->getChildEdges().size() == 1;
};
auto isSutableChildNode = [&](MKLDNNNodePtr parentNode, MKLDNNNodePtr childNode) {
auto isSuitableChildNode = [&](MKLDNNNodePtr parentNode, MKLDNNNodePtr childNode) {
if (parentNode->isConstant() && !childNode->isConstant())
return false;
for (auto &childParentEdge : childNode->getParentEdges()) {
@@ -1312,13 +1312,13 @@ void MKLDNNGraphOptimizer::FuseEltwiseAndSimple(MKLDNNGraph &graph) {
auto parent = graphNodes.begin();
while (parent != graphNodes.end()) {
auto parentNode = *parent;
if (!isSutableParentNode(parentNode)) {
if (!isSuitableParentNode(parentNode)) {
parent++;
continue;
}
auto childNode = parentNode->getChildEdgeAt(0)->getChild();
if (!isSutableChildNode(parentNode, childNode)) {
if (!isSuitableChildNode(parentNode, childNode)) {
parent++;
continue;
}
@@ -1422,10 +1422,10 @@ void MKLDNNGraphOptimizer::DropDoubleReorders(MKLDNNGraph &graph) {
if (nn == nullptr)
IE_THROW() << "Cannot get reorder layer " << nextNode->getName();
MKLDNNNodePtr p = n->getParentEdgeAt(0)->getParent();
MKLDNNNodePtr c = nn->getChildEdgeAt(0)->getChild();
MKLDNNNodePtr p = n->getParentEdgesAtPort(0)[0]->getParent();
MKLDNNNodePtr c = nn->getChildEdgesAtPort(0)[0]->getChild();
auto oldEdgeNum = n->getParentEdgeAt(0)->getInputNum();
auto oldEdgeNum = n->getParentEdgesAtPort(0)[0]->getInputNum();
graph.DropNode(node);
graph.DropNode(nextNode);
@@ -1461,11 +1461,11 @@ void MKLDNNGraphOptimizer::FuseBroadcastAndEltwise(MKLDNNGraph &graph) {
MKLDNNNodePtr& broadcastNode = graphNode;
MKLDNNNodePtr eltwiseNode = broadcastNode->getChildEdgeAt(0)->getChild();
eltwiseNode->inputShapes[broadcastNode->getChildEdgeAt(0)->getOutputNum()]
= broadcastNode->getParentEdgeAt(0)->getShape();
= broadcastNode->getInputShapeAtPort(0);
auto& edges = graph.GetEdges();
for (size_t i = 1lu; i < broadcastNode->getParentEdges().size(); i++) {
auto constParent = broadcastNode->getParentEdgeAt(i)->getParent();
auto constParent = broadcastNode->getParentEdgesAtPort(i)[0]->getParent();
for (auto it = edges.begin(); it != edges.end(); it++) {
if ((*it) == constParent->getChildEdgeAt(0)) {
edges.erase(it);
@@ -1481,11 +1481,11 @@ void MKLDNNGraphOptimizer::FuseBroadcastAndEltwise(MKLDNNGraph &graph) {
void MKLDNNGraphOptimizer::FuseClampAndFakeQuantize(MKLDNNGraph &graph) {
auto& graphNodes = graph.GetNodes();
auto isSutableClampNode = [](MKLDNNNodePtr node) {
auto isSuitableClampNode = [](MKLDNNNodePtr node) {
return node->getType() == Eltwise && node->getChildEdges().size() == 1 && node->getAlgorithm() == EltwiseClamp;
};
auto isSutableFakeQuantizeNode = [](MKLDNNNodePtr node) {
auto isSuitableFakeQuantizeNode = [](MKLDNNNodePtr node) {
return node->getType() == FakeQuantize && node->getAlgorithm() != FQBinarization;
};
@@ -1516,10 +1516,10 @@ void MKLDNNGraphOptimizer::FuseClampAndFakeQuantize(MKLDNNGraph &graph) {
for (int i = 0; i < graphNodes.size(); i++) {
auto parent = graphNodes[i];
if (!isSutableClampNode(parent)) continue;
if (!isSuitableClampNode(parent)) continue;
auto child = parent->getChildEdgeAt(0)->getChild();
if (!isSutableFakeQuantizeNode(child)) continue;
if (!isSuitableFakeQuantizeNode(child)) continue;
if (fuseClampAndFakeQuantizeNodes(parent, child)) {
graph.DropNode(parent);
@@ -1531,16 +1531,16 @@ void MKLDNNGraphOptimizer::FusePerformedAsScaleShiftAndFakeQuantize(MKLDNNGraph
auto& graphNodes = graph.GetNodes();
auto getConstPort = [](const MKLDNNNodePtr node) -> int {
if (node->getParentEdgeAt(0)->getParent()->getType() == Input && node->getParentEdgeAt(0)->getParent()->isConstant()) {
if (node->getParentEdgesAtPort(0)[0]->getParent()->getType() == Input && node->getParentEdgesAtPort(0)[0]->getParent()->isConstant()) {
return 0;
} else if (node->getParentEdgeAt(1)->getParent()->getType() == Input && node->getParentEdgeAt(1)->getParent()->isConstant()) {
} else if (node->getParentEdgesAtPort(1)[0]->getParent()->getType() == Input && node->getParentEdgesAtPort(1)[0]->getParent()->isConstant()) {
return 1;
} else {
return -1;
}
};
auto isSutableScaleShiftNode = [getConstPort](MKLDNNNodePtr node) {
auto isSuitableScaleShiftNode = [getConstPort](MKLDNNNodePtr node) {
if (one_of(node->getAlgorithm(), EltwiseAdd, EltwiseSubtract, EltwiseMultiply, EltwiseDivide, EltwiseMulAdd)) {
MKLDNNNode *parent = nullptr;
if (node->getAlgorithm() != EltwiseMulAdd) {
@@ -1548,14 +1548,14 @@ void MKLDNNGraphOptimizer::FusePerformedAsScaleShiftAndFakeQuantize(MKLDNNGraph
if (constPort == -1) {
return false;
}
parent = node->getParentEdgeAt(1 - constPort)->getParent().get();
parent = node->getParentEdgesAtPort(1 - constPort)[0]->getParent().get();
}
return node->getType() == Eltwise && node->getChildEdges().size() == 1 && node->canBePerformedAsScaleShift(parent);
}
return false;
};
auto isSutableFakeQuantizeNode = [](MKLDNNNodePtr node) {
auto isSuitableFakeQuantizeNode = [](MKLDNNNodePtr node) {
return node->getType() == FakeQuantize && node->getAlgorithm() != FQBinarization;
};
@@ -1566,7 +1566,7 @@ void MKLDNNGraphOptimizer::FusePerformedAsScaleShiftAndFakeQuantize(MKLDNNGraph
std::vector<float> scalesBuffer;
std::vector<float> shiftsBuffer;
parent->fillScalesAndShifts(parent->getParentEdgeAt(1 - getConstPort(parent))->getParent().get(), scalesBuffer, shiftsBuffer, 1);
parent->fillScalesAndShifts(parent->getParentEdgesAtPort(1 - getConstPort(parent))[0]->getParent().get(), scalesBuffer, shiftsBuffer, 1);
for (int i = 0; i < scalesBuffer.size(); i++)
if (scalesBuffer[i] == 0.f)
@@ -1644,10 +1644,10 @@ void MKLDNNGraphOptimizer::FusePerformedAsScaleShiftAndFakeQuantize(MKLDNNGraph
for (int i = 0; i < graphNodes.size(); i++) {
auto parent = graphNodes[i];
if (!isSutableScaleShiftNode(parent)) continue;
if (!isSuitableScaleShiftNode(parent)) continue;
auto child = parent->getChildEdgeAt(0)->getChild();
if (!isSutableFakeQuantizeNode(child)) continue;
if (!isSuitableFakeQuantizeNode(child)) continue;
if (fuseScaleShiftAndFakeQuantizeNodes(parent, child)) {
auto parentEdges = parent->parentEdges;
@@ -1667,11 +1667,11 @@ void MKLDNNGraphOptimizer::FusePerformedAsScaleShiftAndFakeQuantize(MKLDNNGraph
void MKLDNNGraphOptimizer::MergeTransposeAndReorder(MKLDNNGraph &graph) {
auto& graphNodes = graph.GetNodes();
auto isSutableParentNode = [](MKLDNNNodePtr node) {
auto isSuitableParentNode = [](MKLDNNNodePtr node) {
return node->getType() == Transpose && node->getChildEdges().size() == 1;
};
auto isSutableChildNode = [](MKLDNNNodePtr node) {
auto isSuitableChildNode = [](MKLDNNNodePtr node) {
return node->getType() == Reorder && node->getChildEdges().size() == 1;
};
@@ -1685,33 +1685,32 @@ void MKLDNNGraphOptimizer::MergeTransposeAndReorder(MKLDNNGraph &graph) {
}
auto& transposeOrder = transposeNode->getOrder();
auto layoutOrder = MemoryDescUtils::convertToBlockedDescriptor(
*transposeNode->getSelectedPrimitiveDescriptor()->getConfig().outConfs[0].desc).getOrder();
auto layoutOrder = transposeNode->getSelectedPrimitiveDescriptor()->getConfig().outConfs[0].desc->as<BlockedMemoryDesc>()->getOrder();
auto inBlockedDesc = MemoryDescUtils::convertToBlockedDescriptor(*reorderNode->getSelectedPrimitiveDescriptor()->getConfig().inConfs[0].desc);
auto outBlockedDesc = MemoryDescUtils::convertToBlockedDescriptor(*reorderNode->getSelectedPrimitiveDescriptor()->getConfig().outConfs[0].desc);
auto inBlockedDesc = reorderNode->getSelectedPrimitiveDescriptor()->getConfig().inConfs[0].desc->as<BlockedMemoryDesc>();
auto outBlockedDesc = reorderNode->getSelectedPrimitiveDescriptor()->getConfig().outConfs[0].desc->as<BlockedMemoryDesc>();
auto& inOrder = inBlockedDesc.getOrder();
auto& outOrder = outBlockedDesc.getOrder();
auto& inOrder = inBlockedDesc->getOrder();
auto& outOrder = outBlockedDesc->getOrder();
if (transposeOrder.size() != layoutOrder.size() || layoutOrder.size() != inOrder.size() || inOrder.size() != outOrder.size()) {
return false;
}
// revLayoutOrder - reverse permutation for layoutOrder
auto revLayoutOrder = SizeVector(layoutOrder.size());
auto revLayoutOrder = VectorDims(layoutOrder.size());
for (int i = 0; i < revLayoutOrder.size(); i++) {
revLayoutOrder[layoutOrder[i]] = i;
}
// newTransposeOrder - Transpose layout-aware permutation
auto newTransposeOrder = SizeVector(transposeOrder.size());
auto newTransposeOrder = VectorDims(transposeOrder.size());
for (int i = 0; i < newTransposeOrder.size(); i++) {
newTransposeOrder[i] = layoutOrder[transposeOrder[revLayoutOrder[i]]];
}
// reorderOrder - Reorder layout-aware permutation
auto reorderOrder = SizeVector(outOrder.size());
auto reorderOrder = VectorDims(outOrder.size());
for (int i = 0; i < reorderOrder.size(); i++) {
for (int j = 0; j < reorderOrder.size(); j++) {
if (outOrder[i] == inOrder[j]) {
@@ -1722,7 +1721,7 @@ void MKLDNNGraphOptimizer::MergeTransposeAndReorder(MKLDNNGraph &graph) {
}
// summaryOrder - resulting Transpose+Reorder permutation
auto summaryOrder = SizeVector(transposeOrder.size());
auto summaryOrder = VectorDims(transposeOrder.size());
for (int i = 0; i < summaryOrder.size(); i++) {
summaryOrder[i] = reorderOrder[newTransposeOrder[i]];
}
@@ -1774,9 +1773,8 @@ void MKLDNNGraphOptimizer::MergeTransposeAndReorder(MKLDNNGraph &graph) {
auto inPrec = inDesc->getPrecision();
auto outPrec = outDesc->getPrecision();
auto reorderInDesc = inDesc->clone();
auto reorderOutDesc = outDesc->clone();
reorderOutDesc->setPrecision(inPrec);
auto reorderInDesc = inDesc;
auto reorderOutDesc = MemoryDescUtils::cloneWithNewPrecision(*outDesc, inPrec);
std::string reorderlayerName = parentParentNode->getName() + "_" +
MKLDNNReorderNode::getReorderArgs(*reorderInDesc, *reorderOutDesc) + "_" + "fake";
@@ -1796,8 +1794,8 @@ void MKLDNNGraphOptimizer::MergeTransposeAndReorder(MKLDNNGraph &graph) {
// case 2
if (inPrec != outPrec) {
auto reorderInDesc2 = reorderOutDesc->clone();
auto reorderOutDesc2 = outDesc->clone();
auto reorderInDesc2 = reorderOutDesc;
auto reorderOutDesc2 = outDesc;
std::string reorderLayerName2 = reorderNode->getName() + "_" +
MKLDNNReorderNode::getReorderArgs(*reorderInDesc2, *reorderOutDesc2) + "_" + childChildNode->getName();
@@ -1808,11 +1806,11 @@ void MKLDNNGraphOptimizer::MergeTransposeAndReorder(MKLDNNGraph &graph) {
for (int i = 0; i < graphNodes.size(); i++) {
auto parentNode = graphNodes[i];
if (!isSutableParentNode(parentNode)) {
if (!isSuitableParentNode(parentNode)) {
continue;
}
auto childNode = parentNode->getChildEdgeAt(0)->getChild();
if (!isSutableChildNode(childNode)) {
if (!isSuitableChildNode(childNode)) {
continue;
}
@@ -1860,4 +1858,4 @@ void MKLDNNGraphOptimizer::reshapeRnnSeq(MKLDNNGraph &graph) {
graph.RemoveEdge(edge);
}
}
}
}
inference-engine/src/mkldnn_plugin/mkldnn_infer_request.cpp
+111 -85
View File
@@ -22,6 +22,7 @@
#include <debug.h>
#include "utils/general_utils.h"
#include "utils/cpu_utils.hpp"
#include "memory_desc/dnnl_blocked_memory_desc.h"
MKLDNNPlugin::MKLDNNInferRequest::MKLDNNInferRequest(InferenceEngine::InputsDataMap networkInputs,
InferenceEngine::OutputsDataMap networkOutputs,
@@ -35,11 +36,11 @@ MKLDNNPlugin::MKLDNNInferRequest::MKLDNNInferRequest(InferenceEngine::InputsData
IE_THROW() << "No graph was found";
graph = &(execNetwork->GetGraph()._graph);
// Allocate all input blobs
// Allocate all input blobs if shape is static, delay allocation otherwise
for (const auto& it : _networkInputs) {
MKLDNNInferRequest::GetBlob(it.first);
}
// Allocate all output blobs
// Allocate all output blobs if shape is static, delay allocation otherwise
for (const auto& it : _networkOutputs) {
MKLDNNInferRequest::GetBlob(it.first);
}
@@ -164,6 +165,18 @@ void MKLDNNPlugin::MKLDNNInferRequest::PullStates() {
}
}
void MKLDNNPlugin::MKLDNNInferRequest::redefineMemoryForInputNodes() {
const auto cpuInputNodes = graph->GetInputNodesMap();
for (const auto &blob : _inputs) {
const auto inputNode = cpuInputNodes.find(blob.first);
if (inputNode == cpuInputNodes.end()) {
IE_THROW() << "CPU execution graph doesn't contain input node with name: " << blob.first;
}
if (inputNode->second->isDynamicNode())
inputNode->second->redefineOutputMemory({blob.second->getTensorDesc().getDims()});
}
}
void MKLDNNPlugin::MKLDNNInferRequest::InferImpl() {
using namespace openvino::itt;
@@ -173,6 +186,9 @@ void MKLDNNPlugin::MKLDNNInferRequest::InferImpl() {
ThrowIfCanceled();
if (graph->hasDynamicInput())
redefineMemoryForInputNodes();
execDataPreprocessing(_inputs);
changeDefaultPtr();
@@ -221,30 +237,25 @@ InferenceEngine::Blob::Ptr MKLDNNPlugin::MKLDNNInferRequest::GetBlob(const std::
}
if (_inputs.find(name) == _inputs.end()) {
auto pBlob = graph->getInputBlob(name);
if (!pBlob) {
IE_THROW() << "MKLDNN graph doesn't contain input node with name: " << name;
}
InferenceEngine::TensorDesc desc = pBlob->getTensorDesc();
if (_networkInputs.find(name) != _networkInputs.end()) {
InferenceEngine::Layout l = _networkInputs[name]->getLayout();
InferenceEngine::Precision p = _networkInputs[name]->getPrecision();
InferenceEngine::SizeVector dims = _networkInputs[name]->getTensorDesc().getDims();
InferenceEngine::TensorDesc desc = _networkInputs[name]->getTensorDesc();
bool isDynamic = _networkInputs[name]->getInputData()->isDynamic();
desc = InferenceEngine::TensorDesc(p, dims, l);
}
_inputs[name] = make_blob_with_precision(desc);
_inputs[name]->allocate();
_inputs[name] = make_blob_with_precision(desc);
_inputs[name]->allocate();
if (pBlob->getTensorDesc() == desc &&
graph->_normalizePreprocMap.find(name) == graph->_normalizePreprocMap.end() && !graph->getProperty().batchLimit) {
externalPtr[name] = _inputs[name]->buffer();
if (!isDynamic &&
desc == MemoryDescUtils::convertToTensorDesc(graph->getInputNodeByName(name)->getChildEdgesAtPort(0)[0]->getMemory().getDesc()) &&
graph->_normalizePreprocMap.find(name) == graph->_normalizePreprocMap.end() && !graph->getProperty().batchLimit) {
externalPtr[name] = _inputs[name]->buffer();
}
} else {
IE_THROW() << "Blob with name: " << name << " exists in MKLDNN graph, but absents in network inputs";
}
}
data = _inputs[name];
checkBlob(data, name, true);
// check if preprocess required, but still wasn't set
auto preProcessedInput = std::find_if(std::begin(_networkInputs), std::end(_networkInputs),
[&](const std::pair<std::string, InferenceEngine::InputInfo::Ptr>& pair) {
@@ -263,52 +274,60 @@ InferenceEngine::Blob::Ptr MKLDNNPlugin::MKLDNNInferRequest::GetBlob(const std::
}
if (graph->hasOutputWithName(name)) {
const auto outNode = graph->getOutputNodeByName(name);
if (_outputs.find(name) == _outputs.end()) {
auto pBlob = graph->getOutputBlob(name);
if (!pBlob) {
IE_THROW() << "MKLDNN graph doesn't contain output node with name: " << name;
}
if (_networkOutputs.find(name) != _networkOutputs.end()) {
bool isDynamic = outNode->isDynamicNode();
const auto &desc = outNode->getParentEdgesAtPort(0)[0]->getMemory().getDesc();
if (!data) {
InferenceEngine::TensorDesc desc = _networkOutputs[name]->getTensorDesc();
desc.setPrecision(normalizeToSupportedPrecision(desc.getPrecision()));
if (!data) {
InferenceEngine::TensorDesc desc = _networkOutputs[name]->getTensorDesc();
desc.setPrecision(normalizeToSupportedPrecision(desc.getPrecision()));
// WA: need to avoid exception thrown when we compare blocking desc in SetBlob
// in situation if we push output blobs as inputs for next network (in Hetero plugin)
// it may be that output tensor desc will be different from real input tensor desc for next network
// because the optimal descriptor was chosen (e.g. inPlace case for Split node)
auto currBlockDesc = InferenceEngine::BlockingDesc(desc.getBlockingDesc().getBlockDims(), desc.getBlockingDesc().getOrder());
desc = InferenceEngine::TensorDesc(desc.getPrecision(), desc.getDims(), currBlockDesc);
data = make_blob_with_precision(desc);
data->allocate();
} else {
const auto& expectedTensorDesc = isDynamic ? InferenceEngine::TensorDesc(desc.getPrecision(),
InferenceEngine::TensorDesc::getLayoutByRank(desc.getShape().getRank()))
: MemoryDescUtils::convertToTensorDesc(desc);
const auto &tensorDesc = data->getTensorDesc();
if (expectedTensorDesc.getPrecision() != tensorDesc.getPrecision()) {
IE_THROW(ParameterMismatch) << "Network input and output use the same name: " << name << " but expect blobs with different precision: "
<< tensorDesc.getPrecision() << " for input and " << expectedTensorDesc.getPrecision()
<< " for output.";
}
data = make_blob_with_precision(desc);
data->allocate();
if (expectedTensorDesc.getDims() != tensorDesc.getDims()) {
IE_THROW(ParameterMismatch) << "Network input and output use the same name: " << name << " but expect blobs with different shapes.";
}
if (tensorDesc.getLayout() != InferenceEngine::Layout::ANY && expectedTensorDesc.getLayout() != InferenceEngine::Layout::ANY) {
if (tensorDesc.getLayout() != expectedTensorDesc.getLayout() && !(tensorDesc.getLayout() == InferenceEngine::Layout::BLOCKED &&
InferenceEngine::TensorDesc(tensorDesc.getPrecision(), tensorDesc.getDims(), tensorDesc.getBlockingDesc()).getLayout() ==
expectedTensorDesc.getLayout())) {
IE_THROW(ParameterMismatch) << "Network input and output use the same name: " << name << " but expect blobs" <<
" with different layouts.";
}
if (expectedTensorDesc.getBlockingDesc() != tensorDesc.getBlockingDesc())
IE_THROW(ParameterMismatch) << "Network input and output use the same name: " << name
<< " but expect blobs with different blocking descriptors.";
}
}
_outputs[name] = data;
if (!isDynamic && !externalPtr.count(name) && data->getTensorDesc() == MemoryDescUtils::convertToTensorDesc(desc) &&
!graph->getProperty().batchLimit) {
externalPtr[name] = data->buffer();
}
} else {
const auto& expectedTensorDesc = pBlob->getTensorDesc();
if (expectedTensorDesc.getPrecision() != data->getTensorDesc().getPrecision()) {
IE_THROW(ParameterMismatch) << "Network input and output use the same name: " << name << " but expect blobs with different precision: "
<< data->getTensorDesc().getPrecision() << " for input and " << expectedTensorDesc.getPrecision()
<< " for output.";
}
if (expectedTensorDesc.getDims() != data->getTensorDesc().getDims()) {
IE_THROW(ParameterMismatch) << "Network input and output use the same name: " << name << " but expect blobs with different shapes.";
}
if (data->getTensorDesc().getLayout() != InferenceEngine::Layout::ANY && expectedTensorDesc.getLayout() != InferenceEngine::Layout::ANY &&
expectedTensorDesc.getBlockingDesc() != data->getTensorDesc().getBlockingDesc()) {
IE_THROW(ParameterMismatch) << "Network input and output use the same name: " << name
<< " but expect blobs with different blocking descriptors.";
}
}
_outputs[name] = data;
if (!externalPtr.count(name) && data->getTensorDesc() == pBlob->getTensorDesc() && !graph->getProperty().batchLimit) {
externalPtr[name] = data->buffer();
IE_THROW() << "Blob with name: " << name << " exists in MKLDNN graph, but absents in network outputs";
}
}
data = _outputs[name];
checkBlob(data, name, false);
if (!outNode->isDynamicNode())
checkBlob(data, name, false);
}
if (!data) {
IE_THROW() << "Cannot find blob with name: " << name;
@@ -335,11 +354,12 @@ void MKLDNNPlugin::MKLDNNInferRequest::SetBlob(const std::string& name, const In
InferenceEngine::DataPtr foundOutput;
size_t dataSize = data->size();
findInputAndOutputBlobByName(name, foundInput, foundOutput);
const auto &blobDesc = data->getTensorDesc();
if (foundInput) {
if (foundInput->getPrecision() != data->getTensorDesc().getPrecision()) {
if (foundInput->getPrecision() != blobDesc.getPrecision()) {
IE_THROW(ParameterMismatch) << "Failed to set input blob with precision: "
<< data->getTensorDesc().getPrecision() << ", if CNNNetwork input blob precision is: " << foundInput->getPrecision();
<< blobDesc.getPrecision() << ", if CNNNetwork input blob precision is: " << foundInput->getPrecision();
}
const bool preProcRequired = preProcessingRequired(foundInput, data);
@@ -360,26 +380,29 @@ void MKLDNNPlugin::MKLDNNInferRequest::SetBlob(const std::string& name, const In
size_t inputSize = foundInput->getTensorDesc().getLayout() != InferenceEngine::Layout::SCALAR
? InferenceEngine::details::product(foundInput->getTensorDesc().getDims())
: 1;
if (dataSize != inputSize) {
const bool isDynamic = foundInput->getInputData()->isDynamic();
if (!isDynamic && dataSize != inputSize) {
IE_THROW() << "Input blob size is not equal network input size ("
<< dataSize << "!=" << inputSize << ").";
}
if (foundInput->getTensorDesc().getDims() != data->getTensorDesc().getDims()) {
if (!isDynamic && foundInput->getTensorDesc().getDims() != blobDesc.getDims()) {
IE_THROW(ParameterMismatch) << "Failed to set input blob. Dimensions mismatch.";
}
if (data->getTensorDesc().getLayout() != InferenceEngine::Layout::ANY && foundInput->getTensorDesc().getLayout() != InferenceEngine::Layout::ANY &&
foundInput->getTensorDesc().getBlockingDesc() != data->getTensorDesc().getBlockingDesc()) {
IE_THROW(ParameterMismatch) << "Failed to set input blob. Blocking descriptor mismatch.";
if (blobDesc.getLayout() != InferenceEngine::Layout::ANY && foundInput->getTensorDesc().getLayout() != InferenceEngine::Layout::ANY) {
if (isDynamic && InferenceEngine::TensorDesc(foundInput->getPrecision(), blobDesc.getDims(), foundInput->getLayout()).getBlockingDesc() !=
blobDesc.getBlockingDesc())
IE_THROW(ParameterMismatch) << "Failed to set input blob. Layouts mismatch.";
if (!isDynamic && foundInput->getTensorDesc().getBlockingDesc() != blobDesc.getBlockingDesc())
IE_THROW(ParameterMismatch) << "Failed to set input blob. Blocking descriptor mismatch.";
}
auto pBlob = graph->getInputBlob(name);
if (!pBlob) {
IE_THROW() << "MKLDNN graph doesn't contain input node with name: " << name;
}
if (data->getTensorDesc() == pBlob->getTensorDesc() &&
const auto &actualDesc = graph->getInputNodeByName(name)->getChildEdgesAtPort(0)[0]->getMemory().getDesc();
if (blobDesc.getLayout() != InferenceEngine::Layout::ANY &&
actualDesc.isCompatible(MemoryDescUtils::convertToCpuBlockedMemoryDesc(blobDesc)) &&
graph->_normalizePreprocMap.find(name) == graph->_normalizePreprocMap.end() && !graph->getProperty().batchLimit) {
externalPtr[name] = data->buffer();
} else if (externalPtr.find(name) != externalPtr.end()) {
@@ -393,31 +416,34 @@ void MKLDNNPlugin::MKLDNNInferRequest::SetBlob(const std::string& name, const In
IE_THROW(NotImplemented)
<< "cannot set compound blob: supported only for input pre-processing";
}
if (foundOutput->getPrecision() != data->getTensorDesc().getPrecision()) {
if (foundOutput->getPrecision() != blobDesc.getPrecision()) {
IE_THROW(ParameterMismatch) << "Failed to set output blob with precision: "
<< data->getTensorDesc().getPrecision() << ", if CNNNetwork output blob precision is: " << foundOutput->getPrecision();
<< blobDesc.getPrecision() << ", if CNNNetwork output blob precision is: " << foundOutput->getPrecision();
}
size_t outputSize = foundOutput->getTensorDesc().getLayout() != InferenceEngine::Layout::SCALAR
? InferenceEngine::details::product(foundOutput->getDims())
: 1;
if (dataSize != outputSize) {
const bool isDynamic = foundOutput->isDynamic();
if (!isDynamic && dataSize != outputSize) {
IE_THROW() << "Output blob size is not equal network output size ("
<< dataSize << "!=" << outputSize << ").";
}
if (foundOutput->getTensorDesc().getDims() != data->getTensorDesc().getDims()) {
if (!isDynamic && foundOutput->getTensorDesc().getDims() != blobDesc.getDims()) {
IE_THROW(ParameterMismatch) << "Failed to set output Blob. Dimensions mismatch.";
}
if (data->getTensorDesc().getLayout() != InferenceEngine::Layout::ANY && foundOutput->getTensorDesc().getLayout() != InferenceEngine::Layout::ANY &&
foundOutput->getTensorDesc().getBlockingDesc() != data->getTensorDesc().getBlockingDesc()) {
if (blobDesc.getLayout() != InferenceEngine::Layout::ANY && foundOutput->getTensorDesc().getLayout() != InferenceEngine::Layout::ANY) {
if (isDynamic && InferenceEngine::TensorDesc(foundOutput->getPrecision(), blobDesc.getDims(), foundOutput->getLayout()).getBlockingDesc() !=
blobDesc.getBlockingDesc())
IE_THROW(ParameterMismatch) << "Failed to set input blob. Layouts mismatch.";
if (!isDynamic && foundOutput->getTensorDesc().getBlockingDesc() != blobDesc.getBlockingDesc())
IE_THROW(ParameterMismatch) << "Failed to set output blob. Blocking descriptor mismatch.";
}
auto pBlob = graph->getOutputBlob(name);
if (!pBlob)
IE_THROW() << "MKLDNN graph doesn't contain output node with name: " << name;
if (data->getTensorDesc() == pBlob->getTensorDesc() &&
!graph->getProperty().batchLimit) {
const auto &desc = graph->getOutputNodeByName(name)->getParentEdgesAtPort(0)[0]->getMemory().getDesc();
if (!isDynamic && blobDesc == MemoryDescUtils::convertToTensorDesc(desc) && !graph->getProperty().batchLimit) {
externalPtr[name] = data->buffer();
} else if (externalPtr.find(name) != externalPtr.end()) {
externalPtr.erase(name);
@@ -432,8 +458,8 @@ static inline void changeEdgePtr(const MKLDNNPlugin::MKLDNNEdgePtr &edge, void *
void MKLDNNPlugin::MKLDNNInferRequest::changeDefaultPtr() {
for (auto& it : externalPtr) {
auto input = graph->inputNodesMap.find(it.first);
if (input != graph->inputNodesMap.end()) {
auto input = graph->GetInputNodesMap().find(it.first);
if (input != graph->GetInputNodesMap().end()) {
if (input->second->getChildEdgeAt(0)->getMemory().GetPrimitive().get_data_handle() == it.second)
continue;
// Input cannot be in-place with other primitives
@@ -467,7 +493,7 @@ void MKLDNNPlugin::MKLDNNInferRequest::changeDefaultPtr() {
}
MKLDNNNodePtr output;
for (auto& out : graph->outputNodesMap) {
for (auto& out : graph->GetOutputNodesMap()) {
if (out.first == it.first) {
output = out.second;
break;
inference-engine/src/mkldnn_plugin/mkldnn_infer_request.h
+1
View File
@@ -51,6 +51,7 @@ private:
void PushInputData();
void PushStates();
void PullStates();
void redefineMemoryForInputNodes();
void pushInput(const std::string& inputName, InferenceEngine::Blob::Ptr& inputBlob, InferenceEngine::Precision dataType);
inference-engine/src/mkldnn_plugin/mkldnn_memory.cpp
+51 -790
View File
@@ -18,8 +18,10 @@
#include "nodes/common/cpu_convert.h"
#include "mkldnn/ie_mkldnn.h"
#include "cpu_shape.h"
#include "cpu_memory_desc_utils.h"
#include "mkldnn_extension_utils.h"
#include "memory_desc/dnnl_blocked_memory_desc.h"
#include "utils/cpu_utils.hpp"
#include "nodes/mkldnn_reorder_node.h"
#include "memory_desc/cpu_memory_desc.h"
using namespace InferenceEngine;
using namespace mkldnn;
@@ -39,15 +41,11 @@ namespace {
MKLDNNMemory::MKLDNNMemory(const mkldnn::engine& eng) : eng(eng) {}
size_t MKLDNNMemory::GetSize() const {
uint8_t itemSize = MKLDNNExtensionUtils::sizeOfDataType(mkldnn::memory::data_type(GetDataType()));
return GetElementsCount() * itemSize;
}
size_t MKLDNNMemory::GetElementsCount() const {
auto desc = GetDescriptor();
std::vector<int> dims(desc.data.padded_dims,
desc.data.padded_dims + desc.data.ndims);
return std::accumulate(std::begin(dims), std::end(dims), (size_t) 1, std::multiplies<size_t>());
auto size = getDesc().getCurrentMemSize();
if (size == MemoryDesc::UNDEFINED_SIZE) {
IE_THROW() << "Can't get memory size for undefined shape";
}
return size;
}
void MKLDNNMemory::Create(const memory::dims& dims, memory::data_type data_type, memory::format_tag format, const void* data) {
@@ -55,7 +53,7 @@ void MKLDNNMemory::Create(const memory::dims& dims, memory::data_type data_type,
format = memory::format_tag::any;
}
memory::desc desc = MKLDNNMemoryDesc(MKLDNNExtensionUtils::convertToSizeVector(dims), data_type, format);
memory::desc desc = mkldnn::memory::desc(dims, data_type, format);
Create(desc, data);
}
@@ -91,94 +89,34 @@ void MKLDNNMemory::Create(const mkldnn::memory::desc& desc, const void *data, bo
}
void MKLDNNMemory::Create(const MemoryDesc &desc, const void *data, bool pads_zeroing) {
pMemDesc = desc.clone();
Create(mkldnn::memory::desc(MemoryDescUtils::convertToMKLDNNMemoryDesc(desc)), data, pads_zeroing);
Create(desc.clone(), data, pads_zeroing);
}
void MKLDNNMemory::reorderData(const MKLDNNMemory &input, const MKLDNNMemory &output, size_t size) {
if (size != 0)
IE_ASSERT(size <= output.GetDescriptor().get_size());
if (input.GetDescriptor() == output.GetDescriptor()) {
auto srcPtr = static_cast<uint8_t*>(input.GetPtr());
auto dstPtr = static_cast<uint8_t*>(output.GetPtr());
auto copySize = size == 0 ? output.GetSize() : size;
cpu_memcpy(dstPtr, srcPtr, copySize);
void MKLDNNMemory::Create(MemoryDescPtr desc, const void* data, bool pads_zeroing) {
pMemDesc = std::move(desc);
if (nullptr != data) {
useExternalStorage = true;
} else {
std::unique_ptr<mkldnn::reorder> pReorder;
std::shared_ptr<memory> srcMemoryPtr;
std::vector<uint8_t> tmpBuff;
try {
pReorder = std::unique_ptr<mkldnn::reorder>(new mkldnn::reorder(input.GetPrimitive(), output.GetPrimitive()));
srcMemoryPtr = input.prim;
}
catch (const mkldnn::error& err) {
if (mkldnn_unimplemented == err.status && output.GetDataType() != input.GetDataType()) {
//we probably could not make the reorder because there is no one supporting this precision conversion
//lets try to convert data first using cpu_convert
auto data = static_cast<const uint8_t *>(input.GetPtr());
tmpBuff.resize(input.GetSize());
cpu_convert(data, tmpBuff.data(), MKLDNNExtensionUtils::DataTypeToIEPrecision(input.GetDataType()),
MKLDNNExtensionUtils::DataTypeToIEPrecision(output.GetDataType()), input.GetElementsCount());
MKLDNNMemory tmpMem(output.eng);
tmpMem.Create(input.GetDims(), output.GetDataType(), input.GetMKLDNNDesc().getFormat(), tmpBuff.data());
pReorder = std::unique_ptr<mkldnn::reorder>(new mkldnn::reorder(tmpMem.GetPrimitive(), output.GetPrimitive()));
srcMemoryPtr = tmpMem.prim;
} else {
throw;
}
}
if (pReorder) {
mkldnn::stream loc_stream(output.eng, stream::flags::default_order);
pReorder->execute(loc_stream, *srcMemoryPtr, *output.prim);
} else {
IE_THROW() << "Could not make mkldnn reorder.";
}
useExternalStorage = false;
}
}
// TODO: It should be done via wrap into Memory;
void MKLDNNMemory::SetData(memory::data_type dataType, memory::format_tag format, const void* data, size_t size, bool ftz) const {
IE_ASSERT(!one_of(format, memory::format_tag::undef, memory::format_tag::any));
auto dst_desc = GetDescriptor();
memory::desc src_desc{dst_desc.dims(), dataType, format};
IE_ASSERT(size <= dst_desc.get_size());
if (dst_desc == src_desc) {
uint8_t itemSize = MKLDNNExtensionUtils::sizeOfDataType(mkldnn::memory::data_type(dataType));
uint8_t* dataPtr = static_cast<uint8_t*>(GetData());
// We cannot support strides for i/o blobs because it affects performance.
dataPtr += itemSize * prim->get_desc().data.offset0;
cpu_memcpy(dataPtr, data, size);
if (pMemDesc->isDefined()) {
Create(MemoryDescUtils::convertToDnnlMemoryDesc(pMemDesc)->getDnnlDesc(), data, pads_zeroing);
} else {
auto memData = this->GetDescriptor().data;
memory::dims dims(memData.dims, memData.dims + memData.ndims);
MKLDNNMemory src(this->eng);
src.Create(dims, dataType, format, data);
reorderData(src, *this);
//delayed dynamic allocation
size_t maxMemSize = pMemDesc->getMaxMemSize();
VectorDims dummySize{!pMemDesc->hasDefinedMaxSize() ? 0 : maxMemSize};
DnnlBlockedMemoryDesc dummyDesc(InferenceEngine::Precision::U8, Shape(dummySize));
Create(dummyDesc.getDnnlDesc(), data, false); // no pads zeroing
}
if (ftz
&& dataType == memory::data_type::f32
&& prim->get_desc().data.format_kind != dnnl_format_kind_wino
&& GetDataType() != memory::data_type::bf16) {
// Internal blobs haven't strides yet.
auto *memData = static_cast<float *>(GetData());
memData += prim->get_desc().data.offset0;
setSubnormalsToZero(memData, GetSize() / sizeof(float));
size_t newUpperBound = MKLDNNExtensionUtils::getMemSizeForDnnlDesc(prim->get_desc());
if (newUpperBound > memUpperBound) {
memUpperBound = newUpperBound;
}
}
void MKLDNNMemory::SetData(const MKLDNNMemory& src, size_t size, bool ftz) const {
reorderData(src, *this, size);
MKLDNNReorderNode::reorderData(src, *this, size);
if (ftz
&& src.GetDataType() == memory::data_type::f32
@@ -193,726 +131,49 @@ void MKLDNNMemory::SetData(const MKLDNNMemory& src, size_t size, bool ftz) const
void MKLDNNMemory::FillZero() {
void* dataPtr = GetData();
memset(dataPtr, 0, GetSize());
}
memory::format_tag MKLDNNMemory::GetPlainFormatByRank(size_t rank) {
switch (rank) {
case 0:
case 1:
return memory::format_tag::a;
case 2:
return memory::format_tag::ab;
case 3:
return memory::format_tag::abc;
case 4:
return memory::format_tag::abcd;
case 5:
return memory::format_tag::abcde;
case 6:
return memory::format_tag::abcdef;
default:
return memory::format_tag::undef;
}
}
InferenceEngine::Layout MKLDNNMemory::GetPlainLayout(const memory::dims& dims) {
switch (dims.size()) {
case 0: return Layout::SCALAR;
case 1: return Layout::C;
case 2: return Layout::NC;
case 3: return Layout::CHW;
case 4: return Layout::NCHW;
case 5: return Layout::NCDHW;
default:
return Layout::BLOCKED;
}
}
Precision MKLDNNMemory::convertToIePrec(memory::data_type dataType) {
return MKLDNNExtensionUtils::DataTypeToIEPrecision(dataType);
}
memory::data_type MKLDNNMemory::convertToDataType(const InferenceEngine::Precision &precision) {
return MKLDNNExtensionUtils::IEPrecisionToDataType(precision);
}
memory::format_tag MKLDNNMemory::Convert(const InferenceEngine::Layout layout) {
switch (layout) {
case NCHW:
return memory::format_tag::nchw;
case NHWC:
return memory::format_tag::nhwc;
case NCDHW:
return memory::format_tag::ncdhw;
case NDHWC:
return memory::format_tag::ndhwc;
case CHW:
return memory::format_tag::tnc;
case NC:
return memory::format_tag::nc;
case C:
return memory::format_tag::x;
case SCALAR:
return memory::format_tag::x;
default:
return memory::format_tag::undef;
}
}
std::string MKLDNNMemory::formatToString(memory::format_tag fmt) {
return mkldnn::utils::fmt2str(fmt);
if (dataPtr != nullptr)
memset(dataPtr, 0, getDesc().getMaxMemSize());
}
void *MKLDNNMemory::GetPtr() const {
auto ptr = static_cast<uint8_t*>(GetData());
auto md = GetDescriptor().data;
auto md = prim->get_desc().data;
mkldnn::impl::memory_desc_wrapper wrapper(md);
ptr += wrapper.offset0() * wrapper.data_type_size();
return ptr;
}
template<>
MKLDNNMemoryDesc MKLDNNMemory::GetDescWithType<MKLDNNMemoryDesc, 0, 0>() const {
if (auto descPtr = dynamic_cast<const MKLDNNMemoryDesc*>(pMemDesc.get())) {
return *descPtr;
} else {
switch (pMemDesc->getType()) {
case (MemoryDescType::Blocked):
return MemoryDescUtils::convertToMKLDNNMemoryDesc(*(pMemDesc->as<BlockedMemoryDesc>()));
default:
IE_THROW() << "Can not convert unsupported memory descriptor";
void MKLDNNMemory::redefineDesc(const MemoryDesc& desc, void *data) {
redefineDesc(desc.clone(), data);
}
void MKLDNNMemory::redefineDesc(MemoryDescPtr desc, void *data) {
if (data != nullptr) {
this->Create(std::move(desc), data, false);
} else if (useExternalStorage) {
if (!desc->hasDefinedMaxSize()) {
IE_THROW() << "Can not reset descriptor, memory upper bound is unknown.";
}
}
}
template<>
BlockedMemoryDesc MKLDNNMemory::GetDescWithType<BlockedMemoryDesc, 0, 0>() const {
if (auto descPtr = dynamic_cast<const BlockedMemoryDesc*>(pMemDesc.get())) {
return *descPtr;
} else {
switch (pMemDesc->getType()) {
case (MemoryDescType::Mkldnn):
return MemoryDescUtils::convertToBlockedDescriptor(*(pMemDesc->as<MKLDNNMemoryDesc>()));
default:
IE_THROW() << "Can not convert unsupported memory descriptor";
}
}
}
bool MKLDNNMemoryDesc::operator==(const MKLDNNMemoryDesc &rhs) const {
return this->desc == rhs.desc;
}
bool MKLDNNMemoryDesc::operator!=(const MKLDNNMemoryDesc &rhs) const {
return !(*this == rhs);
}
MKLDNNMemoryDesc::operator mkldnn::memory::desc() const {
return desc;
}
MKLDNNMemoryDesc::MKLDNNMemoryDesc(const mkldnn::memory::desc& desc) :
MemoryDesc(Shape(MKLDNNExtensionUtils::convertToSizeVector(desc.dims())), Mkldnn), desc(desc) {
if (desc.data.format_kind == dnnl::impl::format_kind::any)
IE_THROW(Unexpected) << "Memory format any is prohibited!";
}
MKLDNNMemoryDesc::MKLDNNMemoryDesc(const std::vector<size_t>& _dims, mkldnn::memory::data_type dataType, mkldnn::memory::format_tag format)
: MemoryDesc(Shape(_dims), Mkldnn) {
if (format == memory::format_tag::any)
IE_THROW(Unexpected) << "Memory format any is prohibited!";
if (format != memory::format_tag::undef) {
if (format == memory::format_tag::x && _dims.size() == 0) {
desc = mkldnn::memory::desc(mkldnn::memory::dims(1, 1), dataType, format);
size_t descMaxSize = desc->getMaxMemSize();
if (descMaxSize <= memUpperBound) {
this->Create(std::move(desc), prim->get_data_handle(), false);
} else {
desc = mkldnn::memory::desc(MKLDNNExtensionUtils::convertToDnnlDims(_dims), dataType, format);
this->Create(std::move(desc), nullptr, false);
}
} else {
// Trying to create plain descriptor
// This WA is needed since memory::format_tag doesn't contain plain tag for tensors with rank > 6D
mkldnn::memory::dims strides(_dims.size(), 1);
for (int d = _dims.size() - 2; d >= 0; d--) {
strides[d] = strides[d + 1] * _dims[d + 1];
}
desc = mkldnn::memory::desc(MKLDNNExtensionUtils::convertToDnnlDims(_dims), dataType, strides);
this->Create(std::move(desc), nullptr, false);
}
}
MKLDNNMemoryDesc::MKLDNNMemoryDesc(const std::vector<size_t>& _dims, mkldnn::memory::data_type dataType)
: MemoryDesc(Shape(_dims), Mkldnn), desc() {
const auto ndims = _dims.size();
mkldnn::memory::dims plain_strides(ndims, 1);
for (size_t i = 1; i < ndims; i++) {
plain_strides[ndims - i -1] = plain_strides[ndims - i] * _dims[ndims - i];
}
desc = {MKLDNNExtensionUtils::convertToDnnlDims(_dims), dataType, plain_strides};
template<>
DnnlMemoryDescPtr MKLDNNMemory::GetDescWithType<DnnlMemoryDesc, 0, 0>() const {
return MemoryDescUtils::convertToDnnlMemoryDesc(pMemDesc);
}
static const std::map<int, std::vector<mkldnn::memory::format_tag>> form_tags_by_ndims {
{0, {
mkldnn::memory::format_tag::a // TODO :: really 1d layout for scalar??
}}, {1, {
mkldnn::memory::format_tag::a
}}, {2, {
mkldnn::memory::format_tag::ab,
mkldnn::memory::format_tag::ba
}}, {3, {
mkldnn::memory::format_tag::abc,
mkldnn::memory::format_tag::acb,
mkldnn::memory::format_tag::bac,
mkldnn::memory::format_tag::bca,
mkldnn::memory::format_tag::cba,
mkldnn::memory::format_tag::Abc16a,
mkldnn::memory::format_tag::ABc16a16b,
mkldnn::memory::format_tag::ABc4a4b,
mkldnn::memory::format_tag::aBc16b,
mkldnn::memory::format_tag::aBc32b,
mkldnn::memory::format_tag::ABc16b16a,
mkldnn::memory::format_tag::Abc4a,
mkldnn::memory::format_tag::aBc4b,
mkldnn::memory::format_tag::ABc4b16a4b,
mkldnn::memory::format_tag::ABc2b8a4b,
mkldnn::memory::format_tag::ABc16b16a4b,
mkldnn::memory::format_tag::ABc16b16a2b,
mkldnn::memory::format_tag::ABc4b4a,
mkldnn::memory::format_tag::ABc8a16b2a,
mkldnn::memory::format_tag::ABc8a8b,
mkldnn::memory::format_tag::ABc8a4b,
mkldnn::memory::format_tag::aBc8b,
mkldnn::memory::format_tag::ABc8b16a2b,
mkldnn::memory::format_tag::ABc8b8a,
mkldnn::memory::format_tag::Acb16a,
mkldnn::memory::format_tag::Acb4a,
mkldnn::memory::format_tag::Acb8a,
mkldnn::memory::format_tag::BAc16a16b,
mkldnn::memory::format_tag::BAc16b16a,
}}, {4, { // Popular
mkldnn::memory::format_tag::abcd, // plain
mkldnn::memory::format_tag::acdb, // tail_c
mkldnn::memory::format_tag::aBcd8b, // blocked 8c
mkldnn::memory::format_tag::aBcd16b, // blocked 16c
mkldnn::memory::format_tag::abdc,
mkldnn::memory::format_tag::bacd,
mkldnn::memory::format_tag::bcda,
mkldnn::memory::format_tag::cdba,
mkldnn::memory::format_tag::dcab,
mkldnn::memory::format_tag::Abcd8a,
mkldnn::memory::format_tag::Abcd16a,
mkldnn::memory::format_tag::Abcd32a,
mkldnn::memory::format_tag::ABcd16a16b,
mkldnn::memory::format_tag::aBcd32b,
mkldnn::memory::format_tag::ABcd16b16a,
mkldnn::memory::format_tag::aBCd16b16c,
mkldnn::memory::format_tag::aBCd16c16b,
mkldnn::memory::format_tag::Abcd4a,
mkldnn::memory::format_tag::aBcd4b,
mkldnn::memory::format_tag::ABcd4b16a4b,
mkldnn::memory::format_tag::ABcd2b8a4b,
mkldnn::memory::format_tag::ABcd4b4a,
mkldnn::memory::format_tag::ABcd4a4b,
mkldnn::memory::format_tag::aBCd4c16b4c,
mkldnn::memory::format_tag::aBCd2c8b4c,
mkldnn::memory::format_tag::ABcd16b16a4b,
mkldnn::memory::format_tag::ABcd16b16a2b,
mkldnn::memory::format_tag::aBCd16c16b4c,
mkldnn::memory::format_tag::aBCd16c16b2c,
mkldnn::memory::format_tag::aBCd4c4b,
mkldnn::memory::format_tag::aBCd4b4c,
mkldnn::memory::format_tag::ABcd8a16b2a,
mkldnn::memory::format_tag::ABcd8a8b,
mkldnn::memory::format_tag::ABcd8a32b,
mkldnn::memory::format_tag::ABcd32a32b,
mkldnn::memory::format_tag::ABcd8a4b,
mkldnn::memory::format_tag::ABcd8b16a2b,
mkldnn::memory::format_tag::aBCd8b16c2b,
mkldnn::memory::format_tag::ABcd8b8a,
mkldnn::memory::format_tag::aBCd8b8c,
mkldnn::memory::format_tag::aBCd8b4c,
mkldnn::memory::format_tag::aBCd8c16b2c,
mkldnn::memory::format_tag::aBCd8c8b,
mkldnn::memory::format_tag::ABcd4a8b8a4b,
mkldnn::memory::format_tag::ABcd2a8b8a2b,
mkldnn::memory::format_tag::aBdc16b,
mkldnn::memory::format_tag::aBdc4b,
mkldnn::memory::format_tag::aBdc8b,
mkldnn::memory::format_tag::aCBd16b16c,
mkldnn::memory::format_tag::aCBd16c16b,
mkldnn::memory::format_tag::Acdb16a,
mkldnn::memory::format_tag::Acdb4a,
mkldnn::memory::format_tag::Acdb8a,
mkldnn::memory::format_tag::BAcd16a16b,
mkldnn::memory::format_tag::BAcd16b16a,
mkldnn::memory::format_tag::ABcd32a32b,
mkldnn::memory::format_tag::Acdb32a,
mkldnn::memory::format_tag::aBCd2b4c2b,
mkldnn::memory::format_tag::aBCd2c4b2c,
mkldnn::memory::format_tag::aBCd4b8c2b,
mkldnn::memory::format_tag::aBCd4c8b2c,
}}, {5, { // Popular
mkldnn::memory::format_tag::abcde, // plain
mkldnn::memory::format_tag::acdeb, // tail_c
mkldnn::memory::format_tag::aBcde8b, // blocked 8c
mkldnn::memory::format_tag::aBcde16b, // blocked 16c
mkldnn::memory::format_tag::abdec,
mkldnn::memory::format_tag::acbde,
mkldnn::memory::format_tag::bacde,
mkldnn::memory::format_tag::bcdea,
mkldnn::memory::format_tag::cdeba,
mkldnn::memory::format_tag::decab,
mkldnn::memory::format_tag::Abcde16a,
mkldnn::memory::format_tag::Abcde32a,
mkldnn::memory::format_tag::ABcde16a16b,
mkldnn::memory::format_tag::aBcde32b,
mkldnn::memory::format_tag::ABcde16b16a,
mkldnn::memory::format_tag::aBCde16b16c,
mkldnn::memory::format_tag::aBCde16c16b,
mkldnn::memory::format_tag::aBCde2c8b4c,
mkldnn::memory::format_tag::Abcde4a,
mkldnn::memory::format_tag::aBcde4b,
mkldnn::memory::format_tag::ABcde4b4a,
mkldnn::memory::format_tag::ABcde4a4b,
mkldnn::memory::format_tag::aBCde4b4c,
mkldnn::memory::format_tag::aBCde4c16b4c,
mkldnn::memory::format_tag::aBCde16c16b4c,
mkldnn::memory::format_tag::aBCde16c16b2c,
mkldnn::memory::format_tag::aBCde4c4b,
mkldnn::memory::format_tag::Abcde8a,
mkldnn::memory::format_tag::ABcde8a8b,
mkldnn::memory::format_tag::ABcde8a4b,
mkldnn::memory::format_tag::ABcde8b16a2b,
mkldnn::memory::format_tag::ABcde4b16a4b,
mkldnn::memory::format_tag::ABcde2b8a4b,
mkldnn::memory::format_tag::aBCde8b16c2b,
mkldnn::memory::format_tag::ABcde8b8a,
mkldnn::memory::format_tag::aBCde8b8c,
mkldnn::memory::format_tag::aBCde8b4c,
mkldnn::memory::format_tag::aBCde4b8c8b4c,
mkldnn::memory::format_tag::aBCde2b8c8b2c,
mkldnn::memory::format_tag::aBCde8c16b2c,
mkldnn::memory::format_tag::aBCde8c8b,
mkldnn::memory::format_tag::aBdec16b,
mkldnn::memory::format_tag::aBdec4b,
mkldnn::memory::format_tag::aBdec8b,
mkldnn::memory::format_tag::aCBde16b16c,
mkldnn::memory::format_tag::aCBde16c16b,
mkldnn::memory::format_tag::Acdeb16a,
mkldnn::memory::format_tag::Acdeb4a,
mkldnn::memory::format_tag::Acdeb8a,
mkldnn::memory::format_tag::BAcde16b16a,
mkldnn::memory::format_tag::BAcde16a16b,
mkldnn::memory::format_tag::aBdec32b,
mkldnn::memory::format_tag::aBCde2b4c2b,
mkldnn::memory::format_tag::aBCde2c4b2c,
mkldnn::memory::format_tag::aBCde4b8c2b,
mkldnn::memory::format_tag::aBCde4c8b2c,
}}, {6, { // Popular
mkldnn::memory::format_tag::abcdef, // plain
mkldnn::memory::format_tag::acbdef, // permute
mkldnn::memory::format_tag::defcab, // permute
mkldnn::memory::format_tag::aBcdef16b, // blocked 16c
mkldnn::memory::format_tag::aBCdef16b16c,
mkldnn::memory::format_tag::aBCdef16c16b,
mkldnn::memory::format_tag::aBcdef4b,
mkldnn::memory::format_tag::aBCdef2c8b4c,
mkldnn::memory::format_tag::aBCdef4c4b,
mkldnn::memory::format_tag::aBCdef4b4c,
mkldnn::memory::format_tag::aBCdef8b8c,
mkldnn::memory::format_tag::aBCdef8b4c,
mkldnn::memory::format_tag::aBCdef8c16b2c,
mkldnn::memory::format_tag::aBCdef4c16b4c,
mkldnn::memory::format_tag::aBCdef8c8b,
mkldnn::memory::format_tag::aBdefc16b,
mkldnn::memory::format_tag::aCBdef16c16b,
mkldnn::memory::format_tag::aCBdef16b16c,
mkldnn::memory::format_tag::aBdefc4b,
mkldnn::memory::format_tag::aBdefc8b,
mkldnn::memory::format_tag::Abcdef4a,
mkldnn::memory::format_tag::Abcdef8a,
mkldnn::memory::format_tag::Abcdef16a,
mkldnn::memory::format_tag::Abcdef32a,
mkldnn::memory::format_tag::aBCdef2b4c2b,
mkldnn::memory::format_tag::aBCdef2c4b2c,
mkldnn::memory::format_tag::aBCdef4b8c2b,
mkldnn::memory::format_tag::aBCdef4c8b2c,
}}
};
mkldnn::memory::format_tag MKLDNNMemoryDesc::getFormat() const {
// TODO [OneDNN]: Previously it was a field of tdesc, but now the brute
// force search here. Please avoid of using this method.
const auto ndims = desc.dims().size();
// There are no suitable format_tag for this
if (ndims == 0 || ndims > 6)
return mkldnn::memory::format_tag::undef;
for (const auto fmt : form_tags_by_ndims.at(ndims)) {
if (this->isSame(fmt))
return fmt;
}
return mkldnn::memory::format_tag::undef;
template<>
BlockedMemoryDescPtr MKLDNNMemory::GetDescWithType<BlockedMemoryDesc, 0, 0>() const {
return MemoryDescUtils::convertToBlockedMemoryDesc(pMemDesc);
}
bool MKLDNNMemoryDesc::isSame(mkldnn::memory::format_tag fmt) const {
memory::desc refDesc(desc.dims(), desc.data_type(), fmt);
if (desc.data.ndims != refDesc.data.ndims)
return false;
if (desc.data.format_kind != dnnl_blocked || refDesc.data.format_kind != dnnl_blocked)
IE_THROW() << "MKLDNNMemoryDesc::isSame is not implemented for non blocked memory format";
auto actualBlkDesc = desc.data.format_desc.blocking;
auto refBlkDesc = refDesc.data.format_desc.blocking;
if (actualBlkDesc.inner_nblks != refBlkDesc.inner_nblks)
return false;
for (size_t i = 0; i < actualBlkDesc.inner_nblks; ++i)
if (actualBlkDesc.inner_blks[i] != refBlkDesc.inner_blks[i])
return false;
for (size_t i = 0; i < actualBlkDesc.inner_nblks; ++i)
if (actualBlkDesc.inner_idxs[i] != refBlkDesc.inner_idxs[i])
return false;
auto actualStrides = desc.data.format_desc.blocking.strides;
auto refStrides = refDesc.data.format_desc.blocking.strides;
std::vector<size_t> actualOrder(desc.data.ndims);
{
const auto dims = desc.dims();
std::vector<size_t> total_block_per_dim(dims.size(), 1);
const auto &blk_desc = desc.data.format_desc.blocking;
for (int i = 0; i < blk_desc.inner_nblks; i++) {
total_block_per_dim[blk_desc.inner_idxs[i]] *= blk_desc.inner_blks[i];
}
std::vector<size_t> outer_block_dims(std::begin(dims), std::begin(dims) + dims.size());
for (size_t i = 0; i < outer_block_dims.size(); i++) {
outer_block_dims[i] = div_up(outer_block_dims[i], total_block_per_dim[i]);
}
std::iota(actualOrder.begin(), actualOrder.end(), 0);
std::sort(actualOrder.begin(), actualOrder.end(),
[&actualStrides, &outer_block_dims] (size_t ind_l, size_t ind_r) {
return (actualStrides[ind_l] > actualStrides[ind_r]) ||
(actualStrides[ind_l] == actualStrides[ind_r] && outer_block_dims[ind_l] > outer_block_dims[ind_r]);
});
}
std::vector<size_t> refOrder(refDesc.data.ndims);
{
const auto dims = refDesc.dims();
std::vector<size_t> total_block_per_dim(dims.size(), 1);
const auto &blk_desc = refDesc.data.format_desc.blocking;
for (int i = 0; i < blk_desc.inner_nblks; i++) {
total_block_per_dim[blk_desc.inner_idxs[i]] *= blk_desc.inner_blks[i];
}
std::vector<size_t> outer_block_dims(std::begin(dims), std::begin(dims) + dims.size());
for (size_t i = 0; i < outer_block_dims.size(); i++) {
outer_block_dims[i] = div_up(outer_block_dims[i], total_block_per_dim[i]);
}
std::iota(refOrder.begin(), refOrder.end(), 0);
std::sort(refOrder.begin(), refOrder.end(),
[&refStrides, &outer_block_dims] (size_t ind_l, size_t ind_r) {
return (refStrides[ind_l] > refStrides[ind_r]) ||
(refStrides[ind_l] == refStrides[ind_r] && outer_block_dims[ind_l] > outer_block_dims[ind_r]);
});
}
if (actualOrder != refOrder) {
return false;
}
return true;
}
bool MKLDNNMemoryDesc::isPlainFormat() const {
if (desc.data.format_kind != dnnl_blocked ||
desc.data.format_desc.blocking.inner_nblks != 0)
return false;
const auto ndims = desc.data.ndims;
const auto dims = desc.data.dims;
const auto &strides = desc.data.format_desc.blocking.strides;
bool is_plain_strides = (strides[ndims-1] == 1);
for (int i = 0; i < ndims - 1; i++) {
is_plain_strides &= (strides[i] == strides[i+1] * dims[i+1]);
}
return is_plain_strides;
}
bool MKLDNNMemoryDesc::isBlockedCFormat(size_t blk_size) const {
const auto &blocking = desc.data.format_desc.blocking;
if (desc.data.format_kind != dnnl_blocked ||
blocking.inner_nblks != 1 ||
blocking.inner_idxs[0] != 1)
return false;
const auto &ndims = desc.data.ndims;
const auto &strides = desc.data.format_desc.blocking.strides;
const auto &dims = desc.data.padded_dims;
if (blk_size == UNREACHABLE_DIM) {
blk_size = blocking.inner_blks[0];
} else {
if (blk_size != blocking.inner_blks[0])
return false;
}
bool is_direct_order = (strides[ndims-1] == blocking.inner_blks[0]);
for (int i = 0; i < ndims - 1; i++) {
auto dim = (i == 0) ? div_up(dims[i+1], blk_size) : dims[i+1];
is_direct_order &= (strides[i] >= strides[i+1] * dim);
}
return is_direct_order;
}
bool MKLDNNMemoryDesc::isTailCFormat() const {
const auto &blocking = desc.data.format_desc.blocking;
if (desc.data.format_kind != dnnl_blocked ||
blocking.inner_nblks != 0)
return false;
const auto &ndims = desc.data.ndims;
const auto &strides = desc.data.format_desc.blocking.strides;
const auto &dims = desc.data.padded_dims;
// dense permutation of acd..b
bool is_tailc_strides = (strides[1] == 1 && strides[ndims-1] == dims[1] && strides[0] == dims[2] * strides[2]);
for (int i = 2; i < ndims - 1; i++) {
is_tailc_strides &= (strides[i] == strides[i+1] * dims[i+1]);
}
return is_tailc_strides;
}
bool MKLDNNMemoryDesc::blocksExtended() const {
for (int i = 0; i < desc.data.ndims; i++) {
if (desc.data.dims[i] != desc.data.padded_dims[i])
return true;
}
return false;
}
size_t MKLDNNMemoryDesc::getMemSizeImp() const {
return desc.get_size();
}
size_t MKLDNNMemoryDesc::getElementOffset(size_t elemNumber) const {
mkldnn::impl::memory_desc_wrapper wrapped(desc.data);
return wrapped.off_l(elemNumber);
}
bool MKLDNNMemoryDesc::isCompatible(const MemoryDesc &rhs) const {
if (MemoryDescType::Blocked == rhs.getType()) {
return isCompatible(*(rhs.as<BlockedMemoryDesc>()));
} else if (MemoryDescType::Mkldnn == rhs.getType()) {
return isCompatible(*(rhs.as<MKLDNNMemoryDesc>()));
} else {
return false;
}
}
bool MKLDNNMemoryDesc::isCompatible(const MKLDNNMemoryDesc &rhs) const {
using namespace dnnl;
using namespace impl;
using namespace dnnl::impl::utils;
if (this->desc == rhs.desc) {
return true;
}
mkldnn::impl::memory_desc_wrapper wrappedThis(this->desc.data);
mkldnn::impl::memory_desc_wrapper wrappedRhs(rhs.desc.data);
if (one_of(wrappedThis.format_kind(), format_kind::undef, format_kind::any))
return false;
if (wrappedThis.is_wino_desc() || wrappedThis.is_rnn_packed_desc()) return false;
const auto &blk = wrappedThis.blocking_desc();
const auto &r_blk = wrappedRhs.blocking_desc();
int stride_start = wrappedThis.ndims() >0 && wrappedThis.dims()[0] == 1 ? 1 : 0; //ignore batch axis stride if batch size == 1
// Here is a slightly modified version of mkldnn::impl::memory_desc_wrapper::similar_to() call able to skip specific strides check.
return wrappedThis.ndims() == wrappedRhs.ndims()
&& wrappedThis.format_kind() == wrappedRhs.format_kind()
&& wrappedThis.data_type() == wrappedRhs.data_type()
&& array_cmp(wrappedThis.dims(), wrappedRhs.dims(), wrappedThis.ndims())
&& array_cmp(blk.strides + stride_start, r_blk.strides + stride_start, wrappedThis.ndims() - stride_start)
&& blk.inner_nblks == r_blk.inner_nblks
&& array_cmp(blk.inner_blks, r_blk.inner_blks, blk.inner_nblks)
&& array_cmp(blk.inner_idxs, r_blk.inner_idxs, blk.inner_nblks)
&& array_cmp(wrappedThis.padded_dims(), wrappedRhs.padded_dims(), wrappedRhs.ndims())
&& array_cmp(wrappedThis.padded_offsets(), wrappedRhs.padded_offsets(), wrappedThis.ndims())
&& dimsEqualWeak(wrappedThis.offset0(), wrappedRhs.offset0());
}
/**
* Check compatibility with BlockedMemoryDesc
*
* mkl: IOhw_4i16o4i dims {32, 64, 128, 128}
* strides // the order of outer dims is encoded here
* inner_blks 4 16 4
* inner_idxs 1 0 1
*
* BlockedMemoryDesc desc has more expressive ability.
* How to check compatibility with BlockedMemoryDesc representation:
* 0. Detect a new_outer_order of outer_dims via descending strides.
* 1. BlockedMemoryDesc strides : concatenate strides in new_outer_order and inner strides.
* 2. BlockedMemoryDesc dims : concatenate outer dims in new_outer_order with auto padding and inner blocks
* 3. BlockedMemoryDesc order : concatenate new_outer_order and inner_idxs
*/
bool MKLDNNMemoryDesc::isCompatible(const BlockedMemoryDesc &rhs) const {
if (this->getShape() != rhs.getShape() || this->getPrecision() != rhs.getPrecision()) {
return false;
}
const auto dims = desc.dims();
if (desc.data.format_kind != dnnl_blocked) {
return false;
}
const auto &blk_desc = desc.data.format_desc.blocking;
const size_t outer_ndims = dims.size();
const size_t inner_ndims = blk_desc.inner_nblks;
const size_t total_ndims = outer_ndims + inner_ndims;
// strides of inner dims. In case of 4i16o4i will be {64, 4, 1}
std::vector<size_t> inner_strides(inner_ndims, 1);
for (size_t i = 1; i < blk_desc.inner_nblks; i++) {
inner_strides[blk_desc.inner_nblks - 1 - i] = inner_strides[blk_desc.inner_nblks - i] * blk_desc.inner_blks[blk_desc.inner_nblks - i];
}
// total inner block size. in case of 4i16o4i will be {16, 16, 1, 1}
std::vector<size_t> total_block_per_dim(outer_ndims, 1);
for (int i = 0; i < inner_ndims; i++) {
total_block_per_dim[blk_desc.inner_idxs[i]] *= blk_desc.inner_blks[i];
}
std::vector<size_t> outer_block_dims(std::begin(dims), std::begin(dims) + outer_ndims);
for (size_t i = 0; i < outer_block_dims.size(); i++) {
outer_block_dims[i] = div_up(outer_block_dims[i], total_block_per_dim[i]);
}
// order of outer dims. In case of IOhw_ will be {1, 0, 2, 3}
std::vector<size_t> outer_order(outer_ndims);
std::iota(outer_order.begin(), outer_order.end(), 0);
std::sort(outer_order.begin(), outer_order.end(),
[&blk_desc, &outer_block_dims] (size_t ind_l, size_t ind_r) {
return (blk_desc.strides[ind_l] > blk_desc.strides[ind_r]) ||
(blk_desc.strides[ind_l] == blk_desc.strides[ind_r] && outer_block_dims[ind_l] > outer_block_dims[ind_r]);
});
// blocked order
// [new_outer_order] U [inner_idxs]
SizeVector blk_order(total_ndims, 0);
std::copy(outer_order.begin(), outer_order.end(), blk_order.begin());
std::copy(blk_desc.inner_idxs, blk_desc.inner_idxs + blk_desc.inner_nblks, blk_order.begin() + dims.size());
if (!dimsEqualWeak(blk_order, rhs.getOrder())) {
return false;
}
//TODO [DS]: undefined offset is also used now as an indicator of undefined strides
if (desc.data.offset0 != Shape::UNDEFINED_DIM) {
// blocked strides
// [outer_strides via new_outer_order] U [inner_strides]
SizeVector blk_strides(total_ndims, 0);
std::copy(inner_strides.rbegin(), inner_strides.rend(), blk_strides.rbegin());
std::transform(outer_order.begin(), outer_order.end(), blk_strides.begin(),
[&](size_t i) { return blk_desc.strides[i]; });
size_t skipAxis = this->getShape().getRank() > 0 && this->getShape().getDims().front() == 1 ? 0 :
Shape::UNDEFINED_DIM; //ignore batch axis if batch size == 1
if (!dimsEqualWeak(blk_strides, rhs.getStrides(), skipAxis)) {
return false;
}
}
// blocked dims
// [dims via new_outer_order with auto pad] U [inner_blk_dims]
SizeVector blk_dims(total_ndims, 0);
std::copy(blk_desc.inner_blks, blk_desc.inner_blks + blk_desc.inner_nblks,
blk_dims.end() - blk_desc.inner_nblks);
std::transform(outer_order.begin(), outer_order.end(), blk_dims.begin(),
[&] (size_t i) { return outer_block_dims[i]; });
if (!dimsEqualWeak(blk_dims, rhs.getBlockDims())) {
return false;
}
// offset padded to data. Same as for oneDNN
SizeVector blk_offset_to_data {desc.data.padded_offsets, desc.data.padded_offsets + desc.data.ndims};
// TODO: The BlockedMemoryDesc implementation allow to specify offset_to_data for inner blocked dims.
// Which is not obvious behavior. It required offset_to_data.size == total_ndims, so will
// fill it with zero.
blk_offset_to_data.insert(blk_offset_to_data.end(), inner_ndims, 0);
if (!dimsEqualWeak(blk_offset_to_data, rhs.getOffsetPaddingToData())) {
return false;
}
return dimsEqualWeak(desc.data.offset0, rhs.getOffsetPadding());
}
bool MKLDNNMemoryDesc::hasLayoutType(LayoutType layoutType) const {
switch (layoutType) {
case LayoutType::ncsp:
return isPlainFormat();
case LayoutType::nspc:
return isTailCFormat();
case LayoutType::nCsp8c:
return isBlockedCFormat(8);
case LayoutType::nCsp16c:
return isBlockedCFormat(16);
default:
return false;
}
}
std::string MKLDNNMemoryDesc::serializeFormat() const {
if (desc.data.format_kind == dnnl_format_kind_wino) {
switch (desc.data.format_desc.wino_desc.wino_format) {
case dnnl_wino_memory_format_t::dnnl_wino_wei_aaOIoi: return "wino_aaOIoi";
case dnnl_wino_memory_format_t::dnnl_wino_wei_aaOio: return "wino_aaOio";
case dnnl_wino_memory_format_t::dnnl_wino_wei_aaOBiOo: return "wino_aaOBiOo";
case dnnl_wino_memory_format_t::dnnl_wino_wei_OBaaIBOIio: return "wino_OBaaIBOIio";
default: return "wino_undef";
}
}
auto fmt = getFormat();
return mkldnn::utils::fmt2str(fmt);
}
bool MKLDNNMemoryDesc::isDefined() const {
return desc.data.offset0 != Shape::UNDEFINED_DIM;
}
InferenceEngine::Precision MKLDNNMemoryDesc::getPrecision() const {
return MKLDNNExtensionUtils::DataTypeToIEPrecision(desc.data_type());
}
void MKLDNNMemoryDesc::setPrecision(InferenceEngine::Precision prc) {
desc.data.data_type = static_cast<dnnl_data_type_t>(MKLDNNExtensionUtils::IEPrecisionToDataType(prc));
}
} // namespace MKLDNNPlugin
inference-engine/src/mkldnn_plugin/mkldnn_memory.h
+31 -108
View File
@@ -5,13 +5,14 @@
#pragma once
#include "ie_layouts.h"
#include "mkldnn_dims.h"
#include "cpu_memory_desc.h"
#include "memory_desc/cpu_memory_desc.h"
#include "mkldnn_extension_utils.h"
#include "memory_desc/cpu_memory_desc_utils.h"
#include <mkldnn.hpp>
#include <mkldnn_types.h>
#include <cpu_shape.h>
#include <cpu_blocked_memory_desc.h>
#include "memory_desc/dnnl_memory_desc.h"
#include <string>
#include <functional>
@@ -22,11 +23,6 @@
/**
* @file contains a concept classes to work with memory/tensor/blob abstractions on plugin level.
*
* MKLDNNMemoryDesc - the descriptor of tensor representation in memory. Describes all required information
* for proper allocation and handling tensor in some buffer. The real memory is not present, just description.
* This object answers on question how and where data with logical index [x1, x2, .. xN] placed in real buffer.
* In the simplest case it describe a mapping between "logical offset" and "real offset".
*
* MKLDNNMemory is an abstraction of some real tensor which contains some data. As in short it's a pair of
* memory descriptor and raw buffer handler to contains data. In case of system memory raw buffer it's simple
* "void*" on some system memory buffer.
@@ -35,82 +31,6 @@
namespace MKLDNNPlugin {
/**
* Represent internal plugin abstraction of tensor description
*
*/
class MKLDNNMemoryDesc : public MemoryDesc {
public:
/** Construct a tensor desc with plain layout format (like ND C array) */
MKLDNNMemoryDesc(const std::vector<size_t>& _dims, mkldnn::memory::data_type dataType);
/** Construct a tensor desc with specified layout format tag. Any and Undef is not supported */
MKLDNNMemoryDesc(const std::vector<size_t>& _dims, mkldnn::memory::data_type dataType, mkldnn::memory::format_tag format);
explicit MKLDNNMemoryDesc(const mkldnn::memory::desc& desc);
/**
* Try to define original format tag use on creation
*
* @return format tag if was able to define it
*/
mkldnn::memory::format_tag getFormat() const;
mkldnn::memory::data_type getDataType() const {
return static_cast<mkldnn::memory::data_type>(desc.data.data_type);
}
MKLDNNDims getDims() const {
return MKLDNNDims(desc.data.dims, desc.data.ndims);
}
bool blocksExtended() const;
operator bool() const {
return getFormat() != mkldnn::memory::format_tag::any && getFormat() != mkldnn::memory::format_tag::undef;
}
bool operator == (const MKLDNNMemoryDesc& rhs) const;
bool operator != (const MKLDNNMemoryDesc& rhs) const;
operator mkldnn::memory::desc() const;
bool isSame(mkldnn::memory::format_tag fmt) const;
dnnl_format_kind_t getFormatKind() const {
return desc.data.format_kind;
}
std::unique_ptr<MemoryDesc> clone() const override {
return MKLDNNPlugin::make_unique<MKLDNNMemoryDesc>(*this);
}
bool hasLayoutType(LayoutType layoutType) const override;
std::string serializeFormat() const override;
bool isDefined() const override;
InferenceEngine::Precision getPrecision() const override;
void setPrecision(InferenceEngine::Precision prc) override;
bool isCompatible(const MemoryDesc& rhs) const override;
bool isCompatible(const BlockedMemoryDesc& rhs) const;
bool isCompatible(const MKLDNNMemoryDesc& rhs) const;
private:
size_t getElementOffset(size_t elemNumber) const override;
size_t getMemSizeImp() const override;
bool isPlainFormat() const;
bool isBlockedCFormat(size_t blk_size = UNREACHABLE_DIM) const;
bool isTailCFormat() const;
private:
static constexpr size_t UNREACHABLE_DIM = std::numeric_limits<size_t>::max();
mkldnn::memory::desc desc;
};
class MKLDNNMemory {
public:
explicit MKLDNNMemory(const mkldnn::engine& eng);
@@ -129,18 +49,14 @@ public:
return prim;
}
mkldnn::memory::desc GetDescriptor() const {
return prim->get_desc();
}
const MemoryDesc& GetDesc() const {
const MemoryDesc& getDesc() const {
return *pMemDesc;
}
template <typename T,
typename std::enable_if<!std::is_pointer<T>::value && !std::is_reference<T>::value, int>::type = 0,
typename std::enable_if<std::is_base_of<MemoryDesc, T>::value, int>::type = 0>
T GetDescWithType() const;
std::shared_ptr<T> GetDescWithType() const;
/**
* Return handler of buffer. Real data may starts from some other offset
@@ -161,33 +77,42 @@ public:
void* GetPtr() const;
mkldnn::memory::data_type GetDataType() const {
return static_cast<mkldnn::memory::data_type>(GetDescriptor().data.data_type);
return MKLDNNExtensionUtils::IEPrecisionToDataType(getDesc().getPrecision());
}
size_t GetSize() const;
size_t GetElementsCount() const;
mkldnn::memory::dims GetDims() const {
auto data = GetDescriptor().data;
return {std::begin(data.dims), std::begin(data.dims) + data.ndims};
const Shape& GetShape() const {
return getDesc().getShape();
}
void Create(const MemoryDesc& desc, const void* data = nullptr, bool pads_zeroing = true);
void Create(MemoryDescPtr desc, const void* data = nullptr, bool pads_zeroing = true);
// Redefines descriptor. The memory descriptor will be replaced with the new one.
// Memory will not be reallocated if the new tensor size is less or equal the upper bound.
// Caution!!! This action invalidates the previous data layout. The old data may become unreachable.
void redefineDesc(const MemoryDesc& desc, void *data = nullptr);
void redefineDesc(MemoryDescPtr desc, void *data = nullptr);
// Like a plain format
void SetData(mkldnn::memory::data_type dataType, mkldnn::memory::format_tag format, const void* data, size_t size, bool ftz = true) const;
void SetData(const MKLDNNMemory& memory, size_t size = 0, bool ftz = true) const;
void FillZero();
static mkldnn::memory::format_tag GetPlainFormatByRank(size_t rank);
static InferenceEngine::Layout GetPlainLayout(const mkldnn::memory::dims& dims);
static mkldnn::memory::format_tag Convert(const InferenceEngine::Layout layout);
static InferenceEngine::Precision convertToIePrec(mkldnn::memory::data_type dataType);
static mkldnn::memory::data_type convertToDataType(const InferenceEngine::Precision &precision);
bool hasExternalStorage() const {
return useExternalStorage;
}
static std::string formatToString(mkldnn::memory::format_tag fmt);
const VectorDims& getStaticDims() const {
return getDesc().getShape().getStaticDims();
}
static void reorderData(const MKLDNNMemory& input, const MKLDNNMemory& output, size_t size = 0);
mkldnn::engine getEngine() const {
return eng;
}
bool isUsedExternalStorage() const {
return useExternalStorage;
}
private:
void Create(const mkldnn::memory::dims& dims, mkldnn::memory::data_type data_type, mkldnn::memory::format_tag format,
@@ -195,14 +120,12 @@ private:
void Create(const mkldnn::memory::desc& desc, const void* data = nullptr, bool pads_zeroing = true);
const MKLDNNMemoryDesc GetMKLDNNDesc() const {
return MKLDNNMemoryDesc(prim->get_desc());
}
private:
MemoryDescPtr pMemDesc;
std::shared_ptr<mkldnn::memory> prim;
mkldnn::engine eng;
bool useExternalStorage = false;
size_t memUpperBound = 0ul;
};
using MKLDNNMemoryPtr = std::shared_ptr<MKLDNNMemory>;
inference-engine/src/mkldnn_plugin/mkldnn_memory_state.h
+2 -2
View File
@@ -8,7 +8,7 @@
#include "blob_factory.hpp"
#include "mkldnn_memory.h"
#include "nodes/common/cpu_memcpy.h"
#include "cpu_memory_desc_utils.h"
#include "memory_desc/cpu_memory_desc_utils.h"
#include <string>
@@ -18,7 +18,7 @@ class MKLDNNVariableState : public InferenceEngine::IVariableStateInternal {
public:
MKLDNNVariableState(std::string name, MKLDNNMemoryPtr storage) :
InferenceEngine::IVariableStateInternal{name} {
state = make_blob_with_precision(MemoryDescUtils::convertToTensorDesc(storage->GetDesc()));
state = make_blob_with_precision(MemoryDescUtils::convertToTensorDesc(storage->getDesc()));
state->allocate();
cpu_memcpy(state->buffer(), storage->GetData(), storage->GetSize());
}
inference-engine/src/mkldnn_plugin/mkldnn_node.cpp
+147 -51
View File
@@ -55,7 +55,8 @@
#include "utils/general_utils.h"
#include "utils/cpu_utils.hpp"
#include "nodes/common/cpu_convert.h"
#include "cpu_memory_desc_utils.h"
#include "memory_desc/cpu_memory_desc_utils.h"
#include "memory_desc/dnnl_blocked_memory_desc.h"
using namespace mkldnn;
using namespace MKLDNNPlugin;
@@ -226,6 +227,7 @@ static const InferenceEngine::details::caseless_unordered_map<std::string, Type>
{ "ExperimentalDetectronPriorGridGenerator", ExperimentalDetectronPriorGridGenerator},
{ "ExperimentalDetectronGenerateProposalsSingleImage", ExperimentalDetectronGenerateProposalsSingleImage},
{ "ExtractImagePatches", ExtractImagePatches},
{ "NonMaxSuppression", NonMaxSuppression},
{ "NonMaxSuppressionIEInternal", NonMaxSuppression},
{ "MatrixNms", MatrixNms},
{ "MulticlassNms", MulticlassNms}
@@ -240,6 +242,26 @@ Type TypeFromName(const std::string type) {
}
}
template<>
DnnlMemoryDescPtr MKLDNNNode::getInputMemDescAtPort<DnnlMemoryDesc, 0, 0>(size_t portNum) const {
return MemoryDescUtils::convertToDnnlMemoryDesc(getBaseMemDescAtInputPort(portNum));
}
template<>
BlockedMemoryDescPtr MKLDNNNode::getInputMemDescAtPort<BlockedMemoryDesc, 0, 0>(size_t portNum) const {
return MemoryDescUtils::convertToBlockedMemoryDesc(getBaseMemDescAtInputPort(portNum));
}
template<>
DnnlMemoryDescPtr MKLDNNNode::getOutputMemDescAtPort<DnnlMemoryDesc, 0, 0>(size_t portNum) const {
return MemoryDescUtils::convertToDnnlMemoryDesc(getBaseMemDescAtOutputPort(portNum));
}
template<>
BlockedMemoryDescPtr MKLDNNNode::getOutputMemDescAtPort<BlockedMemoryDesc, 0, 0>(size_t portNum) const {
return MemoryDescUtils::convertToBlockedMemoryDesc(getBaseMemDescAtOutputPort(portNum));
}
} // namespace MKLDNNPlugin
MKLDNNNode::NodesFactory & MKLDNNNode::factory() {
@@ -257,11 +279,11 @@ MKLDNNNode::MKLDNNNode(const std::shared_ptr<ngraph::Node>& op, const mkldnn::en
for (size_t i = 0; i < op->get_input_size(); i++) {
const auto &shape = op->get_input_partial_shape(i);
bool isScalar = false;
if (shape.rank().is_static()) {
isScalar = shape.rank().get_length() == 0;
if (shape.rank().is_dynamic()) {
IE_THROW(Unexpected) << "CPU plug-in doesn't support operation with dynamic rank";
}
bool isScalar = shape.rank().get_length() == 0;
inputShapes.emplace_back(isScalar ? ngraph::PartialShape{1} : shape);
originalInputPrecisions.emplace_back(details::convertPrecision(op->get_input_element_type(i)));
}
@@ -272,16 +294,19 @@ MKLDNNNode::MKLDNNNode(const std::shared_ptr<ngraph::Node>& op, const mkldnn::en
}
for (size_t i = 0; i < op->get_output_size(); i++) {
const auto &shape = op->get_output_partial_shape(i);
bool isScalar = false;
if (shape.rank().is_static()) {
isScalar = shape.rank().get_length() == 0;
if (shape.rank().is_dynamic()) {
IE_THROW(Unexpected) << "CPU plug-in doesn't support operation with dynamic rank";
}
bool isScalar = shape.rank().get_length() == 0;
outputShapes.emplace_back(isScalar ? ngraph::PartialShape{1} : shape);
originalOutputPrecisions.emplace_back(details::convertPrecision(op->get_output_element_type(i)));
}
}
isDynamic = std::any_of(inputShapes.begin(), inputShapes.end(), [](const Shape& shape){ return shape.isDynamic(); }) ||
std::any_of(outputShapes.begin(), outputShapes.end(), [](const Shape& shape){ return shape.isDynamic(); });
const auto& rtInfo = op->get_rt_info();
if (rtInfo.count("originalLayersNames")) {
originalLayers = getRTInfoValue(rtInfo, "originalLayersNames");
@@ -462,16 +487,18 @@ bool MKLDNNNode::canBeInPlace() const {
return false;
}
auto inShape = getParentEdgeAt(0)->getShape();
for (size_t cIdx = 0; cIdx < getChildEdges().size(); cIdx++) {
if (getChildEdgeAt(cIdx)->getShape() != inShape) {
auto inShape = getInputShapeAtPort(0);
for (size_t cIdx = 0; cIdx < outputShapes.size(); cIdx++) {
if (getOutputShapeAtPort(cIdx) != inShape) {
return false;
}
}
return true;
}
void MKLDNNNode::resolveNotAllocatedEdges() {
void MKLDNNNode::resolveInPlaceEdges() {
// TODO [DS]: first version dynamic shapes do not support inPlace logic
// after enabling inPlace logic for dynamic shapes we need to update this method for nodes with several edges at single port
const NodeDesc *selected_pd = getSelectedPrimitiveDescriptor();
if (!selected_pd)
IE_THROW() << "Cannot find selected primitive descriptor for node: " << getName();
@@ -501,6 +528,28 @@ void MKLDNNNode::resolveNotAllocatedEdges() {
}
}
MemoryDescPtr MKLDNNNode::getBaseMemDescAtInputPort(size_t portNum) const {
if (auto primDesc = getSelectedPrimitiveDescriptor()) {
const auto& inConfs = primDesc->getConfig().inConfs;
if (inConfs.size() < portNum) {
IE_THROW() << "Can't get input memory desc at port: " << portNum << ", incorrect port number";
}
return inConfs[portNum].desc;
}
IE_THROW() << "Can't get input memory desc, primitive descriptor is not selected";
}
MemoryDescPtr MKLDNNNode::getBaseMemDescAtOutputPort(size_t portNum) const {
if (auto primDesc = getSelectedPrimitiveDescriptor()) {
const auto& outConfs = primDesc->getConfig().outConfs;
if (outConfs.size() < portNum) {
IE_THROW() << "Can't get output memory desc at port: " << portNum << ", incorrect port number";
}
return outConfs[portNum].desc;
}
IE_THROW() << "Can't get output memory desc, primitive descriptor is not selected";
}
std::string MKLDNNNode::getPrimitiveDescriptorType() {
auto selectedPrimitiveDesc = getSelectedPrimitiveDescriptor();
@@ -636,6 +685,37 @@ void MKLDNNNode::execute(mkldnn::stream strm) {
}
}
void MKLDNNNode::executeDynamic(mkldnn::stream strm) {
const auto newShapes = shapeInfer();
if (!newShapes.empty())
redefineOutputMemory(newShapes);
executeDynamicImpl(strm);
}
void MKLDNNNode::redefineOutputMemory(const std::vector<VectorDims> &newShapes) {
if (newShapes.size() != outputShapes.size()) {
IE_THROW() << "Number shapes mismatch with real outputs number for node with name: " << getName();
}
for (size_t i = 0; i < outputShapes.size(); i++) {
const auto edges = getChildEdgesAtPort(i);
const auto memDesc = getBaseMemDescAtOutputPort(i)->cloneWithNewDims(newShapes[i]);
size_t sharedEdgeNum = 0;
for (size_t j = 0; j < edges.size(); j++) {
if (!edges[j]->getMemory().isUsedExternalStorage()) {
sharedEdgeNum = j;
break;
}
}
edges[sharedEdgeNum]->getMemoryPtr()->redefineDesc(*memDesc);
void *data = edges[sharedEdgeNum]->getMemoryPtr()->GetData();
for (size_t j = 0; j < edges.size(); j++) {
if (j == sharedEdgeNum)
continue;
edges[j]->getMemoryPtr()->redefineDesc(*memDesc, data);
}
}
}
void MKLDNNNode::initSupportedPrimitiveDescriptors() {
if (!supportedPrimitiveDescriptors.empty())
return;
@@ -650,7 +730,12 @@ void MKLDNNNode::initSupportedPrimitiveDescriptors() {
PortConfig portConfig;
portConfig.inPlace = -1;
portConfig.constant = false;
portConfig.desc = MemoryDescUtils::applyUndefinedOffset(*getSrcMemDesc(itpd, i));
auto desc = getSrcMemDesc(itpd, i);
if (desc->getType() & MemoryDescType::Blocked) {
portConfig.desc = MemoryDescUtils::cloneWithUndefStridesAndOffset(*desc);
} else {
portConfig.desc = std::move(desc);
}
config.inConfs.push_back(portConfig);
}
@@ -658,7 +743,12 @@ void MKLDNNNode::initSupportedPrimitiveDescriptors() {
PortConfig portConfig;
portConfig.inPlace = canBeInPlace() ? 0 : -1;
portConfig.constant = false;
portConfig.desc = MemoryDescUtils::applyUndefinedOffset(*getDstMemDesc(itpd, i));
auto desc = getDstMemDesc(itpd, i);
if (desc->getType() & MemoryDescType::Blocked) {
portConfig.desc = MemoryDescUtils::cloneWithUndefStridesAndOffset(*desc);
} else {
portConfig.desc = std::move(desc);
}
config.outConfs.push_back(portConfig);
}
impl_desc_type impl_type = parse_impl_name(itpd.impl_info_str());
@@ -673,10 +763,9 @@ void MKLDNNNode::initSupportedPrimitiveDescriptors() {
void MKLDNNNode::filterSupportedPrimitiveDescriptors() {
// Compare by partial layout descriptor (without particular strides values)
auto areCompatible = [](const MemoryDesc& desc, mkldnn::memory::format_tag fmt) -> bool {
MKLDNNMemoryDesc fmt_tdesc = MKLDNNMemoryDesc{desc.getShape().getStaticDims(),
MKLDNNExtensionUtils::IEPrecisionToDataType(desc.getPrecision()),
fmt};
auto fmt_tdesc = DnnlBlockedMemoryDesc(desc.getShape(),
MKLDNNExtensionUtils::IEPrecisionToDataType(desc.getPrecision()),
fmt);
return desc.isCompatible(fmt_tdesc);
};
@@ -710,12 +799,12 @@ void MKLDNNNode::initDescriptor(const NodeConfig& config) {
if (!selectedPD) {
return;
}
std::vector<const MemoryDesc*> inDescs;
std::vector<MemoryDescPtr> inDescs;
for (const auto& inConf : config.inConfs)
inDescs.push_back(inConf.desc.get());
std::vector<const MemoryDesc*> outDescs;
inDescs.emplace_back(inConf.desc);
std::vector<MemoryDescPtr> outDescs;
for (const auto& outConf : config.outConfs)
outDescs.push_back(outConf.desc.get());
outDescs.emplace_back(outConf.desc);
createDescriptor(inDescs, outDescs);
std::shared_ptr<mkldnn::primitive_attr> attr = initPrimitiveAttr();
@@ -773,12 +862,12 @@ void MKLDNNNode::initDescriptor(const NodeConfig& config) {
for (size_t i = 0; i < selectedConfig.inConfs.size(); i++) {
if (!selectedConfig.inConfs[i].desc->isCompatible(*config.inConfs[i].desc))
IE_THROW() << "Incorrect descriptor for node: " << getName();
IE_THROW() << "Incorrect descriptor for node: " << getName() << " on " << i << " intput port";
}
for (size_t i = 0; i < selectedConfig.outConfs.size(); i++) {
if (!selectedConfig.outConfs[i].desc->isCompatible(*config.outConfs[i].desc))
IE_THROW() << "Incorrect descriptor for node: " << getName();
IE_THROW() << "Incorrect descriptor for node: " << getName() << " on " << i << " output port";
}
rightConfig = config;
}
@@ -799,7 +888,7 @@ void MKLDNNNode::prepareMemory(const NodeDesc *selected_pd, mkldnn::primitive_de
IE_THROW() << "Destination memory didn't allocate for node " << getName()
<< " from node " << getParentEdgeAt(i)->getParent()->getName() << ".";
}
std::vector<MKLDNNMemoryDesc> intDescs;
std::vector<DnnlMemoryDescPtr> intDescs;
for (auto &it : internalBlobDesc)
intDescs.push_back(it(itpd, 0));
@@ -809,13 +898,13 @@ void MKLDNNNode::prepareMemory(const NodeDesc *selected_pd, mkldnn::primitive_de
auto create = [&] () {
// TODO [DS]: internal blobs should be removed or rewritten using Memory object
auto newDesc = MemoryDescUtils::convertToMKLDNNMemoryDesc(internalBlob->getTensorDesc());
auto newDesc = MemoryDescUtils::convertToDnnlBlockedMemoryDesc(internalBlob->getTensorDesc());
MKLDNNMemory memory{ engine };
memory.Create(newDesc, internalBlob->buffer());
MKLDNNMemoryPtr _ptr = MKLDNNMemoryPtr(new MKLDNNMemory(engine));
_ptr->Create(intDescs[i]);
_ptr->Create(*intDescs[i]);
_ptr->SetData(memory);
return _ptr;
@@ -950,14 +1039,14 @@ const std::vector<impl_desc_type>& MKLDNNNode::getPrimitivesPriority() {
return implPriorities;
}
std::unique_ptr<MemoryDesc> MKLDNNNode::getDefinedInputDesc(const NodeConfig &config, size_t idx) const {
MemoryDescPtr MKLDNNNode::getDefinedInputDesc(const NodeConfig &config, size_t idx) const {
int num = getParentEdgeAt(idx)->getInputNum();
auto *selectedPD = getParentEdgeAt(idx)->getParent()->getSelectedPrimitiveDescriptor();
if (!selectedPD)
IE_THROW() << "Cannot get selected primitive descriptor for node: " << getParentEdgeAt(idx)->getParent()->getName();
if (config.inConfs[idx].desc->isDefined()) {
return config.inConfs[idx].desc->clone();
return config.inConfs[idx].desc;
}
if (config.inConfs[idx].inPlace >= 0) {
@@ -966,26 +1055,26 @@ std::unique_ptr<MemoryDesc> MKLDNNNode::getDefinedInputDesc(const NodeConfig &co
if (num >= 0) {
auto parentConf = selectedPD->getConfig().outConfs[num];
parentConf.desc->setPrecision(config.inConfs[idx].desc->getPrecision());
parentConf.desc = MemoryDescUtils::cloneWithNewPrecision(*parentConf.desc, config.inConfs[idx].desc->getPrecision());
if (!parentConf.desc->isDefined() && parentConf.inPlace >= 0)
getParentEdgeAt(idx)->getParent()->initOptimalPrimitiveDescriptor();
parentConf = getParentEdgeAt(idx)->getParent()->getSelectedPrimitiveDescriptor()->getConfig().outConfs[num];
if (parentConf.desc->isDefined() && parentConf.desc->isCompatible(*config.inConfs[idx].desc)) {
return parentConf.desc->clone();
return parentConf.desc;
}
}
return MemoryDescUtils::resetOffset(config.inConfs[idx].desc.get());
return MemoryDescUtils::cloneWithDefaultStridesAndOffset(*config.inConfs[idx].desc);
}
std::unique_ptr<MemoryDesc> MKLDNNNode::getDefinedOutputDesc(const NodeConfig &config, size_t idx) const {
MemoryDescPtr MKLDNNNode::getDefinedOutputDesc(const NodeConfig &config, size_t idx) const {
int num = getChildEdgeAt(idx)->getOutputNum();
auto *selectedPD = getChildEdgeAt(idx)->getChild()->getSelectedPrimitiveDescriptor();
if (!selectedPD)
IE_THROW() << "Cannot get selected primitive descriptor for node: " << getChildEdgeAt(idx)->getChild()->getName();
if (config.outConfs[idx].desc->isDefined()) {
return config.outConfs[idx].desc->clone();
return config.outConfs[idx].desc;
}
if (config.outConfs[idx].inPlace >= 0) {
@@ -994,16 +1083,16 @@ std::unique_ptr<MemoryDesc> MKLDNNNode::getDefinedOutputDesc(const NodeConfig &c
if (num >= 0) {
auto childConf = selectedPD->getConfig().inConfs[num];
childConf.desc->setPrecision(config.outConfs[idx].desc->getPrecision());
childConf.desc = MemoryDescUtils::cloneWithNewPrecision(*childConf.desc, config.outConfs[idx].desc->getPrecision());
if (!childConf.desc->isDefined() && childConf.inPlace >= 0)
getChildEdgeAt(idx)->getChild()->initOptimalPrimitiveDescriptor();
childConf = getChildEdgeAt(idx)->getChild()->getSelectedPrimitiveDescriptor()->getConfig().inConfs[num];
if (childConf.desc->isDefined() && childConf.desc->isCompatible(*config.outConfs[idx].desc)) {
return childConf.desc->clone();
return childConf.desc;
}
}
return MemoryDescUtils::resetOffset(config.outConfs[idx].desc.get());
return MemoryDescUtils::cloneWithDefaultStridesAndOffset(*config.outConfs[idx].desc);
}
void MKLDNNNode::initOptimalPrimitiveDescriptor() {
@@ -1036,19 +1125,20 @@ bool MKLDNNNode::isConfigDefined(const NodeConfig &config) const {
return true;
}
std::unique_ptr<MKLDNNMemoryDesc> MKLDNNNode::getSrcMemDesc(mkldnn::primitive_desc_iterator &primitive_desc_it, size_t idx) {
return MKLDNNPlugin::make_unique<MKLDNNMemoryDesc>(primitive_desc_it.src_desc(idx));
MemoryDescPtr MKLDNNNode::getSrcMemDesc(mkldnn::primitive_desc_iterator &primitive_desc_it, size_t idx) {
return MKLDNNExtensionUtils::makeDescriptor(primitive_desc_it.src_desc(idx));
}
std::unique_ptr<MKLDNNMemoryDesc> MKLDNNNode::getDstMemDesc(mkldnn::primitive_desc_iterator &primitive_desc_it, size_t idx) {
return MKLDNNPlugin::make_unique<MKLDNNMemoryDesc>(primitive_desc_it.dst_desc(idx));
MemoryDescPtr MKLDNNNode::getDstMemDesc(mkldnn::primitive_desc_iterator &primitive_desc_it, size_t idx) {
return MKLDNNExtensionUtils::makeDescriptor(primitive_desc_it.dst_desc(idx));
}
int MKLDNNNode::batchToProcess() {
return dynBatchLim == 0 ? getMaxBatch() : std::min<int>(getMaxBatch(), dynBatchLim);
}
int MKLDNNNode::getMaxBatch() {
// TODO [DS]: how we should process this for dynamic shape?
size_t MKLDNNNode::getMaxBatch() {
// FIXME: batch != 0 dims number
if (!inputShapes.empty()) {
if (inputShapes[0].getRank())
@@ -1175,7 +1265,9 @@ MKLDNNNode* MKLDNNNode::NodesFactory::create(const std::shared_ptr<ngraph::Node>
if (ex.getStatus() != NOT_IMPLEMENTED) {
throw;
} else {
errorMessage += getExceptionDescWithoutStatus(ex);
const auto currErrorMess = getExceptionDescWithoutStatus(ex);
if (!currErrorMess.empty())
errorMessage += "\n" + currErrorMess;
}
IE_SUPPRESS_DEPRECATED_END
}
@@ -1190,7 +1282,9 @@ MKLDNNNode* MKLDNNNode::NodesFactory::create(const std::shared_ptr<ngraph::Node>
if (ex.getStatus() != NOT_IMPLEMENTED) {
throw;
} else {
errorMessage += getExceptionDescWithoutStatus(ex);
const auto currErrorMess = getExceptionDescWithoutStatus(ex);
if (!currErrorMess.empty())
errorMessage += "\n" + currErrorMess;
}
IE_SUPPRESS_DEPRECATED_END
}
@@ -1206,7 +1300,9 @@ MKLDNNNode* MKLDNNNode::NodesFactory::create(const std::shared_ptr<ngraph::Node>
if (ex.getStatus() != NOT_IMPLEMENTED) {
throw;
} else {
errorMessage += getExceptionDescWithoutStatus(ex);
const auto currErrorMess = getExceptionDescWithoutStatus(ex);
if (!currErrorMess.empty())
errorMessage += "\n" + currErrorMess;
}
IE_SUPPRESS_DEPRECATED_END
}
@@ -1222,7 +1318,7 @@ MKLDNNNode* MKLDNNNode::NodesFactory::create(const std::shared_ptr<ngraph::Node>
if (!newNode) {
std::string errorDetails;
if (!errorMessage.empty()) {
errorDetails = "\nDetails: \n" + errorMessage;
errorDetails = "\nDetails: " + errorMessage;
}
IE_THROW() << "Unsupported operation of type: " << op->get_type_name() << " name: " << op->get_friendly_name() << errorDetails;
}
@@ -1233,7 +1329,7 @@ MKLDNNNode* MKLDNNNode::NodesFactory::create(const std::shared_ptr<ngraph::Node>
bool MKLDNNNode::canBePerformedAsScaleShift(const MKLDNNNode *parentNode) const {
size_t fusingPort = 0;
for (size_t i = (parentNode == nullptr ? 1 : 0); i < getParentEdges().size(); i++) {
MKLDNNNode *node = getParentEdgeAt(i)->getParent().get();
MKLDNNNode *node = getParentEdgesAtPort(i)[0]->getParent().get();
if (node == nullptr) {
IE_THROW() << "Cannot get parent node for " << getName() << " on " << i << " port";
}
@@ -1247,11 +1343,11 @@ bool MKLDNNNode::canBePerformedAsScaleShift(const MKLDNNNode *parentNode) const
}
const auto isBroadcastableToDataInput = [&]() {
const auto dataShape = getParentEdgeAt(fusingPort)->getShape().getStaticDims();
auto& dataShape = getInputShapeAtPort(fusingPort).getDims();
for (size_t i = 0; i < getParentEdges().size(); i++) {
if (i == fusingPort)
continue;
auto weightShape = getParentEdgeAt(i)->getShape().getStaticDims();
auto& weightShape = getInputShapeAtPort(i).getDims();
if (getParentEdgesAtPort(i)[0]->getParent()->getChildEdges().size() != 1 || !isPerTensorOrPerChannelBroadcastable(dataShape, weightShape))
return false;
}
@@ -1295,7 +1391,7 @@ void MKLDNNNode::fillScalesAndShifts(const MKLDNNNode *parentNode, std::vector<f
const auto fillValuesFrom = [&](const MKLDNNNodePtr& constInput, std::vector<float>& buffer) {
auto *constInputNode = dynamic_cast<MKLDNNInputNode *>(constInput.get());
auto constBlob = constInputNode->getMemoryPtr();
auto const elementsCount = constBlob->GetElementsCount();
const auto elementsCount = constBlob->GetDescWithType<BlockedMemoryDesc>()->getPaddedElementsCount();
buffer.resize(elementsCount);
cpu_convert(constBlob->GetPtr(),
&buffer[0],
inference-engine/src/mkldnn_plugin/mkldnn_node.h
+100 -17
View File
@@ -11,7 +11,6 @@
#include <cassert>
#include <algorithm>
#include <caseless.hpp>
#include "mkldnn_dims.h"
#include "mkldnn_memory.h"
#include "mkldnn_edge.h"
#include "mkldnn_descriptor.h"
@@ -29,11 +28,12 @@
#include <nodes/common/blocked_desc_creator.h>
#include "cpu_types.h"
#include "cpu_shape.h"
#include "cpu_memory_desc.h"
#include "memory_desc/cpu_memory_desc.h"
namespace MKLDNNPlugin {
using MKLDNNNodePtr = std::shared_ptr<MKLDNNNode>;
using MKLDNNNodeConstPtr = std::shared_ptr<const MKLDNNNode>;
using MKLDNNNodeWeakPtr = std::weak_ptr<MKLDNNNode>;
Type TypeFromName(const std::string type);
@@ -238,7 +238,7 @@ struct PortConfig {
this->constant = rhs.constant;
this->inPlace = rhs.inPlace;
if (rhs.desc) {
this->desc = rhs.desc->clone();
this->desc = rhs.desc;
}
}
@@ -246,7 +246,7 @@ struct PortConfig {
this->constant = rhs.constant;
this->inPlace = rhs.inPlace;
if (rhs.desc) {
this->desc = rhs.desc->clone();
this->desc = rhs.desc;
}
return *this;
}
@@ -257,7 +257,7 @@ struct PortConfig {
// TODO [DS]: better to make private and const
bool constant = false;
int inPlace = -1;
std::unique_ptr<MemoryDesc> desc;
MemoryDescPtr desc;
};
struct NodeConfig {
@@ -454,6 +454,44 @@ public:
return &supportedPrimitiveDescriptors[selectedPrimitiveDescriptorIndex];
}
/**
* @brief Returns input selected primitive descriptor on the specified port
* must be used after selectOptimalPrimitiveDescriptor stage
* @param portNum port number
* @return pointer to selected primitive descriptor with type MemoryDesc
*/
MemoryDescPtr getBaseMemDescAtInputPort(size_t portNum) const;
/**
* @brief Returns output selected primitive descriptor on the specified port
* must be used after selectOptimalPrimitiveDescriptor stage
* @param portNum port number
* @return pointer to selected primitive descriptor with type MemoryDesc
*/
MemoryDescPtr getBaseMemDescAtOutputPort(size_t portNum) const;
/**
* @brief Returns input selected primitive descriptor on the specified port
* must be used after selectOptimalPrimitiveDescriptor stage
* @param portNum port number
* @return pointer to selected primitive descriptor with type T
*/
template <typename T,
typename std::enable_if<!std::is_pointer<T>::value && !std::is_reference<T>::value, int>::type = 0,
typename std::enable_if<std::is_base_of<MemoryDesc, T>::value, int>::type = 0>
std::shared_ptr<T> getInputMemDescAtPort(size_t portNum) const;
/**
* @brief Returns output selected primitive descriptor on the specified port
* must be used after selectOptimalPrimitiveDescriptor stage
* @param portNum port number
* @return pointer to selected primitive descriptor with type T
*/
template <typename T,
typename std::enable_if<!std::is_pointer<T>::value && !std::is_reference<T>::value, int>::type = 0,
typename std::enable_if<std::is_base_of<MemoryDesc, T>::value, int>::type = 0>
std::shared_ptr<T> getOutputMemDescAtPort(size_t portNum) const;
void selectPrimitiveDescriptorByIndex(int index) {
if (index < 0 || index >= supportedPrimitiveDescriptors.size())
selectedPrimitiveDescriptorIndex = -1;
@@ -467,8 +505,12 @@ public:
virtual void setDynamicBatchLim(int lim);
void resolveNotAllocatedEdges();
void resolveInPlaceEdges();
virtual void execute(mkldnn::stream strm);
void executeDynamic(mkldnn::stream strm);
void redefineOutputMemory(const std::vector<VectorDims> &newShapes);
virtual void initSupportedPrimitiveDescriptors();
/**
@@ -484,8 +526,8 @@ public:
virtual void getSupportedDescriptors() = 0;
// TODO [DS]: Should be moved into Node derivative class
virtual void createDescriptor(const std::vector<const MemoryDesc*>& inputDesc,
const std::vector<const MemoryDesc*>& outputDesc) {}
virtual void createDescriptor(const std::vector<MemoryDescPtr>& inputDesc,
const std::vector<MemoryDescPtr>& outputDesc) {}
virtual void initDescriptor(const NodeConfig& config);
virtual bool created() const = 0;
virtual bool created(const MKLDNNExtensionManager::Ptr& extMgr) {
@@ -621,10 +663,23 @@ public:
originalOutputPrecisions.push_back(precision);
}
// TODO: alighn behaviour for original(Input/Output)Precisions and (input/output)Shapes
/**
* @brief Returns inputs number which have ngraph nodes.
* Inputs number compute as size of originalInputPrecisions vector
* IMPORTANT!!!
* FuseConvolutionAndBias and FuseMultiplyAndAdd change originalInputPrecisions vector
* @return original inputs number
*/
size_t getOriginalInputsNumber() const {
return originalInputPrecisions.size();
}
/**
* @brief Returns outputs number which have ngraph nodes.
* Outputs number compute as size of originalOutputPrecisions vector
* @return original outputs number
*/
size_t getOriginalOutputsNumber() const {
return originalOutputPrecisions.size();
}
@@ -647,7 +702,33 @@ public:
bool canBePerformedAsScaleShift(const MKLDNNNode *parentNode = nullptr) const;
bool isDynamicNode() const {
return isDynamic;
}
const Shape& getInputShapeAtPort(size_t port) const {
if (inputShapes.size() <= port) {
IE_THROW() << "Incorrect input port number for node " << getName();
}
return inputShapes[port];
}
const Shape& getOutputShapeAtPort(size_t port) const {
if (outputShapes.size() <= port) {
IE_THROW() << "Incorrect output port number for node " << getName();
}
return outputShapes[port];
}
protected:
// TODO [DS] : make pure after all nodes will be support dynamic shapes
virtual std::vector<VectorDims> shapeInfer() const {
IE_THROW(NotImplemented) << "[DS] MKLDNNNode::shapeInfer is not defined for node with type: " << getTypeStr();
}
virtual void executeDynamicImpl(mkldnn::stream strm) {
IE_THROW(NotImplemented) << "[DS] executeDynamicImpl not implemented for node with type: " << getTypeStr();
}
bool canFuseSimpleOperation(const MKLDNNNodePtr& node) const;
// TODO [mandrono]: place outside of the node API
void fillScalesAndShifts(const MKLDNNNode *parentNode, std::vector<float> &scales, std::vector<float> &shifts, const int align = -1);
@@ -656,13 +737,13 @@ protected:
this->type = type;
}
virtual int getMaxBatch();
virtual size_t getMaxBatch();
virtual std::unique_ptr<MemoryDesc> getDefinedInputDesc(const NodeConfig &config, size_t idx) const;
virtual std::unique_ptr<MemoryDesc> getDefinedOutputDesc(const NodeConfig &config, size_t idx) const;
virtual std::unique_ptr<MKLDNNMemoryDesc> getSrcMemDesc(mkldnn::primitive_desc_iterator &primitive_desc_it, size_t idx);
virtual std::unique_ptr<MKLDNNMemoryDesc> getDstMemDesc(mkldnn::primitive_desc_iterator &primitive_desc_it, size_t idx);
virtual MemoryDescPtr getDefinedInputDesc(const NodeConfig &config, size_t idx) const;
virtual MemoryDescPtr getDefinedOutputDesc(const NodeConfig &config, size_t idx) const;
virtual MemoryDescPtr getSrcMemDesc(mkldnn::primitive_desc_iterator &primitive_desc_it, size_t idx);
virtual MemoryDescPtr getDstMemDesc(mkldnn::primitive_desc_iterator &primitive_desc_it, size_t idx);
/**
* @brief Appends new item into ops list with the information on how the node should be executed as post operation.
@@ -672,7 +753,7 @@ protected:
virtual void appendPostOps(mkldnn::post_ops& ops);
virtual std::shared_ptr<mkldnn::primitive_attr> initPrimitiveAttr() const { return nullptr; }
typedef std::function<MKLDNNMemoryDesc (mkldnn::primitive_desc_iterator &primitive_desc_it, size_t idx)>
typedef std::function<DnnlMemoryDescPtr (mkldnn::primitive_desc_iterator &primitive_desc_it, size_t idx)>
GetPrimitiveMemoryFormatFunc;
std::vector<GetPrimitiveMemoryFormatFunc> internalBlobDesc;
@@ -755,7 +836,7 @@ protected:
PortConfig portConfig;
portConfig.inPlace = portConfigurator.inPlace;
portConfig.constant = portConfigurator.constant;
portConfig.desc = portConfigurator.blockedDescCreator->createUniqueDesc(prc, shape.getStaticDims());
portConfig.desc = portConfigurator.blockedDescCreator->createSharedDesc(prc, shape);
port.push_back(std::move(portConfig));
@@ -764,14 +845,14 @@ protected:
NodeConfig config;
for (size_t i = 0; i < inPortConfigs.size(); i++) {
auto shape = inPortConfigs[i].shape.getRank() == 0 ? getParentEdgesAtPort(i)[0]->getShape() : inPortConfigs[i].shape;
auto shape = inPortConfigs[i].shape.getRank() == 0 ? getInputShapeAtPort(i) : inPortConfigs[i].shape;
auto prc = inPortConfigs[i].prc == InferenceEngine::Precision::UNSPECIFIED ? getOriginalInputPrecisionAtPort(i) : inPortConfigs[i].prc;
if (!fill_port(inPortConfigs[i], shape, prc, config.inConfs))
return;
}
for (size_t i = 0; i < outPortConfigs.size(); i++) {
auto dims = outPortConfigs[i].shape.getRank() == 0 ? getChildEdgesAtPort(i)[0]->getShape() : outPortConfigs[i].shape;
auto dims = outPortConfigs[i].shape.getRank() == 0 ? getOutputShapeAtPort(i) : outPortConfigs[i].shape;
auto prc = outPortConfigs[i].prc == InferenceEngine::Precision::UNSPECIFIED ? getOriginalOutputPrecisionAtPort(i) : outPortConfigs[i].prc;
if (!fill_port(outPortConfigs[i], dims, prc, config.outConfs))
return;
@@ -782,6 +863,8 @@ protected:
}
private:
bool isDynamic = false;
std::vector<MKLDNNEdgeWeakPtr> parentEdges;
std::vector<MKLDNNEdgeWeakPtr> childEdges;
inference-engine/src/mkldnn_plugin/mkldnn_plugin.cpp
+13
View File
@@ -289,6 +289,19 @@ static void Transformation(CNNNetwork& clonedNetwork, const Config& conf) {
return node->input_value(0).get_partial_shape().rank().get_length() > 5;
});
pass_config->set_callback<ngraph::pass::ConvertNMSToNMSIEInternal>(
[](const_node_ptr &node) -> bool {
for (size_t i = 0; i < node->get_output_size(); i++) {
const auto outputs = node->get_output_target_inputs(i);
for (const auto &out : outputs) {
if (out.get_node()->get_type_info() != ngraph::op::v0::Result::type_info) {
return false;
}
}
}
return true;
});
// List of enabled/disabled transformations
pass_config->disable<ngraph::pass::ConvertGELU>();
pass_config->disable<ngraph::pass::ConvertShuffleChannels3>();
inference-engine/src/mkldnn_plugin/nodes/common/blocked_desc_creator.cpp
+13 -13
View File
@@ -13,21 +13,21 @@ constexpr size_t channelsPos = 1lu;
class PlainFormatCreator : public BlockedDescCreator {
public:
BlockedMemoryDesc createDesc(const InferenceEngine::Precision& precision, const InferenceEngine::SizeVector& srcDims) const override {
SizeVector order(srcDims.size());
CpuBlockedMemoryDesc createDesc(const InferenceEngine::Precision& precision, const Shape& srcShape) const override {
SizeVector order(srcShape.getRank());
std::iota(order.begin(), order.end(), 0);
return BlockedMemoryDesc(precision, srcDims, srcDims, order);
return CpuBlockedMemoryDesc(precision, srcShape, srcShape.getDims(), order);
}
size_t getMinimalRank() const override { return 0lu; }
};
class PerChannelCreator : public BlockedDescCreator {
public:
BlockedMemoryDesc createDesc(const InferenceEngine::Precision &precision, const InferenceEngine::SizeVector &srcDims) const override {
SizeVector order(srcDims.size());
CpuBlockedMemoryDesc createDesc(const InferenceEngine::Precision &precision, const Shape& srcShape) const override {
SizeVector order(srcShape.getRank());
std::iota(order.begin(), order.end(), 0);
SizeVector blkDims = srcDims;
if (srcDims.size() > 2) {
SizeVector blkDims = srcShape.getDims();
if (srcShape.getRank() > 2) {
auto moveElementBack = [](SizeVector& vector, size_t indx) {
auto itr = vector.begin() + indx;
std::rotate(itr, itr + 1, vector.end());
@@ -37,7 +37,7 @@ public:
moveElementBack(blkDims, channelsPos);
}
return BlockedMemoryDesc(precision, srcDims, blkDims, order);
return CpuBlockedMemoryDesc(precision, srcShape, blkDims, order);
}
size_t getMinimalRank() const override { return 3lu; }
};
@@ -45,22 +45,22 @@ public:
class ChannelBlockedCreator : public BlockedDescCreator {
public:
ChannelBlockedCreator(size_t blockSize) : _blockSize(blockSize) {}
BlockedMemoryDesc createDesc(const InferenceEngine::Precision& precision, const InferenceEngine::SizeVector& srcDims) const override {
if (srcDims.size() < 2) {
CpuBlockedMemoryDesc createDesc(const InferenceEngine::Precision& precision, const Shape& srcShape) const override {
if (srcShape.getRank() < 2) {
IE_THROW() << "Can't create blocked tensor descriptor!";
}
SizeVector order(srcDims.size());
SizeVector order(srcShape.getRank());
std::iota(order.begin(), order.end(), 0);
order.push_back(channelsPos);
SizeVector blkDims = srcDims;
SizeVector blkDims = srcShape.getDims();
if (Shape::UNDEFINED_DIM != blkDims[channelsPos]) {
blkDims[channelsPos] = blkDims[channelsPos] / _blockSize + (blkDims[channelsPos] % _blockSize ? 1 : 0);
}
blkDims.push_back(_blockSize);
return BlockedMemoryDesc(precision, srcDims, blkDims, order);
return CpuBlockedMemoryDesc(precision, srcShape, blkDims, order);
}
size_t getMinimalRank() const override { return 3lu; }
inference-engine/src/mkldnn_plugin/nodes/common/blocked_desc_creator.h
+6 -4
View File
@@ -6,7 +6,7 @@
#include <functional>
#include "cpu_shape.h"
#include "cpu_blocked_memory_desc.h"
#include "memory_desc/cpu_blocked_memory_desc.h"
namespace MKLDNNPlugin {
@@ -27,10 +27,12 @@ public:
makeFilteredRange(const CreatorsMap& map, unsigned rank, const std::vector<LayoutType>& supportedTypes);
static std::pair<CreatorsMapFilterConstIterator, CreatorsMapFilterConstIterator>
makeFilteredRange(const CreatorsMap& map, Predicate predicate);
virtual BlockedMemoryDesc createDesc(const InferenceEngine::Precision& precision, const InferenceEngine::SizeVector& srcDims) const = 0;
std::unique_ptr<BlockedMemoryDesc> createUniqueDesc(const InferenceEngine::Precision& precision, const InferenceEngine::SizeVector& srcDims) const {
return MKLDNNPlugin::make_unique<BlockedMemoryDesc>(createDesc(precision, srcDims));
virtual CpuBlockedMemoryDesc createDesc(const InferenceEngine::Precision& precision, const Shape& srcShape) const = 0;
std::shared_ptr<CpuBlockedMemoryDesc> createSharedDesc(const InferenceEngine::Precision& precision, const Shape& srcShape) const {
return std::make_shared<CpuBlockedMemoryDesc>(createDesc(precision, srcShape));
}
virtual size_t getMinimalRank() const = 0;
virtual ~BlockedDescCreator() = default;
};
inference-engine/src/mkldnn_plugin/nodes/mkldnn_adaptive_pooling.cpp
+37 -32
View File
@@ -21,16 +21,20 @@ using namespace InferenceEngine;
using namespace mkldnn;
using namespace mkldnn::impl::cpu::x64;
bool MKLDNNAdaptivePoolingNode::isSupportedOperation(const std::shared_ptr<ngraph::Node>& op, std::string& errorMessage) noexcept {
bool MKLDNNAdaptivePoolingNode::isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept {
try {
if (isDynamicNgraphNode(op)) {
errorMessage = "Doesn't support op with dynamic shapes";
return false;
}
if (one_of(op->get_type_info(), ngraph::op::v8::AdaptiveAvgPool::type_info)) {
auto adaPool = std::dynamic_pointer_cast<ngraph::opset8::AdaptiveAvgPool>(op);
auto adaPool = std::dynamic_pointer_cast<const ngraph::opset8::AdaptiveAvgPool>(op);
if (!adaPool) {
errorMessage = "Only opset8 AdaptiveAvgPooling operation is supported";
return false;
}
} else if (one_of(op->get_type_info(), ngraph::op::v8::AdaptiveMaxPool::type_info)) {
auto adaPool = std::dynamic_pointer_cast<ngraph::opset8::AdaptiveMaxPool>(op);
auto adaPool = std::dynamic_pointer_cast<const ngraph::opset8::AdaptiveMaxPool>(op);
if (!adaPool) {
errorMessage = "Only opset8 AdaptiveMaxPooling operation is supported";
return false;
@@ -69,19 +73,19 @@ void MKLDNNAdaptivePoolingNode::getSupportedDescriptors() {
if (getChildEdges().size() != (algorithm == AdaptivePoolingMax ? 2 : 1))
IE_THROW() << errorPrefix << "has incorrect number of output edges: " << getParentEdges().size();
auto parentDims = getParentEdgeAt(0)->getShape().getStaticDims();
auto childDims = getChildEdgeAt(0)->getShape().getStaticDims();
auto parentDims = getInputShapeAtPort(0).getStaticDims();
auto childDims = getOutputShapeAtPort(0).getStaticDims();
spatialDimsCount = parentDims.size() - 2;
if (!one_of(spatialDimsCount, 1, 2, 3)) {
IE_THROW() << errorPrefix << "doesn't support 0th input with rank: " << getParentEdgeAt(0)->getShape().getRank();
IE_THROW() << errorPrefix << "doesn't support 0th input with rank: " << getInputShapeAtPort(0).getRank();
}
if (getParentEdgeAt(1)->getShape().getRank() != 1) {
IE_THROW() << errorPrefix << "doesn't support 1st input with rank: " << getParentEdgeAt(1)->getShape().getRank();
if (getInputShapeAtPort(1).getRank() != 1) {
IE_THROW() << errorPrefix << "doesn't support 1st input with rank: " << getInputShapeAtPort(1).getRank();
}
if (getChildEdgeAt(0)->getShape().getRank() != getParentEdgeAt(0)->getShape().getRank()) {
if (getOutputShapeAtPort(0).getRank() != getInputShapeAtPort(0).getRank()) {
IE_THROW() << errorPrefix << "must keep data rank";
}
}
@@ -99,7 +103,7 @@ void MKLDNNAdaptivePoolingNode::initSupportedPrimitiveDescriptors() {
config.outConfs.resize((algorithm == Algorithm::AdaptivePoolingAvg ? 1 : 2));
std::vector<LayoutType> dataFormats{ LayoutType::ncsp };
if (getParentEdgeAt(0)->getShape().getStaticDims()[1] != 1) {
if (getInputShapeAtPort(0).getStaticDims()[1] != 1) {
dataFormats.push_back(LayoutType::nspc);
dataFormats.push_back(LayoutType::nCsp16c);
dataFormats.push_back(LayoutType::nCsp8c);
@@ -118,8 +122,8 @@ void MKLDNNAdaptivePoolingNode::initSupportedPrimitiveDescriptors() {
}
void MKLDNNAdaptivePoolingNode::execute(mkldnn::stream strm) {
auto inputPrec = getParentEdgeAt(0)->getMemory().GetDescriptor().data.data_type;
auto outputPrec = getChildEdgeAt(0)->getMemory().GetDescriptor().data.data_type;
auto inputPrec = getParentEdgeAt(0)->getMemory().GetDataType();
auto outputPrec = getChildEdgeAt(0)->getMemory().GetDataType();
if (!(inputPrec == mkldnn_f32 && outputPrec == mkldnn_f32))
IE_THROW() << errorPrefix << "doesn't support demanded precisions";
@@ -131,21 +135,22 @@ void MKLDNNAdaptivePoolingNode::execute(mkldnn::stream strm) {
indexDst = reinterpret_cast<int *>(getChildEdgeAt(1)->getMemoryPtr()->GetPtr());
}
auto srcBlockDesc = srcMemory0.GetDescriptor().data.format_desc.blocking;
auto isPlainFmt = srcMemory0.getDesc().hasLayoutType(LayoutType::ncsp);
auto isTailCFmt = srcMemory0.getDesc().hasLayoutType(LayoutType::nspc);
auto isBlkFmt = srcMemory0.getDesc().hasLayoutType(LayoutType::nCsp16c) || srcMemory0.getDesc().hasLayoutType(LayoutType::nCsp8c);
int blockSize = srcBlockDesc.inner_nblks > 0 ? srcBlockDesc.inner_blks[0] : 1;
auto isPlainFmt = srcMemory0.GetDesc().hasLayoutType(LayoutType::ncsp);
auto isTailCFmt = srcMemory0.GetDesc().hasLayoutType(LayoutType::nspc);
auto srcBlockDesc = srcMemory0.GetDescWithType<BlockedMemoryDesc>();
int blockSize = isBlkFmt ? srcBlockDesc->getBlockDims().back() : 1;
const auto *src = reinterpret_cast<const float *>(getParentEdgeAt(0)->getMemoryPtr()->GetPtr());
const auto *srcPooledSpatialShapes = reinterpret_cast<const int *>(getParentEdgeAt(1)->getMemoryPtr()->GetPtr());
auto *dst = reinterpret_cast<float *>(getChildEdgeAt(0)->getMemoryPtr()->GetPtr());
if (srcMemory1.GetElementsCount() != spatialDimsCount)
IE_THROW() << errorPrefix << "has input spatial dimension (" << srcMemory1.GetElementsCount()
if (srcMemory1.GetShape().getElementsCount() != spatialDimsCount)
IE_THROW() << errorPrefix << "has input spatial dimension (" << srcMemory1.GetShape().getElementsCount()
<< ") inconsistent with pooling vector size (" << spatialDimsCount << ")";
auto inputDimVector = srcMemory0.GetDims();
auto inputDimVector = srcMemory0.getStaticDims();
const int N = static_cast<int>(inputDimVector[0]);
const int C = static_cast<int>(inputDimVector[1]);
const int ID = static_cast<int>(spatialDimsCount == 3 ? inputDimVector[2] : 1);
@@ -159,14 +164,14 @@ void MKLDNNAdaptivePoolingNode::execute(mkldnn::stream strm) {
const int iHW = IH * IW;
const int oDHW = OD * OH * OW, oHW = OH * OW;
const int chPadding = srcMemory0.GetDescriptor().data.padded_dims[1];
const int chPadding = blockSize * (isBlkFmt ? srcBlockDesc->getBlockDims()[1] : srcMemory0.GetShape().getStaticDims()[1]);
const int blockCount = (isTailCFmt ? 1 : chPadding / blockSize);
auto selectedPrimitiveDescriptor = getSelectedPrimitiveDescriptor();
if (!selectedPrimitiveDescriptor)
IE_THROW() << errorPrefix << "doesn't have primitive descriptors.";
auto config = selectedPrimitiveDescriptor->getConfig();
auto srcStrides = getParentEdgesAtPort(0)[0]->getMemory().GetDescWithType<BlockedMemoryDesc>().getStrides();
auto dstStrides = getChildEdgesAtPort(0)[0]->getMemory().GetDescWithType<BlockedMemoryDesc>().getStrides();
auto srcStrides = srcBlockDesc->getStrides();
auto dstStrides = getChildEdgesAtPort(0)[0]->getMemory().GetDescWithType<BlockedMemoryDesc>()->getStrides();
// unified strides array
const size_t tailDimsOffset = (isTailCFmt ? -1 : 0);
@@ -233,20 +238,20 @@ void MKLDNNAdaptivePoolingNode::execute(mkldnn::stream strm) {
[&](int n, int blkIdx, int od, int oh, int ow) {
auto srcData = src + n * inStrides[0] + blkIdx * inStrides[1];
auto dstData = dst + n * outStrides[0] + blkIdx * outStrides[1] +
od * outStrides[2] + oh * outStrides[3] + ow * outStrides[4];
od * outStrides[2] + oh * outStrides[3] + ow * outStrides[4];
int cStart = 0, cEnd = C, inResidual = 0, outResidual = 0;
if (!isTailCFmt) {
cStart = blkIdx * blockSize;
cEnd = (blkIdx == blockCount - 1 ? C : cStart + blockSize);
cStart = blkIdx * blockSize;
cEnd = (blkIdx == blockCount - 1 ? C : cStart + blockSize);
}
for (int c = cStart; c < cEnd; c++) {
if (isTailCFmt) {
inResidual = c * inStrides[1];
outResidual = c * outStrides[1];
} else if (!isPlainFmt) {
inResidual = outResidual = c % blockSize;
}
pool(srcData + inResidual, dstData + outResidual, od, oh, ow, n * C + c);
if (isTailCFmt) {
inResidual = c * inStrides[1];
outResidual = c * outStrides[1];
} else if (!isPlainFmt) {
inResidual = outResidual = c % blockSize;
}
pool(srcData + inResidual, dstData + outResidual, od, oh, ow, n * C + c);
}});
}
inference-engine/src/mkldnn_plugin/nodes/mkldnn_adaptive_pooling.h
+1 -1
View File
@@ -22,7 +22,7 @@ public:
void execute(mkldnn::stream strm) override;
bool created() const override;
static bool isSupportedOperation(const std::shared_ptr<ngraph::Node>& op, std::string& errorMessage) noexcept;
static bool isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept;
private:
int spatialDimsCount;
inference-engine/src/mkldnn_plugin/nodes/mkldnn_batch_to_space_node.cpp
+15 -11
View File
@@ -16,8 +16,12 @@
using namespace MKLDNNPlugin;
using namespace InferenceEngine;
bool MKLDNNBatchToSpaceNode::isSupportedOperation(const std::shared_ptr<ngraph::Node>& op, std::string& errorMessage) noexcept {
bool MKLDNNBatchToSpaceNode::isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept {
try {
if (isDynamicNgraphNode(op)) {
errorMessage = "Doesn't support op with dynamic shapes";
return false;
}
const auto batchToSpace = std::dynamic_pointer_cast<const ngraph::opset2::BatchToSpace>(op);
if (!batchToSpace) {
errorMessage = "Only opset2 BatchToSpace operation is supported";
@@ -47,8 +51,8 @@ MKLDNNBatchToSpaceNode::MKLDNNBatchToSpaceNode(const std::shared_ptr<ngraph::Nod
if (op->get_input_size() != 4 || op->get_output_size() != 1)
IE_THROW() << errorPrefix << " has incorrect number of input or output edges!";
inDims = op->get_input_shape(0);
outDims = op->get_output_shape(0);
inDims = getInputShapeAtPort(0).getStaticDims();
outDims = getOutputShapeAtPort(0).getStaticDims();
if (inDims.size() < 4 || inDims.size() > 5)
IE_THROW() << errorPrefix << " has unsupported 'data' input rank: " << inDims.size();
if (inDims.size() != outDims.size())
@@ -114,14 +118,14 @@ void MKLDNNBatchToSpaceNode::batchToSpaceKernel() {
auto srcDesc = getParentEdgeAt(0)->getMemory().GetDescWithType<BlockedMemoryDesc>();
const bool blocked = srcDesc.hasLayoutType(LayoutType::nCsp8c) || srcDesc.hasLayoutType(LayoutType::nCsp16c);
const bool blocked = srcDesc->hasLayoutType(LayoutType::nCsp8c) || srcDesc->hasLayoutType(LayoutType::nCsp16c);
const auto dimsSize = inDims.size();
auto inShape5D = getShape5D(inDims);
auto outShape5D = getShape5D(outDims);
auto blockShape = getShape5D(blockShapeIn);
if (srcDesc.hasLayoutType(LayoutType::nspc) && one_of(srcDesc.getShape().getRank(), 4, 5)) {
if (srcDesc->hasLayoutType(LayoutType::nspc) && one_of(srcDesc->getShape().getRank(), 4, 5)) {
inShape5D.push_back(inShape5D[1]);
inShape5D.erase(inShape5D.begin() + 1);
outShape5D.push_back(outShape5D[1]);
@@ -132,9 +136,9 @@ void MKLDNNBatchToSpaceNode::batchToSpaceKernel() {
auto dstDesc = getChildEdgeAt(0)->getMemory().GetDescWithType<BlockedMemoryDesc>();
const size_t blockSize = blocked ? dstDesc.getBlockDims().back() : 1lu;
const size_t blockCountInput = srcDesc.getBlockDims()[1];
const size_t blockCountOutput = dstDesc.getBlockDims()[1];
const size_t blockSize = blocked ? dstDesc->getBlockDims().back() : 1lu;
const size_t blockCountInput = srcDesc->getBlockDims()[1];
const size_t blockCountOutput = dstDesc->getBlockDims()[1];
const auto blockRemainder = inShape5D[1] % blockSize;
const auto lastBlock = blockRemainder == 0 ? blockSize : blockRemainder;
@@ -169,7 +173,7 @@ void MKLDNNBatchToSpaceNode::batchToSpaceKernel() {
oAdd[2] = dimsSize == 5 ? bIdx % blockShapeIn[2] - cropsBeginIn[2] : 0lu;
bIdx = dimsSize == 5 ? bIdx / blockShapeIn[2] : bIdx;
oAdd[1] = bIdx % blockShapeIn[1] - cropsBeginIn[1];
if (srcDesc.hasLayoutType(LayoutType::nspc) && one_of(srcDesc.getShape().getRank(), 4, 5)) {
if (srcDesc->hasLayoutType(LayoutType::nspc) && one_of(srcDesc->getShape().getRank(), 4, 5)) {
oAdd.push_back(oAdd[1]);
oAdd.erase(oAdd.begin() + 1);
}
@@ -224,13 +228,13 @@ void MKLDNNBatchToSpaceNode::batchToSpaceKernel() {
}
void MKLDNNBatchToSpaceNode::execute(mkldnn::stream strm) {
switch (getParentEdgeAt(0)->getMemory().GetDesc().getPrecision().size()) {
switch (getParentEdgeAt(0)->getMemory().getDesc().getPrecision().size()) {
case 1: batchToSpaceKernel<PrecisionTrait<Precision::U8>::value_type>(); break;
case 2: batchToSpaceKernel<PrecisionTrait<Precision::U16>::value_type>(); break;
case 4: batchToSpaceKernel<PrecisionTrait<Precision::I32>::value_type>(); break;
default:
IE_THROW() << "BatchToSpace layer does not support precision '" <<
std::string(getParentEdgeAt(0)->getMemory().GetDesc().getPrecision().name()) << "'";
std::string(getParentEdgeAt(0)->getMemory().getDesc().getPrecision().name()) << "'";
}
}
inference-engine/src/mkldnn_plugin/nodes/mkldnn_batch_to_space_node.h
+1 -1
View File
@@ -22,7 +22,7 @@ public:
void execute(mkldnn::stream strm) override;
bool created() const override;
static bool isSupportedOperation(const std::shared_ptr<ngraph::Node>& op, std::string& errorMessage) noexcept;
static bool isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept;
private:
template<typename T>
inference-engine/src/mkldnn_plugin/nodes/mkldnn_bin_conv_node.cpp
+30 -25
View File
@@ -872,8 +872,13 @@ private:
}
};
bool MKLDNNBinaryConvolutionNode::isSupportedOperation(const std::shared_ptr<ngraph::Node>& op, std::string& errorMessage) noexcept {
bool MKLDNNBinaryConvolutionNode::isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept {
try {
if (isDynamicNgraphNode(op)) {
errorMessage = "Doesn't support op with dynamic shapes";
return false;
}
const auto binConv = std::dynamic_pointer_cast<const ngraph::opset1::BinaryConvolution>(op);
if (!binConv) {
errorMessage = "Only opset1 BinaryConvolution operation is supported";
@@ -942,16 +947,16 @@ void MKLDNNBinaryConvolutionNode::getSupportedDescriptors() {
if (getChildEdges().empty())
IE_THROW() << errorPrefix << "has incorrect number of output edges";
if (getParentEdgeAt(0)->getShape().getRank() != 4) {
IE_THROW() << errorPrefix << "doesn't support 0th input with rank: " << getParentEdgeAt(0)->getShape().getRank();
if (getInputShapeAtPort(0).getRank() != 4) {
IE_THROW() << errorPrefix << "doesn't support 0th input with rank: " << getInputShapeAtPort(0).getRank();
}
if (getParentEdgeAt(1)->getShape().getRank() != 4) {
IE_THROW() << errorPrefix << "doesn't support 1st input with rank: " << getParentEdgeAt(1)->getShape().getRank();
if (getInputShapeAtPort(1).getRank() != 4) {
IE_THROW() << errorPrefix << "doesn't support 1st input with rank: " << getInputShapeAtPort(1).getRank();
}
if (getChildEdgeAt(0)->getShape().getRank() != 4) {
IE_THROW() << errorPrefix << "doesn't support output with rank: " << getChildEdgeAt(0)->getShape().getRank();
if (getOutputShapeAtPort(0).getRank() != 4) {
IE_THROW() << errorPrefix << "doesn't support output with rank: " << getOutputShapeAtPort(0).getRank();
}
}
@@ -979,20 +984,20 @@ void MKLDNNBinaryConvolutionNode::initSupportedPrimitiveDescriptors() {
//activation
auto nspcCreator = BlockedDescCreator::getCommonCreators().at(LayoutType::nspc);
config.inConfs[0].desc = nspcCreator->createUniqueDesc(Precision::BIN, getParentEdgeAt(0)->getShape().getStaticDims());
config.inConfs[0].desc = nspcCreator->createSharedDesc(Precision::BIN, getInputShapeAtPort(0));
//weights
size_t weiFirstDimBlockSize = implType == impl_desc_type::jit_avx512 ? 16 : 8; //memory::format_tag::OIhw16o32i : memory::format_tag::OIhw8o32i;
auto weiDims = getParentEdgeAt(1)->getShape().getStaticDims();
auto weiDims = getInputShapeAtPort(1).getStaticDims();
std::vector<size_t> weiBlockDims = {div_up(weiDims[0], weiFirstDimBlockSize), div_up(weiDims[1], 32),
weiDims[2], weiDims[3], weiFirstDimBlockSize, 32};
std::vector<size_t> weiOrder = {0, 1, 2, 3, 0, 1};
config.inConfs[1].desc = MKLDNNPlugin::make_unique<BlockedMemoryDesc>(Precision::BIN, weiDims, weiBlockDims, weiOrder);
config.inConfs[1].desc = std::make_shared<CpuBlockedMemoryDesc>(Precision::BIN, Shape(weiDims), weiBlockDims, weiOrder);
//result
auto outputPrecision = withBinarization ? Precision::BIN : Precision::FP32;
config.outConfs[0].desc = nspcCreator->createUniqueDesc(outputPrecision, getChildEdgeAt(0)->getShape().getStaticDims());
config.outConfs[0].desc = nspcCreator->createSharedDesc(outputPrecision, getOutputShapeAtPort(0));
if (withSum) {
config.inConfs.push_back(config.outConfs[0]);
config.outConfs[0].inPlace = 2;
@@ -1003,9 +1008,9 @@ void MKLDNNBinaryConvolutionNode::initSupportedPrimitiveDescriptors() {
auto weiCreator = BlockedDescCreator::getCommonCreators().at(LayoutType::ncsp);
auto nspcCreator = BlockedDescCreator::getCommonCreators().at(LayoutType::nspc);
config.inConfs[0].desc = nspcCreator->createUniqueDesc(Precision::BIN, getParentEdgeAt(0)->getShape().getStaticDims());
config.inConfs[1].desc = weiCreator->createUniqueDesc(Precision::BIN, getParentEdgeAt(1)->getShape().getStaticDims());
config.outConfs[0].desc = nspcCreator->createUniqueDesc(Precision::FP32, getChildEdgeAt(0)->getShape().getStaticDims());
config.inConfs[0].desc = nspcCreator->createSharedDesc(Precision::BIN, getInputShapeAtPort(0));
config.inConfs[1].desc = weiCreator->createSharedDesc(Precision::BIN, getInputShapeAtPort(1));
config.outConfs[0].desc = nspcCreator->createSharedDesc(Precision::FP32, getOutputShapeAtPort(0));
supportedPrimitiveDescriptors.push_back({config, implType});
}
}
@@ -1015,9 +1020,9 @@ void MKLDNNBinaryConvolutionNode::createPrimitive() {
if (!selectedPrimitiveDescriptor)
IE_THROW() << "CPU binary convolution with name '" << getName() << "' doesn't have primitive descriptors.";
auto srcDims = getParentEdgeAt(0)->getShape().getStaticDims();
auto weiDims = getParentEdgeAt(1)->getShape().getStaticDims();
auto dstDims = getChildEdgeAt(0)->getShape().getStaticDims();
auto srcDims = getParentEdgesAtPort(0)[0]->getMemory().getStaticDims();
auto weiDims = getParentEdgesAtPort(1)[0]->getMemory().getStaticDims();
auto dstDims = getChildEdgesAtPort(0)[0]->getMemory().getStaticDims();
auto implType = selectedPrimitiveDescriptor->getImplementationType();
@@ -1071,8 +1076,8 @@ void MKLDNNBinaryConvolutionNode::createPrimitive() {
jcp.nb_oc_blocking = nstl::min(implType == impl_desc_type::jit_sse42 ? 2 : implType == impl_desc_type::jit_avx2 ? 4 : 6, jcp.nb_oc);
auto srcPrecision = getParentEdgeAt(0)->getMemory().GetDesc().getPrecision();
auto dstPrecision = getChildEdgeAt(0)->getMemory().GetDesc().getPrecision();
auto srcPrecision = getParentEdgeAt(0)->getMemory().getDesc().getPrecision();
auto dstPrecision = getChildEdgeAt(0)->getMemory().getDesc().getPrecision();
jcp.dst_dt = MKLDNNExtensionUtils::IEPrecisionToDataType(dstPrecision);
jcp.typesize_in = srcPrecision == Precision::BIN ? 1 : srcPrecision.size();
@@ -1295,21 +1300,21 @@ void MKLDNNBinaryConvolutionNode::execute(mkldnn::stream strm) {
auto dst = reinterpret_cast<uint8_t*>(dstMemory->GetPtr());
auto srcDesc = getParentEdgeAt(0)->getMemory().GetDescWithType<BlockedMemoryDesc>();
std::vector<size_t> srcStride(srcDesc.getStrides().size());
std::vector<size_t> srcStride(srcDesc->getStrides().size());
for (int i = 0; i < srcStride.size(); i++) {
srcStride[srcDesc.getOrder()[i]] = srcDesc.getStrides()[i];
srcStride[srcDesc->getOrder()[i]] = srcDesc->getStrides()[i];
}
auto weiDesc = getParentEdgeAt(1)->getMemory().GetDescWithType<BlockedMemoryDesc>();
std::vector<size_t> weightsStride(weiDesc.getShape().getRank());
std::vector<size_t> weightsStride(weiDesc->getShape().getRank());
for (int i = 0; i < weightsStride.size(); i++) {
weightsStride[weiDesc.getOrder()[i]] = weiDesc.getStrides()[i];
weightsStride[weiDesc->getOrder()[i]] = weiDesc->getStrides()[i];
}
auto dstDesc = getChildEdgeAt(0)->getMemory().GetDescWithType<BlockedMemoryDesc>();
std::vector<size_t> dstStride(dstDesc.getStrides().size());
std::vector<size_t> dstStride(dstDesc->getStrides().size());
for (int i = 0; i < dstStride.size(); i++) {
dstStride[dstDesc.getOrder()[i]] = dstDesc.getStrides()[i];
dstStride[dstDesc->getOrder()[i]] = dstDesc->getStrides()[i];
}
auto selectedPrimitiveDescriptor = getSelectedPrimitiveDescriptor();
inference-engine/src/mkldnn_plugin/nodes/mkldnn_bin_conv_node.h
+1 -1
View File
@@ -87,7 +87,7 @@ public:
void setPostOps(mkldnn::primitive_attr &attr);
bool canFuse(const MKLDNNNodePtr& node) const override;
static bool isSupportedOperation(const std::shared_ptr<ngraph::Node>& op, std::string& errorMessage) noexcept;
static bool isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept;
impl_desc_type getImplType() { return implType; }
inference-engine/src/mkldnn_plugin/nodes/mkldnn_broadcast_node.cpp
+11 -7
View File
@@ -17,8 +17,12 @@
using namespace MKLDNNPlugin;
using namespace InferenceEngine;
bool MKLDNNBroadcastNode::isSupportedOperation(const std::shared_ptr<ngraph::Node>& op, std::string& errorMessage) noexcept {
bool MKLDNNBroadcastNode::isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept {
try {
if (isDynamicNgraphNode(op)) {
errorMessage = "Doesn't support op with dynamic shapes";
return false;
}
const auto broadcast = std::dynamic_pointer_cast<const ngraph::opset1::Broadcast>(op);
if (!broadcast) {
errorMessage = "Only opset1 Broadcast operation is supported";
@@ -67,13 +71,13 @@ void MKLDNNBroadcastNode::initSupportedPrimitiveDescriptors() {
}
void MKLDNNBroadcastNode::execute(mkldnn::stream strm) {
size_t shape_size = (getParentEdgeAt(BROADCAST_SHAPE)->getMemory().GetDesc().getShape().getStaticDims())[0];
SizeVector dst_dims = getChildEdgeAt(0)->getMemory().GetDesc().getShape().getStaticDims();
SizeVector src_dims = getParentEdgeAt(BROADCAST_INPUT)->getMemory().GetDesc().getShape().getStaticDims();
size_t shape_size = (getParentEdgeAt(BROADCAST_SHAPE)->getMemory().getStaticDims())[0];
SizeVector dst_dims = getChildEdgeAt(0)->getMemory().getStaticDims();
SizeVector src_dims = getParentEdgeAt(BROADCAST_INPUT)->getMemory().getStaticDims();
auto srcDesc = getParentEdgeAt(BROADCAST_INPUT)->getMemory().GetDescWithType<BlockedMemoryDesc>();
SizeVector srcStrides = srcDesc.getStrides();
size_t data_size = srcDesc.getPrecision().size();
SizeVector srcStrides = srcDesc->getStrides();
size_t data_size = srcDesc->getPrecision().size();
if (!src_dims.size())
src_dims = SizeVector(1, 1);
@@ -89,7 +93,7 @@ void MKLDNNBroadcastNode::execute(mkldnn::stream strm) {
}
auto dstDesc = getChildEdgeAt(0)->getMemory().GetDescWithType<BlockedMemoryDesc>();
InferenceEngine::SizeVector dstStrides = dstDesc.getStrides();
InferenceEngine::SizeVector dstStrides = dstDesc->getStrides();
InferenceEngine::SizeVector src_aligned(dst_dims.size());
InferenceEngine::SizeVector srcStrides_aligned(dst_dims.size());
size_t prefix_size = dst_dims.size() - src_dims.size();
inference-engine/src/mkldnn_plugin/nodes/mkldnn_broadcast_node.h
+1 -1
View File
@@ -22,7 +22,7 @@ public:
void execute(mkldnn::stream strm) override;
bool created() const override;
static bool isSupportedOperation(const std::shared_ptr<ngraph::Node>& op, std::string& errorMessage) noexcept;
static bool isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept;
private:
static const size_t BROADCAST_INPUT = 0;
inference-engine/src/mkldnn_plugin/nodes/mkldnn_bucketize_node.cpp
+5 -1
View File
@@ -13,8 +13,12 @@
using namespace MKLDNNPlugin;
using namespace InferenceEngine;
bool MKLDNNBucketizeNode::isSupportedOperation(const std::shared_ptr<ngraph::Node>& op, std::string& errorMessage) noexcept {
bool MKLDNNBucketizeNode::isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept {
try {
if (isDynamicNgraphNode(op)) {
errorMessage = "Doesn't support op with dynamic shapes";
return false;
}
const auto bucketsize = std::dynamic_pointer_cast<const ngraph::opset3::Bucketize>(op);
if (!bucketsize) {
errorMessage = "Only opset3 Bucketize operation is supported";
inference-engine/src/mkldnn_plugin/nodes/mkldnn_bucketize_node.h
+1 -1
View File
@@ -19,7 +19,7 @@ public:
void execute(mkldnn::stream strm) override;
bool created() const override;
static bool isSupportedOperation(const std::shared_ptr<ngraph::Node>& op, std::string& errorMessage) noexcept;
static bool isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept;
private:
template <typename T, typename T_BOUNDARIES, typename T_IND>
inference-engine/src/mkldnn_plugin/nodes/mkldnn_concat_node.cpp
+57 -53
View File
@@ -11,7 +11,6 @@
#include "mkldnn.hpp"
#include "mkldnn/iml_type_mapper.h"
#include "mkldnn_dims.h"
#include "mkldnn_edge.h"
#include "mkldnn_memory.h"
#include "ie_parallel.hpp"
@@ -22,7 +21,7 @@
#include <limits>
#include "common/cpu_memcpy.h"
#include "common/blocked_desc_creator.h"
#include <cpu_memory_desc_utils.h>
#include <memory_desc/cpu_memory_desc_utils.h>
using namespace mkldnn;
using namespace MKLDNNPlugin;
@@ -34,6 +33,11 @@ namespace {
bool MKLDNNConcatNode::isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept {
try {
if (isDynamicNgraphNode(op)) {
errorMessage = "Doesn't support op with dynamic shapes";
return false;
}
const auto concatOp = ngraph::as_type_ptr<const ngraph::op::v0::Concat>(op);
if (!concatOp) {
errorMessage = "Node is not an instance of the Concat operation.";
@@ -62,9 +66,9 @@ MKLDNNConcatNode::MKLDNNConcatNode(const std::shared_ptr<ngraph::Node>& op, cons
}
void MKLDNNConcatNode::getSupportedDescriptors() {
auto& firstParentDims = getParentEdgeAt(0)->getShape().getStaticDims();
auto& firstParentDims = getInputShapeAtPort(0).getStaticDims();
for (size_t i = 1; i < getParentEdges().size(); i++) {
auto& dims = getParentEdgeAt(i)->getShape().getStaticDims();
auto& dims = getInputShapeAtPort(i).getStaticDims();
bool incorrectDims = false;
for (size_t j = 0; j < firstParentDims.size(); j++) {
if (j == axis)
@@ -87,7 +91,7 @@ void MKLDNNConcatNode::initSupportedPrimitiveDescriptors() {
auto& originInputPrecisions = getOriginalInputPrecisions();
inputPrecision = originInputPrecisions[0];
bool isMixedPrecision = false;
for (int i = 1; i < getOriginalInputsNumber(); i++) {
for (int i = 1; i < inputShapes.size(); i++) {
if (originInputPrecisions[0] != originInputPrecisions[i]) {
isMixedPrecision = true;
break;
@@ -101,19 +105,19 @@ void MKLDNNConcatNode::initSupportedPrimitiveDescriptors() {
// Concat supports only equal precisions for inputs and output
outputPrecision = inputPrecision;
auto& dstDims = getChildEdgeAt(0)->getShape().getStaticDims();
const auto& dstShape = getOutputShapeAtPort(0);
std::vector<LayoutType> tdCreatorTypes = {LayoutType::ncsp, LayoutType::nspc};
// check if blocked layouts are available the channels size should be evenly divided by the block size to avoid slow oneDNN ref implementation
if (dstDims.size() > channelAxis) {
if (dstShape.getRank() > channelAxis) {
for (auto item : { std::make_pair(8lu, LayoutType::nCsp8c), std::make_pair(16lu, LayoutType::nCsp16c)}) {
SizeVector blkDims = dstDims;
const VectorDims &blkDims = dstShape.getStaticDims();
if (blkDims[channelAxis] % item.first)
continue;
bool blocked = true;
for (size_t i = 0; i < getParentEdges().size(); i++) {
auto& srcDims = getParentEdgeAt(i)->getShape().getStaticDims();
auto& srcDims = getInputShapeAtPort(i).getStaticDims();
if (srcDims[channelAxis] % item.first) {
blocked = false;
break;
@@ -128,7 +132,8 @@ void MKLDNNConcatNode::initSupportedPrimitiveDescriptors() {
std::vector<size_t> pdIndexesToReuse;
auto& creatorsMap = BlockedDescCreator::getCommonCreators();
auto itrRange = BlockedDescCreator::makeFilteredRange(creatorsMap, static_cast<unsigned>(dstDims.size()), tdCreatorTypes);
auto itrRange = BlockedDescCreator::makeFilteredRange(creatorsMap, static_cast<unsigned>(dstShape.getRank()), tdCreatorTypes);
for (auto itr = itrRange.first; itr != itrRange.second; ++itr) {
NodeConfig config;
@@ -136,15 +141,14 @@ void MKLDNNConcatNode::initSupportedPrimitiveDescriptors() {
config.outConfs.resize(1);
config.outConfs[0].inPlace = -1;
config.outConfs[0].constant = false;
config.outConfs[0].desc = itr->second->createUniqueDesc(outputPrecision, dstDims);
config.outConfs[0].desc = itr->second->createSharedDesc(outputPrecision, dstShape);
config.inConfs.resize(getParentEdges().size());
for (size_t i = 0; i < getParentEdges().size(); ++i) {
config.inConfs[i].inPlace = -1;
config.inConfs[i].constant = false;
config.inConfs[i].desc = MemoryDescUtils::applyUndefinedOffset(
itr->second->createDesc(inputPrecision, getParentEdgeAt(i)->getShape().getStaticDims()));
config.inConfs[i].desc = MemoryDescUtils::cloneWithUndefStridesAndOffset(itr->second->createDesc(inputPrecision, getInputShapeAtPort(i)));
}
supportedPrimitiveDescriptors.emplace_back(config, impl_desc_type::ref);
if (itr->first != LayoutType::nspc) {
@@ -167,8 +171,8 @@ void MKLDNNConcatNode::initSupportedPrimitiveDescriptors() {
const auto& refConfig = supportedPrimitiveDescriptors[refPdIndex].getConfig();
auto config = refConfig;
const auto &order = refConfig.outConfs[0].desc->as<BlockedMemoryDesc>()->getOrder();
const auto &blkDims = refConfig.outConfs[0].desc->as<BlockedMemoryDesc>()->getBlockDims();
const auto &order = refConfig.outConfs[0].desc->as<CpuBlockedMemoryDesc>()->getOrder();
const auto &blkDims = refConfig.outConfs[0].desc->as<CpuBlockedMemoryDesc>()->getBlockDims();
auto numOfDim = blkDims.size();
SizeVector offsets(numOfDim, 0lu);
@@ -184,14 +188,14 @@ void MKLDNNConcatNode::initSupportedPrimitiveDescriptors() {
}
}
config.outConfs[0].desc = MKLDNNPlugin::make_unique<BlockedMemoryDesc>(outputPrecision, dstDims, blkDims, order, offset, offsets, strides);
config.outConfs[0].desc = std::make_shared<CpuBlockedMemoryDesc>(outputPrecision, dstShape, blkDims, order, offset, offsets, strides);
for (size_t i = 0; i < getParentEdges().size(); i++) {
const auto& srcBlkDims = refConfig.inConfs[i].desc->as<BlockedMemoryDesc>()->getBlockDims();
const auto& dims = refConfig.inConfs[i].desc->getShape().getStaticDims();
const auto& srcBlkDims = refConfig.inConfs[i].desc->as<CpuBlockedMemoryDesc>()->getBlockDims();
const auto& shape = refConfig.inConfs[i].desc->getShape();
config.inConfs[i].inPlace = 0;
config.inConfs[i].desc = MKLDNNPlugin::make_unique<BlockedMemoryDesc>(inputPrecision, dims, srcBlkDims, order, offset, offsets, strides);
config.inConfs[i].desc = std::make_shared<CpuBlockedMemoryDesc>(inputPrecision, shape, srcBlkDims, order, offset, offsets, strides);
}
supportedPrimitiveDescriptors.emplace_back(config, impl_desc_type::unknown);
}
@@ -257,7 +261,7 @@ void MKLDNNConcatNode::selectOptimalPrimitiveDescriptor() {
}
size_t maxCount = 0;
auto outDims = getChildEdgeAt(0)->getShape().getStaticDims();
auto outDims = getOutputShapeAtPort(0).getStaticDims();
LayoutType convertTo = LayoutType::ncsp;
for (auto &it : formatFrequency) {
if (it.second > maxCount) {
@@ -279,7 +283,7 @@ void MKLDNNConcatNode::selectOptimalPrimitiveDescriptor() {
break;
}
for (size_t i = 0; i < getParentEdges().size(); i++) {
auto& inpDims = getParentEdgeAt(i)->getShape().getStaticDims();
auto& inpDims = getInputShapeAtPort(i).getStaticDims();
if (inpDims[1] % item.first != 0) {
convertTo = LayoutType::ncsp;
break;
@@ -339,7 +343,7 @@ void MKLDNNConcatNode::createPrimitive() {
IE_THROW() << "Preferable primitive descriptor is not set.";
//check if selected Tensor descriptor has nspc layout and concat axis is C
if (axis == channelAxis && getChildEdgeAt(0)->getMemory().GetDesc().hasLayoutType(LayoutType::nspc)) {
if (axis == channelAxis && getChildEdgeAt(0)->getMemory().getDesc().hasLayoutType(LayoutType::nspc)) {
canOptimizeNspc = true;
return;
}
@@ -353,9 +357,9 @@ void MKLDNNConcatNode::createPrimitive() {
IE_THROW() << "Source memory from " << parent->getName() << " didn't allocate for node "
<< getName() << ".";
}
auto desc = srcMemPtr->GetDescriptor();
auto& dims = getParentEdgeAt(i)->getShape().getStaticDims();
// DnnlBlockedMemoryDesc
auto desc = srcMemPtr->GetDescWithType<DnnlMemoryDesc>()->getDnnlDesc();
auto& dims = getInputShapeAtPort(i).getStaticDims();
for (size_t j = 0; j < dims.size(); j++) {
desc.data.dims[j] = dims[j];
}
@@ -363,8 +367,8 @@ void MKLDNNConcatNode::createPrimitive() {
srcs_d.emplace_back(desc);
}
auto desc = getChildEdgeAt(0)->getMemory().GetDescriptor();
auto& dims = getChildEdgeAt(0)->getShape().getStaticDims();
auto desc = getChildEdgeAt(0)->getMemory().GetDescWithType<DnnlMemoryDesc>()->getDnnlDesc();
auto& dims = getOutputShapeAtPort(0).getStaticDims();
for (size_t i = 0; i < dims.size(); i++) {
desc.data.dims[i] = dims[i];
desc.data.padded_dims[i] = dims[i];
@@ -393,14 +397,12 @@ void MKLDNNConcatNode::initOptimalPrimitiveDescriptor() {
auto config = selected_pd->getConfig();
if (!isConfigDefined(config)) {
for (size_t i = 0; i < config.inConfs.size(); i++) {
config.inConfs[i].desc = getDefinedInputDesc(config, i);
// Concat doesn't support different precision on inputs
config.inConfs[i].desc->setPrecision(inputPrecision);
config.inConfs[i].desc = MemoryDescUtils::cloneWithNewPrecision(*getDefinedInputDesc(config, i), inputPrecision);
}
for (size_t i = 0; i < config.outConfs.size(); i++) {
config.outConfs[i].desc = getDefinedOutputDesc(config, i);
config.outConfs[i].desc->setPrecision(outputPrecision);
config.outConfs[i].desc = MemoryDescUtils::cloneWithNewPrecision(*getDefinedOutputDesc(config, i), outputPrecision);
}
initDescriptor(config);
@@ -418,47 +420,49 @@ void MKLDNNConcatNode::initOptimalPrimitiveDescriptor() {
int num = getChildEdgeAt(i)->getOutputNum();
if (num >= 0) {
auto childConf = getChildEdgeAt(i)->getChild()->getSelectedPrimitiveDescriptor()->getConfig().inConfs[num];
childConf.desc->setPrecision(config.outConfs[i].desc->getPrecision());
childConf.desc = MemoryDescUtils::cloneWithNewPrecision(*childConf.desc, config.outConfs[i].desc->getPrecision());
if (getChildEdgeAt(i)->getChild()->getSelectedPrimitiveDescriptor()) {
if (!childConf.desc->isDefined() && childConf.inPlace >= 0)
getChildEdgeAt(i)->getChild()->initOptimalPrimitiveDescriptor();
if (childConf.desc->isDefined() && childConf.desc->isCompatible(*config.outConfs[i].desc)) {
config.outConfs[i].desc = childConf.desc->clone();
config.outConfs[i].desc = childConf.desc;
continue;
}
}
}
// reset undefined offsets
config.outConfs[i].desc = MemoryDescUtils::resetOffset(config.outConfs[i].desc.get());
config.outConfs[i].desc = MemoryDescUtils::cloneWithDefaultStridesAndOffset(*config.outConfs[i].desc);
}
auto firstOutBlockingDesc = MemoryDescUtils::convertToBlockedDescriptor(*config.outConfs[0].desc);
auto firstOutBlockingDesc = config.outConfs[0].desc->as<BlockedMemoryDesc>();
size_t offset = 0;
for (size_t i = 0; i < config.inConfs.size(); i++) {
auto inpBlockingDesc = MemoryDescUtils::convertToBlockedDescriptor(*config.inConfs[i].desc);
config.inConfs[i].desc = MKLDNNPlugin::make_unique<BlockedMemoryDesc>(inpBlockingDesc.getPrecision(),
inpBlockingDesc.getShape().getStaticDims(),
inpBlockingDesc.getBlockDims(),
inpBlockingDesc.getOrder(),
firstOutBlockingDesc.getOffsetPadding() + offset,
firstOutBlockingDesc.getOffsetPaddingToData(),
firstOutBlockingDesc.getStrides());
auto oldDesc = config.inConfs[i].desc;
auto inpBlockingDesc = oldDesc->as<BlockedMemoryDesc>();
config.inConfs[i].desc = std::make_shared<CpuBlockedMemoryDesc>(inpBlockingDesc->getPrecision(),
inpBlockingDesc->getShape(),
inpBlockingDesc->getBlockDims(),
inpBlockingDesc->getOrder(),
firstOutBlockingDesc->getOffsetPadding() + offset,
firstOutBlockingDesc->getOffsetPaddingToData(),
firstOutBlockingDesc->getStrides());
size_t axisSize = 1;
auto firstInpBlockingDesc = MemoryDescUtils::convertToBlockedDescriptor(*config.inConfs[0].desc);
if (firstInpBlockingDesc.hasLayoutType(LayoutType::nspc)) {
auto firstInpBlockingDesc = config.inConfs[0].desc->as<BlockedMemoryDesc>();
if (firstInpBlockingDesc->hasLayoutType(LayoutType::nspc)) {
// This is more general and works for any "direct" Layout (such as nchw or nhwc), but it doesn't work for blocked
size_t realAxis = inverseOrder(firstInpBlockingDesc.getOrder(), axis);
for (size_t j = realAxis; j < inpBlockingDesc.getBlockDims().size(); j++) {
size_t jj = firstInpBlockingDesc.getOrder()[j];
axisSize *= inpBlockingDesc.getBlockDims()[jj];
size_t realAxis = inverseOrder(firstInpBlockingDesc->getOrder(), axis);
for (size_t j = realAxis; j < inpBlockingDesc->getBlockDims().size(); j++) {
size_t jj = firstInpBlockingDesc->getOrder()[j];
axisSize *= inpBlockingDesc->getBlockDims()[jj];
}
} else {
// This works for nchw and nchw8c/nchw16c
for (size_t j = axis; j < inpBlockingDesc.getBlockDims().size(); j++) {
axisSize *= inpBlockingDesc.getBlockDims()[j];
for (size_t j = axis; j < inpBlockingDesc->getBlockDims().size(); j++) {
axisSize *= inpBlockingDesc->getBlockDims()[j];
}
}
offset += axisSize;
@@ -502,7 +506,7 @@ void MKLDNNConcatNode::execNspcSpecCase() {
for (size_t i = 0; i < num_src; i++) {
const MKLDNNMemory& src_mem = getParentEdgeAt(i)->getMemory();
const size_t num_channels = src_mem.GetDims()[channelAxis];
const size_t num_channels = src_mem.getStaticDims()[channelAxis];
channelsDataSize.push_back(num_channels * dataSize);
src_ptrs.push_back(reinterpret_cast<const uint8_t*>(src_mem.GetData()));
@@ -520,4 +524,4 @@ void MKLDNNConcatNode::execNspcSpecCase() {
});
}
REG_MKLDNN_PRIM_FOR(MKLDNNConcatNode, Concatenation);
REG_MKLDNN_PRIM_FOR(MKLDNNConcatNode, Concatenation);
inference-engine/src/mkldnn_plugin/nodes/mkldnn_conv_node.cpp
+78 -78
View File
@@ -18,14 +18,20 @@
#include <ngraph/ops.hpp>
#include <cpu/x64/jit_generator.hpp>
#include "common/cpu_convert.h"
#include <cpu_memory_desc_utils.h>
#include <memory_desc/cpu_memory_desc_utils.h>
#include "memory_desc/dnnl_blocked_memory_desc.h"
using namespace mkldnn;
using namespace MKLDNNPlugin;
using namespace InferenceEngine;
bool MKLDNNConvolutionNode::isSupportedOperation(const std::shared_ptr<ngraph::Node>& op, std::string& errorMessage) noexcept {
bool MKLDNNConvolutionNode::isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept {
try {
if (isDynamicNgraphNode(op)) {
errorMessage = "Doesn't support op with dynamic shapes";
return false;
}
if (!ngraph::is_type<ngraph::op::v1::Convolution>(op) && !ngraph::is_type<ngraph::op::v1::GroupConvolution>(op)) {
errorMessage = "Only opset1 Convolution and GroupConvolution operations are supported";
return false;
@@ -194,8 +200,7 @@ void MKLDNNConvolutionNode::getSupportedDescriptors() {
if (getChildEdges().empty())
IE_THROW() << "Incorrect number of output edges for layer " << getName();
int ndims = getParentEdgesAtPort(0)[0]->getShape().getRank();
MKLDNNDims weightsDims = MKLDNNDims(weightDims);
int ndims = getInputShapeAtPort(0).getRank();
withDWConv = isFusedWith(Convolution);
@@ -226,9 +231,9 @@ void MKLDNNConvolutionNode::getSupportedDescriptors() {
for (int j = 0; j < paddingR.size(); j++) {
int with_group = isGrouped ? 1 : 0;
int krn = weightsDims[with_group + 2 + j];
int src = getParentEdgeAt(0)->getShape().getStaticDims()[2 + j];
int dst = getChildEdgeAt(0)->getShape().getStaticDims()[2 + j];
int krn = weightDims[with_group + 2 + j];
int src = getInputShapeAtPort(0).getStaticDims()[2 + j];
int dst = getOutputShapeAtPort(0).getStaticDims()[2 + j];
krn = (krn - 1)*(dilation[j] + 1) + 1;
int calc_dst = (src - krn + paddingL[j]) / stride[j] + 1;
@@ -244,11 +249,11 @@ void MKLDNNConvolutionNode::getSupportedDescriptors() {
outputDataType = memory::data_type::f32;
if (eltwisePrecision == Precision::BF16)
eltwisePrecision = Precision::FP32;
in_candidate = MKLDNNPlugin::make_unique<MKLDNNMemoryDesc>(getParentEdgeAt(0)->getShape().getStaticDims(),
in_candidate = std::make_shared<DnnlBlockedMemoryDesc>(getInputShapeAtPort(0),
inputDataType, ndims == 5 ? memory::format_tag::ndhwc : memory::format_tag::nhwc);
out_candidate = MKLDNNPlugin::make_unique<MKLDNNMemoryDesc>(getChildEdgeAt(0)->getShape().getStaticDims(),
out_candidate = std::make_shared<DnnlBlockedMemoryDesc>(getOutputShapeAtPort(0),
outputDataType, ndims == 5 ? memory::format_tag::ndhwc : memory::format_tag::nhwc);
createDescriptor({ in_candidate.get() }, { out_candidate.get() });
createDescriptor({ in_candidate }, { out_candidate });
} else {
inputDataType = (getOriginalInputPrecisionAtPort(0) == Precision::BF16
&& !(isDepthWise() && ndims == 5)) ? memory::data_type::bf16 : memory::data_type::f32;
@@ -285,36 +290,36 @@ void MKLDNNConvolutionNode::getSupportedDescriptors() {
memory::format_tag nCsp16c = ndims == 4 ? memory::format_tag::nChw16c : memory::format_tag::nCdhw16c;
memory::format_tag nCsp8c = ndims == 4 ? memory::format_tag::nChw8c : memory::format_tag::nCdhw8c;
auto inputDims = getParentEdgeAt(0)->getShape().getStaticDims();
auto outputDims = getChildEdgeAt(0)->getShape().getStaticDims();
auto inputShape = getInputShapeAtPort(0);
auto outputShape = getOutputShapeAtPort(0);
if (IC == 1 && groupOC == 1) {
in_candidate = MKLDNNPlugin::make_unique<MKLDNNMemoryDesc>(inputDims, inputDataType, ncsp);
out_candidate = MKLDNNPlugin::make_unique<MKLDNNMemoryDesc>(outputDims, outputDataType, ncsp);
createDescriptor({ in_candidate.get() }, { out_candidate.get() });
in_candidate = std::make_shared<DnnlBlockedMemoryDesc>(inputShape, inputDataType, ncsp);
out_candidate = std::make_shared<DnnlBlockedMemoryDesc>(outputShape, outputDataType, ncsp);
createDescriptor({ in_candidate }, { out_candidate });
} else if (IC < 4) {
in_candidate = MKLDNNPlugin::make_unique<MKLDNNMemoryDesc>(inputDims, inputDataType, ncsp);
out_candidate = MKLDNNPlugin::make_unique<MKLDNNMemoryDesc>(outputDims, outputDataType, nCsp16c);
createDescriptor({ in_candidate.get() }, { out_candidate.get() });
out_candidate = MKLDNNPlugin::make_unique<MKLDNNMemoryDesc>(outputDims, outputDataType, nCsp8c);
createDescriptor({ in_candidate.get() }, { out_candidate.get() });
in_candidate = std::make_shared<DnnlBlockedMemoryDesc>(inputShape, inputDataType, ncsp);
out_candidate = std::make_shared<DnnlBlockedMemoryDesc>(outputShape, outputDataType, nCsp16c);
createDescriptor({ in_candidate }, { out_candidate });
out_candidate = std::make_shared<DnnlBlockedMemoryDesc>(outputShape, outputDataType, nCsp8c);
createDescriptor({ in_candidate }, { out_candidate });
} else {
in_candidate = MKLDNNPlugin::make_unique<MKLDNNMemoryDesc>(inputDims, inputDataType, nCsp16c);
out_candidate = MKLDNNPlugin::make_unique<MKLDNNMemoryDesc>(outputDims, outputDataType, nCsp16c);
createDescriptor({ in_candidate.get() }, { out_candidate.get() });
in_candidate = MKLDNNPlugin::make_unique<MKLDNNMemoryDesc>(inputDims, inputDataType, nCsp8c);
out_candidate = MKLDNNPlugin::make_unique<MKLDNNMemoryDesc>(outputDims, outputDataType, nCsp8c);
createDescriptor({ in_candidate.get() }, { out_candidate.get() });
in_candidate = std::make_shared<DnnlBlockedMemoryDesc>(inputShape, inputDataType, nCsp16c);
out_candidate = std::make_shared<DnnlBlockedMemoryDesc>(outputShape, outputDataType, nCsp16c);
createDescriptor({ in_candidate }, { out_candidate });
in_candidate = std::make_shared<DnnlBlockedMemoryDesc>(inputShape, inputDataType, nCsp8c);
out_candidate = std::make_shared<DnnlBlockedMemoryDesc>(outputShape, outputDataType, nCsp8c);
createDescriptor({ in_candidate }, { out_candidate });
}
in_candidate = MKLDNNPlugin::make_unique<MKLDNNMemoryDesc>(inputDims, inputDataType, ncsp);
out_candidate = MKLDNNPlugin::make_unique<MKLDNNMemoryDesc>(outputDims, outputDataType, ncsp);
createDescriptor({ in_candidate.get() }, { out_candidate.get() });
in_candidate = std::make_shared<DnnlBlockedMemoryDesc>(inputShape, inputDataType, ncsp);
out_candidate = std::make_shared<DnnlBlockedMemoryDesc>(outputShape, outputDataType, ncsp);
createDescriptor({ in_candidate }, { out_candidate });
if (inputDataType != memory::data_type::bf16 && isNspcAvailable()) {
in_candidate = MKLDNNPlugin::make_unique<MKLDNNMemoryDesc>(inputDims, inputDataType, nspc);
out_candidate = MKLDNNPlugin::make_unique<MKLDNNMemoryDesc>(outputDims, outputDataType, nspc);
createDescriptor({ in_candidate.get() }, { out_candidate.get() });
in_candidate = std::make_shared<DnnlBlockedMemoryDesc>(inputShape, inputDataType, nspc);
out_candidate = std::make_shared<DnnlBlockedMemoryDesc>(outputShape, outputDataType, nspc);
createDescriptor({ in_candidate }, { out_candidate });
}
}
}
@@ -395,11 +400,12 @@ void MKLDNNConvolutionNode::initSupportedPrimitiveDescriptors() {
PortConfig dataConfig;
dataConfig.inPlace = -1;
dataConfig.constant = false;
auto srcDesc = getSrcMemDesc(itpd, i);
if (isGrouped || srcDesc->getFormatKind() != dnnl_format_kind_t::dnnl_blocked)
dataConfig.desc = std::move(srcDesc);
else
dataConfig.desc = MemoryDescUtils::applyUndefinedOffset(*srcDesc);
auto desc = getSrcMemDesc(itpd, i);
if (desc->getType() & MemoryDescType::Blocked && !isGrouped) {
dataConfig.desc = MemoryDescUtils::cloneWithUndefStridesAndOffset(*desc);
} else {
dataConfig.desc = std::move(desc);
}
config.inConfs.push_back(dataConfig);
}
@@ -414,10 +420,10 @@ void MKLDNNConvolutionNode::initSupportedPrimitiveDescriptors() {
PortConfig dataConfig;
dataConfig.inPlace = -1;
dataConfig.constant = false;
dataConfig.desc = MKLDNNPlugin::make_unique<MKLDNNMemoryDesc>(dwWeightsDims, weightsPrc, memory::format_tag::Goihw8g);
dataConfig.desc = std::make_shared<DnnlBlockedMemoryDesc>(Shape(dwWeightsDims), weightsPrc, memory::format_tag::Goihw8g);
config.inConfs.push_back(dataConfig);
dataConfig.desc = MKLDNNPlugin::make_unique<MKLDNNMemoryDesc>(dwBiasesDims, biasPrc, memory::format_tag::x);
dataConfig.desc = std::make_shared<DnnlBlockedMemoryDesc>(Shape(dwBiasesDims), biasPrc, memory::format_tag::x);
config.inConfs.push_back(dataConfig);
}
@@ -428,18 +434,18 @@ void MKLDNNConvolutionNode::initSupportedPrimitiveDescriptors() {
}
dataConfig.constant = false;
auto dstDesc = getDstMemDesc(itpd, i);
if (isGrouped || dstDesc->getFormatKind() != dnnl_format_kind_t::dnnl_blocked)
dataConfig.desc = std::move(dstDesc);
else
dataConfig.desc = MemoryDescUtils::applyUndefinedOffset(*dstDesc);
auto desc = getDstMemDesc(itpd, i);
if (desc->getType() & MemoryDescType::Blocked && !isGrouped) {
dataConfig.desc = MemoryDescUtils::cloneWithUndefStridesAndOffset(*desc);
} else {
dataConfig.desc = std::move(desc);
}
config.outConfs.push_back(dataConfig);
if (withSum) {
dataConfig.inPlace = -1;
dataConfig.desc->setPrecision(eltwisePrecision);
dataConfig.desc = MemoryDescUtils::cloneWithNewPrecision(*dataConfig.desc, dataConfig.desc->getPrecision());
config.inConfs.push_back(dataConfig);
}
}
@@ -480,21 +486,19 @@ bool MKLDNNConvolutionNode::created() const {
return getType() == Convolution;
}
void MKLDNNConvolutionNode::createDescriptor(const std::vector<const MemoryDesc*>& inputDesc,
const std::vector<const MemoryDesc*>& outputDesc) {
auto inDesc = MemoryDescUtils::convertToMKLDNNMemoryDesc(*inputDesc[0]);
auto outDesc = MemoryDescUtils::convertToMKLDNNMemoryDesc(*outputDesc[0]);
void MKLDNNConvolutionNode::createDescriptor(const std::vector<MemoryDescPtr>& inputDesc,
const std::vector<MemoryDescPtr>& outputDesc) {
const auto inDesc = MemoryDescUtils::convertToDnnlMemoryDesc(inputDesc[0])->getDnnlDesc();
const auto outDesc = MemoryDescUtils::convertToDnnlMemoryDesc(outputDesc[0])->getDnnlDesc();
memory::data_type wdt = MKLDNNExtensionUtils::IEPrecisionToDataType(inDesc.getPrecision());
memory::data_type wdt = static_cast<memory::data_type>(inDesc.data.data_type);
memory::data_type bdt = memory::data_type::f32;
if (inDesc.getPrecision() == Precision::U8 || inDesc.getPrecision() == Precision::I8) {
if (inDesc.data.data_type == mkldnn_s8 || inDesc.data.data_type == mkldnn_u8) {
wdt = memory::data_type::s8;
}
MKLDNNDims blocked_weightDims(weightDims);
MKLDNNDims blocked_biasesDims(biasesDims);
mkldnn::memory::desc wgh_candidate(blocked_weightDims, wdt, memory::format_tag::any);
mkldnn::memory::desc wgh_candidate(MKLDNNExtensionUtils::convertToDnnlDims(weightDims), wdt, memory::format_tag::any);
std::vector<mkldnn::algorithm> algorithms;
@@ -506,7 +510,7 @@ void MKLDNNConvolutionNode::createDescriptor(const std::vector<const MemoryDesc*
try {
std::shared_ptr<mkldnn::convolution_forward::desc> conv_desc;
if (withBiases) {
mkldnn::memory::desc bias_candidate(blocked_biasesDims, bdt, memory::format_tag::any);
mkldnn::memory::desc bias_candidate(MKLDNNExtensionUtils::convertToDnnlDims(biasesDims), bdt, memory::format_tag::any);
conv_desc.reset(new convolution_forward::desc(prop_kind::forward_scoring, alg,
inDesc, wgh_candidate, bias_candidate, outDesc,
@@ -556,13 +560,9 @@ void MKLDNNConvolutionNode::initDescriptor(const NodeConfig& config) {
if (canBeExecutedInInt8()) {
isStridedBlobsSupported = false;
}
// TODO [NM]: fix strided blobs feature support for dynamic weights
// if (getOriginalInputsNumber() != 1) {
// isStridedBlobsSupported = false;
// }
if (isStridedBlobsSupported) {
createDescriptor({config.inConfs[0].desc.get()}, {config.outConfs[0].desc.get()});
createDescriptor({config.inConfs[0].desc}, {config.outConfs[0].desc});
}
mkldnn::primitive_attr attr;
@@ -600,10 +600,10 @@ void MKLDNNConvolutionNode::initDescriptor(const NodeConfig& config) {
PortConfig dataConfig;
dataConfig.inPlace = -1;
dataConfig.constant = false;
dataConfig.desc = MKLDNNPlugin::make_unique<MKLDNNMemoryDesc>(dwWeightsDims, weightsPrc, memory::format_tag::Goihw8g);
dataConfig.desc = std::make_shared<DnnlBlockedMemoryDesc>(Shape(dwWeightsDims), weightsPrc, memory::format_tag::Goihw8g);
cfg.inConfs.push_back(dataConfig);
dataConfig.desc = MKLDNNPlugin::make_unique<MKLDNNMemoryDesc>(dwBiasesDims, biasPrc, memory::format_tag::x);
dataConfig.desc = std::make_shared<DnnlBlockedMemoryDesc>(Shape(dwBiasesDims), biasPrc, memory::format_tag::x);
cfg.inConfs.push_back(dataConfig);
}
@@ -614,7 +614,7 @@ void MKLDNNConvolutionNode::initDescriptor(const NodeConfig& config) {
dataConfig.desc = getDstMemDesc(itpd, j);
if (withSum) {
auto eltwiseConfig = dataConfig;
eltwiseConfig.desc->setPrecision(eltwisePrecision);
eltwiseConfig.desc = MemoryDescUtils::cloneWithNewPrecision(*eltwiseConfig.desc, eltwisePrecision);
cfg.inConfs.push_back(eltwiseConfig);
dataConfig.inPlace = getParentEdges().size() - 1;
}
@@ -659,12 +659,12 @@ void MKLDNNConvolutionNode::filterSupportedDescriptors() {
while (itd != descs.end()) {
bool isSuitableDesc = true;
if (!inputMemoryFormatsFilter.empty()) {
MKLDNNMemoryDesc src_tdesc(std::shared_ptr<mkldnn::convolution_forward::desc>(*itd)->data.src_desc);
isSuitableDesc &= src_tdesc.isSame(inputMemoryFormatsFilter[0]);
auto src_tdesc = MKLDNNExtensionUtils::makeDescriptor(std::shared_ptr<mkldnn::convolution_forward::desc>(*itd)->data.src_desc);
isSuitableDesc &= src_tdesc->isSame(inputMemoryFormatsFilter[0]);
}
if (!outputMemoryFormatsFilter.empty()) {
MKLDNNMemoryDesc dst_tdesc(std::shared_ptr<mkldnn::convolution_forward::desc>(*itd)->data.dst_desc);
isSuitableDesc &= dst_tdesc.isSame(outputMemoryFormatsFilter[0]);
auto dst_tdesc = MKLDNNExtensionUtils::makeDescriptor(std::shared_ptr<mkldnn::convolution_forward::desc>(*itd)->data.dst_desc);
isSuitableDesc &= dst_tdesc->isSame(outputMemoryFormatsFilter[0]);
}
if (!isSuitableDesc) {
itd = descs.erase(itd);
@@ -698,21 +698,21 @@ bool MKLDNNConvolutionNode::isPossibleToSkipInitConfig(MKLDNNDescriptor &desc) c
isPossibleJitPlanar = false;
std::shared_ptr<mkldnn::convolution_forward::desc> convDesc(desc);
auto srcMemDesc = MKLDNNMemoryDesc {convDesc->data.src_desc};
auto dstMemDesc = MKLDNNMemoryDesc {convDesc->data.dst_desc};
auto srcMemDesc = MKLDNNExtensionUtils::makeDescriptor(convDesc->data.src_desc);
auto dstMemDesc = MKLDNNExtensionUtils::makeDescriptor(convDesc->data.dst_desc);
auto srcDataType = convDesc->data.src_desc.data_type;
auto dstDataType = convDesc->data.dst_desc.data_type;
bool isPlanarFloatConv = srcMemDesc.hasLayoutType(LayoutType::ncsp)
&& dstMemDesc.hasLayoutType(LayoutType::ncsp)
bool isPlanarFloatConv = srcMemDesc->hasLayoutType(LayoutType::ncsp)
&& dstMemDesc->hasLayoutType(LayoutType::ncsp)
&& srcDataType == memory::data_type::f32
&& dstDataType == memory::data_type::f32;
return !isPossibleJitPlanar && isPlanarFloatConv;
}
std::unique_ptr<MKLDNNMemoryDesc> MKLDNNConvolutionNode::getSrcMemDesc(mkldnn::primitive_desc_iterator &primitive_desc_it, size_t idx) {
auto desc = idx > 0 ? MKLDNNMemoryDesc(primitive_desc_it.weights_desc(idx - 1)) : MKLDNNMemoryDesc(primitive_desc_it.src_desc(idx));
return MKLDNNPlugin::make_unique<MKLDNNMemoryDesc>(std::move(desc));
std::shared_ptr<MemoryDesc> MKLDNNConvolutionNode::getSrcMemDesc(mkldnn::primitive_desc_iterator &primitive_desc_it, size_t idx) {
auto desc = idx > 0 ? primitive_desc_it.weights_desc(idx - 1) : primitive_desc_it.src_desc(idx);
return MKLDNNExtensionUtils::makeDescriptor(desc);
}
bool MKLDNNConvolutionNode::canFuse(const MKLDNNNodePtr& node) const {
@@ -755,8 +755,8 @@ bool MKLDNNConvolutionNode::isNspcAvailable() const {
}
// A bunch of heuristics are designed to cut off not optimal nspc convolution applications
auto inpDims = getParentEdgeAt(0)->getShape().getStaticDims();
auto outDims = getChildEdgeAt(0)->getShape().getStaticDims();
auto inpDims = getInputShapeAtPort(0).getStaticDims();
auto outDims = getOutputShapeAtPort(0).getStaticDims();
auto ndims = inpDims.size();
if (isDepthWise()) {
@@ -826,7 +826,7 @@ InferenceEngine::Blob::Ptr MKLDNNConvolutionNode::createInternalBlob(InferenceEn
if (blb == nullptr)
IE_THROW() << "Cannot get const blob for node " << getName() << ".";
auto const elementsCount = blb->GetElementsCount();
auto const elementsCount = blb->GetDescWithType<BlockedMemoryDesc>()->getPaddedElementsCount();
InferenceEngine::TensorDesc desc(InferenceEngine::Precision::FP32, dims, getWeightsLayoutByDims(dims, isGrouped));
inference-engine/src/mkldnn_plugin/nodes/mkldnn_conv_node.h
+4 -4
View File
@@ -18,10 +18,10 @@ class MKLDNNConvolutionNode : public MKLDNNNode {
public:
MKLDNNConvolutionNode(const std::shared_ptr<ngraph::Node>& op, const mkldnn::engine& eng, MKLDNNWeightsSharing::Ptr &cache);
static bool isSupportedOperation(const std::shared_ptr<ngraph::Node>& op, std::string& errorMessage) noexcept;
static bool isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept;
void getSupportedDescriptors() override;
void createDescriptor(const std::vector<const MemoryDesc*>& inputDesc,
const std::vector<const MemoryDesc*>& outputDesc) override;
void createDescriptor(const std::vector<MemoryDescPtr>& inputDesc,
const std::vector<MemoryDescPtr>& outputDesc) override;
void initDescriptor(const NodeConfig& config) override;
void createPrimitive() override;
void selectOptimalPrimitiveDescriptor() override;
@@ -32,7 +32,7 @@ public:
return false;
}
InferenceEngine::Precision getRuntimePrecision() const override;
std::unique_ptr<MKLDNNMemoryDesc> getSrcMemDesc(mkldnn::primitive_desc_iterator &primitive_desc_it, size_t idx) override;
std::shared_ptr<MemoryDesc> getSrcMemDesc(mkldnn::primitive_desc_iterator &primitive_desc_it, size_t idx) override;
const mkldnn::memory& getWeights() const;
const mkldnn::memory& getBias() const;
inference-engine/src/mkldnn_plugin/nodes/mkldnn_convert_node.cpp
+37 -13
View File
@@ -7,13 +7,19 @@
#include "common/cpu_convert.h"
#include "common/blocked_desc_creator.h"
#include <ngraph/opsets/opset1.hpp>
#include "utils/ngraph_utils.hpp"
using namespace mkldnn;
using namespace MKLDNNPlugin;
using namespace InferenceEngine;
bool MKLDNNConvertNode::isSupportedOperation(const std::shared_ptr<ngraph::Node>& op, std::string& errorMessage) noexcept {
bool MKLDNNConvertNode::isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept {
try {
if (isDynamicNgraphNode(op)) {
errorMessage = "Doesn't support op with dynamic shapes";
return false;
}
const auto convert = std::dynamic_pointer_cast<const ngraph::opset1::Convert>(op);
if (!convert) {
errorMessage = "Only opset1 Convert operation is supported";
@@ -59,6 +65,13 @@ void MKLDNNConvertNode::getSupportedDescriptors() {
IE_THROW() << errorPrefix << " has incorrect number of output edges";
}
bool MKLDNNConvertNode::isSupportedDesc(const MemoryDesc &desc) {
bool isSupported = desc.getType() & MemoryDescType::Blocked;
if (desc.getType() == MemoryDescType::DnnlBlocked)
isSupported &= desc.as<const DnnlMemoryDesc>()->hasEmptyExtraData();
return isSupported;
}
void MKLDNNConvertNode::initSupportedPrimitiveDescriptors() {
if (!supportedPrimitiveDescriptors.empty())
return;
@@ -69,21 +82,28 @@ void MKLDNNConvertNode::initSupportedPrimitiveDescriptors() {
config.dynBatchSupport = false;
// if input and output pointers are not null, then the inp/output tensor descriptors were set using setDescs method, so
// they should be used as the actual descriptors.
bool canInitExternalDesc = false;
if (input && output) {
dataIn.desc = input->clone();
canInitExternalDesc = true;
canInitExternalDesc &= isSupportedDesc(*input);
canInitExternalDesc &= isSupportedDesc(*output);
}
// if input and output pointers are not null and not contain extra data, then the inp/output tensor descriptors were set using setDescs method, so
// they should be used as the actual descriptors.
if (canInitExternalDesc) {
dataIn.desc = input;
config.inConfs.push_back(dataIn);
// inp/out layouts must be the same
dataConfigOut.desc = config.inConfs[0].desc->clone();
dataConfigOut.desc->setPrecision(output->getPrecision());
dataConfigOut.desc = config.inConfs[0].desc;
dataConfigOut.desc = MemoryDescUtils::cloneWithNewPrecision(*dataConfigOut.desc, output->getPrecision());
config.outConfs.push_back(dataConfigOut);
supportedPrimitiveDescriptors.emplace_back(config, impl_desc_type::unknown);
} else if (getOriginalInputsNumber() == 1 && getOriginalOutputsNumber() == 1) {
const Shape& insShape = getParentEdgeAt(0)->getShape();
} else if (inputShapes.size() == 1 && outputShapes.size() == 1) {
const Shape& insShape = getInputShapeAtPort(0);
auto insPrecision = getOriginalInputPrecisionAtPort(0);
const Shape& outputShape = getChildEdgeAt(0)->getShape();
const Shape& outputShape = getOutputShapeAtPort(0);
auto outPrecision = getOriginalOutputPrecisionAtPort(0);
config.inConfs.push_back(dataIn);
@@ -93,8 +113,8 @@ void MKLDNNConvertNode::initSupportedPrimitiveDescriptors() {
auto range = BlockedDescCreator::makeFilteredRange(creators, insShape.getRank());
for (auto itr = range.first; itr != range.second; ++itr) {
config.inConfs[0].desc = MKLDNNPlugin::make_unique<BlockedMemoryDesc>(itr->second->createDesc(insPrecision, insShape.getDims()));
config.outConfs[0].desc = MKLDNNPlugin::make_unique<BlockedMemoryDesc>(itr->second->createDesc(outPrecision, outputShape.getDims()));
config.inConfs[0].desc = std::make_shared<CpuBlockedMemoryDesc>(itr->second->createDesc(insPrecision, insShape));
config.outConfs[0].desc = std::make_shared<CpuBlockedMemoryDesc>(itr->second->createDesc(outPrecision, outputShape));
supportedPrimitiveDescriptors.emplace_back(config, impl_desc_type::unknown);
}
@@ -117,12 +137,16 @@ void MKLDNNConvertNode::createPrimitive() {
void MKLDNNConvertNode::execute(mkldnn::stream strm) {
auto& parentMem = getParentEdgeAt(0)->getMemory();
auto& childMem = getChildEdgeAt(0)->getMemory();
if (parentMem.GetElementsCount() != childMem.GetElementsCount())
const auto parentPaddElemCount = parentMem.GetDescWithType<BlockedMemoryDesc>()->getPaddedElementsCount();
const auto childPaddElemCount = childMem.GetDescWithType<BlockedMemoryDesc>()->getPaddedElementsCount();
if (parentPaddElemCount != childPaddElemCount)
IE_THROW() << errorPrefix << " has different elements number in input and output buffers";
void* srcPtr = parentMem.GetPtr();
void* dstPtr = childMem.GetPtr();
cpu_convert(srcPtr, dstPtr, parentMem.GetDesc().getPrecision(), childMem.GetDesc().getPrecision(), parentMem.GetElementsCount());
cpu_convert(srcPtr, dstPtr, parentMem.getDesc().getPrecision(), childMem.getDesc().getPrecision(), parentPaddElemCount);
}
bool MKLDNNConvertNode::created() const {
inference-engine/src/mkldnn_plugin/nodes/mkldnn_convert_node.h
+5 -3
View File
@@ -38,11 +38,13 @@ public:
const MemoryDesc& getInput() const { return *input; }
const MemoryDesc& getOutput() const { return *output; }
static bool isSupportedOperation(const std::shared_ptr<ngraph::Node>& op, std::string& errorMessage) noexcept;
static bool isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept;
static bool isSupportedDesc(const MemoryDesc &desc);
private:
std::unique_ptr<MemoryDesc> input;
std::unique_ptr<MemoryDesc> output;
MemoryDescPtr input;
MemoryDescPtr output;
std::string errorPrefix;
};
inference-engine/src/mkldnn_plugin/nodes/mkldnn_ctc_greedy_decoder_node.cpp
+8 -4
View File
@@ -12,8 +12,12 @@
using namespace MKLDNNPlugin;
using namespace InferenceEngine;
bool MKLDNNCTCGreedyDecoderNode::isSupportedOperation(const std::shared_ptr<ngraph::Node>& op, std::string& errorMessage) noexcept {
bool MKLDNNCTCGreedyDecoderNode::isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept {
try {
if (isDynamicNgraphNode(op)) {
errorMessage = "Doesn't support op with dynamic shapes";
return false;
}
const auto greedyDecOp = ngraph::as_type_ptr<const ngraph::op::v0::CTCGreedyDecoder>(op);
if (!greedyDecOp) {
errorMessage = "Node is not an instance of the CTCGreedyDecoder operation from operation set v0.";
@@ -69,9 +73,9 @@ void MKLDNNCTCGreedyDecoderNode::execute(mkldnn::stream strm) {
const float* sequenceMask = reinterpret_cast<const float *>(getParentEdgeAt(SEQUENCE_LENGTH_INDEX)->getMemoryPtr()->GetPtr());
float* outputSequences = reinterpret_cast<float *>(getChildEdgesAtPort(0)[0]->getMemoryPtr()->GetPtr());
const size_t T = getParentEdgeAt(DATA_INDEX)->getShape().getStaticDims()[0];
const size_t B = getParentEdgeAt(DATA_INDEX)->getShape().getStaticDims()[1];
const int C = getParentEdgeAt(DATA_INDEX)->getShape().getStaticDims()[2];
const size_t T = getParentEdgeAt(DATA_INDEX)->getMemory().getStaticDims()[0];
const size_t B = getParentEdgeAt(DATA_INDEX)->getMemory().getStaticDims()[1];
const int C = getParentEdgeAt(DATA_INDEX)->getMemory().getStaticDims()[2];
const size_t BC = B * C;
const size_t CB1 = C * (B - 1);
inference-engine/src/mkldnn_plugin/nodes/mkldnn_ctc_greedy_decoder_node.h
+1 -1
View File
@@ -19,7 +19,7 @@ public:
void execute(mkldnn::stream strm) override;
bool created() const override;
static bool isSupportedOperation(const std::shared_ptr<ngraph::Node>& op, std::string& errorMessage) noexcept;
static bool isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept;
private:
const size_t DATA_INDEX = 0lu;
inference-engine/src/mkldnn_plugin/nodes/mkldnn_ctc_greedy_decoder_seq_len_node.cpp
+11 -7
View File
@@ -12,8 +12,12 @@
using namespace MKLDNNPlugin;
using namespace InferenceEngine;
bool MKLDNNCTCGreedyDecoderSeqLenNode::isSupportedOperation(const std::shared_ptr<ngraph::Node>& op, std::string& errorMessage) noexcept {
bool MKLDNNCTCGreedyDecoderSeqLenNode::isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept {
try {
if (isDynamicNgraphNode(op)) {
errorMessage = "Doesn't support op with dynamic shapes";
return false;
}
const auto greedyDecOp = ngraph::as_type_ptr<const ngraph::op::v6::CTCGreedyDecoderSeqLen>(op);
if (!greedyDecOp) {
errorMessage = "Node is not an instance of the CTCGreedyDecoderSeqLen operation from operation set v6.";
@@ -58,9 +62,9 @@ void MKLDNNCTCGreedyDecoderSeqLenNode::initSupportedPrimitiveDescriptors() {
IE_THROW() << errorPrefix << "has unsupported 'sequence_length' input precision: " << seqLenPrecision;
std::vector<PortConfigurator> inDataConf;
inDataConf.reserve(getOriginalInputsNumber());
inDataConf.reserve(inputShapes.size());
inDataConf.emplace_back(LayoutType::ncsp, Precision::FP32);
for (int i = 1; i < getOriginalInputsNumber(); ++i)
for (int i = 1; i < inputShapes.size(); ++i)
inDataConf.emplace_back(LayoutType::ncsp, Precision::I32);
addSupportedPrimDesc(inDataConf,
@@ -75,9 +79,9 @@ void MKLDNNCTCGreedyDecoderSeqLenNode::execute(mkldnn::stream strm) {
int* decodedClasses = reinterpret_cast<int *>(getChildEdgesAtPort(DECODED_CLASSES_INDEX)[0]->getMemoryPtr()->GetPtr());
int* decodedClassesLength = reinterpret_cast<int *>(getChildEdgesAtPort(DECODED_CLASSES_LENGTH_INDEX)[0]->getMemoryPtr()->GetPtr());
const size_t B = getParentEdgeAt(DATA_INDEX)->getShape().getStaticDims()[0];;
const size_t T = getParentEdgeAt(DATA_INDEX)->getShape().getStaticDims()[1];;
const int C = getParentEdgeAt(DATA_INDEX)->getShape().getStaticDims()[2];;
const size_t B = getParentEdgeAt(DATA_INDEX)->getMemory().getStaticDims()[0];;
const size_t T = getParentEdgeAt(DATA_INDEX)->getMemory().getStaticDims()[1];;
const int C = getParentEdgeAt(DATA_INDEX)->getMemory().getStaticDims()[2];;
const size_t TC = T * C;
int blankIndex = C - 1;
@@ -90,7 +94,7 @@ void MKLDNNCTCGreedyDecoderSeqLenNode::execute(mkldnn::stream strm) {
std::string errorMsg = errorPrefix
+ ". Sequence length " + std::to_string(sequenceLengths[b])
+ " cannot be greater than according decoded classes dimension size "
+ std::to_string(getChildEdgesAtPort(DECODED_CLASSES_INDEX)[0]->getShape().getStaticDims()[1]);
+ std::to_string(getChildEdgesAtPort(DECODED_CLASSES_INDEX)[0]->getMemory().getStaticDims()[1]);
IE_THROW() << errorMsg;
}
workAmount += sequenceLengths[b];
inference-engine/src/mkldnn_plugin/nodes/mkldnn_ctc_greedy_decoder_seq_len_node.h
+1 -1
View File
@@ -19,7 +19,7 @@ public:
void execute(mkldnn::stream strm) override;
bool created() const override;
static bool isSupportedOperation(const std::shared_ptr<ngraph::Node>& op, std::string& errorMessage) noexcept;
static bool isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept;
private:
const size_t DATA_INDEX = 0lu;
inference-engine/src/mkldnn_plugin/nodes/mkldnn_ctc_loss_node.cpp
+11 -6
View File
@@ -11,8 +11,12 @@
using namespace MKLDNNPlugin;
using namespace InferenceEngine;
bool MKLDNNCTCLossNode::isSupportedOperation(const std::shared_ptr<ngraph::Node>& op, std::string& errorMessage) noexcept {
bool MKLDNNCTCLossNode::isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept {
try {
if (isDynamicNgraphNode(op)) {
errorMessage = "Doesn't support op with dynamic shapes";
return false;
}
const auto ctcLossOp = ngraph::as_type_ptr<const ngraph::op::v4::CTCLoss>(op);
if (!ctcLossOp) {
errorMessage = "Node is not an instance of the CTCLoss operation from operation set v4.";
@@ -47,9 +51,9 @@ void MKLDNNCTCLossNode::initSupportedPrimitiveDescriptors() {
return;
std::vector<PortConfigurator> inDataConf;
inDataConf.reserve(getOriginalInputsNumber());
inDataConf.reserve(inputShapes.size());
inDataConf.emplace_back(LayoutType::ncsp, Precision::FP32);
for (int i = 1; i < getOriginalInputsNumber(); ++i)
for (int i = 1; i < inputShapes.size(); ++i)
inDataConf.emplace_back(LayoutType::ncsp, Precision::I32);
addSupportedPrimDesc(inDataConf,
@@ -66,9 +70,10 @@ void MKLDNNCTCLossNode::execute(mkldnn::stream strm) {
const int* labelsLength = reinterpret_cast<const int *>(getParentEdgeAt(3)->getMemoryPtr()->GetPtr());
float* dstData = reinterpret_cast<float *>(getChildEdgesAtPort(0)[0]->getMemoryPtr()->GetPtr());
const size_t batchNum = getParentEdgeAt(0)->getShape().getStaticDims()[0];
const size_t maxTime = getParentEdgeAt(0)->getShape().getStaticDims()[1];
const size_t classesNum = getParentEdgeAt(0)->getShape().getStaticDims()[2];
const auto &inDims = getParentEdgeAt(0)->getMemory().getStaticDims();
const size_t batchNum = inDims[0];
const size_t maxTime = inDims[1];
const size_t classesNum = inDims[2];
int blankIndex = classesNum - 1;
if (inputShapes.size() > 4) {
inference-engine/src/mkldnn_plugin/nodes/mkldnn_ctc_loss_node.h
+1 -1
View File
@@ -19,7 +19,7 @@ public:
void execute(mkldnn::stream strm) override;
bool created() const override;
static bool isSupportedOperation(const std::shared_ptr<ngraph::Node>& op, std::string& errorMessage) noexcept;
static bool isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept;
private:
bool ctcMergeRepeated;
inference-engine/src/mkldnn_plugin/nodes/mkldnn_cum_sum_node.cpp
+12 -8
View File
@@ -16,8 +16,12 @@
using namespace MKLDNNPlugin;
using namespace InferenceEngine;
bool MKLDNNCumSumNode::isSupportedOperation(const std::shared_ptr<ngraph::Node>& op, std::string& errorMessage) noexcept {
bool MKLDNNCumSumNode::isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept {
try {
if (isDynamicNgraphNode(op)) {
errorMessage = "Doesn't support op with dynamic shapes";
return false;
}
const auto cumsum = std::dynamic_pointer_cast<const ngraph::opset3::CumSum>(op);
if (!cumsum) {
errorMessage = "Only opset3 CumSum operation is supported";
@@ -71,16 +75,16 @@ void MKLDNNCumSumNode::initSupportedPrimitiveDescriptors() {
dataPrecision != Precision::FP32 && dataPrecision != Precision::I64 && dataPrecision != Precision::U64 && dataPrecision != Precision::BF16)
IE_THROW() << errorPrefix << " has unsupported 'data' input precision: " << dataPrecision.name();
if (getOriginalInputsNumber() == numOfInputs) {
if (inputShapes.size() == numOfInputs) {
const auto &axisTensorPrec = getOriginalInputPrecisionAtPort(AXIS);
if (axisTensorPrec != Precision::I32 && axisTensorPrec != Precision::I64)
IE_THROW() << errorPrefix << " has unsupported 'axis' input precision: " << axisTensorPrec.name();
}
std::vector<PortConfigurator> inDataConf;
inDataConf.reserve(getOriginalInputsNumber());
inDataConf.reserve(inputShapes.size());
inDataConf.emplace_back(LayoutType::ncsp, dataPrecision);
for (int i = 1; i < getOriginalInputsNumber(); ++i)
for (int i = 1; i < inputShapes.size(); ++i)
inDataConf.emplace_back(LayoutType::ncsp, Precision::I32);
addSupportedPrimDesc(inDataConf,
@@ -133,7 +137,7 @@ template <typename dataType>
void MKLDNNCumSumNode::exec() {
const auto *input = reinterpret_cast<const dataType *>(getParentEdgeAt(CUM_SUM_DATA)->getMemoryPtr()->GetPtr());
auto *output = reinterpret_cast<dataType *>(getChildEdgesAtPort(0)[0]->getMemoryPtr()->GetPtr());
const std::vector<size_t> strides = getParentEdgeAt(CUM_SUM_DATA)->getMemory().GetDescWithType<BlockedMemoryDesc>().getStrides();
const VectorDims strides = getParentEdgeAt(CUM_SUM_DATA)->getMemory().GetDescWithType<BlockedMemoryDesc>()->getStrides();
if (reverse) {
if (exclusive) {
@@ -151,7 +155,7 @@ void MKLDNNCumSumNode::exec() {
}
template <bool reverse, bool exclusive, typename dataType>
void MKLDNNCumSumNode::cumSum(const dataType *input, dataType *output, const std::vector<size_t> &strides) {
void MKLDNNCumSumNode::cumSum(const dataType *input, dataType *output, const VectorDims &strides) {
SizeVector iterationRange(numOfDims - 1);
size_t j = 0;
for (size_t i = 0; i < shape.size(); i++) {
@@ -248,8 +252,8 @@ inline size_t MKLDNNCumSumNode::getStartOffset(const std::vector<size_t> &forSta
}
size_t MKLDNNCumSumNode::getAxis(const MKLDNNMemory& _axis, const MKLDNNMemory& _data) const {
const auto& axisPrecision = _axis.GetDesc().getPrecision();
const int64_t dataShapeSize = static_cast<int64_t>(_data.GetDesc().getShape().getRank());
const auto& axisPrecision = _axis.getDesc().getPrecision();
const int64_t dataShapeSize = static_cast<int64_t>(_data.GetShape().getRank());
int64_t axisValueFromBlob;
switch (axisPrecision) {
case Precision::I32 : {
inference-engine/src/mkldnn_plugin/nodes/mkldnn_cum_sum_node.h
+1 -1
View File
@@ -19,7 +19,7 @@ public:
void execute(mkldnn::stream strm) override;
bool created() const override;
static bool isSupportedOperation(const std::shared_ptr<ngraph::Node>& op, std::string& errorMessage) noexcept;
static bool isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept;
private:
template <typename dataType>
inference-engine/src/mkldnn_plugin/nodes/mkldnn_deconv_node.cpp
+46 -45
View File
@@ -16,14 +16,20 @@
#include <ngraph/opsets/opset1.hpp>
#include <cpu/x64/cpu_isa_traits.hpp>
#include <nodes/common/cpu_memcpy.h>
#include <cpu_memory_desc_utils.h>
#include <memory_desc/cpu_memory_desc_utils.h>
#include "memory_desc/dnnl_blocked_memory_desc.h"
using namespace mkldnn;
using namespace MKLDNNPlugin;
using namespace InferenceEngine;
bool MKLDNNDeconvolutionNode::isSupportedOperation(const std::shared_ptr<ngraph::Node>& op, std::string& errorMessage) noexcept {
bool MKLDNNDeconvolutionNode::isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept {
try {
if (isDynamicNgraphNode(op)) {
errorMessage = "Doesn't support op with dynamic shapes";
return false;
}
if (std::dynamic_pointer_cast<const ngraph::opset1::ConvolutionBackpropData>(op) == nullptr &&
std::dynamic_pointer_cast<const ngraph::opset1::GroupConvolutionBackpropData>(op) == nullptr) {
errorMessage = "Only opset1 ConvolutionBackpropData and GroupConvolutionBackpropData operations are supported";
@@ -42,8 +48,8 @@ bool MKLDNNDeconvolutionNode::isSupportedOperation(const std::shared_ptr<ngraph:
MKLDNNDeconvolutionNode::MKLDNNDeconvolutionNode(const std::shared_ptr<ngraph::Node>& op,
const mkldnn::engine& eng, MKLDNNWeightsSharing::Ptr &cache) : MKLDNNNode(op, eng, cache) {
internalBlobDesc.emplace_back([&](primitive_desc_iterator &primitive_desc_it, size_t idx) -> MKLDNNMemoryDesc {
return MKLDNNMemoryDesc(primitive_desc_it.weights_desc(0));
internalBlobDesc.emplace_back([&](primitive_desc_iterator &primitive_desc_it, size_t idx) -> DnnlMemoryDescPtr {
return MKLDNNExtensionUtils::makeDescriptor(primitive_desc_it.weights_desc(0));
});
std::string errorMessage;
if (isSupportedOperation(op, errorMessage)) {
@@ -152,7 +158,7 @@ bool MKLDNNDeconvolutionNode::canBeExecutedInInt8() const {
if (!withGroups && stride.back() > 3)
return false;
if (!impl::cpu::x64::mayiuse(impl::cpu::x64::avx512_common)) {
auto inDims = getChildEdgeAt(0)->getShape().getStaticDims();
auto inDims = getOutputShapeAtPort(0).getStaticDims();
// heuristicConst = 2^26
// heuristicParam = IC^2 * SP
auto heuristicConst = 67108864;
@@ -231,8 +237,8 @@ void MKLDNNDeconvolutionNode::getSupportedDescriptors() {
for (int i = 0; i < paddingR.size(); i++) {
int with_group = getAlgorithm() == DeconvolutionGrouped ? 1 : 0;
int krn = weightDims[with_group + 2 + i];
int src = getChildEdgeAt(0)->getShape().getStaticDims()[2 + i];
int dst = getParentEdgeAt(0)->getShape().getStaticDims()[2 + i];
int src = getOutputShapeAtPort(0).getStaticDims()[2 + i];
int dst = getInputShapeAtPort(0).getStaticDims()[2 + i];
krn = (krn - 1)*(dilation[i] + 1) + 1;
int calc_dst = (src - krn + paddingL[i]) / stride[i] + 1;
@@ -243,15 +249,15 @@ void MKLDNNDeconvolutionNode::getSupportedDescriptors() {
// WA: if int8 deconvolution is supported, we create internal weights blob in IO format
std::swap(weightDims[withGroups + 0], weightDims[withGroups + 1]);
internalBlobs.push_back(createWeiBlobAsIO(weightDims));
auto format = getParentEdgeAt(0)->getShape().getRank() == 5 ? dnnl::memory::format_tag::ndhwc : dnnl::memory::format_tag::nhwc;
MKLDNNMemoryDesc in_candidate(getParentEdgeAt(0)->getShape().getStaticDims(), inputDataType, format);
MKLDNNMemoryDesc out_candidate(getChildEdgeAt(0)->getShape().getStaticDims(), outputDataType, format);
createDescriptor({&in_candidate}, {&out_candidate});
auto format = getInputShapeAtPort(0).getRank() == 5 ? dnnl::memory::format_tag::ndhwc : dnnl::memory::format_tag::nhwc;
MemoryDescPtr in_candidate = std::make_shared<DnnlBlockedMemoryDesc>(getInputShapeAtPort(0), inputDataType, format);
MemoryDescPtr out_candidate = std::make_shared<DnnlBlockedMemoryDesc>(getOutputShapeAtPort(0), outputDataType, format);
createDescriptor({in_candidate}, {out_candidate});
} else {
for (auto format : getAvailableFormatsForDims(getParentEdgeAt(0)->getShape())) {
MKLDNNMemoryDesc in_candidate(getParentEdgeAt(0)->getShape().getStaticDims(), inputDataType, format);
MKLDNNMemoryDesc out_candidate(getChildEdgeAt(0)->getShape().getStaticDims(), outputDataType, format);
createDescriptor({&in_candidate}, {&out_candidate});
for (auto format : getAvailableFormatsForDims(getInputShapeAtPort(0))) {
MemoryDescPtr in_candidate = std::make_shared<DnnlBlockedMemoryDesc>(getInputShapeAtPort(0), inputDataType, format);
MemoryDescPtr out_candidate = std::make_shared<DnnlBlockedMemoryDesc>(getOutputShapeAtPort(0), outputDataType, format);
createDescriptor({in_candidate}, {out_candidate});
}
}
setPostOps(attr);
@@ -292,20 +298,20 @@ void MKLDNNDeconvolutionNode::filterSupportedDescriptors() {
bool isSuitableDesc = true;
if (!inputMemoryFormatsFilter.empty()) {
if (isInt8) {
auto src_tdesc = MKLDNNMemoryDesc(std::shared_ptr<dnnl::deconvolution_forward::desc>(*itd)->data.src_desc);
isSuitableDesc &= src_tdesc.isSame(inputMemoryFormatsFilter[0]);
auto src_tdesc = MKLDNNExtensionUtils::makeDescriptor(std::shared_ptr<dnnl::deconvolution_forward::desc>(*itd)->data.src_desc);
isSuitableDesc &= src_tdesc->isSame(inputMemoryFormatsFilter[0]);
} else {
auto src_tdesc = MKLDNNMemoryDesc(std::shared_ptr<mkldnn::convolution_backward_data::desc>(*itd)->data.diff_src_desc);
isSuitableDesc &= src_tdesc.isSame(inputMemoryFormatsFilter[0]);
auto src_tdesc = MKLDNNExtensionUtils::makeDescriptor(std::shared_ptr<mkldnn::convolution_backward_data::desc>(*itd)->data.diff_src_desc);
isSuitableDesc &= src_tdesc->isSame(inputMemoryFormatsFilter[0]);
}
}
if (!outputMemoryFormatsFilter.empty()) {
if (isInt8) {
auto dst_tdesc = MKLDNNMemoryDesc(std::shared_ptr<mkldnn::deconvolution_forward::desc>(*itd)->data.dst_desc);
isSuitableDesc &= dst_tdesc.isSame(outputMemoryFormatsFilter[0]);
auto dst_tdesc = MKLDNNExtensionUtils::makeDescriptor(std::shared_ptr<mkldnn::deconvolution_forward::desc>(*itd)->data.dst_desc);
isSuitableDesc &= dst_tdesc->isSame(outputMemoryFormatsFilter[0]);
} else {
auto dst_tdesc = MKLDNNMemoryDesc(std::shared_ptr<mkldnn::convolution_backward_data::desc>(*itd)->data.diff_dst_desc);
isSuitableDesc &= dst_tdesc.isSame(outputMemoryFormatsFilter[0]);
auto dst_tdesc = MKLDNNExtensionUtils::makeDescriptor(std::shared_ptr<mkldnn::convolution_backward_data::desc>(*itd)->data.diff_dst_desc);
isSuitableDesc &= dst_tdesc->isSame(outputMemoryFormatsFilter[0]);
}
}
if (!isSuitableDesc) {
@@ -347,10 +353,10 @@ void MKLDNNDeconvolutionNode::createPrimitive() {
}
}
void MKLDNNDeconvolutionNode::createDescriptor(const std::vector<const MemoryDesc*> &inputDesc,
const std::vector<const MemoryDesc*> &outputDesc) {
const MKLDNNMemoryDesc in_candidate = MemoryDescUtils::convertToMKLDNNMemoryDesc(*inputDesc[0]);
const MKLDNNMemoryDesc out_candidate = MemoryDescUtils::convertToMKLDNNMemoryDesc(*outputDesc[0]);
void MKLDNNDeconvolutionNode::createDescriptor(const std::vector<MemoryDescPtr> &inputDesc,
const std::vector<MemoryDescPtr> &outputDesc) {
const auto in_candidate = MemoryDescUtils::convertToDnnlBlockedMemoryDesc(*inputDesc[0]);
const auto out_candidate = MemoryDescUtils::convertToDnnlBlockedMemoryDesc(*outputDesc[0]);
// grouping and autoblicking is not compatible
if ((withGroups && !isDW) && (in_candidate.blocksExtended() || out_candidate.blocksExtended()))
@@ -361,29 +367,27 @@ void MKLDNNDeconvolutionNode::createDescriptor(const std::vector<const MemoryDes
};
if (isInt8) {
MKLDNNDims weightsDims = MKLDNNDims(weightDims);
mkldnn::memory::desc wgh_candidate(weightsDims, memory::data_type::s8, memory::format_tag::any);
mkldnn::memory::desc wgh_candidate(MKLDNNExtensionUtils::convertToDnnlDims(weightDims), memory::data_type::s8, memory::format_tag::any);
std::shared_ptr<mkldnn::deconvolution_forward::desc> deconv_desc;
deconv_desc.reset(new deconvolution_forward::desc(prop_kind::forward_inference, mkldnn::algorithm::deconvolution_direct,
in_candidate, wgh_candidate, out_candidate,
in_candidate.getDnnlDesc(), wgh_candidate, out_candidate.getDnnlDesc(),
convertDims(stride), convertDims(dilation),
convertDims(paddingL), convertDims(paddingR)));
descs.emplace_back(deconv_desc);
} else {
MKLDNNDims weightsDims = MKLDNNDims(weightDims);
mkldnn::memory::desc wgh_candidate(weightsDims, in_candidate.getDataType(), memory::format_tag::any);
mkldnn::memory::desc wgh_candidate(MKLDNNExtensionUtils::convertToDnnlDims(weightDims), in_candidate.getDataType(), memory::format_tag::any);
for (auto alg : {mkldnn::algorithm::convolution_winograd, mkldnn::algorithm::convolution_direct}) {
std::shared_ptr<mkldnn::convolution_forward::desc> conv_desc;
conv_desc.reset(new convolution_forward::desc(prop_kind::forward_inference, alg,
out_candidate, wgh_candidate, in_candidate,
out_candidate.getDnnlDesc(), wgh_candidate, in_candidate.getDnnlDesc(),
convertDims(stride),
convertDims(dilation),
convertDims(paddingL),
convertDims(paddingR)));
std::shared_ptr<mkldnn::convolution_backward_data::desc> deconv_desc;
deconv_desc.reset(new convolution_backward_data::desc(alg, out_candidate, wgh_candidate,
in_candidate,
deconv_desc.reset(new convolution_backward_data::desc(alg, out_candidate.getDnnlDesc(), wgh_candidate,
in_candidate.getDnnlDesc(),
convertDims(stride),
convertDims(dilation),
convertDims(paddingL),
@@ -400,21 +404,18 @@ void MKLDNNDeconvolutionNode::createDescriptor(const std::vector<const MemoryDes
}
}
std::unique_ptr<MKLDNNMemoryDesc> MKLDNNDeconvolutionNode::getSrcMemDesc(mkldnn::primitive_desc_iterator &primitive_desc_it, size_t idx) {
std::shared_ptr<MemoryDesc> MKLDNNDeconvolutionNode::getSrcMemDesc(mkldnn::primitive_desc_iterator &primitive_desc_it, size_t idx) {
if (idx == 2) {
auto dataType = MKLDNNExtensionUtils::IEPrecisionToDataType(getOriginalInputPrecisionAtPort(2));
return MKLDNNPlugin::make_unique<MKLDNNMemoryDesc>(getParentEdgeAt(2)->getShape().getStaticDims(), dataType,
MKLDNNMemory::GetPlainFormatByRank(getParentEdgeAt(2)->getShape().getRank()));
return std::make_shared<CpuBlockedMemoryDesc>(getOriginalInputPrecisionAtPort(2), Shape(getInputShapeAtPort(2).getStaticDims()));
}
MKLDNNMemoryDesc desc = idx > 0 ? MKLDNNMemoryDesc(primitive_desc_it.weights_desc(idx - 1))
: isInt8 ? MKLDNNMemoryDesc(primitive_desc_it.src_desc(idx)) : MKLDNNMemoryDesc(primitive_desc_it.diff_dst_desc(idx));
return MKLDNNPlugin::make_unique<MKLDNNMemoryDesc>(std::move(desc));
auto desc = idx > 0 ? primitive_desc_it.weights_desc(idx - 1) : isInt8 ? primitive_desc_it.src_desc(idx) : primitive_desc_it.diff_dst_desc(idx);
return MKLDNNExtensionUtils::makeDescriptor(desc);
}
std::unique_ptr<MKLDNNMemoryDesc> MKLDNNDeconvolutionNode::getDstMemDesc(mkldnn::primitive_desc_iterator &primitive_desc_it, size_t idx) {
return isInt8 ? MKLDNNPlugin::make_unique<MKLDNNMemoryDesc>(primitive_desc_it.dst_desc(idx)) :
MKLDNNPlugin::make_unique<MKLDNNMemoryDesc>(primitive_desc_it.diff_src_desc(idx));
std::shared_ptr<MemoryDesc> MKLDNNDeconvolutionNode::getDstMemDesc(mkldnn::primitive_desc_iterator &primitive_desc_it, size_t idx) {
auto desc = isInt8 ? primitive_desc_it.dst_desc(idx) : primitive_desc_it.diff_src_desc(idx);
return MKLDNNExtensionUtils::makeDescriptor(desc);
}
InferenceEngine::Precision MKLDNNDeconvolutionNode::getRuntimePrecision() const {
inference-engine/src/mkldnn_plugin/nodes/mkldnn_deconv_node.h
+5 -5
View File
@@ -17,8 +17,8 @@ public:
MKLDNNDeconvolutionNode(const std::shared_ptr<ngraph::Node>& op, const mkldnn::engine& eng, MKLDNNWeightsSharing::Ptr &cache);
void getSupportedDescriptors() override;
void createDescriptor(const std::vector<const MemoryDesc*>& inputDesc,
const std::vector<const MemoryDesc*>& outputDesc) override;
void createDescriptor(const std::vector<MemoryDescPtr>& inputDesc,
const std::vector<MemoryDescPtr>& outputDesc) override;
void createPrimitive() override;
void filterSupportedPrimitiveDescriptors() override;
void filterSupportedDescriptors();
@@ -31,12 +31,12 @@ public:
return static_cast<size_t>(getParentEdges().size());
}
std::unique_ptr<MKLDNNMemoryDesc> getSrcMemDesc(mkldnn::primitive_desc_iterator &primitive_desc_it, size_t idx) override;
std::unique_ptr<MKLDNNMemoryDesc> getDstMemDesc(mkldnn::primitive_desc_iterator &primitive_desc_it, size_t idx) override;
std::shared_ptr<MemoryDesc> getSrcMemDesc(mkldnn::primitive_desc_iterator &primitive_desc_it, size_t idx) override;
std::shared_ptr<MemoryDesc> getDstMemDesc(mkldnn::primitive_desc_iterator &primitive_desc_it, size_t idx) override;
InferenceEngine::Precision getRuntimePrecision() const override;
static bool isSupportedOperation(const std::shared_ptr<ngraph::Node>& op, std::string& errorMessage) noexcept;
static bool isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept;
bool canFuse(const MKLDNNNodePtr& node) const override;
private:
inference-engine/src/mkldnn_plugin/nodes/mkldnn_def_conv_node.cpp
+35 -30
View File
@@ -14,6 +14,7 @@
#include <mkldnn_extension_utils.h>
#include <cpu/x64/jit_generator.hpp>
#include "ie_parallel.hpp"
#include "memory_desc/dnnl_blocked_memory_desc.h"
using namespace mkldnn;
using namespace MKLDNNPlugin;
@@ -741,6 +742,10 @@ private:
bool MKLDNNDeformableConvolutionNode::isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept {
try {
if (isDynamicNgraphNode(op)) {
errorMessage = "Doesn't support op with dynamic shapes";
return false;
}
if (!one_of(op->get_type_info(),
ngraph::op::v1::DeformableConvolution::type_info,
ngraph::op::v8::DeformableConvolution::type_info)) {
@@ -793,20 +798,20 @@ void MKLDNNDeformableConvolutionNode::getSupportedDescriptors() {
if (getChildEdges().empty())
IE_THROW() << errorPrefix << "has incorrect number of output edges";
if (getParentEdgeAt(0)->getShape().getRank() != 4) {
if (getInputShapeAtPort(0).getRank() != 4) {
IE_THROW() << "Deformable convolution layer. Unsupported mode. Only 4D blobs are supported as input.";
}
if (getParentEdgeAt(1)->getShape().getRank() != 4) {
IE_THROW() << errorPrefix << "doesn't support 1st input with rank: " << getParentEdgeAt(1)->getShape().getRank();
if (getInputShapeAtPort(1).getRank() != 4) {
IE_THROW() << errorPrefix << "doesn't support 1st input with rank: " << getInputShapeAtPort(1).getRank();
}
if (getParentEdgeAt(2)->getShape().getRank() != 4) {
IE_THROW() << errorPrefix << "doesn't support 2nd input with rank: " << getParentEdgeAt(2)->getShape().getRank();
if (getInputShapeAtPort(2).getRank() != 4) {
IE_THROW() << errorPrefix << "doesn't support 2nd input with rank: " << getInputShapeAtPort(2).getRank();
}
if (getChildEdgeAt(0)->getShape().getRank() != 4) {
IE_THROW() << errorPrefix << "doesn't support output with rank: " << getChildEdgeAt(0)->getShape().getRank();
if (getOutputShapeAtPort(0).getRank() != 4) {
IE_THROW() << errorPrefix << "doesn't support output with rank: " << getOutputShapeAtPort(0).getRank();
}
}
@@ -853,45 +858,45 @@ void MKLDNNDeformableConvolutionNode::initSupportedPrimitiveDescriptors() {
auto weiFormat = group > 1 ? mayiuse(avx512_common) ? memory::format_tag::gOIhw16i16o : memory::format_tag::gOIhw8i8o
: mayiuse(avx512_common) ? memory::format_tag::OIhw16i16o : memory::format_tag::OIhw8i8o;
config.inConfs[0].desc = MKLDNNPlugin::make_unique<MKLDNNMemoryDesc>(getParentEdgeAt(0)->getShape().getStaticDims(),
config.inConfs[0].desc = std::make_shared<DnnlBlockedMemoryDesc>(getInputShapeAtPort(0),
memory::data_type::f32, dataFormat);
config.inConfs[1].desc = MKLDNNPlugin::make_unique<MKLDNNMemoryDesc>(getParentEdgeAt(1)->getShape().getStaticDims(),
config.inConfs[1].desc = std::make_shared<DnnlBlockedMemoryDesc>(getInputShapeAtPort(1),
memory::data_type::f32, offFormat);
auto& wDims = getParentEdgeAt(2)->getShape().getStaticDims();
auto& wDims = getInputShapeAtPort(2).getStaticDims();
if (group > 1 && wDims.size() != 5) {
auto new_dims = InferenceEngine::SizeVector({group, div_up(wDims[0], group)});
for (int i = 1; i < wDims.size(); i++) {
new_dims.push_back(wDims[i]);
}
config.inConfs[2].desc = MKLDNNPlugin::make_unique<MKLDNNMemoryDesc>(getParentEdgeAt(2)->getShape().getStaticDims(),
config.inConfs[2].desc = std::make_shared<DnnlBlockedMemoryDesc>(getInputShapeAtPort(2),
memory::data_type::f32, weiFormat);
} else {
config.inConfs[2].desc = MKLDNNPlugin::make_unique<MKLDNNMemoryDesc>(getParentEdgeAt(2)->getShape().getStaticDims(),
config.inConfs[2].desc = std::make_shared<DnnlBlockedMemoryDesc>(getInputShapeAtPort(2),
memory::data_type::f32, weiFormat);
}
if (inputsNumber > 3) {
config.inConfs[3].desc = MKLDNNPlugin::make_unique<MKLDNNMemoryDesc>(getParentEdgeAt(3)->getShape().getStaticDims(),
config.inConfs[3].desc = std::make_shared<DnnlBlockedMemoryDesc>(getInputShapeAtPort(3),
memory::data_type::f32, memory::format_tag::nchw);
}
config.outConfs[0].desc = MKLDNNPlugin::make_unique<MKLDNNMemoryDesc>(getChildEdgeAt(0)->getShape().getStaticDims(),
config.outConfs[0].desc = std::make_shared<DnnlBlockedMemoryDesc>(getOutputShapeAtPort(0),
memory::data_type::f32, dataFormat);
supportedPrimitiveDescriptors.push_back({config, impl_type});
} else {
// reference implementation
config.inConfs[0].desc = MKLDNNPlugin::make_unique<MKLDNNMemoryDesc>(getParentEdgeAt(0)->getShape().getStaticDims(), memory::data_type::f32,
config.inConfs[0].desc = std::make_shared<DnnlBlockedMemoryDesc>(getInputShapeAtPort(0), memory::data_type::f32,
memory::format_tag::nchw);
config.inConfs[1].desc = MKLDNNPlugin::make_unique<MKLDNNMemoryDesc>(getParentEdgeAt(1)->getShape().getStaticDims(), memory::data_type::f32,
config.inConfs[1].desc = std::make_shared<DnnlBlockedMemoryDesc>(getInputShapeAtPort(1), memory::data_type::f32,
memory::format_tag::nchw);
config.inConfs[2].desc = MKLDNNPlugin::make_unique<MKLDNNMemoryDesc>(getParentEdgeAt(2)->getShape().getStaticDims(), memory::data_type::f32,
config.inConfs[2].desc = std::make_shared<DnnlBlockedMemoryDesc>(getInputShapeAtPort(2), memory::data_type::f32,
memory::format_tag::oihw);
if (inputsNumber > 3) {
config.inConfs[3].desc = MKLDNNPlugin::make_unique<MKLDNNMemoryDesc>(getParentEdgeAt(3)->getShape().getStaticDims(), memory::data_type::f32,
config.inConfs[3].desc = std::make_shared<DnnlBlockedMemoryDesc>(getInputShapeAtPort(3), memory::data_type::f32,
memory::format_tag::nchw);
}
config.outConfs[0].desc = MKLDNNPlugin::make_unique<MKLDNNMemoryDesc>(getChildEdgeAt(0)->getShape().getStaticDims(), memory::data_type::f32,
config.outConfs[0].desc = std::make_shared<DnnlBlockedMemoryDesc>(getOutputShapeAtPort(0), memory::data_type::f32,
memory::format_tag::nchw);
supportedPrimitiveDescriptors.push_back({config, impl_type});
}
@@ -903,9 +908,9 @@ void MKLDNNDeformableConvolutionNode::createPrimitive() {
IE_THROW() << "CPU deformable convolution with name '" << getName() << "' doesn't have primitive descriptors.";
auto config = selectedPrimitiveDescriptor->getConfig();
auto srcDims = getParentEdgeAt(0)->getShape().getStaticDims();
auto weiDims = getParentEdgeAt(2)->getShape().getStaticDims();
auto dstDims = getChildEdgeAt(0)->getShape().getStaticDims();
auto srcDims = getParentEdgeAt(0)->getMemory().getStaticDims();
auto weiDims = getParentEdgeAt(2)->getMemory().getStaticDims();
auto dstDims = getChildEdgesAtPort(0)[0]->getMemory().getStaticDims();
jcp.dg = deformable_group;
@@ -1136,23 +1141,23 @@ void MKLDNNDeformableConvolutionNode::execute(mkldnn::stream strm) {
auto config = selectedPrimitiveDescriptor->getConfig();
auto src_block_desc = getParentEdgeAt(0)->getMemory().GetDescWithType<BlockedMemoryDesc>();
std::vector<size_t> src_strides(src_block_desc.getStrides().size());
std::vector<size_t> src_strides(src_block_desc->getStrides().size());
for (int i = 0; i < src_strides.size(); i++) {
src_strides[src_block_desc.getOrder()[i]] = src_block_desc.getStrides()[i];
src_strides[src_block_desc->getOrder()[i]] = src_block_desc->getStrides()[i];
}
auto dst_block_desc = getChildEdgeAt(0)->getMemory().GetDescWithType<BlockedMemoryDesc>();
std::vector<size_t> dst_strides(dst_block_desc.getStrides().size());
std::vector<size_t> dst_strides(dst_block_desc->getStrides().size());
for (int i = 0; i < dst_strides.size(); i++) {
dst_strides[dst_block_desc.getOrder()[i]] = dst_block_desc.getStrides()[i];
dst_strides[dst_block_desc->getOrder()[i]] = dst_block_desc->getStrides()[i];
}
auto off_strides = getParentEdgeAt(1)->getMemory().GetDescWithType<BlockedMemoryDesc>().getStrides();
auto wei_strides = getParentEdgeAt(2)->getMemory().GetDescWithType<BlockedMemoryDesc>().getStrides();
auto off_strides = getParentEdgeAt(1)->getMemory().GetDescWithType<BlockedMemoryDesc>()->getStrides();
auto wei_strides = getParentEdgeAt(2)->getMemory().GetDescWithType<BlockedMemoryDesc>()->getStrides();
InferenceEngine::SizeVector modulation_strides;
if (inputsNumber > 3) {
modulation_strides = getParentEdgeAt(3)->getMemory().GetDescWithType<BlockedMemoryDesc>().getStrides();
modulation_strides = getParentEdgeAt(3)->getMemory().GetDescWithType<BlockedMemoryDesc>()->getStrides();
}
@@ -1171,4 +1176,4 @@ InferenceEngine::Precision MKLDNNDeformableConvolutionNode::getRuntimePrecision(
return getMaxPrecision(getInputPrecisions());
}
REG_MKLDNN_PRIM_FOR(MKLDNNDeformableConvolutionNode, DeformableConvolution);
REG_MKLDNN_PRIM_FOR(MKLDNNDeformableConvolutionNode, DeformableConvolution);
inference-engine/src/mkldnn_plugin/nodes/mkldnn_depth_to_space_node.cpp
+13 -13
View File
@@ -21,8 +21,12 @@ using namespace mkldnn;
using namespace mkldnn::impl;
using namespace mkldnn::impl::cpu::x64;
bool MKLDNNDepthToSpaceNode::isSupportedOperation(const std::shared_ptr<ngraph::Node>& op, std::string& errorMessage) noexcept {
bool MKLDNNDepthToSpaceNode::isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept {
try {
if (isDynamicNgraphNode(op)) {
errorMessage = "Doesn't support op with dynamic shapes";
return false;
}
const auto depthToSpace = std::dynamic_pointer_cast<const ngraph::opset1::DepthToSpace>(op);
if (!depthToSpace) {
errorMessage = "Only opset1 DepthToSpace operation is supported";
@@ -99,7 +103,7 @@ void MKLDNNDepthToSpaceNode::initSupportedPrimitiveDescriptors() {
return;
InferenceEngine::Precision precision = getOriginalInputPrecisionAtPort(0);
auto srcDims = getParentEdgeAt(0)->getShape().getStaticDims();
auto srcDims = getInputShapeAtPort(0).getStaticDims();
const size_t nDims = srcDims.size();
impl_desc_type impl_type;
@@ -140,8 +144,8 @@ void MKLDNNDepthToSpaceNode::initSupportedPrimitiveDescriptors() {
auto range = BlockedDescCreator::makeFilteredRange(creators, nDims, supportedTypes);
for (auto itr = range.first; itr != range.second; ++itr) {
config.inConfs[0].desc = itr->second->createUniqueDesc(precision, getParentEdgeAt(0)->getShape().getStaticDims());
config.outConfs[0].desc = itr->second->createUniqueDesc(precision, getChildEdgeAt(0)->getShape().getStaticDims());
config.inConfs[0].desc = itr->second->createSharedDesc(precision, getInputShapeAtPort(0));
config.outConfs[0].desc = itr->second->createSharedDesc(precision, getOutputShapeAtPort(0));
supportedPrimitiveDescriptors.emplace_back(config, impl_type);
}
}
@@ -156,13 +160,12 @@ void MKLDNNDepthToSpaceNode::createPrimitive() {
if (getSelectedPrimitiveDescriptor() == nullptr)
THROW_ERROR << "has unidentified preferable primitive descriptor";
SizeVector srcDims = getParentEdgeAt(0)->getShape().getStaticDims();
SizeVector dstDims = getChildEdgeAt(0)->getShape().getStaticDims();
VectorDims srcDims = srcMemPtr->getStaticDims();
size_t nDims = srcDims.size();
const size_t nSpatialDims = nDims - 2;
const bool isBlocked = getParentEdgeAt(0)->getMemory().GetDesc().hasLayoutType(LayoutType::nCsp8c) ||
getParentEdgeAt(0)->getMemory().GetDesc().hasLayoutType(LayoutType::nCsp16c);
const bool isBlocked = getParentEdgeAt(0)->getMemory().getDesc().hasLayoutType(LayoutType::nCsp8c) ||
getParentEdgeAt(0)->getMemory().getDesc().hasLayoutType(LayoutType::nCsp16c);
const size_t reshapedRank = nDims + nSpatialDims + static_cast<int>(isBlocked) + static_cast<int>(isBlocked && mode == Mode::DEPTH_FIRST);
const size_t lastIdx = reshapedRank - 1;
size_t firstSpatialOrder = 2;
@@ -194,8 +197,7 @@ void MKLDNNDepthToSpaceNode::createPrimitive() {
};
if (isBlocked) {
SizeVector srcBlockedDims = getParentEdgeAt(0)->getMemory().GetDescWithType<BlockedMemoryDesc>().getBlockDims();
SizeVector dstBlockedDims = getChildEdgeAt(0)->getMemory().GetDescWithType<BlockedMemoryDesc>().getBlockDims();
VectorDims srcBlockedDims = getParentEdgeAt(0)->getMemory().GetDescWithType<BlockedMemoryDesc>()->getBlockDims();
size_t orderShiftForBlocks, orderShiftForDims;
if (mode == Mode::BLOCKS_FIRST) {
@@ -224,11 +226,9 @@ void MKLDNNDepthToSpaceNode::createPrimitive() {
}
reshapeAndSetPermOrder(orderShiftForDims, orderShiftForBlocks, firstSpatialOrder, srcBlockedDims);
} else if (getParentEdgeAt(0)->getMemory().GetDesc().hasLayoutType(LayoutType::nspc)) {
} else if (getParentEdgeAt(0)->getMemory().getDesc().hasLayoutType(LayoutType::nspc)) {
srcDims.push_back(srcDims[1]);
dstDims.push_back(dstDims[1]);
srcDims.erase(srcDims.begin() + 1);
dstDims.erase(dstDims.begin() + 1);
firstSpatialOrder = 1;
size_t shift = static_cast<size_t>(mode == DEPTH_FIRST) + nSpatialDims + 1;
inference-engine/src/mkldnn_plugin/nodes/mkldnn_depth_to_space_node.h
+1 -1
View File
@@ -21,7 +21,7 @@ public:
void execute(mkldnn::stream strm) override;
bool created() const override;
static bool isSupportedOperation(const std::shared_ptr<ngraph::Node>& op, std::string& errorMessage) noexcept;
static bool isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept;
private:
enum Mode {
inference-engine/src/mkldnn_plugin/nodes/mkldnn_detection_output_node.cpp
+11 -6
View File
@@ -24,8 +24,12 @@ bool SortScorePairDescend<std::pair<int, int>>(const std::pair<float, std::pair<
return (pair1.first > pair2.first) || (pair1.first == pair2.first && pair1.second.second < pair2.second.second);
}
bool MKLDNNDetectionOutputNode::isSupportedOperation(const std::shared_ptr<ngraph::Node>& op, std::string& errorMessage) noexcept {
bool MKLDNNDetectionOutputNode::isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept {
try {
if (isDynamicNgraphNode(op)) {
errorMessage = "Doesn't support op with dynamic shapes";
return false;
}
const auto doOp = ngraph::as_type_ptr<const ngraph::op::v0::DetectionOutput>(op);
if (!doOp) {
errorMessage = "Node is not an instance of the DetectionOutput from the operations set v0.";
@@ -116,8 +120,8 @@ void MKLDNNDetectionOutputNode::initSupportedPrimitiveDescriptors() {
return;
std::vector<PortConfigurator> inDataConf;
inDataConf.reserve(getOriginalInputsNumber());
for (int i = 0; i < getOriginalInputsNumber(); ++i)
inDataConf.reserve(inputShapes.size());
for (int i = 0; i < inputShapes.size(); ++i)
inDataConf.emplace_back(LayoutType::ncsp, Precision::FP32);
addSupportedPrimDesc(inDataConf,
@@ -136,7 +140,7 @@ void MKLDNNDetectionOutputNode::execute(mkldnn::stream strm) {
const float *arm_loc_data = inputShapes.size() > 4 ?
reinterpret_cast<const float *>(getParentEdgeAt(idx_arm_location)->getMemoryPtr()->GetPtr()) : nullptr;
const int N = getParentEdgeAt(idx_confidence)->getShape().getStaticDims()[0];
const int N = getParentEdgeAt(idx_confidence)->getMemory().getStaticDims()[0];
float *decoded_bboxes_data = _decoded_bboxes.data();
float *reordered_conf_data = _reordered_conf.data();
@@ -285,8 +289,9 @@ void MKLDNNDetectionOutputNode::execute(mkldnn::stream strm) {
}
}
const int num_results = getChildEdgesAtPort(0)[0]->getShape().getStaticDims()[2];
const int DETECTION_SIZE = getChildEdgesAtPort(0)[0]->getShape().getStaticDims()[3];
const auto outDims = getChildEdgesAtPort(0)[0]->getMemory().getStaticDims();
const int num_results = outDims[2];
const int DETECTION_SIZE = outDims[3];
if (DETECTION_SIZE != 7) {
IE_THROW() << NOT_IMPLEMENTED;
}
inference-engine/src/mkldnn_plugin/nodes/mkldnn_detection_output_node.h
+1 -1
View File
@@ -19,7 +19,7 @@ public:
void execute(mkldnn::stream strm) override;
bool created() const override;
static bool isSupportedOperation(const std::shared_ptr<ngraph::Node>& op, std::string& errorMessage) noexcept;
static bool isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept;
private:
const int idx_location = 0;
inference-engine/src/mkldnn_plugin/nodes/mkldnn_dft_node.cpp
+12 -8
View File
@@ -19,8 +19,12 @@ using namespace mkldnn;
using namespace MKLDNNPlugin;
using namespace InferenceEngine;
bool MKLDNNDFTNode::isSupportedOperation(const std::shared_ptr<ngraph::Node>& op, std::string& errorMessage) noexcept {
bool MKLDNNDFTNode::isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept {
try {
if (isDynamicNgraphNode(op)) {
errorMessage = "Doesn't support op with dynamic shapes";
return false;
}
const auto interpDFT = std::dynamic_pointer_cast<const ngraph::opset7::DFT>(op);
const auto interpIDFT = std::dynamic_pointer_cast<const ngraph::opset7::IDFT>(op);
@@ -86,7 +90,7 @@ void MKLDNNDFTNode::initSupportedPrimitiveDescriptors() {
IE_THROW() << layerErrorPrefix << " has unsupported 'axes' input precision: " << axesPrecision.name();
}
if (getOriginalInputsNumber() > SIGNAL_SIZE_INDEX) {
if (inputShapes.size() > SIGNAL_SIZE_INDEX) {
const auto& signalSizeTensorPrec = getOriginalInputPrecisionAtPort(SIGNAL_SIZE_INDEX);
if (signalSizeTensorPrec != Precision::I32 && signalSizeTensorPrec != Precision::I64) {
IE_THROW() << layerErrorPrefix << " has unsupported 'signal_size' input precision: " << signalSizeTensorPrec.name();
@@ -95,7 +99,7 @@ void MKLDNNDFTNode::initSupportedPrimitiveDescriptors() {
std::vector<PortConfigurator> inDataConfigurators({{LayoutType::ncsp, Precision::FP32},
{LayoutType::ncsp, Precision::I32}});
if (getOriginalInputsNumber() > SIGNAL_SIZE_INDEX)
if (inputShapes.size() > SIGNAL_SIZE_INDEX)
inDataConfigurators.push_back({LayoutType::ncsp, Precision::I32});
addSupportedPrimDesc(inDataConfigurators, {{LayoutType::ncsp, Precision::FP32}}, impl_desc_type::ref_any);
@@ -225,7 +229,7 @@ void copyDataToOutputWithSignalSize(const float* input, const std::vector<size_t
void MKLDNNDFTNode::execute(mkldnn::stream strm) {
auto axesEdge = getParentEdgeAt(AXES_INDEX);
const auto* axesStartPtr = reinterpret_cast<const int32_t*>(axesEdge->getMemoryPtr()->GetPtr());
axes = std::vector<int32_t>(axesStartPtr, axesStartPtr + axesEdge->getShape().getStaticDims()[0]);
axes = std::vector<int32_t>(axesStartPtr, axesStartPtr + axesEdge->getMemory().getStaticDims()[0]);
for (auto& axis : axes) {
if (axis < 0) {
axis += inputShape.size() - 1;
@@ -233,7 +237,7 @@ void MKLDNNDFTNode::execute(mkldnn::stream strm) {
}
std::sort(axes.begin(), axes.end());
outputShape = getChildEdgeAt(0)->getShape().getStaticDims();
outputShape = getChildEdgesAtPort(0)[0]->getMemory().getStaticDims();
for (size_t axis : axes) {
size_t nComplex = outputShape[axis];
// FFT uses different twiddle factors
@@ -247,8 +251,8 @@ void MKLDNNDFTNode::execute(mkldnn::stream strm) {
const auto *input = reinterpret_cast<const float*>(inputDataEdge->getMemoryPtr()->GetPtr());
auto *output = reinterpret_cast<float*>(outputDataEdge->getMemoryPtr()->GetPtr());
auto inputStrides = inputDataEdge->getMemory().GetDescWithType<BlockedMemoryDesc>().getStrides();
auto outputStrides = outputDataEdge->getMemory().GetDescWithType<BlockedMemoryDesc>().getStrides();
auto inputStrides = inputDataEdge->getMemory().GetDescWithType<BlockedMemoryDesc>()->getStrides();
auto outputStrides = outputDataEdge->getMemory().GetDescWithType<BlockedMemoryDesc>()->getStrides();
if (inputShape != outputShape) {
copyDataToOutputWithSignalSize(input, inputShape, inputStrides, output, outputShape, outputStrides);
} else {
@@ -257,7 +261,7 @@ void MKLDNNDFTNode::execute(mkldnn::stream strm) {
}
// 1d case
if (inputDataEdge->getShape().getRank() == 2) {
if (inputDataEdge->getMemory().GetShape().getRank() == 2) {
size_t nComplex = outputShape[0];
if (IsPowerOfTwo(nComplex)) {
fft(output, nComplex * 2, true);
inference-engine/src/mkldnn_plugin/nodes/mkldnn_dft_node.h
+1 -1
View File
@@ -21,7 +21,7 @@ public:
void execute(mkldnn::stream strm) override;
bool created() const override;
static bool isSupportedOperation(const std::shared_ptr<ngraph::Node>& op, std::string& errorMessage) noexcept;
static bool isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept;
private:
void dftNd(float* output, const std::vector<size_t>& outputStrides) const;
inference-engine/src/mkldnn_plugin/nodes/mkldnn_eltwise_node.cpp
+68 -51
View File
@@ -37,6 +37,7 @@
#include <cmath>
#include <map>
#include <functional>
#include "memory_desc/dnnl_blocked_memory_desc.h"
using namespace MKLDNNPlugin;
using namespace InferenceEngine;
@@ -372,7 +373,7 @@ private:
Vmm vmm_d_bias = Vmm(13);
Vmm vmm_zero = Vmm(15);
std::unique_ptr<jit_emu_vcvtneps2bf16> emu_vcvtneps2bf16;
std::shared_ptr<jit_emu_vcvtneps2bf16> emu_vcvtneps2bf16;
std::shared_ptr<jit_emitter> eltwise_emitter = nullptr;
std::vector<std::shared_ptr<jit_emitter>> post_op_emitters = {};
@@ -958,14 +959,30 @@ std::map<const ngraph::DiscreteTypeInfo, std::function<void(const std::shared_pt
}},
};
bool MKLDNNEltwiseNode::isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept {
try {
if (isDynamicNgraphNode(op)) {
errorMessage = "Doesn't support op with dynamic shapes";
return false;
}
if (initializers.find(op->get_type_info()) == initializers.end()) {
errorMessage = "Doesn't support Eltwise algorithm: " + std::string(op->get_type_name());
return false;
}
} catch (...) {
return false;
}
return true;
}
MKLDNNEltwiseNode::MKLDNNEltwiseNode(const std::shared_ptr<ngraph::Node>& op, const mkldnn::engine& eng, MKLDNNWeightsSharing::Ptr &cache) :
MKLDNNNode(op, eng, cache) {
if (initializers.find(op->get_type_info()) != initializers.end()) {
initializers[op->get_type_info()](op, *this);
} else {
IE_THROW(NotImplemented)
<< "CPU Eltwise node doesn't support ngraph operation " << op->get_type_name() << " with name " << op->get_friendly_name();
std::string errorMessage;
if (!isSupportedOperation(op, errorMessage)) {
IE_THROW(NotImplemented) << errorMessage;
}
initializers[op->get_type_info()](op, *this);
}
size_t MKLDNNEltwiseNode::getOpInputsNum() const {
@@ -1103,9 +1120,9 @@ void MKLDNNEltwiseNode::initSupportedPrimitiveDescriptors() {
};
auto initDesc = [&] (LayoutType lt) -> NodeDesc {
auto createMemoryDesc = [lt](MKLDNNEdgePtr edge, Precision prc, size_t offset) -> std::unique_ptr<BlockedMemoryDesc> {
if (lt == ChannelsFirst && edge->getShape().getRank() != 1) {
auto dims = edge->getShape().getStaticDims();
auto createMemoryDesc = [lt](const Shape &shape, Precision prc, size_t offset) -> std::shared_ptr<CpuBlockedMemoryDesc> {
if (lt == ChannelsFirst && shape.getRank() != 1) {
auto dims = shape.getStaticDims();
auto ndims = dims.size();
std::vector<size_t> order(ndims);
std::iota(order.begin(), order.end(), 0);
@@ -1119,11 +1136,11 @@ void MKLDNNEltwiseNode::initSupportedPrimitiveDescriptors() {
blocks[i] = dims[order[i]];
}
return MKLDNNPlugin::make_unique<BlockedMemoryDesc>(prc, edge->getShape().getStaticDims(), blocks, order, offset);
} else if (lt == Blocked && edge->getShape().getRank() != 1 && edge->getShape().getStaticDims()[1] != 1) {
return std::make_shared<CpuBlockedMemoryDesc>(prc, shape, blocks, order, offset);
} else if (lt == Blocked && shape.getRank() != 1 && shape.getStaticDims()[1] != 1) {
size_t blockSize = mayiuse(x64::avx512_common) ? 16 : 8;
std::vector<size_t> blocks = edge->getShape().getStaticDims();
std::vector<size_t> blocks = shape.getStaticDims();
std::vector<size_t> order(blocks.size());
std::iota(order.begin(), order.end(), 0);
@@ -1131,27 +1148,27 @@ void MKLDNNEltwiseNode::initSupportedPrimitiveDescriptors() {
blocks.push_back(blockSize);
order.push_back(1);
return MKLDNNPlugin::make_unique<BlockedMemoryDesc>(prc, edge->getShape().getStaticDims(), blocks, order, offset);
return std::make_shared<CpuBlockedMemoryDesc>(prc, shape, blocks, order, offset);
} else {
std::vector<size_t> blocks = edge->getShape().getStaticDims();
std::vector<size_t> blocks = shape.getStaticDims();
std::vector<size_t> order(blocks.size());
std::iota(order.begin(), order.end(), 0);
return MKLDNNPlugin::make_unique<BlockedMemoryDesc>(prc, edge->getShape().getStaticDims(), blocks, order, offset);
return std::make_shared<CpuBlockedMemoryDesc>(prc, shape, blocks, order, offset);
}
};
size_t offset = std::numeric_limits<size_t>::max();
NodeConfig config;
config.dynBatchSupport = getChildEdgeAt(0)->getShape().getRank() > 1 && getChildEdgeAt(0)->getShape() ==
getParentEdgeAt(0)->getShape();
config.dynBatchSupport = getOutputShapeAtPort(0).getRank() > 1 && getOutputShapeAtPort(0) ==
getInputShapeAtPort(0);
for (size_t i = 0; i < getParentEdges().size(); i++) {
PortConfig portConfig;
portConfig.inPlace = (!i && canBeInPlace() && inputPrecisions[i] == outputPrecision) ? 0 : -1;
portConfig.constant = false;
portConfig.desc = createMemoryDesc(getParentEdgeAt(i), inputPrecisions[i], offset);
portConfig.desc = createMemoryDesc(getInputShapeAtPort(i), inputPrecisions[i], offset);
config.inConfs.push_back(portConfig);
}
@@ -1160,7 +1177,7 @@ void MKLDNNEltwiseNode::initSupportedPrimitiveDescriptors() {
portConfig.inPlace = -1;
portConfig.constant = false;
portConfig.desc = createMemoryDesc(getChildEdgeAt(0), outputPrecision, offset);
portConfig.desc = createMemoryDesc(getOutputShapeAtPort(0), outputPrecision, offset);
config.outConfs.push_back(portConfig);
@@ -1178,20 +1195,20 @@ void MKLDNNEltwiseNode::initSupportedPrimitiveDescriptors() {
return {config, impl_type};
};
bool isChannelsFirstApplicable = one_of(getChildEdgeAt(0)->getShape().getRank(), 1, 2, 4, 5);
bool isChannelsFirstApplicable = one_of(getOutputShapeAtPort(0).getRank(), 1, 2, 4, 5);
for (size_t i = 0; i < getParentEdges().size(); i++) {
isChannelsFirstApplicable = isChannelsFirstApplicable && one_of(getParentEdgeAt(i)->getShape().getRank(), 1, 2, 4, 5);
isChannelsFirstApplicable = isChannelsFirstApplicable && implication(getParentEdgeAt(i)->getShape().getRank() != 1,
getChildEdgeAt(0)->getShape().getRank() ==
getParentEdgeAt(i)->getShape().getRank());
isChannelsFirstApplicable = isChannelsFirstApplicable && one_of(getInputShapeAtPort(i).getRank(), 1, 2, 4, 5);
isChannelsFirstApplicable = isChannelsFirstApplicable && implication(getInputShapeAtPort(i).getRank() != 1,
getOutputShapeAtPort(0).getRank() ==
getInputShapeAtPort(i).getRank());
}
bool isBlockedApplicable = one_of(getChildEdgeAt(0)->getShape().getRank(), 1, 4, 5);
bool isBlockedApplicable = one_of(getOutputShapeAtPort(0).getRank(), 1, 4, 5);
for (size_t i = 0; i < getParentEdges().size(); i++) {
isBlockedApplicable = isBlockedApplicable && one_of(getParentEdgeAt(i)->getShape().getRank(), 1, 4, 5);
isBlockedApplicable = isBlockedApplicable && implication(getParentEdgeAt(i)->getShape().getRank() != 1,
getChildEdgeAt(0)->getShape().getRank() ==
getParentEdgeAt(i)->getShape().getRank());
isBlockedApplicable = isBlockedApplicable && one_of(getInputShapeAtPort(i).getRank(), 1, 4, 5);
isBlockedApplicable = isBlockedApplicable && implication(getInputShapeAtPort(i).getRank() != 1,
getOutputShapeAtPort(0).getRank() ==
getInputShapeAtPort(i).getRank());
}
if (isChannelsFirstApplicable)
@@ -1215,7 +1232,7 @@ void MKLDNNEltwiseNode::createPrimitive() {
auto outBlockingDesc = getChildEdgeAt(0)->getMemory().GetDescWithType<BlockedMemoryDesc>();
std::vector<size_t> order(maxInputSize);
auto outOrder = outBlockingDesc.getOrder();
auto outOrder = outBlockingDesc->getOrder();
for (size_t i = 0; i < order.size(); i++) {
if (i < order.size() - outOrder.size())
order[i] = i;
@@ -1223,18 +1240,18 @@ void MKLDNNEltwiseNode::createPrimitive() {
order[i] = outOrder[i - (order.size() - outOrder.size())] + (order.size() - outOrder.size());
}
size_t outRank = outBlockingDesc.getBlockDims().size();
size_t outRank = outBlockingDesc->getBlockDims().size();
for (int i = 0; i < outRank; i++) {
dims_out[dims_out.size() - 1 - i] = outBlockingDesc.getBlockDims()[outRank - 1 - i];
dims_out[dims_out.size() - 1 - i] = outBlockingDesc->getBlockDims()[outRank - 1 - i];
}
for (int i = 0; i < inputNum; i++) {
auto inBlockingDesc = getParentEdgeAt(i)->getMemory().GetDescWithType<BlockedMemoryDesc>();
size_t inRank = inBlockingDesc.getBlockDims().size();
size_t inRank = inBlockingDesc->getBlockDims().size();
// WA to normalize blocked and planar layouts
auto inOrder = inBlockingDesc.getOrder();
size_t startOff = outOrder.size() != outBlockingDesc.getShape().getRank() &&
auto inOrder = inBlockingDesc->getOrder();
size_t startOff = outOrder.size() != outBlockingDesc->getShape().getRank() &&
outOrder[outOrder.size() - 1] != inOrder[inOrder.size() - 1] ? 1 : 0;
// WA to handle nspc layout with 1D tensors
@@ -1243,7 +1260,7 @@ void MKLDNNEltwiseNode::createPrimitive() {
}
for (int j = 0; j < inRank; j++) {
dims_in[i][dims_in[i].size() - 1 - j - startOff] = inBlockingDesc.getBlockDims()[inRank - 1 - j];
dims_in[i][dims_in[i].size() - 1 - j - startOff] = inBlockingDesc->getBlockDims()[inRank - 1 - j];
}
}
@@ -1259,7 +1276,7 @@ void MKLDNNEltwiseNode::createPrimitive() {
offsets_out.resize(maxInputSize, 1);
offset_out_calc(offsets_out, dims_out);
for (int j = 0; j < maxInputSize; j++) {
offsets_out[j] *= getChildEdgeAt(0)->getMemory().GetDesc().getPrecision().size();
offsets_out[j] *= getChildEdgeAt(0)->getMemory().getDesc().getPrecision().size();
}
offsets_in.resize(inputNum);
@@ -1267,17 +1284,17 @@ void MKLDNNEltwiseNode::createPrimitive() {
offsets_in[i].resize(maxInputSize, 1);
offset_in_calc(offsets_in[i], dims_in[i], dims_out);
for (int j = 0; j < maxInputSize; j++) {
offsets_in[i][j] *= getParentEdgeAt(i)->getMemory().GetDesc().getPrecision().size();
offsets_in[i][j] *= getParentEdgeAt(i)->getMemory().getDesc().getPrecision().size();
}
}
start_offset_in.resize(inputNum);
for (size_t i = 0; i < inputNum; i++) {
start_offset_in[i] = getParentEdgeAt(i)->getMemory().GetDescriptor().data.offset0 *
MKLDNNExtensionUtils::sizeOfDataType(mkldnn::memory::data_type(getParentEdgeAt(i)->getMemory().GetDescriptor().data.data_type));
const auto desc = getParentEdgeAt(i)->getMemory().GetDescWithType<BlockedMemoryDesc>();
start_offset_in[i] = desc->getOffsetPadding() * desc->getPrecision().size();
}
start_offset_out = getChildEdgeAt(0)->getMemory().GetDescriptor().data.offset0 *
MKLDNNExtensionUtils::sizeOfDataType(mkldnn::memory::data_type(getChildEdgeAt(0)->getMemory().GetDescriptor().data.data_type));
const auto desc = getChildEdgeAt(0)->getMemory().GetDescWithType<BlockedMemoryDesc>();
start_offset_out = desc->getOffsetPadding() * desc->getPrecision().size();
};
auto collapseLastDims = [](std::vector<size_t>& dims, int dimsToCollapse) {
@@ -1312,10 +1329,10 @@ void MKLDNNEltwiseNode::createPrimitive() {
};
auto outBlockingDesc = getChildEdgeAt(0)->getMemory().GetDescWithType<BlockedMemoryDesc>();
tensorRank = std::max(static_cast<size_t>(optimalTensorRank), outBlockingDesc.getBlockDims().size());
tensorRank = std::max(static_cast<size_t>(optimalTensorRank), outBlockingDesc->getBlockDims().size());
initDims(tensorRank);
auto outOrder = outBlockingDesc.getOrder();
auto outOrder = outBlockingDesc->getOrder();
size_t oc_size = 0;
offsets_oc.resize(tensorRank, 0);
if (isFusedWith(FakeQuantize)) {
@@ -1345,7 +1362,7 @@ void MKLDNNEltwiseNode::createPrimitive() {
bool hasDifferentDims = false;
while (currentJitWorkAmount < minimalJitWorkAmount && currentJitWorkAmount < fullWorkAmount &&
// we shouldn't collapse batch dimension in case dynamic batch is enabled
(!isDynBatchEnabled || (outBlockingDesc.getBlockDims().size() - collapsedDims > 2))) {
(!isDynBatchEnabled || (outBlockingDesc->getBlockDims().size() - collapsedDims > 2))) {
if (dims_out.size() - collapsedDims - 2 < 0)
break;
@@ -1397,7 +1414,7 @@ void MKLDNNEltwiseNode::createPrimitive() {
}
}
batchDimIdx = tensorRank - outBlockingDesc.getBlockDims().size() + collapsedDims;
batchDimIdx = tensorRank - outBlockingDesc->getBlockDims().size() + collapsedDims;
schedulerWorkAmount = fullWorkAmount / dims_out[dims_out.size() - 1];
initOffsets(tensorRank);
@@ -1414,10 +1431,10 @@ void MKLDNNEltwiseNode::createPrimitive() {
for (int i = 0; i < inputNum; i++) {
jep.src_size[i] = dims_in[i][dims_in[i].size() - 1];
jep.src_prc[i] = getParentEdgesAtPort(i).front()->getMemory().GetDesc().getPrecision();
jep.src_prc[i] = getParentEdgesAtPort(i).front()->getMemory().getDesc().getPrecision();
}
jep.dst_size = dims_out[dims_out.size() - 1];
jep.dst_prc = getChildEdgesAtPort(0).front()->getMemory().GetDesc().getPrecision();
jep.dst_prc = getChildEdgesAtPort(0).front()->getMemory().getDesc().getPrecision();
jep.oc_size = oc_size;
jep.work_amount = dims_out.back();
@@ -1638,13 +1655,13 @@ bool MKLDNNEltwiseNode::canBeInPlace() const {
}
}
return getParentEdgesAtPort(0)[0].get()->getShape() == getChildEdgesAtPort(0)[0].get()->getShape();
return getInputShapeAtPort(0) == getOutputShapeAtPort(0);
}
void MKLDNNEltwiseNode::fuseInto(MKLDNNNodePtr& parentNode) {
// Handling Convolution custom Add node fusing case which is processed via dnnl append_sum() API.
specialConvolutionAddFusing = (parentNode->getType() == Convolution || parentNode->getType() == BinaryConvolution) && getAlgorithm() == EltwiseAdd &&
getParentEdgesAtPort(0)[0]->getShape() == getParentEdgesAtPort(1)[0]->getShape();
getInputShapeAtPort(0) == getInputShapeAtPort(1);
if (!specialConvolutionAddFusing && canBePerformedAsScaleShift(parentNode.get())) {
fillScalesAndShifts(parentNode.get(), scales, shifts, 16);
}
@@ -1748,7 +1765,7 @@ bool MKLDNNEltwiseNode::canFuse(const MKLDNNNodePtr& node) const {
}
// We can use optimized execution with fusions only in cases when dim rank is less or equal to the maximum possible
if (node->getParentEdgesAtPort(0).front()->getShape().getRank() > MAX_ELTWISE_DIM_RANK)
if (node->getInputShapeAtPort(0).getRank() > MAX_ELTWISE_DIM_RANK)
return false;
return true;
inference-engine/src/mkldnn_plugin/nodes/mkldnn_eltwise_node.h
+2
View File
@@ -88,6 +88,8 @@ public:
bool isWithBroadcast();
bool isSpecialConvolutionAddFusing() const { return specialConvolutionAddFusing; }
static bool isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept;
private:
mkldnn::algorithm mkldnnAlgorithm = mkldnn::algorithm::undef;
inference-engine/src/mkldnn_plugin/nodes/mkldnn_embedding_bag_offset_sum_node.cpp
+10 -5
View File
@@ -11,8 +11,12 @@
using namespace MKLDNNPlugin;
using namespace InferenceEngine;
bool MKLDNNEmbeddingBagOffsetSumNode::isSupportedOperation(const std::shared_ptr<ngraph::Node>& op, std::string& errorMessage) noexcept {
bool MKLDNNEmbeddingBagOffsetSumNode::isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept {
try {
if (isDynamicNgraphNode(op)) {
errorMessage = "Doesn't support op with dynamic shapes";
return false;
}
const auto embBagOffsetSumOp = ngraph::as_type_ptr<const ngraph::op::v3::EmbeddingBagOffsetsSum>(op);
if (!embBagOffsetSumOp) {
errorMessage = "Node is not an instance of the EmbeddingBagOffsetsSum operation from opset v3.";
@@ -65,9 +69,9 @@ void MKLDNNEmbeddingBagOffsetSumNode::initSupportedPrimitiveDescriptors() {
std::vector<PortConfigurator> inDataConfigurators({{LayoutType::ncsp, inDataPrecision},
{LayoutType::ncsp, Precision::I32},
{LayoutType::ncsp, Precision::I32}});
if (getOriginalInputsNumber() > DEFAULT_INDEX_IDX)
if (inputShapes.size() > DEFAULT_INDEX_IDX)
inDataConfigurators.push_back({LayoutType::ncsp, Precision::I32});
if (getOriginalInputsNumber() > PER_SAMPLE_WEIGHTS_IDX)
if (inputShapes.size() > PER_SAMPLE_WEIGHTS_IDX)
inDataConfigurators.push_back({LayoutType::ncsp, inDataPrecision});
addSupportedPrimDesc(inDataConfigurators, {{LayoutType::ncsp, inDataPrecision}}, impl_desc_type::ref_any);
@@ -122,8 +126,9 @@ void MKLDNNEmbeddingBagOffsetSumNode::execute(mkldnn::stream strm) {
if (_withWeights)
weightsData = reinterpret_cast<const uint8_t *>(getParentEdgeAt(PER_SAMPLE_WEIGHTS_IDX)->getMemoryPtr()->GetPtr());
MKLDNNEmbeddingBagSumNode::execute(srcData, weightsData, dstData, getParentEdgeAt(0)->getMemory().GetDesc().getPrecision(),
getParentEdgeAt(0)->getShape().getStaticDims(), getChildEdgeAt(0)->getShape().getStaticDims());
const auto &inputMem = getParentEdgeAt(0)->getMemory();
MKLDNNEmbeddingBagSumNode::execute(srcData, weightsData, dstData, inputMem .getDesc().getPrecision(),
inputMem .getStaticDims(), getChildEdgesAtPort(0)[0]->getMemory().GetShape().getStaticDims());
}
bool MKLDNNEmbeddingBagOffsetSumNode::created() const {
inference-engine/src/mkldnn_plugin/nodes/mkldnn_embedding_bag_offset_sum_node.h
+1 -1
View File
@@ -23,7 +23,7 @@ public:
void execute(mkldnn::stream strm) override;
bool created() const override;
static bool isSupportedOperation(const std::shared_ptr<ngraph::Node>& op, std::string& errorMessage) noexcept;
static bool isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept;
private:
void initFromInputs() override;
inference-engine/src/mkldnn_plugin/nodes/mkldnn_embedding_bag_packed_sum_node.cpp
+9 -4
View File
@@ -11,8 +11,12 @@
using namespace MKLDNNPlugin;
using namespace InferenceEngine;
bool MKLDNNEmbeddingBagPackedSumNode::isSupportedOperation(const std::shared_ptr<ngraph::Node>& op, std::string& errorMessage) noexcept {
bool MKLDNNEmbeddingBagPackedSumNode::isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept {
try {
if (isDynamicNgraphNode(op)) {
errorMessage = "Doesn't support op with dynamic shapes";
return false;
}
const auto embBagPackedSumOp = ngraph::as_type_ptr<const ngraph::op::v3::EmbeddingBagPackedSum>(op);
if (!embBagPackedSumOp) {
errorMessage = "Node is not an instance of the EmbeddingBagPackedSum operation from opset v3.";
@@ -60,7 +64,7 @@ void MKLDNNEmbeddingBagPackedSumNode::initSupportedPrimitiveDescriptors() {
std::vector<PortConfigurator> inDataConfigurators({{LayoutType::ncsp, inDataPrecision},
{LayoutType::ncsp, Precision::I32}});
if (getOriginalInputsNumber() > PER_SAMPLE_WEIGHTS_IDX)
if (inputShapes.size() > PER_SAMPLE_WEIGHTS_IDX)
inDataConfigurators.push_back({LayoutType::ncsp, inDataPrecision});
addSupportedPrimDesc(inDataConfigurators, {{LayoutType::ncsp, inDataPrecision}}, impl_desc_type::ref_any);
@@ -89,8 +93,9 @@ void MKLDNNEmbeddingBagPackedSumNode::execute(mkldnn::stream strm) {
if (_withWeights)
weightsData = reinterpret_cast<const uint8_t *>(getParentEdgeAt(PER_SAMPLE_WEIGHTS_IDX)->getMemoryPtr()->GetPtr());
MKLDNNEmbeddingBagSumNode::execute(srcData, weightsData, dstData, getParentEdgeAt(0)->getMemory().GetDesc().getPrecision(),
getParentEdgeAt(0)->getShape().getStaticDims(), getChildEdgeAt(0)->getShape().getStaticDims());
const auto &inputMem = getParentEdgeAt(0)->getMemory();
MKLDNNEmbeddingBagSumNode::execute(srcData, weightsData, dstData, inputMem .getDesc().getPrecision(),
inputMem .getStaticDims(), getChildEdgesAtPort(0)[0]->getMemory().GetShape().getStaticDims());
}
bool MKLDNNEmbeddingBagPackedSumNode::created() const {
inference-engine/src/mkldnn_plugin/nodes/mkldnn_embedding_bag_packed_sum_node.h
+1 -1
View File
@@ -23,7 +23,7 @@ public:
void execute(mkldnn::stream strm) override;
bool created() const override;
static bool isSupportedOperation(const std::shared_ptr<ngraph::Node>& op, std::string& errorMessage) noexcept;
static bool isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept;
private:
void initFromInputs() override;
inference-engine/src/mkldnn_plugin/nodes/mkldnn_embedding_segments_sum_node.cpp
+11 -6
View File
@@ -11,8 +11,12 @@
using namespace MKLDNNPlugin;
using namespace InferenceEngine;
bool MKLDNNEmbeddingSegmentsSumNode::isSupportedOperation(const std::shared_ptr<ngraph::Node>& op, std::string& errorMessage) noexcept {
bool MKLDNNEmbeddingSegmentsSumNode::isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept {
try {
if (isDynamicNgraphNode(op)) {
errorMessage = "Doesn't support op with dynamic shapes";
return false;
}
const auto embBagSegSumOp = ngraph::as_type_ptr<const ngraph::op::v3::EmbeddingSegmentsSum>(op);
if (!embBagSegSumOp) {
errorMessage = "Node is not an instance of the EmbeddingSegmentsSum operation from opset v3.";
@@ -66,9 +70,9 @@ void MKLDNNEmbeddingSegmentsSumNode::initSupportedPrimitiveDescriptors() {
{LayoutType::ncsp, Precision::I32},
{LayoutType::ncsp, Precision::I32},
{LayoutType::ncsp, Precision::I32}});
if (getOriginalInputsNumber() > DEFAULT_INDEX_IDX)
if (inputShapes.size() > DEFAULT_INDEX_IDX)
inDataConfigurators.push_back({LayoutType::ncsp, Precision::I32});
if (getOriginalInputsNumber() > PER_SAMPLE_WEIGHTS_IDX)
if (inputShapes.size() > PER_SAMPLE_WEIGHTS_IDX)
inDataConfigurators.push_back({LayoutType::ncsp, inDataPrecision});
addSupportedPrimDesc(inDataConfigurators, {{LayoutType::ncsp, inDataPrecision}}, impl_desc_type::ref_any);
@@ -76,7 +80,7 @@ void MKLDNNEmbeddingSegmentsSumNode::initSupportedPrimitiveDescriptors() {
void MKLDNNEmbeddingSegmentsSumNode::initFromInputs() {
indices_ = reinterpret_cast<const int *>(getParentEdgeAt(INDICES_IDX)->getMemoryPtr()->GetPtr());
indicesSize_ = getParentEdgeAt(INDICES_IDX)->getShape().getElementsCount();
indicesSize_ = getParentEdgeAt(INDICES_IDX)->getMemory().GetShape().getElementsCount();
segmentIds_ = reinterpret_cast<const int *>(getParentEdgeAt(SEGMENT_ID_IDX)->getMemoryPtr()->GetPtr());
@@ -124,8 +128,9 @@ void MKLDNNEmbeddingSegmentsSumNode::execute(mkldnn::stream strm) {
if (_withWeights)
weightsData = reinterpret_cast<const uint8_t *>(getParentEdgeAt(PER_SAMPLE_WEIGHTS_IDX)->getMemoryPtr()->GetPtr());
MKLDNNEmbeddingBagSumNode::execute(srcData, weightsData, dstData, getParentEdgeAt(0)->getMemory().GetDesc().getPrecision(),
getParentEdgeAt(0)->getShape().getStaticDims(), getChildEdgeAt(0)->getShape().getStaticDims());
const auto &inputMem = getParentEdgeAt(0)->getMemory();
MKLDNNEmbeddingBagSumNode::execute(srcData, weightsData, dstData, inputMem .getDesc().getPrecision(),
inputMem .getStaticDims(), getChildEdgesAtPort(0)[0]->getMemory().GetShape().getStaticDims());
}
bool MKLDNNEmbeddingSegmentsSumNode::created() const {
inference-engine/src/mkldnn_plugin/nodes/mkldnn_embedding_segments_sum_node.h
+1 -1
View File
@@ -23,7 +23,7 @@ public:
void execute(mkldnn::stream strm) override;
bool created() const override;
static bool isSupportedOperation(const std::shared_ptr<ngraph::Node>& op, std::string& errorMessage) noexcept;
static bool isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept;
private:
void initFromInputs() override;
inference-engine/src/mkldnn_plugin/nodes/mkldnn_experimental_detectron_detection_output_node.cpp
+10 -6
View File
@@ -215,8 +215,12 @@ static void nms_cf(const float* conf_data,
detections = (post_nms_topn == -1 ? detections : (std::min)(post_nms_topn, detections));
}
bool MKLDNNExperimentalDetectronDetectionOutputNode::isSupportedOperation(const std::shared_ptr<ngraph::Node>& op, std::string& errorMessage) noexcept {
bool MKLDNNExperimentalDetectronDetectionOutputNode::isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept {
try {
if (isDynamicNgraphNode(op)) {
errorMessage = "Doesn't support op with dynamic shapes";
return false;
}
const auto doOp = ngraph::as_type_ptr<const ngraph::op::v6::ExperimentalDetectronDetectionOutput>(op);
if (!doOp) {
errorMessage = "Node is not an instance of the ExperimentalDetectronDetectionOutput from the operations set v6.";
@@ -253,8 +257,8 @@ void MKLDNNExperimentalDetectronDetectionOutputNode::initSupportedPrimitiveDescr
return;
std::vector<PortConfigurator> inDataConf;
inDataConf.reserve(getOriginalInputsNumber());
for (int i = 0; i < getOriginalInputsNumber(); ++i)
inDataConf.reserve(inputShapes.size());
for (int i = 0; i < inputShapes.size(); ++i)
inDataConf.emplace_back(LayoutType::ncsp, Precision::FP32);
addSupportedPrimDesc(inDataConf,
@@ -265,9 +269,9 @@ void MKLDNNExperimentalDetectronDetectionOutputNode::initSupportedPrimitiveDescr
}
void MKLDNNExperimentalDetectronDetectionOutputNode::execute(mkldnn::stream strm) {
const int rois_num = getParentEdgeAt(INPUT_ROIS)->getShape().getStaticDims()[0];
assert(classes_num_ == static_cast<int>(getParentEdgeAt(INPUT_SCORES)->getShape().getStaticDims()[1]));
assert(4 * classes_num_ == static_cast<int>(getParentEdgeAt(INPUT_DELTAS)->getShape().getStaticDims()[1]));
const int rois_num = getParentEdgeAt(INPUT_ROIS)->getMemory().getStaticDims()[0];
assert(classes_num_ == static_cast<int>(getParentEdgeAt(INPUT_SCORES)->getMemory().getStaticDims()[1]));
assert(4 * classes_num_ == static_cast<int>(getParentEdgeAt(INPUT_DELTAS)->getMemory().getStaticDims()[1]));
const auto* boxes = reinterpret_cast<const float *>(getParentEdgeAt(INPUT_ROIS)->getMemoryPtr()->GetPtr());
const auto* deltas = reinterpret_cast<const float *>(getParentEdgeAt(INPUT_DELTAS)->getMemoryPtr()->GetPtr());
inference-engine/src/mkldnn_plugin/nodes/mkldnn_experimental_detectron_detection_output_node.h
+1 -1
View File
@@ -19,7 +19,7 @@ public:
void execute(mkldnn::stream strm) override;
bool created() const override;
static bool isSupportedOperation(const std::shared_ptr<ngraph::Node>& op, std::string& errorMessage) noexcept;
static bool isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept;
private:
const int INPUT_ROIS {0};
inference-engine/src/mkldnn_plugin/nodes/mkldnn_experimental_detectron_generate_proposals_single_image_node.cpp
+17 -10
View File
@@ -273,8 +273,12 @@ void fill_output_blobs(const float* proposals, const int* roi_indices,
}
bool MKLDNNExperimentalDetectronGenerateProposalsSingleImageNode::isSupportedOperation
(const std::shared_ptr<ngraph::Node>& op, std::string& errorMessage) noexcept {
(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept {
try {
if (isDynamicNgraphNode(op)) {
errorMessage = "Doesn't support op with dynamic shapes";
return false;
}
const auto proposalOp = ngraph::as_type_ptr<const ngraph::op::v6::ExperimentalDetectronGenerateProposalsSingleImage>(op);
if (!proposalOp) {
errorMessage = "Node is not an instance of the Proposal from the operations set v0.";
@@ -327,20 +331,23 @@ void MKLDNNExperimentalDetectronGenerateProposalsSingleImageNode::execute(mkldnn
}
size_t anchor_dims_size = 1;
for (size_t i = 0; i < getParentEdgeAt(INPUT_ANCHORS)->getShape().getRank(); i++) {
anchor_dims_size *= getParentEdgeAt(INPUT_ANCHORS)->getShape().getStaticDims()[i];
const auto &anchorDims = getParentEdgeAt(INPUT_ANCHORS)->getMemory().getStaticDims();
for (size_t i = 0; i < anchorDims.size(); i++) {
anchor_dims_size *= anchorDims[i];
}
size_t deltas_dims_size = 1;
for (size_t i = 0; i < getParentEdgeAt(INPUT_DELTAS)->getShape().getRank(); i++) {
deltas_dims_size *= getParentEdgeAt(INPUT_DELTAS)->getShape().getStaticDims()[i];
const auto &deltaDims = getParentEdgeAt(INPUT_DELTAS)->getMemory().getStaticDims();
for (size_t i = 0; i < deltaDims.size(); i++) {
deltas_dims_size *= deltaDims[i];
}
if (anchor_dims_size != deltas_dims_size)
IE_THROW() << "'Anchors' blob size for ONNXProposal is incompatible with 'deltas' blob size!";
size_t score_dims_size = 1;
for (size_t i = 0; i < getParentEdgeAt(INPUT_SCORES)->getShape().getRank(); i++) {
score_dims_size *= getParentEdgeAt(INPUT_SCORES)->getShape().getStaticDims()[i];
const auto &scoreDims = getParentEdgeAt(INPUT_SCORES)->getMemory().getStaticDims();
for (size_t i = 0; i < scoreDims.size(); i++) {
score_dims_size *= scoreDims[i];
}
if (deltas_dims_size != (4 * score_dims_size))
IE_THROW() << "'Deltas' blob size for ONNXProposal is incompatible with 'scores' blob size!";
@@ -354,11 +361,11 @@ void MKLDNNExperimentalDetectronGenerateProposalsSingleImageNode::execute(mkldnn
float *p_roi_item = reinterpret_cast<float *>(getChildEdgesAtPort(OUTPUT_ROIS)[0]->getMemoryPtr()->GetPtr());
float *p_roi_score_item = reinterpret_cast<float *>(getChildEdgesAtPort(OUTPUT_SCORES)[0]->getMemoryPtr()->GetPtr());
const int anchors_num = getParentEdgeAt(INPUT_SCORES)->getShape().getStaticDims()[0];
const int anchors_num = scoreDims[0];
// bottom shape: (num_anchors) x H x W
const int bottom_H = getParentEdgeAt(INPUT_DELTAS)->getShape().getStaticDims()[1];
const int bottom_W = getParentEdgeAt(INPUT_DELTAS)->getShape().getStaticDims()[2];
const int bottom_H = deltaDims[1];
const int bottom_W = deltaDims[2];
// input image height & width
const float img_H = p_img_info_cpu[0];
inference-engine/src/mkldnn_plugin/nodes/mkldnn_experimental_detectron_generate_proposals_single_image_node.h
+1 -1
View File
@@ -20,7 +20,7 @@ public:
void execute(mkldnn::stream strm) override;
bool created() const override;
static bool isSupportedOperation(const std::shared_ptr<ngraph::Node>& op, std::string& errorMessage) noexcept;
static bool isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept;
private:
// Inputs:
inference-engine/src/mkldnn_plugin/nodes/mkldnn_experimental_detectron_priorgridgenerator_node.cpp
+12 -7
View File
@@ -11,8 +11,13 @@
using namespace MKLDNNPlugin;
using namespace InferenceEngine;
bool MKLDNNExperimentalDetectronPriorGridGeneratorNode::isSupportedOperation(const std::shared_ptr<ngraph::Node>& op, std::string& errorMessage) noexcept {
bool MKLDNNExperimentalDetectronPriorGridGeneratorNode::isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op,
std::string& errorMessage) noexcept {
try {
if (isDynamicNgraphNode(op)) {
errorMessage = "Doesn't support op with dynamic shapes";
return false;
}
const auto priorGridGen = std::dynamic_pointer_cast<const ngraph::opset6::ExperimentalDetectronPriorGridGenerator>(op);
if (!priorGridGen) {
errorMessage = "Only opset6 ExperimentalDetectronPriorGridGenerator operation is supported";
@@ -61,14 +66,14 @@ void MKLDNNExperimentalDetectronPriorGridGeneratorNode::initSupportedPrimitiveDe
}
void MKLDNNExperimentalDetectronPriorGridGeneratorNode::execute(mkldnn::stream strm) {
const int num_priors_ = getParentEdgeAt(INPUT_PRIORS)->getShape().getStaticDims()[0];
assert(getParentEdgeAt(INPUT_PRIORS)->getShape().getStaticDims()[1] == 4);
const int num_priors_ = getParentEdgeAt(INPUT_PRIORS)->getMemory().getStaticDims()[0];
assert(getParentEdgeAt(INPUT_PRIORS)->getMemory().getStaticDims()[1] == 4);
// Execute
const int layer_width = grid_w_ ? grid_w_ : getParentEdgeAt(INPUT_FEATUREMAP)->getShape().getStaticDims()[3];
const int layer_height = grid_h_ ? grid_h_ : getParentEdgeAt(INPUT_FEATUREMAP)->getShape().getStaticDims()[2];
const float step_w = stride_w_ ? stride_w_ : static_cast<float>(getParentEdgeAt(INPUT_IMAGE)->getShape().getStaticDims()[3]) / layer_width;
const float step_h = stride_h_ ? stride_h_ : static_cast<float>(getParentEdgeAt(INPUT_IMAGE)->getShape().getStaticDims()[2]) / layer_height;
const int layer_width = grid_w_ ? grid_w_ : getParentEdgeAt(INPUT_FEATUREMAP)->getMemory().getStaticDims()[3];
const int layer_height = grid_h_ ? grid_h_ : getParentEdgeAt(INPUT_FEATUREMAP)->getMemory().getStaticDims()[2];
const float step_w = stride_w_ ? stride_w_ : static_cast<float>(getParentEdgeAt(INPUT_IMAGE)->getMemory().getStaticDims()[3]) / layer_width;
const float step_h = stride_h_ ? stride_h_ : static_cast<float>(getParentEdgeAt(INPUT_IMAGE)->getMemory().getStaticDims()[2]) / layer_height;
const auto *bottom_data_0 = reinterpret_cast<const float *>(getParentEdgeAt(0)->getMemoryPtr()->GetPtr());
auto *top_data_0 = reinterpret_cast<float *>(getChildEdgesAtPort(OUTPUT_ROIS)[0]->getMemoryPtr()->GetPtr());
inference-engine/src/mkldnn_plugin/nodes/mkldnn_experimental_detectron_priorgridgenerator_node.h
+1 -1
View File
@@ -19,7 +19,7 @@ public:
void execute(mkldnn::stream strm) override;
bool created() const override;
static bool isSupportedOperation(const std::shared_ptr<ngraph::Node>& op, std::string& errorMessage) noexcept;
static bool isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept;
private:
// Inputs:
inference-engine/src/mkldnn_plugin/nodes/mkldnn_experimental_detectron_roifeatureextractor_node.cpp
+12 -7
View File
@@ -304,8 +304,13 @@ void reorder_rois(const float *rois, const int* ids, int* mapping, const int roi
}
}
bool MKLDNNExperimentalDetectronROIFeatureExtractorNode::isSupportedOperation(const std::shared_ptr<ngraph::Node>& op, std::string& errorMessage) noexcept {
bool MKLDNNExperimentalDetectronROIFeatureExtractorNode::isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op,
std::string& errorMessage) noexcept {
try {
if (isDynamicNgraphNode(op)) {
errorMessage = "Doesn't support op with dynamic shapes";
return false;
}
const auto roiFeatureExtractor = std::dynamic_pointer_cast<const ngraph::opset6::ExperimentalDetectronROIFeatureExtractor>(op);
if (!roiFeatureExtractor) {
errorMessage = "Only opset6 ExperimentalDetectronROIFeatureExtractor operation is supported";
@@ -340,8 +345,8 @@ void MKLDNNExperimentalDetectronROIFeatureExtractorNode::initSupportedPrimitiveD
return;
std::vector<PortConfigurator> inDataConf;
inDataConf.reserve(getOriginalInputsNumber());
for (int i = 0; i < getOriginalInputsNumber(); ++i)
inDataConf.reserve(inputShapes.size());
for (int i = 0; i < inputShapes.size(); ++i)
inDataConf.emplace_back(LayoutType::ncsp, Precision::FP32);
addSupportedPrimDesc(inDataConf,
@@ -352,8 +357,8 @@ void MKLDNNExperimentalDetectronROIFeatureExtractorNode::initSupportedPrimitiveD
void MKLDNNExperimentalDetectronROIFeatureExtractorNode::execute(mkldnn::stream strm) {
const int levels_num = inputShapes.size() - INPUT_FEATURES_START;
const int num_rois = getParentEdgeAt(INPUT_ROIS)->getShape().getStaticDims()[0];
const int channels_num = getParentEdgeAt(INPUT_FEATURES_START)->getShape().getStaticDims()[1];
const int num_rois = getParentEdgeAt(INPUT_ROIS)->getMemory().getStaticDims()[0];
const int channels_num = getParentEdgeAt(INPUT_FEATURES_START)->getMemory().getStaticDims()[1];
const int feaxels_per_roi = pooled_height_ * pooled_width_ * channels_num;
auto *input_rois = reinterpret_cast<const float *>(getParentEdgeAt(INPUT_ROIS)->getMemoryPtr()->GetPtr());
@@ -379,8 +384,8 @@ void MKLDNNExperimentalDetectronROIFeatureExtractorNode::execute(mkldnn::stream
const int level_rois_num = rois_per_level[i + 1] - level_rois_offset;
if (level_rois_num > 0) {
auto *featuremap = reinterpret_cast<const float *>(getParentEdgeAt(INPUT_FEATURES_START + i)->getMemoryPtr()->GetPtr());
const int featuremap_height = getParentEdgeAt(INPUT_FEATURES_START + i)->getShape().getStaticDims()[2];
const int featuremap_width = getParentEdgeAt(INPUT_FEATURES_START + i)->getShape().getStaticDims()[3];
const int featuremap_height = getParentEdgeAt(INPUT_FEATURES_START + i)->getMemory().getStaticDims()[2];
const int featuremap_width = getParentEdgeAt(INPUT_FEATURES_START + i)->getMemory().getStaticDims()[3];
ROIAlignForward_cpu_kernel<float>(feaxels_per_roi * level_rois_num,
featuremap,
1.0f / pyramid_scales_[i],
inference-engine/src/mkldnn_plugin/nodes/mkldnn_experimental_detectron_roifeatureextractor_node.h
+1 -1
View File
@@ -19,7 +19,7 @@ public:
void execute(mkldnn::stream strm) override;
bool created() const override;
static bool isSupportedOperation(const std::shared_ptr<ngraph::Node>& op, std::string& errorMessage) noexcept;
static bool isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept;
private:
const int INPUT_ROIS {0};
inference-engine/src/mkldnn_plugin/nodes/mkldnn_experimental_detectron_topkrois_node.cpp
+6 -2
View File
@@ -14,8 +14,12 @@
using namespace MKLDNNPlugin;
using namespace InferenceEngine;
bool MKLDNNExperimentalDetectronTopKROIsNode::isSupportedOperation(const std::shared_ptr<ngraph::Node>& op, std::string& errorMessage) noexcept {
bool MKLDNNExperimentalDetectronTopKROIsNode::isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept {
try {
if (isDynamicNgraphNode(op)) {
errorMessage = "Doesn't support op with dynamic shapes";
return false;
}
const auto topKROI = std::dynamic_pointer_cast<const ngraph::opset6::ExperimentalDetectronTopKROIs>(op);
if (!topKROI) {
errorMessage = "Only opset6 ExperimentalDetectronTopKROIs operation is supported";
@@ -56,7 +60,7 @@ void MKLDNNExperimentalDetectronTopKROIsNode::initSupportedPrimitiveDescriptors(
}
void MKLDNNExperimentalDetectronTopKROIsNode::execute(mkldnn::stream strm) {
const int input_rois_num = getParentEdgeAt(INPUT_ROIS)->getShape().getStaticDims()[0];
const int input_rois_num = getParentEdgeAt(INPUT_ROIS)->getMemory().getStaticDims()[0];
const int top_rois_num = (std::min)(max_rois_num_, input_rois_num);
auto *input_rois = reinterpret_cast<const float *>(getParentEdgeAt(INPUT_ROIS)->getMemoryPtr()->GetPtr());
inference-engine/src/mkldnn_plugin/nodes/mkldnn_experimental_detectron_topkrois_node.h
+1 -1
View File
@@ -19,7 +19,7 @@ public:
void execute(mkldnn::stream strm) override;
bool created() const override;
static bool isSupportedOperation(const std::shared_ptr<ngraph::Node>& op, std::string& errorMessage) noexcept;
static bool isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept;
private:
// Inputs:
inference-engine/src/mkldnn_plugin/nodes/mkldnn_extract_image_patches_node.cpp
+9 -5
View File
@@ -268,8 +268,12 @@ private:
}
};
bool MKLDNNExtractImagePatchesNode::isSupportedOperation(const std::shared_ptr<ngraph::Node>& op, std::string& errorMessage) noexcept {
bool MKLDNNExtractImagePatchesNode::isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept {
try {
if (isDynamicNgraphNode(op)) {
errorMessage = "Doesn't support op with dynamic shapes";
return false;
}
const auto extImgPatcher = std::dynamic_pointer_cast<const ngraph::opset3::ExtractImagePatches>(op);
if (!extImgPatcher) {
errorMessage = "Only opset3 ExtractImagePatches operation is supported";
@@ -419,12 +423,12 @@ void MKLDNNExtractImagePatchesNode::execute(mkldnn::stream strm) {
char *dst_data = reinterpret_cast<char *>(getChildEdgesAtPort(0)[0]->getMemoryPtr()->GetPtr());
const size_t dtype_size = getOriginalInputPrecisionAtPort(0).size();
const auto& inDims = getParentEdgeAt(0)->getShape().getStaticDims();
const auto& inDims = getParentEdgeAt(0)->getMemory().getStaticDims();
const size_t IC = inDims[1];
const size_t IH = inDims[2];
const size_t IW = inDims[3];
const auto& outDims = getChildEdgesAtPort(0)[0]->getShape().getStaticDims();
const auto& outDims = getChildEdgesAtPort(0)[0]->getMemory().getStaticDims();
const size_t OB = outDims[0];
const size_t OH = outDims[2];
const size_t OW = outDims[3];
@@ -434,8 +438,8 @@ void MKLDNNExtractImagePatchesNode::execute(mkldnn::stream strm) {
const size_t RH = _rates[0], RW = _rates[1];
const size_t PT = _pad_top, PL = _pad_left;
const std::vector<size_t> istrides = getParentEdgeAt(0)->getMemory().GetDescWithType<BlockedMemoryDesc>().getStrides();
const std::vector<size_t> ostrides = getChildEdgesAtPort(0)[0]->getMemory().GetDescWithType<BlockedMemoryDesc>().getStrides();
const std::vector<size_t> istrides = getParentEdgeAt(0)->getMemory().GetDescWithType<BlockedMemoryDesc>()->getStrides();
const std::vector<size_t> ostrides = getChildEdgesAtPort(0)[0]->getMemory().GetDescWithType<BlockedMemoryDesc>()->getStrides();
const std::vector<size_t> ostrides_partial = {ostrides[0], KW * IC * ostrides[1], IC * ostrides[1], ostrides[1]};
if (extract_image_patches_kernel) {
inference-engine/src/mkldnn_plugin/nodes/mkldnn_extract_image_patches_node.h
+1 -1
View File
@@ -50,7 +50,7 @@ public:
void execute(mkldnn::stream strm) override;
bool created() const override;
static bool isSupportedOperation(const std::shared_ptr<ngraph::Node>& op, std::string& errorMessage) noexcept;
static bool isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept;
private:
enum class ExtImgPatcherPadType {
inference-engine/src/mkldnn_plugin/nodes/mkldnn_fake_quantize_node.cpp
+24 -17
View File
@@ -19,7 +19,9 @@
#include "ie_parallel.hpp"
#include <ngraph/opsets/opset1.hpp>
#include <cpu_memory_desc_utils.h>
#include <memory_desc/cpu_memory_desc_utils.h>
#include "memory_desc/dnnl_blocked_memory_desc.h"
#include "utils/ngraph_utils.hpp"
// Quantization ranges validation is switched off by default in order to avoid regressions on user side
// #define VALIDATE_QUANTIZATION_RANGES
@@ -820,6 +822,11 @@ private:
bool MKLDNNFakeQuantizeNode::isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept {
try {
if (isDynamicNgraphNode(op)) {
errorMessage = "Doesn't support op with dynamic shapes";
return false;
}
const auto fq = std::dynamic_pointer_cast<const ngraph::opset1::FakeQuantize>(op);
if (!fq) {
errorMessage = "Only opset1 FakeQuantize operation is supported";
@@ -1093,13 +1100,13 @@ MKLDNNFakeQuantizeNode::MKLDNNFakeQuantizeNode(const std::shared_ptr<ngraph::Nod
std::vector<LayoutType> MKLDNNFakeQuantizeNode::getDataFormats() const {
// Special case for first FQ in the network
if (getParentEdgesAtPort(0)[0]->getShape().getStaticDims()[getAxis()] == 3) {
if (getInputShapeAtPort(0).getStaticDims()[getAxis()] == 3) {
return { LayoutType::ncsp };
} else {
if (isBinarization()) {
return { LayoutType::nspc };
} else {
if (one_of(getParentEdgesAtPort(0)[0]->getShape().getRank(), 4, 5)) {
if (one_of(getInputShapeAtPort(0).getRank(), 4, 5)) {
if (getAxis() == 1) {
auto blkFormat = mayiuse(cpu::x64::avx512_common) ? LayoutType::nCsp16c : LayoutType::nCsp8c;
return { blkFormat, LayoutType::nspc, LayoutType::ncsp };
@@ -1140,12 +1147,12 @@ void MKLDNNFakeQuantizeNode::getSupportedDescriptors() {
IE_THROW() << errorPrefix << "has unsupported number of parent edges at port " << i;
}
if (getParentEdgesAtPort(0)[0]->getShape().getRank() != getChildEdgesAtPort(0)[0]->getShape().getRank()) {
if (getInputShapeAtPort(0).getRank() != getInputShapeAtPort(0).getRank()) {
IE_THROW() << errorPrefix << "has different ranks for input and output tensors";
}
if (isBinarization()) {
if (getParentEdgesAtPort(0)[0]->getShape().getRank() != 4ul) {
if (getInputShapeAtPort(0).getRank() != 4ul) {
IE_THROW() << errorPrefix << "doesn't support input/output rank != 4";
}
}
@@ -1192,10 +1199,10 @@ void MKLDNNFakeQuantizeNode::initSupportedPrimitiveDescriptors() {
if (i == 0) {
auto descCreator = BlockedDescCreator::getCommonCreators().at(fmt);
dataConfig.desc = descCreator->createUniqueDesc(getInputPrecision(), getParentEdgeAt(i)->getShape().getStaticDims());
dataConfig.desc = descCreator->createSharedDesc(getInputPrecision(), getInputShapeAtPort(i));
} else {
auto descCreator = BlockedDescCreator::getCommonCreators().at(LayoutType::ncsp);
dataConfig.desc = descCreator->createUniqueDesc(Precision::FP32, getParentEdgeAt(i)->getShape().getStaticDims());
dataConfig.desc = descCreator->createSharedDesc(Precision::FP32, getInputShapeAtPort(i));
}
config.inConfs.push_back(dataConfig);
}
@@ -1204,7 +1211,7 @@ void MKLDNNFakeQuantizeNode::initSupportedPrimitiveDescriptors() {
dataConfig.inPlace = -1;
dataConfig.constant = false;
auto descCreator = BlockedDescCreator::getCommonCreators().at(fmt);
dataConfig.desc = descCreator->createUniqueDesc(getOutputPrecision(), getChildEdgeAt(0)->getShape().getStaticDims());
dataConfig.desc = descCreator->createSharedDesc(getOutputPrecision(), getOutputShapeAtPort(0));
config.outConfs.push_back(dataConfig);
supportedPrimitiveDescriptors.push_back({config, impl_type});
@@ -1222,12 +1229,12 @@ void MKLDNNFakeQuantizeNode::createPrimitive() {
jqp.dst_prc = config.outConfs[0].desc->getPrecision();
auto srcDesc = getParentEdgeAt(0)->getMemory().GetDescWithType<BlockedMemoryDesc>();
jqp.s_str = srcDesc.getStrides();
jqp.s_str = srcDesc->getStrides();
auto dstDesc = getChildEdgeAt(0)->getMemory().GetDescWithType<BlockedMemoryDesc>();
jqp.d_str = dstDesc.getStrides();
jqp.d_str = dstDesc->getStrides();
jqp.is_planar = srcDesc.hasLayoutType(LayoutType::ncsp) && one_of(srcDesc.getShape().getRank(), 3, 4, 5);
jqp.is_planar = srcDesc->hasLayoutType(LayoutType::ncsp) && one_of(srcDesc->getShape().getRank(), 3, 4, 5);
jqp.op_type = getAlgorithm();
@@ -1256,10 +1263,10 @@ void MKLDNNFakeQuantizeNode::createPrimitive() {
if (quantize_kernel)
quantize_kernel->create_ker();
size_t axisSize = getParentEdgeAt(0)->getShape().getStaticDims()[getAxis()];
size_t axisSize = getParentEdgesAtPort(0)[0]->getMemory().GetShape().getStaticDims()[getAxis()];
size_t axisPaddedSize = rnd_up(axisSize, 16);
MKLDNNMemoryDesc weightsDataDesc = {{(uint32_t)axisPaddedSize}, memory::data_type::f32, memory::format_tag::x};
DnnlBlockedMemoryDesc weightsDataDesc(Shape(InferenceEngine::SizeVector{axisPaddedSize}), memory::data_type::f32, memory::format_tag::x);
if (isBinarization()) {
auto binarizationThresholdsDataMem = std::make_shared<MKLDNNMemory>(getEngine());
@@ -1295,8 +1302,8 @@ void MKLDNNFakeQuantizeNode::executeReference() {
auto src = reinterpret_cast<const float *>(srcMemory->GetPtr());
auto srcDims = srcMemory->GetDesc().getShape().getStaticDims();
auto dstDims = dstMemory->GetDesc().getShape().getStaticDims();
auto srcDims = srcMemory->getStaticDims();
auto dstDims = dstMemory->getStaticDims();
auto s_str = jqp.s_str;
auto d_str = jqp.d_str;
@@ -1416,7 +1423,7 @@ void MKLDNNFakeQuantizeNode::executeBinarization() {
auto thresholds = reinterpret_cast<const float*>(internalBlobMemory[0]->GetData());
auto output_mask = reinterpret_cast<const float*>(internalBlobMemory[1]->GetData());
auto src_dims = srcMemory->GetDesc().getShape().getStaticDims();
auto src_dims = srcMemory->getStaticDims();
std::vector<size_t> s_str = jqp.s_str;
size_t tmp = s_str[s_str.size() - 1];
@@ -1459,7 +1466,7 @@ void MKLDNNFakeQuantizeNode::executeQuantization() {
auto output_scale = reinterpret_cast<const float*>(internalBlobMemory[4]->GetData());
auto output_shift = reinterpret_cast<const float*>(internalBlobMemory[5]->GetData());
auto& srcDesc = srcMemory->GetDesc();
auto& srcDesc = srcMemory->getDesc();
auto srcDims = srcDesc.getShape().getStaticDims();
bool is_blk_format = !srcDesc.hasLayoutType(LayoutType::nspc) && one_of(srcDesc.getShape().getRank(), 4, 5);
inference-engine/src/mkldnn_plugin/nodes/mkldnn_fullyconnected_node.cpp
+33 -25
View File
@@ -12,14 +12,20 @@
#include <mkldnn_extension_utils.h>
#include <mkldnn.hpp>
#include "utils/general_utils.h"
#include <cpu_memory_desc_utils.h>
#include <memory_desc/cpu_memory_desc_utils.h>
#include "memory_desc/dnnl_blocked_memory_desc.h"
using namespace mkldnn;
using namespace MKLDNNPlugin;
using namespace InferenceEngine;
bool MKLDNNFullyConnectedNode::isSupportedOperation(const std::shared_ptr<ngraph::Node>& op, std::string& errorMessage) noexcept {
bool MKLDNNFullyConnectedNode::isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept {
try {
if (isDynamicNgraphNode(op)) {
errorMessage = "Doesn't support op with dynamic shapes";
return false;
}
const auto fc = std::dynamic_pointer_cast<const FullyConnectedNode>(op);
if (!fc) {
errorMessage = "Only legacy FullyConnected operation is supported";
@@ -101,8 +107,8 @@ void MKLDNNFullyConnectedNode::getSupportedDescriptors() {
outputDataType = memory::data_type::bf16;
}
const auto inDims = getParentEdgeAt(0)->getShape().getStaticDims();
const auto outDims = getChildEdgeAt(0)->getShape().getStaticDims();
const auto& inDims = getInputShapeAtPort(0).getStaticDims();
const auto& outDims = getOutputShapeAtPort(0).getStaticDims();
if (inDims.size() == 3) {
weightsDims = InferenceEngine::SizeVector({static_cast<size_t>(outDims[2]), static_cast<size_t>(inDims[2])});
@@ -113,7 +119,7 @@ void MKLDNNFullyConnectedNode::getSupportedDescriptors() {
}
biasesDims.push_back(weightsDims[0]);
for (auto format : getAvailableFormatsForDims(getParentEdgeAt(0)->getShape())) {
for (auto format : getAvailableFormatsForDims(getInputShapeAtPort(0))) {
auto in_candidate = mkldnn::memory::desc(MKLDNNExtensionUtils::convertToDnnlDims(inDims), inputDataType, format);
auto out_candidate = mkldnn::memory::desc(MKLDNNExtensionUtils::convertToDnnlDims(outDims), outputDataType, mkldnn::memory::format_tag::any);
@@ -149,7 +155,7 @@ void MKLDNNFullyConnectedNode::execute(mkldnn::stream strm) {
auto oldMem = param->second;
auto dims = oldMem.get_desc().dims();
if (dims.size() == 3) {
MKLDNNDims normalizedDims({static_cast<ptrdiff_t>(dims[0] * dims[1]), static_cast<ptrdiff_t>(dims[2])});
std::vector<dnnl::memory::dim> normalizedDims({dims[0] * dims[1], dims[2]});
mkldnn::memory::desc newMemDesc(oldMem.get_desc().reshape(normalizedDims));
mkldnn::memory newMem(newMemDesc, oldMem.get_engine(), oldMem.get_data_handle());
primArgs.at(argType) = newMem;
@@ -237,7 +243,7 @@ std::shared_ptr<mkldnn::primitive_attr> MKLDNNFullyConnectedNode::initPrimitiveA
return attr;
}
// WA: creation MKLDNNMemoryDesc with format == any is prohibited
// WA: creation DnnlMemoryDesc with format == any is prohibited
// so we create mkldnn::memory::desc directly
// we need specific method and can't remove createDescriptor from base class because its used into initDescriptor
void MKLDNNFullyConnectedNode::createDescriptorInternal(const mkldnn::memory::desc &inputDesc,
@@ -261,12 +267,12 @@ void MKLDNNFullyConnectedNode::createDescriptorInternal(const mkldnn::memory::de
auto normalizedInDims = {inDims[0] * inDims[1], inDims[2]};
auto normalizedOutDims = {outDims[0] * outDims[1], outDims[2]};
in_candidate = mkldnn::memory::desc(normalizedInDims, in_candidate.data_type(),
MKLDNNMemory::GetPlainFormatByRank(normalizedInDims.size()));
MKLDNNExtensionUtils::GetPlainFormatByRank(normalizedInDims.size()));
out_candidate = mkldnn::memory::desc(normalizedOutDims, out_candidate.data_type(),
MKLDNNMemory::GetPlainFormatByRank(normalizedOutDims.size()));
MKLDNNExtensionUtils::GetPlainFormatByRank(normalizedOutDims.size()));
}
mkldnn::memory::desc wgh_candidate(MKLDNNDims(weightsDims), wdt, mkldnn::memory::format_tag::any);
mkldnn::memory::desc wgh_candidate(MKLDNNExtensionUtils::convertToDnnlDims(weightsDims), wdt, mkldnn::memory::format_tag::any);
if (withBiases) {
mkldnn::memory::desc bias_candidate(MKLDNNExtensionUtils::convertToDnnlDims(inputShapes[BIAS_ID].getStaticDims()), bdt,
@@ -283,28 +289,30 @@ void MKLDNNFullyConnectedNode::createDescriptorInternal(const mkldnn::memory::de
}
}
void MKLDNNFullyConnectedNode::createDescriptor(const std::vector<const MemoryDesc*> &inputDesc,
const std::vector<const MemoryDesc*> &outputDesc) {
createDescriptorInternal(MemoryDescUtils::convertToMKLDNNMemoryDesc(*inputDesc[0]), MemoryDescUtils::convertToMKLDNNMemoryDesc(*outputDesc[0]));
void MKLDNNFullyConnectedNode::createDescriptor(const std::vector<MemoryDescPtr> &inputDesc,
const std::vector<MemoryDescPtr> &outputDesc) {
createDescriptorInternal(MemoryDescUtils::convertToDnnlMemoryDesc(inputDesc[0])->getDnnlDesc(),
MemoryDescUtils::convertToDnnlMemoryDesc(outputDesc[0])->getDnnlDesc());
}
std::unique_ptr<MKLDNNMemoryDesc> MKLDNNFullyConnectedNode::getSrcMemDesc(mkldnn::primitive_desc_iterator &primitive_desc_it, size_t idx) {
auto desc = idx > 0 ? MKLDNNMemoryDesc(primitive_desc_it.weights_desc(idx - 1)) : MKLDNNMemoryDesc(primitive_desc_it.src_desc(idx));
std::shared_ptr<MemoryDesc> MKLDNNFullyConnectedNode::getSrcMemDesc(mkldnn::primitive_desc_iterator &primitive_desc_it, size_t idx) {
auto desc = idx > 0 ? primitive_desc_it.weights_desc(idx - 1) : primitive_desc_it.src_desc(idx);
if (getParentEdgeAt(idx)->getShape().getRank() == 3) {
desc = MKLDNNMemoryDesc(getParentEdgeAt(idx)->getShape().getStaticDims(), MKLDNNExtensionUtils::IEPrecisionToDataType(desc.getPrecision()),
MKLDNNMemory::GetPlainFormatByRank(getParentEdgeAt(idx)->getShape().getRank()));
if (getInputShapeAtPort(idx).getRank() == 3) {
return std::make_shared<CpuBlockedMemoryDesc>(MKLDNNExtensionUtils::DataTypeToIEPrecision(
static_cast<mkldnn::memory::data_type>(desc.data.data_type)), getInputShapeAtPort(idx));
}
return MKLDNNPlugin::make_unique<MKLDNNMemoryDesc>(std::move(desc));
return MKLDNNExtensionUtils::makeDescriptor(desc);
}
std::unique_ptr<MKLDNNMemoryDesc> MKLDNNFullyConnectedNode::getDstMemDesc(mkldnn::primitive_desc_iterator &primitive_desc_it, size_t idx) {
auto desc = MKLDNNMemoryDesc(primitive_desc_it.dst_desc(idx));
if (getChildEdgeAt(idx)->getShape().getRank() == 3) {
desc = MKLDNNMemoryDesc(getChildEdgeAt(idx)->getShape().getStaticDims(), MKLDNNExtensionUtils::IEPrecisionToDataType(desc.getPrecision()),
MKLDNNMemory::GetPlainFormatByRank(getChildEdgeAt(idx)->getShape().getRank()));
std::shared_ptr<MemoryDesc> MKLDNNFullyConnectedNode::getDstMemDesc(mkldnn::primitive_desc_iterator &primitive_desc_it, size_t idx) {
auto desc = primitive_desc_it.dst_desc(idx);
if (getOutputShapeAtPort(idx).getRank() == 3) {
return std::make_shared<CpuBlockedMemoryDesc>(MKLDNNExtensionUtils::DataTypeToIEPrecision(
static_cast<mkldnn::memory::data_type>(desc.data.data_type)), getOutputShapeAtPort(idx));
}
return MKLDNNPlugin::make_unique<MKLDNNMemoryDesc>(std::move(desc));
return MKLDNNExtensionUtils::makeDescriptor(desc);
}
InferenceEngine::Precision MKLDNNFullyConnectedNode::getRuntimePrecision() const {
inference-engine/src/mkldnn_plugin/nodes/mkldnn_fullyconnected_node.h
+5 -5
View File
@@ -27,21 +27,21 @@ public:
}
const std::vector<impl_desc_type>& getPrimitivesPriority() override;
void createDescriptor(const std::vector<const MemoryDesc*>& inputDesc,
const std::vector<const MemoryDesc*>& outputDesc) override;
void createDescriptor(const std::vector<MemoryDescPtr>& inputDesc,
const std::vector<MemoryDescPtr>& outputDesc) override;
size_t descInputNumbers(MKLDNNDescriptor desc) override {
return static_cast<size_t>(getOriginalInputsNumber());
}
std::unique_ptr<MKLDNNMemoryDesc> getSrcMemDesc(mkldnn::primitive_desc_iterator &primitive_desc_it, size_t idx) override;
std::unique_ptr<MKLDNNMemoryDesc> getDstMemDesc(mkldnn::primitive_desc_iterator &primitive_desc_it, size_t idx) override;
std::shared_ptr<MemoryDesc> getSrcMemDesc(mkldnn::primitive_desc_iterator &primitive_desc_it, size_t idx) override;
std::shared_ptr<MemoryDesc> getDstMemDesc(mkldnn::primitive_desc_iterator &primitive_desc_it, size_t idx) override;
InferenceEngine::Precision getRuntimePrecision() const override;
bool canFuse(const MKLDNNNodePtr& node) const override;
static bool isSupportedOperation(const std::shared_ptr<ngraph::Node>& op, std::string& errorMessage) noexcept;
static bool isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept;
protected:
std::shared_ptr<mkldnn::primitive_attr> initPrimitiveAttr();
inference-engine/src/mkldnn_plugin/nodes/mkldnn_gather_elements_node.cpp
+6 -2
View File
@@ -16,8 +16,12 @@
using namespace MKLDNNPlugin;
using namespace InferenceEngine;
bool MKLDNNGatherElementsNode::isSupportedOperation(const std::shared_ptr<ngraph::Node>& op, std::string& errorMessage) noexcept {
bool MKLDNNGatherElementsNode::isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept {
try {
if (isDynamicNgraphNode(op)) {
errorMessage = "Doesn't support op with dynamic shapes";
return false;
}
const auto gatherElementsOp = ngraph::as_type_ptr<const ngraph::op::v6::GatherElements>(op);
if (!gatherElementsOp) {
errorMessage = "Node is not an instance of the GatherElements operation from operation set v6.";
@@ -98,7 +102,7 @@ void MKLDNNGatherElementsNode::directExecution() {
const auto *indices = reinterpret_cast<const int *>(getParentEdgeAt(indicesIndex_)->getMemoryPtr()->GetPtr());
auto *dstData = reinterpret_cast<dataType *>(getChildEdgeAt(0)->getMemoryPtr()->GetPtr());
const int outSize = getChildEdgeAt(0)->getShape().getElementsCount();
const int outSize = getChildEdgesAtPort(0)[0]->getMemory().GetShape().getElementsCount();
auto threadBody = [&](const int ithr, const int nthr) {
int start(0lu), end(0lu);
splitter(outSize, nthr, ithr, start, end);
inference-engine/src/mkldnn_plugin/nodes/mkldnn_gather_elements_node.h
+1 -1
View File
@@ -22,7 +22,7 @@ public:
void execute(mkldnn::stream strm) override;
bool created() const override;
static bool isSupportedOperation(const std::shared_ptr<ngraph::Node>& op, std::string& errorMessage) noexcept;
static bool isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept;
private:
const size_t dataIndex_ = 0;
inference-engine/src/mkldnn_plugin/nodes/mkldnn_gather_nd_node.cpp
+9 -5
View File
@@ -16,8 +16,12 @@
using namespace MKLDNNPlugin;
using namespace InferenceEngine;
bool MKLDNNGatherNDNode::isSupportedOperation(const std::shared_ptr<ngraph::Node>& op, std::string& errorMessage) noexcept {
bool MKLDNNGatherNDNode::isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept {
try {
if (isDynamicNgraphNode(op)) {
errorMessage = "Doesn't support op with dynamic shapes";
return false;
}
const auto gatherElementsOp = ngraph::as_type_ptr<const ngraph::op::v5::GatherND>(op);
if (!gatherElementsOp) {
errorMessage = "Node is not an instance of the GatherND operation from operation set v5.";
@@ -101,11 +105,11 @@ void MKLDNNGatherNDNode::gatherElementwise() {
const auto *indices = reinterpret_cast<const int *>(getParentEdgeAt(_indicesIndex)->getMemoryPtr()->GetPtr());
auto *dstData = reinterpret_cast<dataType *>(getChildEdgeAt(0)->getMemoryPtr()->GetPtr());
auto strides = getParentEdgeAt(_dataIndex)->getMemory().GetDescWithType<BlockedMemoryDesc>().getStrides();
auto strides = getParentEdgeAt(_dataIndex)->getMemory().GetDescWithType<BlockedMemoryDesc>()->getStrides();
const size_t* srcMultipliers = strides.data() + _batchDims;
const size_t cycles = getChildEdgeAt(0)->getShape().getElementsCount() *
getChildEdgeAt(0)->getMemory().GetDesc().getPrecision().size() / (sizeof(dataType) * _batchNum);
const size_t cycles = getChildEdgeAt(0)->getMemory().GetShape().getElementsCount() *
getChildEdgeAt(0)->getMemory().getDesc().getPrecision().size() / (sizeof(dataType) * _batchNum);
const size_t CS = cycles * _sliceRank;
const size_t CB = cycles * _blockSize;
const size_t workAmount = _batchNum * cycles;
@@ -150,7 +154,7 @@ void MKLDNNGatherNDNode::gatherBlocks() {
std::vector<size_t> srcMultipliers(_sliceRank);
for (size_t i = 0; i < _sliceRank ; i++)
srcMultipliers[i] = _dataTypeSize * getParentEdgeAt(_dataIndex)->getMemory().GetDescWithType<BlockedMemoryDesc>().getStrides()[i + _batchDims];
srcMultipliers[i] = _dataTypeSize * getParentEdgeAt(_dataIndex)->getMemory().GetDescWithType<BlockedMemoryDesc>()->getStrides()[i + _batchDims];
const size_t batchStep = _batchStep * _dataTypeSize;
const size_t dataStep = _blockSize * _dataTypeSize;
inference-engine/src/mkldnn_plugin/nodes/mkldnn_gather_nd_node.h
+1 -1
View File
@@ -22,7 +22,7 @@ public:
void execute(mkldnn::stream strm) override;
bool created() const override;
static bool isSupportedOperation(const std::shared_ptr<ngraph::Node>& op, std::string& errorMessage) noexcept;
static bool isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept;
private:
size_t _dataRank;
inference-engine/src/mkldnn_plugin/nodes/mkldnn_gather_node.cpp
+9 -5
View File
@@ -13,8 +13,12 @@
using namespace MKLDNNPlugin;
using namespace InferenceEngine;
bool MKLDNNGatherNode::isSupportedOperation(const std::shared_ptr<ngraph::Node>& op, std::string& errorMessage) noexcept {
bool MKLDNNGatherNode::isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept {
try {
if (isDynamicNgraphNode(op)) {
errorMessage = "Doesn't support op with dynamic shapes";
return false;
}
const auto gatherOp = ngraph::as_type_ptr<const ngraph::op::v7::Gather>(op);
if (!gatherOp) {
errorMessage = "Only opset7 Gather operation is supported";
@@ -92,10 +96,10 @@ void MKLDNNGatherNode::createPrimitive() {
if (getSelectedPrimitiveDescriptor() == nullptr)
IE_THROW() << errorPrefix_ << " has unidentified preferable primitive descriptor.";
const SizeVector srcDims = getParentEdgeAt(GATHER_DATA)->getShape().getStaticDims();
const SizeVector idxDims = getParentEdgeAt(GATHER_INDEXES)->getShape().getStaticDims();
const SizeVector dstDims = getChildEdgeAt(0)->getShape().getStaticDims();
dataSize = getParentEdgeAt(GATHER_DATA)->getMemory().GetDesc().getPrecision().size();
const SizeVector srcDims = getParentEdgeAt(GATHER_DATA)->getMemory().getStaticDims();
const SizeVector idxDims = getParentEdgeAt(GATHER_INDEXES)->getMemory().getStaticDims();
const SizeVector dstDims = getChildEdgesAtPort(0)[0]->getMemory().getStaticDims();
dataSize = getParentEdgeAt(GATHER_DATA)->getMemory().getDesc().getPrecision().size();
indexRange = srcDims[axis];
batchSize = std::accumulate(srcDims.begin(), srcDims.begin() + batchDims, 1, std::multiplies<size_t>());
inference-engine/src/mkldnn_plugin/nodes/mkldnn_gather_node.h
+1 -1
View File
@@ -22,7 +22,7 @@ public:
void execute(mkldnn::stream strm) override;
bool created() const override;
static bool isSupportedOperation(const std::shared_ptr<ngraph::Node>& op, std::string& errorMessage) noexcept;
static bool isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept;
private:
int axis = 0;
inference-engine/src/mkldnn_plugin/nodes/mkldnn_gather_tree_node.cpp
+15 -10
View File
@@ -14,8 +14,12 @@
using namespace MKLDNNPlugin;
using namespace InferenceEngine;
bool MKLDNNGatherTreeNode::isSupportedOperation(const std::shared_ptr<ngraph::Node>& op, std::string& errorMessage) noexcept {
bool MKLDNNGatherTreeNode::isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept {
try {
if (isDynamicNgraphNode(op)) {
errorMessage = "Doesn't support op with dynamic shapes";
return false;
}
const auto gatherElementsOp = ngraph::as_type_ptr<const ngraph::op::v1::GatherTree>(op);
if (!gatherElementsOp) {
errorMessage = "Node is not an instance of the GatherTree operation from operation set v1.";
@@ -66,9 +70,9 @@ void MKLDNNGatherTreeNode::initSupportedPrimitiveDescriptors() {
}
addSupportedPrimDesc({{LayoutType::ncsp, precision},
{LayoutType::ncsp, precision},
{LayoutType::ncsp, precision},
{LayoutType::ncsp, precision}},
{LayoutType::ncsp, precision},
{LayoutType::ncsp, precision},
{LayoutType::ncsp, precision}},
{{LayoutType::ncsp, precision}},
impl_desc_type::ref_any);
}
@@ -84,16 +88,17 @@ template<typename DATA_T>
void MKLDNNGatherTreeNode::gatherTreeKernel() noexcept {
const auto *step_idx = reinterpret_cast<DATA_T *>(getParentEdgeAt(GATHER_TREE_STEP_IDX)->getMemoryPtr()->GetPtr());
const auto * const parent_idx = reinterpret_cast<DATA_T *>(getParentEdgeAt(GATHER_TREE_PARENT_IDX)->getMemoryPtr()->GetPtr());
const size_t parent_idx_size = getParentEdgeAt(GATHER_TREE_PARENT_IDX)->getShape().getElementsCount()
- getParentEdgeAt(GATHER_TREE_PARENT_IDX)->getMemory().GetDescWithType<BlockedMemoryDesc>().getOffsetPadding();
const size_t parent_idx_size = getParentEdgeAt(GATHER_TREE_PARENT_IDX)->getMemory().GetShape().getElementsCount()
- getParentEdgeAt(GATHER_TREE_PARENT_IDX)->getMemory().GetDescWithType<BlockedMemoryDesc>()->getOffsetPadding();
const auto *max_seq_len = reinterpret_cast<DATA_T *>(getParentEdgeAt(GATHER_TREE_MAX_SEQ_LEN)->getMemoryPtr()->GetPtr());
auto end_token = (reinterpret_cast<DATA_T *>(getParentEdgeAt(GATHER_TREE_END_TOKEN)->getMemoryPtr()->GetPtr()))[0];
auto * final_idx = reinterpret_cast<DATA_T *>(getChildEdgesAtPort(0)[0]->getMemoryPtr()->GetPtr());
SizeVector step_idx_dims = getParentEdgeAt(GATHER_TREE_STEP_IDX)->getShape().getStaticDims();
SizeVector parent_idx_dims = getParentEdgeAt(GATHER_TREE_PARENT_IDX)->getShape().getStaticDims();
SizeVector max_seq_len_dims = getParentEdgeAt(GATHER_TREE_MAX_SEQ_LEN)->getShape().getStaticDims();
SizeVector final_idx_dims = getChildEdgesAtPort(0)[0]->getShape().getStaticDims();
SizeVector step_idx_dims = getParentEdgeAt(GATHER_TREE_STEP_IDX)->getMemory().getStaticDims();
SizeVector parent_idx_dims = getParentEdgeAt(GATHER_TREE_PARENT_IDX)->getMemory().getStaticDims();
SizeVector max_seq_len_dims = getParentEdgeAt(GATHER_TREE_MAX_SEQ_LEN)->getMemory().getStaticDims();
SizeVector final_idx_dims = getChildEdgesAtPort(0)[0]->getMemory().getStaticDims();
int32_t max_time = step_idx_dims[0];
const size_t batch_size = step_idx_dims[1];
const size_t beam_width = step_idx_dims[2];
inference-engine/src/mkldnn_plugin/nodes/mkldnn_gather_tree_node.h
+1 -1
View File
@@ -19,7 +19,7 @@ public:
void execute(mkldnn::stream strm) override;
bool created() const override;
static bool isSupportedOperation(const std::shared_ptr<ngraph::Node>& op, std::string& errorMessage) noexcept;
static bool isSupportedOperation(const std::shared_ptr<const ngraph::Node>& op, std::string& errorMessage) noexcept;
template<typename DATA_T>
void gatherTreeKernel() noexcept;
Loaded 100 of 197 files, more files were not shown because too many files have changed in this diff. Show more