IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[CPU] Remove legacy dynamic batch processing from the plugin (#17052)

Browse Source 
* Intermediate state

* Remove old dyn batch path in the new api

* Remove legacy dyn batch support

* Remove dyn batch support field from the config

* Revert changes to the common part

* Revert accidental change in the test file

* Minor fixes

* Fix support for dyn batch without setting current

* Typo fix
This commit is contained in:
Maksim Kutakov 2023-04-24 23:18:10 +02:00 committed by GitHub
parent 758ec32001
commit 9fce01f8cc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
60 changed files with 54 additions and 531 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

144
src/plugins/intel_cpu/src/exec_network.cpp
View File

@ -81,14 +81,7 @@ ExecNetwork::ExecNetwork(const InferenceEngine::CNNNetwork &network,
_cfg.isNewApi = !isLegacyAPI();
_mutex = std::make_shared<std::mutex>();
// WA for inference dynamic batch cases in new API
if (_cfg.isNewApi) {
int64_t maxBatchSize = -1;
if (canBeExecViaLegacyDynBatch(function, maxBatchSize)) {
IE_ASSERT(maxBatchSize > -1);
_cfg.batchLimit = maxBatchSize;
}
} else if (_cfg.batchLimit > 1) {
if (_cfg.batchLimit > 1) {
// check topology for applicability
if (!CanProcessDynBatch(_network)) {
IE_THROW() << "Graph::CreateGraph: such topology cannot be compiled for dynamic batch!";
@ -378,141 +371,6 @@ InferenceEngine::Parameter ExecNetwork::GetMetric(const std::string &name) const
return GetMetricLegacy(name, graph);
}
bool ExecNetwork::canBeExecViaLegacyDynBatch(std::shared_ptr<const ov::Model> function, int64_t& maxBatchSize) const {
maxBatchSize = -1;
auto isDynBatchWithUpperBound = [maxBatchSize](const ov::PartialShape& shape) -> bool {
if (shape.rank().is_dynamic()) {
return false;
}
bool retVal = shape[0].is_dynamic() && shape[0].get_max_length() != maxBatchSize;
for (size_t i = 1; i < shape.size(); i++) {
retVal = retVal && shape[i].is_static();
}
return retVal;
};
if (function->get_parameters().size() != 1) {
return false;
}
auto param = *function->get_parameters().begin();
const auto shape = param->get_output_partial_shape(0);
if (shape.rank().is_dynamic()) {
return false;
}
if (shape.rank().get_length() < 2) {
return false;
} else {
if (maxBatchSize == -1) {
maxBatchSize = shape[0].get_max_length();
if (maxBatchSize == -1) {
return false;
}
}
if (!isDynBatchWithUpperBound(shape)) {
return false;
}
}
auto ops = function->get_ordered_ops();
for (auto op : ops) {
if (op->get_type_info() == ngraph::op::Constant::get_type_info_static()) {
continue;
}
auto type = TypeFromName(op->get_type_name());
if (!one_of(type, Type::Input,
Type::Output,
Type::Convolution,
Type::Deconvolution,
Type::Lrn,
Type::Pooling,
Type::FullyConnected,
Type::MatMul,
Type::Softmax,
Type::Split,
Type::Concatenation,
Type::Eltwise,
Type::Reshape,
Type::Tile)) {
return false;
}
for (size_t i = 0; i < op->get_output_size(); i++) {
if (!isDynBatchWithUpperBound(op->get_output_partial_shape(i))) {
return false;
}
}
if (type == Type::Tile) {
const auto repeatsNode = std::dynamic_pointer_cast<const ngraph::opset1::Constant>(op->get_input_node_shared_ptr(1));
const auto tile = std::dynamic_pointer_cast<const ngraph::opset1::Tile>(op);
if (!(tile && repeatsNode && repeatsNode->cast_vector<int64_t>()[0] == 1)) {
return false;
}
}
if (type == Type::Reshape) {
const auto inShape = op->get_input_partial_shape(0);
const auto outShape = op->get_output_partial_shape(0);
if (isDynBatchWithUpperBound(inShape) && isDynBatchWithUpperBound(outShape)) {
size_t inSize = 1, outSize = 1;
for (size_t i = 1; i < inShape.size(); i++) {
inSize *= inShape[i].get_length();
}
for (size_t i = 1; i < outShape.size(); i++) {
outSize *= outShape[i].get_length();
}
if (inSize != outSize) {
return false;
}
} else {
return false;
}
}
if (type == Type::Split) {
const auto axis = std::dynamic_pointer_cast<const ngraph::opset1::Constant>(op->get_input_node_shared_ptr(1));
if (!axis || axis->cast_vector<int64_t>()[0] == 0) {
return false;
}
}
if (type == Type::Concatenation) {
const auto concat = std::dynamic_pointer_cast<const ngraph::op::v0::Concat>(op);
if (!concat || concat->get_axis() == 0) {
return false;
}
}
if (type == Type::Softmax) {
const auto softmax = std::dynamic_pointer_cast<const ngraph::opset1::Softmax>(op);
if (!softmax || softmax->get_axis() == 0) {
return false;
}
}
if ((type == Type::MatMul || type == Type::FullyConnected) &&
(op->get_input_node_ptr(1)->get_type_info() != ngraph::op::Constant::get_type_info_static() ||
op->get_input_partial_shape(0).rank().get_length() < 2)) {
return false;
}
if (type == Type::Eltwise && std::dynamic_pointer_cast<ov::op::util::BinaryElementwiseArithmetic>(op) &&
!(op->get_input_node_ptr(0)->get_type_info() == ngraph::op::Constant::get_type_info_static() ||
op->get_input_node_ptr(1)->get_type_info() == ngraph::op::Constant::get_type_info_static()) &&
op->get_input_partial_shape(0).rank().get_length() != op->get_input_partial_shape(1).rank().get_length()) {
return false;
}
}
return true;
}
bool ExecNetwork::CanProcessDynBatch(const InferenceEngine::CNNNetwork &network) const {
InputsDataMap inputs = network.getInputsInfo();

1
src/plugins/intel_cpu/src/exec_network.h
View File

@ -76,7 +76,6 @@ protected:
*/
GraphGuard::Lock GetGraph() const;
bool canBeExecViaLegacyDynBatch(std::shared_ptr<const ov::Model> function, int64_t& maxBatchSize) const;
bool CanProcessDynBatch(const InferenceEngine::CNNNetwork &network) const;
bool isLegacyAPI() const;

61
src/plugins/intel_cpu/src/graph.cpp
View File

@ -204,24 +204,7 @@ void Graph::Replicate(const CNNNetwork &network) {
this->_name = network.getName();
std::shared_ptr<const ov::Model> func = nullptr;
// we perform model cloning and reshaping on Replicate stage to preserve input/output information
// it help to perform a graph compilation like in static case
// and handle dynamic batch case in inference stage with minimal code changes
if (getConfig().isNewApi && getConfig().batchLimit > 0) {
auto upperBoundModel = ngraph::clone_function(*network.getFunction());
std::map<ov::Output<ov::Node>, ov::PartialShape> newInShape;
for (const auto& in : upperBoundModel->get_parameters()) {
auto newShape = in->get_output_partial_shape(0);
newShape[0] = getConfig().batchLimit;
newInShape[in] = newShape;
}
upperBoundModel->reshape(newInShape);
func = upperBoundModel;
} else {
func = network.getFunction();
}
std::shared_ptr<const ov::Model> func = network.getFunction();
if (!func) {
IE_THROW() << "Function pointer inside CNNNetwork is nullptr";
@ -911,19 +894,7 @@ void Graph::PushInputData(const std::string& name, const InferenceEngine::Blob::
Memory ext_mem(getEngine());
ext_mem.Create(ext_tdesc, ext_data_ptr, false);
// branch for handling dynamic batch feature in new API
if (getConfig().isNewApi && getConfig().batchLimit > 0 && ext_mem.getStaticDims()[0] != childEdge->getMemory().getStaticDims()[0]) {
auto newDims = childEdge->getMemory().getStaticDims();
newDims[0] = ext_mem.getStaticDims()[0];
Memory tmpMem(getEngine());
auto newDesc = childEdge->getMemory().getDesc().cloneWithNewDims(newDims, true);
tmpMem.Create(newDesc, childEdge->getMemory().GetData(), false);
tmpMem.SetData(ext_mem, false);
} else {
childEdge->getMemory().SetData(ext_mem, false);
}
childEdge->getMemory().SetData(ext_mem, false);
}
// todo: make sure 'name' exists in this map...
@ -979,9 +950,6 @@ void Graph::PullOutputData(BlobMap &out) {
if (expectedDesc.getLayout() == InferenceEngine::Layout::BLOCKED) {
expectedDesc = TensorDesc(expectedDesc.getPrecision(), expectedDesc.getLayout());
}
if (getConfig().isNewApi && getConfig().batchLimit > 0) {
outDims[0] = node->batchToProcess();
}
out[name]->setShape(outDims);
}
@ -993,7 +961,7 @@ void Graph::PullOutputData(BlobMap &out) {
auto srcPrec = actualDesc.getPrecision();
auto dstPrec = expectedDesc.getPrecision();
if ((getConfig().isNewApi && !getConfig().batchLimit) && srcPrec == dstPrec && ext_blob->byteSize() != intr_blob.GetSize())
if (getConfig().isNewApi && srcPrec == dstPrec && ext_blob->byteSize() != intr_blob.GetSize())
IE_THROW() << "Output blob byte size is not equal network output byte size ("
<< ext_blob->byteSize() << "!=" << intr_blob.GetSize() << ").";
@ -1012,29 +980,16 @@ void Graph::PullOutputData(BlobMap &out) {
Memory outBloMem(getEngine());
outBloMem.Create(outBlobDesc, ext_blob_ptr, false);
// branch for handling dynamic batch feature in new API
if (getConfig().isNewApi && getConfig().batchLimit > 0 && outBloMem.getStaticDims()[0] != intr_blob.getStaticDims()[0]) {
auto newDims = intr_blob.getStaticDims();
newDims[0] = outBloMem.getStaticDims()[0];
Memory tmpMem(getEngine());
auto newDesc = intr_blob.getDesc().cloneWithNewDims(newDims, true);
tmpMem.Create(newDesc, intr_blob.GetData(), false);
outBloMem.SetData(tmpMem, false);
} else {
outBloMem.SetData(intr_blob, false);
}
outBloMem.SetData(intr_blob, false);
} else {
size_t size_to_copy = intr_blob.GetDescWithType<BlockedMemoryDesc>()->getPaddedElementsCount();
// TODO: Should we support InferenceEngine::PluginConfigParams::KEY_DYN_BATCH_LIMIT???
// TODO [DS]: phase 2: should we support this behaviour? Looks obsolete in the dynamic shapes paradigm
if (getConfig().batchLimit) {
// used only for backward compatibility with the legacy API
if (getConfig().batchLimit && dynBatch > 0) {
if (node->isDynamicNode() && !getConfig().isNewApi) {
IE_THROW(NotImplemented) << "[DS] not implemented dynamic batch for node with dynamic shape";
}
int MB_to_process = node->batchToProcess();
size_to_copy = std::accumulate(outDims.begin() + 1, outDims.end(), (size_t)1, std::multiplies<size_t>()) * MB_to_process;
size_to_copy = std::accumulate(outDims.begin() + 1, outDims.end(), (size_t)1, std::multiplies<size_t>()) * static_cast<size_t>(dynBatch);
}
cpu_convert(intr_blob_ptr, ext_blob_ptr, srcPrec, dstPrec, size_to_copy);

15
src/plugins/intel_cpu/src/graph.h
View File

@ -190,6 +190,17 @@ public:
return graphHasDynamicInput;
}
/**
* @brief This call updates the dynamic batch value
*
* @note It is used for backward compatibility with legacy API only.
* @param newDynBatch
* new dynamic batch value
*/
void setDynBatch(int newDynBatch) {
dynBatch = newDynBatch;
}
protected:
void VisitNode(NodePtr node, std::vector<NodePtr>& sortedNodes);
@ -255,6 +266,10 @@ private:
GraphContext::CPtr context;
// this field stores the dynamic batch value to provide backward compatibility
// with the legacy API dyn batch behaviour
int dynBatch = -1;
void EnforceBF16();
};

24
src/plugins/intel_cpu/src/infer_request.cpp
View File

@ -160,9 +160,6 @@ void InferRequestBase::InferImpl() {
if (graph->hasDynamicInput()) {
redefineMemoryForInputNodes();
} else if (graph->getConfig().isNewApi && graph->getConfig().batchLimit > 0) {
const auto batch = _inputs.begin()->second->getTensorDesc().getDims()[0];
SetBatch(batch);
}
execDataPreprocessing(_inputs);
@ -367,10 +364,7 @@ void LegacyInferRequest::SetBatch(int new_batch) {
}
m_curBatch = new_batch;
for (const auto& node : graph->GetNodes()) {
node->setDynamicBatchLim(new_batch);
}
graph->setDynBatch(m_curBatch);
}
void LegacyInferRequest::changeDefaultPtr() {
@ -650,22 +644,6 @@ void InferRequest::initBlobs() {
}
}
void InferRequest::SetBatch(int new_batch) {
if (!graph->getConfig().batchLimit || modelInputsMap.begin()->second->get_output_partial_shape(0).is_static()) {
IE_THROW() << "Can't set batch for model that can't be executed via legacy dynamic batch or for static model";
}
if (new_batch < 1 || new_batch > graph->getConfig().batchLimit) {
IE_THROW() << "Can't set batch that is bigger than upper bound";
}
m_curBatch = new_batch;
for (const auto& node : graph->GetNodes()) {
node->setDynamicBatchLim(new_batch);
}
}
void InferRequest::SetBlob(const std::string& name, const InferenceEngine::Blob::Ptr &data) {
OV_ITT_SCOPED_TASK(itt::domains::intel_cpu, "SetBlob");
if (name.empty()) {

1
src/plugins/intel_cpu/src/infer_request.h
View File

@ -101,7 +101,6 @@ public:
private:
void PushInputData() override;
void initBlobs() override;
void SetBatch(int batch = -1) override;
std::unordered_map<std::string, std::shared_ptr<const ov::Node>> modelInputsMap;
std::unordered_map<std::string, std::shared_ptr<const ov::Node>> modelOutputsMap;

53
src/plugins/intel_cpu/src/node.cpp
View File

@ -628,7 +628,6 @@ void Node::initSupportedPrimitiveDescriptors() {
while (static_cast<bool>(itpd)) {
NodeConfig config;
config.dynBatchSupport = true;
for (size_t i = 0; i < descInputNumbers(); i++) {
PortConfig portConfig;
portConfig.inPlace(-1);
@ -1159,53 +1158,6 @@ MemoryDescPtr Node::getDstMemDesc(dnnl::primitive_desc_iterator &primitive_desc_
return DnnlExtensionUtils::makeDescriptor(primitive_desc_it.dst_desc(idx));
}
int Node::batchToProcess() const {
return dynBatchLim == 0 ? getMaxBatch() : std::min<int>(getMaxBatch(), dynBatchLim);
}
// TODO [DS]: how we should process this for dynamic shape?
size_t Node::getMaxBatch() const {
// FIXME: batch != 0 dims number
if (!inputShapes.empty()) {
if (inputShapes[0].getRank())
return static_cast<int>(inputShapes[0].getStaticDims()[0]);
else
return 1;
}
if (!outputShapes.empty()) {
if (outputShapes[0].getRank())
return static_cast<int>(outputShapes[0].getStaticDims()[0]);
else
return 1;
}
return 0;
}
void Node::setDynamicBatchLim(int lim) {
dynBatchLim = lim;
auto setDynamicBatch = [this](int argType, int newBatch) {
auto param = primArgs.find(argType);
if (param != primArgs.end()) {
auto oldMem = param->second;
dnnl::memory::desc newMemDesc(oldMem.get_desc());
newMemDesc.get()->dims[0] = newBatch;
newMemDesc.get()->padded_dims[0] = newBatch;
dnnl::memory newMem(newMemDesc, oldMem.get_engine(), oldMem.get_data_handle());
primArgs.at(argType) = newMem;
}
};
if (!primArgs.empty()) {
int newBatch = batchToProcess();
setDynamicBatch(DNNL_ARG_SRC, newBatch);
setDynamicBatch(DNNL_ARG_DST, newBatch);
setDynamicBatch(DNNL_ARG_DIFF_SRC, newBatch);
setDynamicBatch(DNNL_ARG_DIFF_DST, newBatch);
}
}
void Node::appendPostOpArgs(const dnnl::primitive_attr& attr,
std::unordered_map<int, dnnl::memory>& primArgs,
const std::unordered_map<int, MemoryPtr>& postOpsArgs) {
@ -1643,8 +1595,7 @@ void Node::addFusedNode(const NodePtr &fusingNode) {
void Node::addSupportedPrimDesc(const std::vector<PortConfigurator>& inPortConfigs,
const std::vector<PortConfigurator>& outPortConfigs,
impl_desc_type implType,
bool dynBatchSupport) {
impl_desc_type implType) {
auto fill_port = [] (const PortConfigurator& portConfigurator, const Shape& shape,
InferenceEngine::Precision prc, std::vector<PortConfig>& port) -> bool {
// In order to simplify particular node initialization logic we just don't add config in case target shape is not supported by blockedDescCreator.
@ -1676,8 +1627,6 @@ void Node::addSupportedPrimDesc(const std::vector<PortConfigurator>& inPortConfi
if (!fill_port(outPortConfigs[i], dims, prc, config.outConfs))
return;
}
config.dynBatchSupport = dynBatchSupport;
supportedPrimitiveDescriptors.push_back({config, implType});
}

11
src/plugins/intel_cpu/src/node.h
View File

@ -357,8 +357,6 @@ public:
PerfCount &PerfCounter() { return perfCounter; }
virtual void setDynamicBatchLim(int lim);
void resolveInPlaceEdges();
virtual void execute(dnnl::stream strm) = 0;
@ -559,9 +557,6 @@ protected:
this->type = type;
}
virtual size_t getMaxBatch() const;
virtual PortDescBasePtr getConsistentInputDesc(const NodeConfig &config, size_t idx) const;
virtual PortDescBasePtr getConsistentOutputDesc(const NodeConfig &config, size_t idx) const;
virtual MemoryDescPtr getSrcMemDesc(dnnl::primitive_desc_iterator &primitive_desc_it, size_t idx);
@ -591,7 +586,7 @@ protected:
int selectedPrimitiveDescriptorIndex = -1;
bool permanent = false;
bool temporary = false;
int dynBatchLim = 0;
enum class InPlaceType {
Unknown,
InPlace,
@ -626,7 +621,6 @@ protected:
virtual const std::vector<impl_desc_type>& getPrimitivesPriority();
virtual std::vector<dnnl::memory::format_tag> getAvailableFormatsForDims(const Shape& dims) const;
int batchToProcess() const;
InferenceEngine::Layout getWeightsLayoutByDims(InferenceEngine::SizeVector dims, bool isGrouped);
@ -644,8 +638,7 @@ protected:
void addSupportedPrimDesc(const std::vector<PortConfigurator>& inPortConfigs,
const std::vector<PortConfigurator>& outPortConfigs,
impl_desc_type implType,
bool dynBatchSupport = false);
impl_desc_type implType);
void prepareMemory(const std::vector<DnnlMemoryDescPtr>& intDescs);
void prepareMemory(const DnnlMemoryDescPtr& intDesc, size_t indx);

1
src/plugins/intel_cpu/src/nodes/adaptive_pooling.cpp
View File

@ -159,7 +159,6 @@ void AdaptivePooling::initSupportedPrimitiveDescriptors() {
precision = Precision::FP32;
InferenceEngine::LayerConfig config;
config.dynBatchSupport = false;
config.inConfs.resize(2);
config.outConfs.resize((algorithm == Algorithm::AdaptivePoolingAvg ? 1 : 2));

1
src/plugins/intel_cpu/src/nodes/bin_conv.cpp
View File

@ -969,7 +969,6 @@ void BinaryConvolution::initSupportedPrimitiveDescriptors() {
setPostOps(attr);
NodeConfig config;
config.dynBatchSupport = false;
config.inConfs.resize(2);
config.inConfs[0].constant(false);
config.inConfs[0].inPlace(-1);

6
src/plugins/intel_cpu/src/nodes/color_convert.cpp
View File

@ -1064,8 +1064,7 @@ void ColorConvert::initSupportedPrimitiveDescriptors() {
const auto & inPortConfigs = std::get<0>(desc);
const auto & outPortConfigs = std::get<1>(desc);
const auto implType = std::get<2>(desc);
const auto dynBatchSupport = std::get<3>(desc);
addSupportedPrimDesc(inPortConfigs, outPortConfigs, implType, dynBatchSupport);
addSupportedPrimDesc(inPortConfigs, outPortConfigs, implType);
}
initSupportedNV12Impls();
break;
@ -1076,8 +1075,7 @@ void ColorConvert::initSupportedPrimitiveDescriptors() {
const auto & inPortConfigs = std::get<0>(desc);
const auto & outPortConfigs = std::get<1>(desc);
const auto implType = std::get<2>(desc);
const auto dynBatchSupport = std::get<3>(desc);
addSupportedPrimDesc(inPortConfigs, outPortConfigs, implType, dynBatchSupport);
addSupportedPrimDesc(inPortConfigs, outPortConfigs, implType);
}
initSupportedI420Impls();
break;

1
src/plugins/intel_cpu/src/nodes/common/tile_broadcast_utils.cpp
View File

@ -104,7 +104,6 @@ std::vector<NodeDesc> TileBroadcastCommon::getSupportedConfigs(const Node *node)
IE_THROW() << node->getTypeStr() << " node with name " << node->getName() << " has incorrect Repeats vector."
"Repeats rank must be equal to output shape rank. Repeats rank: " << repeats.size() << ", output shape rank: " << outDataShapeRank;
config.dynBatchSupport = false;
config.inConfs.resize(node->getParentEdges().size());
config.inConfs[0].inPlace(-1);
config.inConfs[0].constant(constMap[0]);

1
src/plugins/intel_cpu/src/nodes/concat.cpp
View File

@ -148,7 +148,6 @@ void Concat::initSupportedPrimitiveDescriptors() {
for (auto itr = itrRange.first; itr != itrRange.second; ++itr) {
NodeConfig config;
config.dynBatchSupport = true;
config.outConfs.resize(1);
config.outConfs[0].inPlace(-1);
config.outConfs[0].constant(false);

11
src/plugins/intel_cpu/src/nodes/conv.cpp
View File

@ -748,7 +748,6 @@ void Convolution::initSupportedPrimitiveDescriptors() {
auto itpd = desc;
while (itpd) {
NodeConfig config;
config.dynBatchSupport = true;
for (size_t i = 0; i < descInputNumbers(); i++) {
PortConfig dataConfig;
@ -1177,16 +1176,6 @@ dnnl::memory Convolution::getWeights() const {
return getParentEdgeAt(1)->getMemory().GetPrimitive();
}
void Convolution::setDynamicBatchLim(int lim) {
if (!execPtr) {
IE_THROW() << "Can't set dynamic batch for Convolution node with name: " << getName() << ", because executor is not compiled";
}
if (execPtr->needReordering()) {
IE_THROW() << "Can't execute Convolution node with dynamic batch via executor with reorders";
}
Node::setDynamicBatchLim(lim);
}
dnnl::memory Convolution::getBias() const {
return getParentEdgeAt(2)->getMemory().GetPrimitive();
}

2
src/plugins/intel_cpu/src/nodes/conv.h
View File

@ -69,8 +69,6 @@ public:
bool isWinograd() const { return isWino; }
void setDynamicBatchLim(int lim) override;
protected:
InferenceEngine::Precision fusedEltwisePrecision(const NodePtr& fusingNode) const;
void redefineOutputMemory(const std::vector<VectorDims> &newOutputShapes) override;

2
src/plugins/intel_cpu/src/nodes/convert.cpp
View File

@ -89,8 +89,6 @@ void Convert::initSupportedPrimitiveDescriptors() {
PortConfig dataIn;
PortConfig dataConfigOut;
config.dynBatchSupport = false;
bool canInitExternalDesc = false;
if (input && output) {
canInitExternalDesc = true;

10
src/plugins/intel_cpu/src/nodes/deconv.cpp
View File

@ -602,16 +602,6 @@ VectorDims Deconvolution::shapeInferInternal(const VectorDims &inDims, std::vect
return std::move(result.dims.back());
}
void Deconvolution::setDynamicBatchLim(int lim) {
if (!execPtr) {
IE_THROW() << "Can't set dynamic batch for Deconvolution node with name: " << getName() << ", because executor is not compiled";
}
if (execPtr->needReordering()) {
IE_THROW() << "Can't execute Deconvolution node with dynamic batch via executor with reorders";
}
Node::setDynamicBatchLim(lim);
}
void Deconvolution::execute(dnnl::stream strm) {
if (!execPtr) {
IE_THROW() << "Can't execute Deconvolution node with name: " << getName() << ", because executor is not compiled";

1
src/plugins/intel_cpu/src/nodes/deconv.h
View File

@ -50,7 +50,6 @@ public:
void executeDynamicImpl(dnnl::stream strm) override { execute(strm); }
bool needShapeInfer() const override;
void setDynamicBatchLim(int lim) override;
bool canBeExecutedInInt8() const;
bool canFuseBias() const;

1
src/plugins/intel_cpu/src/nodes/def_conv.cpp
View File

@ -806,7 +806,6 @@ void DeformableConvolution::initSupportedPrimitiveDescriptors() {
size_t inputsNumber = getOriginalInputsNumber();
NodeConfig config;
config.dynBatchSupport = false;
config.inConfs.resize(inputsNumber);
config.inConfs[0].constant(false);
config.inConfs[0].inPlace(-1);

3
src/plugins/intel_cpu/src/nodes/depth_to_space.cpp
View File

@ -125,7 +125,6 @@ void DepthToSpace::initSupportedPrimitiveDescriptors() {
}
NodeConfig config;
config.dynBatchSupport = true;
config.inConfs.resize(1);
config.outConfs.resize(1);
config.inConfs[0].inPlace(-1);
@ -304,7 +303,7 @@ void DepthToSpace::execute(dnnl::stream strm) {
THROW_ERROR << "doesn't have a compiled executor.";
}
int MB = isDynamicNode() ? getParentEdgeAt(0)->getMemoryPtr()->getStaticDims()[0] : batchToProcess();
int MB = getParentEdgeAt(0)->getMemoryPtr()->getStaticDims()[0];
execPtr->exec(getParentEdgeAt(0)->getMemoryPtr(), getChildEdgeAt(0)->getMemoryPtr(), MB);
}

43
src/plugins/intel_cpu/src/nodes/eltwise.cpp
View File

@ -1220,7 +1220,6 @@ struct EltwiseKey {
std::vector<InferenceEngine::Precision> inpPrc;
InferenceEngine::Precision outPrc;
dnnl::post_ops postOps;
bool useDynBatch;
EltwiseImplType implType;
size_t hash() const {
@ -1256,7 +1255,6 @@ struct EltwiseKey {
});
seed = hash_combine(seed, outPrc.getPrecVal());
seed = get_post_op_hash(seed, *postOps.get());
seed = hash_combine(seed, useDynBatch);
seed = hash_combine(seed, implType);
return seed;
}
@ -1271,7 +1269,6 @@ struct EltwiseKey {
inpPrc == rhs.inpPrc &&
outPrc == rhs.outPrc &&
*postOps.get() == *rhs.postOps.get() &&
useDynBatch == rhs.useDynBatch &&
implType == rhs.implType;
if (result) {
@ -1322,7 +1319,6 @@ public:
const std::vector<InferenceEngine::Precision>& inpPrc,
const InferenceEngine::Precision& outPrc,
const dnnl::post_ops& post_ops,
bool useDynBatch,
bool useRuntimePtrs) {
auto collapseLastDims = [](std::vector<size_t>& dims, int dimsToCollapse) {
for (int i = dims.size() - 2; i > dims.size() - dimsToCollapse - 2; i--) {
@ -1428,9 +1424,7 @@ public:
int collapsedDims = 0;
bool hasDifferentDims = false;
while (!useRuntimePtrs && currentJitWorkAmount < minimalJitWorkAmount && currentJitWorkAmount < fullWorkAmount &&
// we shouldn't collapse batch dimension in case dynamic batch is enabled
(!useDynBatch || (outBlkDims.size() - collapsedDims > 2))) {
while (!useRuntimePtrs && currentJitWorkAmount < minimalJitWorkAmount && currentJitWorkAmount < fullWorkAmount) {
if (collapsedDims >= maxCollapsedDims)
break;
@ -1786,7 +1780,6 @@ static Eltwise::executorPtr buildExecutor(const EltwiseKey& key) {
key.inpPrc,
key.outPrc,
key.postOps,
key.useDynBatch,
key.implType == EltwiseImplType::optimizedShapeAgnostic);
} else {
execPtr = std::make_shared<EltwiseRefExecutor>(key.eltwise_data.front(),
@ -2062,10 +2055,6 @@ void Eltwise::initSupportedPrimitiveDescriptors() {
// TODO [DS]: inplace
size_t offset = 0;
NodeConfig config;
if (!isDynamicNode()) {
config.dynBatchSupport = getOutputShapeAtPort(0).getRank() > 1 && getOutputShapeAtPort(0) ==
getInputShapeAtPort(0);
}
for (size_t i = 0; i < getParentEdges().size(); i++) {
BlockedMemoryDesc::CmpMask inputMask = BLOCKED_DESC_SKIP_OFFSET_MASK;
@ -2182,8 +2171,6 @@ void Eltwise::createPrimitive() {
memPtrs.push_back(getChildEdgeAt(0)->getMemoryPtr());
}
isDynBatchEnabled = getSelectedPrimitiveDescriptor()->getConfig().dynBatchSupport;
start_offset_in.resize(inputNum);
for (size_t i = 0; i < inputNum; i++) {
const auto desc = getParentEdgeAt(i)->getMemory().GetDescWithType<BlockedMemoryDesc>();
@ -2276,7 +2263,7 @@ void Eltwise::prepareParams() {
if (!canSkipSearchInCache) {
EltwiseData thisOp{getAlgorithm(), getOneDnnAlgorithm(), getAlpha(), getBeta(), getGamma()};
EltwiseKey key = {{thisOp}, {getType()}, currentOutBlkDims, outOrder, dims_in, inpPrc, outPrc, dnnl::post_ops(), isDynBatchEnabled, implType};
EltwiseKey key = {{thisOp}, {getType()}, currentOutBlkDims, outOrder, dims_in, inpPrc, outPrc, dnnl::post_ops(), implType};
fqDataPtrs.clear();
for (const auto &node : fusedWith) {
key.ops_list.push_back(node->getType());
@ -2364,14 +2351,6 @@ void Eltwise::execute(dnnl::stream strm) {
args_ptrs.src_ptr[i] = reinterpret_cast<const uint8_t*>(memPtrs[i]->GetData()) + start_offset_in[i];
args_ptrs.dst_ptr = reinterpret_cast<uint8_t*>(memPtrs.back()->GetData()) + start_offset_out;
// In general case we need to recompute offsets as well but currently all supported layout assumes batch to be outermost dimension
if (isDynBatchEnabled) {
auto batchDimIdx = execPtr->getBatchDimIdx();
if (dims_out.size() <= batchDimIdx)
IE_THROW() << "Can't set batch dims for eltwise node with rank: " << dims_out.size() << " and batch idx: " << batchDimIdx;
dims_out[batchDimIdx] = static_cast<size_t>(batchToProcess());
}
args_ptrs.post_op_data = fqDataPtrs.data();
// shape agnostic kernel: offsets and work amount initialization
@ -2402,24 +2381,6 @@ void Eltwise::executeDynamicImpl(dnnl::stream strm) {
execute(strm);
}
void Eltwise::setDynamicBatchLim(int lim) {
Node::setDynamicBatchLim(lim);
ov::PartialShape outShape = getParentEdgesAtPort(0)[0]->getMemory().GetShape().toPartialShape();
if (!getParentEdgesAtPort(0)[0]->getParent()->isConstant()) {
outShape[0] = batchToProcess();
}
for (size_t i = 1; i < getParentEdges().size(); i++) {
auto currentShape = getParentEdgesAtPort(i)[0]->getMemory().GetShape().toPartialShape();
if (!getParentEdgesAtPort(i)[0]->getParent()->isConstant()) {
currentShape[0] = batchToProcess();
}
if (!ov::PartialShape::broadcast_merge_into(outShape, currentShape, ov::op::AutoBroadcastType::NUMPY)) {
IE_THROW() << "Can't execute eltwise node with dynamic batch. Input shapes are incompatible";
}
}
}
bool Eltwise::created() const {
return getType() == Type::Eltwise;
}

3
src/plugins/intel_cpu/src/nodes/eltwise.h
View File

@ -133,8 +133,6 @@ public:
void executeDynamicImpl(dnnl::stream strm) override;
void setDynamicBatchLim(int lim) override;
enum BroadcastingPolicy {
PerChannel,
PerTensor,
@ -153,7 +151,6 @@ private:
EltwiseImplType implType = EltwiseImplType::reference;
std::vector<bool> broadcastPolicy;
bool isDynBatchEnabled = false;
bool specialConvolutionAddFusing = false;
size_t inputNum = 0;
std::vector<ptrdiff_t> start_offset_in = {};

1
src/plugins/intel_cpu/src/nodes/fake_quantize.cpp
View File

@ -1344,7 +1344,6 @@ void FakeQuantize::initSupportedPrimitiveDescriptors() {
for (auto& fmt : dataFormats) {
NodeConfig config;
config.dynBatchSupport = true;
for (size_t i = 0; i < getParentEdges().size(); i++) {
PortConfig dataConfig;
dataConfig.inPlace(-1);

33
src/plugins/intel_cpu/src/nodes/fullyconnected.cpp
View File

@ -446,38 +446,6 @@ void FullyConnected::prepareParams() {
}
}
void FullyConnected::setDynamicBatchLim(int lim) {
if (!execPtr) {
IE_THROW() << "Can't set dynamic batch for FullyConnected node with name: " << getName() << ", because executor is not compiled";
}
if (execPtr->needReordering()) {
IE_THROW() << "Can't execute FullyConnected node with dynamic batch via executor with reorders";
}
auto setBatchPrimArgs = [this](int argType, const dnnl::memory& oldMem) {
dnnl::memory::desc newMemDesc(oldMem.get_desc());
newMemDesc.get()->dims[0] = batchToProcess();
newMemDesc.get()->padded_dims[0] = batchToProcess();
const auto dims = newMemDesc.get_dims();
if (dims.size() == 3) {
std::vector<dnnl::memory::dim> normalizedDims({dims[0] * dims[1], dims[2]});
newMemDesc = newMemDesc.reshape(normalizedDims);
}
primArgs.at(argType) = dnnl::memory(newMemDesc, oldMem.get_engine(), oldMem.get_data_handle());
};
if (useConv1x1) {
Node::setDynamicBatchLim(lim);
} else {
dynBatchLim = lim;
setBatchPrimArgs(DNNL_ARG_SRC, getParentEdgesAtPort(0)[0]->getMemory().GetPrimitive());
setBatchPrimArgs(DNNL_ARG_DST, getChildEdgesAtPort(0)[0]->getMemory().GetPrimitive());
}
}
void FullyConnected::execute(dnnl::stream strm) {
if (!execPtr) {
IE_THROW() << "Can't execute FullyConnected node with name: " << getName() << ", because executor is not compiled";
@ -705,7 +673,6 @@ void FullyConnected::initSupportedPrimitiveDescriptors() {
};
NodeConfig config;
config.dynBatchSupport = true;
for (size_t i = 0; i < descInputNumbers(); i++) {
PortConfig portConfig;
portConfig.inPlace(-1);

2
src/plugins/intel_cpu/src/nodes/fullyconnected.h
View File

@ -56,8 +56,6 @@ public:
void prepareParams() override;
void executeDynamicImpl(dnnl::stream strm) override;
void setDynamicBatchLim(int lim) override;
bool withBiasFused() const {
return withBiases;
}

3
src/plugins/intel_cpu/src/nodes/gather.cpp
View File

@ -200,8 +200,7 @@ void Gather::initSupportedPrimitiveDescriptors() {
{LayoutType::ncsp, Precision::I32},
{LayoutType::ncsp, Precision::I32, isAxisInputConst}},
{{LayoutType::ncsp, dataPrecision}},
ref_any,
isDynamicNode());
ref_any);
}
void Gather::createPrimitive() {

16
src/plugins/intel_cpu/src/nodes/generic.cpp
View File

@ -55,7 +55,6 @@ void Generic::getSupportedDescriptors() {
NodeConfig Generic::convertLayerToNodeConfig(const InferenceEngine::LayerConfig &layerConfig) {
NodeConfig config;
config.dynBatchSupport = layerConfig.dynBatchSupport;
config.inConfs.resize(layerConfig.inConfs.size());
for (size_t i = 0; i < layerConfig.inConfs.size(); i++) {
config.inConfs[i].inPlace(layerConfig.inConfs[i].inPlace);
@ -73,7 +72,6 @@ NodeConfig Generic::convertLayerToNodeConfig(const InferenceEngine::LayerConfig
InferenceEngine::LayerConfig Generic::convertNodeToLayerConfig(const NodeConfig &nodeConfig) {
InferenceEngine::LayerConfig config;
config.dynBatchSupport = nodeConfig.dynBatchSupport;
config.inConfs.resize(nodeConfig.inConfs.size());
for (size_t i = 0; i < nodeConfig.inConfs.size(); i++) {
config.inConfs[i].inPlace = nodeConfig.inConfs[i].inPlace();
@ -146,7 +144,6 @@ void Generic::cleanup() {
}
void Generic::execLayer() {
bool isDynBatch = dynBatchLim > 0;
std::vector<InferenceEngine::Blob::Ptr> inputs;
std::vector<InferenceEngine::Blob::CPtr> constInputs;
std::vector<InferenceEngine::TensorDesc> inputDescs;
@ -155,19 +152,10 @@ void Generic::execLayer() {
auto inputBlob = MemoryDescUtils::interpretAsBlob(getParentEdgeAt(i)->getMemory());
inputs.push_back(inputBlob);
constInputs.push_back(inputBlob);
if (isDynBatch && dynBatchLim >= inputs[inputs.size() - 1]->getTensorDesc().getDims()[0]) {
isDynBatch = false;
} else {
// TODO: Ask the right dims using getShape() from previous node
inputDescs.push_back(inputs[inputs.size() - 1]->getTensorDesc());
if (inputDescs[inputDescs.size() - 1].getDims().size() > 0)
inputDescs[inputDescs.size() - 1].getDims()[0] = static_cast<size_t>(batchToProcess());
}
// TODO: Ask the right dims using getShape() from previous node
inputDescs.push_back(inputs[inputs.size() - 1]->getTensorDesc());
}
// TODO: use ngraph-based extension mechnism if needed to recompute shape
isDynBatch = false;
std::vector<InferenceEngine::Blob::Ptr> outputs;
for (size_t i = 0; i < outputShapes.size(); i++) {
outputs.push_back(MemoryDescUtils::interpretAsBlob(getChildEdgesAtPort(i)[0]->getMemory()));

3
src/plugins/intel_cpu/src/nodes/grid_sample.cpp
View File

@ -106,8 +106,7 @@ void GridSample::initSupportedPrimitiveDescriptors() {
addSupportedPrimDesc({{LayoutType::ncsp, dataPrecision},
{LayoutType::ncsp, gridPrecision}},
{{LayoutType::ncsp, dataPrecision}},
implType,
isDynamicNode());
implType);
}
void GridSample::createPrimitive() {

2
src/plugins/intel_cpu/src/nodes/if.cpp
View File

@ -170,8 +170,6 @@ void If::initSupportedPrimitiveDescriptors() {
config.outConfs.push_back(dataConf);
}
config.dynBatchSupport = true;
supportedPrimitiveDescriptors.emplace_back(config, impl_desc_type::unknown);
}

2
src/plugins/intel_cpu/src/nodes/interaction.cpp
View File

@ -207,7 +207,7 @@ void Interaction::initSupportedPrimitiveDescriptors() {
}
};
//add descriptor
addSupportedPrimDesc(inPortConfigs, outPortConfigs, impl_desc_type::ref_any, true);
addSupportedPrimDesc(inPortConfigs, outPortConfigs, impl_desc_type::ref_any);
}
static inline void cat(uint8_t* out,

1
src/plugins/intel_cpu/src/nodes/interpolate.cpp
View File

@ -2040,7 +2040,6 @@ void Interpolate::initSupportedPrimitiveDescriptors() {
auto axesType = Precision::I32;
NodeConfig config;
config.dynBatchSupport = false;
config.outConfs.resize(1);
if (is_version11) {
if (isAxesSpecified) {

7
src/plugins/intel_cpu/src/nodes/matmul.cpp
View File

@ -510,7 +510,6 @@ void MatMul::initSupportedPrimitiveDescriptors() {
auto itpd = desc;
while (itpd) {
NodeConfig config;
config.dynBatchSupport = true;
for (size_t i = 0; i < descInputNumbers(); i++) {
PortConfig portConfig;
portConfig.inPlace(-1);
@ -553,12 +552,6 @@ bool MatMul::created() const {
return getType() == Type::MatMul;
}
size_t MatMul::getMaxBatch() const {
if (!outputShapes.empty())
return outputShapes[0].getStaticDims()[0];
return 0;
}
InferenceEngine::Precision MatMul::getRuntimePrecision() const {
return getMaxPrecision(getInputPrecisions());
}

1
src/plugins/intel_cpu/src/nodes/matmul.h
View File

@ -27,7 +27,6 @@ public:
MemoryDescPtr getSrcMemDesc(dnnl::primitive_desc_iterator &primitive_desc_it, size_t idx) override;
bool canFuse(const NodePtr& node) const override;
bool created() const override;
size_t getMaxBatch() const override;
InferenceEngine::Precision getRuntimePrecision() const override;
size_t descInputNumbers() override {

1
src/plugins/intel_cpu/src/nodes/memory.cpp
View File

@ -71,7 +71,6 @@ void MemoryOutput::initSupportedPrimitiveDescriptors() {
InferenceEngine::Precision precision = getOriginalInputPrecisionAtPort(0);
NodeConfig config;
config.dynBatchSupport = true;
config.inConfs.resize(1);
config.inConfs[0].inPlace(-1);
config.inConfs[0].constant(false);

3
src/plugins/intel_cpu/src/nodes/mha.cpp
View File

@ -785,8 +785,7 @@ void MHA::initSupportedPrimitiveDescriptors() {
{LayoutType::ncsp, Precision::FP32},
{LayoutType::ncsp, inputPrecisions[3]}},
{{LayoutType::ncsp, getOriginalOutputPrecisionAtPort(0)}},
ref_any,
isDynamicNode());
ref_any);
}
void MHA::init_brgemm(brgemmCtx& ctx, std::unique_ptr<brgemm_kernel_t>& brgKernel, bool use_amx) {

1
src/plugins/intel_cpu/src/nodes/mvn.cpp
View File

@ -1191,7 +1191,6 @@ void MVN::initSupportedPrimitiveDescriptors() {
const size_t inputsNum = getParentEdges().size();
NodeConfig config;
config.dynBatchSupport = false;
config.inConfs.resize(inputsNum);
config.outConfs.resize(1);
config.inConfs[0].constant(false);

3
src/plugins/intel_cpu/src/nodes/ngram.cpp
View File

@ -94,8 +94,7 @@ void Ngram::initSupportedPrimitiveDescriptors() {
addSupportedPrimDesc({{LayoutType::ncsp, InferenceEngine::Precision::FP32},
{LayoutType::ncsp, idcesPrecision}},
{{LayoutType::ncsp, InferenceEngine::Precision::FP32}},
ref_any,
isDynamicNode());
ref_any);
}
void Ngram::prepareParams() {

1
src/plugins/intel_cpu/src/nodes/node_config.h
View File

@ -147,7 +147,6 @@ private:
};
struct NodeConfig {
bool dynBatchSupport = false;
std::vector<PortConfig> inConfs;
std::vector<PortConfig> outConfs;
};

1
src/plugins/intel_cpu/src/nodes/normalize.cpp
View File

@ -813,7 +813,6 @@ void NormalizeL2::initSupportedPrimitiveDescriptors() {
getParentEdgeAt(DATA)->getParent()->getChildEdges().size() == 1;
NodeConfig config;
config.dynBatchSupport = false;
config.inConfs.resize(2);
config.outConfs.resize(1);
config.outConfs[0].inPlace(canBeInplace ? 0 : -1);

1
src/plugins/intel_cpu/src/nodes/pad.cpp
View File

@ -129,7 +129,6 @@ void Pad::initSupportedPrimitiveDescriptors() {
const size_t numOfDims = inputDataShape.getRank();
NodeConfig config;
config.dynBatchSupport = false;
config.inConfs.resize(isPadValueSpecified ? 4 : 3);
config.outConfs.resize(1);

2
src/plugins/intel_cpu/src/nodes/pooling.cpp
View File

@ -593,7 +593,6 @@ void Pooling::initSupportedPrimitiveDescriptors() {
auto& creatorsMap = BlockedDescCreator::getCommonCreators();
auto pushDesc = [&](LayoutType format) {
NodeConfig config;
config.dynBatchSupport = false;
config.inConfs.resize(getParentEdges().size());
config.outConfs.resize(getOriginalOutputsNumber());
@ -625,7 +624,6 @@ void Pooling::initSupportedPrimitiveDescriptors() {
while (static_cast<bool>(itpd)) {
NodeConfig config;
config.dynBatchSupport = true;
for (size_t i = 0; i < descInputNumbers(); i++) {
PortConfig dataConfig;
dataConfig.inPlace(-1);

1
src/plugins/intel_cpu/src/nodes/reduce.cpp
View File

@ -1827,7 +1827,6 @@ void Reduce::initSupportedPrimitiveDescriptors() {
dst_data_size = output_prec.size();
NodeConfig config;
config.dynBatchSupport = false;
config.inConfs.resize(2);
config.outConfs.resize(1);
config.inConfs[REDUCE_DATA].constant(false);

26
src/plugins/intel_cpu/src/nodes/reorder.cpp
View File

@ -52,7 +52,6 @@ void Reorder::initSupportedPrimitiveDescriptors() {
auto child = getChildEdgeAt(0)->getChild();
NodeConfig config;
config.dynBatchSupport = true;
config.inConfs.resize(1);
config.outConfs.resize(1);
config.inConfs[0].inPlace(-1);
@ -351,31 +350,6 @@ void Reorder::execute(dnnl::stream strm) {
}
}
void Reorder::setDynamicBatchLim(int lim) {
std::cout << getName() << " setDynamicBatchLim" << "\n";
dynBatchLim = lim;
if (!prim)
return;
auto &dstMemPtr = getChildEdgeAt(0)->getMemoryPtr();
auto &srcMemPtr = getParentEdgeAt(0)->getMemoryPtr();
memory::desc src_d = srcMemPtr->GetDescWithType<DnnlMemoryDesc>()->getDnnlDesc();
memory::desc dst_d = dstMemPtr->GetDescWithType<DnnlMemoryDesc>()->getDnnlDesc();
void *src_data_hdl = srcMemPtr->GetData();
void *dst_data_hdl = dstMemPtr->GetData();
// @ TODO ONEDNN_3_0 direct access to internal elements should be avoided
src_d.get()->dims[0] = batchToProcess();
src_d.get()->padded_dims[0] = batchToProcess();
dst_d.get()->dims[0] = batchToProcess();
dst_d.get()->padded_dims[0] = batchToProcess();
createReorderPrimitive(src_d, src_data_hdl, dst_d, dst_data_hdl);
}
std::string Reorder::getReorderArgs(const MemoryDesc &parentDesc, const MemoryDesc &childDesc) {
std::string inArgs, outArgs;
if (parentDesc.getPrecision() != childDesc.getPrecision()) {

2
src/plugins/intel_cpu/src/nodes/reorder.h
View File

@ -52,8 +52,6 @@ public:
this->isOptimized = isOptimized;
}
void setDynamicBatchLim(int lim) override;
bool canBeInPlace() const override {
return false;
}

1
src/plugins/intel_cpu/src/nodes/reshape.cpp
View File

@ -303,7 +303,6 @@ void Reshape::initSupportedPrimitiveDescriptors() {
canBeInPlace = false;
NodeConfig config;
config.dynBatchSupport = true;
config.inConfs.resize(getParentEdges().size());
auto& creatorsMap = BlockedDescCreator::getCommonCreators();
for (size_t i = 0; i < getParentEdges().size(); i++) {

1
src/plugins/intel_cpu/src/nodes/rnn.cpp
View File

@ -975,7 +975,6 @@ void RNN::createDescriptor(const std::vector<MemoryDescPtr> &inputDesc,
// Fill supported config
NodeConfig config;
config.dynBatchSupport = false;
for (size_t i = 0; i < inputDesc.size(); i++) {
PortConfig dataConfig;
dataConfig.inPlace(-1);

1
src/plugins/intel_cpu/src/nodes/roi_align.cpp
View File

@ -778,7 +778,6 @@ void ROIAlign::initSupportedPrimitiveDescriptors() {
}
NodeConfig config;
config.dynBatchSupport = false;
config.inConfs.resize(3);
config.outConfs.resize(1);

1
src/plugins/intel_cpu/src/nodes/scatter_update.cpp
View File

@ -192,7 +192,6 @@ void ScatterUpdate::initSupportedPrimitiveDescriptors() {
!getParentEdgeAt(DATA_ID)->getParent()->isConstant();
NodeConfig config;
config.dynBatchSupport = false;
if (axisRelaxed) {
config.inConfs.resize(4);
} else {

14
src/plugins/intel_cpu/src/nodes/shuffle_channels.cpp
View File

@ -81,8 +81,6 @@ ShuffleChannels::ShuffleChannels(const std::shared_ptr<ngraph::Node>& op, const
attrs.dataRank = getInputShapeAtPort(0).getRank();
if (attrs.axis < 0)
attrs.axis += attrs.dataRank;
supportDynamicBatch = (attrs.axis != 0);
}
void ShuffleChannels::initSupportedPrimitiveDescriptors() {
@ -111,18 +109,18 @@ void ShuffleChannels::initSupportedPrimitiveDescriptors() {
addSupportedPrimDesc({{firstCreatorType, precision}},
{{firstCreatorType, precision}},
impl_type, supportDynamicBatch);
impl_type);
addSupportedPrimDesc({{secondCreatorType, precision}},
{{secondCreatorType, precision}},
impl_type, supportDynamicBatch);
impl_type);
// canUseBlocked
if (attrs.axis != 1) {
addSupportedPrimDesc({{LayoutType::nCsp8c, precision}},
{{LayoutType::nCsp8c, precision}},
impl_type, supportDynamicBatch);
impl_type);
addSupportedPrimDesc({{LayoutType::nCsp16c, precision}},
{{LayoutType::nCsp16c, precision}},
impl_type, supportDynamicBatch);
impl_type);
}
}
@ -294,9 +292,7 @@ void ShuffleChannels::execute(dnnl::stream strm) {
if (!execPtr)
THROW_SHCH_ERROR << "doesn't have a compiled executor.";
int MB = -1;
if (supportDynamicBatch)
MB = isDynamicNode() ? getParentEdgeAt(0)->getMemoryPtr()->getStaticDims()[0] : batchToProcess();
int MB = (attrs.axis != 0) ? getParentEdgeAt(0)->getMemoryPtr()->getStaticDims()[0] : -1;
const uint8_t* srcData = reinterpret_cast<const uint8_t*>(getParentEdgeAt(0)->getMemoryPtr()->GetPtr());
uint8_t* dstData = reinterpret_cast<uint8_t*>(getChildEdgeAt(0)->getMemoryPtr()->GetPtr());

2
src/plugins/intel_cpu/src/nodes/shuffle_channels.h
View File

@ -57,8 +57,6 @@ private:
};
using executorPtr = std::shared_ptr<ShuffleChannelsExecutor>;
executorPtr execPtr = nullptr;
bool supportDynamicBatch = false;
};
} // namespace node

3
src/plugins/intel_cpu/src/nodes/space_to_depth.cpp
View File

@ -128,7 +128,6 @@ void SpaceToDepth::initSupportedPrimitiveDescriptors() {
}
NodeConfig config;
config.dynBatchSupport = true;
config.inConfs.resize(1);
config.outConfs.resize(1);
config.inConfs[0].inPlace(-1);
@ -316,7 +315,7 @@ void SpaceToDepth::execute(dnnl::stream strm) {
}
const uint8_t* srcData = reinterpret_cast<const uint8_t *>(getParentEdgeAt(0)->getMemoryPtr()->GetPtr());
uint8_t* dstData = reinterpret_cast<uint8_t *>(getChildEdgeAt(0)->getMemoryPtr()->GetPtr());
const int MB = isDynamicNode() ? getParentEdgeAt(0)->getMemoryPtr()->getStaticDims()[0] : batchToProcess();
const int MB = getParentEdgeAt(0)->getMemoryPtr()->getStaticDims()[0];
execPtr->exec(srcData, dstData, MB);
}

25
src/plugins/intel_cpu/src/nodes/split.cpp
View File

@ -101,11 +101,6 @@ void Split::initSupportedPrimitiveDescriptors() {
const auto axisPrecision = Precision::I32;
auto outPrecision = inpPrecision; // the split layer doesn't convert precisions
bool dynBatchSupport = true;
if (axis < 1) {
dynBatchSupport = false;
}
// Set plain and tailC formats
std::vector<LayoutType> tdCreatorTypes{ LayoutType::ncsp, LayoutType::nspc };
@ -137,7 +132,6 @@ void Split::initSupportedPrimitiveDescriptors() {
for (auto itr = itrRange.first; itr != itrRange.second; ++itr) {
NodeConfig config;
config.dynBatchSupport = dynBatchSupport;
config.inConfs.resize(INPUTS_NUM);
config.inConfs[0].inPlace(-1);
config.inConfs[0].constant(false);
@ -215,7 +209,6 @@ void Split::initSupportedPrimitiveDescriptors() {
if (axis == 1 && (dstFirstDims.size() == 4 || dstFirstDims.size() == 5)) {
NodeConfig config;
config.dynBatchSupport = dynBatchSupport;
config.inConfs.resize(INPUTS_NUM);
config.inConfs[0].inPlace(-1);
config.inConfs[0].constant(false);
@ -315,17 +308,15 @@ void Split::execute(dnnl::stream strm) {
THROW_ERROR << "Output data pointers have not been initialized.";
const auto &srcMem = getParentEdgesAtPort(0)[0]->getMemory();
size_t batch = srcMem.getStaticDims()[0];
Dim MB = isDynamicNode() ? batch : batchToProcess();
if (canUseOptimizedNspc2Ncsp) {
optimizedNspc2Ncsp(MB);
optimizedNspc2Ncsp(srcMem.getStaticDims()[0]);
return;
}
uint8_t* srcData = reinterpret_cast<uint8_t*>(srcMem.GetPtr());
IE_ASSERT(execPtr != nullptr);
execPtr->exec(srcData, getRawDstMemPtrs(), batch, MB);
execPtr->exec(srcData, getRawDstMemPtrs());
}
bool Split::created() const {
@ -499,13 +490,6 @@ void Split::selectOptimalPrimitiveDescriptor() {
selectPrimitiveDescriptorByIndex(0);
}
void Split::setDynamicBatchLim(int lim) {
if (axis == 0)
THROW_ERROR << "Dynamic batch is not supported by split layer with axis == 0 parameter";
dynBatchLim = lim;
}
void Split::optimizedNspc2Ncsp(size_t MB) {
auto parentEdge = getParentEdgeAt(0);
const int rank = parentEdge->getMemory().GetShape().getRank();
@ -606,11 +590,8 @@ Split::SplitOptimizedExecutor::SplitOptimizedExecutor(BlockedMemoryDescCPtr inDe
}
}
void Split::SplitOptimizedExecutor::exec(const uint8_t* srcData, const std::vector<uint8_t*>& dstRawMemPtrs,
const Dim origBatch, const Dim perInferBatch) {
void Split::SplitOptimizedExecutor::exec(const uint8_t* srcData, const std::vector<uint8_t*>& dstRawMemPtrs) {
size_t execCountStrides = countStrides;
if (origBatch != perInferBatch)
execCountStrides = execCountStrides / origBatch * perInferBatch;
parallel_for2d(dstRawMemPtrs.size(), execCountStrides, [&](size_t i, size_t j) {
uint8_t* dstData = dstRawMemPtrs[i];

7
src/plugins/intel_cpu/src/nodes/split.h
View File

@ -26,7 +26,6 @@ public:
bool isOptimized() const;
void initOptimalPrimitiveDescriptor() override;
void setDynamicBatchLim(int lim) override;
bool isExecutable() const override;
bool needPrepareParams() const override;
@ -36,8 +35,7 @@ public:
private:
struct SplitExecutor {
virtual void exec(const uint8_t* srcData, const std::vector<uint8_t*>& dstRawMemPtrs,
const Dim origBatch, const Dim perInferBatch) = 0;
virtual void exec(const uint8_t* srcData, const std::vector<uint8_t*>& dstRawMemPtrs) = 0;
virtual ~SplitExecutor() = default;
};
std::shared_ptr<SplitExecutor> execPtr = nullptr;
@ -45,8 +43,7 @@ private:
struct SplitOptimizedExecutor : public SplitExecutor {
public:
SplitOptimizedExecutor(BlockedMemoryDescCPtr inDesc, const std::vector<BlockedMemoryDescCPtr> &outDescs, const size_t axis);
void exec(const uint8_t* srcData, const std::vector<uint8_t*>& dstRawMemPtrs,
const Dim origBatch, const Dim perInferBatch) override;
void exec(const uint8_t* srcData, const std::vector<uint8_t*>& dstRawMemPtrs) override;
private:
std::vector<size_t> dataSize;

1
src/plugins/intel_cpu/src/nodes/strided_slice.cpp
View File

@ -327,7 +327,6 @@ void StridedSlice::initSupportedPrimitiveDescriptors() {
const size_t nDims = getInputShapeAtPort(DATA_ID).getRank();
NodeConfig config;
config.dynBatchSupport = false;
config.inConfs.resize(getParentEdges().size());
config.inConfs[DATA_ID].inPlace(-1);
config.inConfs[BEGIN_ID].inPlace(-1);

1
src/plugins/intel_cpu/src/nodes/subgraph.cpp
View File

@ -172,7 +172,6 @@ void Snippet::initSupportedPrimitiveDescriptors() {
size_t offset = 0;
NodeConfig config;
config.dynBatchSupport = false;
config.inConfs.resize(inputShapes.size());
for (size_t i = 0; i < inputShapes.size(); i++) {
const auto originalInputPrecision = getOriginalInputPrecisionAtPort(i);

1
src/plugins/intel_cpu/src/nodes/tensoriterator.cpp
View File

@ -51,7 +51,6 @@ static NodeConfig make_plain_config(const std::shared_ptr<ov::Node>& op) {
config.outConfs.push_back(data_conf);
}
config.dynBatchSupport = true;
return config;
}

7
src/plugins/intel_cpu/src/nodes/tile.cpp
View File

@ -164,12 +164,7 @@ void Tile::plainExecute(dnnl::stream strm) {
for (int i = axis; i < inDims.size(); i++ )
m_inner_dim *= inDims[i];
int MB = 0;
if (isDynamicNode()) {
MB = srcMemory.getStaticDims()[0];
} else {
MB = batchToProcess();
}
int MB = srcMemory.getStaticDims()[0];
if (axis > 0) {
m_outer_dim /= inDims[0];
m_outer_dim *= MB;

8
src/plugins/intel_cpu/src/nodes/transpose.cpp
View File

@ -133,7 +133,6 @@ void Transpose::initSupportedPrimitiveDescriptors() {
auto& creatorsMap = BlockedDescCreator::getCommonCreators();
NodeConfig config;
config.dynBatchSupport = true;
config.inConfs.resize(2);
config.outConfs.resize(1);
config.inConfs[INPUT_DATA_IDX].inPlace(-1);
@ -382,12 +381,7 @@ void Transpose::execute(dnnl::stream strm) {
auto &dstMemPtr = getChildEdgeAt(0)->getMemoryPtr();
auto &srcMemPtr = getParentEdgeAt(INPUT_DATA_IDX)->getMemoryPtr();
int MB = 0;
if (isDynamicNode()) {
MB = srcMemPtr->getStaticDims()[0];
} else {
MB = batchToProcess();
}
int MB = srcMemPtr->getStaticDims()[0];
execPtr->exec(this, srcMemPtr, dstMemPtr, MB);
} else {

2
src/plugins/intel_cpu/src/nodes/unique.cpp
View File

@ -80,7 +80,7 @@ void Unique::initSupportedPrimitiveDescriptors() {
outPortConfigs.push_back({LayoutType::ncsp, i == 0 ? dataPrecision : axisPrecision});
}
addSupportedPrimDesc(inPortConfigs, outPortConfigs, implType, isDynamicNode());
addSupportedPrimDesc(inPortConfigs, outPortConfigs, implType);
}
void Unique::createPrimitive() {

1
src/plugins/intel_cpu/tests/functional/extension/extension.cpp
View File

@ -21,7 +21,6 @@ public:
InferenceEngine::StatusCode getSupportedConfigurations(std::vector<InferenceEngine::LayerConfig>& conf,
InferenceEngine::ResponseDesc* /*resp*/) noexcept override {
InferenceEngine::LayerConfig layerConfig;
layerConfig.dynBatchSupport = true;
if (node->outputs().size() != 1 && node->inputs().size() != 1)
return InferenceEngine::GENERAL_ERROR;