From ae3e3af521aede85a2411ae680bdec064401e40e Mon Sep 17 00:00:00 2001 From: Alexandra Sidorova Date: Fri, 24 Sep 2021 09:31:41 +0300 Subject: [PATCH] [CPU] Removed optimized out nodes from infer stage (#7440) --- .../src/mkldnn_plugin/mkldnn_graph.cpp | 14 +++++++------- inference-engine/src/mkldnn_plugin/mkldnn_graph.h | 8 +++++--- inference-engine/src/mkldnn_plugin/mkldnn_node.h | 5 +++++ .../src/mkldnn_plugin/nodes/mkldnn_concat_node.h | 3 +++ .../src/mkldnn_plugin/nodes/mkldnn_input_node.h | 3 +++ .../src/mkldnn_plugin/nodes/mkldnn_memory_node.hpp | 3 +++ .../src/mkldnn_plugin/nodes/mkldnn_reorder_node.h | 4 ++++ .../src/mkldnn_plugin/nodes/mkldnn_reshape_node.h | 3 +++ .../src/mkldnn_plugin/nodes/mkldnn_split_node.cpp | 2 +- .../src/mkldnn_plugin/nodes/mkldnn_split_node.h | 5 ++++- 10 files changed, 38 insertions(+), 12 deletions(-) diff --git a/inference-engine/src/mkldnn_plugin/mkldnn_graph.cpp b/inference-engine/src/mkldnn_plugin/mkldnn_graph.cpp index 4bb7253c3a1..d692288e5e2 100644 --- a/inference-engine/src/mkldnn_plugin/mkldnn_graph.cpp +++ b/inference-engine/src/mkldnn_plugin/mkldnn_graph.cpp @@ -344,7 +344,7 @@ void MKLDNNGraph::InitGraph() { graphNode->cleanup(); } #endif - ExtractConstantNodes(); + ExtractConstantAndExecutableNodes(); ExecuteConstantNodesOnly(); } @@ -389,13 +389,13 @@ void MKLDNNGraph::InitOptimalPrimitiveDescriptors() { } } -void MKLDNNGraph::ExtractConstantNodes() { - OV_ITT_SCOPE(FIRST_INFERENCE, itt::domains::MKLDNN_LT, "MKLDNNGraph::ExtractConstantNodes"); - for (auto& graphNode : graphNodes) { +void MKLDNNGraph::ExtractConstantAndExecutableNodes() { + OV_ITT_SCOPE(FIRST_INFERENCE, itt::domains::MKLDNN_LT, "MKLDNNGraph::ExtractConstantAndExecutableNodes"); + for (const auto& graphNode : graphNodes) { if (graphNode->isConstant()) constantGraphNodes.emplace_back(graphNode); - else - mutableGraphNodes.emplace_back(graphNode); + else if (graphNode->isExecutable()) + executableGraphNodes.emplace_back(graphNode); } } @@ -827,7 +827,7 @@ void MKLDNNGraph::Infer(MKLDNNInferRequest* request, int batch) { mkldnn::stream stream(eng); - for (const auto& node : mutableGraphNodes) { + for (const auto& node : executableGraphNodes) { PERF(config.collectPerfCounters, node); if (request) request->ThrowIfCanceled(); diff --git a/inference-engine/src/mkldnn_plugin/mkldnn_graph.h b/inference-engine/src/mkldnn_plugin/mkldnn_graph.h index cc87acd3eaf..4371b6aa6f5 100644 --- a/inference-engine/src/mkldnn_plugin/mkldnn_graph.h +++ b/inference-engine/src/mkldnn_plugin/mkldnn_graph.h @@ -235,7 +235,7 @@ protected: void Allocate(); void AllocateWithReuse(); void CreatePrimitives(); - void ExtractConstantNodes(); + void ExtractConstantAndExecutableNodes(); void ExecuteNode(const MKLDNNNodePtr& node, const mkldnn::stream& stream) const; void ExecuteConstantNodesOnly() const; @@ -247,10 +247,12 @@ private: // TODO: change std::map to std::unordered_map std::map inputNodesMap; std::map outputNodesMap; + // these node pointers (from graphNodes) are to avoid regular checking for - // constant node in ExecuteConstantNodesOnly and Infer methods + // constantness of nodes in ExecuteConstantNodesOnly, Infer methods and calls of + // non-executable (optimized out) nodes, such as Input, Reshape, etc. std::vector constantGraphNodes; - std::vector mutableGraphNodes; + std::vector executableGraphNodes; void EnforceBF16(); }; diff --git a/inference-engine/src/mkldnn_plugin/mkldnn_node.h b/inference-engine/src/mkldnn_plugin/mkldnn_node.h index db47cb905d4..162e33ee19c 100644 --- a/inference-engine/src/mkldnn_plugin/mkldnn_node.h +++ b/inference-engine/src/mkldnn_plugin/mkldnn_node.h @@ -195,6 +195,11 @@ public: return engine; } + // must be called only after MKLDNNGraph::InitEdges() + virtual bool isExecutable() const { + return true; + } + bool isConstant(); bool isInplace() const; diff --git a/inference-engine/src/mkldnn_plugin/nodes/mkldnn_concat_node.h b/inference-engine/src/mkldnn_plugin/nodes/mkldnn_concat_node.h index abb3a38ac17..e9f60b4dec2 100644 --- a/inference-engine/src/mkldnn_plugin/nodes/mkldnn_concat_node.h +++ b/inference-engine/src/mkldnn_plugin/nodes/mkldnn_concat_node.h @@ -27,6 +27,9 @@ public: bool isOptimized() const; InferenceEngine::Precision getRuntimePrecision() const override; + bool isExecutable() const override { + return !isOptimized(); + } private: size_t axis = 0; diff --git a/inference-engine/src/mkldnn_plugin/nodes/mkldnn_input_node.h b/inference-engine/src/mkldnn_plugin/nodes/mkldnn_input_node.h index 9e32f9bd0e4..b71cc6c79bf 100644 --- a/inference-engine/src/mkldnn_plugin/nodes/mkldnn_input_node.h +++ b/inference-engine/src/mkldnn_plugin/nodes/mkldnn_input_node.h @@ -26,6 +26,9 @@ public: MKLDNNMemoryCPtr getMemoryPtr() const; void executeDynamicImpl(mkldnn::stream strm) override {} + bool isExecutable() const override { + return false; + } std::vector shapeInfer() const override { return std::vector(); diff --git a/inference-engine/src/mkldnn_plugin/nodes/mkldnn_memory_node.hpp b/inference-engine/src/mkldnn_plugin/nodes/mkldnn_memory_node.hpp index c7810ba1c9a..5d98378bd38 100644 --- a/inference-engine/src/mkldnn_plugin/nodes/mkldnn_memory_node.hpp +++ b/inference-engine/src/mkldnn_plugin/nodes/mkldnn_memory_node.hpp @@ -89,6 +89,9 @@ public: bool created() const override { return getType() == MemoryInput; } + bool isExecutable() const override { + return true; + } void execute(mkldnn::stream strm) override; void createPrimitive() override; diff --git a/inference-engine/src/mkldnn_plugin/nodes/mkldnn_reorder_node.h b/inference-engine/src/mkldnn_plugin/nodes/mkldnn_reorder_node.h index 51ee28a5614..5ebcc03b2f6 100644 --- a/inference-engine/src/mkldnn_plugin/nodes/mkldnn_reorder_node.h +++ b/inference-engine/src/mkldnn_plugin/nodes/mkldnn_reorder_node.h @@ -25,6 +25,10 @@ public: bool created() const override; const std::vector& getPrimitivesPriority() override; + bool isExecutable() const override { + return !isOptimized; + } + void setDescs(const MemoryDesc& input, const MemoryDesc& output) { this->input = input.clone(); inputShapes.clear(); diff --git a/inference-engine/src/mkldnn_plugin/nodes/mkldnn_reshape_node.h b/inference-engine/src/mkldnn_plugin/nodes/mkldnn_reshape_node.h index c262e02a87b..db14860a9e5 100644 --- a/inference-engine/src/mkldnn_plugin/nodes/mkldnn_reshape_node.h +++ b/inference-engine/src/mkldnn_plugin/nodes/mkldnn_reshape_node.h @@ -26,6 +26,9 @@ public: void initSupportedPrimitiveDescriptors() override; void createPrimitive() override; bool created() const override; + bool isExecutable() const override { + return false; + } static bool isSupportedOperation(const std::shared_ptr& op, std::string& errorMessage) noexcept; }; diff --git a/inference-engine/src/mkldnn_plugin/nodes/mkldnn_split_node.cpp b/inference-engine/src/mkldnn_plugin/nodes/mkldnn_split_node.cpp index b815b02111a..d7bf08a11d3 100644 --- a/inference-engine/src/mkldnn_plugin/nodes/mkldnn_split_node.cpp +++ b/inference-engine/src/mkldnn_plugin/nodes/mkldnn_split_node.cpp @@ -298,7 +298,7 @@ bool MKLDNNSplitNode::created() const { return getType() == Split; } -bool MKLDNNSplitNode::isOptimized() { +bool MKLDNNSplitNode::isOptimized() const { return getSelectedPrimitiveDescriptor() && getSelectedPrimitiveDescriptor()->getConfig().outConfs[0].inPlace >= 0; } diff --git a/inference-engine/src/mkldnn_plugin/nodes/mkldnn_split_node.h b/inference-engine/src/mkldnn_plugin/nodes/mkldnn_split_node.h index 8428e20a9fd..e7f99105d1d 100644 --- a/inference-engine/src/mkldnn_plugin/nodes/mkldnn_split_node.h +++ b/inference-engine/src/mkldnn_plugin/nodes/mkldnn_split_node.h @@ -22,10 +22,13 @@ public: void execute(mkldnn::stream strm) override; bool created() const override; - bool isOptimized(); + bool isOptimized() const; void initOptimalPrimitiveDescriptor() override; void setDynamicBatchLim(int lim) override; + bool isExecutable() const override { + return !isOptimized(); + } private: void prepareOptimizedParams();