[IE][VPU][GT]: Fixes Eltwise+ReLU merge for dynamic networks (#2777)

Eltwise + ReLU merge is expected to be performed unconditionally
in all cases and since it does not require strides to be defined
could be performed before adjustDataLayout pass.

Unfortunately, there are cases with unexpected degradation after
such a change is introduced. In specific case it seems to be
caused by degradation in HW operation (convolution). It was not
investigated completely and reason is still unknown (convolution
itself remains unchanged in network, but for some reason works
slower).

It has been decided to introduce change only in case of dynamic
models to have performance benefit for some cases and avoid
degradations in others.

Moving mergeEltwiseAndReLU pass before adjustDataLayout for
dynamic cases allows to get additional performance gain due to
lack of extra copy stages introduced in adjustDataLayout.

Signed-off-by: Gladilov, Gleb <gleb.gladilov@intel.com>

inference-engine/src/vpu/graph_transformer/include/vpu/middleend/pass_manager.hpp

@@ -148,7 +148,8 @@ public:
     //
 
     Pass::Ptr mergeReLUAndBias();
-    Pass::Ptr mergeEltwiseAndReLU();
+    Pass::Ptr mergeEltwiseAndReLUDynamic();
+    Pass::Ptr mergeEltwiseAndReLUStatic();
     Pass::Ptr replaceWithSCReLU();
     Pass::Ptr replaceWithReduceMean();
 

inference-engine/src/vpu/graph_transformer/include/vpu/model/model.hpp

@@ -322,6 +322,9 @@ public:
         return _orderedStageList | asRange();
     }
 
+    bool isDynamic() const;
+    bool isStatic() const;
+
     //
     // Allocator
     //

inference-engine/src/vpu/graph_transformer/src/middleend/pass_manager.cpp

@@ -255,6 +255,9 @@ PassSet::Ptr PassManager::buildMiddleEnd() {
     ADD_PASS(mergeReLUAndBias);
     ADD_DUMP_PASS("mergeReLUAndBias");
 
+    ADD_PASS(mergeEltwiseAndReLUDynamic);
+    ADD_DUMP_PASS("mergeEltwiseAndReLUDynamic");
+
     //
     // Data layout adjustment
     //
@@ -277,8 +280,8 @@ PassSet::Ptr PassManager::buildMiddleEnd() {
     // Model SW-specific optimizations after data layout adjustment
     //
 
-    ADD_PASS(mergeEltwiseAndReLU);
+    ADD_PASS(mergeEltwiseAndReLUStatic);
-    ADD_DUMP_PASS("mergeEltwiseAndReLU");
+    ADD_DUMP_PASS("mergeEltwiseAndReLUStatic");
 
     //
     // Model special stages processing

inference-engine/src/vpu/graph_transformer/src/middleend/passes/merge_eltwise_and_relu.cpp

@@ -13,18 +13,32 @@ namespace vpu {
 
 namespace {
 
+enum class MergeMode {
+    DYNAMIC_NETWORK,
+    STATIC_NETWORK
+};
+
 class PassImpl final : public Pass {
 public:
-    explicit PassImpl(const StageBuilder::Ptr& stageBuilder) : _stageBuilder(stageBuilder) {}
+    explicit PassImpl(MergeMode mode) : m_mode(mode) {}
-
     void run(const Model& model) override;
 
 private:
-    StageBuilder::Ptr _stageBuilder;
+    MergeMode m_mode;
 };
 
 void PassImpl::run(const Model& model) {
-    VPU_PROFILE(mergeEltwiseAndReLU);
+    if (m_mode == MergeMode::DYNAMIC_NETWORK) {
+        VPU_PROFILE(mergeEltwiseAndReLUDynamic);
+        if (model->isStatic()) {
+            return;
+        }
+    } else if (m_mode == MergeMode::STATIC_NETWORK) {
+        VPU_PROFILE(mergeEltwiseAndReLUStatic);
+        if (model->isDynamic()) {
+            return;
+        }
+    }
 
     for (const auto& eltwiseStage : model->getStages()) {
         if (eltwiseStage == nullptr) {
@@ -66,7 +80,7 @@ void PassImpl::run(const Model& model) {
             auto reluInput = reluStage->input(0);
             auto reluOutput = reluStage->output(0);
 
-            if (reluInput->strides() == reluOutput->strides() || reluOutput->checkStrides(StridesRequirement::compact())) {
+            if (model->isDynamic() || reluInput->strides() == reluOutput->strides() || reluOutput->checkStrides(StridesRequirement::compact())) {
                 auto reluStageType = reluStage->type();
                 auto reluStageName = reluStage->name();
 
@@ -90,8 +104,12 @@ void PassImpl::run(const Model& model) {
 
 }  // namespace
 
-Pass::Ptr PassManager::mergeEltwiseAndReLU() {
+Pass::Ptr PassManager::mergeEltwiseAndReLUStatic() {
-    return std::make_shared<PassImpl>(_stageBuilder);
+    return std::make_shared<PassImpl>(MergeMode::STATIC_NETWORK);
+}
+
+Pass::Ptr PassManager::mergeEltwiseAndReLUDynamic() {
+    return std::make_shared<PassImpl>(MergeMode::DYNAMIC_NETWORK);
 }
 
 }  // namespace vpu

inference-engine/src/vpu/graph_transformer/src/model/model.cpp

@@ -2166,4 +2166,14 @@ void ModelObj::removeUnusedData(const Data& data) {
     _dataPtrList.erase(data->_ptrPosInModel);
 }
 
+bool ModelObj::isDynamic() const {
+    const auto& dataObjects = datas();
+    return std::any_of(dataObjects.begin(), dataObjects.end(),
+        [](const Data& data) { return data->parentDataToShapeEdge() || !data->childDataToShapeEdges().empty(); });
+}
+
+bool ModelObj::isStatic() const {
+    return !isDynamic();
+}
+
 }  // namespace vpu