[IE][VPU]: Enables dynamic output from middle of network support (#930)

* [IE][VPU]: Enables dynamic output from middle of network support

This feature is very useful for debugging dynamic networks.
Changes include modification of existing addCopyForOutputsInsideNetwork
pass to respect dynamic outputs and moving propagateDynamismToOutputs
pass after addCopyForOutputsInsideNetwork. The motivation for last change
is to avoid unnecessary copy stages due to not synchronized logic, because
previously:

* First in Front-End (parseDSR) we mark shape data object as output
* Then in propagateDynamismToOutputs we insert copy stage for that case.
  It's necessary if shape data object had other consumers
* Then in convertShapeNotation we insert Gather consumer for output data object
* Finally, addCopyForOutputsInsideNetwork inserts one more copy stage to leave
  output data object without consumers.

Signed-off-by: Gladilov, Gleb <gleb.gladilov@intel.com>

* [IE][VPU]: Replaces attrs.has + attrs.get with attrs.getOrDefault

* [IE][VPU]: Fixes setting IE-notation and converted-notation to the same data object

inference-engine/src/vpu/graph_transformer/src/middleend/pass_manager.cpp

@@ -90,15 +90,6 @@ PassSet::Ptr PassManager::buildMiddleEnd() {
     //
     // Convert shape notation
     //
-
-
-    // MyriadInferRequest::GetResult expects output shape data object
-    // to be in IE notation in case of dynamic data object
-    // propagateDynamismToOutputs must be applied before convertShapeNotation
-    // to mark shape in IE notation, not MDK notation as output
-    ADD_PASS(propagateDynamismToOutputs);
-    ADD_DUMP_PASS("propagateDynamismToOutputs");
-
     ADD_PASS(convertShapeNotation);
     ADD_DUMP_PASS("convertShapeNotation");
 
@@ -120,6 +111,13 @@ PassSet::Ptr PassManager::buildMiddleEnd() {
     ADD_PASS(addCopyForOutputsInsideNetwork);
     ADD_DUMP_PASS("addCopyForOutputsInsideNetwork");
 
+    // MyriadInferRequest::GetResult expects output shape data object
+    // to be in IE notation in case of dynamic data object
+    // propagateDynamismToOutputs must be applied after convertShapeNotation
+    // and addCopyForOutputsInsideNetwork to mark shape in IE notation, not MDK notation as output
+    ADD_PASS(propagateDynamismToOutputs);
+    ADD_DUMP_PASS("propagateDynamismToOutputs");
+
     ADD_PASS(initialCheck);
 
     //

inference-engine/src/vpu/graph_transformer/src/middleend/passes/add_copy_for_outputs_inside_network.cpp

@@ -14,13 +14,18 @@ public:
     explicit PassImpl(const StageBuilder::Ptr& stageBuilder) : _stageBuilder(stageBuilder) {}
 
     void run(const Model& model) override {
-        VPU_PROFILE(initialCheck);
+        VPU_PROFILE(addCopyForOutputsInsideNetwork);
 
         for (const auto& outputData : model->datas()) {
             if (outputData->usage() != DataUsage::Output || outputData->numConsumers() == 0) {
                 continue;
             }
 
+            VPU_THROW_UNLESS(outputData->childDataToShapeEdges().empty(),
+                "Output data object cannot be a shape parent for another data object, since notation conversion "
+                " has already happened, but {} with usage {} has {} data to shape children",
+                outputData->name(), outputData->usage(), outputData->childDataToShapeEdges().size());
+
             auto newIntermediateData = model->duplicateData(
                 outputData,
                 "@intermediate",
@@ -39,6 +44,10 @@ public:
                 newIntermediateData,
                 outputData,
                 "addCopyForOutputsInsideNetwork");
+
+            if (const auto& parentShapeEdge = outputData->parentDataToShapeEdge()) {
+                model->connectDataWithShape(parentShapeEdge->parent(), newIntermediateData);
+            }
         }
     }
 

inference-engine/src/vpu/graph_transformer/src/middleend/passes/convert_shape_notation.cpp

@@ -35,6 +35,11 @@ void PassImpl::run(const Model& model) {
         // Revert shape from IE to MDK notation
         auto convertedShape = model->duplicateData(shape, "@converted-notation");
 
+        // Settings IE-notation attribute to shape must be done after duplicateData
+        // Since duplicateData does deep attributes copy
+        shape->attrs().set<bool>("IE-notation", true);
+        convertedShape->attrs().set<bool>("converted-notation", true);
+
         const auto generator = [&convertedShape](const ie::Blob::Ptr& blob) {
             std::vector<int32_t> gatherIndices(static_cast<size_t>(convertedShape->desc().totalDimSize()));
             std::iota(gatherIndices.rbegin(), gatherIndices.rend(), 0);

inference-engine/src/vpu/graph_transformer/src/middleend/passes/propagate_dynamism_to_outputs.cpp

@@ -39,34 +39,35 @@ public:
             }
             const auto parent = parentDataToShapeEdge->parent();
 
+            const auto& parentAttrs = parent->attrs();
+            VPU_THROW_UNLESS(parentAttrs.getOrDefault("converted-notation", false),
+                "All shape parent data object must be already converted to MDK notation, but {} is in IE notation",
+                parent->name());
+
+            const auto& parentInIENotation = parent->producer()->input(0);
+            const auto& parentInIENotationAttrs = parentInIENotation->attrs();
+            VPU_THROW_UNLESS(parentInIENotationAttrs.getOrDefault("IE-notation", false),
+                 "Data object {} is expected to be shape in IE notation, but is not marked as it",
+                 parentInIENotation->name());
+
+            VPU_THROW_UNLESS(parentInIENotation->usage() == DataUsage::Intermediate,
+                "Shape data object in IE notation {} is expected to be an {} data object, but it has usage {}",
+                parentInIENotation->name(), DataUsage::Intermediate, parentInIENotation->usage());
+
+            model->connectDataWithShape(parent, data);
+
             // MyriadInferRequest::GetResult assumes that dynamic data object has shape data object
             // with the same name + suffix "@shape"
             const auto shapeName = data->name() + "@shape";
-            const auto& shapeOutput = model->addOutputData(shapeName, parent->desc());
+            const auto& shapeOutput = model->addOutputData(shapeName, parentInIENotation->desc());
 
-            if (parent->numConsumers() > 0) {
+            _stageBuilder->addCopyStage(
-                _stageBuilder->addCopyStage(
+                model,
-                    model,
+                "copy-for-dynamic-output",
-                    "copy-for-dynamic-output",
+                nullptr,
-                    nullptr,
+                parentInIENotation,
-                    parent,
+                shapeOutput,
-                    shapeOutput,
+                "PropagateDynamismToOutput");
-                    "PropagateDynamismToOutput");
-
-            } else {
-                const auto parentProducerEdge = parent->producerEdge();
-                VPU_THROW_UNLESS(parentProducerEdge != nullptr,
-                    "Data containing shape is expected to have a producer, but {} doesn't have", parent->name());
-
-                for (const auto& dataToShapeEdge : parent->childDataToShapeEdges()) {
-                    model->replaceDataToShapeParent(dataToShapeEdge, shapeOutput);
-                }
-
-                model->replaceStageOutput(parentProducerEdge, shapeOutput);
-                model->removeUnusedData(parent);
-            }
-
-            model->connectDataWithShape(shapeOutput, data);
         }
     }