From 4a859833ffc744f219a1311774c19306cbadc6f5 Mon Sep 17 00:00:00 2001 From: "Gladilov, Gleb" Date: Tue, 16 Jun 2020 16:17:36 +0300 Subject: [PATCH] [IE][VPU]: Enables dynamic output from middle of network support (#930) * [IE][VPU]: Enables dynamic output from middle of network support This feature is very useful for debugging dynamic networks. Changes include modification of existing addCopyForOutputsInsideNetwork pass to respect dynamic outputs and moving propagateDynamismToOutputs pass after addCopyForOutputsInsideNetwork. The motivation for last change is to avoid unnecessary copy stages due to not synchronized logic, because previously: * First in Front-End (parseDSR) we mark shape data object as output * Then in propagateDynamismToOutputs we insert copy stage for that case. It's necessary if shape data object had other consumers * Then in convertShapeNotation we insert Gather consumer for output data object * Finally, addCopyForOutputsInsideNetwork inserts one more copy stage to leave output data object without consumers. Signed-off-by: Gladilov, Gleb * [IE][VPU]: Replaces attrs.has + attrs.get with attrs.getOrDefault * [IE][VPU]: Fixes setting IE-notation and converted-notation to the same data object --- .../src/middleend/pass_manager.cpp | 16 +++--- .../add_copy_for_outputs_inside_network.cpp | 11 ++++- .../passes/convert_shape_notation.cpp | 5 ++ .../passes/propagate_dynamism_to_outputs.cpp | 49 ++++++++++--------- 4 files changed, 47 insertions(+), 34 deletions(-) diff --git a/inference-engine/src/vpu/graph_transformer/src/middleend/pass_manager.cpp b/inference-engine/src/vpu/graph_transformer/src/middleend/pass_manager.cpp index d2108ae4364..b18038aeb2e 100644 --- a/inference-engine/src/vpu/graph_transformer/src/middleend/pass_manager.cpp +++ b/inference-engine/src/vpu/graph_transformer/src/middleend/pass_manager.cpp @@ -90,15 +90,6 @@ PassSet::Ptr PassManager::buildMiddleEnd() { // // Convert shape notation // - - - // MyriadInferRequest::GetResult expects output shape data object - // to be in IE notation in case of dynamic data object - // propagateDynamismToOutputs must be applied before convertShapeNotation - // to mark shape in IE notation, not MDK notation as output - ADD_PASS(propagateDynamismToOutputs); - ADD_DUMP_PASS("propagateDynamismToOutputs"); - ADD_PASS(convertShapeNotation); ADD_DUMP_PASS("convertShapeNotation"); @@ -120,6 +111,13 @@ PassSet::Ptr PassManager::buildMiddleEnd() { ADD_PASS(addCopyForOutputsInsideNetwork); ADD_DUMP_PASS("addCopyForOutputsInsideNetwork"); + // MyriadInferRequest::GetResult expects output shape data object + // to be in IE notation in case of dynamic data object + // propagateDynamismToOutputs must be applied after convertShapeNotation + // and addCopyForOutputsInsideNetwork to mark shape in IE notation, not MDK notation as output + ADD_PASS(propagateDynamismToOutputs); + ADD_DUMP_PASS("propagateDynamismToOutputs"); + ADD_PASS(initialCheck); // diff --git a/inference-engine/src/vpu/graph_transformer/src/middleend/passes/add_copy_for_outputs_inside_network.cpp b/inference-engine/src/vpu/graph_transformer/src/middleend/passes/add_copy_for_outputs_inside_network.cpp index a541e11be20..859b267f1a1 100644 --- a/inference-engine/src/vpu/graph_transformer/src/middleend/passes/add_copy_for_outputs_inside_network.cpp +++ b/inference-engine/src/vpu/graph_transformer/src/middleend/passes/add_copy_for_outputs_inside_network.cpp @@ -14,13 +14,18 @@ public: explicit PassImpl(const StageBuilder::Ptr& stageBuilder) : _stageBuilder(stageBuilder) {} void run(const Model& model) override { - VPU_PROFILE(initialCheck); + VPU_PROFILE(addCopyForOutputsInsideNetwork); for (const auto& outputData : model->datas()) { if (outputData->usage() != DataUsage::Output || outputData->numConsumers() == 0) { continue; } + VPU_THROW_UNLESS(outputData->childDataToShapeEdges().empty(), + "Output data object cannot be a shape parent for another data object, since notation conversion " + " has already happened, but {} with usage {} has {} data to shape children", + outputData->name(), outputData->usage(), outputData->childDataToShapeEdges().size()); + auto newIntermediateData = model->duplicateData( outputData, "@intermediate", @@ -39,6 +44,10 @@ public: newIntermediateData, outputData, "addCopyForOutputsInsideNetwork"); + + if (const auto& parentShapeEdge = outputData->parentDataToShapeEdge()) { + model->connectDataWithShape(parentShapeEdge->parent(), newIntermediateData); + } } } diff --git a/inference-engine/src/vpu/graph_transformer/src/middleend/passes/convert_shape_notation.cpp b/inference-engine/src/vpu/graph_transformer/src/middleend/passes/convert_shape_notation.cpp index a12ab97a905..4f1c4c5c92e 100644 --- a/inference-engine/src/vpu/graph_transformer/src/middleend/passes/convert_shape_notation.cpp +++ b/inference-engine/src/vpu/graph_transformer/src/middleend/passes/convert_shape_notation.cpp @@ -35,6 +35,11 @@ void PassImpl::run(const Model& model) { // Revert shape from IE to MDK notation auto convertedShape = model->duplicateData(shape, "@converted-notation"); + // Settings IE-notation attribute to shape must be done after duplicateData + // Since duplicateData does deep attributes copy + shape->attrs().set("IE-notation", true); + convertedShape->attrs().set("converted-notation", true); + const auto generator = [&convertedShape](const ie::Blob::Ptr& blob) { std::vector gatherIndices(static_cast(convertedShape->desc().totalDimSize())); std::iota(gatherIndices.rbegin(), gatherIndices.rend(), 0); diff --git a/inference-engine/src/vpu/graph_transformer/src/middleend/passes/propagate_dynamism_to_outputs.cpp b/inference-engine/src/vpu/graph_transformer/src/middleend/passes/propagate_dynamism_to_outputs.cpp index 555eb842b59..1110fab520e 100644 --- a/inference-engine/src/vpu/graph_transformer/src/middleend/passes/propagate_dynamism_to_outputs.cpp +++ b/inference-engine/src/vpu/graph_transformer/src/middleend/passes/propagate_dynamism_to_outputs.cpp @@ -39,34 +39,35 @@ public: } const auto parent = parentDataToShapeEdge->parent(); + const auto& parentAttrs = parent->attrs(); + VPU_THROW_UNLESS(parentAttrs.getOrDefault("converted-notation", false), + "All shape parent data object must be already converted to MDK notation, but {} is in IE notation", + parent->name()); + + const auto& parentInIENotation = parent->producer()->input(0); + const auto& parentInIENotationAttrs = parentInIENotation->attrs(); + VPU_THROW_UNLESS(parentInIENotationAttrs.getOrDefault("IE-notation", false), + "Data object {} is expected to be shape in IE notation, but is not marked as it", + parentInIENotation->name()); + + VPU_THROW_UNLESS(parentInIENotation->usage() == DataUsage::Intermediate, + "Shape data object in IE notation {} is expected to be an {} data object, but it has usage {}", + parentInIENotation->name(), DataUsage::Intermediate, parentInIENotation->usage()); + + model->connectDataWithShape(parent, data); + // MyriadInferRequest::GetResult assumes that dynamic data object has shape data object // with the same name + suffix "@shape" const auto shapeName = data->name() + "@shape"; - const auto& shapeOutput = model->addOutputData(shapeName, parent->desc()); + const auto& shapeOutput = model->addOutputData(shapeName, parentInIENotation->desc()); - if (parent->numConsumers() > 0) { - _stageBuilder->addCopyStage( - model, - "copy-for-dynamic-output", - nullptr, - parent, - shapeOutput, - "PropagateDynamismToOutput"); - - } else { - const auto parentProducerEdge = parent->producerEdge(); - VPU_THROW_UNLESS(parentProducerEdge != nullptr, - "Data containing shape is expected to have a producer, but {} doesn't have", parent->name()); - - for (const auto& dataToShapeEdge : parent->childDataToShapeEdges()) { - model->replaceDataToShapeParent(dataToShapeEdge, shapeOutput); - } - - model->replaceStageOutput(parentProducerEdge, shapeOutput); - model->removeUnusedData(parent); - } - - model->connectDataWithShape(shapeOutput, data); + _stageBuilder->addCopyStage( + model, + "copy-for-dynamic-output", + nullptr, + parentInIENotation, + shapeOutput, + "PropagateDynamismToOutput"); } }