[VPU] Add support for dynamic shape API (#8470)

This PR adds support for new dynamic shape API to MyriadX

inference-engine/src/vpu/myriad_plugin/myriad_executable_network.cpp

@@ -18,6 +18,10 @@
 #include <vpu/configuration/options/log_level.hpp>
 #include <vpu/configuration/options/throughput_streams.hpp>
 #include <vpu/configuration/options/exclusive_async_requests.hpp>
+#include <vpu/ngraph/operations/dynamic_shape_resolver.hpp>
+#include <vpu/ngraph/transformations/dynamic_to_static_shape.hpp>
+#include <ngraph/opsets/opset3.hpp>
+#include "../../../ngraph/core/include/openvino/core/interval.hpp"
 
 using namespace InferenceEngine;
 
@@ -72,8 +76,40 @@ ExecutableNetwork::ExecutableNetwork(
         _config.get<LogLevelOption>(),
         consoleOutput());
 
+    ie::CNNNetwork copyNetwork = network;
+    if (copyNetwork.getFunction() && copyNetwork.getFunction()->is_dynamic()) {
+        copyNetwork = InferenceEngine::details::cloneNetwork(network);
+        auto function = copyNetwork.getFunction();
+        for (const auto& input : function->get_parameters()) {
+            if (input->get_partial_shape().is_dynamic()) {
+                auto inputShape = input->get_partial_shape();
+                const auto inDataParam = std::make_shared<ngraph::opset3::Parameter>(
+                    input->get_output_element_type(0), inputShape.get_max_shape());
+                const auto inDataShapeParam = std::make_shared<ngraph::opset3::Parameter>(
+                    ngraph::element::i32, ov::Shape{inputShape.get_max_shape().size()});
+                inDataShapeParam->set_friendly_name(input->get_friendly_name()+"_real_shape");
+                inDataParam->set_friendly_name(input->get_friendly_name());
+                inDataParam->get_output_tensor(0).set_names(input->get_output_tensor(0).get_names());
+                const auto dsr = std::make_shared<ngraph::vpu::op::DynamicShapeResolver>(
+                    inDataParam, inDataShapeParam, ngraph::vpu::op::DynamicShapeResolverMode::INFER_DYNAMIC_SHAPE);
+                function->replace_node(input, dsr);
+                function->remove_parameter(input);
+                function->add_parameters({inDataShapeParam, inDataParam});
+            }
+        }
+        copyNetwork = ie::CNNNetwork(function);
+        for (const auto& inputInf : network.getInputsInfo()) {
+            auto& copyInput = copyNetwork.getInputsInfo()[inputInf.first];
+            copyInput->setPrecision(inputInf.second->getPrecision());
+            copyInput->setLayout(inputInf.second->getLayout());
+            copyInput->getPreProcess() = inputInf.second->getPreProcess();
+        }
+        for (const auto& outputInf : network.getOutputsInfo()) {
+            *copyNetwork.getOutputsInfo()[outputInf.first].get() = *outputInf.second.get();
+        }
+    }
     auto compiledGraph = compileNetwork(
-        network,
+        copyNetwork,
         _config,
         compilerLog,
         _core);

inference-engine/src/vpu/myriad_plugin/myriad_infer_request.cpp

@@ -166,6 +166,17 @@ void MyriadInferRequest::InferAsync() {
         } else {
             MEMCPY(&inputBuffer[offset], blob->buffer().as<uint8_t*>(), byteSize);
         }
+        const auto offsetShape = inputInfo.offset.find(name+"_real_shape");
+        if (offsetShape == inputInfo.offset.end()) {
+            continue;
+        }
+        auto dimSize = sizeof(int32_t);
+        const auto offsetDims = (offsetShape->second) / dimSize;
+        const auto blobDims = blob->getTensorDesc().getDims();
+        for (size_t i = 0; i < blobDims.size(); ++i) {
+            int32_t dim = static_cast<int32_t>(blobDims[i]);
+            reinterpret_cast<int32_t*>(inputBuffer.data())[offsetDims + i] = dim;
+        }
     }
 
     _executor->queueInference(_graphDesc, inputBuffer.data(),

inference-engine/tests/functional/plugin/myriad/shared_tests_instances/behavior/infer_request_dynamic.cpp

@@ -0,0 +1,88 @@
+// Copyright (C) 2018-2021 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#include <vector>
+
+#include "behavior/ov_infer_request/infer_request_dynamic.hpp"
+#include <vpu/private_plugin_config.hpp>
+#include <vpu/configuration/options/dump_internal_graph_file_name.hpp>
+#include <vpu/configuration/options/dump_all_passes_directory.hpp>
+#include <vpu/configuration/options/dump_all_passes.hpp>
+
+using namespace ov::test::behavior;
+
+namespace {
+
+const std::vector<InferenceEngine::Precision> netPrecisions = {
+    InferenceEngine::Precision::FP16
+};
+
+const std::vector<std::map<std::string, std::string>> configs = {
+    {}
+};
+
+const std::vector<std::map<std::string, std::string>> HeteroConfigs = {
+    {{"TARGET_FALLBACK", CommonTestUtils::DEVICE_CPU}}
+};
+
+std::shared_ptr<ngraph::Function> getFunction1() {
+    const std::vector<size_t> inputShape = {1, 4, 20, 20};
+    const ngraph::element::Type_t ngPrc = ngraph::element::Type_t::f32;
+
+    auto params = ngraph::builder::makeParams(ngPrc, {inputShape});
+    params.front()->set_friendly_name("Param_1");
+    params.front()->get_output_tensor(0).set_names({"input_tensor"});
+
+    auto in2add = ngraph::builder::makeConstant(ngPrc, {1, 4, 1, 1}, std::vector<float>{}, true);
+    auto add = ngraph::builder::makeEltwise(params[0], in2add, ngraph::helpers::EltwiseTypes::ADD);
+    auto relu1 = std::make_shared<ngraph::opset1::Relu>(add->output(0));
+    relu1->get_output_tensor(0).set_names({"relu1"});
+
+    ngraph::NodeVector results{relu1};
+    return std::make_shared<ngraph::Function>(results, params, "AddOneOutputEdge");
+}
+
+std::shared_ptr<ngraph::Function> getFunction2() {
+    const std::vector<size_t> inputShape = {1, 4, 20, 20};
+    const ngraph::element::Type_t ngPrc = ngraph::element::Type_t::f32;
+
+    auto params = ngraph::builder::makeParams(ngPrc, {inputShape});
+    params.front()->set_friendly_name("Param_1");
+    params.front()->get_output_tensor(0).set_names({"input_tensor"});
+    auto split = ngraph::builder::makeSplit(params[0], ngPrc, 2, 1);
+
+    auto in2add = ngraph::builder::makeConstant(ngPrc, {1, 2, 1, 1}, std::vector<float>{}, true);
+    auto add = ngraph::builder::makeEltwise(split->output(0), in2add, ngraph::helpers::EltwiseTypes::ADD);
+    auto relu1 = std::make_shared<ngraph::opset1::Relu>(add);
+
+    auto in2mult = ngraph::builder::makeConstant(ngPrc, {1, 2, 1, 1}, std::vector<float>{}, true);
+    auto mult = ngraph::builder::makeEltwise(split->output(1), in2mult, ngraph::helpers::EltwiseTypes::MULTIPLY);
+    auto relu2 = std::make_shared<ngraph::opset1::Relu>(mult);
+
+    auto concat = std::make_shared<ngraph::opset1::Concat>(ngraph::OutputVector{relu1->output(0), relu2->output(0)}, 3);
+    concat->get_output_tensor(0).set_names({"concat"});
+
+    return std::make_shared<ngraph::Function>(concat, params, "SplitAddConcat");
+}
+
+INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests_1, OVInferRequestDynamicTests,
+                        ::testing::Combine(
+                                ::testing::Values(getFunction1()),
+                                ::testing::Values(std::vector<std::pair<std::vector<size_t>, std::vector<size_t>>>{
+                                    {{1, 4, 20, 20}, {1, 4, 20, 20}},
+                                    {{2, 4, 20, 20}, {2, 4, 20, 20}}}),
+                                ::testing::Values(CommonTestUtils::DEVICE_MYRIAD),
+                                ::testing::ValuesIn(configs)),
+                        OVInferRequestDynamicTests::getTestCaseName);
+
+INSTANTIATE_TEST_SUITE_P(smoke_Hetero_BehaviorTests, OVInferRequestDynamicTests,
+                        ::testing::Combine(
+                                ::testing::Values(getFunction2()),
+                                ::testing::Values(std::vector<std::pair<std::vector<size_t>, std::vector<size_t>>>{
+                                    {{1, 4, 20, 20}, {1, 2, 20, 40}},
+                                    {{2, 4, 20, 20}, {2, 2, 20, 40}}}),
+                                ::testing::Values(CommonTestUtils::DEVICE_HETERO),
+                                ::testing::ValuesIn(HeteroConfigs)),
+                        OVInferRequestDynamicTests::getTestCaseName);
+}  // namespace

inference-engine/tests/functional/plugin/myriad/shared_tests_instances/skip_tests_config.cpp

@@ -15,6 +15,15 @@ std::vector<std::string> disabledTestPatterns() {
         ".*ActivationLayerTest\\.CompareWithRefs/Log.*netPRC=FP32.*",
         ".*ActivationLayerTest\\.CompareWithRefs/Sigmoid.*netPRC=FP32.*",
         ".*ActivationLayerTest\\.CompareWithRefs/Relu.*netPRC=FP32.*",
+        // Not supported dynamic shapes without upper bound
+        ".*InferDynamicNetworkWithGetTensor2times.function.*",
+        ".*InferFullyDynamicNetworkWithGetTensor.function.*",
+        ".*InferDynamicNetworkWithGetTensor.function.*",
+        ".*InferDynamicNetworkWithoutSetShape.function.*",
+        ".*InferFullyDynamicNetworkWithSetTensor/function.*",
+        ".*InferDynamicNetworkWithSetTensor2times.*",
+        ".*InferRequestDynamicTests.GetSameTensor2times.*",
+        ".*InferRequestDynamicTests.InferDynamicNetworkWithSetTensor.*",
         // TODO: Issue: 26268
         ".*ConcatLayerTest.*axis=0.*",
         // TODO: Issue 31197