auto dynamic shape (#8731)

* initial commit Signed-off-by: fishbell <bell.song@intel.com> * overwrite GetBlob/SetBlob to support dynamic shape when needed Signed-off-by: fishbell <bell.song@intel.com> * extend workers Signed-off-by: fishbell <bell.song@intel.com> * clean up code Signed-off-by: fishbell <bell.song@intel.com> * fix lock and add case for CPU,GPU Signed-off-by: fishbell <bell.song@intel.com> * fix mem leak Signed-off-by: fishbell <bell.song@intel.com> * clean up allocated worker Signed-off-by: fishbell <bell.song@intel.com> * fix test case failure Signed-off-by: fishbell <bell.song@intel.com> * enable case for chaining Signed-off-by: fishbell <bell.song@intel.com> * fix arm compile issue Signed-off-by: fishbell <bell.song@intel.com> * clang format, fix potential threading issue Signed-off-by: fishbell <bell.song@intel.com> * clean up idle request queue if infer request destroyed Signed-off-by: fishbell <bell.song@intel.com> * keep multi legacy behavior Signed-off-by: fishbell <bell.song@intel.com> * elegant way to handle recycling Signed-off-by: fishbell <bell.song@intel.com> * avoid the exception with set output blob in auto dynamic case Signed-off-by: fishbell <bell.song@intel.com> * remove increase workers logic Signed-off-by: fishbell <bell.song@intel.com> * formatting Signed-off-by: fishbell <bell.song@intel.com> * remove the irrelevant code to dynamic shape Signed-off-by: fishbell <bell.song@intel.com> * filter target device to CPU in the case of dynamic shape Signed-off-by: fishbell <bell.song@intel.com> * fix logic in selectdevice, add case for GPU Signed-off-by: fishbell <bell.song@intel.com> * add case for AUTO:GPU Signed-off-by: fishbell <bell.song@intel.com> * fix build error Signed-off-by: fishbell <bell.song@intel.com> * add comment for skipped cases Signed-off-by: fishbell <bell.song@intel.com> * fix the case failure caused by naming conflict Signed-off-by: fishbell <bell.song@intel.com> * move file to new folder Signed-off-by: fishbell <bell.song@intel.com> * change to 2.0 API in plugin Signed-off-by: fishbell <bell.song@intel.com> * refactor mkldnn code change Signed-off-by: fishbell <bell.song@intel.com> * formatting Signed-off-by: fishbell <bell.song@intel.com>
2021-12-20 18:06:50 +03:00
1 parent cf92c54a12
commit 1901f33bd3
@@ -468,14 +468,16 @@ void MKLDNNPlugin::MKLDNNInferRequest::SetBlob(const std::string& name, const In
            IE_THROW(ParameterMismatch) << "Failed to set output blob with precision: "
                               << blobDesc.getPrecision() << ", if CNNNetwork output blob precision is: " << foundOutput->getPrecision();
        }
-        size_t outputSize = foundOutput->getTensorDesc().getLayout() != InferenceEngine::Layout::SCALAR
-            ? InferenceEngine::details::product(foundOutput->getDims())
-            : 1;

        const bool isDynamic = outputNode->isDynamicNode();
-        if (!isDynamic && dataSize != outputSize) {
-            IE_THROW() << "Output blob size is not equal network output size ("
-                               << dataSize << "!=" << outputSize << ").";
+        if (!isDynamic) {
+            size_t outputSize = foundOutput->getTensorDesc().getLayout() != InferenceEngine::Layout::SCALAR
+                ? InferenceEngine::details::product(foundOutput->getDims())
+                : 1;
+            if (dataSize != outputSize) {
+                IE_THROW() << "Output blob size is not equal network output size ("
+                                   << dataSize << "!=" << outputSize << ").";
+            }
        }
        if (!isDynamic && foundOutput->getTensorDesc().getDims() != blobDesc.getDims()) {
            IE_THROW(ParameterMismatch) << "Failed to set output Blob. Dimensions mismatch.";
@@ -103,6 +103,8 @@ void IInferRequestInternal::SetBlob(const std::string& name, const Blob::Ptr& us
        !((foundInput && foundInput->getInputData()->isDynamic()) || (foundOutput && foundOutput->isDynamic()))) {
        IE_THROW() << "Input data is empty. Input name: \'" << name << "\'";
    }
+    const bool isInputDynamic = foundInput && foundInput->getInputData()->isDynamic();
+    const bool isOutputDynamic = foundOutput && foundOutput->isDynamic();
    IE_SUPPRESS_DEPRECATED_END

    size_t dataSize = userBlob->size();
@@ -126,7 +128,7 @@ void IInferRequestInternal::SetBlob(const std::string& name, const Blob::Ptr& us
            size_t inputSize = foundInput->getTensorDesc().getLayout() != InferenceEngine::Layout::SCALAR
                                   ? InferenceEngine::details::product(foundInput->getTensorDesc().getDims())
                                   : 1;
-            if (dataSize != inputSize) {
+            if (!isInputDynamic && dataSize != inputSize) {
                IE_THROW() << "Input blob size is not equal network input size (" << dataSize << "!=" << inputSize
                           << ").";
            }
@@ -140,7 +142,7 @@ void IInferRequestInternal::SetBlob(const std::string& name, const Blob::Ptr& us
        size_t outputSize = foundOutput->getTensorDesc().getLayout() != InferenceEngine::Layout::SCALAR
                                ? details::product(foundOutput->getTensorDesc().getDims())
                                : 1;
-        if (dataSize != outputSize) {
+        if (!isOutputDynamic && dataSize != outputSize) {
            IE_THROW() << "Output blob size is not equal network output size (" << dataSize << "!=" << outputSize
                       << ").";
        }
@@ -164,6 +166,9 @@ Blob::Ptr IInferRequestInternal::GetBlob(const std::string& name) {
    DataPtr foundOutput;
    const SizeVector oneVector = {1};
    if (findInputAndOutputBlobByName(name, foundInput, foundOutput)) {
+        IE_SUPPRESS_DEPRECATED_START
+        const bool isInputDynamic = foundInput && foundInput->getInputData()->isDynamic();
+        IE_SUPPRESS_DEPRECATED_END
        // ROI blob is returned only if it was set previously. Otherwise default blob is returned.
        auto it = _preProcData.find(name);
        if (it != _preProcData.end()) {
@@ -171,7 +176,10 @@ Blob::Ptr IInferRequestInternal::GetBlob(const std::string& name) {
        } else {
            data = _inputs[name];
            const auto& dims = foundInput->getTensorDesc().getDims();
-            checkBlob(data, name, true, foundInput->getTensorDesc().getLayout() != SCALAR ? dims : oneVector);
+            if (isInputDynamic)
+                checkBlob(data, name, true);
+            else
+                checkBlob(data, name, true, foundInput->getTensorDesc().getLayout() != SCALAR ? dims : oneVector);

            auto& devBlob = _deviceInputs[name];
            if (preProcessingRequired(foundInput, data, devBlob)) {
@@ -180,9 +188,15 @@ Blob::Ptr IInferRequestInternal::GetBlob(const std::string& name) {
            }
        }
    } else {
+        IE_SUPPRESS_DEPRECATED_START
+        const bool isOutputDynamic = foundOutput && foundOutput->isDynamic();
+        IE_SUPPRESS_DEPRECATED_END
        data = _outputs[name];
        const auto& dims = foundOutput->getTensorDesc().getDims();
-        checkBlob(data, name, false, foundOutput->getTensorDesc().getLayout() != SCALAR ? dims : oneVector);
+        if (isOutputDynamic)
+            checkBlob(data, name, false);
+        else
+            checkBlob(data, name, false, foundOutput->getTensorDesc().getLayout() != SCALAR ? dims : oneVector);
    }
    return data;
 }
@@ -269,10 +269,11 @@ IExecutableNetworkInternal::Ptr MultiDeviceInferencePlugin::LoadNetworkImpl(cons
        // filter the device that supports filter configure
        auto strDevices = GetDeviceList(fullConfig);
        auto metaDevices = ParseMetaDevices(strDevices, fullConfig);
-        auto supportDevices = FilterDevice(metaDevices, filterConfig);
-        if (supportDevices.size() == 0) {
+        auto supportDevicesByConfig = FilterDevice(metaDevices, filterConfig);
+        if (supportDevicesByConfig.size() == 0) {
             IE_THROW() << "there is no device support the configure";
        }
+        auto supportDevices = FilterDeviceByNetwork(supportDevicesByConfig, network);
        // replace the configure with configure that auto want to pass to device
        // and reset the strDevices to support devices
        auto validConfigKey = PerfHintsConfig::SupportedKeys();
@@ -636,4 +637,21 @@ std::vector<DeviceInformation> MultiDeviceInferencePlugin::FilterDevice(const st
    }
    return filterDevice;
 }
+std::vector<DeviceInformation> MultiDeviceInferencePlugin::FilterDeviceByNetwork(const std::vector<DeviceInformation>& metaDevices,
+                                                InferenceEngine::CNNNetwork network) {
+    std::vector<DeviceInformation> filterDevice;
+    auto model = network.getFunction();
+    if (model->is_dynamic()) {
+        for (auto& iter : metaDevices) {
+            if (iter.deviceName.find("CPU") != std::string::npos) {
+                filterDevice.push_back(iter);
+                break;
+            }
+        }
+        if (filterDevice.size() == 0)
+            IE_THROW(NotFound) << "No available device for dynamic shape network !";
+        return filterDevice;
+    }
+    return metaDevices;
+}
 }  // namespace MultiDevicePlugin
@@ -64,6 +64,8 @@ private:
                            std::map<std::string, std::string>& filterConfig);
    std::vector<DeviceInformation> FilterDevice(const std::vector<DeviceInformation>& metaDevices,
                                                const std::map<std::string, std::string>& config);
+    std::vector<DeviceInformation> FilterDeviceByNetwork(const std::vector<DeviceInformation>& metaDevices,
+                                                InferenceEngine::CNNNetwork network);
    static std::mutex _mtx;
    static std::map<unsigned int, std::list<std::string>> _priorityMap;
 };
@@ -18,6 +18,10 @@ const std::vector<std::map<std::string, std::string>> HeteroConfigs = {
    {{"TARGET_FALLBACK", CommonTestUtils::DEVICE_CPU}}
 };

+const std::vector<std::map<std::string, std::string>> AutoConfigs = {
+    {{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES, CommonTestUtils::DEVICE_CPU}}
+};
+
 std::shared_ptr<ngraph::Function> getFunction1() {
    const std::vector<size_t> inputShape = {1, 4, 20, 20};
    const ngraph::element::Type_t ngPrc = ngraph::element::Type_t::f32;
@@ -90,4 +94,14 @@ INSTANTIATE_TEST_SUITE_P(smoke_Hetero_BehaviorTests, OVInferRequestDynamicTests,
                                ::testing::ValuesIn(HeteroConfigs)),
                        OVInferRequestDynamicTests::getTestCaseName);

+INSTANTIATE_TEST_SUITE_P(smoke_Auto_BehaviorTests, OVInferRequestDynamicTests,
+                        ::testing::Combine(
+                                ::testing::Values(getFunction2()),
+                                ::testing::Values(std::vector<std::pair<std::vector<size_t>, std::vector<size_t>>>{
+                                    {{1, 4, 20, 20}, {1, 2, 20, 40}},
+                                    {{2, 4, 20, 20}, {2, 2, 20, 40}}}),
+                                ::testing::Values(CommonTestUtils::DEVICE_AUTO),
+                                ::testing::ValuesIn(AutoConfigs)),
+                        OVInferRequestDynamicTests::getTestCaseName);
+
 }  // namespace
@@ -17,6 +17,10 @@ const std::vector<std::map<std::string, std::string>> HeteroConfigs = {
    {{"TARGET_FALLBACK", CommonTestUtils::DEVICE_CPU}}
 };

+const std::vector<std::map<std::string, std::string>> AutoConfigs = {
+    {{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES, CommonTestUtils::DEVICE_CPU}}
+};
+
 INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVInferenceChaining,
                        ::testing::Combine(
                                ::testing::Values(CommonTestUtils::DEVICE_CPU),
@@ -29,4 +33,9 @@ INSTANTIATE_TEST_SUITE_P(smoke_Hetero_BehaviorTests, OVInferenceChaining,
                                ::testing::ValuesIn(HeteroConfigs)),
                        OVInferenceChaining::getTestCaseName);

+INSTANTIATE_TEST_SUITE_P(smoke_Auto_BehaviorTests, OVInferenceChaining,
+                        ::testing::Combine(
+                                ::testing::Values(CommonTestUtils::DEVICE_AUTO),
+                                ::testing::ValuesIn(AutoConfigs)),
+                        OVInferenceChaining::getTestCaseName);
 }  // namespace
@@ -109,11 +109,13 @@ std::vector<std::string> disabledTestPatterns() {
        // Issue: CVS-66778
        R"(.*smoke_BehaviorTests.*InferFullyDynamicNetworkWith(S|G)etTensor.*)",
        R"(.*smoke_Hetero_BehaviorTests.*InferFullyDynamicNetworkWith(S|G)etTensor.*)",
+        R"(.*smoke_Auto_BehaviorTests.*InferFullyDynamicNetworkWith(S|G)etTensor.*)",
        R"(.*smoke_BehaviorTests.*DynamicOutputToDynamicInput.*)",
        R"(.*smoke_BehaviorTests.*DynamicInputToDynamicOutput.*)",
        R"(.*smoke_Hetero_BehaviorTests.*DynamicOutputToDynamicInput.*)",
        R"(.*smoke_Hetero_BehaviorTests.*DynamicInputToDynamicOutput.*)",
-
+        R"(.*smoke_Auto_BehaviorTests.*DynamicOutputToDynamicInput.*)",
+        R"(.*smoke_Auto_BehaviorTests.*DynamicInputToDynamicOutput.*)",
        // CPU dynamism: empty tensor returns size() == 1. Looks like layout is SCALAR
        // Issue: CVS-66780
        R"(.*smoke_BehaviorTests.*InferUpperBoundNetworkWithGetTensor.*)",
@@ -0,0 +1,70 @@
+// Copyright (C) 2018-2021 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#include <vector>
+
+#include "behavior/ov_infer_request/infer_request_dynamic.hpp"
+#include "behavior/ov_infer_request/inference_chaining.hpp"
+
+using namespace ov::test::behavior;
+
+namespace {
+
+const std::vector<std::map<std::string, std::string>> AutoConfigs = {
+    {{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES,
+    CommonTestUtils::DEVICE_GPU + std::string(",") + CommonTestUtils::DEVICE_CPU}},
+    {}
+};
+const std::vector<std::map<std::string, std::string>> AutoNotSupportConfigs = {
+    {{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES, CommonTestUtils::DEVICE_GPU}}
+};
+
+std::shared_ptr<ngraph::Function> getFunction2() {
+    const std::vector<size_t> inputShape = {1, 4, 20, 20};
+    const ngraph::element::Type_t ngPrc = ngraph::element::Type_t::f32;
+
+    auto params = ngraph::builder::makeParams(ngPrc, {inputShape});
+    params.front()->set_friendly_name("Param_1");
+    params.front()->get_output_tensor(0).set_names({"input_tensor"});
+    auto split = ngraph::builder::makeSplit(params[0], ngPrc, 2, 1);
+
+    auto in2add = ngraph::builder::makeConstant(ngPrc, {1, 2, 1, 1}, std::vector<float>{}, true);
+    auto add = ngraph::builder::makeEltwise(split->output(0), in2add, ngraph::helpers::EltwiseTypes::ADD);
+    auto relu1 = std::make_shared<ngraph::opset1::Relu>(add);
+
+    auto in2mult = ngraph::builder::makeConstant(ngPrc, {1, 2, 1, 1}, std::vector<float>{}, true);
+    auto mult = ngraph::builder::makeEltwise(split->output(1), in2mult, ngraph::helpers::EltwiseTypes::MULTIPLY);
+    auto relu2 = std::make_shared<ngraph::opset1::Relu>(mult);
+
+    auto concat = std::make_shared<ngraph::opset1::Concat>(ngraph::OutputVector{relu1->output(0), relu2->output(0)}, 3);
+    concat->get_output_tensor(0).set_names({"concat"});
+
+    return std::make_shared<ngraph::Function>(concat, params, "SplitAddConcat");
+}
+
+INSTANTIATE_TEST_SUITE_P(smoke_Auto_BehaviorTests, OVInferRequestDynamicTests,
+                        ::testing::Combine(
+                                ::testing::Values(getFunction2()),
+                                ::testing::Values(std::vector<std::pair<std::vector<size_t>, std::vector<size_t>>>{
+                                    {{1, 4, 20, 20}, {1, 2, 20, 40}},
+                                    {{2, 4, 20, 20}, {2, 2, 20, 40}}}),
+                                ::testing::Values(CommonTestUtils::DEVICE_AUTO),
+                                ::testing::ValuesIn(AutoConfigs)),
+                        OVInferRequestDynamicTests::getTestCaseName);
+
+INSTANTIATE_TEST_SUITE_P(smoke_Auto_BehaviorTests, OVInferenceChaining,
+                        ::testing::Combine(
+                                ::testing::Values(CommonTestUtils::DEVICE_AUTO),
+                                ::testing::ValuesIn(AutoConfigs)),
+                        OVInferenceChaining::getTestCaseName);
+INSTANTIATE_TEST_SUITE_P(smoke_Auto_BehaviorTests, OVNotSupportRequestDynamicTests,
+                        ::testing::Combine(
+                                ::testing::Values(getFunction2()),
+                                ::testing::Values(std::vector<std::pair<std::vector<size_t>, std::vector<size_t>>>{
+                                    {{1, 4, 20, 20}, {1, 2, 20, 40}},
+                                    {{2, 4, 20, 20}, {2, 2, 20, 40}}}),
+                                ::testing::Values(CommonTestUtils::DEVICE_AUTO),
+                                ::testing::ValuesIn(AutoNotSupportConfigs)),
+                        OVInferRequestDynamicTests::getTestCaseName);
+}  // namespace
@@ -91,5 +91,9 @@ std::vector<std::string> disabledTestPatterns() {
            R"(smoke_Behavior/InferRequestSetBlobByType.setInputBlobsByType/BlobType=Batched_Device=GPU_Config=().*)",
            // TODO: Issue 72624
            R"(smoke_PrePostProcess.*resize_dynamic.*)",
+            // Issue: CVS-66778
+            R"(.*smoke_Auto_BehaviorTests.*DynamicOutputToDynamicInput.*)",
+            R"(.*smoke_Auto_BehaviorTests.*DynamicInputToDynamicOutput.*)",
+            R"(.*smoke_Auto_BehaviorTests.*InferFullyDynamicNetworkWith(S|G)etTensor.*)",
    };
 }
@@ -33,6 +33,9 @@ protected:
        auto mul = std::make_shared<opset8::Multiply>(input4, constant);
        auto input5 = std::make_shared<opset8::Parameter>(type, constGroupConvBackpropShape);
        auto mul2 = std::make_shared<opset8::Multiply>(input5, constantGroupConv);
+        // explicitly set the output name, to avoid global conflict
+        mul2->set_friendly_name("Multiply_0");
+        mul->set_friendly_name("Multiply_1");
        function = std::make_shared<ngraph::Function>(NodeVector{convBprop, conv, groupConvBprop, mul2, mul},
                ParameterVector{input1, input2, input3, input4, input5});
    }
@@ -57,6 +57,7 @@ protected:
    std::map<std::string, std::string> configuration;
    std::vector<std::pair<std::vector<size_t>, std::vector<size_t>>> inOutShapes;
 };
+using OVNotSupportRequestDynamicTests = OVInferRequestDynamicTests;
 }  // namespace behavior
 }  // namespace test
 }  // namespace ov
@@ -359,6 +359,20 @@ TEST_P(OVInferRequestDynamicTests, InferDynamicNetworkWithSetTensor2times) {
    ASSERT_EQ(tensor.get_shape(), refOutShape2);
 }

+TEST_P(OVNotSupportRequestDynamicTests, InferDynamicNotSupported) {
+    const std::string tensor_name = "input_tensor";
+    const ov::Shape refShape = inOutShapes[0].first;
+    const ov::Shape refShape2 = inOutShapes[1].first;
+    const ov::Shape refOutShape = inOutShapes[0].second;
+    const ov::Shape refOutShape2 = inOutShapes[1].second;
+    std::map<std::string, ov::PartialShape> shapes;
+    shapes[tensor_name] = {ov::Dimension::dynamic(), 4, 20, 20};
+    OV_ASSERT_NO_THROW(function->reshape(shapes));
+    const std::string outputName = function->outputs().back().get_any_name();
+    // Load ov::Function to target plugins
+    ov::runtime::CompiledModel execNet;
+    ASSERT_THROW((execNet = ie->compile_model(function, targetDevice, configuration)), ov::Exception);
+}
 }  // namespace behavior
 }  // namespace test
 }  // namespace ov