From 4cc53c97a106ac7fe3f5c09b09e2779a7fc40370 Mon Sep 17 00:00:00 2001 From: Anton Pankratv Date: Fri, 15 Oct 2021 10:44:11 +0300 Subject: [PATCH] Blob processing in HETERO forwarded to sub requests (#8012) * Blob processing in HETERO forwarded to subrequests * Added template dynamic hetero tests --- .../tests/functional/CMakeLists.txt | 1 + .../infer_request/infer_request_dynamic.cpp | 12 +++ .../infer_request/inference_chaining.cpp | 10 ++ .../hetero_async_infer_request.cpp | 5 - .../hetero_async_infer_request.hpp | 1 - .../hetero_plugin/hetero_infer_request.cpp | 94 ++++++++----------- .../hetero_plugin/hetero_infer_request.hpp | 15 ++- 7 files changed, 71 insertions(+), 67 deletions(-) diff --git a/docs/template_plugin/tests/functional/CMakeLists.txt b/docs/template_plugin/tests/functional/CMakeLists.txt index a2962cea0ae..7632b4bb8c3 100644 --- a/docs/template_plugin/tests/functional/CMakeLists.txt +++ b/docs/template_plugin/tests/functional/CMakeLists.txt @@ -10,6 +10,7 @@ addIeTargetTest( ROOT ${CMAKE_CURRENT_SOURCE_DIR} DEPENDENCIES templatePlugin + HeteroPlugin LINK_LIBRARIES IE::funcSharedTests INCLUDES diff --git a/docs/template_plugin/tests/functional/shared_tests_instances/behavior/infer_request/infer_request_dynamic.cpp b/docs/template_plugin/tests/functional/shared_tests_instances/behavior/infer_request/infer_request_dynamic.cpp index df9bc757088..edfdfa24af4 100644 --- a/docs/template_plugin/tests/functional/shared_tests_instances/behavior/infer_request/infer_request_dynamic.cpp +++ b/docs/template_plugin/tests/functional/shared_tests_instances/behavior/infer_request/infer_request_dynamic.cpp @@ -14,6 +14,9 @@ const std::vector> configs = { {} }; +const std::vector> HeteroConfigs = { + {{"TARGET_FALLBACK", CommonTestUtils::DEVICE_TEMPLATE}}}; + INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, InferRequestDynamicTests, ::testing::Combine( ::testing::Values(ngraph::builder::subgraph::makeSplitConvConcat()), @@ -23,4 +26,13 @@ INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, InferRequestDynamicTests, ::testing::ValuesIn(configs)), InferRequestDynamicTests::getTestCaseName); +INSTANTIATE_TEST_SUITE_P(smoke_Hetero_BehaviorTests, InferRequestDynamicTests, + ::testing::Combine( + ::testing::Values(ngraph::builder::subgraph::makeSplitConvConcat()), + ::testing::Values(std::vector, std::vector>>{{{1, 4, 20, 20}, {1, 10, 18, 18}}, + {{2, 4, 20, 20}, {2, 10, 18, 18}}}), + ::testing::Values(CommonTestUtils::DEVICE_HETERO), + ::testing::ValuesIn(HeteroConfigs)), + InferRequestDynamicTests::getTestCaseName); + } // namespace diff --git a/docs/template_plugin/tests/functional/shared_tests_instances/behavior/infer_request/inference_chaining.cpp b/docs/template_plugin/tests/functional/shared_tests_instances/behavior/infer_request/inference_chaining.cpp index 6733b507c4f..1080e99c55e 100644 --- a/docs/template_plugin/tests/functional/shared_tests_instances/behavior/infer_request/inference_chaining.cpp +++ b/docs/template_plugin/tests/functional/shared_tests_instances/behavior/infer_request/inference_chaining.cpp @@ -13,6 +13,9 @@ const std::vector> configs = { {} }; +const std::vector> HeteroConfigs = { + {{"TARGET_FALLBACK", CommonTestUtils::DEVICE_TEMPLATE}}}; + INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVInferenceChaining, ::testing::Combine( ::testing::Values(ov::element::f32), @@ -20,4 +23,11 @@ INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVInferenceChaining, ::testing::ValuesIn(configs)), OVInferenceChaining::getTestCaseName); +INSTANTIATE_TEST_SUITE_P(smoke_Hetero_BehaviorTests, OVInferenceChaining, + ::testing::Combine( + ::testing::Values(ov::element::f32), + ::testing::Values(CommonTestUtils::DEVICE_HETERO), + ::testing::ValuesIn(HeteroConfigs)), + OVInferenceChaining::getTestCaseName); + } // namespace diff --git a/inference-engine/src/hetero_plugin/hetero_async_infer_request.cpp b/inference-engine/src/hetero_plugin/hetero_async_infer_request.cpp index ee770c12ed2..019332f0a62 100644 --- a/inference-engine/src/hetero_plugin/hetero_async_infer_request.cpp +++ b/inference-engine/src/hetero_plugin/hetero_async_infer_request.cpp @@ -43,11 +43,6 @@ HeteroAsyncInferRequest::HeteroAsyncInferRequest(const IInferRequestInternal::Pt } } -void HeteroAsyncInferRequest::StartAsync_ThreadUnsafe() { - _heteroInferRequest->updateInOutIfNeeded(); - RunFirstStage(_pipeline.begin(), _pipeline.end()); -} - StatusCode HeteroAsyncInferRequest::Wait(int64_t millis_timeout) { auto waitStatus = StatusCode::OK; try { diff --git a/inference-engine/src/hetero_plugin/hetero_async_infer_request.hpp b/inference-engine/src/hetero_plugin/hetero_async_infer_request.hpp index 22f8d065432..85d6a13afba 100644 --- a/inference-engine/src/hetero_plugin/hetero_async_infer_request.hpp +++ b/inference-engine/src/hetero_plugin/hetero_async_infer_request.hpp @@ -18,7 +18,6 @@ public: const InferenceEngine::ITaskExecutor::Ptr& taskExecutor, const InferenceEngine::ITaskExecutor::Ptr& callbackExecutor); ~HeteroAsyncInferRequest(); - void StartAsync_ThreadUnsafe() override; InferenceEngine::StatusCode Wait(int64_t millis_timeout) override; private: diff --git a/inference-engine/src/hetero_plugin/hetero_infer_request.cpp b/inference-engine/src/hetero_plugin/hetero_infer_request.cpp index 9d7002304c5..7ecf4a3a8e9 100644 --- a/inference-engine/src/hetero_plugin/hetero_infer_request.cpp +++ b/inference-engine/src/hetero_plugin/hetero_infer_request.cpp @@ -36,11 +36,14 @@ HeteroInferRequest::HeteroInferRequest(InferenceEngine::InputsDataMap networkInp bool emplaced = false; std::tie(itBlob, emplaced) = _blobs.emplace(intermediateBlobName, Blob::Ptr{}); if (emplaced) { - itBlob->second = r->GetBlob(blobName); - if (InferenceEngine::details::contains(networkInputs, blobName)) { - _inputs[blobName] = itBlob->second; - } else if (InferenceEngine::details::contains(networkOutputs, blobName)) { - _outputs[blobName] = itBlob->second; + if (InferenceEngine::details::contains(_networkInputs, blobName)) { + _subRequestFromBlobName.emplace(blobName, r._ptr.get()); + _blobs.erase(intermediateBlobName); + } else if (InferenceEngine::details::contains(_networkOutputs, blobName)) { + _subRequestFromBlobName.emplace(blobName, r._ptr.get()); + _blobs.erase(intermediateBlobName); + } else { + itBlob->second = r->GetBlob(blobName); } } else { r->SetBlob(blobName, itBlob->second); @@ -64,25 +67,39 @@ HeteroInferRequest::HeteroInferRequest(InferenceEngine::InputsDataMap networkInp } } -void HeteroInferRequest::SetBlob(const std::string& name, const InferenceEngine::Blob::Ptr& data) { - InferenceEngine::IInferRequestInternal::SetBlob(name, data); - assert(!_inferRequests.empty()); - for (auto &&desc : _inferRequests) { - auto &r = desc._request; - assert(r); - InputInfo::Ptr foundInput; - DataPtr foundOutput; - try { - // if `name` is input blob - if (findInputAndOutputBlobByName(name, foundInput, foundOutput)) { - r->SetBlob(name, data, foundInput->getPreProcess()); - } - } catch (const InferenceEngine::NotFound&) {} +void HeteroInferRequest::SetBlob(const std::string& name, const InferenceEngine::Blob::Ptr& blob) { + auto itRequest = _subRequestFromBlobName.find(name); + if (itRequest == _subRequestFromBlobName.end()) { + IE_THROW() << "There is no infer requests binded to blob with name: " << name; } + itRequest->second->SetBlob(name, blob); +} + +InferenceEngine::Blob::Ptr HeteroInferRequest::GetBlob(const std::string& name) { + auto itRequest = _subRequestFromBlobName.find(name); + if (itRequest == _subRequestFromBlobName.end()) { + IE_THROW() << "There is no infer requests binded to blob with name: " << name; + } + return itRequest->second->GetBlob(name); +} + +void HeteroInferRequest::SetBlob(const std::string& name, const Blob::Ptr& blob, const PreProcessInfo& info) { + auto itRequest = _subRequestFromBlobName.find(name); + if (itRequest == _subRequestFromBlobName.end()) { + IE_THROW() << "There is no infer requests binded to blob with name: " << name; + } + itRequest->second->SetBlob(name, blob, info); +} + +const InferenceEngine::PreProcessInfo& HeteroInferRequest::GetPreProcess(const std::string& name) const { + auto itRequest = _subRequestFromBlobName.find(name); + if (itRequest == _subRequestFromBlobName.end()) { + IE_THROW() << "There is no infer requests binded to blob with name: " << name; + } + return itRequest->second->GetPreProcess(name); } void HeteroInferRequest::InferImpl() { - updateInOutIfNeeded(); for (auto &&desc : _inferRequests) { OV_ITT_SCOPED_TASK(itt::domains::HeteroPlugin, desc._profilingTask); auto &r = desc._request; @@ -101,40 +118,3 @@ std::map HeteroInferRequest::GetPerform } return perfMap; } - -void HeteroInferRequest::updateInOutIfNeeded() { - OV_ITT_SCOPED_TASK(itt::domains::HeteroPlugin, "updateInOutIfNeeded"); - assert(!_inferRequests.empty()); - for (auto &&desc : _inferRequests) { - auto &r = desc._request; - assert(r); - for (auto&& inputInfo : desc._network->GetInputsInfo()) { - auto& ioname = inputInfo.first; - auto iti = _inputs.find(ioname); - if (iti != _inputs.end()) { - auto it = _preProcData.find(ioname); - if (it != _preProcData.end()) { - if (it->second->getRoiBlob() != _blobs[ioname]) { - r->SetBlob(ioname.c_str(), it->second->getRoiBlob()); - _blobs[ioname] = iti->second; - } - } else { - if (iti->second != _blobs[ioname]) { - r->SetBlob(ioname.c_str(), iti->second); - _blobs[ioname] = iti->second; - } - } - } - } - for (auto&& outputInfo : desc._network->GetOutputsInfo()) { - auto& ioname = outputInfo.first; - auto ito = _outputs.find(ioname); - if (ito != _outputs.end()) { - if (ito->second != _blobs[ioname]) { - r->SetBlob(ioname.c_str(), ito->second); - _blobs[ioname] = ito->second; - } - } - } - } -} diff --git a/inference-engine/src/hetero_plugin/hetero_infer_request.hpp b/inference-engine/src/hetero_plugin/hetero_infer_request.hpp index 9327d2c410d..0b4e55b6606 100644 --- a/inference-engine/src/hetero_plugin/hetero_infer_request.hpp +++ b/inference-engine/src/hetero_plugin/hetero_infer_request.hpp @@ -34,14 +34,21 @@ public: void InferImpl() override; - void SetBlob(const std::string& name, const InferenceEngine::Blob::Ptr& data) override; + void SetBlob(const std::string& name, const InferenceEngine::Blob::Ptr& blob) override; + + InferenceEngine::Blob::Ptr GetBlob(const std::string& name) override; + + void SetBlob(const std::string& name, + const InferenceEngine::Blob::Ptr& blob, + const InferenceEngine::PreProcessInfo& info) override; + + const InferenceEngine::PreProcessInfo& GetPreProcess(const std::string& name) const override; std::map GetPerformanceCounts() const override; - void updateInOutIfNeeded(); - SubRequestsList _inferRequests; - std::map _blobs; + std::map _blobs; + std::map _subRequestFromBlobName; }; } // namespace HeteroPlugin