diff --git a/src/inference/dev_api/ie_icore.hpp b/src/inference/dev_api/ie_icore.hpp index 5a0a6bd9241..08bbd925693 100644 --- a/src/inference/dev_api/ie_icore.hpp +++ b/src/inference/dev_api/ie_icore.hpp @@ -180,6 +180,13 @@ public: virtual bool isNewAPI() const = 0; + /** + * @brief Get a pointer to default shared context object for the specified device. + * @param deviceName - A name of a device to get create shared context from. + * @return A shared pointer to a default remote context. + */ + virtual RemoteContext::Ptr GetDefaultContext(const std::string& deviceName) = 0; + /** * @brief Default virtual destructor */ diff --git a/src/inference/src/ie_core.cpp b/src/inference/src/ie_core.cpp index 789a7a553aa..914be504088 100644 --- a/src/inference/src/ie_core.cpp +++ b/src/inference/src/ie_core.cpp @@ -497,6 +497,11 @@ public: return newAPI; } + ie::RemoteContext::Ptr GetDefaultContext(const std::string& deviceName) override { + auto parsed = ov::parseDeviceNameIntoConfig(deviceName, ParamMap{}); + return GetCPPPluginByName(parsed._deviceName).get_default_context(parsed._config)._ptr; + } + ov::SoPtr LoadNetwork(const ie::CNNNetwork& network, const std::shared_ptr& context, const std::map& config) override { @@ -1424,9 +1429,7 @@ RemoteContext::Ptr Core::GetDefaultContext(const std::string& deviceName) { if (deviceName.find("AUTO") == 0) { IE_THROW() << "AUTO device does not support remote context"; } - - auto parsed = ov::parseDeviceNameIntoConfig(deviceName, ParamMap()); - return _impl->GetCPPPluginByName(parsed._deviceName).get_default_context(parsed._config)._ptr; + return _impl->GetDefaultContext(deviceName); } void Core::AddExtension(IExtensionPtr extension, const std::string& deviceName_) { diff --git a/src/plugins/auto/executable_network.cpp b/src/plugins/auto/executable_network.cpp index df5d6f517d0..de6ce1220aa 100644 --- a/src/plugins/auto/executable_network.cpp +++ b/src/plugins/auto/executable_network.cpp @@ -557,16 +557,24 @@ InferenceEngine::IInferRequestInternal::Ptr MultiDeviceExecutableNetwork::Create auto num = _numRequestsCreated++; size_t sum = 0; InferenceEngine::SoIInferRequestInternal request_to_share_blobs_with; + InferenceEngine::RemoteContext::Ptr ctx = nullptr; if (_workModeIsAUTO) { if (!_loadContext[CPU].isEnabled && _loadContext[ACTUALDEVICE].isAlready) { - auto& dev_requests = _workerRequests[_loadContext[ACTUALDEVICE].deviceInfo.deviceName]; - if (num < dev_requests.size()) { - request_to_share_blobs_with = dev_requests.at(num)._inferRequest; + try { + ctx = GetCore()->GetDefaultContext(_loadContext[ACTUALDEVICE].deviceInfo.deviceName); + } catch (InferenceEngine::Exception& ex) { + // plugin does not support context, say CPU + LOG_DEBUG("[AUTOPLUGIN]context not supported for %s, fallback to default memory", + _loadContext[ACTUALDEVICE].deviceInfo.deviceName.c_str()); + // for dynamic shape support + auto& dev_requests = _workerRequests[_loadContext[ACTUALDEVICE].deviceInfo.deviceName]; + if (num < dev_requests.size()) { + request_to_share_blobs_with = dev_requests.at(num)._inferRequest; + } } } - // if user creates more infer request than the device optimal value, fall back to default memory - return std::make_shared(inputs, outputs, request_to_share_blobs_with); + return std::make_shared(inputs, outputs, request_to_share_blobs_with, ctx); } // borrowing device-specific blobs from the underlying requests for the device-agnostic, user-facing requests @@ -587,16 +595,23 @@ InferenceEngine::IInferRequestInternal::Ptr MultiDeviceExecutableNetwork::Create auto num = _numRequestsCreated++; size_t sum = 0; InferenceEngine::SoIInferRequestInternal request_to_share_blobs_with; + InferenceEngine::RemoteContext::Ptr ctx = nullptr; if (_workModeIsAUTO) { if (!_loadContext[CPU].isEnabled && _loadContext[ACTUALDEVICE].isAlready) { - auto& dev_requests = _workerRequests[_loadContext[ACTUALDEVICE].deviceInfo.deviceName]; - if (num < dev_requests.size()) { - request_to_share_blobs_with = dev_requests.at(num)._inferRequest; + try { + ctx = GetCore()->GetDefaultContext(_loadContext[ACTUALDEVICE].deviceInfo.deviceName); + } catch (InferenceEngine::Exception& ex) { + // plugin does not support context + LOG_DEBUG("[AUTOPLUGIN]context not supported for %s, fallback to default memory", + _loadContext[ACTUALDEVICE].deviceInfo.deviceName.c_str()); + auto& dev_requests = _workerRequests[_loadContext[ACTUALDEVICE].deviceInfo.deviceName]; + if (num < dev_requests.size()) { + request_to_share_blobs_with = dev_requests.at(num)._inferRequest; + } } } - // if user creates more infer request than the device optimal value, fall back to default memory - return std::make_shared(networkInputs, networkOutputs, request_to_share_blobs_with); + return std::make_shared(networkInputs, networkOutputs, request_to_share_blobs_with, ctx); } // borrowing device-specific blobs from the underlying requests for the device-agnostic, user-facing requests diff --git a/src/plugins/auto/infer_request.cpp b/src/plugins/auto/infer_request.cpp index 1372ae82103..abaf946f9a6 100644 --- a/src/plugins/auto/infer_request.cpp +++ b/src/plugins/auto/infer_request.cpp @@ -16,19 +16,22 @@ using namespace InferenceEngine; // ------------------------------MultiDeviceInferRequest---------------------------- MultiDeviceInferRequest::MultiDeviceInferRequest(const std::vector>& inputs, const std::vector>& outputs, - const InferenceEngine::SoIInferRequestInternal & request_to_share_blobs_with) + const InferenceEngine::SoIInferRequestInternal & request_to_share_blobs_with, + InferenceEngine::RemoteContext::Ptr ctx) : IInferRequestInternal(inputs, outputs) { - CreateInferRequest(request_to_share_blobs_with); + CreateInferRequest(request_to_share_blobs_with, ctx); } MultiDeviceInferRequest::MultiDeviceInferRequest(const InputsDataMap& networkInputs, const OutputsDataMap& networkOutputs, - const SoIInferRequestInternal & request_to_share_blobs_with) + const SoIInferRequestInternal & request_to_share_blobs_with, + InferenceEngine::RemoteContext::Ptr ctx) : IInferRequestInternal(networkInputs, networkOutputs) { - CreateInferRequest(request_to_share_blobs_with); + CreateInferRequest(request_to_share_blobs_with, ctx); } -void MultiDeviceInferRequest::CreateInferRequest(const InferenceEngine::SoIInferRequestInternal& request_to_share_blobs_with) { +void MultiDeviceInferRequest::CreateInferRequest(const InferenceEngine::SoIInferRequestInternal& request_to_share_blobs_with, + InferenceEngine::RemoteContext::Ptr ctx) { if (request_to_share_blobs_with) { // borrow device-friendly blobs from the request for (const auto &it : _networkInputs) @@ -39,22 +42,30 @@ void MultiDeviceInferRequest::CreateInferRequest(const InferenceEngine::SoIInfer } // Allocate all input blobs for (const auto &it : _networkInputs) { - Layout l = it.second->getLayout(); - Precision p = it.second->getPrecision(); - SizeVector dims = it.second->getTensorDesc().getDims(); + auto l = it.second->getLayout(); + auto p = it.second->getPrecision(); + auto dims = it.second->getTensorDesc().getDims(); TensorDesc desc = TensorDesc(p, dims, l); - _inputs[it.first] = make_blob_with_precision(desc); + if (ctx) { + _inputs[it.first] = ctx->CreateHostBlob(desc); + } else { + _inputs[it.first] = make_blob_with_precision(desc); + } _inputs[it.first]->allocate(); } // Allocate all output blobs for (const auto &it : _networkOutputs) { - Layout l = it.second->getLayout(); - Precision p = it.second->getPrecision(); - SizeVector dims = it.second->getTensorDesc().getDims(); + auto l = it.second->getLayout(); + auto p = it.second->getPrecision(); + auto dims = it.second->getTensorDesc().getDims(); TensorDesc desc = TensorDesc(p, dims, l); - _outputs[it.first] = make_blob_with_precision(desc); + if (ctx) { + _outputs[it.first] = ctx->CreateHostBlob(desc); + } else { + _outputs[it.first] = make_blob_with_precision(desc); + } _outputs[it.first]->allocate(); } } diff --git a/src/plugins/auto/infer_request.hpp b/src/plugins/auto/infer_request.hpp index dac399c26ea..5457682fd59 100644 --- a/src/plugins/auto/infer_request.hpp +++ b/src/plugins/auto/infer_request.hpp @@ -15,6 +15,7 @@ #include #include #include +#include "ie_remote_context.hpp" #ifdef MULTIUNITTEST #define MOCKTESTMACRO virtual @@ -30,17 +31,20 @@ public: using Ptr = std::shared_ptr; explicit MultiDeviceInferRequest(const InferenceEngine::InputsDataMap& networkInputs, const InferenceEngine::OutputsDataMap& networkOutputs, - const InferenceEngine::SoIInferRequestInternal & request_to_share_blobs_with); + const InferenceEngine::SoIInferRequestInternal & request_to_share_blobs_with, + InferenceEngine::RemoteContext::Ptr ctx = nullptr); explicit MultiDeviceInferRequest(const std::vector>& inputs, const std::vector>& outputs, - const InferenceEngine::SoIInferRequestInternal & request_to_share_blobs_with); + const InferenceEngine::SoIInferRequestInternal & request_to_share_blobs_with, + InferenceEngine::RemoteContext::Ptr ctx = nullptr); std::map GetPerformanceCounts() const override; void InferImpl() override; // Multi-Device impl specific: sets the data (blobs from the device-less requests to the specific device request) void SetBlobsToAnotherRequest(const InferenceEngine::SoIInferRequestInternal& req); private: - void CreateInferRequest(const InferenceEngine::SoIInferRequestInternal& request_to_share_blobs_with); + void CreateInferRequest(const InferenceEngine::SoIInferRequestInternal& request_to_share_blobs_with, + InferenceEngine::RemoteContext::Ptr ctx); }; } // namespace MultiDevicePlugin diff --git a/src/tests/ie_test_utils/unit_test_utils/mocks/cpp_interfaces/interface/mock_icore.hpp b/src/tests/ie_test_utils/unit_test_utils/mocks/cpp_interfaces/interface/mock_icore.hpp index 639eec709bb..63f6cad66a4 100644 --- a/src/tests/ie_test_utils/unit_test_utils/mocks/cpp_interfaces/interface/mock_icore.hpp +++ b/src/tests/ie_test_utils/unit_test_utils/mocks/cpp_interfaces/interface/mock_icore.hpp @@ -38,6 +38,7 @@ public: MOCK_CONST_METHOD1(DeviceSupportsImportExport, bool(const std::string&)); // NOLINT not a cast to bool MOCK_METHOD2(GetSupportedConfig, std::map(const std::string&, const std::map&)); MOCK_CONST_METHOD0(isNewAPI, bool()); + MOCK_METHOD1(GetDefaultContext, InferenceEngine::RemoteContext::Ptr(const std::string&)); ~MockICore() = default; };