diff --git a/src/bindings/c/tests/ie_c_api_test.cpp b/src/bindings/c/tests/ie_c_api_test.cpp index f4d527d6180..013001b85c2 100644 --- a/src/bindings/c/tests/ie_c_api_test.cpp +++ b/src/bindings/c/tests/ie_c_api_test.cpp @@ -954,36 +954,6 @@ TEST_P(ie_c_api_test, ie_exec_network_get_config) { ie_core_free(&core); } -TEST_P(ie_c_api_test, ie_exec_network_set_config) { - ie_core_t *core = nullptr; - IE_ASSERT_OK(ie_core_create("", &core)); - ASSERT_NE(nullptr, core); - - ie_param_t param; - if (ie_core_get_metric(core, "GPU", "AVAILABLE_DEVICES", ¶m) != IEStatusCode::OK) { - ie_core_free(&core); - GTEST_SKIP(); - } - - ie_network_t *network = nullptr; - IE_EXPECT_OK(ie_core_read_network(core, xml_file_name.c_str(), bin_file_name.c_str(), &network)); - EXPECT_NE(nullptr, network); - - const char *device_name = "MULTI:GPU,CPU"; - ie_config_t config = {nullptr, nullptr, nullptr}; - ie_executable_network_t *exe_network = nullptr; - IE_EXPECT_OK(ie_core_load_network(core, network, device_name, &config, &exe_network)); - EXPECT_NE(nullptr, exe_network); - - ie_config_t config_param = {"MULTI_DEVICE_PRIORITIES", "GPU,CPU", nullptr}; - IE_EXPECT_OK(ie_exec_network_set_config(exe_network, &config_param)); - - ie_exec_network_free(&exe_network); - ie_network_free(&network); - ie_core_free(&core); - ie_param_free(¶m); -} - TEST_P(ie_c_api_test, ie_exec_network_get_metric) { ie_core_t *core = nullptr; IE_ASSERT_OK(ie_core_create("", &core)); diff --git a/src/bindings/c/tests/ov_compiled_model_test.cpp b/src/bindings/c/tests/ov_compiled_model_test.cpp index 0ff98f5ba2d..c1b6ed33cad 100644 --- a/src/bindings/c/tests/ov_compiled_model_test.cpp +++ b/src/bindings/c/tests/ov_compiled_model_test.cpp @@ -121,45 +121,6 @@ TEST_P(ov_compiled_model_test, ov_compiled_model_input_by_name) { ov_core_free(core); } -TEST_P(ov_compiled_model_test, set_and_get_property) { - // It seems that all set_property() for CPU plugin are not implement in compiled_model. - auto device_name = "MULTI:GPU,CPU"; - ov_core_t* core = nullptr; - OV_EXPECT_OK(ov_core_create(&core)); - EXPECT_NE(nullptr, core); - - char* info = nullptr; - const char* key_0 = ov_property_key_available_devices; - if (ov_core_get_property(core, "GPU", key_0, &info) != ov_status_e::OK) { - ov_core_free(core); - GTEST_SKIP(); - } - - ov_model_t* model = nullptr; - OV_EXPECT_OK(ov_core_read_model(core, xml_file_name.c_str(), bin_file_name.c_str(), &model)); - EXPECT_NE(nullptr, model); - - ov_compiled_model_t* compiled_model = nullptr; - OV_EXPECT_OK(ov_core_compile_model(core, model, device_name, 0, &compiled_model)); - EXPECT_NE(nullptr, compiled_model); - - const char* key_1 = ov_property_key_device_priorities; - const char* value_1 = "GPU,CPU"; - OV_EXPECT_OK(ov_compiled_model_set_property(compiled_model, key_1, value_1)); - char* result = nullptr; - OV_EXPECT_OK(ov_compiled_model_get_property(compiled_model, key_1, &result)); - EXPECT_STREQ(value_1, result); - ov_free(result); - - const char* key_2 = ov_property_key_supported_properties; - OV_EXPECT_OK(ov_compiled_model_get_property(compiled_model, key_2, &result)); - ov_free(result); - - ov_compiled_model_free(compiled_model); - ov_model_free(model); - ov_core_free(core); -} - TEST_P(ov_compiled_model_test, get_property) { auto device_name = GetParam(); ov_core_t* core = nullptr; diff --git a/src/plugins/auto/auto_executable_network.cpp b/src/plugins/auto/auto_executable_network.cpp index 64e0e8f1a91..fa7b814e13e 100644 --- a/src/plugins/auto/auto_executable_network.cpp +++ b/src/plugins/auto/auto_executable_network.cpp @@ -17,12 +17,21 @@ AutoExecutableNetwork::AutoExecutableNetwork(AutoScheduleContext::Ptr& context, } std::shared_ptr AutoExecutableNetwork::GetContext() const { - std::lock_guard lock(_autoSContext->_fallbackMutex); - if (_autoSchedule->_loadContext[FALLBACKDEVICE].isAlready) { - return _autoSchedule->_loadContext[FALLBACKDEVICE].executableNetwork->GetContext(); + if (_autoSchedule->_pCTPUTLoadContext) { + for (size_t i = 0; i < _autoSchedule->_nCTputDeviceNums; i++) { + if (_autoSchedule->_pCTPUTLoadContext[i].isAlready) { + return _autoSchedule->_pCTPUTLoadContext[i].executableNetwork->GetContext(); + } + } + return nullptr; } else { - _autoSchedule->WaitActualNetworkReady(); - return _autoSchedule->_loadContext[ACTUALDEVICE].executableNetwork->GetContext(); + std::lock_guard lock(_autoSContext->_fallbackMutex); + if (_autoSchedule->_loadContext[FALLBACKDEVICE].isAlready) { + return _autoSchedule->_loadContext[FALLBACKDEVICE].executableNetwork->GetContext(); + } else { + _autoSchedule->WaitActualNetworkReady(); + return _autoSchedule->_loadContext[ACTUALDEVICE].executableNetwork->GetContext(); + } } } @@ -82,11 +91,11 @@ IE::Parameter AutoExecutableNetwork::GetMetric(const std::string& name) const { all_devices[context.deviceInfo.deviceName] = device_properties; }; if (_autoSchedule->_pCTPUTLoadContext) { - // need lock for inference failure - std::lock_guard lock(_autoSContext->_fallbackMutex); - auto load_count = _autoSContext->_devicePriorities.size(); - for (size_t i = 0; i < load_count; i++) - get_device_supported_metrics(_autoSchedule->_pCTPUTLoadContext[i]); + for (size_t i = 0; i < _autoSchedule->_nCTputDeviceNums; i++) { + if (_autoSchedule->_pCTPUTLoadContext[i].isAlready) { + get_device_supported_metrics(_autoSchedule->_pCTPUTLoadContext[i]); + } + } } else { { std::lock_guard lock(_autoSContext->_fallbackMutex); @@ -118,11 +127,13 @@ IE::Parameter AutoExecutableNetwork::GetMetric(const std::string& name) const { if (_autoSchedule->_pCTPUTLoadContext) { std::lock_guard lock(_autoSContext->_fallbackMutex); unsigned int res = 0u; - auto load_count = _autoSContext->_devicePriorities.size(); - for (size_t i = 0; i < load_count; i++) { + for (size_t i = 0; i < _autoSchedule->_nCTputDeviceNums; i++) { try { - res += (_autoSchedule->_pCTPUTLoadContext[i]).executableNetwork->GetMetric( - METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS)).as(); + if (_autoSchedule->_pCTPUTLoadContext[i].isAlready) { + res += (_autoSchedule->_pCTPUTLoadContext[i]) + .executableNetwork->GetMetric(METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS)) + .as(); + } } catch (const IE::Exception& iie) { IE_THROW() << "Every device used in cumulative mode should " diff --git a/src/plugins/auto/auto_schedule.cpp b/src/plugins/auto/auto_schedule.cpp index 7e5a85809b8..8e5eaf8c648 100644 --- a/src/plugins/auto/auto_schedule.cpp +++ b/src/plugins/auto/auto_schedule.cpp @@ -234,9 +234,7 @@ void AutoSchedule::init(const ScheduleContext::Ptr& sContext) { bool isCumulative = (_autoSContext->_performanceHint == IE::PluginConfigParams::CUMULATIVE_THROUGHPUT) ? true : false; if (isCumulative) { - std::list validDevices = - _autoSContext->_plugin->GetValidDevice(_autoSContext->_devicePriorities, - _loadContext[ACTUALDEVICE].networkPrecision); + const auto& validDevices = _autoSContext->_devicePriorities; // When the hint is ctput and there is only one device, the single-device logic is used if (validDevices.size() == 1) { _loadContext[ACTUALDEVICE].deviceInfo = validDevices.front(); @@ -244,14 +242,10 @@ void AutoSchedule::init(const ScheduleContext::Ptr& sContext) { IE::PluginConfigParams::THROUGHPUT; } else if (validDevices.size() > 1) { _loadContext[ACTUALDEVICE].isEnabled = false; - _autoSContext->_devicePriorities.clear(); - std::copy(std::begin(validDevices), - std::end(validDevices), - std::back_inserter(_autoSContext->_devicePriorities)); // Total number of devices in CTPUT - auto nCTputDeviceNums = validDevices.size(); + _nCTputDeviceNums = validDevices.size(); // Generate contexts for loading each device - _pCTPUTLoadContext.reset(new AutoLoadContext[nCTputDeviceNums]); + _pCTPUTLoadContext.reset(new AutoLoadContext[_nCTputDeviceNums]); int idx = 0; DeviceInformation cpuDeviceInformation; for (auto& device : validDevices) { @@ -272,6 +266,10 @@ void AutoSchedule::init(const ScheduleContext::Ptr& sContext) { IE::PluginConfigParams::THROUGHPUT; } } + if (_autoSContext->_LogTag == "MULTI") { + // MULTI's performance hint always is tput + _autoSContext->_performanceHint = IE::PluginConfigParams::THROUGHPUT; + } } else { _loadContext[ACTUALDEVICE].deviceInfo = _autoSContext->_plugin->SelectDevice(_autoSContext->_devicePriorities, @@ -388,12 +386,12 @@ void AutoSchedule::init(const ScheduleContext::Ptr& sContext) { std::vector otherDevicesloads; std::vector cpuLoads; if (_pCTPUTLoadContext) { - for (size_t i = 0; i < _autoSContext->_devicePriorities.size(); i++) { + for (size_t i = 0; i < _nCTputDeviceNums; i++) { auto* contextPtr = &_pCTPUTLoadContext[i]; auto modelPath = _autoSContext->_modelPath; auto network = _autoSContext->_network; _pCTPUTLoadContext[i].task = std::bind(loadDeviceTask, contextPtr, modelPath, network, isCumulative); - if (i == _autoSContext->_devicePriorities.size() - 1 && + if (i == _nCTputDeviceNums - 1 && _pCTPUTLoadContext[i].deviceInfo.deviceName.find("CPU") != std::string::npos) { cpuLoads.push_back(_pCTPUTLoadContext[i].task); } else { @@ -518,7 +516,7 @@ void AutoSchedule::init(const ScheduleContext::Ptr& sContext) { _passthroughExeNet = _loadContext[ACTUALDEVICE].executableNetwork; } } - WaitFirstNetworkReady(); + _autoSContext->_hwExecutableNetwork = WaitFirstNetworkReady(); } void AutoSchedule::TryToLoadNetWork(AutoLoadContext& context, const std::string& modelPath, const IE::CNNNetwork& network, bool isCumulative) { @@ -618,7 +616,7 @@ void AutoSchedule::TryToLoadNetWork(AutoLoadContext& context, const std::string& TryToLoadNetWork(context, modelPath, network, isCumulative); } -void AutoSchedule::WaitFirstNetworkReady() { +SoExecNetwork AutoSchedule::WaitFirstNetworkReady() { if (_firstLoadFuture.valid()) { // wait for the first loading finished _firstLoadFuture.wait(); @@ -626,7 +624,7 @@ void AutoSchedule::WaitFirstNetworkReady() { // check if there is any device that have loaded network successfully for (int i = CONTEXTNUM - 2; i >= 0; i--) { if (_loadContext[i].isEnabled && _loadContext[i].isAlready) { - return; + return _loadContext[i].executableNetwork; } } // the first loading is failed, wait for another loading @@ -635,7 +633,7 @@ void AutoSchedule::WaitFirstNetworkReady() { _loadContext[i].future.wait(); // check if loading is successful if (_loadContext[i].isAlready) { - return; + return _loadContext[i].executableNetwork; } } } @@ -646,17 +644,21 @@ void AutoSchedule::WaitFirstNetworkReady() { } } // devices loaded successfully in CTPUT + SoExecNetwork execNetwork; if (_pCTPUTLoadContext) { int nLoadSucNums = 0; - for (size_t i = 0; i < _autoSContext->_devicePriorities.size(); i++) { + for (size_t i = 0; i < _nCTputDeviceNums; i++) { // check if device loaded successfully if (_pCTPUTLoadContext[i].isAlready) { + if (!execNetwork) { + execNetwork = _pCTPUTLoadContext[i].executableNetwork; + } nLoadSucNums++; } } // one or more devices loaded successfully if (nLoadSucNums > 0) { - return; + return execNetwork; } } IE_THROW() << GetLogTag() << "load all devices failed"; @@ -784,7 +786,6 @@ IInferPtr AutoSchedule::CreateInferRequest() { so = _passthroughExeNet._so; syncRequestImpl->setPointerToSo(so); } else if (std::static_pointer_cast(syncRequestImpl)->GetSharedRequest()) { - // cumulative case, load to MULTI:* auto sharedMultiRequest = std::static_pointer_cast(syncRequestImpl)->GetSharedRequest(); if (sharedMultiRequest._ptr->getPointerToSo()) syncRequestImpl->setPointerToSo(sharedMultiRequest._ptr->getPointerToSo()); diff --git a/src/plugins/auto/auto_schedule.hpp b/src/plugins/auto/auto_schedule.hpp index 08be8e5a5e5..fc1308064b5 100644 --- a/src/plugins/auto/auto_schedule.hpp +++ b/src/plugins/auto/auto_schedule.hpp @@ -51,6 +51,7 @@ public: public: AutoLoadContext _loadContext[CONTEXTNUM]; std::unique_ptr _pCTPUTLoadContext = nullptr; + size_t _nCTputDeviceNums; protected: void GenerateWorkers(const std::string& device, const SoExecNetwork& executableNetwork) override; @@ -58,9 +59,15 @@ protected: static bool RunPipelineTask(IE::Task& inferPipelineTask, NotBusyPriorityWorkerRequests& idleWorkerRequests, const DeviceName& preferred_device); DeviceMap _idleWorkerRequests; + AutoScheduleContext::Ptr _autoSContext; private: - void WaitFirstNetworkReady(); + /** + * @brief wait for one of the executable network to finish loading. + * @return An SoPtr object hold an available executable network loaded to HW device. + * @note An exception will be thrown if all loading of network to hw device fails. + */ + SoExecNetwork WaitFirstNetworkReady(); void TryToLoadNetWork(AutoLoadContext& context, const std::string& modelPath, const IE::CNNNetwork& network, bool isCumulative); bool selectOtherDevice(const std::string& currentDeviceName); IE::Task releaseActualdeviceTask; @@ -73,7 +80,6 @@ private: std::promise _firstLoadPromise; bool _exitFlag = {false}; size_t _cpuHelpInferCount = 0; - AutoScheduleContext::Ptr _autoSContext; }; } // namespace MultiDevicePlugin diff --git a/src/plugins/auto/bind_multi_schedule.cpp b/src/plugins/auto/bind_multi_schedule.cpp index 3ab8f30547d..3e623bd984c 100644 --- a/src/plugins/auto/bind_multi_schedule.cpp +++ b/src/plugins/auto/bind_multi_schedule.cpp @@ -6,140 +6,40 @@ #include "async_infer_request.hpp" #include "plugin.hpp" #include "bind_multi_schedule.hpp" -#include "multi_executable_network.hpp" // ------------------------------MultiSchedule---------------------------- namespace MultiDevicePlugin { -thread_local IE::IInferRequestInternal* BinderMultiSchedule::_sharedRequest = nullptr; - void BinderMultiSchedule::init(const ScheduleContext::Ptr& sContext) { - MultiSchedule::init(sContext); + AutoSchedule::init(sContext); + LOG_INFO_TAG("enable bind buffer for AUTO"); } Pipeline BinderMultiSchedule::GetPipeline(const IInferPtr& syncInferRequest, WorkerInferRequest** workerInferRequest) { - Pipeline pipeline = { - // if the request is coming with device-specific remote blobs make sure it is scheduled to the specific device only: - Stage { - /*TaskExecutor*/ std::make_shared(), /*task*/ [this, &syncInferRequest, workerInferRequest]() { - // by default, no preferred device: - _thisPreferredDeviceName = ""; - auto execNetwork = _multiSContext->_executableNetwork.lock(); - // if any input is remote (e.g. was set with SetBlob), let' use the corresponding device - for (const auto& it : execNetwork->GetInputsInfo()) { - auto b = syncInferRequest->GetBlob(it.first); - auto r = b->as(); - if (r) { - const auto name = r->getDeviceName(); - const auto res = std::find_if( - _multiSContext->_devicePrioritiesInitial.cbegin(), - _multiSContext->_devicePrioritiesInitial.cend(), - [&name](const MultiDevicePlugin::DeviceInformation & d) { - return (d.defaultDeviceID.empty() ? d.deviceName : (d.deviceName + "." + - d.defaultDeviceID)) == name; - }); - if (_multiSContext->_devicePrioritiesInitial.cend() == res) { - IE_THROW() << - "None of the devices (for which current MULTI-device configuration was " - "initialized) supports a remote blob created on the device named " << name; - } else { - // it is ok to take the c_str() here (as pointed in the executable_network.hpp we need to use const char*) - // as the original strings are from the "persistent" vector (with the right lifetime) - _thisPreferredDeviceName = res->deviceName.c_str(); - break; - } - } - } - _thisWorkerInferRequest = *workerInferRequest; - _sharedRequest = std::dynamic_pointer_cast(syncInferRequest)->GetSharedRequest()._ptr.get(); - }}, - // as the scheduling algo may select any device, this stage accepts the scheduling decision (actual workerRequest) - // then sets the device-agnostic blobs to the actual (device-specific) request - Stage { - /*TaskExecutor*/std::dynamic_pointer_cast(shared_from_this()), /*task*/ [&syncInferRequest, workerInferRequest]() { - *workerInferRequest = _thisWorkerInferRequest; - auto multiSyncInferRequest = std::dynamic_pointer_cast(syncInferRequest); - multiSyncInferRequest->SetBlobsToAnotherRequest(_thisWorkerInferRequest->_inferRequest); - INFO_RUN([workerInferRequest]() { - (*workerInferRequest)->_startTimes.push_back(std::chrono::steady_clock::now()); - }); - }}, - // final task in the pipeline: - Stage { - /*TaskExecutor*/std::make_shared(workerInferRequest), /*task*/ [this, &syncInferRequest, workerInferRequest]() { - if (nullptr != (*workerInferRequest)->_exceptionPtr) { - std::rethrow_exception((*workerInferRequest)->_exceptionPtr); - } - if (_multiSContext->_needPerfCounters) { - auto multiSyncInferRequest = std::dynamic_pointer_cast - (syncInferRequest); - multiSyncInferRequest->_scheduledRequest = - (*workerInferRequest)->_inferRequest; - } - INFO_RUN([workerInferRequest]() { - (*workerInferRequest)->_endTimes.push_back(std::chrono::steady_clock::now()); - }); - }} - }; - return pipeline; -} - -bool BinderMultiSchedule::ScheduleToWorkerInferRequest(IE::Task inferPipelineTask, DeviceName preferred_device) { - std::vector devices; - devices = [&] { - std::lock_guard lock(_multiSContext->_mutex); - return _multiSContext->_devicePriorities; - }(); - for (auto&& device : devices) { - if (!preferred_device.empty() && (device.deviceName != preferred_device)) { - continue; - } - if (RunPipelineTask(inferPipelineTask, _idleWorkerRequests[device.deviceName], preferred_device)) { - return true; - } - } - // no vacant requests this time, storing the task to the respective queue - if (!preferred_device.empty()) { - _inferPipelineTasksDeviceSpecific[preferred_device]->push(std::move(inferPipelineTask)); - } else { - _inferPipelineTasks.push(std::move(inferPipelineTask)); - } - return false; -} - -bool BinderMultiSchedule::RunPipelineTask(IE::Task& inferPipelineTask, - NotBusyWorkerRequests& idleWorkerRequests, - const DeviceName& preferred_device) { - WorkerInferRequest* workerRequestPtr = nullptr; - WorkerInferRequest* headWorker = nullptr; - bool flag = false; - while (idleWorkerRequests.try_pop(workerRequestPtr)) { - if (flag && workerRequestPtr == headWorker) - break; - if (!flag) { - headWorker = workerRequestPtr; - flag = true; - } - IdleGuard idleGuard{workerRequestPtr, idleWorkerRequests}; - if (_sharedRequest == workerRequestPtr->_inferRequest._ptr.get()) { - _thisWorkerInferRequest = workerRequestPtr; - { - auto capturedTask = std::move(inferPipelineTask); + Pipeline pipeline; + struct RequestExecutor : ITaskExecutor { + explicit RequestExecutor(InferenceEngine::SoIInferRequestInternal& inferRequest) : _inferRequest(inferRequest) { + _inferRequest->SetCallback([this](std::exception_ptr exceptionPtr) mutable { + _exceptionPtr = exceptionPtr; + auto capturedTask = std::move(_task); capturedTask(); - } - idleGuard.Release(); - return true; + }); } - } - return false; -} - -void BinderMultiSchedule::run(IE::Task inferPipelineTask) { - if (_thisWorkerInferRequest) { - auto capturedTask = std::move(inferPipelineTask); - capturedTask(); - } else { - ScheduleToWorkerInferRequest(std::move(inferPipelineTask), _thisPreferredDeviceName); - } + void run(InferenceEngine::Task task) override { + _task = std::move(task); + _inferRequest->StartAsync(); + }; + InferenceEngine::SoIInferRequestInternal& _inferRequest; + std::exception_ptr _exceptionPtr; + InferenceEngine::Task _task; + }; + auto requestExecutor = std::make_shared( + std::static_pointer_cast(syncInferRequest)->GetSharedRequest()); + pipeline.emplace_back(requestExecutor, [requestExecutor] { + if (nullptr != requestExecutor->_exceptionPtr) { + std::rethrow_exception(requestExecutor->_exceptionPtr); + } + }); + return pipeline; } BinderMultiSchedule::~BinderMultiSchedule() { @@ -153,7 +53,7 @@ IInferPtr BinderMultiSchedule::CreateInferRequestImpl( SoInfer request_to_share_blobs_with; // borrowing device-specific blobs from the underlying requests for the device-agnostic, user-facing requests // this allows to potentially save on the data-copy later (if the requests are scheduled in the same order) - for (const auto& device : _multiSContext->_devicePrioritiesInitial) { + for (const auto& device : _autoSContext->_devicePrioritiesInitial) { auto& dev_requests = _workerRequests[device.deviceName]; if ((num - sum) < dev_requests.size()) { request_to_share_blobs_with = dev_requests.at(num - sum)._inferRequest; @@ -177,7 +77,7 @@ IInferPtr BinderMultiSchedule::CreateInferRequestImpl(IE::InputsDataMap networkI size_t sum = 0; // borrowing device-specific blobs from the underlying requests for the device-agnostic, user-facing requests // this allows to potentially save on the data-copy later (if the requests are scheduled in the same order) - for (const auto& device : _multiSContext->_devicePrioritiesInitial) { + for (const auto& device : _autoSContext->_devicePrioritiesInitial) { auto& dev_requests = _workerRequests[device.deviceName]; if ((num - sum) < dev_requests.size()) { request_to_share_blobs_with = dev_requests.at(num - sum)._inferRequest; diff --git a/src/plugins/auto/bind_multi_schedule.hpp b/src/plugins/auto/bind_multi_schedule.hpp index 2217f4fb6d7..7f24af39bac 100644 --- a/src/plugins/auto/bind_multi_schedule.hpp +++ b/src/plugins/auto/bind_multi_schedule.hpp @@ -5,7 +5,7 @@ /////////////////////////////////////////////////////////////////////////////////////////////////// #pragma once -#include "multi_schedule.hpp" +#include "auto_schedule.hpp" #ifdef MULTIUNITTEST #define MOCKTESTMACRO virtual @@ -15,22 +15,14 @@ #endif namespace MultiDevicePlugin { -class BinderMultiSchedule : public MultiSchedule { +class BinderMultiSchedule : public AutoSchedule { public: using Ptr = std::shared_ptr; IInferPtr CreateInferRequestImpl(IE::InputsDataMap networkInputs, IE::OutputsDataMap networkOutputs) override; IE::IInferRequestInternal::Ptr CreateInferRequestImpl(const std::vector>& inputs, const std::vector>& outputs) override; - void run(IE::Task inferTask) override; void init(const ScheduleContext::Ptr& sContext) override; Pipeline GetPipeline(const IInferPtr& syncRequestImpl, WorkerInferRequest** WorkerInferRequest) override; virtual ~BinderMultiSchedule(); - -protected: - static bool RunPipelineTask(IE::Task& inferPipelineTask, NotBusyWorkerRequests& idleWorkerRequests, const DeviceName& preferred_device); - bool ScheduleToWorkerInferRequest(IE::Task, DeviceName preferred_device = "") override; - -protected: - thread_local static IE::IInferRequestInternal* _sharedRequest; }; } // namespace MultiDevicePlugin diff --git a/src/plugins/auto/common.hpp b/src/plugins/auto/common.hpp index 1b2fb9a85ca..891e0bebb01 100644 --- a/src/plugins/auto/common.hpp +++ b/src/plugins/auto/common.hpp @@ -143,7 +143,6 @@ public: std::mutex _mutex; bool _needPerfCounters; bool _batchingDisabled = {false}; - bool _bindBuffer = false; bool _startupfallback = true; bool _runtimeFallback = true; virtual ~MultiScheduleContext() = default; @@ -161,6 +160,7 @@ public: std::mutex _confMutex; std::mutex _fallbackMutex; MultiDeviceInferencePlugin* _plugin; + SoExecNetwork _hwExecutableNetwork; virtual ~AutoScheduleContext() = default; }; diff --git a/src/plugins/auto/multi_schedule.cpp b/src/plugins/auto/multi_schedule.cpp index b2dd7097587..27a84196245 100644 --- a/src/plugins/auto/multi_schedule.cpp +++ b/src/plugins/auto/multi_schedule.cpp @@ -312,7 +312,7 @@ IInferPtr MultiSchedule::CreateInferRequest() { if (!so) so = _passthroughExeNet._so; syncRequestImpl->setPointerToSo(so); - } else if (_multiSContext->_bindBuffer) { + } else if (std::static_pointer_cast(syncRequestImpl)->GetSharedRequest()) { auto sharedRequest = std::static_pointer_cast(syncRequestImpl)->GetSharedRequest(); if (sharedRequest._ptr->getPointerToSo()) syncRequestImpl->setPointerToSo(sharedRequest._ptr->getPointerToSo()); diff --git a/src/plugins/auto/plugin.cpp b/src/plugins/auto/plugin.cpp index 65bd8793f71..531af4ef31f 100644 --- a/src/plugins/auto/plugin.cpp +++ b/src/plugins/auto/plugin.cpp @@ -351,19 +351,21 @@ IExecutableNetworkInternal::Ptr MultiDeviceInferencePlugin::LoadNetworkImpl(cons auto loadConfig = _pluginConfig; // if no perf hint from user with compiled model, or already been set with plugin // apply latency for AUTO, tput for MULTI - bool isHintSet = _pluginConfig.is_set_by_user(ov::hint::performance_mode) || config.find(ov::hint::performance_mode.name()) != config.end(); - if (!isHintSet) { - if (workModeAuto) { - // set performance hint to 'LATENCY' model for AutoExecutable Network. - loadConfig.set_property(ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY)); - } else { - // set performance hint to 'THROUGHPUT' model for MultiExecutable Network. - loadConfig.set_property(ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT)); - } + auto itorConfig = config.find(ov::hint::performance_mode.name()); + bool isHintSet = _pluginConfig.is_set_by_user(ov::hint::performance_mode) || itorConfig != config.end(); + if (!isHintSet && workModeAuto) { + // NO user sets perfHint, then set perfhint to 'LATENCY' for AutoExecutableNetwork. + loadConfig.set_property(ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY)); } // updateFromMap will check config valid loadConfig.set_user_property(PreProcessConfig(config), workModeAuto); loadConfig.apply_user_properties(); + if (!workModeAuto) { + if (itorConfig != config.end() && itorConfig->second != InferenceEngine::PluginConfigParams::THROUGHPUT) { + LOG_WARNING_TAG("User set perf_hint:%s, but MULTI supports THROUGHPUT only", itorConfig->second.c_str()); + } + loadConfig.set_property(ov::hint::performance_mode(ov::hint::PerformanceMode::CUMULATIVE_THROUGHPUT)); + } auto fullProperty = loadConfig.get_full_properties(); // this can be updated when plugin switch to 2.0 API std::map fullConfig = ConvertToStringMap(fullProperty); @@ -378,6 +380,8 @@ IExecutableNetworkInternal::Ptr MultiDeviceInferencePlugin::LoadNetworkImpl(cons std::unordered_map multiNetworkConfig; std::vector metaDevices; auto priorities = loadConfig.get_property(ov::device::priorities); + if (priorities.empty() && !workModeAuto) + IE_THROW() << "KEY_MULTI_DEVICE_PRIORITIES key is not set for " << GetName() << " device"; if (priorities.find("AUTO") != std::string::npos || priorities.find("MULTI") != std::string::npos) { IE_THROW() << "The device candidate list should not include the meta plugin for " << GetName() << " device"; } @@ -397,256 +401,104 @@ IExecutableNetworkInternal::Ptr MultiDeviceInferencePlugin::LoadNetworkImpl(cons } }; - // if workMode is AUTO - // only AUTO uses CheckConfig() to check fullConfig's parameters, MULTI does not - if (workModeAuto) { - // check the configure and check if need to set PerfCounters configure to device - // and set filter configure - - OV_ITT_SCOPED_TASK(itt::domains::MULTIPlugin, "MultiDeviceInferencePlugin::LoadNetworkImpl::AutoMode"); - auto autoSContext = std::make_shared(); - std::map filterConfig; - auto strDevices = GetDeviceList(fullConfig); - // fill in the context for auto - if (loadConfig.get_property(ov::enable_profiling)) { - filterConfig.insert({ov::enable_profiling.name(), PluginConfigParams::YES}); - autoSContext->_needPerfCounters = true; - } - autoSContext->_modelPriority = MapPriorityValues(loadConfig.get_property(ov::hint::model_priority)); - autoSContext->_batchingDisabled = !(loadConfig.get_property(ov::hint::allow_auto_batching)); - autoSContext->_performanceHint = loadConfig.get_property(ov::hint::performance_mode.name()).as(); - // filter the device that supports filter configure - auto metaDevices = ParseMetaDevices(strDevices, fullConfig); - auto supportDevicesByConfig = FilterDevice(metaDevices, filterConfig); - if (supportDevicesByConfig.empty()) { - IE_THROW() << "There is no device support the configure"; - } - auto supportDevices = supportDevicesByConfig; - CNNNetwork clonedNetwork; - std::string clonedModelPath = modelPath; - if (modelPath.empty()) { - // if network is valid - LOG_INFO_TAG("load with CNN network"); - supportDevices = FilterDeviceByNetwork(supportDevicesByConfig, network); - // clone the network, in case of reshape conflict - clonedNetwork = InferenceEngine::details::cloneNetwork(network); - } else { - // model path, enable model load with single device situation - if (supportDevices.size() > 1) { - clonedNetwork = GetCore()->ReadNetwork(modelPath, std::string()); - // do we really need to disable model path? - clonedModelPath = ""; - LOG_INFO_TAG("load with CNN network"); - } else { - LOG_INFO_TAG("load with model path"); - } - } - // reset the strDevices to support devices - strDevices = ""; - // calling GetValidDevices() to get a prioritized list of devices - auto devicesWithPriority = GetValidDevice(supportDevices, networkPrecision); - for (auto iter = devicesWithPriority.begin(); iter != devicesWithPriority.end(); iter++) { - strDevices += iter->deviceName; - strDevices += ","; - } - strDevices.pop_back(); - for (auto iter = supportDevices.begin(); iter != supportDevices.end(); iter++) { - auto& configs = iter->config; - for (auto& config : configs) { - LOG_INFO_TAG("device:%s, config:%s=%s", - iter->deviceName.c_str(), - config.first.c_str(), - config.second.c_str()); - } - // carry on batch configs only if user explicitly sets - if (loadConfig.is_set_by_user(ov::hint::allow_auto_batching)) - insertPropToConfig(ov::hint::allow_auto_batching.name(), iter->deviceName, configs); - if (loadConfig.is_set_by_user(ov::auto_batch_timeout)) - insertPropToConfig(ov::auto_batch_timeout.name(), iter->deviceName, configs); - insertPropToConfig(ov::cache_dir.name(), iter->deviceName, configs); - LOG_INFO_TAG("device:%s, priority:%ld", iter->deviceName.c_str(), iter->devicePriority); - } - autoSContext->_modelPath = clonedModelPath; + // check the configure and check if need to set PerfCounters configure to device + // and set filter configure + OV_ITT_SCOPED_TASK(itt::domains::MULTIPlugin, "MultiDeviceInferencePlugin::LoadNetworkImpl::AutoMode"); + auto autoSContext = std::make_shared(); + std::map filterConfig; + auto strDevices = GetDeviceList(fullConfig); + // fill in the context for auto + if (loadConfig.get_property(ov::enable_profiling)) { + filterConfig.insert({ov::enable_profiling.name(), PluginConfigParams::YES}); + autoSContext->_needPerfCounters = true; + } + autoSContext->_modelPriority = MapPriorityValues(loadConfig.get_property(ov::hint::model_priority)); + autoSContext->_batchingDisabled = !(loadConfig.get_property(ov::hint::allow_auto_batching)); + // set performanceHint for AutoExecutableNetwork + autoSContext->_performanceHint = loadConfig.get_property(ov::hint::performance_mode.name()).as(); + // filter the device that supports filter configure + metaDevices = ParseMetaDevices(strDevices, fullConfig); + auto supportDevicesByConfig = FilterDevice(metaDevices, filterConfig); + if (supportDevicesByConfig.empty()) { + IE_THROW() << "There is no device support the configure"; + } + auto supportDevices = supportDevicesByConfig; + CNNNetwork clonedNetwork; + std::string clonedModelPath = modelPath; + // reset the strDevices to support devices + strDevices = ""; + // calling GetValidDevices() to get a prioritized list of devices + bool isCumulative = + (autoSContext->_performanceHint == IE::PluginConfigParams::CUMULATIVE_THROUGHPUT) ? true : false; + std::list devicesWithPriority(supportDevices.begin(), supportDevices.end()); + if (modelPath.empty()) { + // if network is valid + LOG_INFO_TAG("load with CNN network"); + supportDevices = FilterDeviceByNetwork(supportDevicesByConfig, network); + clonedNetwork = InferenceEngine::details::cloneNetwork(network); // clone the network, in case of reshape conflict - autoSContext->_network = clonedNetwork; - autoSContext->_devicePriorities = supportDevices; - autoSContext->_devicePrioritiesInitial = supportDevices; - autoSContext->_strDevices = strDevices; - autoSContext->_plugin = this; - autoSContext->_core = GetCore(); - autoSContext->_LogTag = _LogTag; - autoSContext->_bindBuffer = loadConfig.get_property(ov::intel_auto::device_bind_buffer); - autoSContext->_startupfallback = loadConfig.get_property(ov::intel_auto::enable_startup_fallback); - autoSContext->_runtimeFallback = loadConfig.get_property(ov::intel_auto::enable_runtime_fallback); - return std::make_shared(autoSContext, std::make_shared()); - } - OV_ITT_SCOPED_TASK(itt::domains::MULTIPlugin, "MultiDeviceInferencePlugin::LoadNetworkImpl:MultiMode"); - // if is cumulative, PERFORMANCE_HINT set to THROUGHPUT and _LogTag set to AUTO - auto configIter = - std::find_if(fullConfig.begin(), fullConfig.end(), [](const std::pair& config) { - return (config.first == CONFIG_KEY(PERFORMANCE_HINT)); - }); - if (configIter != fullConfig.end() && configIter->second == InferenceEngine::PluginConfigParams::CUMULATIVE_THROUGHPUT) { - configIter->second = InferenceEngine::PluginConfigParams::THROUGHPUT; - _LogTag = "AUTO"; - LOG_INFO_TAG("CUMULATIVE Call MULTI PERFORMANCE_HINT set to THROUGHPUT"); - } - if (priorities.empty()) { - IE_THROW() << "KEY_MULTI_DEVICE_PRIORITIES key is not set for " << GetName() << " device"; - } else { // for use case -d MULTI:xPU or -d AUTO:xPU - auto metaDevicesByConfig = ParseMetaDevices(priorities, fullConfig); - metaDevices = modelPath.empty() ? FilterDeviceByNetwork(metaDevicesByConfig, network) - : metaDevicesByConfig; - if (metaDevicesByConfig.size() != metaDevices.size()) { - LOG_DEBUG_TAG("stateful/dynamic model, loaded to single device"); - multiNetworkConfig[ov::device::priorities.name()] - = metaDevices[0].deviceName; - } else { - multiNetworkConfig[ov::device::priorities.name()] = priorities; - } - } - auto multiSContext = std::make_shared(); - DeviceMap executableNetworkPerDevice; - std::mutex load_mutex; - std::vector loads; - - auto loadInferEngTask = [&](DeviceInformation& p) { - auto tmpiter = fullConfig.find(CONFIG_KEY(ALLOW_AUTO_BATCHING)); - if (tmpiter != fullConfig.end()) { - if (tmpiter->second == PluginConfigParams::NO) { - LOG_INFO_TAG("set %s=%s", tmpiter->first.c_str(), tmpiter->second.c_str()); - multiSContext->_batchingDisabled = true; - } - if (loadConfig.is_set_by_user(ov::hint::allow_auto_batching)) - p.config.insert({tmpiter->first, tmpiter->second}); - } - if (loadConfig.is_set_by_user(ov::auto_batch_timeout)) - insertPropToConfig(ov::auto_batch_timeout.name(), p.deviceName, p.config); - insertPropToConfig(ov::cache_dir.name(), p.deviceName, p.config); - const auto& deviceName = p.deviceName; - const auto& deviceConfig = p.config; - SoExecutableNetworkInternal exec_net; - LOG_DEBUG_TAG("load network to device:%s", deviceName.c_str()); - try { - if (modelPath.empty()) { - exec_net = GetCore()->LoadNetwork(network, deviceName, deviceConfig); - } else { - exec_net = GetCore()->LoadNetwork(modelPath, deviceName, deviceConfig); - } - } catch (const IE::Exception& iie) { - if (_LogTag == "AUTO") { - LOG_DEBUG_TAG("Failed to load network to device:%s with error: %s", deviceName.c_str(), iie.what()); - return; - } else { - IE_THROW() << "Failed to load network to device: " << deviceName.c_str() << " with error:" << - iie.what(); - } - } - - try { - std::string sStreamNums = ""; - std::string sThreadNums = ""; - if (deviceName.find("CPU") != std::string::npos) { - sStreamNums = exec_net->GetMetric(ov::num_streams.name()).as(); - sThreadNums = exec_net->GetMetric(ov::inference_num_threads.name()).as(); - } else if (deviceName.find("GPU") != std::string::npos) { - sStreamNums = exec_net->GetConfig(ov::num_streams.name()).as(); - sThreadNums = exec_net->GetConfig(ov::compilation_num_threads.name()).as(); - } - - // print CPU or GPU streams num and threads num - if (!sStreamNums.empty() && !sThreadNums.empty()) { - LOG_INFO_TAG("after load network, %s streamNums:%s, %s threadNums:%s", - deviceName.c_str(), - sStreamNums.c_str(), - deviceName.c_str(), - sThreadNums.c_str()); - } - } catch (const IE::Exception&) { - LOG_DEBUG_TAG("deviceName:%s cannot get streamNums and threadNums from exec_net", deviceName.c_str()); - } - std::unique_lock lock{load_mutex}; - executableNetworkPerDevice.insert({deviceName, exec_net}); - multiNetworkConfig.insert(deviceConfig.begin(), deviceConfig.end()); - }; - - // Check if CPU is in device list - auto iterCPU = std::find_if(metaDevices.begin(), metaDevices.end(), [&](DeviceInformation& d) { - return d.deviceName.find("CPU") != std::string::npos; - }); - // Load devices other than CPU first - for (auto& p : metaDevices) { - if (iterCPU != metaDevices.end() && p.deviceName == iterCPU->deviceName) { - continue; - } - loads.push_back([&]() { - loadInferEngTask(p); - }); - } - - auto executor = executorManager()->getIdleCPUStreamsExecutor( - IStreamsExecutor::Config{"MultiDeviceAsyncLoad", - static_cast(std::thread::hardware_concurrency()) /* max possible #streams*/, - 0 /*default threads per stream, workaround for ticket 62376*/, - IStreamsExecutor::ThreadBindingType::NONE}); - if (loads.size() > 0) { - // Wait for the device to load the network - executor->runAndWait(loads); - loads.clear(); - } - - // Finally load the CPU - if (iterCPU != metaDevices.end()) { - if (!executableNetworkPerDevice.empty() && iterCPU->config.find(ov::affinity.name()) == iterCPU->config.end()) { - LOG_DEBUG_TAG("set affinity to NUMA and disable hyper thread for CPU"); - // If the other devices load successfully and no user set affinity then set NUMA to CPU - iterCPU->config.insert({ov::affinity.name(), ov::affinity(ov::Affinity::NUMA).second.as()}); - iterCPU->config.insert({CONFIG_KEY_INTERNAL(ENABLE_HYPER_THREAD), CONFIG_VALUE(NO)}); - } - loads.push_back([&]() { - loadInferEngTask(*iterCPU); - }); - // Wait for CPU to load the network - executor->runAndWait(loads); - } - - if (executableNetworkPerDevice.empty()) - IE_THROW(NotFound) << "Failed to load network to any device " - << "that the " << GetName() << " device is initialized to work with"; - - // checking the perf counters config from the loaded network to respect both device's plugin and load-specific setting - size_t num_plugins_supporting_perf_counters = 0; - for (auto& n : executableNetworkPerDevice) { - try { - num_plugins_supporting_perf_counters += - n.second->GetConfig(PluginConfigParams::KEY_PERF_COUNT).as() == - PluginConfigParams::YES; - } catch (const IE::Exception&) { - } - } - // MULTI can enable the perf counters only if all devices support/enable that - bool enablePerfCounters = num_plugins_supporting_perf_counters == executableNetworkPerDevice.size(); - multiSContext->_devicePriorities = metaDevices; - multiSContext->_devicePrioritiesInitial = metaDevices; - multiSContext->_networksPerDevice = executableNetworkPerDevice; - multiSContext->_config = multiNetworkConfig; - multiSContext->_needPerfCounters = enablePerfCounters; - multiSContext->_core = GetCore(); - multiSContext->_LogTag = _LogTag; - IExecutableNetworkInternal::Ptr impl; - auto tmp = loadConfig.get_property(ov::intel_auto::device_bind_buffer); - if (tmp) { - multiSContext->_bindBuffer = true; - impl = std::make_shared(multiSContext, std::make_shared()); } else { - impl = std::make_shared(multiSContext, std::make_shared()); + // model path, enable model load with single device situation + if (supportDevices.size() > 1 && !isCumulative) { + clonedNetwork = GetCore()->ReadNetwork(modelPath, std::string()); + // do we really need to disable model path? + clonedModelPath = ""; + LOG_INFO_TAG("load with CNN network"); + } else { + LOG_INFO_TAG("load with model path"); + } + } + if (!isCumulative) { + devicesWithPriority = GetValidDevice(supportDevices, networkPrecision); + } + for (auto iter = devicesWithPriority.begin(); iter != devicesWithPriority.end(); iter++) { + strDevices += iter->deviceName; + strDevices += ","; + } + strDevices.pop_back(); + for (auto iter = supportDevices.begin(); iter != supportDevices.end(); iter++) { + auto& configs = iter->config; + for (auto& config : configs) { + LOG_INFO_TAG("device:%s, config:%s=%s", + iter->deviceName.c_str(), + config.first.c_str(), + config.second.c_str()); + } + // carry on batch configs only if user explicitly sets + if (loadConfig.is_set_by_user(ov::hint::allow_auto_batching)) + insertPropToConfig(ov::hint::allow_auto_batching.name(), iter->deviceName, configs); + if (loadConfig.is_set_by_user(ov::auto_batch_timeout)) + insertPropToConfig(ov::auto_batch_timeout.name(), iter->deviceName, configs); + insertPropToConfig(ov::cache_dir.name(), iter->deviceName, configs); + LOG_INFO_TAG("device:%s, priority:%ld", iter->deviceName.c_str(), iter->devicePriority); + } + autoSContext->_modelPath = clonedModelPath; + // clone the network, in case of reshape conflict + autoSContext->_network = clonedNetwork; + autoSContext->_devicePriorities = supportDevices; + autoSContext->_devicePrioritiesInitial = supportDevices; + autoSContext->_strDevices = strDevices; + autoSContext->_plugin = this; + autoSContext->_core = GetCore(); + autoSContext->_LogTag = _LogTag; + autoSContext->_startupfallback = loadConfig.get_property(ov::intel_auto::enable_startup_fallback); + autoSContext->_runtimeFallback = loadConfig.get_property(ov::intel_auto::enable_runtime_fallback); + IExecutableNetworkInternal::Ptr impl; + // enable bind only in cumulative_throughput mode + if (loadConfig.get_property(ov::intel_auto::device_bind_buffer) && + autoSContext->_performanceHint == "CUMULATIVE_THROUGHPUT") { + LOG_INFO_TAG("runtime fallback set to disabled in binder mode"); + autoSContext->_runtimeFallback = false; + impl = std::make_shared(autoSContext, std::make_shared()); + } else { + impl = std::make_shared(autoSContext, std::make_shared()); } if (!modelPath.empty()) { SetExeNetworkInfo(impl, - executableNetworkPerDevice.begin()->second->GetInputsInfo(), - executableNetworkPerDevice.begin()->second->GetOutputsInfo()); - impl->setInputs(executableNetworkPerDevice.begin()->second->getInputs()); - impl->setOutputs(executableNetworkPerDevice.begin()->second->getOutputs()); + autoSContext->_hwExecutableNetwork->GetInputsInfo(), + autoSContext->_hwExecutableNetwork->GetOutputsInfo()); + impl->setInputs(autoSContext->_hwExecutableNetwork->getInputs()); + impl->setOutputs(autoSContext->_hwExecutableNetwork->getOutputs()); } return impl; } diff --git a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/behavior/ov_plugin/core_integration.cpp b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/behavior/ov_plugin/core_integration.cpp index 1db9e1864e1..af5e0dc09ee 100644 --- a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/behavior/ov_plugin/core_integration.cpp +++ b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/behavior/ov_plugin/core_integration.cpp @@ -411,12 +411,12 @@ INSTANTIATE_TEST_SUITE_P(smoke_HETERO_OVClassLoadNetworkWithSecondaryPropertiesT // IE Class load and check network with ov::device::properties INSTANTIATE_TEST_SUITE_P(smoke_CPU_OVClassLoadNetworkAndCheckWithSecondaryPropertiesTest, OVClassLoadNetworkAndCheckSecondaryPropertiesTest, - ::testing::Combine(::testing::Values("CPU", "MULTI:CPU"), + ::testing::Combine(::testing::Values("CPU"), ::testing::ValuesIn(configsDeviceProperties))); INSTANTIATE_TEST_SUITE_P(smoke_CPU_OVClassLoadNetworkAndCheckWithSecondaryPropertiesDoubleTest, OVClassLoadNetworkAndCheckSecondaryPropertiesTest, - ::testing::Combine(::testing::Values("CPU", "MULTI:CPU"), + ::testing::Combine(::testing::Values("CPU"), ::testing::ValuesIn(configsDevicePropertiesDouble))); INSTANTIATE_TEST_SUITE_P( smoke_OVClassLoadNetworkTest, OVClassLoadNetworkTest, diff --git a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/behavior/ov_plugin/properties_tests.cpp b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/behavior/ov_plugin/properties_tests.cpp index 550acd81644..5362f5af947 100644 --- a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/behavior/ov_plugin/properties_tests.cpp +++ b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/behavior/ov_plugin/properties_tests.cpp @@ -56,27 +56,13 @@ INSTANTIATE_TEST_SUITE_P(smoke_cpuCompileModelBehaviorTests, OVSetPropComplieModleGetPropTests::getTestCaseName); const std::vector multi_setcore_properties = { - {ov::device::priorities(CommonTestUtils::DEVICE_CPU), - ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT), - ov::hint::num_requests(2), - ov::hint::allow_auto_batching(false), - ov::enable_profiling(false)}, {ov::device::priorities(CommonTestUtils::DEVICE_CPU), ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY), - ov::hint::num_requests(8), - ov::hint::allow_auto_batching(true), - ov::enable_profiling(true)}}; + ov::hint::model_priority(ov::hint::Priority::HIGH)}}; const std::vector multi_compileModel_properties = { - {ov::device::priorities(CommonTestUtils::DEVICE_CPU), - ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY), - ov::hint::num_requests(10), - ov::hint::allow_auto_batching(true), - ov::enable_profiling(true)}, {ov::device::priorities(CommonTestUtils::DEVICE_CPU), ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT), - ov::hint::num_requests(2), - ov::hint::allow_auto_batching(false), - ov::enable_profiling(false)}}; + ov::hint::model_priority(ov::hint::Priority::MEDIUM)}}; INSTANTIATE_TEST_SUITE_P(smoke_MultiCompileModelBehaviorTests, OVSetPropComplieModleGetPropTests, @@ -88,36 +74,24 @@ INSTANTIATE_TEST_SUITE_P(smoke_MultiCompileModelBehaviorTests, const std::vector auto_setcore_properties = { {ov::device::priorities(CommonTestUtils::DEVICE_CPU), ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT), - ov::hint::num_requests(2), - ov::hint::allow_auto_batching(false), - ov::enable_profiling(false)}, + ov::hint::model_priority(ov::hint::Priority::HIGH)}, {ov::device::priorities(CommonTestUtils::DEVICE_CPU), ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY), - ov::hint::num_requests(8), - ov::hint::allow_auto_batching(true), - ov::enable_profiling(true)}, + ov::hint::model_priority(ov::hint::Priority::HIGH)}, {ov::device::priorities(CommonTestUtils::DEVICE_CPU), ov::hint::performance_mode(ov::hint::PerformanceMode::CUMULATIVE_THROUGHPUT), - ov::hint::num_requests(10), - ov::hint::allow_auto_batching(false), - ov::enable_profiling(true)}, + ov::hint::model_priority(ov::hint::Priority::HIGH)}, }; const std::vector auto_compileModel_properties = { {ov::device::priorities(CommonTestUtils::DEVICE_CPU), ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY), - ov::hint::num_requests(8), - ov::hint::allow_auto_batching(true), - ov::enable_profiling(true)}, + ov::hint::model_priority(ov::hint::Priority::MEDIUM)}, {ov::device::priorities(CommonTestUtils::DEVICE_CPU), ov::hint::performance_mode(ov::hint::PerformanceMode::CUMULATIVE_THROUGHPUT), - ov::hint::num_requests(10), - ov::hint::allow_auto_batching(false), - ov::enable_profiling(false)}, + ov::hint::model_priority(ov::hint::Priority::MEDIUM)}, {ov::device::priorities(CommonTestUtils::DEVICE_CPU), ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT), - ov::hint::num_requests(2), - ov::hint::allow_auto_batching(true), - ov::enable_profiling(false)}}; + ov::hint::model_priority(ov::hint::Priority::MEDIUM)}}; INSTANTIATE_TEST_SUITE_P(smoke_AutoCompileModelBehaviorTests, OVSetPropComplieModleGetPropTests, ::testing::Combine(::testing::Values(CommonTestUtils::DEVICE_AUTO), diff --git a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/behavior/plugin/configuration_tests.cpp b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/behavior/plugin/configuration_tests.cpp index 27368152408..5d62856303f 100644 --- a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/behavior/plugin/configuration_tests.cpp +++ b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/behavior/plugin/configuration_tests.cpp @@ -281,18 +281,12 @@ namespace { const std::vector> auto_multi_prop_config = { {{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES, CommonTestUtils::DEVICE_CPU}, {InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT, InferenceEngine::PluginConfigParams::THROUGHPUT}, - {InferenceEngine::PluginConfigParams::KEY_EXCLUSIVE_ASYNC_REQUESTS, InferenceEngine::PluginConfigParams::YES}, - {InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT_NUM_REQUESTS, "2"}, - {InferenceEngine::PluginConfigParams::KEY_ALLOW_AUTO_BATCHING, InferenceEngine::PluginConfigParams::NO}, - {InferenceEngine::PluginConfigParams::KEY_PERF_COUNT, InferenceEngine::PluginConfigParams::NO}}}; + {InferenceEngine::PluginConfigParams::KEY_MODEL_PRIORITY, InferenceEngine::PluginConfigParams::MODEL_PRIORITY_MED}}}; const std::vector> auto_multi_loadNetWork_config = { {{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES, CommonTestUtils::DEVICE_CPU}, - {InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT, InferenceEngine::PluginConfigParams::LATENCY}, - {InferenceEngine::PluginConfigParams::KEY_EXCLUSIVE_ASYNC_REQUESTS, InferenceEngine::PluginConfigParams::NO}, - {InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT_NUM_REQUESTS, "10"}, - {InferenceEngine::PluginConfigParams::KEY_ALLOW_AUTO_BATCHING, InferenceEngine::PluginConfigParams::YES}, - {InferenceEngine::PluginConfigParams::KEY_PERF_COUNT, InferenceEngine::PluginConfigParams::YES}}}; + {InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT, InferenceEngine::PluginConfigParams::THROUGHPUT}, + {InferenceEngine::PluginConfigParams::KEY_MODEL_PRIORITY, InferenceEngine::PluginConfigParams::MODEL_PRIORITY_HIGH}}}; INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, SetPropLoadNetWorkGetPropTests, diff --git a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/skip_tests_config.cpp b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/skip_tests_config.cpp index 5e246017855..a2fe8fcfb6f 100644 --- a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/skip_tests_config.cpp +++ b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/skip_tests_config.cpp @@ -93,7 +93,9 @@ std::vector disabledTestPatterns() { R"(.*Behavior.*ExecutableNetworkBaseTest.*canSetConfigToExecNetWithIncorrectConfig.*)", R"(.*Hetero.*Behavior.*ExecutableNetworkBaseTest.*ExecGraphInfo.*)", R"(.*Hetero.*Behavior.*ExecutableNetworkBaseTest.*CanCreateTwoExeNetworksAndCheckFunction.*)", - + // TODO: CVS-104942 + R"(.*(Auto|Multi).*Behavior.*ExecutableNetworkBaseTest.*canLoadCorrectNetworkToGetExecutableAndCheckConfig.*)", + R"(.*(Auto|Multi).*SetPropLoadNetWorkGetPropTests.*)", // CPU does not support dynamic rank // Issue: CVS-66778 R"(.*smoke_BehaviorTests.*InferFullyDynamicNetworkWith(S|G)etTensor.*)", diff --git a/src/tests/functional/plugin/gpu/shared_tests_instances/behavior/ov_plugin/properties_tests.cpp b/src/tests/functional/plugin/gpu/shared_tests_instances/behavior/ov_plugin/properties_tests.cpp index 479141520a7..e051908f204 100644 --- a/src/tests/functional/plugin/gpu/shared_tests_instances/behavior/ov_plugin/properties_tests.cpp +++ b/src/tests/functional/plugin/gpu/shared_tests_instances/behavior/ov_plugin/properties_tests.cpp @@ -109,25 +109,11 @@ INSTANTIATE_TEST_SUITE_P(smoke_gpuCompileModelBehaviorTests, const std::vector multi_setcore_properties = { {ov::device::priorities(CommonTestUtils::DEVICE_GPU), ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT), - ov::hint::num_requests(2), - ov::hint::allow_auto_batching(false), - ov::enable_profiling(false)}, - {ov::device::priorities(CommonTestUtils::DEVICE_GPU), - ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY), - ov::hint::num_requests(8), - ov::hint::allow_auto_batching(true), - ov::enable_profiling(true)}}; + ov::hint::model_priority(ov::hint::Priority::HIGH)}}; const std::vector multi_compileModel_properties = { - {ov::device::priorities(CommonTestUtils::DEVICE_GPU), - ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY), - ov::hint::num_requests(10), - ov::hint::allow_auto_batching(true), - ov::enable_profiling(true)}, {ov::device::priorities(CommonTestUtils::DEVICE_GPU), ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT), - ov::hint::num_requests(2), - ov::hint::allow_auto_batching(false), - ov::enable_profiling(false)}}; + ov::hint::model_priority(ov::hint::Priority::MEDIUM)}}; INSTANTIATE_TEST_SUITE_P(smoke_MultiCompileModelBehaviorTests, OVSetPropComplieModleGetPropTests, @@ -139,36 +125,24 @@ INSTANTIATE_TEST_SUITE_P(smoke_MultiCompileModelBehaviorTests, const std::vector auto_setcore_properties = { {ov::device::priorities(CommonTestUtils::DEVICE_GPU), ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT), - ov::hint::num_requests(2), - ov::hint::allow_auto_batching(false), - ov::enable_profiling(false)}, + ov::hint::model_priority(ov::hint::Priority::HIGH)}, {ov::device::priorities(CommonTestUtils::DEVICE_GPU), ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY), - ov::hint::num_requests(8), - ov::hint::allow_auto_batching(true), - ov::enable_profiling(true)}, + ov::hint::model_priority(ov::hint::Priority::HIGH)}, {ov::device::priorities(CommonTestUtils::DEVICE_GPU), ov::hint::performance_mode(ov::hint::PerformanceMode::CUMULATIVE_THROUGHPUT), - ov::hint::num_requests(10), - ov::hint::allow_auto_batching(false), - ov::enable_profiling(true)}, + ov::hint::model_priority(ov::hint::Priority::HIGH)}, }; const std::vector auto_compileModel_properties = { {ov::device::priorities(CommonTestUtils::DEVICE_GPU), ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY), - ov::hint::num_requests(8), - ov::hint::allow_auto_batching(true), - ov::enable_profiling(true)}, + ov::hint::model_priority(ov::hint::Priority::MEDIUM)}, {ov::device::priorities(CommonTestUtils::DEVICE_GPU), ov::hint::performance_mode(ov::hint::PerformanceMode::CUMULATIVE_THROUGHPUT), - ov::hint::num_requests(10), - ov::hint::allow_auto_batching(false), - ov::enable_profiling(false)}, + ov::hint::model_priority(ov::hint::Priority::MEDIUM)}, {ov::device::priorities(CommonTestUtils::DEVICE_GPU), ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT), - ov::hint::num_requests(2), - ov::hint::allow_auto_batching(true), - ov::enable_profiling(false)}}; + ov::hint::model_priority(ov::hint::Priority::MEDIUM)}}; INSTANTIATE_TEST_SUITE_P(smoke_AutoCompileModelBehaviorTests, OVSetPropComplieModleGetPropTests, ::testing::Combine(::testing::Values(CommonTestUtils::DEVICE_AUTO), diff --git a/src/tests/functional/plugin/gpu/shared_tests_instances/behavior/plugin/configuration_tests.cpp b/src/tests/functional/plugin/gpu/shared_tests_instances/behavior/plugin/configuration_tests.cpp index f6dadba3496..b68d29ac758 100644 --- a/src/tests/functional/plugin/gpu/shared_tests_instances/behavior/plugin/configuration_tests.cpp +++ b/src/tests/functional/plugin/gpu/shared_tests_instances/behavior/plugin/configuration_tests.cpp @@ -309,22 +309,16 @@ namespace { {{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES, CommonTestUtils::DEVICE_GPU}, {InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT, InferenceEngine::PluginConfigParams::THROUGHPUT}, - {InferenceEngine::PluginConfigParams::KEY_EXCLUSIVE_ASYNC_REQUESTS, - InferenceEngine::PluginConfigParams::YES}, - {InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT_NUM_REQUESTS, "2"}, - {InferenceEngine::PluginConfigParams::KEY_ALLOW_AUTO_BATCHING, InferenceEngine::PluginConfigParams::NO}, - {InferenceEngine::PluginConfigParams::KEY_PERF_COUNT, InferenceEngine::PluginConfigParams::NO}}}; + {InferenceEngine::PluginConfigParams::KEY_MODEL_PRIORITY, + InferenceEngine::PluginConfigParams::MODEL_PRIORITY_MED}}}; }; auto auto_multi_loadNetWork_config = []() { return std::vector>{ {{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES, CommonTestUtils::DEVICE_GPU}, - {InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT, InferenceEngine::PluginConfigParams::LATENCY}, - {InferenceEngine::PluginConfigParams::KEY_EXCLUSIVE_ASYNC_REQUESTS, - InferenceEngine::PluginConfigParams::NO}, - {InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT_NUM_REQUESTS, "10"}, - {InferenceEngine::PluginConfigParams::KEY_ALLOW_AUTO_BATCHING, InferenceEngine::PluginConfigParams::YES}, - {InferenceEngine::PluginConfigParams::KEY_PERF_COUNT, InferenceEngine::PluginConfigParams::YES}}}; + {InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT, InferenceEngine::PluginConfigParams::THROUGHPUT}, + {InferenceEngine::PluginConfigParams::KEY_MODEL_PRIORITY, + InferenceEngine::PluginConfigParams::MODEL_PRIORITY_HIGH}}}; }; INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, diff --git a/src/tests/functional/plugin/gpu/shared_tests_instances/multi/gpu_remote_blob_tests.cpp b/src/tests/functional/plugin/gpu/shared_tests_instances/multi/gpu_remote_blob_tests.cpp index e95d69995b6..10b67ab10d3 100644 --- a/src/tests/functional/plugin/gpu/shared_tests_instances/multi/gpu_remote_blob_tests.cpp +++ b/src/tests/functional/plugin/gpu/shared_tests_instances/multi/gpu_remote_blob_tests.cpp @@ -106,12 +106,17 @@ TEST_P(MultiDevice_Bind_oversubsciption_test, oversubsciptionOfInferRequest) { auto device_names_and_support_for_remote_blobs2 = []() { return std::vector{ + // another GPU (the test will test its presence), different OCL contexts + // use GPU.0 as reference, expect auto to throw exception on other hardware contexts #ifdef ENABLE_INTEL_CPU //{{CPU}, {}}, // stand-alone CPU via MULTI (no GPU), no OCL context - {{CPU}, {ov::intel_auto::device_bind_buffer(true)}}, // stand-alone CPU via MULTI (no GPU), no OCL context + {{"GPU.1", CPU}, + {ov::intel_auto::device_bind_buffer(true)}}, + {{"GPU.1", CPU}, + {ov::intel_auto::device_bind_buffer(false)}}, #endif - {{"GPU.1"}, {}}, // another GPU (the test will test its presence), different OCL contexts - {{"GPU.1"}, {ov::intel_auto::device_bind_buffer(true)}}, // another GPU (the test will test its presence), different OCL contexts + {{"GPU.1"}, {}}, + {{"GPU.1"}, {ov::intel_auto::device_bind_buffer(true)}}, }; }; diff --git a/src/tests/functional/plugin/gpu/shared_tests_instances/skip_tests_config.cpp b/src/tests/functional/plugin/gpu/shared_tests_instances/skip_tests_config.cpp index df4e53897f4..9ef080844e9 100644 --- a/src/tests/functional/plugin/gpu/shared_tests_instances/skip_tests_config.cpp +++ b/src/tests/functional/plugin/gpu/shared_tests_instances/skip_tests_config.cpp @@ -124,5 +124,9 @@ std::vector disabledTestPatterns() { // Looks like the test is targeting CPU plugin and doesn't respect that execution graph may vary from plugin to plugin R"(.*ExecGraphSerializationTest.*)", + + // TODO: support getconfig in auto/multi CVS-104942 + // TODO: move auto/multi cases to dedicated unit tests + R"(.*(Auto|Multi).*SetPropLoadNetWorkGetPropTests.*)", }; } diff --git a/src/tests/functional/plugin/shared/include/behavior/executable_network/exec_network_base.hpp b/src/tests/functional/plugin/shared/include/behavior/executable_network/exec_network_base.hpp index 15da490b23e..e5331582cd9 100644 --- a/src/tests/functional/plugin/shared/include/behavior/executable_network/exec_network_base.hpp +++ b/src/tests/functional/plugin/shared/include/behavior/executable_network/exec_network_base.hpp @@ -70,10 +70,6 @@ TEST_P(ExecutableNetworkBaseTest, checkGetMetric) { TEST_P(ExecutableNetworkBaseTest, canLoadCorrectNetworkToGetExecutableAndCheckConfig) { auto execNet = ie->LoadNetwork(cnnNet, target_device, configuration); - if (target_device == CommonTestUtils::DEVICE_AUTO) { - // AUTO executable network didn't support to read any config. - GTEST_SKIP(); - } for (const auto& configItem : configuration) { InferenceEngine::Parameter param; ASSERT_NO_THROW(param = execNet.GetConfig(configItem.first)); diff --git a/src/tests/functional/plugin/shared/include/behavior/ov_plugin/core_integration.hpp b/src/tests/functional/plugin/shared/include/behavior/ov_plugin/core_integration.hpp index c6566cac1a0..fe62869619a 100644 --- a/src/tests/functional/plugin/shared/include/behavior/ov_plugin/core_integration.hpp +++ b/src/tests/functional/plugin/shared/include/behavior/ov_plugin/core_integration.hpp @@ -1355,6 +1355,7 @@ TEST_P(OVClassLoadNetworkAndCheckSecondaryPropertiesTest, LoadNetworkAndCheckSec ASSERT_TRUE(property.count(ov::num_streams.name())); auto actual = property.at(ov::num_streams.name()).as(); ov::Any value; + //AutoExcutableNetwork GetMetric() does not support key ov::num_streams OV_ASSERT_NO_THROW(value = model.get_property(ov::num_streams.name())); int32_t expect = value.as(); ASSERT_EQ(actual, expect); @@ -1382,7 +1383,7 @@ TEST_P(OVClassLoadNetWorkDoNotReturnDefaultHintTest, LoadNetworkDoNotReturnDefau if (target_device.find("AUTO") != std::string::npos) { ASSERT_NE(value, ov::hint::PerformanceMode::LATENCY); } else { - ASSERT_NE(value, ov::hint::PerformanceMode::THROUGHPUT); + ASSERT_EQ(value, ov::hint::PerformanceMode::THROUGHPUT); } } diff --git a/src/tests/functional/plugin/shared/src/behavior/ov_plugin/properties_tests.cpp b/src/tests/functional/plugin/shared/src/behavior/ov_plugin/properties_tests.cpp index 6633a562c85..c8fc23815f2 100644 --- a/src/tests/functional/plugin/shared/src/behavior/ov_plugin/properties_tests.cpp +++ b/src/tests/functional/plugin/shared/src/behavior/ov_plugin/properties_tests.cpp @@ -178,8 +178,6 @@ TEST_P(OVSetPropComplieModleGetPropTests, SetPropertyComplieModelGetProperty) { ov::CompiledModel exeNetWork; OV_ASSERT_NO_THROW(exeNetWork = core->compile_model(model, target_device, compileModelProperties)); - if (target_device == CommonTestUtils::DEVICE_AUTO) - GTEST_SKIP(); for (const auto& property_item : compileModelProperties) { Any exeNetProperty; diff --git a/src/tests/functional/plugin/shared/src/behavior/plugin/configuration_tests.cpp b/src/tests/functional/plugin/shared/src/behavior/plugin/configuration_tests.cpp index 4626cb98d8d..74d37ae30ad 100644 --- a/src/tests/functional/plugin/shared/src/behavior/plugin/configuration_tests.cpp +++ b/src/tests/functional/plugin/shared/src/behavior/plugin/configuration_tests.cpp @@ -197,9 +197,8 @@ TEST_P(SetPropLoadNetWorkGetPropTests, SetPropLoadNetWorkGetProperty) { InferenceEngine::ExecutableNetwork exeNetWork; ASSERT_NO_THROW(exeNetWork = ie->LoadNetwork(cnnNet, target_device, loadNetWorkConfig)); - if (target_device == CommonTestUtils::DEVICE_AUTO) - GTEST_SKIP(); + //ie's setConfig and LoadNetwork should not affect each other, for config settings for (const auto& property_item : loadNetWorkConfig) { InferenceEngine::Parameter exeNetProperty; ASSERT_NO_THROW(exeNetProperty = exeNetWork.GetConfig(property_item.first)); @@ -213,5 +212,4 @@ TEST_P(SetPropLoadNetWorkGetPropTests, SetPropLoadNetWorkGetProperty) { ASSERT_EQ(property_item.second, property.as()); } } - } // namespace BehaviorTestsDefinitions diff --git a/src/tests/unit/auto/auto_ctput_call_multi.cpp b/src/tests/unit/auto/auto_ctput_call_multi.cpp index c61db673125..afd893ad7d5 100644 --- a/src/tests/unit/auto/auto_ctput_call_multi.cpp +++ b/src/tests/unit/auto/auto_ctput_call_multi.cpp @@ -45,6 +45,11 @@ public: ov::SoPtr cpuMockExeNetwork; NiceMock* cpuMockIPlugin; std::shared_ptr cpuMockPlugin; + // mock gpu exeNetwork + std::shared_ptr> gpuMockIExeNet; + ov::SoPtr gpuMockExeNetwork; + NiceMock* gpuMockIPlugin; + std::shared_ptr gpuMockPlugin; std::shared_ptr> inferReqInternal; public: @@ -85,6 +90,16 @@ public: EXPECT_CALL(*cpuMockIPluginPtr, LoadNetwork(MatcherCast(_), _)).Times(1); cpuMockExeNetwork = ov::SoPtr(cpuMockPlugin->LoadNetwork(CNNNetwork{}, {}), {}); + // prepare gpuMockExeNetwork + gpuMockIExeNet = std::make_shared>(); + auto gpuMockIPluginPtr = std::make_shared>(); + ON_CALL(*gpuMockIPluginPtr, LoadNetwork(MatcherCast(_), _)) + .WillByDefault(Return(gpuMockIExeNet)); + gpuMockPlugin = gpuMockIPluginPtr; + // remove annoying ON CALL message + EXPECT_CALL(*gpuMockIPluginPtr, LoadNetwork(MatcherCast(_), _)).Times(1); + gpuMockExeNetwork = ov::SoPtr(gpuMockPlugin->LoadNetwork(CNNNetwork{}, {}), {}); + // prepare mockicore and cnnNetwork for loading core = std::shared_ptr>(new NiceMock()); auto* origin_plugin = new NiceMock(); @@ -95,6 +110,9 @@ public: ON_CALL(*cpuMockIExeNet.get(), CreateInferRequest()).WillByDefault(Return(inferReqInternal)); ON_CALL(*cpuMockIExeNet.get(), GetMetric(StrEq(METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS)))) .WillByDefault(Return("0")); + ON_CALL(*gpuMockIExeNet.get(), CreateInferRequest()).WillByDefault(Return(inferReqInternal)); + ON_CALL(*gpuMockIExeNet.get(), GetMetric(StrEq(METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS)))) + .WillByDefault(Return("0")); std::vector availableDevs = {"CPU", "GPU"}; ON_CALL(*core, GetAvailableDevices()).WillByDefault(Return(availableDevs)); @@ -161,58 +179,53 @@ public: LoadNetwork(::testing::Matcher(_), ::testing::Matcher(StrEq("GPU")), ::testing::Matcher&>(_))) - .WillByDefault(Throw(InferenceEngine::GeneralError{""})); + .WillByDefault(Return(gpuMockExeNetwork)); + std::shared_ptr simpleNetwork = ngraph::builder::subgraph::makeSingleConv(); ASSERT_NO_THROW(simpleCnnNetwork = InferenceEngine::CNNNetwork(simpleNetwork)); } }; -TEST_P(AutoCTPUTCallMulti, CTPUTDevicesLogicTest) { +TEST_P(AutoCTPUTCallMulti, CTPUTDeviceLoadFailedNoExceptionThrowTest) { std::vector targetDevices; std::string targetDev; bool AutoCallMulti; Config config; std::tie(AutoCallMulti, targetDevices) = this->GetParam(); + std::string loadFailedDevice = targetDevices.size() > 0 ? targetDevices[0] : ""; + std::string secondDevice = targetDevices.size() > 1 ? targetDevices[1] : ""; plugin->SetName("MULTI"); for (auto& deviceName : targetDevices) { targetDev += deviceName; targetDev += ((deviceName == targetDevices.back()) ? "" : ","); } - if (AutoCallMulti) { - std::shared_ptr exeNetwork; - config.insert({{CONFIG_KEY(PERFORMANCE_HINT), InferenceEngine::PluginConfigParams::CUMULATIVE_THROUGHPUT}}); - config.insert({InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES, targetDev}); + std::shared_ptr exeNetwork; + config.insert({{CONFIG_KEY(PERFORMANCE_HINT), InferenceEngine::PluginConfigParams::CUMULATIVE_THROUGHPUT}}); + config.insert({InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES, targetDev}); + ON_CALL(*core, + LoadNetwork(::testing::Matcher(_), + ::testing::Matcher(StrEq(loadFailedDevice)), + ::testing::Matcher&>(_))) + .WillByDefault(Throw(InferenceEngine::GeneralError{""})); + if (loadFailedDevice != CommonTestUtils::DEVICE_CPU) { EXPECT_CALL(*core, LoadNetwork(::testing::Matcher(_), ::testing::Matcher(CommonTestUtils::DEVICE_CPU), ::testing::Matcher&>(_))) .Times(1); - EXPECT_CALL(*core, - LoadNetwork(::testing::Matcher(_), - ::testing::Matcher(CommonTestUtils::DEVICE_GPU), - ::testing::Matcher&>(_))) - .Times(1); - ASSERT_NO_THROW(exeNetwork = plugin->LoadExeNetworkImpl(simpleCnnNetwork, config)); - EXPECT_EQ(exeNetwork->GetMetric(ov::execution_devices.name()).as(), CommonTestUtils::DEVICE_CPU); - } else { - config.insert({InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES, targetDev}); - EXPECT_CALL(*core, - LoadNetwork(::testing::Matcher(_), - ::testing::Matcher(CommonTestUtils::DEVICE_CPU), - ::testing::Matcher&>(_))) - .Times(0); - EXPECT_CALL(*core, - LoadNetwork(::testing::Matcher(_), - ::testing::Matcher(CommonTestUtils::DEVICE_GPU), - ::testing::Matcher&>(_))) - .Times(1); - EXPECT_THROW(plugin->LoadExeNetworkImpl(simpleCnnNetwork, config), IE::Exception); } + if (loadFailedDevice != CommonTestUtils::DEVICE_GPU) { + EXPECT_CALL(*core, + LoadNetwork(::testing::Matcher(_), + ::testing::Matcher(CommonTestUtils::DEVICE_GPU), + ::testing::Matcher&>(_))) + .Times(1); + } + ASSERT_NO_THROW(exeNetwork = plugin->LoadExeNetworkImpl(simpleCnnNetwork, config)); + EXPECT_EQ(exeNetwork->GetMetric(ov::execution_devices.name()).as(), secondDevice); } const std::vector testConfigs = { - ConfigParams{false, {"CPU", "GPU"}}, - ConfigParams{false, {"GPU", "CPU"}}, ConfigParams{true, {"CPU", "GPU"}}, ConfigParams{true, {"GPU", "CPU"}}, }; diff --git a/src/tests/unit/auto/auto_default_perf_hint_test.cpp b/src/tests/unit/auto/auto_default_perf_hint_test.cpp index a213d31df9f..0657f4fc484 100644 --- a/src/tests/unit/auto/auto_default_perf_hint_test.cpp +++ b/src/tests/unit/auto/auto_default_perf_hint_test.cpp @@ -180,13 +180,13 @@ public: testConfigs.push_back(ConfigParams{ "MULTI:CPU,GPU", {"CPU", "GPU"}, - {{"DEVICE_PROPERTIES", "{GPU:{PERFORMANCE_HINT:LATENCY}}"}, - {"MULTI_DEVICE_PRIORITIES", "CPU,GPU"}}}); // CPU: get default_hint:tput GPU: get perf_hint:latency + {{"DEVICE_PROPERTIES", "{GPU:{PERFORMANCE_HINT:THROUGHPUT}}"}, + {"MULTI_DEVICE_PRIORITIES", "CPU,GPU"}}}); // CPU: get default_hint:tput GPU: get perf_hint:tput testConfigs.push_back(ConfigParams{ "MULTI:CPU,GPU", {"CPU", "GPU"}, - {{"DEVICE_PROPERTIES", "{CPU:{PERFORMANCE_HINT:LATENCY},GPU:{PERFORMANCE_HINT:LATENCY}}"}, - {"MULTI_DEVICE_PRIORITIES", "CPU,GPU"}}}); // CPU: get perf_hint:lantency GPU: get perf_hint:lantency + {{"DEVICE_PROPERTIES", "{CPU:{PERFORMANCE_HINT:THROUGHPUT},GPU:{PERFORMANCE_HINT:THROUGHPUT}}"}, + {"MULTI_DEVICE_PRIORITIES", "CPU,GPU"}}}); // CPU: get perf_hint:tput GPU: get perf_hint:tput return testConfigs; } @@ -221,17 +221,17 @@ public: "MULTI:CPU,GPU", {"CPU", "GPU"}, {{"CPU", "{ALLOW_AUTO_BATCHING:FALSE}"}, - {"MULTI_DEVICE_PRIORITIES", "CPU,GPU"}}}); // CPU: no perf_hint GPU: get default_hint:tput + {"MULTI_DEVICE_PRIORITIES", "CPU,GPU"}}}); // CPU: get default_hint:tput GPU: get default_hint:tput testConfigs.push_back(ConfigParams{ "MULTI:CPU,GPU", {"CPU", "GPU"}, {{"DEVICE_PROPERTIES", "GPU:{ALLOW_AUTO_BATCHING:FALSE}}"}, - {"MULTI_DEVICE_PRIORITIES", "CPU,GPU"}}}); // CPU: get default_hint:tput GPU: no perf_hint + {"MULTI_DEVICE_PRIORITIES", "CPU,GPU"}}}); // CPU: get default_hint:tput GPU: get default_hint:tput testConfigs.push_back(ConfigParams{ "MULTI:CPU,GPU", {"CPU", "GPU"}, {{"DEVICE_PROPERTIES", "CPU:{ALLOW_AUTO_BATCHING:TRUE},GPU:{ALLOW_AUTO_BATCHING:FALSE}}"}, - {"MULTI_DEVICE_PRIORITIES", "CPU,GPU"}}}); // CPU: no perf_hint GPU: no perf_hint + {"MULTI_DEVICE_PRIORITIES", "CPU,GPU"}}}); // CPU: no perf_hint GPU: get default_hint:tput return testConfigs; } @@ -499,11 +499,6 @@ TEST_P(SecPropAndDefaultPerfHintMockTest, SecPropAndDefaultPerfHintTest) { // HW default perf_hint HW_PerfHint = bIsAuto ? "LATENCY" : "THROUGHPUT"; } - auto item = config.find(deviceName); - if (item != config.end() && !isCPUHelper) { - // do not pass default perf_hint to HW - HW_PerfHint = "No PERFORMANCE_HINT"; - } EXPECT_CALL( *core,