Unify code path for MULTI and AUTO CTPUT hint (#16349)

[MULTI] pass through to AUTO with CTPUT hint
After this change
-- MULTI doesn't support setting infer request via CPU(4),GPU(8).
-- MULTI doesn't support CompiledModel::set_property() and ExecutableNetwork::GetConfig().

src/bindings/c/tests/ie_c_api_test.cpp

@@ -954,36 +954,6 @@ TEST_P(ie_c_api_test, ie_exec_network_get_config) {
     ie_core_free(&core);
 }
 
-TEST_P(ie_c_api_test, ie_exec_network_set_config) {
-    ie_core_t *core = nullptr;
-    IE_ASSERT_OK(ie_core_create("", &core));
-    ASSERT_NE(nullptr, core);
-
-    ie_param_t param;
-    if (ie_core_get_metric(core, "GPU", "AVAILABLE_DEVICES", &param) != IEStatusCode::OK) {
-        ie_core_free(&core);
-        GTEST_SKIP();
-    }
-
-    ie_network_t *network = nullptr;
-    IE_EXPECT_OK(ie_core_read_network(core, xml_file_name.c_str(), bin_file_name.c_str(), &network));
-    EXPECT_NE(nullptr, network);
-
-    const char *device_name = "MULTI:GPU,CPU";
-    ie_config_t config = {nullptr, nullptr, nullptr};
-    ie_executable_network_t *exe_network = nullptr;
-    IE_EXPECT_OK(ie_core_load_network(core, network, device_name, &config, &exe_network));
-    EXPECT_NE(nullptr, exe_network);
-
-    ie_config_t config_param = {"MULTI_DEVICE_PRIORITIES", "GPU,CPU", nullptr};
-    IE_EXPECT_OK(ie_exec_network_set_config(exe_network, &config_param));
-
-    ie_exec_network_free(&exe_network);
-    ie_network_free(&network);
-    ie_core_free(&core);
-    ie_param_free(&param);
-}
-
 TEST_P(ie_c_api_test, ie_exec_network_get_metric) {
     ie_core_t *core = nullptr;
     IE_ASSERT_OK(ie_core_create("", &core));

src/bindings/c/tests/ov_compiled_model_test.cpp

@@ -121,45 +121,6 @@ TEST_P(ov_compiled_model_test, ov_compiled_model_input_by_name) {
     ov_core_free(core);
 }
 
-TEST_P(ov_compiled_model_test, set_and_get_property) {
-    // It seems that all set_property() for CPU plugin are not implement in compiled_model.
-    auto device_name = "MULTI:GPU,CPU";
-    ov_core_t* core = nullptr;
-    OV_EXPECT_OK(ov_core_create(&core));
-    EXPECT_NE(nullptr, core);
-
-    char* info = nullptr;
-    const char* key_0 = ov_property_key_available_devices;
-    if (ov_core_get_property(core, "GPU", key_0, &info) != ov_status_e::OK) {
-        ov_core_free(core);
-        GTEST_SKIP();
-    }
-
-    ov_model_t* model = nullptr;
-    OV_EXPECT_OK(ov_core_read_model(core, xml_file_name.c_str(), bin_file_name.c_str(), &model));
-    EXPECT_NE(nullptr, model);
-
-    ov_compiled_model_t* compiled_model = nullptr;
-    OV_EXPECT_OK(ov_core_compile_model(core, model, device_name, 0, &compiled_model));
-    EXPECT_NE(nullptr, compiled_model);
-
-    const char* key_1 = ov_property_key_device_priorities;
-    const char* value_1 = "GPU,CPU";
-    OV_EXPECT_OK(ov_compiled_model_set_property(compiled_model, key_1, value_1));
-    char* result = nullptr;
-    OV_EXPECT_OK(ov_compiled_model_get_property(compiled_model, key_1, &result));
-    EXPECT_STREQ(value_1, result);
-    ov_free(result);
-
-    const char* key_2 = ov_property_key_supported_properties;
-    OV_EXPECT_OK(ov_compiled_model_get_property(compiled_model, key_2, &result));
-    ov_free(result);
-
-    ov_compiled_model_free(compiled_model);
-    ov_model_free(model);
-    ov_core_free(core);
-}
-
 TEST_P(ov_compiled_model_test, get_property) {
     auto device_name = GetParam();
     ov_core_t* core = nullptr;

src/plugins/auto/auto_executable_network.cpp

@@ -17,12 +17,21 @@ AutoExecutableNetwork::AutoExecutableNetwork(AutoScheduleContext::Ptr& context,
 }
 
 std::shared_ptr<IE::RemoteContext> AutoExecutableNetwork::GetContext() const {
-    std::lock_guard<std::mutex> lock(_autoSContext->_fallbackMutex);
+    if (_autoSchedule->_pCTPUTLoadContext) {
-    if (_autoSchedule->_loadContext[FALLBACKDEVICE].isAlready) {
+        for (size_t i = 0; i < _autoSchedule->_nCTputDeviceNums; i++) {
-        return _autoSchedule->_loadContext[FALLBACKDEVICE].executableNetwork->GetContext();
+            if (_autoSchedule->_pCTPUTLoadContext[i].isAlready) {
+                return _autoSchedule->_pCTPUTLoadContext[i].executableNetwork->GetContext();
+            }
+        }
+        return nullptr;
     } else {
-        _autoSchedule->WaitActualNetworkReady();
+        std::lock_guard<std::mutex> lock(_autoSContext->_fallbackMutex);
-        return _autoSchedule->_loadContext[ACTUALDEVICE].executableNetwork->GetContext();
+        if (_autoSchedule->_loadContext[FALLBACKDEVICE].isAlready) {
+            return _autoSchedule->_loadContext[FALLBACKDEVICE].executableNetwork->GetContext();
+        } else {
+            _autoSchedule->WaitActualNetworkReady();
+            return _autoSchedule->_loadContext[ACTUALDEVICE].executableNetwork->GetContext();
+        }
     }
 }
 
@@ -82,11 +91,11 @@ IE::Parameter AutoExecutableNetwork::GetMetric(const std::string& name) const {
             all_devices[context.deviceInfo.deviceName] = device_properties;
         };
         if (_autoSchedule->_pCTPUTLoadContext) {
-            // need lock for inference failure
+            for (size_t i = 0; i < _autoSchedule->_nCTputDeviceNums; i++) {
-            std::lock_guard<std::mutex> lock(_autoSContext->_fallbackMutex);
+                if (_autoSchedule->_pCTPUTLoadContext[i].isAlready) {
-            auto load_count = _autoSContext->_devicePriorities.size();
+                    get_device_supported_metrics(_autoSchedule->_pCTPUTLoadContext[i]);
-            for (size_t i = 0; i < load_count; i++)
+                }
-                get_device_supported_metrics(_autoSchedule->_pCTPUTLoadContext[i]);
+            }
         } else {
             {
                 std::lock_guard<std::mutex> lock(_autoSContext->_fallbackMutex);
@@ -118,11 +127,13 @@ IE::Parameter AutoExecutableNetwork::GetMetric(const std::string& name) const {
         if (_autoSchedule->_pCTPUTLoadContext) {
             std::lock_guard<std::mutex> lock(_autoSContext->_fallbackMutex);
             unsigned int res = 0u;
-            auto load_count = _autoSContext->_devicePriorities.size();
+            for (size_t i = 0; i < _autoSchedule->_nCTputDeviceNums; i++) {
-            for (size_t i = 0; i < load_count; i++) {
                 try {
-                    res += (_autoSchedule->_pCTPUTLoadContext[i]).executableNetwork->GetMetric(
+                    if (_autoSchedule->_pCTPUTLoadContext[i].isAlready) {
-                        METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS)).as<unsigned int>();
+                        res += (_autoSchedule->_pCTPUTLoadContext[i])
+                                   .executableNetwork->GetMetric(METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS))
+                                   .as<unsigned int>();
+                    }
                 } catch (const IE::Exception& iie) {
                     IE_THROW()
                         << "Every device used in cumulative mode should "

src/plugins/auto/auto_schedule.cpp

@@ -234,9 +234,7 @@ void AutoSchedule::init(const ScheduleContext::Ptr& sContext) {
     bool isCumulative =
         (_autoSContext->_performanceHint == IE::PluginConfigParams::CUMULATIVE_THROUGHPUT) ? true : false;
     if (isCumulative) {
-        std::list<DeviceInformation> validDevices =
+        const auto& validDevices = _autoSContext->_devicePriorities;
-            _autoSContext->_plugin->GetValidDevice(_autoSContext->_devicePriorities,
-                                                   _loadContext[ACTUALDEVICE].networkPrecision);
         // When the hint is ctput and there is only one device, the single-device logic is used
         if (validDevices.size() == 1) {
             _loadContext[ACTUALDEVICE].deviceInfo = validDevices.front();
@@ -244,14 +242,10 @@ void AutoSchedule::init(const ScheduleContext::Ptr& sContext) {
                 IE::PluginConfigParams::THROUGHPUT;
         } else if (validDevices.size() > 1) {
             _loadContext[ACTUALDEVICE].isEnabled = false;
-            _autoSContext->_devicePriorities.clear();
-            std::copy(std::begin(validDevices),
-                      std::end(validDevices),
-                      std::back_inserter(_autoSContext->_devicePriorities));
             // Total number of devices in CTPUT
-            auto nCTputDeviceNums = validDevices.size();
+            _nCTputDeviceNums = validDevices.size();
             // Generate contexts for loading each device
-            _pCTPUTLoadContext.reset(new AutoLoadContext[nCTputDeviceNums]);
+            _pCTPUTLoadContext.reset(new AutoLoadContext[_nCTputDeviceNums]);
             int idx = 0;
             DeviceInformation cpuDeviceInformation;
             for (auto& device : validDevices) {
@@ -272,6 +266,10 @@ void AutoSchedule::init(const ScheduleContext::Ptr& sContext) {
                     IE::PluginConfigParams::THROUGHPUT;
             }
         }
+        if (_autoSContext->_LogTag == "MULTI") {
+            // MULTI's performance hint always is tput
+            _autoSContext->_performanceHint = IE::PluginConfigParams::THROUGHPUT;
+        }
     } else {
         _loadContext[ACTUALDEVICE].deviceInfo =
             _autoSContext->_plugin->SelectDevice(_autoSContext->_devicePriorities,
@@ -388,12 +386,12 @@ void AutoSchedule::init(const ScheduleContext::Ptr& sContext) {
     std::vector<Task> otherDevicesloads;
     std::vector<Task> cpuLoads;
     if (_pCTPUTLoadContext) {
-        for (size_t i = 0; i < _autoSContext->_devicePriorities.size(); i++) {
+        for (size_t i = 0; i < _nCTputDeviceNums; i++) {
             auto* contextPtr = &_pCTPUTLoadContext[i];
             auto modelPath = _autoSContext->_modelPath;
             auto network = _autoSContext->_network;
             _pCTPUTLoadContext[i].task = std::bind(loadDeviceTask, contextPtr, modelPath, network, isCumulative);
-            if (i == _autoSContext->_devicePriorities.size() - 1 &&
+            if (i == _nCTputDeviceNums - 1 &&
                 _pCTPUTLoadContext[i].deviceInfo.deviceName.find("CPU") != std::string::npos) {
                 cpuLoads.push_back(_pCTPUTLoadContext[i].task);
             } else {
@@ -518,7 +516,7 @@ void AutoSchedule::init(const ScheduleContext::Ptr& sContext) {
             _passthroughExeNet = _loadContext[ACTUALDEVICE].executableNetwork;
         }
     }
-    WaitFirstNetworkReady();
+    _autoSContext->_hwExecutableNetwork = WaitFirstNetworkReady();
 }
 
 void AutoSchedule::TryToLoadNetWork(AutoLoadContext& context, const std::string& modelPath, const IE::CNNNetwork& network, bool isCumulative) {
@@ -618,7 +616,7 @@ void AutoSchedule::TryToLoadNetWork(AutoLoadContext& context, const std::string&
     TryToLoadNetWork(context, modelPath, network, isCumulative);
 }
 
-void AutoSchedule::WaitFirstNetworkReady() {
+SoExecNetwork AutoSchedule::WaitFirstNetworkReady() {
     if (_firstLoadFuture.valid()) {
         // wait for the first loading finished
         _firstLoadFuture.wait();
@@ -626,7 +624,7 @@ void AutoSchedule::WaitFirstNetworkReady() {
     // check if there is any device that have loaded network successfully
     for (int i = CONTEXTNUM - 2; i >= 0; i--) {
         if (_loadContext[i].isEnabled && _loadContext[i].isAlready) {
-            return;
+            return _loadContext[i].executableNetwork;
         }
     }
     // the first loading is failed, wait for another loading
@@ -635,7 +633,7 @@ void AutoSchedule::WaitFirstNetworkReady() {
             _loadContext[i].future.wait();
             // check if loading is successful
             if (_loadContext[i].isAlready) {
-                return;
+                return _loadContext[i].executableNetwork;
             }
         }
     }
@@ -646,17 +644,21 @@ void AutoSchedule::WaitFirstNetworkReady() {
         }
     }
     // devices loaded successfully in CTPUT
+    SoExecNetwork execNetwork;
     if (_pCTPUTLoadContext) {
         int nLoadSucNums = 0;
-        for (size_t i = 0; i < _autoSContext->_devicePriorities.size(); i++) {
+        for (size_t i = 0; i < _nCTputDeviceNums; i++) {
             // check if device loaded successfully
             if (_pCTPUTLoadContext[i].isAlready) {
+                if (!execNetwork) {
+                    execNetwork = _pCTPUTLoadContext[i].executableNetwork;
+                }
                 nLoadSucNums++;
             }
         }
         // one or more devices loaded successfully
         if (nLoadSucNums > 0) {
-            return;
+            return execNetwork;
         }
     }
     IE_THROW() << GetLogTag() << "load all devices failed";
@@ -784,7 +786,6 @@ IInferPtr AutoSchedule::CreateInferRequest() {
             so = _passthroughExeNet._so;
         syncRequestImpl->setPointerToSo(so);
     } else if (std::static_pointer_cast<MultiDeviceInferRequest>(syncRequestImpl)->GetSharedRequest()) {
-        // cumulative case, load to MULTI:*
         auto sharedMultiRequest = std::static_pointer_cast<MultiDeviceInferRequest>(syncRequestImpl)->GetSharedRequest();
         if (sharedMultiRequest._ptr->getPointerToSo())
             syncRequestImpl->setPointerToSo(sharedMultiRequest._ptr->getPointerToSo());

src/plugins/auto/auto_schedule.hpp

@@ -51,6 +51,7 @@ public:
 public:
     AutoLoadContext                           _loadContext[CONTEXTNUM];
     std::unique_ptr<AutoLoadContext[]>        _pCTPUTLoadContext = nullptr;
+    size_t                                    _nCTputDeviceNums;
 
 protected:
     void GenerateWorkers(const std::string& device, const SoExecNetwork& executableNetwork) override;
@@ -58,9 +59,15 @@ protected:
     static bool RunPipelineTask(IE::Task& inferPipelineTask, NotBusyPriorityWorkerRequests& idleWorkerRequests,
                                 const DeviceName& preferred_device);
     DeviceMap<NotBusyPriorityWorkerRequests> _idleWorkerRequests;
+    AutoScheduleContext::Ptr                 _autoSContext;
 
 private:
-    void WaitFirstNetworkReady();
+    /**
+     * @brief wait for one of the executable network to finish loading.
+     * @return An SoPtr object hold an available executable network loaded to HW device.
+     * @note An exception will be thrown if all loading of network to hw device fails.
+     */
+    SoExecNetwork WaitFirstNetworkReady();
     void TryToLoadNetWork(AutoLoadContext& context, const std::string& modelPath, const IE::CNNNetwork& network, bool isCumulative);
     bool selectOtherDevice(const std::string& currentDeviceName);
     IE::Task releaseActualdeviceTask;
@@ -73,7 +80,6 @@ private:
     std::promise<void>                       _firstLoadPromise;
     bool                                     _exitFlag = {false};
     size_t                                   _cpuHelpInferCount = 0;
-    AutoScheduleContext::Ptr                 _autoSContext;
 };
 
 }  // namespace MultiDevicePlugin

src/plugins/auto/bind_multi_schedule.cpp

@@ -6,140 +6,40 @@
 #include "async_infer_request.hpp"
 #include "plugin.hpp"
 #include "bind_multi_schedule.hpp"
-#include "multi_executable_network.hpp"
 // ------------------------------MultiSchedule----------------------------
 namespace MultiDevicePlugin {
 
-thread_local IE::IInferRequestInternal* BinderMultiSchedule::_sharedRequest = nullptr;
-
 void BinderMultiSchedule::init(const ScheduleContext::Ptr& sContext) {
-     MultiSchedule::init(sContext);
+    AutoSchedule::init(sContext);
+    LOG_INFO_TAG("enable bind buffer for AUTO");
 }
 
 Pipeline BinderMultiSchedule::GetPipeline(const IInferPtr& syncInferRequest, WorkerInferRequest** workerInferRequest) {
-    Pipeline pipeline = {
+    Pipeline pipeline;
-        // if the request is coming with device-specific remote blobs make sure it is scheduled to the specific device only:
+    struct RequestExecutor : ITaskExecutor {
-        Stage {
+        explicit RequestExecutor(InferenceEngine::SoIInferRequestInternal& inferRequest) : _inferRequest(inferRequest) {
-            /*TaskExecutor*/ std::make_shared<IE::ImmediateExecutor>(), /*task*/ [this, &syncInferRequest, workerInferRequest]() {
+            _inferRequest->SetCallback([this](std::exception_ptr exceptionPtr) mutable {
-                // by default, no preferred device:
+                _exceptionPtr = exceptionPtr;
-                _thisPreferredDeviceName = "";
+                auto capturedTask = std::move(_task);
-                auto execNetwork = _multiSContext->_executableNetwork.lock();
-                // if any input is remote (e.g. was set with SetBlob), let' use the corresponding device
-                for (const auto& it : execNetwork->GetInputsInfo()) {
-                    auto b = syncInferRequest->GetBlob(it.first);
-                    auto r = b->as<IE::RemoteBlob>();
-                    if (r) {
-                        const auto name = r->getDeviceName();
-                        const auto res = std::find_if(
-                            _multiSContext->_devicePrioritiesInitial.cbegin(),
-                            _multiSContext->_devicePrioritiesInitial.cend(),
-                        [&name](const MultiDevicePlugin::DeviceInformation & d) {
-                            return (d.defaultDeviceID.empty() ? d.deviceName : (d.deviceName + "." +
-                                    d.defaultDeviceID)) == name;
-                        });
-                        if (_multiSContext->_devicePrioritiesInitial.cend() == res) {
-                            IE_THROW() <<
-                                "None of the devices (for which current MULTI-device configuration was "
-                                "initialized) supports a remote blob created on the device named " << name;
-                        } else {
-                            // it is ok to take the c_str() here (as pointed in the executable_network.hpp we need to use const char*)
-                            // as the original strings are from the "persistent" vector (with the right lifetime)
-                            _thisPreferredDeviceName = res->deviceName.c_str();
-                            break;
-                        }
-                    }
-                }
-                _thisWorkerInferRequest = *workerInferRequest;
-                _sharedRequest = std::dynamic_pointer_cast<MultiDeviceInferRequest>(syncInferRequest)->GetSharedRequest()._ptr.get();
-            }},
-        // as the scheduling algo may select any device, this stage accepts the scheduling decision (actual workerRequest)
-        // then sets the device-agnostic blobs to the actual (device-specific) request
-        Stage {
-            /*TaskExecutor*/std::dynamic_pointer_cast<IE::ITaskExecutor>(shared_from_this()), /*task*/ [&syncInferRequest, workerInferRequest]() {
-                *workerInferRequest = _thisWorkerInferRequest;
-                auto multiSyncInferRequest = std::dynamic_pointer_cast<MultiDeviceInferRequest>(syncInferRequest);
-                multiSyncInferRequest->SetBlobsToAnotherRequest(_thisWorkerInferRequest->_inferRequest);
-                INFO_RUN([workerInferRequest]() {
-                    (*workerInferRequest)->_startTimes.push_back(std::chrono::steady_clock::now());
-                });
-            }},
-        // final task in the pipeline:
-        Stage {
-            /*TaskExecutor*/std::make_shared<ThisRequestExecutor>(workerInferRequest), /*task*/ [this, &syncInferRequest, workerInferRequest]() {
-                if (nullptr != (*workerInferRequest)->_exceptionPtr) {
-                    std::rethrow_exception((*workerInferRequest)->_exceptionPtr);
-                }
-                if (_multiSContext->_needPerfCounters) {
-                    auto multiSyncInferRequest = std::dynamic_pointer_cast<MultiDeviceInferRequest>
-                        (syncInferRequest);
-                    multiSyncInferRequest->_scheduledRequest =
-                        (*workerInferRequest)->_inferRequest;
-                }
-                INFO_RUN([workerInferRequest]() {
-                   (*workerInferRequest)->_endTimes.push_back(std::chrono::steady_clock::now());
-                });
-            }}
-    };
-    return pipeline;
-}
-
-bool BinderMultiSchedule::ScheduleToWorkerInferRequest(IE::Task inferPipelineTask, DeviceName preferred_device) {
-    std::vector<DeviceInformation> devices;
-    devices = [&] {
-        std::lock_guard<std::mutex> lock(_multiSContext->_mutex);
-        return _multiSContext->_devicePriorities;
-    }();
-    for (auto&& device : devices) {
-        if (!preferred_device.empty() && (device.deviceName != preferred_device)) {
-            continue;
-        }
-        if (RunPipelineTask(inferPipelineTask, _idleWorkerRequests[device.deviceName], preferred_device)) {
-            return true;
-        }
-    }
-    // no vacant requests this time, storing the task to the respective queue
-    if (!preferred_device.empty()) {
-        _inferPipelineTasksDeviceSpecific[preferred_device]->push(std::move(inferPipelineTask));
-    } else {
-        _inferPipelineTasks.push(std::move(inferPipelineTask));
-    }
-    return false;
-}
-
-bool BinderMultiSchedule::RunPipelineTask(IE::Task& inferPipelineTask,
-    NotBusyWorkerRequests& idleWorkerRequests,
-    const DeviceName& preferred_device) {
-    WorkerInferRequest* workerRequestPtr = nullptr;
-    WorkerInferRequest* headWorker = nullptr;
-    bool flag = false;
-    while (idleWorkerRequests.try_pop(workerRequestPtr)) {
-        if (flag && workerRequestPtr == headWorker)
-            break;
-        if (!flag) {
-            headWorker = workerRequestPtr;
-            flag = true;
-        }
-        IdleGuard<NotBusyWorkerRequests> idleGuard{workerRequestPtr, idleWorkerRequests};
-        if (_sharedRequest == workerRequestPtr->_inferRequest._ptr.get()) {
-            _thisWorkerInferRequest = workerRequestPtr;
-            {
-                auto capturedTask = std::move(inferPipelineTask);
                 capturedTask();
-            }
+            });
-            idleGuard.Release();
-            return true;
         }
-    }
+        void run(InferenceEngine::Task task) override {
-    return false;
+            _task = std::move(task);
-}
+            _inferRequest->StartAsync();
-
+        };
-void BinderMultiSchedule::run(IE::Task inferPipelineTask) {
+        InferenceEngine::SoIInferRequestInternal& _inferRequest;
-    if (_thisWorkerInferRequest) {
+        std::exception_ptr _exceptionPtr;
-        auto capturedTask = std::move(inferPipelineTask);
+        InferenceEngine::Task _task;
-        capturedTask();
+    };
-    } else {
+    auto requestExecutor = std::make_shared<RequestExecutor>(
-        ScheduleToWorkerInferRequest(std::move(inferPipelineTask), _thisPreferredDeviceName);
+        std::static_pointer_cast<MultiDeviceInferRequest>(syncInferRequest)->GetSharedRequest());
-    }
+    pipeline.emplace_back(requestExecutor, [requestExecutor] {
+        if (nullptr != requestExecutor->_exceptionPtr) {
+            std::rethrow_exception(requestExecutor->_exceptionPtr);
+        }
+    });
+    return pipeline;
 }
 
 BinderMultiSchedule::~BinderMultiSchedule() {
@@ -153,7 +53,7 @@ IInferPtr BinderMultiSchedule::CreateInferRequestImpl(
     SoInfer request_to_share_blobs_with;
     // borrowing device-specific blobs from the underlying requests for the device-agnostic, user-facing requests
     // this allows to potentially save on the data-copy later (if the requests are scheduled in the same order)
-    for (const auto& device : _multiSContext->_devicePrioritiesInitial) {
+    for (const auto& device : _autoSContext->_devicePrioritiesInitial) {
         auto& dev_requests = _workerRequests[device.deviceName];
         if ((num - sum) < dev_requests.size()) {
             request_to_share_blobs_with = dev_requests.at(num - sum)._inferRequest;
@@ -177,7 +77,7 @@ IInferPtr BinderMultiSchedule::CreateInferRequestImpl(IE::InputsDataMap networkI
     size_t sum = 0;
     // borrowing device-specific blobs from the underlying requests for the device-agnostic, user-facing requests
     // this allows to potentially save on the data-copy later (if the requests are scheduled in the same order)
-    for (const auto& device : _multiSContext->_devicePrioritiesInitial) {
+    for (const auto& device : _autoSContext->_devicePrioritiesInitial) {
         auto& dev_requests = _workerRequests[device.deviceName];
         if ((num - sum) < dev_requests.size()) {
             request_to_share_blobs_with = dev_requests.at(num - sum)._inferRequest;

src/plugins/auto/bind_multi_schedule.hpp

@@ -5,7 +5,7 @@
 ///////////////////////////////////////////////////////////////////////////////////////////////////
 #pragma once
 
-#include "multi_schedule.hpp"
+#include "auto_schedule.hpp"
 
 #ifdef  MULTIUNITTEST
 #define MOCKTESTMACRO virtual
@@ -15,22 +15,14 @@
 #endif
 
 namespace MultiDevicePlugin {
-class BinderMultiSchedule : public MultiSchedule {
+class BinderMultiSchedule : public AutoSchedule {
 public:
     using Ptr = std::shared_ptr<BinderMultiSchedule>;
     IInferPtr CreateInferRequestImpl(IE::InputsDataMap networkInputs, IE::OutputsDataMap networkOutputs) override;
     IE::IInferRequestInternal::Ptr CreateInferRequestImpl(const std::vector<std::shared_ptr<const ov::Node>>& inputs,
                                                           const std::vector<std::shared_ptr<const ov::Node>>& outputs) override;
-    void run(IE::Task inferTask) override;
     void init(const ScheduleContext::Ptr& sContext) override;
     Pipeline GetPipeline(const IInferPtr& syncRequestImpl, WorkerInferRequest** WorkerInferRequest) override;
     virtual ~BinderMultiSchedule();
-
-protected:
-    static bool RunPipelineTask(IE::Task& inferPipelineTask, NotBusyWorkerRequests& idleWorkerRequests, const DeviceName& preferred_device);
-    bool ScheduleToWorkerInferRequest(IE::Task, DeviceName preferred_device = "") override;
-
-protected:
-    thread_local static IE::IInferRequestInternal*                     _sharedRequest;
 };
 }  // namespace MultiDevicePlugin

src/plugins/auto/common.hpp

@@ -143,7 +143,6 @@ public:
     std::mutex                                     _mutex;
     bool                                           _needPerfCounters;
     bool                                           _batchingDisabled = {false};
-    bool                                           _bindBuffer = false;
     bool                                           _startupfallback = true;
     bool                                           _runtimeFallback = true;
     virtual ~MultiScheduleContext() = default;
@@ -161,6 +160,7 @@ public:
     std::mutex                  _confMutex;
     std::mutex                  _fallbackMutex;
     MultiDeviceInferencePlugin* _plugin;
+    SoExecNetwork               _hwExecutableNetwork;
     virtual ~AutoScheduleContext() = default;
 };
 

src/plugins/auto/multi_schedule.cpp

@@ -312,7 +312,7 @@ IInferPtr MultiSchedule::CreateInferRequest() {
         if (!so)
             so = _passthroughExeNet._so;
         syncRequestImpl->setPointerToSo(so);
-    } else if (_multiSContext->_bindBuffer) {
+    } else if (std::static_pointer_cast<MultiDeviceInferRequest>(syncRequestImpl)->GetSharedRequest()) {
         auto sharedRequest = std::static_pointer_cast<MultiDeviceInferRequest>(syncRequestImpl)->GetSharedRequest();
         if (sharedRequest._ptr->getPointerToSo())
             syncRequestImpl->setPointerToSo(sharedRequest._ptr->getPointerToSo());

src/plugins/auto/plugin.cpp

@@ -351,19 +351,21 @@ IExecutableNetworkInternal::Ptr MultiDeviceInferencePlugin::LoadNetworkImpl(cons
     auto loadConfig = _pluginConfig;
     // if no perf hint from user with compiled model, or already been set with plugin
     // apply latency for AUTO, tput for MULTI
-    bool isHintSet = _pluginConfig.is_set_by_user(ov::hint::performance_mode) || config.find(ov::hint::performance_mode.name()) != config.end();
+    auto itorConfig = config.find(ov::hint::performance_mode.name());
-    if (!isHintSet) {
+    bool isHintSet = _pluginConfig.is_set_by_user(ov::hint::performance_mode) || itorConfig != config.end();
-        if (workModeAuto) {
+    if (!isHintSet && workModeAuto) {
-            // set performance hint to 'LATENCY' model for AutoExecutable Network.
+        // NO user sets perfHint, then set perfhint to 'LATENCY' for AutoExecutableNetwork.
-            loadConfig.set_property(ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY));
+        loadConfig.set_property(ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY));
-        } else {
-            // set performance hint to 'THROUGHPUT' model for MultiExecutable Network.
-            loadConfig.set_property(ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT));
-        }
     }
     // updateFromMap will check config valid
     loadConfig.set_user_property(PreProcessConfig(config), workModeAuto);
     loadConfig.apply_user_properties();
+    if (!workModeAuto) {
+        if (itorConfig != config.end() && itorConfig->second != InferenceEngine::PluginConfigParams::THROUGHPUT) {
+            LOG_WARNING_TAG("User set perf_hint:%s, but MULTI supports THROUGHPUT only", itorConfig->second.c_str());
+        }
+        loadConfig.set_property(ov::hint::performance_mode(ov::hint::PerformanceMode::CUMULATIVE_THROUGHPUT));
+    }
     auto fullProperty = loadConfig.get_full_properties();
     // this can be updated when plugin switch to 2.0 API
     std::map<std::string, std::string> fullConfig = ConvertToStringMap(fullProperty);
@@ -378,6 +380,8 @@ IExecutableNetworkInternal::Ptr MultiDeviceInferencePlugin::LoadNetworkImpl(cons
     std::unordered_map<std::string, ov::Any> multiNetworkConfig;
     std::vector<DeviceInformation> metaDevices;
     auto priorities = loadConfig.get_property(ov::device::priorities);
+    if (priorities.empty() && !workModeAuto)
+        IE_THROW() << "KEY_MULTI_DEVICE_PRIORITIES key is not set for " << GetName() << " device";
     if (priorities.find("AUTO") != std::string::npos || priorities.find("MULTI") != std::string::npos) {
         IE_THROW() << "The device candidate list should not include the meta plugin for " << GetName() << " device";
     }
@@ -397,256 +401,104 @@ IExecutableNetworkInternal::Ptr MultiDeviceInferencePlugin::LoadNetworkImpl(cons
         }
     };
 
-    // if workMode is AUTO
+    // check the configure and check if need to set PerfCounters configure to device
-    // only AUTO uses CheckConfig() to check fullConfig's parameters, MULTI does not
+    // and set filter configure
-    if (workModeAuto) {
+    OV_ITT_SCOPED_TASK(itt::domains::MULTIPlugin, "MultiDeviceInferencePlugin::LoadNetworkImpl::AutoMode");
-        // check the configure and check if need to set PerfCounters configure to device
+    auto autoSContext = std::make_shared<AutoScheduleContext>();
-        // and set filter configure
+    std::map<std::string, std::string> filterConfig;
-
+    auto strDevices = GetDeviceList(fullConfig);
-        OV_ITT_SCOPED_TASK(itt::domains::MULTIPlugin, "MultiDeviceInferencePlugin::LoadNetworkImpl::AutoMode");
+    // fill in the context for auto
-        auto autoSContext = std::make_shared<AutoScheduleContext>();
+    if (loadConfig.get_property(ov::enable_profiling)) {
-        std::map<std::string, std::string> filterConfig;
+        filterConfig.insert({ov::enable_profiling.name(), PluginConfigParams::YES});
-        auto strDevices = GetDeviceList(fullConfig);
+        autoSContext->_needPerfCounters = true;
-        // fill in the context for auto
+    }
-        if (loadConfig.get_property(ov::enable_profiling)) {
+    autoSContext->_modelPriority = MapPriorityValues(loadConfig.get_property(ov::hint::model_priority));
-            filterConfig.insert({ov::enable_profiling.name(), PluginConfigParams::YES});
+    autoSContext->_batchingDisabled = !(loadConfig.get_property(ov::hint::allow_auto_batching));
-            autoSContext->_needPerfCounters = true;
+    // set performanceHint for AutoExecutableNetwork
-        }
+    autoSContext->_performanceHint = loadConfig.get_property(ov::hint::performance_mode.name()).as<std::string>();
-        autoSContext->_modelPriority = MapPriorityValues(loadConfig.get_property(ov::hint::model_priority));
+    // filter the device that supports filter configure
-        autoSContext->_batchingDisabled = !(loadConfig.get_property(ov::hint::allow_auto_batching));
+    metaDevices = ParseMetaDevices(strDevices, fullConfig);
-        autoSContext->_performanceHint = loadConfig.get_property(ov::hint::performance_mode.name()).as<std::string>();
+    auto supportDevicesByConfig = FilterDevice(metaDevices, filterConfig);
-        // filter the device that supports filter configure
+    if (supportDevicesByConfig.empty()) {
-        auto metaDevices = ParseMetaDevices(strDevices, fullConfig);
+        IE_THROW() << "There is no device support the configure";
-        auto supportDevicesByConfig = FilterDevice(metaDevices, filterConfig);
+    }
-        if (supportDevicesByConfig.empty()) {
+    auto supportDevices = supportDevicesByConfig;
-             IE_THROW() << "There is no device support the configure";
+    CNNNetwork clonedNetwork;
-        }
+    std::string clonedModelPath = modelPath;
-        auto supportDevices = supportDevicesByConfig;
+    // reset the strDevices to support devices
-        CNNNetwork clonedNetwork;
+    strDevices = "";
-        std::string clonedModelPath = modelPath;
+    // calling GetValidDevices() to get a prioritized list of devices
-        if (modelPath.empty()) {
+    bool isCumulative =
-            // if network is valid
+        (autoSContext->_performanceHint == IE::PluginConfigParams::CUMULATIVE_THROUGHPUT) ? true : false;
-            LOG_INFO_TAG("load with CNN network");
+    std::list<DeviceInformation> devicesWithPriority(supportDevices.begin(), supportDevices.end());
-            supportDevices = FilterDeviceByNetwork(supportDevicesByConfig, network);
+    if (modelPath.empty()) {
-            // clone the network, in case of reshape conflict
+        // if network is valid
-            clonedNetwork = InferenceEngine::details::cloneNetwork(network);
+        LOG_INFO_TAG("load with CNN network");
-        } else {
+        supportDevices = FilterDeviceByNetwork(supportDevicesByConfig, network);
-            // model path, enable model load with single device situation
+        clonedNetwork = InferenceEngine::details::cloneNetwork(network);
-            if (supportDevices.size() > 1) {
-                clonedNetwork = GetCore()->ReadNetwork(modelPath, std::string());
-                // do we really need to disable model path?
-                clonedModelPath = "";
-                LOG_INFO_TAG("load with CNN network");
-            } else {
-                LOG_INFO_TAG("load with model path");
-            }
-        }
-        // reset the strDevices to support devices
-        strDevices = "";
-        // calling GetValidDevices() to get a prioritized list of devices
-        auto devicesWithPriority = GetValidDevice(supportDevices, networkPrecision);
-        for (auto iter = devicesWithPriority.begin(); iter != devicesWithPriority.end(); iter++) {
-            strDevices += iter->deviceName;
-            strDevices += ",";
-        }
-        strDevices.pop_back();
-        for (auto iter = supportDevices.begin(); iter != supportDevices.end(); iter++) {
-            auto& configs = iter->config;
-            for (auto& config : configs) {
-                LOG_INFO_TAG("device:%s, config:%s=%s",
-                             iter->deviceName.c_str(),
-                             config.first.c_str(),
-                             config.second.c_str());
-            }
-            // carry on batch configs only if user explicitly sets
-            if (loadConfig.is_set_by_user(ov::hint::allow_auto_batching))
-                insertPropToConfig(ov::hint::allow_auto_batching.name(), iter->deviceName, configs);
-            if (loadConfig.is_set_by_user(ov::auto_batch_timeout))
-                insertPropToConfig(ov::auto_batch_timeout.name(), iter->deviceName, configs);
-            insertPropToConfig(ov::cache_dir.name(), iter->deviceName, configs);
-            LOG_INFO_TAG("device:%s, priority:%ld", iter->deviceName.c_str(), iter->devicePriority);
-        }
-        autoSContext->_modelPath = clonedModelPath;
         // clone the network, in case of reshape conflict
-        autoSContext->_network = clonedNetwork;
-        autoSContext->_devicePriorities = supportDevices;
-        autoSContext->_devicePrioritiesInitial = supportDevices;
-        autoSContext->_strDevices = strDevices;
-        autoSContext->_plugin = this;
-        autoSContext->_core = GetCore();
-        autoSContext->_LogTag = _LogTag;
-        autoSContext->_bindBuffer = loadConfig.get_property(ov::intel_auto::device_bind_buffer);
-        autoSContext->_startupfallback = loadConfig.get_property(ov::intel_auto::enable_startup_fallback);
-        autoSContext->_runtimeFallback = loadConfig.get_property(ov::intel_auto::enable_runtime_fallback);
-        return std::make_shared<AutoExecutableNetwork>(autoSContext, std::make_shared<AutoSchedule>());
-    }
-    OV_ITT_SCOPED_TASK(itt::domains::MULTIPlugin, "MultiDeviceInferencePlugin::LoadNetworkImpl:MultiMode");
-    // if is cumulative, PERFORMANCE_HINT set to THROUGHPUT and _LogTag set to AUTO
-    auto configIter =
-        std::find_if(fullConfig.begin(), fullConfig.end(), [](const std::pair<std::string, std::string>& config) {
-            return (config.first == CONFIG_KEY(PERFORMANCE_HINT));
-        });
-    if (configIter != fullConfig.end() && configIter->second == InferenceEngine::PluginConfigParams::CUMULATIVE_THROUGHPUT) {
-        configIter->second = InferenceEngine::PluginConfigParams::THROUGHPUT;
-        _LogTag = "AUTO";
-        LOG_INFO_TAG("CUMULATIVE Call MULTI PERFORMANCE_HINT set to THROUGHPUT");
-    }
-    if (priorities.empty()) {
-        IE_THROW() << "KEY_MULTI_DEVICE_PRIORITIES key is not set for " << GetName() << " device";
-    } else {  // for use case -d MULTI:xPU or -d AUTO:xPU
-        auto metaDevicesByConfig = ParseMetaDevices(priorities, fullConfig);
-        metaDevices = modelPath.empty() ? FilterDeviceByNetwork(metaDevicesByConfig, network)
-                                        : metaDevicesByConfig;
-        if (metaDevicesByConfig.size() != metaDevices.size()) {
-            LOG_DEBUG_TAG("stateful/dynamic model, loaded to single device");
-            multiNetworkConfig[ov::device::priorities.name()]
-                    = metaDevices[0].deviceName;
-        } else {
-            multiNetworkConfig[ov::device::priorities.name()] = priorities;
-        }
-    }
-    auto multiSContext = std::make_shared<MultiScheduleContext>();
-    DeviceMap<SoExecutableNetworkInternal> executableNetworkPerDevice;
-    std::mutex load_mutex;
-    std::vector<Task> loads;
-
-    auto loadInferEngTask = [&](DeviceInformation& p) {
-        auto tmpiter = fullConfig.find(CONFIG_KEY(ALLOW_AUTO_BATCHING));
-        if (tmpiter != fullConfig.end()) {
-            if (tmpiter->second == PluginConfigParams::NO) {
-                LOG_INFO_TAG("set %s=%s", tmpiter->first.c_str(), tmpiter->second.c_str());
-                multiSContext->_batchingDisabled = true;
-            }
-            if (loadConfig.is_set_by_user(ov::hint::allow_auto_batching))
-                p.config.insert({tmpiter->first, tmpiter->second});
-        }
-        if (loadConfig.is_set_by_user(ov::auto_batch_timeout))
-            insertPropToConfig(ov::auto_batch_timeout.name(), p.deviceName, p.config);
-        insertPropToConfig(ov::cache_dir.name(), p.deviceName, p.config);
-        const auto& deviceName = p.deviceName;
-        const auto& deviceConfig = p.config;
-        SoExecutableNetworkInternal exec_net;
-        LOG_DEBUG_TAG("load network to device:%s", deviceName.c_str());
-        try {
-            if (modelPath.empty()) {
-                exec_net = GetCore()->LoadNetwork(network, deviceName, deviceConfig);
-            } else {
-                exec_net = GetCore()->LoadNetwork(modelPath, deviceName, deviceConfig);
-            }
-        } catch (const IE::Exception& iie) {
-            if (_LogTag == "AUTO") {
-                LOG_DEBUG_TAG("Failed to load network to device:%s with error: %s", deviceName.c_str(), iie.what());
-                return;
-            } else {
-                IE_THROW() << "Failed to load network to device: " << deviceName.c_str() << " with error:" <<
-                    iie.what();
-            }
-        }
-
-        try {
-            std::string sStreamNums = "";
-            std::string sThreadNums = "";
-            if (deviceName.find("CPU") != std::string::npos) {
-                sStreamNums = exec_net->GetMetric(ov::num_streams.name()).as<std::string>();
-                sThreadNums = exec_net->GetMetric(ov::inference_num_threads.name()).as<std::string>();
-            } else if (deviceName.find("GPU") != std::string::npos) {
-                sStreamNums = exec_net->GetConfig(ov::num_streams.name()).as<std::string>();
-                sThreadNums = exec_net->GetConfig(ov::compilation_num_threads.name()).as<std::string>();
-            }
-
-            // print CPU or GPU streams num and threads num
-            if (!sStreamNums.empty() && !sThreadNums.empty()) {
-                LOG_INFO_TAG("after load network, %s streamNums:%s, %s threadNums:%s",
-                             deviceName.c_str(),
-                             sStreamNums.c_str(),
-                             deviceName.c_str(),
-                             sThreadNums.c_str());
-            }
-        } catch (const IE::Exception&) {
-            LOG_DEBUG_TAG("deviceName:%s cannot get streamNums and threadNums from exec_net", deviceName.c_str());
-        }
-        std::unique_lock<std::mutex> lock{load_mutex};
-        executableNetworkPerDevice.insert({deviceName, exec_net});
-        multiNetworkConfig.insert(deviceConfig.begin(), deviceConfig.end());
-    };
-
-    // Check if CPU is in device list
-    auto iterCPU = std::find_if(metaDevices.begin(), metaDevices.end(), [&](DeviceInformation& d) {
-        return d.deviceName.find("CPU") != std::string::npos;
-    });
-    // Load devices other than CPU first
-    for (auto& p : metaDevices) {
-        if (iterCPU != metaDevices.end() && p.deviceName == iterCPU->deviceName) {
-            continue;
-        }
-        loads.push_back([&]() {
-            loadInferEngTask(p);
-        });
-    }
-
-    auto executor = executorManager()->getIdleCPUStreamsExecutor(
-        IStreamsExecutor::Config{"MultiDeviceAsyncLoad",
-                                 static_cast<int>(std::thread::hardware_concurrency()) /* max possible #streams*/,
-                                 0 /*default threads per stream, workaround for ticket 62376*/,
-                                 IStreamsExecutor::ThreadBindingType::NONE});
-    if (loads.size() > 0) {
-        // Wait for the device to load the network
-        executor->runAndWait(loads);
-        loads.clear();
-    }
-
-    // Finally load the CPU
-    if (iterCPU != metaDevices.end()) {
-        if (!executableNetworkPerDevice.empty() && iterCPU->config.find(ov::affinity.name()) == iterCPU->config.end()) {
-            LOG_DEBUG_TAG("set affinity to NUMA and disable hyper thread for CPU");
-            // If the other devices load successfully and no user set affinity then set NUMA to CPU
-            iterCPU->config.insert({ov::affinity.name(), ov::affinity(ov::Affinity::NUMA).second.as<std::string>()});
-            iterCPU->config.insert({CONFIG_KEY_INTERNAL(ENABLE_HYPER_THREAD), CONFIG_VALUE(NO)});
-        }
-        loads.push_back([&]() {
-            loadInferEngTask(*iterCPU);
-        });
-        // Wait for CPU to load the network
-        executor->runAndWait(loads);
-    }
-
-    if (executableNetworkPerDevice.empty())
-        IE_THROW(NotFound) << "Failed to load network to any device "
-                           <<  "that the " << GetName() << " device is initialized to work with";
-
-    // checking the perf counters config from the loaded network to respect both device's plugin and load-specific setting
-    size_t num_plugins_supporting_perf_counters = 0;
-    for (auto& n : executableNetworkPerDevice) {
-            try {
-                num_plugins_supporting_perf_counters +=
-                        n.second->GetConfig(PluginConfigParams::KEY_PERF_COUNT).as<std::string>() ==
-                        PluginConfigParams::YES;
-            } catch (const IE::Exception&) {
-            }
-    }
-    // MULTI can enable the perf counters only if all  devices support/enable that
-    bool enablePerfCounters = num_plugins_supporting_perf_counters == executableNetworkPerDevice.size();
-    multiSContext->_devicePriorities = metaDevices;
-    multiSContext->_devicePrioritiesInitial = metaDevices;
-    multiSContext->_networksPerDevice = executableNetworkPerDevice;
-    multiSContext->_config = multiNetworkConfig;
-    multiSContext->_needPerfCounters = enablePerfCounters;
-    multiSContext->_core = GetCore();
-    multiSContext->_LogTag = _LogTag;
-    IExecutableNetworkInternal::Ptr impl;
-    auto tmp = loadConfig.get_property(ov::intel_auto::device_bind_buffer);
-    if (tmp) {
-        multiSContext->_bindBuffer = true;
-        impl = std::make_shared<MultiExecutableNetwork>(multiSContext, std::make_shared<BinderMultiSchedule>());
     } else {
-        impl = std::make_shared<MultiExecutableNetwork>(multiSContext, std::make_shared<MultiSchedule>());
+        // model path, enable model load with single device situation
+        if (supportDevices.size() > 1 && !isCumulative) {
+            clonedNetwork = GetCore()->ReadNetwork(modelPath, std::string());
+            // do we really need to disable model path?
+            clonedModelPath = "";
+            LOG_INFO_TAG("load with CNN network");
+        } else {
+            LOG_INFO_TAG("load with model path");
+        }
+    }
+    if (!isCumulative) {
+        devicesWithPriority = GetValidDevice(supportDevices, networkPrecision);
+    }
+    for (auto iter = devicesWithPriority.begin(); iter != devicesWithPriority.end(); iter++) {
+        strDevices += iter->deviceName;
+        strDevices += ",";
+    }
+    strDevices.pop_back();
+    for (auto iter = supportDevices.begin(); iter != supportDevices.end(); iter++) {
+        auto& configs = iter->config;
+        for (auto& config : configs) {
+            LOG_INFO_TAG("device:%s, config:%s=%s",
+                         iter->deviceName.c_str(),
+                         config.first.c_str(),
+                         config.second.c_str());
+        }
+        // carry on batch configs only if user explicitly sets
+        if (loadConfig.is_set_by_user(ov::hint::allow_auto_batching))
+            insertPropToConfig(ov::hint::allow_auto_batching.name(), iter->deviceName, configs);
+        if (loadConfig.is_set_by_user(ov::auto_batch_timeout))
+            insertPropToConfig(ov::auto_batch_timeout.name(), iter->deviceName, configs);
+        insertPropToConfig(ov::cache_dir.name(), iter->deviceName, configs);
+        LOG_INFO_TAG("device:%s, priority:%ld", iter->deviceName.c_str(), iter->devicePriority);
+    }
+    autoSContext->_modelPath = clonedModelPath;
+    // clone the network, in case of reshape conflict
+    autoSContext->_network = clonedNetwork;
+    autoSContext->_devicePriorities = supportDevices;
+    autoSContext->_devicePrioritiesInitial = supportDevices;
+    autoSContext->_strDevices = strDevices;
+    autoSContext->_plugin = this;
+    autoSContext->_core = GetCore();
+    autoSContext->_LogTag = _LogTag;
+    autoSContext->_startupfallback = loadConfig.get_property(ov::intel_auto::enable_startup_fallback);
+    autoSContext->_runtimeFallback = loadConfig.get_property(ov::intel_auto::enable_runtime_fallback);
+    IExecutableNetworkInternal::Ptr impl;
+    // enable bind only in cumulative_throughput mode
+    if (loadConfig.get_property(ov::intel_auto::device_bind_buffer) &&
+        autoSContext->_performanceHint == "CUMULATIVE_THROUGHPUT") {
+        LOG_INFO_TAG("runtime fallback set to disabled in binder mode");
+        autoSContext->_runtimeFallback = false;
+        impl = std::make_shared<AutoExecutableNetwork>(autoSContext, std::make_shared<BinderMultiSchedule>());
+    } else {
+        impl = std::make_shared<AutoExecutableNetwork>(autoSContext, std::make_shared<AutoSchedule>());
     }
     if (!modelPath.empty()) {
         SetExeNetworkInfo(impl,
-                          executableNetworkPerDevice.begin()->second->GetInputsInfo(),
+                          autoSContext->_hwExecutableNetwork->GetInputsInfo(),
-                          executableNetworkPerDevice.begin()->second->GetOutputsInfo());
+                          autoSContext->_hwExecutableNetwork->GetOutputsInfo());
-        impl->setInputs(executableNetworkPerDevice.begin()->second->getInputs());
+        impl->setInputs(autoSContext->_hwExecutableNetwork->getInputs());
-        impl->setOutputs(executableNetworkPerDevice.begin()->second->getOutputs());
+        impl->setOutputs(autoSContext->_hwExecutableNetwork->getOutputs());
     }
     return impl;
 }

src/plugins/intel_cpu/tests/functional/shared_tests_instances/behavior/ov_plugin/core_integration.cpp

@@ -411,12 +411,12 @@ INSTANTIATE_TEST_SUITE_P(smoke_HETERO_OVClassLoadNetworkWithSecondaryPropertiesT
 // IE Class load and check network with ov::device::properties
 INSTANTIATE_TEST_SUITE_P(smoke_CPU_OVClassLoadNetworkAndCheckWithSecondaryPropertiesTest,
                          OVClassLoadNetworkAndCheckSecondaryPropertiesTest,
-                         ::testing::Combine(::testing::Values("CPU", "MULTI:CPU"),
+                         ::testing::Combine(::testing::Values("CPU"),
                                             ::testing::ValuesIn(configsDeviceProperties)));
 
 INSTANTIATE_TEST_SUITE_P(smoke_CPU_OVClassLoadNetworkAndCheckWithSecondaryPropertiesDoubleTest,
                          OVClassLoadNetworkAndCheckSecondaryPropertiesTest,
-                         ::testing::Combine(::testing::Values("CPU", "MULTI:CPU"),
+                         ::testing::Combine(::testing::Values("CPU"),
                                             ::testing::ValuesIn(configsDevicePropertiesDouble)));
 INSTANTIATE_TEST_SUITE_P(
         smoke_OVClassLoadNetworkTest, OVClassLoadNetworkTest,

src/plugins/intel_cpu/tests/functional/shared_tests_instances/behavior/ov_plugin/properties_tests.cpp

@@ -56,27 +56,13 @@ INSTANTIATE_TEST_SUITE_P(smoke_cpuCompileModelBehaviorTests,
                          OVSetPropComplieModleGetPropTests::getTestCaseName);
 
 const std::vector<ov::AnyMap> multi_setcore_properties = {
-    {ov::device::priorities(CommonTestUtils::DEVICE_CPU),
-     ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT),
-     ov::hint::num_requests(2),
-     ov::hint::allow_auto_batching(false),
-     ov::enable_profiling(false)},
     {ov::device::priorities(CommonTestUtils::DEVICE_CPU),
      ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY),
-     ov::hint::num_requests(8),
+     ov::hint::model_priority(ov::hint::Priority::HIGH)}};
-     ov::hint::allow_auto_batching(true),
-     ov::enable_profiling(true)}};
 const std::vector<ov::AnyMap> multi_compileModel_properties = {
-    {ov::device::priorities(CommonTestUtils::DEVICE_CPU),
-     ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY),
-     ov::hint::num_requests(10),
-     ov::hint::allow_auto_batching(true),
-     ov::enable_profiling(true)},
     {ov::device::priorities(CommonTestUtils::DEVICE_CPU),
      ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT),
-     ov::hint::num_requests(2),
+     ov::hint::model_priority(ov::hint::Priority::MEDIUM)}};
-     ov::hint::allow_auto_batching(false),
-     ov::enable_profiling(false)}};
 
 INSTANTIATE_TEST_SUITE_P(smoke_MultiCompileModelBehaviorTests,
                          OVSetPropComplieModleGetPropTests,
@@ -88,36 +74,24 @@ INSTANTIATE_TEST_SUITE_P(smoke_MultiCompileModelBehaviorTests,
 const std::vector<ov::AnyMap> auto_setcore_properties = {
     {ov::device::priorities(CommonTestUtils::DEVICE_CPU),
      ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT),
-     ov::hint::num_requests(2),
+     ov::hint::model_priority(ov::hint::Priority::HIGH)},
-     ov::hint::allow_auto_batching(false),
-     ov::enable_profiling(false)},
     {ov::device::priorities(CommonTestUtils::DEVICE_CPU),
      ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY),
-     ov::hint::num_requests(8),
+     ov::hint::model_priority(ov::hint::Priority::HIGH)},
-     ov::hint::allow_auto_batching(true),
-     ov::enable_profiling(true)},
     {ov::device::priorities(CommonTestUtils::DEVICE_CPU),
      ov::hint::performance_mode(ov::hint::PerformanceMode::CUMULATIVE_THROUGHPUT),
-     ov::hint::num_requests(10),
+     ov::hint::model_priority(ov::hint::Priority::HIGH)},
-     ov::hint::allow_auto_batching(false),
-     ov::enable_profiling(true)},
 };
 const std::vector<ov::AnyMap> auto_compileModel_properties = {
     {ov::device::priorities(CommonTestUtils::DEVICE_CPU),
      ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY),
-     ov::hint::num_requests(8),
+     ov::hint::model_priority(ov::hint::Priority::MEDIUM)},
-     ov::hint::allow_auto_batching(true),
-     ov::enable_profiling(true)},
     {ov::device::priorities(CommonTestUtils::DEVICE_CPU),
      ov::hint::performance_mode(ov::hint::PerformanceMode::CUMULATIVE_THROUGHPUT),
-     ov::hint::num_requests(10),
+     ov::hint::model_priority(ov::hint::Priority::MEDIUM)},
-     ov::hint::allow_auto_batching(false),
-     ov::enable_profiling(false)},
     {ov::device::priorities(CommonTestUtils::DEVICE_CPU),
      ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT),
-     ov::hint::num_requests(2),
+     ov::hint::model_priority(ov::hint::Priority::MEDIUM)}};
-     ov::hint::allow_auto_batching(true),
-     ov::enable_profiling(false)}};
 INSTANTIATE_TEST_SUITE_P(smoke_AutoCompileModelBehaviorTests,
                          OVSetPropComplieModleGetPropTests,
                          ::testing::Combine(::testing::Values(CommonTestUtils::DEVICE_AUTO),

src/plugins/intel_cpu/tests/functional/shared_tests_instances/behavior/plugin/configuration_tests.cpp

@@ -281,18 +281,12 @@ namespace {
     const std::vector<std::map<std::string, std::string>> auto_multi_prop_config = {
         {{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES, CommonTestUtils::DEVICE_CPU},
          {InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT, InferenceEngine::PluginConfigParams::THROUGHPUT},
-         {InferenceEngine::PluginConfigParams::KEY_EXCLUSIVE_ASYNC_REQUESTS, InferenceEngine::PluginConfigParams::YES},
+         {InferenceEngine::PluginConfigParams::KEY_MODEL_PRIORITY, InferenceEngine::PluginConfigParams::MODEL_PRIORITY_MED}}};
-         {InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT_NUM_REQUESTS, "2"},
-         {InferenceEngine::PluginConfigParams::KEY_ALLOW_AUTO_BATCHING, InferenceEngine::PluginConfigParams::NO},
-         {InferenceEngine::PluginConfigParams::KEY_PERF_COUNT, InferenceEngine::PluginConfigParams::NO}}};
 
     const std::vector<std::map<std::string, std::string>> auto_multi_loadNetWork_config = {
         {{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES, CommonTestUtils::DEVICE_CPU},
-         {InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT, InferenceEngine::PluginConfigParams::LATENCY},
+         {InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT, InferenceEngine::PluginConfigParams::THROUGHPUT},
-         {InferenceEngine::PluginConfigParams::KEY_EXCLUSIVE_ASYNC_REQUESTS, InferenceEngine::PluginConfigParams::NO},
+         {InferenceEngine::PluginConfigParams::KEY_MODEL_PRIORITY, InferenceEngine::PluginConfigParams::MODEL_PRIORITY_HIGH}}};
-         {InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT_NUM_REQUESTS, "10"},
-         {InferenceEngine::PluginConfigParams::KEY_ALLOW_AUTO_BATCHING, InferenceEngine::PluginConfigParams::YES},
-         {InferenceEngine::PluginConfigParams::KEY_PERF_COUNT, InferenceEngine::PluginConfigParams::YES}}};
 
     INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests,
                              SetPropLoadNetWorkGetPropTests,

src/plugins/intel_cpu/tests/functional/shared_tests_instances/skip_tests_config.cpp

@@ -93,7 +93,9 @@ std::vector<std::string> disabledTestPatterns() {
         R"(.*Behavior.*ExecutableNetworkBaseTest.*canSetConfigToExecNetWithIncorrectConfig.*)",
         R"(.*Hetero.*Behavior.*ExecutableNetworkBaseTest.*ExecGraphInfo.*)",
         R"(.*Hetero.*Behavior.*ExecutableNetworkBaseTest.*CanCreateTwoExeNetworksAndCheckFunction.*)",
-
+        // TODO: CVS-104942
+        R"(.*(Auto|Multi).*Behavior.*ExecutableNetworkBaseTest.*canLoadCorrectNetworkToGetExecutableAndCheckConfig.*)",
+        R"(.*(Auto|Multi).*SetPropLoadNetWorkGetPropTests.*)",
         // CPU does not support dynamic rank
         // Issue: CVS-66778
         R"(.*smoke_BehaviorTests.*InferFullyDynamicNetworkWith(S|G)etTensor.*)",

src/tests/functional/plugin/gpu/shared_tests_instances/behavior/ov_plugin/properties_tests.cpp

@@ -109,25 +109,11 @@ INSTANTIATE_TEST_SUITE_P(smoke_gpuCompileModelBehaviorTests,
 const std::vector<ov::AnyMap> multi_setcore_properties = {
     {ov::device::priorities(CommonTestUtils::DEVICE_GPU),
      ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT),
-     ov::hint::num_requests(2),
+     ov::hint::model_priority(ov::hint::Priority::HIGH)}};
-     ov::hint::allow_auto_batching(false),
-     ov::enable_profiling(false)},
-    {ov::device::priorities(CommonTestUtils::DEVICE_GPU),
-     ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY),
-     ov::hint::num_requests(8),
-     ov::hint::allow_auto_batching(true),
-     ov::enable_profiling(true)}};
 const std::vector<ov::AnyMap> multi_compileModel_properties = {
-    {ov::device::priorities(CommonTestUtils::DEVICE_GPU),
-     ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY),
-     ov::hint::num_requests(10),
-     ov::hint::allow_auto_batching(true),
-     ov::enable_profiling(true)},
     {ov::device::priorities(CommonTestUtils::DEVICE_GPU),
      ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT),
-     ov::hint::num_requests(2),
+     ov::hint::model_priority(ov::hint::Priority::MEDIUM)}};
-     ov::hint::allow_auto_batching(false),
-     ov::enable_profiling(false)}};
 
 INSTANTIATE_TEST_SUITE_P(smoke_MultiCompileModelBehaviorTests,
                          OVSetPropComplieModleGetPropTests,
@@ -139,36 +125,24 @@ INSTANTIATE_TEST_SUITE_P(smoke_MultiCompileModelBehaviorTests,
 const std::vector<ov::AnyMap> auto_setcore_properties = {
     {ov::device::priorities(CommonTestUtils::DEVICE_GPU),
      ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT),
-     ov::hint::num_requests(2),
+     ov::hint::model_priority(ov::hint::Priority::HIGH)},
-     ov::hint::allow_auto_batching(false),
-     ov::enable_profiling(false)},
     {ov::device::priorities(CommonTestUtils::DEVICE_GPU),
      ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY),
-     ov::hint::num_requests(8),
+     ov::hint::model_priority(ov::hint::Priority::HIGH)},
-     ov::hint::allow_auto_batching(true),
-     ov::enable_profiling(true)},
     {ov::device::priorities(CommonTestUtils::DEVICE_GPU),
      ov::hint::performance_mode(ov::hint::PerformanceMode::CUMULATIVE_THROUGHPUT),
-     ov::hint::num_requests(10),
+     ov::hint::model_priority(ov::hint::Priority::HIGH)},
-     ov::hint::allow_auto_batching(false),
-     ov::enable_profiling(true)},
 };
 const std::vector<ov::AnyMap> auto_compileModel_properties = {
     {ov::device::priorities(CommonTestUtils::DEVICE_GPU),
      ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY),
-     ov::hint::num_requests(8),
+     ov::hint::model_priority(ov::hint::Priority::MEDIUM)},
-     ov::hint::allow_auto_batching(true),
-     ov::enable_profiling(true)},
     {ov::device::priorities(CommonTestUtils::DEVICE_GPU),
      ov::hint::performance_mode(ov::hint::PerformanceMode::CUMULATIVE_THROUGHPUT),
-     ov::hint::num_requests(10),
+     ov::hint::model_priority(ov::hint::Priority::MEDIUM)},
-     ov::hint::allow_auto_batching(false),
-     ov::enable_profiling(false)},
     {ov::device::priorities(CommonTestUtils::DEVICE_GPU),
      ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT),
-     ov::hint::num_requests(2),
+     ov::hint::model_priority(ov::hint::Priority::MEDIUM)}};
-     ov::hint::allow_auto_batching(true),
-     ov::enable_profiling(false)}};
 INSTANTIATE_TEST_SUITE_P(smoke_AutoCompileModelBehaviorTests,
                          OVSetPropComplieModleGetPropTests,
                          ::testing::Combine(::testing::Values(CommonTestUtils::DEVICE_AUTO),

src/tests/functional/plugin/gpu/shared_tests_instances/behavior/plugin/configuration_tests.cpp

@@ -309,22 +309,16 @@ namespace {
             {{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES, CommonTestUtils::DEVICE_GPU},
              {InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT,
               InferenceEngine::PluginConfigParams::THROUGHPUT},
-             {InferenceEngine::PluginConfigParams::KEY_EXCLUSIVE_ASYNC_REQUESTS,
+             {InferenceEngine::PluginConfigParams::KEY_MODEL_PRIORITY,
-              InferenceEngine::PluginConfigParams::YES},
+              InferenceEngine::PluginConfigParams::MODEL_PRIORITY_MED}}};
-             {InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT_NUM_REQUESTS, "2"},
-             {InferenceEngine::PluginConfigParams::KEY_ALLOW_AUTO_BATCHING, InferenceEngine::PluginConfigParams::NO},
-             {InferenceEngine::PluginConfigParams::KEY_PERF_COUNT, InferenceEngine::PluginConfigParams::NO}}};
     };
 
     auto auto_multi_loadNetWork_config = []() {
         return std::vector<std::map<std::string, std::string>>{
             {{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES, CommonTestUtils::DEVICE_GPU},
-             {InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT, InferenceEngine::PluginConfigParams::LATENCY},
+             {InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT, InferenceEngine::PluginConfigParams::THROUGHPUT},
-             {InferenceEngine::PluginConfigParams::KEY_EXCLUSIVE_ASYNC_REQUESTS,
+             {InferenceEngine::PluginConfigParams::KEY_MODEL_PRIORITY,
-              InferenceEngine::PluginConfigParams::NO},
+              InferenceEngine::PluginConfigParams::MODEL_PRIORITY_HIGH}}};
-             {InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT_NUM_REQUESTS, "10"},
-             {InferenceEngine::PluginConfigParams::KEY_ALLOW_AUTO_BATCHING, InferenceEngine::PluginConfigParams::YES},
-             {InferenceEngine::PluginConfigParams::KEY_PERF_COUNT, InferenceEngine::PluginConfigParams::YES}}};
     };
 
     INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests,

src/tests/functional/plugin/gpu/shared_tests_instances/multi/gpu_remote_blob_tests.cpp

@@ -106,12 +106,17 @@ TEST_P(MultiDevice_Bind_oversubsciption_test, oversubsciptionOfInferRequest) {
 
 auto device_names_and_support_for_remote_blobs2 = []() {
     return std::vector<DevicesNamseAndProperties>{
+    // another GPU (the test will test its presence), different OCL contexts
+    // use GPU.0 as reference, expect auto to throw exception on other hardware contexts
 #ifdef ENABLE_INTEL_CPU
         //{{CPU}, {}},  // stand-alone CPU via MULTI (no GPU), no OCL context
-        {{CPU}, {ov::intel_auto::device_bind_buffer(true)}},  // stand-alone CPU via MULTI (no GPU), no OCL context
+        {{"GPU.1", CPU},
+         {ov::intel_auto::device_bind_buffer(true)}},
+        {{"GPU.1", CPU},
+         {ov::intel_auto::device_bind_buffer(false)}},
 #endif
-        {{"GPU.1"}, {}},  // another GPU (the test will test its presence), different OCL contexts
+        {{"GPU.1"}, {}},
-        {{"GPU.1"}, {ov::intel_auto::device_bind_buffer(true)}},  // another GPU (the test will test its presence), different OCL contexts
+        {{"GPU.1"}, {ov::intel_auto::device_bind_buffer(true)}},
     };
 };
 

src/tests/functional/plugin/gpu/shared_tests_instances/skip_tests_config.cpp

@@ -124,5 +124,9 @@ std::vector<std::string> disabledTestPatterns() {
 
             // Looks like the test is targeting CPU plugin and doesn't respect that execution graph may vary from plugin to plugin
             R"(.*ExecGraphSerializationTest.*)",
+
+            // TODO: support getconfig in auto/multi CVS-104942
+            // TODO: move auto/multi cases to dedicated unit tests
+            R"(.*(Auto|Multi).*SetPropLoadNetWorkGetPropTests.*)",
     };
 }

src/tests/functional/plugin/shared/include/behavior/executable_network/exec_network_base.hpp

@@ -70,10 +70,6 @@ TEST_P(ExecutableNetworkBaseTest, checkGetMetric) {
 
 TEST_P(ExecutableNetworkBaseTest, canLoadCorrectNetworkToGetExecutableAndCheckConfig) {
     auto execNet = ie->LoadNetwork(cnnNet, target_device, configuration);
-    if (target_device == CommonTestUtils::DEVICE_AUTO) {
-        // AUTO executable network didn't support to read any config.
-        GTEST_SKIP();
-    }
     for (const auto& configItem : configuration) {
         InferenceEngine::Parameter param;
         ASSERT_NO_THROW(param = execNet.GetConfig(configItem.first));

src/tests/functional/plugin/shared/include/behavior/ov_plugin/core_integration.hpp

@@ -1355,6 +1355,7 @@ TEST_P(OVClassLoadNetworkAndCheckSecondaryPropertiesTest, LoadNetworkAndCheckSec
     ASSERT_TRUE(property.count(ov::num_streams.name()));
     auto actual = property.at(ov::num_streams.name()).as<int32_t>();
     ov::Any value;
+    //AutoExcutableNetwork GetMetric() does not support key ov::num_streams
     OV_ASSERT_NO_THROW(value = model.get_property(ov::num_streams.name()));
     int32_t expect = value.as<int32_t>();
     ASSERT_EQ(actual, expect);
@@ -1382,7 +1383,7 @@ TEST_P(OVClassLoadNetWorkDoNotReturnDefaultHintTest, LoadNetworkDoNotReturnDefau
     if (target_device.find("AUTO") != std::string::npos) {
         ASSERT_NE(value, ov::hint::PerformanceMode::LATENCY);
     } else {
-        ASSERT_NE(value, ov::hint::PerformanceMode::THROUGHPUT);
+        ASSERT_EQ(value, ov::hint::PerformanceMode::THROUGHPUT);
     }
 }
 

src/tests/functional/plugin/shared/src/behavior/ov_plugin/properties_tests.cpp

@@ -178,8 +178,6 @@ TEST_P(OVSetPropComplieModleGetPropTests, SetPropertyComplieModelGetProperty) {
 
     ov::CompiledModel exeNetWork;
     OV_ASSERT_NO_THROW(exeNetWork = core->compile_model(model, target_device, compileModelProperties));
-    if (target_device == CommonTestUtils::DEVICE_AUTO)
-        GTEST_SKIP();
 
     for (const auto& property_item : compileModelProperties) {
         Any exeNetProperty;

src/tests/functional/plugin/shared/src/behavior/plugin/configuration_tests.cpp

@@ -197,9 +197,8 @@ TEST_P(SetPropLoadNetWorkGetPropTests, SetPropLoadNetWorkGetProperty) {
 
     InferenceEngine::ExecutableNetwork exeNetWork;
     ASSERT_NO_THROW(exeNetWork = ie->LoadNetwork(cnnNet, target_device, loadNetWorkConfig));
-    if (target_device == CommonTestUtils::DEVICE_AUTO)
-        GTEST_SKIP();
 
+    //ie's setConfig and LoadNetwork should not affect each other, for config settings
     for (const auto& property_item : loadNetWorkConfig) {
         InferenceEngine::Parameter exeNetProperty;
         ASSERT_NO_THROW(exeNetProperty = exeNetWork.GetConfig(property_item.first));
@@ -213,5 +212,4 @@ TEST_P(SetPropLoadNetWorkGetPropTests, SetPropLoadNetWorkGetProperty) {
         ASSERT_EQ(property_item.second, property.as<std::string>());
     }
 }
-
 } // namespace BehaviorTestsDefinitions

src/tests/unit/auto/auto_ctput_call_multi.cpp

@@ -45,6 +45,11 @@ public:
     ov::SoPtr<IExecutableNetworkInternal> cpuMockExeNetwork;
     NiceMock<MockIInferencePlugin>* cpuMockIPlugin;
     std::shared_ptr<InferenceEngine::IInferencePlugin> cpuMockPlugin;
+    // mock gpu exeNetwork
+    std::shared_ptr<NiceMock<MockIExecutableNetworkInternal>> gpuMockIExeNet;
+    ov::SoPtr<IExecutableNetworkInternal> gpuMockExeNetwork;
+    NiceMock<MockIInferencePlugin>* gpuMockIPlugin;
+    std::shared_ptr<InferenceEngine::IInferencePlugin> gpuMockPlugin;
     std::shared_ptr<NiceMock<MockIInferRequestInternal>> inferReqInternal;
 
 public:
@@ -85,6 +90,16 @@ public:
         EXPECT_CALL(*cpuMockIPluginPtr, LoadNetwork(MatcherCast<const CNNNetwork&>(_), _)).Times(1);
         cpuMockExeNetwork = ov::SoPtr<InferenceEngine::IExecutableNetworkInternal>(cpuMockPlugin->LoadNetwork(CNNNetwork{}, {}), {});
 
+        // prepare gpuMockExeNetwork
+        gpuMockIExeNet = std::make_shared<NiceMock<MockIExecutableNetworkInternal>>();
+        auto gpuMockIPluginPtr = std::make_shared<NiceMock<MockIInferencePlugin>>();
+        ON_CALL(*gpuMockIPluginPtr, LoadNetwork(MatcherCast<const CNNNetwork&>(_), _))
+            .WillByDefault(Return(gpuMockIExeNet));
+        gpuMockPlugin = gpuMockIPluginPtr;
+        // remove annoying ON CALL message
+        EXPECT_CALL(*gpuMockIPluginPtr, LoadNetwork(MatcherCast<const CNNNetwork&>(_), _)).Times(1);
+        gpuMockExeNetwork = ov::SoPtr<InferenceEngine::IExecutableNetworkInternal>(gpuMockPlugin->LoadNetwork(CNNNetwork{}, {}), {});
+
         // prepare mockicore and cnnNetwork for loading
         core = std::shared_ptr<NiceMock<MockICore>>(new NiceMock<MockICore>());
         auto* origin_plugin = new NiceMock<MockMultiPluginForLoadNetworkWithPropertiesTest>();
@@ -95,6 +110,9 @@ public:
         ON_CALL(*cpuMockIExeNet.get(), CreateInferRequest()).WillByDefault(Return(inferReqInternal));
         ON_CALL(*cpuMockIExeNet.get(), GetMetric(StrEq(METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS))))
             .WillByDefault(Return("0"));
+        ON_CALL(*gpuMockIExeNet.get(), CreateInferRequest()).WillByDefault(Return(inferReqInternal));
+        ON_CALL(*gpuMockIExeNet.get(), GetMetric(StrEq(METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS))))
+            .WillByDefault(Return("0"));
 
         std::vector<std::string> availableDevs = {"CPU", "GPU"};
         ON_CALL(*core, GetAvailableDevices()).WillByDefault(Return(availableDevs));
@@ -161,58 +179,53 @@ public:
                 LoadNetwork(::testing::Matcher<const InferenceEngine::CNNNetwork&>(_),
                             ::testing::Matcher<const std::string&>(StrEq("GPU")),
                             ::testing::Matcher<const std::map<std::string, std::string>&>(_)))
-            .WillByDefault(Throw(InferenceEngine::GeneralError{""}));
+            .WillByDefault(Return(gpuMockExeNetwork));
+
         std::shared_ptr<ngraph::Function> simpleNetwork = ngraph::builder::subgraph::makeSingleConv();
         ASSERT_NO_THROW(simpleCnnNetwork = InferenceEngine::CNNNetwork(simpleNetwork));
     }
 };
 
-TEST_P(AutoCTPUTCallMulti, CTPUTDevicesLogicTest) {
+TEST_P(AutoCTPUTCallMulti, CTPUTDeviceLoadFailedNoExceptionThrowTest) {
     std::vector<std::string> targetDevices;
     std::string targetDev;
     bool AutoCallMulti;
     Config config;
     std::tie(AutoCallMulti, targetDevices) = this->GetParam();
+    std::string loadFailedDevice = targetDevices.size() > 0 ? targetDevices[0] : "";
+    std::string secondDevice = targetDevices.size() > 1 ? targetDevices[1] : "";
     plugin->SetName("MULTI");
     for (auto& deviceName : targetDevices) {
         targetDev += deviceName;
         targetDev += ((deviceName == targetDevices.back()) ? "" : ",");
     }
-    if (AutoCallMulti) {
+    std::shared_ptr<InferenceEngine::IExecutableNetworkInternal> exeNetwork;
-        std::shared_ptr<InferenceEngine::IExecutableNetworkInternal> exeNetwork;
+    config.insert({{CONFIG_KEY(PERFORMANCE_HINT), InferenceEngine::PluginConfigParams::CUMULATIVE_THROUGHPUT}});
-        config.insert({{CONFIG_KEY(PERFORMANCE_HINT), InferenceEngine::PluginConfigParams::CUMULATIVE_THROUGHPUT}});
+    config.insert({InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES, targetDev});
-        config.insert({InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES, targetDev});
+    ON_CALL(*core,
+            LoadNetwork(::testing::Matcher<const InferenceEngine::CNNNetwork&>(_),
+                        ::testing::Matcher<const std::string&>(StrEq(loadFailedDevice)),
+                        ::testing::Matcher<const std::map<std::string, std::string>&>(_)))
+        .WillByDefault(Throw(InferenceEngine::GeneralError{""}));
+    if (loadFailedDevice != CommonTestUtils::DEVICE_CPU) {
         EXPECT_CALL(*core,
                     LoadNetwork(::testing::Matcher<const InferenceEngine::CNNNetwork&>(_),
                                 ::testing::Matcher<const std::string&>(CommonTestUtils::DEVICE_CPU),
                                 ::testing::Matcher<const std::map<std::string, std::string>&>(_)))
             .Times(1);
-        EXPECT_CALL(*core,
-                    LoadNetwork(::testing::Matcher<const InferenceEngine::CNNNetwork&>(_),
-                                ::testing::Matcher<const std::string&>(CommonTestUtils::DEVICE_GPU),
-                                ::testing::Matcher<const std::map<std::string, std::string>&>(_)))
-            .Times(1);
-        ASSERT_NO_THROW(exeNetwork = plugin->LoadExeNetworkImpl(simpleCnnNetwork, config));
-        EXPECT_EQ(exeNetwork->GetMetric(ov::execution_devices.name()).as<std::string>(), CommonTestUtils::DEVICE_CPU);
-    } else {
-        config.insert({InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES, targetDev});
-        EXPECT_CALL(*core,
-                    LoadNetwork(::testing::Matcher<const InferenceEngine::CNNNetwork&>(_),
-                                ::testing::Matcher<const std::string&>(CommonTestUtils::DEVICE_CPU),
-                                ::testing::Matcher<const std::map<std::string, std::string>&>(_)))
-            .Times(0);
-        EXPECT_CALL(*core,
-                    LoadNetwork(::testing::Matcher<const InferenceEngine::CNNNetwork&>(_),
-                                ::testing::Matcher<const std::string&>(CommonTestUtils::DEVICE_GPU),
-                                ::testing::Matcher<const std::map<std::string, std::string>&>(_)))
-            .Times(1);
-        EXPECT_THROW(plugin->LoadExeNetworkImpl(simpleCnnNetwork, config), IE::Exception);
     }
+    if (loadFailedDevice != CommonTestUtils::DEVICE_GPU) {
+        EXPECT_CALL(*core,
+                    LoadNetwork(::testing::Matcher<const InferenceEngine::CNNNetwork&>(_),
+                                ::testing::Matcher<const std::string&>(CommonTestUtils::DEVICE_GPU),
+                                ::testing::Matcher<const std::map<std::string, std::string>&>(_)))
+            .Times(1);
+    }
+    ASSERT_NO_THROW(exeNetwork = plugin->LoadExeNetworkImpl(simpleCnnNetwork, config));
+    EXPECT_EQ(exeNetwork->GetMetric(ov::execution_devices.name()).as<std::string>(), secondDevice);
 }
 
 const std::vector<ConfigParams> testConfigs = {
-    ConfigParams{false, {"CPU", "GPU"}},
-    ConfigParams{false, {"GPU", "CPU"}},
     ConfigParams{true, {"CPU", "GPU"}},
     ConfigParams{true, {"GPU", "CPU"}},
 };

src/tests/unit/auto/auto_default_perf_hint_test.cpp

@@ -180,13 +180,13 @@ public:
         testConfigs.push_back(ConfigParams{
             "MULTI:CPU,GPU",
             {"CPU", "GPU"},
-            {{"DEVICE_PROPERTIES", "{GPU:{PERFORMANCE_HINT:LATENCY}}"},
+            {{"DEVICE_PROPERTIES", "{GPU:{PERFORMANCE_HINT:THROUGHPUT}}"},
-             {"MULTI_DEVICE_PRIORITIES", "CPU,GPU"}}});  // CPU: get default_hint:tput  GPU: get perf_hint:latency
+             {"MULTI_DEVICE_PRIORITIES", "CPU,GPU"}}});  // CPU: get default_hint:tput  GPU: get perf_hint:tput
         testConfigs.push_back(ConfigParams{
             "MULTI:CPU,GPU",
             {"CPU", "GPU"},
-            {{"DEVICE_PROPERTIES", "{CPU:{PERFORMANCE_HINT:LATENCY},GPU:{PERFORMANCE_HINT:LATENCY}}"},
+            {{"DEVICE_PROPERTIES", "{CPU:{PERFORMANCE_HINT:THROUGHPUT},GPU:{PERFORMANCE_HINT:THROUGHPUT}}"},
-             {"MULTI_DEVICE_PRIORITIES", "CPU,GPU"}}});  // CPU: get perf_hint:lantency  GPU: get perf_hint:lantency
+             {"MULTI_DEVICE_PRIORITIES", "CPU,GPU"}}});  // CPU: get perf_hint:tput  GPU: get perf_hint:tput
         return testConfigs;
     }
 
@@ -221,17 +221,17 @@ public:
             "MULTI:CPU,GPU",
             {"CPU", "GPU"},
             {{"CPU", "{ALLOW_AUTO_BATCHING:FALSE}"},
-             {"MULTI_DEVICE_PRIORITIES", "CPU,GPU"}}});  // CPU: no perf_hint  GPU: get default_hint:tput
+             {"MULTI_DEVICE_PRIORITIES", "CPU,GPU"}}});  // CPU: get default_hint:tput  GPU: get default_hint:tput
         testConfigs.push_back(ConfigParams{
             "MULTI:CPU,GPU",
             {"CPU", "GPU"},
             {{"DEVICE_PROPERTIES", "GPU:{ALLOW_AUTO_BATCHING:FALSE}}"},
-             {"MULTI_DEVICE_PRIORITIES", "CPU,GPU"}}});  // CPU: get default_hint:tput  GPU: no perf_hint
+             {"MULTI_DEVICE_PRIORITIES", "CPU,GPU"}}});  // CPU: get default_hint:tput  GPU: get default_hint:tput
         testConfigs.push_back(ConfigParams{
             "MULTI:CPU,GPU",
             {"CPU", "GPU"},
             {{"DEVICE_PROPERTIES", "CPU:{ALLOW_AUTO_BATCHING:TRUE},GPU:{ALLOW_AUTO_BATCHING:FALSE}}"},
-             {"MULTI_DEVICE_PRIORITIES", "CPU,GPU"}}});  // CPU: no perf_hint GPU: no perf_hint
+             {"MULTI_DEVICE_PRIORITIES", "CPU,GPU"}}});  // CPU: no perf_hint GPU: get default_hint:tput
         return testConfigs;
     }
 
@@ -499,11 +499,6 @@ TEST_P(SecPropAndDefaultPerfHintMockTest, SecPropAndDefaultPerfHintTest) {
             // HW default perf_hint
             HW_PerfHint = bIsAuto ? "LATENCY" : "THROUGHPUT";
         }
-        auto item = config.find(deviceName);
-        if (item != config.end() && !isCPUHelper) {
-            // do not pass default perf_hint to HW
-            HW_PerfHint = "No PERFORMANCE_HINT";
-        }
 
         EXPECT_CALL(
             *core,