From e57ae2e2fac8068043a022307bedc928ba0c877a Mon Sep 17 00:00:00 2001
From: Maxim Shevtsov <maxim.y.shevtsov@intel.com>
Date: Wed, 23 Dec 2020 18:35:27 +0300
Subject: [PATCH] async network loading in the MULTI (#3599)

* async network loading in the MULTI. makes the overall load time as MAX of the individual devices loading timings, as opposite to the current SUM

* correct way of getting perf counters flag for the MULTI (adding to the async load PR, as this is minor change)

* accomodating remark from the code review- MULTI enables the perf counters only if all devices support/enable that
---
 .../src/multi_device/multi_device_plugin.cpp  | 38 +++++++++++++++----
 1 file changed, 30 insertions(+), 8 deletions(-)

diff --git a/inference-engine/src/multi_device/multi_device_plugin.cpp b/inference-engine/src/multi_device/multi_device_plugin.cpp
index 85172e37723..59feb12ecee 100644
--- a/inference-engine/src/multi_device/multi_device_plugin.cpp
+++ b/inference-engine/src/multi_device/multi_device_plugin.cpp
@@ -13,6 +13,7 @@
 
 #include <ie_metric_helpers.hpp>
 #include <multi-device/multi_device_config.hpp>
+#include <threading/ie_executor_manager.hpp>
 #include "multi_device_plugin.hpp"
 
 // ------------------------------MultiDeviceInferencePlugin----------------------------
@@ -170,19 +171,40 @@ ExecutableNetworkInternal::Ptr MultiDeviceInferencePlugin::LoadExeNetworkImpl(co
     multiNetworkConfig.insert(*priorities);
 
     DeviceMap<ExecutableNetwork> executableNetworkPerDevice;
+    std::mutex load_mutex;
+    std::vector<Task> loads;
     for (auto& p : metaDevices) {
-        auto & deviceName = p.deviceName;
-        auto & deviceConfig = p.config;
-        executableNetworkPerDevice.insert({ deviceName, GetCore()->LoadNetwork(network, deviceName, deviceConfig) });
-        multiNetworkConfig.insert(deviceConfig.begin(), deviceConfig.end());
+        loads.push_back([&]() {
+            const auto &deviceName = p.deviceName;
+            const auto &deviceConfig = p.config;
+            auto exec_net = GetCore()->LoadNetwork(network, deviceName, deviceConfig);
+            std::unique_lock<std::mutex> lock{load_mutex};
+            executableNetworkPerDevice.insert({deviceName, exec_net});
+            multiNetworkConfig.insert(deviceConfig.begin(), deviceConfig.end());
+        });
     }
+    auto executor = InferenceEngine::ExecutorManager::getInstance()->getIdleCPUStreamsExecutor(
+            IStreamsExecutor::Config{"MultiDeviceAsyncLoad",
+                                     static_cast<int>(std::thread::hardware_concurrency()) /* max possible #streams*/,
+                                     1 /*single thread per stream*/,
+                                     IStreamsExecutor::ThreadBindingType::NONE});
+    executor->runAndWait(loads);
     if (executableNetworkPerDevice.empty())
-        THROW_IE_EXCEPTION << NOT_FOUND_str << "Failed to load Executable network to any device "
+        THROW_IE_EXCEPTION << NOT_FOUND_str << "Failed to load network to any device "
                                             <<  "that the MULTI device is initialized to work with";
 
-    auto perfConfig = fullConfig.find(PluginConfigParams::KEY_PERF_COUNT);
-    bool enablePerfCounters = (fullConfig.end() != perfConfig) && (perfConfig->second == PluginConfigParams::YES);
-
+    // checking the perf counters config from the loaded network to respect both device's plugin and load-specific setting
+    int num_plugins_supporting_perf_counters = 0;
+    for (auto n : executableNetworkPerDevice) {
+            try {
+                num_plugins_supporting_perf_counters +=
+                        n.second.GetConfig(PluginConfigParams::KEY_PERF_COUNT).as<std::string>() ==
+                        PluginConfigParams::YES;
+            } catch (...) {
+            }
+    }
+    // MULTI can enable the perf counters only if all  devices support/enable that
+    bool enablePerfCounters = num_plugins_supporting_perf_counters == executableNetworkPerDevice.size();
     return std::make_shared<MultiDeviceExecutableNetwork>(executableNetworkPerDevice,
                                                           metaDevices,
                                                           multiNetworkConfig,