IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

async network loading in the MULTI (#3599)

Browse Source 
* async network loading in the MULTI. makes the overall load time as MAX of the individual devices loading timings, as opposite to the current SUM

* correct way of getting perf counters flag for the MULTI (adding to the async load PR, as this is minor change)

* accomodating remark from the code review- MULTI enables the perf counters only if all devices support/enable that
This commit is contained in:
Maxim Shevtsov 2020-12-23 18:35:27 +03:00 committed by GitHub
parent 036f574756
commit e57ae2e2fa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 30 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
inference-engine/src/multi_device/multi_device_plugin.cpp
View File

@ -13,6 +13,7 @@
#include <ie_metric_helpers.hpp>
#include <multi-device/multi_device_config.hpp>
#include <threading/ie_executor_manager.hpp>
#include "multi_device_plugin.hpp"
// ------------------------------MultiDeviceInferencePlugin----------------------------
@ -170,19 +171,40 @@ ExecutableNetworkInternal::Ptr MultiDeviceInferencePlugin::LoadExeNetworkImpl(co
multiNetworkConfig.insert(*priorities);
DeviceMap<ExecutableNetwork> executableNetworkPerDevice;
std::mutex load_mutex;
std::vector<Task> loads;
for (auto& p : metaDevices) {
auto & deviceName = p.deviceName;
auto & deviceConfig = p.config;
executableNetworkPerDevice.insert({ deviceName, GetCore()->LoadNetwork(network, deviceName, deviceConfig) });
multiNetworkConfig.insert(deviceConfig.begin(), deviceConfig.end());
loads.push_back([&]() {
const auto &deviceName = p.deviceName;
const auto &deviceConfig = p.config;
auto exec_net = GetCore()->LoadNetwork(network, deviceName, deviceConfig);
std::unique_lock<std::mutex> lock{load_mutex};
executableNetworkPerDevice.insert({deviceName, exec_net});
multiNetworkConfig.insert(deviceConfig.begin(), deviceConfig.end());
});
}
auto executor = InferenceEngine::ExecutorManager::getInstance()->getIdleCPUStreamsExecutor(
IStreamsExecutor::Config{"MultiDeviceAsyncLoad",
static_cast<int>(std::thread::hardware_concurrency()) /* max possible #streams*/,
1 /*single thread per stream*/,
IStreamsExecutor::ThreadBindingType::NONE});
executor->runAndWait(loads);
if (executableNetworkPerDevice.empty())
THROW_IE_EXCEPTION << NOT_FOUND_str << "Failed to load Executable network to any device "
THROW_IE_EXCEPTION << NOT_FOUND_str << "Failed to load network to any device "
<< "that the MULTI device is initialized to work with";
auto perfConfig = fullConfig.find(PluginConfigParams::KEY_PERF_COUNT);
bool enablePerfCounters = (fullConfig.end() != perfConfig) && (perfConfig->second == PluginConfigParams::YES);
// checking the perf counters config from the loaded network to respect both device's plugin and load-specific setting
int num_plugins_supporting_perf_counters = 0;
for (auto n : executableNetworkPerDevice) {
try {
num_plugins_supporting_perf_counters +=
n.second.GetConfig(PluginConfigParams::KEY_PERF_COUNT).as<std::string>() ==
PluginConfigParams::YES;
} catch (...) {
}
}
// MULTI can enable the perf counters only if all devices support/enable that
bool enablePerfCounters = num_plugins_supporting_perf_counters == executableNetworkPerDevice.size();
return std::make_shared<MultiDeviceExecutableNetwork>(executableNetworkPerDevice,
metaDevices,
multiNetworkConfig,