IntenseWebs/openvino
3
0 Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

multi code refine (#11663)

Browse Source 
* draft

Signed-off-by: Hu, Yuan2 <yuan2.hu@intel.com>

* refactor for multi

Signed-off-by: Hu, Yuan2 <yuan2.hu@intel.com>

* refactor auto draft

Signed-off-by: Hu, Yuan2 <yuan2.hu@intel.com>

* try to fix executable get config test failed issue

Signed-off-by: Hu, Yuan2 <yuan2.hu@intel.com>

* set ExecNetwork only one time

Signed-off-by: Hu, Yuan2 <yuan2.hu@intel.com>

* format code and using alias

Signed-off-by: Hu, Yuan2 <yuan2.hu@intel.com>

* clear head file

Signed-off-by: Hu, Yuan2 <yuan2.hu@intel.com>

* change name from Context to ScheduleContext

Signed-off-by: Hu, Yuan2 <yuan2.hu@intel.com>

* polishing

Signed-off-by: fishbell <bell.song@intel.com>

* polish & new implementation

Signed-off-by: fishbell <bell.song@intel.com>

* enable/test a new schedule

Signed-off-by: fishbell <bell.song@intel.com>

* port fps logs over

Signed-off-by: fishbell <bell.song@intel.com>

* restructure

Signed-off-by: fishbell <bell.song@intel.com>

* fix windows build failure

Signed-off-by: fishbell <bell.song@intel.com>

* clean up code

Signed-off-by: fishbell <bell.song@intel.com>

Co-authored-by: Hu, Yuan2 <yuan2.hu@intel.com>
This commit is contained in:
yanlan songandHu, Yuan2 authored and GitHub committed 2022-05-25 22:53:47 +08:00
1 parent eb36b891eb
commit 0c7840ef28
20 files changed
+1873 -1230
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/plugins/auto/async_infer_request.cpp
+15 -85
View File
@@ -3,103 +3,33 @@
//
///////////////////////////////////////////////////////////////////////////////////////////////////
#include <string>
#include <vector>
#include <memory>
#include <map>
#include "async_infer_request.hpp"
#include <ie_icore.hpp>
#include <ie_metric_helpers.hpp>
#include <ie_plugin_config.hpp>
#include "utils/log_util.hpp"
namespace MultiDevicePlugin {
using namespace InferenceEngine;
MultiDeviceAsyncInferRequest::MultiDeviceAsyncInferRequest(
const MultiDeviceInferRequest::Ptr& inferRequest,
const bool needPerfCounters,
const MultiDeviceExecutableNetwork::Ptr& multiDeviceExecutableNetwork,
const ITaskExecutor::Ptr& callbackExecutor) :
AsyncInferRequest::AsyncInferRequest(const Schedule::Ptr& schedule,
const IInferPtr& inferRequest,
const IE::ITaskExecutor::Ptr& callbackExecutor):
AsyncInferRequestThreadSafeDefault(inferRequest, nullptr, callbackExecutor),
_multiDeviceExecutableNetwork{multiDeviceExecutableNetwork},
_inferRequest{inferRequest},
_needPerfCounters{needPerfCounters} {
// this executor starts the inference while the task (checking the result) is passed to the next stage
struct ThisRequestExecutor : public ITaskExecutor {
explicit ThisRequestExecutor(MultiDeviceAsyncInferRequest* _this_) : _this{_this_} {}
void run(Task task) override {
auto workerInferRequest = _this->_workerInferRequest;
workerInferRequest->_task = std::move(task);
workerInferRequest->_inferRequest->StartAsync();
};
MultiDeviceAsyncInferRequest* _this = nullptr;
};
_pipeline = {
// if the request is coming with device-specific remote blobs make sure it is scheduled to the specific device only:
{ /*TaskExecutor*/ std::make_shared<ImmediateExecutor>(), /*task*/ [this] {
// by default, no preferred device:
_multiDeviceExecutableNetwork->_thisPreferredDeviceName = "";
// if any input is remote (e.g. was set with SetBlob), let' use the corresponding device
for (const auto &it : _multiDeviceExecutableNetwork->GetInputsInfo()) {
auto b = _inferRequest->GetBlob(it.first);
auto r = b->as<RemoteBlob>();
if (r) {
const auto name = r->getDeviceName();
const auto res = std::find_if(
_multiDeviceExecutableNetwork->_devicePrioritiesInitial.cbegin(),
_multiDeviceExecutableNetwork->_devicePrioritiesInitial.cend(),
[&name](const MultiDevicePlugin::DeviceInformation& d) {
return (d.defaultDeviceID.empty() ? d.deviceName : (d.deviceName + "." + d.defaultDeviceID)) == name;
});
if (_multiDeviceExecutableNetwork->_devicePrioritiesInitial.cend() == res) {
IE_THROW() << "None of the devices (for which current MULTI-device configuration was "
"initialized) supports a remote blob created on the device named " << name;
} else {
// it is ok to take the c_str() here (as pointed in the executable_network.hpp we need to use const char*)
// as the original strings are from the "persistent" vector (with the right lifetime)
_multiDeviceExecutableNetwork->_thisPreferredDeviceName = res->deviceName.c_str();
break;
}
}
}
}},
// as the scheduling algo may select any device, this stage accepts the scheduling decision (actual workerRequest)
// then sets the device-agnostic blobs to the actual (device-specific) request
{
/*TaskExecutor*/ _multiDeviceExecutableNetwork, /*task*/ [this] {
_workerInferRequest = MultiDeviceExecutableNetwork::_thisWorkerInferRequest;
_inferRequest->SetBlobsToAnotherRequest(_workerInferRequest->_inferRequest);
INFO_RUN([this]() {
_workerInferRequest->_startTimes.push_back(std::move(std::chrono::steady_clock::now()));
});
}},
// final task in the pipeline:
{ /*TaskExecutor*/std::make_shared<ThisRequestExecutor>(this), /*task*/ [this] {
if (nullptr != _workerInferRequest->_exceptionPtr) {
std::rethrow_exception(_workerInferRequest->_exceptionPtr);
}
if (_needPerfCounters)
_perfMap = _workerInferRequest->_inferRequest->GetPerformanceCounts();
INFO_RUN([this]() {
_workerInferRequest->_endTimes.push_back(std::move(std::chrono::steady_clock::now()));
});
}}
};
_schedule(schedule),
_inferRequest(inferRequest) {
auto pipeline = _schedule->GetPipeline(_inferRequest, &_workerInferRequest);
if (pipeline.size() > 0) {
_pipeline = std::move(pipeline);
}
}
void MultiDeviceAsyncInferRequest::Infer_ThreadUnsafe() {
void AsyncInferRequest::Infer_ThreadUnsafe() {
InferUsingAsync();
}
std::map<std::string, InferenceEngineProfileInfo> MultiDeviceAsyncInferRequest::GetPerformanceCounts() const {
std::map<std::string, IE::InferenceEngineProfileInfo>
AsyncInferRequest::GetPerformanceCounts() const {
CheckState();
return std::move(_perfMap);
auto multiDeviceInfer = std::dynamic_pointer_cast<MultiDeviceInferRequest>(_inferRequest);
return multiDeviceInfer->GetPerformanceCounts();
}
MultiDeviceAsyncInferRequest::~MultiDeviceAsyncInferRequest() {
AsyncInferRequest::~AsyncInferRequest() {
StopAndWait();
}
src/plugins/auto/async_infer_request.hpp
+10 -24
View File
@@ -5,16 +5,8 @@
///////////////////////////////////////////////////////////////////////////////////////////////////
#pragma once
#include <map>
#include <vector>
#include <utility>
#include <memory>
#include <string>
#include <cpp_interfaces/impl/ie_infer_async_request_thread_safe_default.hpp>
#include "schedule.hpp"
#include "infer_request.hpp"
#include "executable_network.hpp"
#include "utils/log_util.hpp"
#ifdef MULTIUNITTEST
#define MOCKTESTMACRO virtual
@@ -24,25 +16,19 @@
#endif
namespace MultiDevicePlugin {
class MultiDeviceAsyncInferRequest : public InferenceEngine::AsyncInferRequestThreadSafeDefault {
class AsyncInferRequest : public IE::AsyncInferRequestThreadSafeDefault {
public:
using Ptr = std::shared_ptr<MultiDeviceAsyncInferRequest>;
explicit MultiDeviceAsyncInferRequest(const MultiDeviceInferRequest::Ptr& inferRequest,
const bool needPerfCounters,
const MultiDeviceExecutableNetwork::Ptr& multiDeviceExecutableNetwork,
const InferenceEngine::ITaskExecutor::Ptr& callbackExecutor);
using Ptr = std::shared_ptr<AsyncInferRequest>;
explicit AsyncInferRequest(const Schedule::Ptr& schedule, const IInferPtr& inferRequest,
const IE::ITaskExecutor::Ptr& callbackExecutor);
void Infer_ThreadUnsafe() override;
std::map<std::string, InferenceEngine::InferenceEngineProfileInfo> GetPerformanceCounts() const override;
~MultiDeviceAsyncInferRequest();
std::map<std::string, IE::InferenceEngineProfileInfo> GetPerformanceCounts() const override;
~AsyncInferRequest();
protected:
MultiDeviceExecutableNetwork::Ptr _multiDeviceExecutableNetwork;
MultiDeviceInferRequest::Ptr _inferRequest;
std::map<std::string, InferenceEngine::InferenceEngineProfileInfo> _perfMap;
bool _needPerfCounters = false;
MultiDeviceExecutableNetwork::WorkerInferRequest* _workerInferRequest = nullptr;
Schedule::Ptr _schedule;
WorkerInferRequest* _workerInferRequest = nullptr;
IInferPtr _inferRequest;
};
} // namespace MultiDevicePlugin
src/plugins/auto/auto_executable_network.cpp
+153
View File
@@ -0,0 +1,153 @@
// Copyright (C) 2018-2022 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
///////////////////////////////////////////////////////////////////////////////////////////////////
#include "auto_executable_network.hpp"
// ------------------------------AutoExecutableNetwork----------------------------
//
namespace MultiDevicePlugin {
AutoExecutableNetwork::AutoExecutableNetwork(AutoScheduleContext::Ptr& context, const AutoSchedule::Ptr& schedule)
:ExecutableNetwork(schedule, context),
_autoSContext(context),
_autoSchedule(schedule) {
}
std::shared_ptr<IE::RemoteContext> AutoExecutableNetwork::GetContext() const {
_autoSchedule->WaitActualNetworkReady();
return _autoSchedule->_loadContext[ACTUALDEVICE].executableNetwork->GetContext();
}
void AutoExecutableNetwork::SetConfig(const std::map<std::string, IE::Parameter>
& config) {
IE_THROW(NotImplemented);
}
IE::Parameter AutoExecutableNetwork::GetConfig(const std::string& name) const {
{
std::lock_guard<std::mutex> lock(_autoSContext->_confMutex);
auto it = _autoSContext->_config.find(name);
if (it != _autoSContext->_config.end()) {
return it->second;
}
}
IE_THROW(NotFound) << name << " not found in the ExecutableNetwork config";
}
IE::Parameter AutoExecutableNetwork::GetMetric(const std::string& name) const {
if (name == ov::supported_properties) {
return decltype(ov::supported_properties)::value_type {
// Metrics
ov::PropertyName{ov::supported_properties.name(), ov::PropertyMutability::RO},
ov::PropertyName{ov::hint::performance_mode.name(), ov::PropertyMutability::RO},
ov::PropertyName{ov::model_name.name(), ov::PropertyMutability::RO},
ov::PropertyName{ov::optimal_number_of_infer_requests.name(), ov::PropertyMutability::RO},
ov::PropertyName{ov::hint::model_priority.name(), ov::PropertyMutability::RO},
ov::PropertyName{ov::device::priorities.name(), ov::PropertyMutability::RO}
};
} else if (name == ov::device::priorities) {
auto value = _autoSContext->_config.find(ov::device::priorities.name());
return decltype(ov::device::priorities)::value_type {value->second.as<std::string>()};
} else if (name == ov::hint::model_priority) {
auto value = _autoSContext->_modelPriority;
if (_autoSContext->_core->isNewAPI()) {
return value ? ((value > 1) ? ov::hint::Priority::LOW :
ov::hint::Priority::MEDIUM) : ov::hint::Priority::HIGH;
} else {
return value ? ((value > 1) ? CONFIG_VALUE(MODEL_PRIORITY_LOW) : CONFIG_VALUE(
MODEL_PRIORITY_MED)) : CONFIG_VALUE(MODEL_PRIORITY_HIGH);
}
} else if (name == ov::optimal_number_of_infer_requests) {
const unsigned int defaultNumForTPUT = 4u;
const unsigned int defaultNumForLatency = 1u;
unsigned int real = 0;
if (_autoSchedule->_loadContext[ACTUALDEVICE].isAlready) {
real = _autoSchedule->_loadContext[ACTUALDEVICE].
executableNetwork->GetMetric(name).as<unsigned int>();
} else {
IE_ASSERT(_autoSchedule->_loadContext[CPU].isAlready == true);
std::unique_lock<std::mutex> lock(_autoSContext->_confMutex);
auto deviceInfo = _autoSchedule->_loadContext[ACTUALDEVICE].deviceInfo;
lock.unlock();
unsigned int optimalBatchSize = 0;
unsigned int requests = 0;
bool bThroughputEnabledInPlugin = false;
try {
// for benchmark through AUTO:CPU,GPU
// SetConfig directly set to CPU/GPU in this case
bThroughputEnabledInPlugin =
_autoSContext->_core->GetConfig(deviceInfo.deviceName,
CONFIG_KEY(PERFORMANCE_HINT)).as<std::string>() == CONFIG_VALUE(THROUGHPUT);
} catch (...) {
LOG_DEBUG("[AUTOPLUGIN]GetMetric:%s for %s", "PERF_HINT config not supported",
deviceInfo.deviceName.c_str());
}
const auto& mode = deviceInfo.config.find(CONFIG_KEY(PERFORMANCE_HINT));
if (bThroughputEnabledInPlugin ||
(mode != deviceInfo.config.end() && mode->second == CONFIG_VALUE(THROUGHPUT))) {
unsigned int upperBoundStreamsNum = 0;
std::map<std::string, IE::Parameter> options;
options["MODEL_PTR"] = std::const_pointer_cast<ngraph::Function>
(_autoSContext->_network.getFunction());
try {
auto rangeOfStreams = _autoSContext->_core->GetMetric(deviceInfo.deviceName,
METRIC_KEY(RANGE_FOR_STREAMS),
options).as<std::tuple<unsigned int, unsigned int>>();
upperBoundStreamsNum = std::get<1>(rangeOfStreams);
} catch (const IE::Exception& iie) {
LOG_DEBUG("[AUTOPLUGIN] GetMetric RANGE_FOR_STREAMS failed");
}
if (!_autoSContext->_batchingDisabled) {
try {
optimalBatchSize = _autoSContext->_core->GetMetric(deviceInfo.deviceName,
METRIC_KEY(OPTIMAL_BATCH_SIZE), options).as<unsigned int>();
LOG_DEBUG("[AUTOPLUGIN]BATCHING:%s:%ld", "optimal batch size",
optimalBatchSize);
} catch (...) {
LOG_DEBUG("[AUTOPLUGIN]BATCHING:%s", "metric OPTIMAL_BATCH_SIZE not supported");
}
}
if (optimalBatchSize > 1) {
// batching is supported with the device
// go with auto-batching
try {
// check if app have set preferred value
auto res =
_autoSContext->_core->GetConfig(deviceInfo.deviceName,
CONFIG_KEY(PERFORMANCE_HINT_NUM_REQUESTS)).as<std::string>();
requests = IE::PerfHintsConfig::CheckPerformanceHintRequestValue(res);
const auto& reqs = deviceInfo.config.find(CONFIG_KEY(
PERFORMANCE_HINT_NUM_REQUESTS));
if (reqs != deviceInfo.config.end()) {
requests = static_cast<unsigned int>
(IE::PerfHintsConfig::CheckPerformanceHintRequestValue(reqs->second));
}
LOG_DEBUG("[AUTOPLUGIN]BATCHING:%s:%ld", "user requested size", requests);
if (!requests) { // no limitations from user
requests = optimalBatchSize * upperBoundStreamsNum * 2;
LOG_DEBUG("[AUTOPLUGIN]BATCHING:%s:%ld", "deduced size:", requests);
}
} catch (const IE::Exception& iie) {
LOG_WARNING("[AUTOPLUGIN]deduce optimal infer requset num for auto-batch failed :%s",
iie.what());
}
real = (std::max)(requests, optimalBatchSize);
} else if (deviceInfo.deviceName.find("VPUX") != std::string::npos) {
real = 8u;
} else {
real = upperBoundStreamsNum ? 2 * upperBoundStreamsNum : defaultNumForTPUT;
}
} else {
real = defaultNumForLatency;
}
}
return decltype(ov::optimal_number_of_infer_requests)::value_type {real};
}
if (_autoSchedule->_loadContext[ACTUALDEVICE].isAlready) {
return _autoSchedule->_loadContext[ACTUALDEVICE].executableNetwork->GetMetric(
name);
}
return _autoSchedule->_loadContext[CPU].executableNetwork->GetMetric(name);
}
} // namespace MultiDevicePlugin
src/plugins/auto/auto_executable_network.hpp
+39
View File
@@ -0,0 +1,39 @@
// Copyright (C) 2018-2022 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
///////////////////////////////////////////////////////////////////////////////////////////////////
#pragma once
#include <string>
#include <map>
#include "auto_schedule.hpp"
#include "executable_network.hpp"
#ifdef MULTIUNITTEST
#define MOCKTESTMACRO virtual
#define MultiDevicePlugin MockMultiDevicePlugin
#else
#define MOCKTESTMACRO
#endif
namespace MultiDevicePlugin {
class AutoExecutableNetwork : public ExecutableNetwork {
friend IInferPtr AutoSchedule::CreateInferRequest();
public:
using Ptr = std::shared_ptr<AutoExecutableNetwork>;
explicit AutoExecutableNetwork(AutoScheduleContext::Ptr& context, const AutoSchedule::Ptr& schedule);
void SetConfig(const std::map<std::string, IE::Parameter>& config) override;
IE::Parameter GetConfig(const std::string& name) const override;
IE::Parameter GetMetric(const std::string& name) const override;
std::shared_ptr<IE::RemoteContext> GetContext() const override;
virtual ~AutoExecutableNetwork() = default;
private:
AutoScheduleContext::Ptr _autoSContext;
AutoSchedule::Ptr _autoSchedule;
};
} // namespace MultiDevicePlugin
src/plugins/auto/auto_schedule.cpp
+572
View File
@@ -0,0 +1,572 @@
// Copyright (C) 2018-2022 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
///////////////////////////////////////////////////////////////////////////////////////////////////
#include "auto_schedule.hpp"
#include "async_infer_request.hpp"
#include "auto_executable_network.hpp"
#include "plugin.hpp"
// ------------------------------AutoSchedule----------------------------
namespace MultiDevicePlugin {
namespace {
std::string GetNetworkPrecision(const IE::CNNNetwork& network) {
auto nGraphFunc = network.getFunction();
bool isINTModel = ngraph::op::util::has_op_with_type<ngraph::op::FakeQuantize>
(nGraphFunc);
if (isINTModel) {
return METRIC_VALUE(INT8);
}
for (auto& node : nGraphFunc->get_ordered_ops()) {
if (std::dynamic_pointer_cast<ngraph::opset1::Convolution>(node) ||
std::dynamic_pointer_cast<ngraph::opset1::GroupConvolution>(node) ||
std::dynamic_pointer_cast<ngraph::opset1::GroupConvolutionBackpropData>(node) ||
std::dynamic_pointer_cast<ngraph::opset1::ConvolutionBackpropData>(node)) {
auto layerType = node->input(1).get_element_type().get_type_name();
if (layerType == "f32") {
return METRIC_VALUE(FP32);
}
if (layerType == "f16") {
return METRIC_VALUE(FP16);
}
}
}
return METRIC_VALUE(FP32);
}
} // namespace
void AutoSchedule::GenerateWorkers(const std::string& device,
const SoExecNetwork& executableNetwork) {
std::string realDeviceName;
if (device == "CPU_HELP") {
realDeviceName = "CPU";
} else {
realDeviceName = device;
}
auto itNumRequests = std::find_if(_autoSContext->_devicePriorities.cbegin(), _autoSContext->_devicePriorities.cend(),
[&realDeviceName](const DeviceInformation & d) {
return d.deviceName == realDeviceName;
});
unsigned int optimalNum = 0;
try {
optimalNum = executableNetwork->GetMetric(METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS)).as<unsigned int>();
} catch (const IE::Exception& iie) {
IE_THROW()
<< "Every device used with the Multi-Device should "
<< "support OPTIMAL_NUMBER_OF_INFER_REQUESTS ExecutableNetwork metric. "
<< "Failed to query the metric for the " << device << " with error:" <<
iie.what();
}
const auto numRequests = (_autoSContext->_devicePriorities.end() == itNumRequests ||
itNumRequests->numRequestsPerDevices == -1) ? optimalNum : itNumRequests->numRequestsPerDevices;
auto& workerRequests = _workerRequests[device];
auto& idleWorkerRequests = _idleWorkerRequests[device];
workerRequests.resize(numRequests);
_inferPipelineTasksDeviceSpecific[device] = std::unique_ptr<IE::ThreadSafeQueue<IE::Task>>(new IE::ThreadSafeQueue<IE::Task>);
auto* idleWorkerRequestsPtr = &(idleWorkerRequests);
idleWorkerRequests.set_capacity(numRequests);
int num = 0;
for (auto&& workerRequest : workerRequests) {
workerRequest._inferRequest = {executableNetwork->CreateInferRequest(), executableNetwork._so};
auto* workerRequestPtr = &workerRequest;
workerRequestPtr->_index = num++;
IE_ASSERT(idleWorkerRequests.try_push(std::make_pair(workerRequestPtr->_index, workerRequestPtr)) == true);
workerRequest._inferRequest->SetCallback(
[workerRequestPtr, this, device, idleWorkerRequestsPtr](std::exception_ptr exceptionPtr) mutable {
IdleGuard<NotBusyPriorityWorkerRequests> idleGuard{workerRequestPtr, *idleWorkerRequestsPtr};
workerRequestPtr->_exceptionPtr = exceptionPtr;
{
auto capturedTask = std::move(workerRequestPtr->_task);
capturedTask();
}
// try to return the request to the idle list (fails if the overall object destruction has began)
if (idleGuard.Release()->try_push(std::make_pair(workerRequestPtr->_index, workerRequestPtr))) {
// let's try to pop a task, as we know there is at least one idle request, schedule if succeeded
// if no device-agnostic tasks, let's try pop the device specific task, schedule if succeeded
IE::Task t;
do {
_inferPipelineTasks.try_pop(t);
} while (t && ScheduleToWorkerInferRequest(std::move(t)));
do {
_inferPipelineTasksDeviceSpecific[device]->try_pop(t);
} while (t && ScheduleToWorkerInferRequest(std::move(t), device));
}
});
}
}
void AutoSchedule::init(const ScheduleContext::Ptr& sContext) {
LOG_INFO("[AUTOPLUGIN]ExecutableNetwork start");
// initialize cpuHelpReleasetime
_cpuHelpReleaseTime = std::chrono::steady_clock::now();
_multiSContext = std::dynamic_pointer_cast<MultiScheduleContext>(sContext);
_autoSContext = std::dynamic_pointer_cast<AutoScheduleContext>(sContext);
if (_autoSContext->_core == nullptr) {
IE_THROW() << "Please, work with Auto device via InferencEngine::Core object";
}
if (_autoSContext->_modelPath.empty() && _autoSContext->_network.getFunction() == nullptr) {
IE_THROW() << "AUTO device supports just ngraph network representation";
}
_autoSContext->_config[IE::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES] = _autoSContext->_strDevices;
std::string profilingTask = "AutoSchedule::AutoSchedule:AutoMode";
bool isCumulative = (_autoSContext->_performanceHint == IE::PluginConfigParams::CUMULATIVE_THROUGHPUT) ? true : false;
// loadContext[ACTUALDEVICE] is always enabled,
// when there is CPU and there are more than two devices, loadContext[CPU] is enabled
_loadContext[ACTUALDEVICE].isEnabled = true;
if (_autoSContext->_modelPath.empty())
_loadContext[ACTUALDEVICE].networkPrecision = GetNetworkPrecision(_autoSContext->_network);
_loadContext[ACTUALDEVICE].metaDevices = _autoSContext->_devicePriorities;
if (isCumulative) {
std::list<DeviceInformation> validDevices =
_autoSContext->_plugin->GetValidDevice(_autoSContext->_devicePriorities, _loadContext[ACTUALDEVICE].networkPrecision);
std::string deviceName = "MULTI:";
for (auto& device : validDevices) {
deviceName += device.deviceName;
deviceName += ((device.deviceName == validDevices.back().deviceName) ? "" : ",");
}
_loadContext[ACTUALDEVICE].deviceInfo.deviceName = deviceName;
_loadContext[ACTUALDEVICE].deviceInfo.config[CONFIG_KEY(PERFORMANCE_HINT)] = InferenceEngine::PluginConfigParams::THROUGHPUT;
} else {
_loadContext[ACTUALDEVICE].deviceInfo = _autoSContext->_plugin->SelectDevice(_autoSContext->_devicePriorities,
_loadContext[ACTUALDEVICE].networkPrecision,
_autoSContext->_modelPriority);
}
LOG_INFO("[AUTOPLUGIN]:select device:%s", _loadContext[ACTUALDEVICE].deviceInfo.deviceName.c_str());
bool isActualDevCPU =
_loadContext[ACTUALDEVICE].deviceInfo.deviceName.find("CPU") !=std::string::npos;
// if Actual device is CPU, disabled _loadContext[CPU], only use _loadContext[ACTUALDEVICE]
if (isActualDevCPU || isCumulative) {
_loadContext[CPU].isEnabled = false;
} else {
const auto CPUIter = std::find_if(_autoSContext->_devicePriorities.begin(), _autoSContext->_devicePriorities.end(),
[=](const DeviceInformation& d) -> bool { return d.deviceName.find("CPU") != std::string::npos; });
// if have CPU Device, enable _loadContext[CPU]
if (CPUIter != _autoSContext->_devicePriorities.end()) {
_loadContext[CPU].isEnabled = true;
_loadContext[CPU].deviceInfo = *CPUIter;
_loadContext[CPU].deviceInfo.config[CONFIG_KEY(PERFORMANCE_HINT)] =
IE::PluginConfigParams::LATENCY;
_loadContext[CPU].workName = "CPU_HELP";
LOG_INFO("[AUTOPLUGIN]:will load CPU for accelerator");
} else {
_loadContext[CPU].isEnabled = false;
}
}
// initialize the rest members of load context
for (int i = 0; i < CONTEXTNUM; i++) {
if (_loadContext[i].isEnabled) {
_loadContext[i].future = _loadContext[i].promise.get_future();
auto* contextPtr = &_loadContext[i];
auto modelPath = _autoSContext->_modelPath;
auto network = _autoSContext->_network;
_loadContext[i].task = [this, contextPtr, modelPath, network, isCumulative]() mutable {
TryToLoadNetWork(*contextPtr, modelPath, network);
if (contextPtr->isLoadSuccess) {
if (contextPtr->workName.empty()) {
contextPtr->workName = contextPtr->deviceInfo.deviceName;
}
GenerateWorkers(contextPtr->workName, contextPtr->executableNetwork);
//need lock
{
std::lock_guard<std::mutex> lock(_autoSContext->_confMutex);
_autoSContext->_config.insert(contextPtr->deviceInfo.config.begin(), contextPtr->deviceInfo.config.end());
}
contextPtr->isAlready = true;
auto& deviceName = contextPtr->deviceInfo.deviceName;
LOG_INFO("[AUTOPLUGIN]:device:%s loading Network finished", deviceName.c_str());
if (!isCumulative) {
auto supported_config_keys =
_autoSContext->_core->GetMetric(deviceName, METRIC_KEY(SUPPORTED_CONFIG_KEYS))
.as<std::vector<std::string>>();
DEBUG_RUN([this, &contextPtr, &deviceName, &supported_config_keys] {
std::lock_guard<std::mutex> lock(_autoSContext->_confMutex);
for (const auto& cfg : supported_config_keys) {
try {
LOG_DEBUG(
"[AUTOPLUGIN]:device:%s, GetConfig:%s=%s",
deviceName.c_str(),
cfg.c_str(),
contextPtr->executableNetwork->GetConfig(cfg).as<std::string>().c_str());
} catch (...) {
}
}
});
}
}
contextPtr->promise.set_value();
// the first load network process finished
std::call_once(_firstLoadOC, [this]() {
_firstLoadPromise.set_value();
});
};
}
}
OV_ITT_SCOPED_TASK(itt::domains::MULTIPlugin,
openvino::itt::handle(profilingTask));
if (_loadContext[CPU].isEnabled) {
_firstLoadFuture = _firstLoadPromise.get_future();
// will not wait for loading accelerator network,
// so the executor can't be destroyed before finished the task,
// so use executor as a member of AutoSchedule.
_executor = _autoSContext->_plugin->executorManager()->getIdleCPUStreamsExecutor(
IE::IStreamsExecutor::Config{"AutoDeviceAsyncLoad",
static_cast<int>(std::thread::hardware_concurrency()) /* max possible #streams*/,
0 /*default threads per stream, workaround for ticket 62376*/,
IE::IStreamsExecutor::ThreadBindingType::NONE});
for (auto&& device : _autoSContext->_devicePriorities) {
// initialize containers before run async task
_idleWorkerRequests[device.deviceName];
_workerRequests[device.deviceName];
_inferPipelineTasksDeviceSpecific[device.deviceName] = nullptr;
}
_idleWorkerRequests["CPU_HELP"];
_workerRequests["CPU_HELP"];
_inferPipelineTasksDeviceSpecific["CPU_HELP"] = nullptr;
_executor->run(_loadContext[CPU].task);
_executor->run(_loadContext[ACTUALDEVICE].task);
auto recycleTask = [this]() mutable {
WaitActualNetworkReady();
while (!_exitFlag && _loadContext[ACTUALDEVICE].isAlready) {
// handle the case of ACTUAL faster than CPU
_loadContext[CPU].future.wait();
// clean up helper infer requests
// first, wait for all the remaining requests to finish
for (auto& iter : _workerRequests["CPU_HELP"]) {
iter._inferRequest._ptr->Wait(IE::InferRequest::WaitMode::RESULT_READY);
}
// late enough to check the idle queue now
// second, check the idle queue if all requests are in place
size_t destroynum = 0;
std::pair<int, WorkerInferRequest*> worker;
std::list<Time> cpuHelpAllStartTimes;
std::list<Time> cpuHelpAllEndTimes;
while (_idleWorkerRequests["CPU_HELP"].try_pop(worker)) {
destroynum++;
INFO_RUN([&cpuHelpAllStartTimes, &cpuHelpAllEndTimes, &worker]() {
cpuHelpAllStartTimes.splice(cpuHelpAllStartTimes.end(), worker.second->_startTimes);
cpuHelpAllEndTimes.splice(cpuHelpAllEndTimes.end(), worker.second->_endTimes);
});
}
INFO_RUN([this, &cpuHelpAllStartTimes, &cpuHelpAllEndTimes]() {
cpuHelpAllStartTimes.sort(std::less<Time>());
cpuHelpAllEndTimes.sort(std::less<Time>());
_cpuHelpInferCount = cpuHelpAllStartTimes.size();
IE_ASSERT(_cpuHelpInferCount == cpuHelpAllEndTimes.size());
});
if (destroynum == _workerRequests["CPU_HELP"].size()) {
std::lock_guard<std::mutex> lock(_autoSContext->_confMutex);
INFO_RUN([this, &cpuHelpAllStartTimes, &cpuHelpAllEndTimes, &destroynum]() {
_cpuHelpReleaseTime = std::chrono::steady_clock::now();
if (cpuHelpAllStartTimes.size() >= destroynum + 1) {
//remove last worksize num requests, so the fps will be more accuracy
cpuHelpAllStartTimes.resize(_cpuHelpInferCount - destroynum);
cpuHelpAllEndTimes.resize(_cpuHelpInferCount - destroynum);
std::chrono::duration<double, std::milli> durtation =
cpuHelpAllEndTimes.back() - cpuHelpAllStartTimes.front();
_cpuHelpFps = cpuHelpAllStartTimes.size() * 1000 / durtation.count();
}
});
LOG_INFO("[AUTOPLUGIN] release all work requests of CPU_HELP");
_workerRequests["CPU_HELP"].clear();
_loadContext[CPU].executableNetwork._ptr.reset();
_loadContext[CPU].executableNetwork._so.reset();
LOG_INFO("[AUTOPLUGIN]:helper released!!");
break;
}
}
};
_executor->run(std::move(recycleTask));
} else {
// only one device need to load network, do not need to load it async
_loadContext[ACTUALDEVICE].task();
_passthroughExeNet = _loadContext[ACTUALDEVICE].executableNetwork;
}
WaitFirstNetworkReady();
}
void AutoSchedule::TryToLoadNetWork(AutoLoadContext& context, const std::string& modelPath, const IE::CNNNetwork& network) {
auto& device = context.deviceInfo.deviceName;
auto& deviceConfig = context.deviceInfo.config;
auto& deviceList = context.metaDevices;
bool curDevIsCPU = (device.find("CPU") != std::string::npos);
bool curDevIsGPU = (device.find("GPU") != std::string::npos);
{
std::lock_guard<std::mutex> lock(_autoSContext->_confMutex);
if (curDevIsGPU && _loadContext[CPU].isEnabled) {
// user does not set the compiling threads
// limit the threads num for compiling
int maxNumThreads = 0;
try {
maxNumThreads = _autoSContext->_core->GetConfig(device, GPU_CONFIG_KEY(MAX_NUM_THREADS)).as<int>();
} catch (...) {
LOG_DEBUG("[AUTOPLUGIN]: cannot get MAX_NUM_THREADS from GPU");
}
if (maxNumThreads == static_cast<int>(std::thread::hardware_concurrency())) {
int threadNum = maxNumThreads / 2;
deviceConfig[GPU_CONFIG_KEY(MAX_NUM_THREADS)] = std::to_string(threadNum).c_str();
LOG_DEBUG("[AUTO PLUGIN]:gpu streams number for compiling: %s",
deviceConfig[GPU_CONFIG_KEY(MAX_NUM_THREADS)].c_str());
} else {
// user set the compiling threads num
// use the user's val anyway
LOG_DEBUG("[AUTOPLUGIN]:user defined compiling threads: %d", maxNumThreads);
}
}
}
try {
if (!modelPath.empty()) {
context.executableNetwork = _autoSContext->_core->LoadNetwork(modelPath, device, deviceConfig);
} else {
context.executableNetwork = _autoSContext->_core->LoadNetwork(network, device, deviceConfig);
}
context.isLoadSuccess = true;
} catch (const std::exception& e) {
context.errMessage += device + ":" + e.what();
context.isLoadSuccess = false;
}
if (context.isLoadSuccess || curDevIsCPU) {
return;
}
// need to reload network, unregister it's priority
// there maybe potential issue.
// for example they are dGPU, VPUX, iGPU, customer want to LoadNetwork with
// configure 0 dGPU, 1 VPUX, if dGPU load failed,
// the result will be not sure, maybe two network are loaded into VPUX,
// maybe 0 is loaded to VPUX, 1 is loaded to iGPU
_autoSContext->_plugin->UnregisterPriority(_autoSContext->_modelPriority, context.deviceInfo.uniqueName);
// remove the current device from deviceList
auto eraseDevice = std::find_if(deviceList.begin(), deviceList.end(),
[device](DeviceInformation & d) {
return d.deviceName == device;
});
deviceList.erase(eraseDevice);
if (deviceList.empty()) {
return;
}
// select next candidate device
try {
std::lock_guard<std::mutex> lock(_autoSContext->_confMutex);
context.deviceInfo = _autoSContext->_plugin->SelectDevice(deviceList,
context.networkPrecision, _autoSContext->_modelPriority);
} catch (const std::exception& e) {
return;
}
// if the select device is CPU, need to check the config of _loadContext[CPU]
// if they are same, do not need to load again
curDevIsCPU = (context.deviceInfo.deviceName.find("CPU") != std::string::npos);
if (curDevIsCPU) {
auto compare = [](std::map<std::string, std::string>& a,
std::map<std::string, std::string>& b) -> bool {
if (a.size() != b.size()) {
return false;
}
for (auto& item : a) {
auto bIter = b.find(item.first);
if (bIter != b.end()) {
if (bIter->second != item.second) {
return false;
}
} else {
return false;
}
}
return true;
};
if (compare(context.deviceInfo.config, _loadContext[CPU].deviceInfo.config)) {
return;
}
}
LOG_DEBUG("[AUTOPLUGIN] try to load %s", context.deviceInfo.deviceName.c_str());
// try to load this candidate device
TryToLoadNetWork(context, modelPath, network);
}
void AutoSchedule::WaitFirstNetworkReady() {
if (_firstLoadFuture.valid()) {
// wait for the first loading finished
_firstLoadFuture.wait();
}
// check if there is any device that have loaded network successfully
for (int i = CONTEXTNUM - 1; i >= 0; i--) {
if (_loadContext[i].isEnabled && _loadContext[i].isAlready) {
return;
}
}
// the first loading is failed, wait for another loading
for (int i = CONTEXTNUM - 1; i >= 0; i--) {
if (_loadContext[i].isEnabled) {
_loadContext[i].future.wait();
// check if loading is successful
if (_loadContext[i].isAlready) {
return;
}
}
}
//print errMessage
for (int i = CONTEXTNUM - 1; i >= 0; i--) {
if (_loadContext[i].isEnabled) {
LOG_ERROR("[AUTOPLUGIN] load failed, %s", _loadContext[i].errMessage.c_str());
}
}
IE_THROW() << "[AUTOPLUGIN] load all devices failed";
}
void AutoSchedule::WaitActualNetworkReady() const {
OV_ITT_SCOPED_TASK(itt::domains::MULTIPlugin, "AutoSchedule::WaitActualNetworkReady");
// Maybe different API will call this function, so add call once here
// for every AutoSchedule instance
std::call_once(_oc, [this]() {
if (_loadContext[ACTUALDEVICE].future.valid()) {
_loadContext[ACTUALDEVICE].future.wait();
}
});
}
bool AutoSchedule::ScheduleToWorkerInferRequest(IE::Task inferPipelineTask, DeviceName preferred_device) {
std::vector<DeviceInformation> devices;
// AUTO work mode
if (!preferred_device.empty()) {
// if the device needed by customer is not ready, need to wait for it
WaitActualNetworkReady();
// the preferred_device should be the selected device in AUTO work mode
if (preferred_device != _loadContext[ACTUALDEVICE].deviceInfo.deviceName) {
IE_THROW(NotFound) << "The preferred device should be the selected device";
}
devices.push_back(_loadContext[ACTUALDEVICE].deviceInfo);
} else {
// _acceleratorDevice could be the same as _cpuDevice, such as AUTO:CPU
if (_loadContext[ACTUALDEVICE].isAlready) {
devices.push_back(_loadContext[ACTUALDEVICE].deviceInfo);
} else {
// replace deviceName with workName, so schedule can select correct
// idleWorkerQueue
auto deviceInfo = _loadContext[CPU].deviceInfo;
deviceInfo.deviceName = _loadContext[CPU].workName;
devices.push_back(std::move(deviceInfo));
}
}
for (auto&& device : devices) {
if (!preferred_device.empty() && (device.deviceName != preferred_device)) {
continue;
}
if (RunPipelineTask(inferPipelineTask, _idleWorkerRequests[device.deviceName], preferred_device)) {
return true;
}
}
// no vacant requests this time, storing the task to the respective queue
if (!preferred_device.empty()) {
_inferPipelineTasksDeviceSpecific[preferred_device]->push(std::move(inferPipelineTask));
} else {
_inferPipelineTasks.push(std::move(inferPipelineTask));
}
return false;
}
bool AutoSchedule::RunPipelineTask(IE::Task& inferPipelineTask,
NotBusyPriorityWorkerRequests& idleWorkerRequests,
const DeviceName& preferred_device) {
WorkerInferRequest* workerRequestPtr = nullptr;
std::pair<int, WorkerInferRequest*> worker;
if (idleWorkerRequests.try_pop(worker)) {
workerRequestPtr = worker.second;
IdleGuard<NotBusyPriorityWorkerRequests> idleGuard{workerRequestPtr, idleWorkerRequests};
_thisWorkerInferRequest = workerRequestPtr;
{
auto capturedTask = std::move(inferPipelineTask);
capturedTask();
}
idleGuard.Release();
return true;
}
return false;
}
AutoSchedule::~AutoSchedule() {
// this is necessary to guarantee member destroyed after getting future
if (_loadContext[CPU].isEnabled) {
_exitFlag = true;
_loadContext[CPU].future.wait();
WaitActualNetworkReady();
// it's necessary to wait the loading network threads to stop here.
_autoSContext->_plugin->executorManager()->clear("AutoDeviceAsyncLoad");
_executor.reset();
}
_autoSContext->_plugin->UnregisterPriority(_autoSContext->_modelPriority,
_loadContext[ACTUALDEVICE].deviceInfo.uniqueName);
LOG_INFO("[AUTOPLUGIN]ExecutableNetwork end");
}
IInferPtr AutoSchedule::CreateInferRequestImpl(
const std::vector<std::shared_ptr<const ov::Node>>& inputs,
const std::vector<std::shared_ptr<const ov::Node>>& outputs) {
auto num = _numRequestsCreated++;
IE::SoIInferRequestInternal request_to_share_blobs_with;
IE::RemoteContext::Ptr ctx = nullptr;
if (!_loadContext[CPU].isEnabled && _loadContext[ACTUALDEVICE].isAlready) {
try {
ctx = _autoSContext->_core->GetDefaultContext(
_loadContext[ACTUALDEVICE].deviceInfo.deviceName);
} catch (IE::Exception& ex) {
// plugin does not support context, say CPU
LOG_DEBUG("[AUTOPLUGIN]context not supported for %s, fallback to default memory",
_loadContext[ACTUALDEVICE].deviceInfo.deviceName.c_str());
// for dynamic shape support
auto& dev_requests =
_workerRequests[_loadContext[ACTUALDEVICE].deviceInfo.deviceName];
if (num < dev_requests.size()) {
request_to_share_blobs_with = dev_requests.at(num)._inferRequest;
}
}
}
return std::make_shared<MultiDeviceInferRequest>(inputs, outputs, request_to_share_blobs_with, ctx);
}
IInferPtr AutoSchedule::CreateInferRequestImpl(IE::InputsDataMap networkInputs,
IE::OutputsDataMap networkOutputs) {
auto num = _numRequestsCreated++;
SoInfer request_to_share_blobs_with;
IE::RemoteContext::Ptr ctx = nullptr;
if (!_loadContext[CPU].isEnabled && _loadContext[ACTUALDEVICE].isAlready) {
try {
ctx = _autoSContext->_core->GetDefaultContext(
_loadContext[ACTUALDEVICE].deviceInfo.deviceName);
} catch (IE::Exception& ex) {
// plugin does not support context
LOG_DEBUG("[AUTOPLUGIN]context not supported for %s, fallback to default memory",
_loadContext[ACTUALDEVICE].deviceInfo.deviceName.c_str());
auto& dev_requests =
_workerRequests[_loadContext[ACTUALDEVICE].deviceInfo.deviceName];
if (num < dev_requests.size()) {
request_to_share_blobs_with = dev_requests.at(num)._inferRequest;
}
}
}
return std::make_shared<MultiDeviceInferRequest>(networkInputs, networkOutputs, request_to_share_blobs_with, ctx);
}
IInferPtr AutoSchedule::CreateInferRequest() {
auto execNetwork = std::dynamic_pointer_cast<AutoExecutableNetwork>(
_autoSContext->_executableNetwork.lock());
if (_passthroughExeNet) {
auto res = _passthroughExeNet->CreateInferRequest();
res->setPointerToExecutableNetworkInternal(execNetwork);
return res;
}
IInferPtr syncRequestImpl;
if (_multiSContext->_core && _multiSContext->_core->isNewAPI())
syncRequestImpl = CreateInferRequestImpl(execNetwork->_parameters, execNetwork->_results);
if (!syncRequestImpl)
syncRequestImpl = CreateInferRequestImpl(execNetwork->_networkInputs, execNetwork->_networkOutputs);
syncRequestImpl->setPointerToExecutableNetworkInternal(execNetwork);
return std::make_shared<AsyncInferRequest>(shared_from_this(),
syncRequestImpl,
execNetwork->_callbackExecutor);
}
} // namespace MultiDevicePlugin
src/plugins/auto/auto_schedule.hpp
+77
View File
@@ -0,0 +1,77 @@
// Copyright (C) 2018-2022 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
///////////////////////////////////////////////////////////////////////////////////////////////////
#pragma once
#include "multi_schedule.hpp"
#ifdef MULTIUNITTEST
#define MOCKTESTMACRO virtual
#define MultiDevicePlugin MockMultiDevicePlugin
#else
#define MOCKTESTMACRO
#endif
namespace MultiDevicePlugin {
struct AutoLoadContext {
std::atomic<bool> isEnabled = {false};
std::atomic<bool> isAlready = {false};
std::atomic<bool> isLoadSuccess = {false};
std::future<void> future;
std::promise<void> promise;
SoExecNetwork executableNetwork;
DeviceInformation deviceInfo;
std::vector<DeviceInformation> metaDevices;
std::string networkPrecision;
std::string errMessage;
IE::Task task;
//ACTUALDEVICE's workName is same as it's deviceName,
//CPU_HELP's workName is "CPU_HELP", and deviceName is "CPU"
//add workName is because of ACTUALDEVICE and CPU maybe all CPU,
//they can't use the same workerQueue
std::string workName = "";
};
enum AutoLoadContextIndex {
CPU = 0,
ACTUALDEVICE = 1,
CONTEXTNUM = 2
};
class AutoSchedule : public MultiSchedule {
public:
using Ptr = std::shared_ptr<AutoSchedule>;
void init(const ScheduleContext::Ptr& sContext) override;
IInferPtr CreateInferRequest() override;
IInferPtr CreateInferRequestImpl(IE::InputsDataMap networkInputs, IE::OutputsDataMap networkOutputs) override;
IInferPtr CreateInferRequestImpl(const std::vector<std::shared_ptr<const ov::Node>>& inputs,
const std::vector<std::shared_ptr<const ov::Node>>& outputs) override;
void WaitActualNetworkReady() const;
virtual ~AutoSchedule();
public:
AutoLoadContext _loadContext[CONTEXTNUM];
protected:
void GenerateWorkers(const std::string& device, const SoExecNetwork& executableNetwork) override;
bool ScheduleToWorkerInferRequest(IE::Task, DeviceName preferred_device = "") override;
static bool RunPipelineTask(IE::Task& inferPipelineTask, NotBusyPriorityWorkerRequests& idleWorkerRequests, const DeviceName& preferred_device);
DeviceMap<NotBusyPriorityWorkerRequests> _idleWorkerRequests;
private:
void WaitFirstNetworkReady();
void TryToLoadNetWork(AutoLoadContext& context, const std::string& modelPath, const IE::CNNNetwork& network);
private:
IE::IStreamsExecutor::Ptr _executor;
mutable std::once_flag _oc;
std::once_flag _firstLoadOC;
std::future<void> _firstLoadFuture;
std::promise<void> _firstLoadPromise;
bool _exitFlag = {false};
unsigned int _cpuHelpInferCount = 0;
std::atomic_size_t _numRequestsCreated = {0};
AutoScheduleContext::Ptr _autoSContext;
};
} // namespace MultiDevicePlugin
src/plugins/auto/bind_multi_schedule.cpp
+188
View File
@@ -0,0 +1,188 @@
// Copyright (C) 2018-2022 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
///////////////////////////////////////////////////////////////////////////////////////////////////
#include "async_infer_request.hpp"
#include "plugin.hpp"
#include "bind_multi_schedule.hpp"
#include "multi_executable_network.hpp"
// ------------------------------MultiSchedule----------------------------
namespace MultiDevicePlugin {
thread_local SoInfer BinderMultiSchedule::_sharedRequest = {};
void BinderMultiSchedule::init(const ScheduleContext::Ptr& sContext) {
MultiSchedule::init(sContext);
}
Pipeline BinderMultiSchedule::GetPipeline(const IInferPtr& syncInferRequest, WorkerInferRequest** workerInferRequest) {
Pipeline pipeline = {
// if the request is coming with device-specific remote blobs make sure it is scheduled to the specific device only:
Stage {
/*TaskExecutor*/ std::make_shared<IE::ImmediateExecutor>(), /*task*/ [this, &syncInferRequest, workerInferRequest]() {
// by default, no preferred device:
_thisPreferredDeviceName = "";
auto execNetwork = _multiSContext->_executableNetwork.lock();
// if any input is remote (e.g. was set with SetBlob), let' use the corresponding device
for (const auto& it : execNetwork->GetInputsInfo()) {
auto b = syncInferRequest->GetBlob(it.first);
auto r = b->as<IE::RemoteBlob>();
if (r) {
const auto name = r->getDeviceName();
const auto res = std::find_if(
_multiSContext->_devicePrioritiesInitial.cbegin(),
_multiSContext->_devicePrioritiesInitial.cend(),
[&name](const MultiDevicePlugin::DeviceInformation & d) {
return (d.defaultDeviceID.empty() ? d.deviceName : (d.deviceName + "." +
d.defaultDeviceID)) == name;
});
if (_multiSContext->_devicePrioritiesInitial.cend() == res) {
IE_THROW() <<
"None of the devices (for which current MULTI-device configuration was "
"initialized) supports a remote blob created on the device named " << name;
} else {
// it is ok to take the c_str() here (as pointed in the executable_network.hpp we need to use const char*)
// as the original strings are from the "persistent" vector (with the right lifetime)
_thisPreferredDeviceName = res->deviceName.c_str();
break;
}
}
}
_thisWorkerInferRequest = *workerInferRequest;
_sharedRequest = std::dynamic_pointer_cast<MultiDeviceInferRequest>(syncInferRequest)->GetSharedRequest();
}},
// as the scheduling algo may select any device, this stage accepts the scheduling decision (actual workerRequest)
// then sets the device-agnostic blobs to the actual (device-specific) request
Stage {
/*TaskExecutor*/std::dynamic_pointer_cast<IE::ITaskExecutor>(shared_from_this()), /*task*/ [this, &syncInferRequest, workerInferRequest]() {
*workerInferRequest = _thisWorkerInferRequest;
auto multiSyncInferRequest = std::dynamic_pointer_cast<MultiDeviceInferRequest>(syncInferRequest);
multiSyncInferRequest->SetBlobsToAnotherRequest(_thisWorkerInferRequest->_inferRequest);
INFO_RUN([workerInferRequest]() {
(*workerInferRequest)->_startTimes.push_back(std::move(std::chrono::steady_clock::now()));
});
}},
// final task in the pipeline:
Stage {
/*TaskExecutor*/std::make_shared<ThisRequestExecutor>(workerInferRequest), /*task*/ [this, &syncInferRequest, workerInferRequest]() {
if (nullptr != (*workerInferRequest)->_exceptionPtr) {
std::rethrow_exception((*workerInferRequest)->_exceptionPtr);
}
if (_multiSContext->_needPerfCounters) {
auto multiSyncInferRequest = std::dynamic_pointer_cast<MultiDeviceInferRequest>
(syncInferRequest);
multiSyncInferRequest->_perfMap =
(*workerInferRequest)->_inferRequest->GetPerformanceCounts();
}
INFO_RUN([workerInferRequest]() {
(*workerInferRequest)->_endTimes.push_back(std::move(std::chrono::steady_clock::now()));
});
}}
};
return pipeline;
}
bool BinderMultiSchedule::RunPipelineTask(IE::Task& inferPipelineTask,
NotBusyWorkerRequests& idleWorkerRequests,
const DeviceName& preferred_device) {
WorkerInferRequest* workerRequestPtr = nullptr;
WorkerInferRequest* headWorker = nullptr;
bool flag = false;
if (_sharedRequest) {
while (idleWorkerRequests.try_pop(workerRequestPtr)) {
if (flag && workerRequestPtr == headWorker)
break;
if (!flag) {
headWorker = workerRequestPtr;
flag = true;
}
IdleGuard<NotBusyWorkerRequests> idleGuard{workerRequestPtr, idleWorkerRequests};
if (_sharedRequest._ptr.get() == workerRequestPtr->_inferRequest._ptr.get()) {
_thisWorkerInferRequest = workerRequestPtr;
{
auto capturedTask = std::move(inferPipelineTask);
capturedTask();
}
idleGuard.Release();
return true;
}
}
} else {
//TBD
}
return false;
}
void BinderMultiSchedule::run(IE::Task inferPipelineTask) {
if (_thisWorkerInferRequest) {
auto capturedTask = std::move(inferPipelineTask);
capturedTask();
} else {
ScheduleToWorkerInferRequest(std::move(inferPipelineTask), _thisPreferredDeviceName);
}
}
BinderMultiSchedule::~BinderMultiSchedule() {
}
IInferPtr BinderMultiSchedule::CreateInferRequestImpl(
const std::vector<std::shared_ptr<const ov::Node>>& inputs,
const std::vector<std::shared_ptr<const ov::Node>>& outputs) {
auto num = _numRequestsCreated++;
size_t sum = 0;
SoInfer request_to_share_blobs_with;
// borrowing device-specific blobs from the underlying requests for the device-agnostic, user-facing requests
// this allows to potentially save on the data-copy later (if the requests are scheduled in the same order)
for (const auto& device : _multiSContext->_devicePrioritiesInitial) {
auto& dev_requests = _workerRequests[device.deviceName];
if ((num - sum) < dev_requests.size()) {
request_to_share_blobs_with = dev_requests.at(num - sum)._inferRequest;
break;
}
sum += dev_requests.size();
}
auto syncImpl = std::make_shared<MultiDeviceInferRequest>(inputs, outputs, request_to_share_blobs_with);
return syncImpl;
}
IInferPtr BinderMultiSchedule::CreateInferRequestImpl(IE::InputsDataMap networkInputs,
IE::OutputsDataMap networkOutputs) {
auto num = _numRequestsCreated++;
SoInfer request_to_share_blobs_with;
size_t sum = 0;
// borrowing device-specific blobs from the underlying requests for the device-agnostic, user-facing requests
// this allows to potentially save on the data-copy later (if the requests are scheduled in the same order)
for (const auto& device : _multiSContext->_devicePrioritiesInitial) {
auto& dev_requests = _workerRequests[device.deviceName];
if ((num - sum) < dev_requests.size()) {
request_to_share_blobs_with = dev_requests.at(num - sum)._inferRequest;
break;
}
sum += dev_requests.size();
}
auto syncImpl = std::make_shared<MultiDeviceInferRequest>(networkInputs, networkOutputs, request_to_share_blobs_with);
return syncImpl;
}
IInferPtr BinderMultiSchedule::CreateInferRequest() {
auto execNetwork = std::dynamic_pointer_cast<MultiExecutableNetwork>(
_multiSContext->_executableNetwork.lock());
if (_passthroughExeNet) {
auto res = _passthroughExeNet->CreateInferRequest();
res->setPointerToExecutableNetworkInternal(execNetwork);
return res;
}
IInferPtr syncRequestImpl;
if (_multiSContext->_core && _multiSContext->_core->isNewAPI())
syncRequestImpl = CreateInferRequestImpl(execNetwork->_parameters, execNetwork->_results);
if (!syncRequestImpl)
syncRequestImpl = CreateInferRequestImpl(execNetwork->_networkInputs, execNetwork->_networkOutputs);
syncRequestImpl->setPointerToExecutableNetworkInternal(execNetwork);
return std::make_shared<AsyncInferRequest>(shared_from_this(),
syncRequestImpl,
execNetwork->_callbackExecutor);
}
} // namespace MultiDevicePlugin
src/plugins/auto/bind_multi_schedule.hpp
+36
View File
@@ -0,0 +1,36 @@
// Copyright (C) 2018-2022 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
///////////////////////////////////////////////////////////////////////////////////////////////////
#pragma once
#include "multi_schedule.hpp"
#ifdef MULTIUNITTEST
#define MOCKTESTMACRO virtual
#define MultiDevicePlugin MockMultiDevicePlugin
#else
#define MOCKTESTMACRO
#endif
namespace MultiDevicePlugin {
class BinderMultiSchedule : public MultiSchedule {
public:
using Ptr = std::shared_ptr<BinderMultiSchedule>;
IInferPtr CreateInferRequest() override;
IInferPtr CreateInferRequestImpl(IE::InputsDataMap networkInputs, IE::OutputsDataMap networkOutputs) override;
IE::IInferRequestInternal::Ptr CreateInferRequestImpl(const std::vector<std::shared_ptr<const ov::Node>>& inputs,
const std::vector<std::shared_ptr<const ov::Node>>& outputs) override;
void run(IE::Task inferTask) override;
void init(const ScheduleContext::Ptr& sContext) override;
Pipeline GetPipeline(const IInferPtr& syncRequestImpl, WorkerInferRequest** WorkerInferRequest) override;
virtual ~BinderMultiSchedule();
protected:
static bool RunPipelineTask(IE::Task& inferPipelineTask, NotBusyWorkerRequests& idleWorkerRequests, const DeviceName& preferred_device);
protected:
static thread_local SoInfer _sharedRequest;
};
} // namespace MultiDevicePlugin
src/plugins/auto/common.hpp
+141
View File
@@ -0,0 +1,141 @@
// Copyright (C) 2018-2022 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
///////////////////////////////////////////////////////////////////////////////////////////////////
#pragma once
#include <map>
#include <string>
#include "ie_icore.hpp"
#include "ie_metric_helpers.hpp"
#include <ie_plugin_config.hpp>
#include "cpp_interfaces/impl/ie_executable_network_thread_safe_default.hpp"
#include "threading/ie_executor_manager.hpp"
#include "threading/ie_immediate_executor.hpp"
#include "threading/ie_istreams_executor.hpp"
#include "threading/ie_itask_executor.hpp"
#include "threading/ie_thread_safe_containers.hpp"
#include "utils/log_util.hpp"
#include <ie_performance_hints.hpp>
#include "openvino/runtime/properties.hpp"
#include "ngraph/opsets/opset1.hpp"
#include "transformations/utils/utils.hpp"
#include "utils/log_util.hpp"
#include "itt.hpp"
#ifdef MULTIUNITTEST
#define MOCKTESTMACRO virtual
#define MultiDevicePlugin MockMultiDevicePlugin
#else
#define MOCKTESTMACRO
#endif
namespace MultiDevicePlugin {
namespace IE = InferenceEngine;
using DeviceName = std::string;
using IInferPtr = IE::IInferRequestInternal::Ptr;
using IExecNetwork = IE::IExecutableNetworkInternal;
using SoInfer = IE::SoIInferRequestInternal;
using SoExecNetwork = IE::SoExecutableNetworkInternal;
using Time = std::chrono::time_point<std::chrono::steady_clock>;
template<typename T>
using DeviceMap = std::unordered_map<DeviceName, T>;
struct DeviceInformation {
DeviceName deviceName;
std::map<std::string, std::string> config;
int numRequestsPerDevices;
std::string defaultDeviceID;
DeviceName uniqueName;
unsigned int devicePriority;
};
struct WorkerInferRequest {
SoInfer _inferRequest;
IE::Task _task;
std::exception_ptr _exceptionPtr = nullptr;
std::list<Time> _startTimes;
std::list<Time> _endTimes;
int _index = 0;
};
using NotBusyPriorityWorkerRequests = IE::ThreadSafeBoundedPriorityQueue<std::pair<int, WorkerInferRequest*>>;
using NotBusyWorkerRequests = IE::ThreadSafeBoundedQueue<WorkerInferRequest*>;
template <typename T>
struct IdleGuard {};
template<>
struct IdleGuard<NotBusyWorkerRequests> {
explicit IdleGuard(WorkerInferRequest* workerInferRequestPtr, NotBusyWorkerRequests& notBusyWorkerRequests) :
_workerInferRequestPtr{workerInferRequestPtr},
_notBusyWorkerRequests{&notBusyWorkerRequests} {
}
~IdleGuard() {
if (nullptr != _notBusyWorkerRequests) {
_notBusyWorkerRequests->try_push(_workerInferRequestPtr);
}
}
NotBusyWorkerRequests* Release() {
auto notBusyWorkerRequests = _notBusyWorkerRequests;
_notBusyWorkerRequests = nullptr;
return notBusyWorkerRequests;
}
WorkerInferRequest* _workerInferRequestPtr = nullptr;
NotBusyWorkerRequests* _notBusyWorkerRequests = nullptr;
};
template<>
struct IdleGuard<NotBusyPriorityWorkerRequests> {
explicit IdleGuard(WorkerInferRequest* workerInferRequestPtr, NotBusyPriorityWorkerRequests& notBusyWorkerRequests) :
_workerInferRequestPtr{workerInferRequestPtr},
_notBusyWorkerRequests{&notBusyWorkerRequests} {
}
~IdleGuard() {
if (nullptr != _notBusyWorkerRequests) {
_notBusyWorkerRequests->try_push(std::make_pair(_workerInferRequestPtr->_index, _workerInferRequestPtr));
}
}
NotBusyPriorityWorkerRequests* Release() {
auto notBusyWorkerRequests = _notBusyWorkerRequests;
_notBusyWorkerRequests = nullptr;
return notBusyWorkerRequests;
}
WorkerInferRequest* _workerInferRequestPtr = nullptr;
NotBusyPriorityWorkerRequests* _notBusyWorkerRequests = nullptr;
};
class ScheduleContext : public std::enable_shared_from_this<ScheduleContext> {
public:
using Ptr = std::shared_ptr<ScheduleContext>;
std::shared_ptr<IE::ICore> _core;
std::weak_ptr<IExecNetwork> _executableNetwork;
virtual ~ScheduleContext() = default;
};
class MultiScheduleContext : public ScheduleContext {
public:
using Ptr = std::shared_ptr<MultiScheduleContext>;
std::vector<DeviceInformation> _devicePriorities;
std::vector<DeviceInformation> _devicePrioritiesInitial;
std::unordered_map<std::string, IE::Parameter> _config;
DeviceMap<SoExecNetwork> _networksPerDevice;
std::mutex _mutex;
bool _needPerfCounters;
virtual ~MultiScheduleContext() = default;
};
class MultiDeviceInferencePlugin;
class AutoScheduleContext : public MultiScheduleContext {
public:
using Ptr = std::shared_ptr<AutoScheduleContext>;
std::string _modelPath;
IE::CNNNetwork _network;
std::string _strDevices;
unsigned int _modelPriority = 0;
bool _batchingDisabled = {false};
std::string _performanceHint;
std::mutex _confMutex;
MultiDeviceInferencePlugin* _plugin;
virtual ~AutoScheduleContext() = default;
};
} // namespace MultiDevicePlugin
src/plugins/auto/executable_network.cpp
+22 -946
View File
@@ -3,967 +3,43 @@
//
///////////////////////////////////////////////////////////////////////////////////////////////////
#include <mutex>
#include <string>
#include <vector>
#include <memory>
#include <utility>
#include <map>
#include <unordered_map>
#include "ie_icore.hpp"
#include "ie_metric_helpers.hpp"
#include <ie_plugin_config.hpp>
#include "executable_network.hpp"
#include "async_infer_request.hpp"
#include "plugin.hpp"
#include "ngraph/opsets/opset1.hpp"
#include "transformations/utils/utils.hpp"
#include "utils/log_util.hpp"
#include "itt.hpp"
// ------------------------------MultiDeviceExecutableNetwork----------------------------
// ------------------------------ExecutableNetwork----------------------------
namespace MultiDevicePlugin {
using namespace InferenceEngine;
namespace {
std::string GetNetworkPrecision(const InferenceEngine::CNNNetwork &network) {
auto nGraphFunc = network.getFunction();
bool isINTModel = ngraph::op::util::has_op_with_type<ngraph::op::FakeQuantize>(nGraphFunc);
if (isINTModel) {
return METRIC_VALUE(INT8);
}
for (auto & node : nGraphFunc->get_ordered_ops()) {
if (std::dynamic_pointer_cast<ngraph::opset1::Convolution>(node) ||
std::dynamic_pointer_cast<ngraph::opset1::GroupConvolution>(node) ||
std::dynamic_pointer_cast<ngraph::opset1::GroupConvolutionBackpropData>(node) ||
std::dynamic_pointer_cast<ngraph::opset1::ConvolutionBackpropData>(node)) {
auto layerType = node->input(1).get_element_type().get_type_name();
if (layerType == "f32")
return METRIC_VALUE(FP32);
if (layerType == "f16")
return METRIC_VALUE(FP16);
}
}
return METRIC_VALUE(FP32);
}
} // namespace
thread_local MultiDeviceExecutableNetwork::WorkerInferRequest* MultiDeviceExecutableNetwork::_thisWorkerInferRequest = nullptr;
// TODO: revert to the plain variable (see header file), when we moved to the next CentOS 8.x in our support matrix
thread_local const char* MultiDeviceExecutableNetwork::_thisPreferredDeviceName = "";
struct IdleGuard {
explicit IdleGuard(MultiDeviceExecutableNetwork::WorkerInferRequest* workerInferRequestPtr,
MultiDeviceExecutableNetwork::NotBusyWorkerRequests& notBusyWorkerRequests) :
_workerInferRequestPtr{workerInferRequestPtr},
_notBusyWorkerRequests{&notBusyWorkerRequests} {
}
~IdleGuard() {
if (nullptr != _notBusyWorkerRequests) {
_notBusyWorkerRequests->try_push(std::make_pair(_workerInferRequestPtr->_index, _workerInferRequestPtr));
}
}
MultiDeviceExecutableNetwork::NotBusyWorkerRequests* Release() {
auto notBusyWorkerRequests = _notBusyWorkerRequests;
_notBusyWorkerRequests = nullptr;
return notBusyWorkerRequests;
}
MultiDeviceExecutableNetwork::WorkerInferRequest* _workerInferRequestPtr = nullptr;
MultiDeviceExecutableNetwork::NotBusyWorkerRequests* _notBusyWorkerRequests = nullptr;
};
MultiDeviceExecutableNetwork::MultiDeviceExecutableNetwork(const DeviceMap<InferenceEngine::SoExecutableNetworkInternal>& networksPerDevice,
const std::vector<DeviceInformation>& networkDevices,
const std::unordered_map<std::string, InferenceEngine::Parameter>& config,
const bool needPerfCounters) :
InferenceEngine::ExecutableNetworkThreadSafeDefault(nullptr, std::make_shared<InferenceEngine::ImmediateExecutor>()),
_devicePriorities{networkDevices},
_devicePrioritiesInitial{networkDevices},
_networksPerDevice{networksPerDevice},
_config{config},
_needPerfCounters{needPerfCounters} {
_cpuHelpReleaseTime = std::chrono::steady_clock::now();
_taskExecutor.reset();
for (auto&& networkValue : _networksPerDevice) {
auto& device = networkValue.first;
auto& network = networkValue.second;
GenerateWorkers(device, network);
}
if (_networksPerDevice.size() == 1)
_passthroughExeNet = _networksPerDevice.begin()->second;
ExecutableNetwork::ExecutableNetwork(const Schedule::Ptr& schedule,
const ScheduleContext::Ptr& sContext):
_schedule(schedule),
_sContext(sContext) {
_schedule->init(_sContext);
}
void MultiDeviceExecutableNetwork::GenerateWorkers(const std::string& device, const SoExecutableNetworkInternal& executableNetwork) {
std::string realDeviceName;
if (device == "CPU_HELP") {
realDeviceName = "CPU";
} else {
realDeviceName = device;
}
auto itNumRequests = std::find_if(_devicePriorities.cbegin(), _devicePriorities.cend(),
[&realDeviceName](const DeviceInformation& d){ return d.deviceName == realDeviceName;});
unsigned int optimalNum = 0;
try {
optimalNum = executableNetwork->GetMetric(METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS)).as<unsigned int>();
} catch (const InferenceEngine::Exception &iie) {
IE_THROW()
<< "Every device used with the Multi-Device should "
<< "support OPTIMAL_NUMBER_OF_INFER_REQUESTS ExecutableNetwork metric. "
<< "Failed to query the metric for the " << device << " with error:" << iie.what();
}
const auto numRequests = (_devicePriorities.end() == itNumRequests ||
itNumRequests->numRequestsPerDevices == -1) ? optimalNum : itNumRequests->numRequestsPerDevices;
auto& workerRequests = _workerRequests[device];
auto& idleWorkerRequests = _idleWorkerRequests[device];
workerRequests.resize(numRequests);
_inferPipelineTasksDeviceSpecific[device] = std::unique_ptr<ThreadSafeQueue<Task>>(new ThreadSafeQueue<Task>);
auto* idleWorkerRequestsPtr = &(idleWorkerRequests);
idleWorkerRequests.set_capacity(numRequests);
int num = 0;
for (auto&& workerRequest : workerRequests) {
workerRequest._inferRequest = {executableNetwork->CreateInferRequest(), executableNetwork._so};
auto* workerRequestPtr = &workerRequest;
workerRequestPtr->_index = num++;
IE_ASSERT(idleWorkerRequests.try_push(std::make_pair(workerRequestPtr->_index, workerRequestPtr)) == true);
workerRequest._inferRequest->SetCallback(
[workerRequestPtr, this, device, idleWorkerRequestsPtr] (std::exception_ptr exceptionPtr) mutable {
IdleGuard idleGuard{workerRequestPtr, *idleWorkerRequestsPtr};
workerRequestPtr->_exceptionPtr = exceptionPtr;
{
auto capturedTask = std::move(workerRequestPtr->_task);
capturedTask();
}
// try to return the request to the idle list (fails if the overall object destruction has began)
if (idleGuard.Release()->try_push(std::make_pair(workerRequestPtr->_index, workerRequestPtr))) {
// let's try to pop a task, as we know there is at least one idle request, schedule if succeeded
// if no device-agnostic tasks, let's try pop the device specific task, schedule if succeeded
Task t;
do {
_inferPipelineTasks.try_pop(t);
} while (t && ScheduleToWorkerInferRequest(std::move(t)));
do {
_inferPipelineTasksDeviceSpecific[device]->try_pop(t);
} while (t && ScheduleToWorkerInferRequest(std::move(t), device));
}
});
}
ExecutableNetwork::~ExecutableNetwork() {
}
MultiDeviceExecutableNetwork::MultiDeviceExecutableNetwork(const std::string& modelPath,
const InferenceEngine::CNNNetwork& network,
const std::vector<DeviceInformation>& metaDevices,
const std::string& strDevices,
MultiDeviceInferencePlugin* plugin,
const AutoContext& context,
const bool needPerfCounters)
: _devicePriorities{metaDevices}
, _devicePrioritiesInitial{metaDevices}
, _needPerfCounters(needPerfCounters)
, _multiPlugin(plugin)
, _context(context)
, _workModeIsAUTO(true)
, _network(network) {
LOG_INFO("[AUTOPLUGIN]ExecutableNetwork start");
// initialize cpuHelpReleasetime
_cpuHelpReleaseTime = std::chrono::steady_clock::now();
if (_multiPlugin->GetCore() == nullptr) {
IE_THROW() << "Please, work with " << _multiPlugin->GetName() << " device via InferencEngine::Core object";
}
if (modelPath.empty() && network.getFunction() == nullptr) {
IE_THROW() << "MULTI " << _multiPlugin->GetName() << " device supports just ngraph network representation";
}
_core = _multiPlugin->GetCore(); // shared_ptr that holds the Core
_config[MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES] = strDevices;
std::string profilingTask = "MultiDeviceExecutableNetwork::MultiDeviceExecutableNetwork:AutoMode";
bool isCumulative = (context.performanceHint == PluginConfigParams::CUMULATIVE_THROUGHPUT) ? true : false;
// loadContext[ACTUALDEVICE] is always enabled,
// when there is CPU and there are more than two devices, loadContext[CPU] is enabled
_loadContext[ACTUALDEVICE].isEnabled = true;
if (modelPath.empty())
_loadContext[ACTUALDEVICE].networkPrecision = GetNetworkPrecision(network);
_loadContext[ACTUALDEVICE].metaDevices = metaDevices;
if (isCumulative) {
std::list<DeviceInformation> validDevices =
_multiPlugin->GetValidDevice(metaDevices, _loadContext[ACTUALDEVICE].networkPrecision);
std::string deviceName = "MULTI:";
for (auto& device : validDevices) {
deviceName += device.deviceName;
deviceName += ((device.deviceName == validDevices.back().deviceName) ? "" : ",");
}
_loadContext[ACTUALDEVICE].deviceInfo.deviceName = deviceName;
_loadContext[ACTUALDEVICE].deviceInfo.config[CONFIG_KEY(PERFORMANCE_HINT)] =
InferenceEngine::PluginConfigParams::THROUGHPUT;
} else {
_loadContext[ACTUALDEVICE].deviceInfo = _multiPlugin->SelectDevice(metaDevices,
_loadContext[ACTUALDEVICE].networkPrecision,
_context.modelPriority);
}
LOG_INFO("[AUTOPLUGIN]:select device:%s", _loadContext[ACTUALDEVICE].deviceInfo.deviceName.c_str());
bool isActualDevCPU =
_loadContext[ACTUALDEVICE].deviceInfo.deviceName.find("CPU") != std::string::npos;
// if Actual device is CPU or hint is cumulative, disabled _loadContext[CPU], only use _loadContext[ACTUALDEVICE]
if (isActualDevCPU || isCumulative) {
_loadContext[CPU].isEnabled = false;
} else {
const auto CPUIter = std::find_if(metaDevices.begin(), metaDevices.end(),
[=](const DeviceInformation& d)->bool{return d.deviceName.find("CPU") != std::string::npos;});
// if have CPU Device, enable _loadContext[CPU]
if (CPUIter != metaDevices.end()) {
_loadContext[CPU].isEnabled = true;
_loadContext[CPU].deviceInfo = *CPUIter;
_loadContext[CPU].deviceInfo.config[CONFIG_KEY(PERFORMANCE_HINT)] =
InferenceEngine::PluginConfigParams::LATENCY;
_loadContext[CPU].workName = "CPU_HELP";
LOG_INFO("[AUTOPLUGIN]:will load CPU for accelerator");
} else {
_loadContext[CPU].isEnabled = false;
}
}
// initialize the rest members of load context
for (int i = 0; i < CONTEXTNUM; i++) {
if (_loadContext[i].isEnabled) {
_loadContext[i].future = _loadContext[i].promise.get_future();
auto* contextPtr = &_loadContext[i];
_loadContext[i].task = [this, contextPtr, modelPath, network, isCumulative]() mutable {
TryToLoadNetWork(*contextPtr, modelPath, network);
if (contextPtr->isLoadSuccess) {
if (contextPtr->workName.empty()) {
contextPtr->workName = contextPtr->deviceInfo.deviceName;
}
GenerateWorkers(contextPtr->workName, contextPtr->executableNetwork);
//need lock
{
std::lock_guard<std::mutex> lock(_confMutex);
_config.insert(contextPtr->deviceInfo.config.begin(),
contextPtr->deviceInfo.config.end());
}
contextPtr->isAlready = true;
auto& deviceName = contextPtr->deviceInfo.deviceName;
LOG_INFO("[AUTOPLUGIN]:device:%s loading Network finished",
deviceName.c_str());
if (!isCumulative) {
auto supported_config_keys =
_core->GetMetric(deviceName, METRIC_KEY(SUPPORTED_CONFIG_KEYS))
.as<std::vector<std::string>>();
DEBUG_RUN([this, &contextPtr, &deviceName, &supported_config_keys] {
std::lock_guard<std::mutex> lock(_confMutex);
for (const auto& cfg : supported_config_keys) {
try {
LOG_DEBUG(
"[AUTOPLUGIN]:device:%s, GetConfig:%s=%s",
deviceName.c_str(),
cfg.c_str(),
contextPtr->executableNetwork->GetConfig(cfg).as<std::string>().c_str());
} catch (...) {
}
}
});
}
}
contextPtr->promise.set_value();
// the first load network process finished
std::call_once(_firstLoadOC, [this] () {
_firstLoadPromise.set_value();
});
};
}
}
OV_ITT_SCOPED_TASK(itt::domains::MULTIPlugin, openvino::itt::handle(profilingTask));
if (_loadContext[CPU].isEnabled) {
_firstLoadFuture = _firstLoadPromise.get_future();
// will not wait for loading accelerator network,
// so the executor can't be destroyed before finished the task,
// so use executor as a member of MultiDeviceExecutableNetwork.
_executor = _multiPlugin->executorManager()->getIdleCPUStreamsExecutor(
IStreamsExecutor::Config{"AutoDeviceAsyncLoad",
static_cast<int>(std::thread::hardware_concurrency()) /* max possible #streams*/,
0 /*default threads per stream, workaround for ticket 62376*/,
IStreamsExecutor::ThreadBindingType::NONE});
for (auto&& device : metaDevices) {
// initialize containers before run async task
_idleWorkerRequests[device.deviceName];
_workerRequests[device.deviceName];
_inferPipelineTasksDeviceSpecific[device.deviceName] = nullptr;
}
_idleWorkerRequests["CPU_HELP"];
_workerRequests["CPU_HELP"];
_inferPipelineTasksDeviceSpecific["CPU_HELP"] = nullptr;
_executor->run(_loadContext[CPU].task);
_executor->run(_loadContext[ACTUALDEVICE].task);
auto recycleTask = [this]() mutable {
WaitActualNetworkReady();
while (!_exitFlag && _loadContext[ACTUALDEVICE].isAlready) {
// handle the case of ACTUAL faster than CPU
_loadContext[CPU].future.wait();
// clean up helper infer requests
// first, wait for all the remaining requests to finish
for (auto& iter : _workerRequests["CPU_HELP"]) {
iter._inferRequest._ptr->Wait(InferRequest::WaitMode::RESULT_READY);
}
// late enough to check the idle queue now
// second, check the idle queue if all requests are in place
size_t destroynum = 0;
std::pair<int, WorkerInferRequest*> worker;
std::list<Time> cpuHelpAllStartTimes;
std::list<Time> cpuHelpAllEndTimes;
while (_idleWorkerRequests["CPU_HELP"].try_pop(worker)) {
destroynum++;
INFO_RUN([&cpuHelpAllStartTimes, &cpuHelpAllEndTimes, &worker]() {
cpuHelpAllStartTimes.splice(cpuHelpAllStartTimes.end(), worker.second->_startTimes);
cpuHelpAllEndTimes.splice(cpuHelpAllEndTimes.end(), worker.second->_endTimes);
});
}
INFO_RUN([this, &cpuHelpAllStartTimes, &cpuHelpAllEndTimes]() {
cpuHelpAllStartTimes.sort(std::less<Time>());
cpuHelpAllEndTimes.sort(std::less<Time>());
_cpuHelpInferCount = cpuHelpAllStartTimes.size();
IE_ASSERT(_cpuHelpInferCount == cpuHelpAllEndTimes.size());
});
if (destroynum == _workerRequests["CPU_HELP"].size()) {
std::lock_guard<std::mutex> lock(_confMutex);
INFO_RUN([this, &cpuHelpAllStartTimes, &cpuHelpAllEndTimes, &destroynum]() {
_cpuHelpReleaseTime = std::chrono::steady_clock::now();
if (cpuHelpAllStartTimes.size() >= destroynum + 1) {
//remove last worksize num requests, so the fps will be more accuracy
cpuHelpAllStartTimes.resize(_cpuHelpInferCount - destroynum);
cpuHelpAllEndTimes.resize(_cpuHelpInferCount - destroynum);
std::chrono::duration<double, std::milli> durtation =
cpuHelpAllEndTimes.back() - cpuHelpAllStartTimes.front();
_cpuHelpFps = cpuHelpAllStartTimes.size() * 1000 / durtation.count();
}
});
LOG_INFO("[AUTOPLUGIN] release all work requests of CPU_HELP");
_workerRequests["CPU_HELP"].clear();
_loadContext[CPU].executableNetwork._ptr.reset();
_loadContext[CPU].executableNetwork._so.reset();
LOG_INFO("[AUTOPLUGIN]:helper released!!");
break;
}
}
};
_executor->run(std::move(recycleTask));
} else {
// only one device need to load network, do not need to load it async
_loadContext[ACTUALDEVICE].task();
_passthroughExeNet = _loadContext[ACTUALDEVICE].executableNetwork;
}
WaitFirstNetworkReady();
}
void MultiDeviceExecutableNetwork::TryToLoadNetWork(AutoLoadContext& context,
const std::string& modelPath,
const InferenceEngine::CNNNetwork& network) {
auto& device = context.deviceInfo.deviceName;
auto& deviceConfig = context.deviceInfo.config;
auto& deviceList = context.metaDevices;
bool curDevIsCPU = (device.find("CPU") != std::string::npos);
bool curDevIsGPU = (device.find("GPU") != std::string::npos);
{
std::lock_guard<std::mutex> lock(_confMutex);
if (curDevIsGPU && _loadContext[CPU].isEnabled) {
// user does not set the compiling threads
// limit the threads num for compiling
int maxNumThreads = 0;
try {
maxNumThreads = _core->GetConfig(device, GPU_CONFIG_KEY(MAX_NUM_THREADS)).as<int>();
} catch (...) {
LOG_DEBUG("[AUTOPLUGIN]: cannot get MAX_NUM_THREADS from GPU");
}
if (maxNumThreads == static_cast<int>(std::thread::hardware_concurrency())) {
int threadNum = maxNumThreads / 2;
deviceConfig[GPU_CONFIG_KEY(MAX_NUM_THREADS)] = std::to_string(threadNum).c_str();
LOG_DEBUG("[AUTO PLUGIN]:gpu streams number for compiling: %s", deviceConfig[GPU_CONFIG_KEY(MAX_NUM_THREADS)].c_str());
} else {
// user set the compiling threads num
// use the user's val anyway
LOG_DEBUG("[AUTOPLUGIN]:user defined compiling threads: %d", maxNumThreads);
}
}
}
try {
if (!modelPath.empty()) {
context.executableNetwork = _core->LoadNetwork(modelPath, device, deviceConfig);
} else {
context.executableNetwork = _core->LoadNetwork(network, device, deviceConfig);
}
context.isLoadSuccess = true;
} catch (const std::exception& e) {
context.errMessage += device + ":" + e.what();
context.isLoadSuccess = false;
}
if (context.isLoadSuccess || curDevIsCPU) {
return;
}
// need to reload network, unregister it's priority
// there maybe potential issue.
// for example they are dGPU, VPUX, iGPU, customer want to LoadNetwork with
// configure 0 dGPU, 1 VPUX, if dGPU load failed,
// the result will be not sure, maybe two network are loaded into VPUX,
// maybe 0 is loaded to VPUX, 1 is loaded to iGPU
_multiPlugin->UnregisterPriority(_context.modelPriority, context.deviceInfo.uniqueName);
// remove the current device from deviceList
auto eraseDevice = std::find_if(deviceList.begin(), deviceList.end(),
[device](DeviceInformation& d){
return d.deviceName == device;
});
deviceList.erase(eraseDevice);
if (deviceList.empty()) {
return;
}
// select next candidate device
try {
std::lock_guard<std::mutex> lock(_confMutex);
context.deviceInfo = _multiPlugin->SelectDevice(deviceList,
context.networkPrecision, _context.modelPriority);
}
catch (const std::exception& e) {
return;
}
// if the select device is CPU, need to check the config of _loadContext[CPU]
// if they are same, do not need to load again
curDevIsCPU = (context.deviceInfo.deviceName.find("CPU") != std::string::npos);
if (curDevIsCPU) {
auto compare = [](std::map<std::string, std::string>& a,
std::map<std::string, std::string>& b) -> bool {
if (a.size() != b.size()) {
return false;
}
for (auto& item : a) {
auto bIter = b.find(item.first);
if (bIter != b.end()) {
if (bIter->second != item.second) {
return false;
}
} else {
return false;
}
}
return true;
};
if (compare(context.deviceInfo.config, _loadContext[CPU].deviceInfo.config)) {
return;
}
}
LOG_DEBUG("[AUTOPLUGIN] try to load %s", context.deviceInfo.deviceName.c_str());
// try to load this candidate device
TryToLoadNetWork(context, modelPath, network);
}
void MultiDeviceExecutableNetwork::WaitFirstNetworkReady() {
if (_firstLoadFuture.valid()) {
// wait for the first loading finished
_firstLoadFuture.wait();
}
// check if there is any device that have loaded network successfully
for (int i = CONTEXTNUM - 1; i >= 0; i--) {
if (_loadContext[i].isEnabled && _loadContext[i].isAlready) {
return;
}
}
// the first loading is failed, wait for another loading
for (int i = CONTEXTNUM - 1; i >= 0; i--) {
if (_loadContext[i].isEnabled) {
_loadContext[i].future.wait();
// check if loading is successful
if (_loadContext[i].isAlready) {
return;
}
}
}
//print errMessage
for (int i = CONTEXTNUM - 1; i >= 0; i--) {
if (_loadContext[i].isEnabled) {
LOG_ERROR("[AUTOPLUGIN] load failed, %s", _loadContext[i].errMessage.c_str());
}
}
IE_THROW() << "[AUTOPLUGIN] load all devices failed";
}
void MultiDeviceExecutableNetwork::WaitActualNetworkReady() const {
OV_ITT_SCOPED_TASK(itt::domains::MULTIPlugin, "MultiDeviceExecutableNetwork::WaitActualNetworkReady");
// Maybe different API will call this function, so add call once here
// for every MultiDeviceExecutableNetwork instance
std::call_once(_oc, [this] () {
if (_loadContext[ACTUALDEVICE].future.valid()) {
_loadContext[ACTUALDEVICE].future.wait();
}
});
}
bool MultiDeviceExecutableNetwork::ScheduleToWorkerInferRequest(Task inferPipelineTask, DeviceName preferred_device) {
std::vector<DeviceInformation> devices;
// AUTO work mode
if (_workModeIsAUTO) {
if (!preferred_device.empty()) {
// if the device needed by customer is not ready, need to wait for it
WaitActualNetworkReady();
// the preferred_device should be the selected device in AUTO work mode
if (preferred_device != _loadContext[ACTUALDEVICE].deviceInfo.deviceName) {
IE_THROW(NotFound) << "The preferred device should be the selected device";
}
devices.push_back(_loadContext[ACTUALDEVICE].deviceInfo);
} else {
// _acceleratorDevice could be the same as _cpuDevice, such as AUTO:CPU
if (_loadContext[ACTUALDEVICE].isAlready) {
devices.push_back(_loadContext[ACTUALDEVICE].deviceInfo);
} else {
// replace deviceName with workName, so schedule can select correct
// idleWorkerQueue
auto deviceInfo = _loadContext[CPU].deviceInfo;
deviceInfo.deviceName = _loadContext[CPU].workName;
devices.push_back(std::move(deviceInfo));
}
}
} else {
devices = [&] {
std::lock_guard<std::mutex> lock(_mutex);
return _devicePriorities;
}();
}
for (auto&& device : devices) {
if (!preferred_device.empty() && (device.deviceName != preferred_device))
continue;
if (RunPipelineTask(inferPipelineTask, _idleWorkerRequests[device.deviceName], preferred_device)) {
return true;
}
}
// no vacant requests this time, storing the task to the respective queue
if (!preferred_device.empty())
_inferPipelineTasksDeviceSpecific[preferred_device]->push(std::move(inferPipelineTask));
else
_inferPipelineTasks.push(std::move(inferPipelineTask));
return false;
}
bool MultiDeviceExecutableNetwork::RunPipelineTask(Task& inferPipelineTask,
NotBusyWorkerRequests& idleWorkerRequests,
const DeviceName& preferred_device) {
WorkerInferRequest *workerRequestPtr = nullptr;
std::pair<int, WorkerInferRequest*> worker;
if (idleWorkerRequests.try_pop(worker)) {
workerRequestPtr = worker.second;
IdleGuard idleGuard{workerRequestPtr, idleWorkerRequests};
_thisWorkerInferRequest = workerRequestPtr;
{
auto capturedTask = std::move(inferPipelineTask);
capturedTask();
}
idleGuard.Release();
return true;
}
return false;
}
void MultiDeviceExecutableNetwork::run(Task inferPipelineTask) {
ScheduleToWorkerInferRequest(std::move(inferPipelineTask), _thisPreferredDeviceName);
}
MultiDeviceExecutableNetwork::~MultiDeviceExecutableNetwork() {
if (_workModeIsAUTO) {
// this is necessary to guarantee member destroyed after getting future
if (_loadContext[CPU].isEnabled) {
_exitFlag = true;
_loadContext[CPU].future.wait();
WaitActualNetworkReady();
// it's necessary to wait the loading network threads to stop here.
_multiPlugin->executorManager()->clear("AutoDeviceAsyncLoad");
_executor.reset();
}
_multiPlugin->UnregisterPriority(_context.modelPriority,
_loadContext[ACTUALDEVICE].deviceInfo.uniqueName);
}
{
std::lock_guard<std::mutex> lock(_mutex);
_devicePriorities.clear();
}
/* NOTE: The only threads that use `MultiDeviceExecutableNetwork` worker infer requests' threads.
* But AsyncInferRequest destructor should wait for all asynchronous tasks by the request
*/
for (auto&& idleWorker : _idleWorkerRequests) {
// stop accepting any idle requests back (for re-scheduling)
idleWorker.second.set_capacity(0);
}
INFO_RUN([this] {
for (auto&& _workerRequest : _workerRequests) {
std::list<Time> reqAllStartTimes;
std::list<Time> reqAllEndTimes;
for (auto& request : _workerRequest.second) {
reqAllStartTimes.splice(reqAllStartTimes.end(), request._startTimes);
reqAllEndTimes.splice(reqAllEndTimes.end(), request._endTimes);
}
unsigned int count = reqAllStartTimes.size();
IE_ASSERT(count == reqAllEndTimes.size());
reqAllStartTimes.sort(std::less<Time>());
reqAllEndTimes.sort(std::less<Time>());
if (_workerRequest.first == "CPU_HELP") {
LOG_INFO("[AUTOPLUGIN]CPU_HELP:infer:%ld", _cpuHelpInferCount + count);
if (_cpuHelpFps > 0.0) {
LOG_INFO("[AUTOPLUGIN]CPU_HELP:fps:%lf", _cpuHelpFps);
} else if (count >= 1) {
std::chrono::duration<double, std::milli> durtation =
reqAllEndTimes.back() - reqAllStartTimes.front();
LOG_INFO("[AUTOPLUGIN]CPU_HELP:fps:%lf", count * 1000 / durtation.count());
}
} else {
LOG_INFO("[AUTOPLUGIN]%s:infer:%ld", _workerRequest.first.c_str(), count);
auto n = reqAllStartTimes.size();
Time time;
while (!reqAllStartTimes.empty()) {
time = reqAllStartTimes.front();
if (time < _cpuHelpReleaseTime) {
reqAllStartTimes.pop_front();
n--;
} else {
break;
}
}
if (n >= 1) {
std::chrono::duration<double, std::milli> durtation =
reqAllEndTimes.back() - time;
LOG_INFO("[AUTOPLUGIN]%s:fps:%lf", _workerRequest.first.c_str(),
n * 1000 / durtation.count());
}
}
}
});
{
std::lock_guard<std::mutex> lock(_confMutex);
_workerRequests.clear();
}
LOG_INFO("[AUTOPLUGIN]ExecutableNetwork end");
}
std::shared_ptr<InferenceEngine::RemoteContext> MultiDeviceExecutableNetwork::GetContext() const {
if (_workModeIsAUTO) {
WaitActualNetworkReady();
return _loadContext[ACTUALDEVICE].executableNetwork->GetContext();
}
auto devices = [&] {
std::lock_guard<std::mutex> lock(_mutex);
return _devicePriorities;
}();
std::string devices_names;
for (auto&& device : devices) {
devices_names += device.deviceName + " ";
const auto& n = _networksPerDevice.at(device.deviceName);
try {
return n->GetContext();
} catch (const InferenceEngine::NotImplemented&) {}
}
IE_THROW(NotImplemented) << "None of the devices in the MULTI device has an associated remote context."
<< " Current list of devices allowed via the DEVICE_PRIORITIES config: " << devices_names;
}
std::shared_ptr<InferenceEngine::ICore> MultiDeviceExecutableNetwork::GetCore() const {
return _plugin->GetCore();
}
InferenceEngine::IInferRequestInternal::Ptr MultiDeviceExecutableNetwork::CreateInferRequestImpl(
IInferPtr ExecutableNetwork::CreateInferRequestImpl(
const std::vector<std::shared_ptr<const ov::Node>>& inputs,
const std::vector<std::shared_ptr<const ov::Node>>& outputs) {
auto num = _numRequestsCreated++;
InferenceEngine::SoIInferRequestInternal request_to_share_blobs_with;
InferenceEngine::RemoteContext::Ptr ctx = nullptr;
if (_workModeIsAUTO) {
if (!_loadContext[CPU].isEnabled && _loadContext[ACTUALDEVICE].isAlready) {
try {
ctx = GetCore()->GetDefaultContext(_loadContext[ACTUALDEVICE].deviceInfo.deviceName);
} catch (InferenceEngine::Exception& ex) {
// plugin does not support context, say CPU
LOG_DEBUG("[AUTOPLUGIN]context not supported for %s, fallback to default memory",
_loadContext[ACTUALDEVICE].deviceInfo.deviceName.c_str());
// for dynamic shape support
auto& dev_requests = _workerRequests[_loadContext[ACTUALDEVICE].deviceInfo.deviceName];
if (num < dev_requests.size()) {
request_to_share_blobs_with = dev_requests.at(num)._inferRequest;
}
}
}
return std::make_shared<MultiDeviceInferRequest>(inputs, outputs, request_to_share_blobs_with, ctx);
}
return std::make_shared<MultiDeviceInferRequest>(inputs, outputs, request_to_share_blobs_with);
return _schedule->CreateInferRequestImpl(inputs, outputs);
}
InferenceEngine::IInferRequestInternal::Ptr MultiDeviceExecutableNetwork::CreateInferRequestImpl(InferenceEngine::InputsDataMap networkInputs,
InferenceEngine::OutputsDataMap networkOutputs) {
auto num = _numRequestsCreated++;
InferenceEngine::SoIInferRequestInternal request_to_share_blobs_with;
InferenceEngine::RemoteContext::Ptr ctx = nullptr;
if (_workModeIsAUTO) {
if (!_loadContext[CPU].isEnabled && _loadContext[ACTUALDEVICE].isAlready) {
try {
ctx = GetCore()->GetDefaultContext(_loadContext[ACTUALDEVICE].deviceInfo.deviceName);
} catch (InferenceEngine::Exception& ex) {
// plugin does not support context
LOG_DEBUG("[AUTOPLUGIN]context not supported for %s, fallback to default memory",
_loadContext[ACTUALDEVICE].deviceInfo.deviceName.c_str());
auto& dev_requests = _workerRequests[_loadContext[ACTUALDEVICE].deviceInfo.deviceName];
if (num < dev_requests.size()) {
request_to_share_blobs_with = dev_requests.at(num)._inferRequest;
}
}
}
return std::make_shared<MultiDeviceInferRequest>(networkInputs, networkOutputs, request_to_share_blobs_with, ctx);
}
return std::make_shared<MultiDeviceInferRequest>(networkInputs, networkOutputs, request_to_share_blobs_with);
IInferPtr ExecutableNetwork::CreateInferRequestImpl(
InferenceEngine::InputsDataMap networkInputs,
InferenceEngine::OutputsDataMap networkOutputs) {
return _schedule->CreateInferRequestImpl(networkInputs, networkOutputs);
}
IInferRequestInternal::Ptr MultiDeviceExecutableNetwork::CreateInferRequest() {
if (_passthroughExeNet) {
auto res = _passthroughExeNet->CreateInferRequest();
res->setPointerToExecutableNetworkInternal(shared_from_this());
return res;
}
IInferRequestInternal::Ptr syncRequestImpl;
if (this->_plugin) {
const auto& core = _plugin->GetCore();
if (core && core->isNewAPI())
syncRequestImpl = CreateInferRequestImpl(_parameters, _results);
}
if (!syncRequestImpl)
syncRequestImpl = CreateInferRequestImpl(_networkInputs, _networkOutputs);
syncRequestImpl->setPointerToExecutableNetworkInternal(shared_from_this());
return std::make_shared<MultiDeviceAsyncInferRequest>(std::static_pointer_cast<MultiDeviceInferRequest>(syncRequestImpl),
_needPerfCounters,
std::static_pointer_cast<MultiDeviceExecutableNetwork>(shared_from_this()),
_callbackExecutor);
IInferRequestInternal::Ptr ExecutableNetwork::CreateInferRequest() {
SetExeNetworkForContext();
return _schedule->CreateInferRequest();
}
void MultiDeviceExecutableNetwork::SetConfig(const std::map<std::string, InferenceEngine::Parameter> &config) {
if (_workModeIsAUTO) {
IE_THROW(NotImplemented);
}
auto priorities = config.find(ov::device::priorities.name());
if (priorities == config.end() || config.size() > 1) {
IE_THROW() << "The only config supported for the Network's SetConfig is MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES";
} else {
auto multiPlugin = std::dynamic_pointer_cast<MultiDeviceInferencePlugin>(this->_plugin);
assert(multiPlugin != nullptr);
auto metaDevices = multiPlugin->ParseMetaDevices(priorities->second.as<std::string>(), {});
if (std::any_of(metaDevices.begin(), metaDevices.end(), [](const DeviceInformation& kvp) {
return kvp.numRequestsPerDevices != -1;
})) {
IE_THROW() << "You can only change device priorities but not number of requests"
<<" with the Network's SetConfig(MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES!";
}
{
std::lock_guard<std::mutex> lock{_mutex};
for (auto && device : metaDevices) {
if (_networksPerDevice.find(device.deviceName) == _networksPerDevice.end()) {
IE_THROW(NotFound) << "You can only change device priorities but not add new devices with"
<< " the Network's SetConfig(MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES. "
<< device.deviceName << " device was not in the original device list!";
}
}
_devicePriorities = metaDevices;
// update value in config
std::lock_guard<std::mutex> lockConf(_confMutex);
_config[MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES] = priorities->second;
}
}
}
InferenceEngine::Parameter MultiDeviceExecutableNetwork::GetConfig(const std::string &name) const {
{
std::lock_guard<std::mutex> lock(_confMutex);
auto it = _config.find(name);
if (it != _config.end()) {
return it->second;
}
}
// find config key among networks config keys
for (const auto& desc : _networksPerDevice) {
const auto& execNetwork = desc.second;
auto param = execNetwork->GetMetric(METRIC_KEY(SUPPORTED_CONFIG_KEYS));
for (auto &&configKey : param.as<std::vector<std::string>>()) {
if (configKey == name) {
return execNetwork->GetConfig(configKey);
}
}
IE_THROW() << "Unsupported ExecutableNetwork config key: " << name;
}
IE_THROW(NotFound) << name <<" not found in the ExecutableNetwork config";
}
InferenceEngine::Parameter MultiDeviceExecutableNetwork::GetMetric(const std::string &name) const {
if (_workModeIsAUTO) {
if (name == ov::supported_properties) {
return decltype(ov::supported_properties)::value_type {
// Metrics
ov::PropertyName{ov::supported_properties.name(), ov::PropertyMutability::RO},
ov::PropertyName{ov::hint::performance_mode.name(), ov::PropertyMutability::RO},
ov::PropertyName{ov::model_name.name(), ov::PropertyMutability::RO},
ov::PropertyName{ov::optimal_number_of_infer_requests.name(), ov::PropertyMutability::RO},
ov::PropertyName{ov::hint::model_priority.name(), ov::PropertyMutability::RO},
ov::PropertyName{ov::device::priorities.name(), ov::PropertyMutability::RO}
};
} else if (name == ov::device::priorities) {
auto value = _config.find(MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES);
return decltype(ov::device::priorities)::value_type {value->second.as<std::string>()};
} else if (name == ov::hint::model_priority) {
auto value = _context.modelPriority;
if (_core->isNewAPI()) {
return value ? ((value > 1) ? ov::hint::Priority::LOW : ov::hint::Priority::MEDIUM) : ov::hint::Priority::HIGH;
} else {
return value ? ((value > 1) ? CONFIG_VALUE(MODEL_PRIORITY_LOW) : CONFIG_VALUE(MODEL_PRIORITY_MED)) : CONFIG_VALUE(MODEL_PRIORITY_HIGH);
}
} else if (name == ov::optimal_number_of_infer_requests) {
const unsigned int defaultNumForTPUT = 4u;
const unsigned int defaultNumForLatency = 1u;
unsigned int real = 0;
if (_loadContext[ACTUALDEVICE].isAlready) {
real = _loadContext[ACTUALDEVICE].
executableNetwork->GetMetric(name).as<unsigned int>();
} else {
IE_ASSERT(_loadContext[CPU].isAlready == true);
std::unique_lock<std::mutex> lock(_confMutex);
auto deviceInfo = _loadContext[ACTUALDEVICE].deviceInfo;
lock.unlock();
unsigned int optimalBatchSize = 0;
unsigned int requests = 0;
bool bThroughputEnabledInPlugin = false;
try {
// for benchmark through AUTO:CPU,GPU
// SetConfig directly set to CPU/GPU in this case
bThroughputEnabledInPlugin =
_core->GetConfig(deviceInfo.deviceName, CONFIG_KEY(PERFORMANCE_HINT)).as<std::string>() == CONFIG_VALUE(THROUGHPUT);
} catch (...) {
LOG_DEBUG("[AUTOPLUGIN]GetMetric:%s for %s", "PERF_HINT config not supported", deviceInfo.deviceName.c_str());
}
const auto& mode = deviceInfo.config.find(CONFIG_KEY(PERFORMANCE_HINT));
if (bThroughputEnabledInPlugin ||
(mode != deviceInfo.config.end() && mode->second == CONFIG_VALUE(THROUGHPUT))) {
unsigned int upperBoundStreamsNum = 0;
std::map<std::string, InferenceEngine::Parameter> options;
options["MODEL_PTR"] = std::const_pointer_cast<ngraph::Function>(_network.getFunction());
try {
auto rangeOfStreams = _core->GetMetric(deviceInfo.deviceName,
METRIC_KEY(RANGE_FOR_STREAMS), options).as<std::tuple<unsigned int, unsigned int>>();
upperBoundStreamsNum = std::get<1>(rangeOfStreams);
} catch (const InferenceEngine::Exception &iie) {
LOG_DEBUG("[AUTOPLUGIN] GetMetric RANGE_FOR_STREAMS failed");
}
if (!_context.batchingDisabled) {
try {
optimalBatchSize = _core->GetMetric(deviceInfo.deviceName,
METRIC_KEY(OPTIMAL_BATCH_SIZE), options).as<unsigned int>();
LOG_DEBUG("[AUTOPLUGIN]BATCHING:%s:%ld", "optimal batch size", optimalBatchSize);
} catch (...) {
LOG_DEBUG("[AUTOPLUGIN]BATCHING:%s", "metric OPTIMAL_BATCH_SIZE not supported");
}
}
if (optimalBatchSize > 1) {
// batching is supported with the device
// go with auto-batching
try {
// check if app have set preferred value
auto res =
_core->GetConfig(deviceInfo.deviceName, CONFIG_KEY(PERFORMANCE_HINT_NUM_REQUESTS)).as<std::string>();
requests = PerfHintsConfig::CheckPerformanceHintRequestValue(res);
const auto& reqs = deviceInfo.config.find(CONFIG_KEY(PERFORMANCE_HINT_NUM_REQUESTS));
if (reqs != deviceInfo.config.end())
requests = static_cast<unsigned int>(PerfHintsConfig::CheckPerformanceHintRequestValue(reqs->second));
LOG_DEBUG("[AUTOPLUGIN]BATCHING:%s:%ld", "user requested size", requests);
if (!requests) { // no limitations from user
requests = optimalBatchSize * upperBoundStreamsNum * 2;
LOG_DEBUG("[AUTOPLUGIN]BATCHING:%s:%ld", "deduced size:", requests);
}
} catch (const InferenceEngine::Exception &iie) {
LOG_WARNING("[AUTOPLUGIN]deduce optimal infer requset num for auto-batch failed :%s", iie.what());
}
real = (std::max)(requests, optimalBatchSize);
} else if (deviceInfo.deviceName.find("VPUX") != std::string::npos) {
real = 8u;
} else {
real = upperBoundStreamsNum ? 2 * upperBoundStreamsNum : defaultNumForTPUT;
}
} else {
real = defaultNumForLatency;
}
}
return decltype(ov::optimal_number_of_infer_requests)::value_type {real};
}
if (_loadContext[ACTUALDEVICE].isAlready) {
return _loadContext[ACTUALDEVICE].executableNetwork->GetMetric(name);
}
return _loadContext[CPU].executableNetwork->GetMetric(name);
}
if (name == ov::supported_properties) {
return decltype(ov::supported_properties)::value_type {
// Metrics
ov::PropertyName{ov::supported_properties.name(), ov::PropertyMutability::RO},
ov::PropertyName{ov::model_name.name(), ov::PropertyMutability::RO},
ov::PropertyName{ov::optimal_number_of_infer_requests.name(), ov::PropertyMutability::RO},
// Configs
// device priority can be changed on-the-fly in MULTI
ov::PropertyName{ov::device::priorities.name(), ov::PropertyMutability::RW}
};
} else if (name == ov::optimal_number_of_infer_requests) {
unsigned int res = 0u;
for (auto n : _networksPerDevice) {
try {
res += n.second->GetMetric(METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS)).as<unsigned int>();
} catch (const InferenceEngine::Exception &iie) {
IE_THROW()
<< "Every device used with the Multi-Device should "
<< "support OPTIMAL_NUMBER_OF_INFER_REQUESTS ExecutableNetwork metric. "
<< "Failed to query the metric for the " << n.first << " with error:" << iie.what();
}
}
return decltype(ov::optimal_number_of_infer_requests)::value_type {res};
} else if (name == ov::model_name) {
auto it = _networksPerDevice.begin();
IE_ASSERT(it != _networksPerDevice.end());
return decltype(ov::model_name)::value_type {it->second->GetMetric(METRIC_KEY(NETWORK_NAME)).as<std::string>()};
} else if (name == METRIC_KEY(SUPPORTED_METRICS)) {
IE_SET_METRIC_RETURN(SUPPORTED_METRICS, {
METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS),
METRIC_KEY(SUPPORTED_METRICS),
METRIC_KEY(NETWORK_NAME),
METRIC_KEY(SUPPORTED_CONFIG_KEYS)
});
} else if (name == METRIC_KEY(SUPPORTED_CONFIG_KEYS)) {
std::vector<std::string> configKeys = { MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES };
IE_SET_METRIC_RETURN(SUPPORTED_CONFIG_KEYS, configKeys);
} else {
IE_THROW() << "Unsupported ExecutableNetwork metric key: " << name;
}
void ExecutableNetwork::SetExeNetworkForContext() {
// Maybe different API will call this function, so add call once here
// for every AutoSchedule instance
std::call_once(_oc, [this]() {
_sContext->_executableNetwork = shared_from_this();
});
}
} // namespace MultiDevicePlugin
src/plugins/auto/executable_network.hpp
+15 -152
View File
@@ -4,21 +4,7 @@
///////////////////////////////////////////////////////////////////////////////////////////////////
#pragma once
#include <atomic>
#include <mutex>
#include <unordered_map>
#include <map>
#include <vector>
#include <string>
#include "cpp_interfaces/impl/ie_executable_network_thread_safe_default.hpp"
#include "threading/ie_thread_safe_containers.hpp"
#include "threading/ie_itask_executor.hpp"
#include "threading/ie_executor_manager.hpp"
#include "ie_icore.hpp"
#include <ie_performance_hints.hpp>
#include "openvino/runtime/properties.hpp"
#include "schedule.hpp"
#ifdef MULTIUNITTEST
#define MOCKTESTMACRO virtual
@@ -28,145 +14,22 @@
#endif
namespace MultiDevicePlugin {
class MultiDeviceInferencePlugin;
using DeviceName = std::string;
using NetworkFuture = std::future<InferenceEngine::SoExecutableNetworkInternal>;
using NetworkPromise = std::promise<InferenceEngine::SoExecutableNetworkInternal>;
struct DeviceInformation {
DeviceName deviceName;
std::map<std::string, std::string> config;
int numRequestsPerDevices;
std::string defaultDeviceID;
DeviceName uniqueName;
unsigned int devicePriority;
};
struct AutoContext {
bool needPerfCounters = {false};
unsigned int modelPriority = 0;
bool batchingDisabled = {false};
std::string performanceHint;
};
struct AutoLoadContext {
std::atomic<bool> isEnabled = {false};
std::atomic<bool> isAlready = {false};
std::atomic<bool> isLoadSuccess = {false};
std::future<void> future;
std::promise<void> promise;
InferenceEngine::SoExecutableNetworkInternal executableNetwork;
DeviceInformation deviceInfo;
std::vector<DeviceInformation> metaDevices;
std::string networkPrecision;
std::string errMessage;
InferenceEngine::Task task;
//ACTUALDEVICE's workName is same as it's deviceName,
//CPU_HELP's workName is "CPU_HELP", and deviceName is "CPU"
//add workName is because of ACTUALDEVICE and CPU maybe all CPU,
//they can't use the same workerQueue
std::string workName = "";
};
enum AutoLoadContextIndex {
CPU = 0,
ACTUALDEVICE = 1,
CONTEXTNUM = 2
};
using Time = std::chrono::time_point<std::chrono::steady_clock>;
template<typename T>
using DeviceMap = std::unordered_map<DeviceName, T>;
class MultiDeviceExecutableNetwork : public InferenceEngine::ExecutableNetworkThreadSafeDefault,
public InferenceEngine::ITaskExecutor {
class ExecutableNetwork : public
InferenceEngine::ExecutableNetworkThreadSafeDefault {
public:
using Ptr = std::shared_ptr<MultiDeviceExecutableNetwork>;
struct WorkerInferRequest {
InferenceEngine::SoIInferRequestInternal _inferRequest;
InferenceEngine::Task _task;
std::exception_ptr _exceptionPtr = nullptr;
std::list<Time> _startTimes;
std::list<Time> _endTimes;
int _index = 0;
};
using NotBusyWorkerRequests = InferenceEngine::ThreadSafeBoundedPriorityQueue<std::pair<int, WorkerInferRequest*>>;
explicit MultiDeviceExecutableNetwork(const DeviceMap<InferenceEngine::SoExecutableNetworkInternal>& networksPerDevice,
const std::vector<DeviceInformation>& networkDevices,
const std::unordered_map<std::string, InferenceEngine::Parameter>& config,
const bool needPerfCounters = false);
MultiDeviceExecutableNetwork(const std::string& modelPath,
const InferenceEngine::CNNNetwork& network,
const std::vector<DeviceInformation>& metaDevices,
const std::string& strDevices,
MultiDeviceInferencePlugin* plugin,
const AutoContext& context,
const bool needPerfCounters = false);
void SetConfig(const std::map<std::string, InferenceEngine::Parameter> &config) override;
InferenceEngine::Parameter GetConfig(const std::string &name) const override;
InferenceEngine::Parameter GetMetric(const std::string &name) const override;
void run(InferenceEngine::Task inferTask) override;
InferenceEngine::IInferRequestInternal::Ptr CreateInferRequest() override;
InferenceEngine::IInferRequestInternal::Ptr CreateInferRequestImpl(InferenceEngine::InputsDataMap networkInputs,
InferenceEngine::OutputsDataMap networkOutputs) override;
InferenceEngine::IInferRequestInternal::Ptr CreateInferRequestImpl(const std::vector<std::shared_ptr<const ov::Node>>& inputs,
const std::vector<std::shared_ptr<const ov::Node>>& outputs) override;
std::shared_ptr<InferenceEngine::RemoteContext> GetContext() const override;
std::shared_ptr<InferenceEngine::ICore> GetCore() const;
~MultiDeviceExecutableNetwork() override;
// return true if current schedule success, fail otherwise
bool ScheduleToWorkerInferRequest(InferenceEngine::Task, DeviceName preferred_device = "");
static thread_local WorkerInferRequest* _thisWorkerInferRequest;
// have to use the const char* ptr rather than std::string due to a bug in old gcc versions,
// the bug is e.g. manifesting on the old CentOS (and it's 4.8.x gcc) used in our testing
// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=81880
static thread_local const char* _thisPreferredDeviceName;
mutable std::mutex _mutex;
std::vector<DeviceInformation> _devicePriorities;
const std::vector<DeviceInformation> _devicePrioritiesInitial;
DeviceMap<InferenceEngine::SoExecutableNetworkInternal> _networksPerDevice;
InferenceEngine::ThreadSafeQueue<InferenceEngine::Task> _inferPipelineTasks;
DeviceMap<std::unique_ptr<InferenceEngine::ThreadSafeQueue<InferenceEngine::Task>>> _inferPipelineTasksDeviceSpecific;
DeviceMap<NotBusyWorkerRequests> _idleWorkerRequests;
DeviceMap<std::vector<WorkerInferRequest>> _workerRequests;
std::unordered_map<std::string, InferenceEngine::Parameter> _config;
bool _needPerfCounters = false;
std::atomic_size_t _numRequestsCreated = {0};
using Ptr = std::shared_ptr<ExecutableNetwork>;
ExecutableNetwork(const Schedule::Ptr& schedule, const ScheduleContext::Ptr& sContext);
IInferPtr CreateInferRequest() override;
IInferPtr CreateInferRequestImpl(InferenceEngine::InputsDataMap networkInputs,
InferenceEngine::OutputsDataMap networkOutputs) override;
IInferPtr CreateInferRequestImpl(const std::vector<std::shared_ptr<const ov::Node>>& inputs,
const std::vector<std::shared_ptr<const ov::Node>>& outputs) override;
~ExecutableNetwork() override;
private:
void GenerateWorkers(const std::string& device, const InferenceEngine::SoExecutableNetworkInternal& executableNetwork);
void WaitActualNetworkReady() const;
void WaitFirstNetworkReady();
static bool RunPipelineTask(InferenceEngine::Task& inferPipelineTask,
NotBusyWorkerRequests& idleWorkerRequests,
const DeviceName& preferred_device);
void TryToLoadNetWork(AutoLoadContext& context,
const std::string& modelPath,
const InferenceEngine::CNNNetwork& network);
private:
std::shared_ptr<InferenceEngine::ICore> _core;
InferenceEngine::IStreamsExecutor::Ptr _executor;
MultiDeviceInferencePlugin* _multiPlugin = nullptr;
AutoContext _context;
bool _workModeIsAUTO = {false};
mutable std::once_flag _oc;
std::once_flag _firstLoadOC;
std::future<void> _firstLoadFuture;
std::promise<void> _firstLoadPromise;
mutable AutoLoadContext _loadContext[CONTEXTNUM];
mutable std::mutex _confMutex;
bool _exitFlag = {false};
const InferenceEngine::CNNNetwork _network;
unsigned int _cpuHelpInferCount = 0;
double _cpuHelpFps = 0.0;
Time _cpuHelpReleaseTime;
InferenceEngine::SoExecutableNetworkInternal _passthroughExeNet;
Schedule::Ptr _schedule;
ScheduleContext::Ptr _sContext;
std::once_flag _oc;
void SetExeNetworkForContext();
};
} // namespace MultiDevicePlugin
src/plugins/auto/infer_request.cpp
+5 -3
View File
@@ -18,7 +18,8 @@ MultiDeviceInferRequest::MultiDeviceInferRequest(const std::vector<std::shared_p
const std::vector<std::shared_ptr<const ov::Node>>& outputs,
const InferenceEngine::SoIInferRequestInternal & request_to_share_blobs_with,
InferenceEngine::RemoteContext::Ptr ctx)
: IInferRequestInternal(inputs, outputs) {
: IInferRequestInternal(inputs, outputs),
_sharedRequest(request_to_share_blobs_with) {
CreateInferRequest(request_to_share_blobs_with, ctx);
}
@@ -26,7 +27,8 @@ MultiDeviceInferRequest::MultiDeviceInferRequest(const InputsDataMap& networkI
const OutputsDataMap& networkOutputs,
const SoIInferRequestInternal & request_to_share_blobs_with,
InferenceEngine::RemoteContext::Ptr ctx)
: IInferRequestInternal(networkInputs, networkOutputs) {
: IInferRequestInternal(networkInputs, networkOutputs),
_sharedRequest(request_to_share_blobs_with) {
CreateInferRequest(request_to_share_blobs_with, ctx);
}
@@ -87,7 +89,7 @@ void MultiDeviceInferRequest::SetBlobsToAnotherRequest(const SoIInferRequestInte
}
std::map<std::string, InferenceEngine::InferenceEngineProfileInfo> MultiDeviceInferRequest::GetPerformanceCounts() const {
IE_THROW(NotImplemented);
return _perfMap;
}
void MultiDeviceInferRequest::InferImpl() {
src/plugins/auto/infer_request.hpp
+3 -1
View File
@@ -41,10 +41,12 @@ public:
void InferImpl() override;
// Multi-Device impl specific: sets the data (blobs from the device-less requests to the specific device request)
void SetBlobsToAnotherRequest(const InferenceEngine::SoIInferRequestInternal& req);
InferenceEngine::SoIInferRequestInternal& GetSharedRequest() { return _sharedRequest; }
std::map<std::string, InferenceEngine::InferenceEngineProfileInfo> _perfMap;
private:
void CreateInferRequest(const InferenceEngine::SoIInferRequestInternal& request_to_share_blobs_with,
InferenceEngine::RemoteContext::Ptr ctx);
InferenceEngine::SoIInferRequestInternal _sharedRequest;
};
} // namespace MultiDevicePlugin
src/plugins/auto/multi_executable_network.cpp
+140
View File
@@ -0,0 +1,140 @@
// Copyright (C) 2018-2022 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
///////////////////////////////////////////////////////////////////////////////////////////////////
#include "multi_executable_network.hpp"
#include "plugin.hpp"
// ------------------------------MultiExecutableNetwork----------------------------
namespace MultiDevicePlugin {
MultiExecutableNetwork::MultiExecutableNetwork(MultiScheduleContext::Ptr& context, const MultiSchedule::Ptr& schedule)
: ExecutableNetwork(schedule, context),
_multiSContext(context) {
}
MultiExecutableNetwork::~MultiExecutableNetwork() {}
std::shared_ptr<IE::RemoteContext> MultiExecutableNetwork::GetContext() const {
auto devices = [&] {
std::lock_guard<std::mutex> lock(_multiSContext->_mutex);
return _multiSContext->_devicePriorities;
}();
std::string devices_names;
for (auto&& device : devices) {
devices_names += device.deviceName + " ";
const auto& n = _multiSContext->_networksPerDevice.at(device.deviceName);
try {
return n->GetContext();
} catch (const IE::NotImplemented&) {}
}
IE_THROW(NotImplemented) <<
"None of the devices in the MULTI device has an associated remote context."
<< " Current list of devices allowed via the DEVICE_PRIORITIES config: " <<
devices_names;
}
void MultiExecutableNetwork::SetConfig(const
std::map<std::string, IE::Parameter>& config) {
auto priorities = config.find(ov::device::priorities.name());
if (priorities == config.end() || config.size() > 1) {
IE_THROW() <<
"The only config supported for the Network's SetConfig is MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES";
} else {
auto multiPlugin = std::dynamic_pointer_cast<MultiDeviceInferencePlugin>
(this->_plugin);
assert(multiPlugin != nullptr);
auto metaDevices = multiPlugin->ParseMetaDevices(
priorities->second.as<std::string>(), {});
if (std::any_of(metaDevices.begin(),
metaDevices.end(), [](const DeviceInformation & kvp) {
return kvp.numRequestsPerDevices != -1;
})) {
IE_THROW() << "You can only change device priorities but not number of requests"
<< " with the Network's SetConfig(MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES!";
}
{
std::lock_guard<std::mutex> lock{_multiSContext->_mutex};
for (auto&& device : metaDevices) {
if (_multiSContext->_networksPerDevice.find(device.deviceName) ==
_multiSContext->_networksPerDevice.end()) {
IE_THROW(NotFound) <<
"You can only change device priorities but not add new devices with"
<< " the Network's SetConfig(MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES. "
<< device.deviceName << " device was not in the original device list!";
}
}
_multiSContext->_devicePriorities = metaDevices;
// update value in config
_multiSContext->_config[IE::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES]
= priorities->second;
}
}
}
IE::Parameter MultiExecutableNetwork::GetConfig(const std::string& name) const {
{
auto it = _multiSContext->_config.find(name);
if (it != _multiSContext->_config.end()) {
return it->second;
}
}
// find config key among networks config keys
for (const auto& desc : _multiSContext->_networksPerDevice) {
const auto& execNetwork = desc.second;
auto param = execNetwork->GetMetric(METRIC_KEY(SUPPORTED_CONFIG_KEYS));
for (auto&& configKey : param.as<std::vector<std::string>>()) {
if (configKey == name) {
return execNetwork->GetConfig(configKey);
}
}
IE_THROW() << "Unsupported ExecutableNetwork config key: " << name;
}
IE_THROW(NotFound) << name << " not found in the ExecutableNetwork config";
}
IE::Parameter MultiExecutableNetwork::GetMetric(const std::string& name) const {
if (name == ov::supported_properties) {
return decltype(ov::supported_properties)::value_type {
// Metrics
ov::PropertyName{ov::supported_properties.name(), ov::PropertyMutability::RO},
ov::PropertyName{ov::model_name.name(), ov::PropertyMutability::RO},
ov::PropertyName{ov::optimal_number_of_infer_requests.name(), ov::PropertyMutability::RO},
// Configs
// device priority can be changed on-the-fly in MULTI
ov::PropertyName{ov::device::priorities.name(), ov::PropertyMutability::RW}
};
} else if (name == ov::optimal_number_of_infer_requests) {
unsigned int res = 0u;
for (auto n : _multiSContext->_networksPerDevice) {
try {
res += n.second->GetMetric(METRIC_KEY(
OPTIMAL_NUMBER_OF_INFER_REQUESTS)).as<unsigned int>();
} catch (const IE::Exception& iie) {
IE_THROW()
<< "Every device used with the Multi-Device should "
<< "support OPTIMAL_NUMBER_OF_INFER_REQUESTS ExecutableNetwork metric. "
<< "Failed to query the metric for the " << n.first << " with error:" <<
iie.what();
}
}
return decltype(ov::optimal_number_of_infer_requests)::value_type {res};
} else if (name == ov::model_name) {
auto it = _multiSContext->_networksPerDevice.begin();
IE_ASSERT(it != _multiSContext->_networksPerDevice.end());
return decltype(ov::model_name)::value_type {it->second->GetMetric(METRIC_KEY(NETWORK_NAME)).as<std::string>()};
} else if (name == METRIC_KEY(SUPPORTED_METRICS)) {
IE_SET_METRIC_RETURN(SUPPORTED_METRICS, {
METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS),
METRIC_KEY(SUPPORTED_METRICS),
METRIC_KEY(NETWORK_NAME),
METRIC_KEY(SUPPORTED_CONFIG_KEYS)
});
} else if (name == METRIC_KEY(SUPPORTED_CONFIG_KEYS)) {
std::vector<std::string> configKeys = {IE::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES};
IE_SET_METRIC_RETURN(SUPPORTED_CONFIG_KEYS, configKeys);
} else {
IE_THROW() << "Unsupported ExecutableNetwork metric key: " << name;
}
}
} // namespace MultiDevicePlugin
src/plugins/auto/multi_executable_network.hpp
+37
View File
@@ -0,0 +1,37 @@
// Copyright (C) 2018-2022 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
///////////////////////////////////////////////////////////////////////////////////////////////////
#pragma once
#include "executable_network.hpp"
#include "multi_schedule.hpp"
#include "bind_multi_schedule.hpp"
#ifdef MULTIUNITTEST
#define MOCKTESTMACRO virtual
#define MultiDevicePlugin MockMultiDevicePlugin
#else
#define MOCKTESTMACRO
#endif
namespace MultiDevicePlugin {
class MultiExecutableNetwork : public ExecutableNetwork {
friend IInferPtr MultiSchedule::CreateInferRequest();
friend IInferPtr BinderMultiSchedule::CreateInferRequest();
public:
using Ptr = std::shared_ptr<MultiExecutableNetwork>;
explicit MultiExecutableNetwork(MultiScheduleContext::Ptr& context, const MultiSchedule::Ptr& schedule);
void SetConfig(const std::map<std::string, IE::Parameter>& config) override;
IE::Parameter GetConfig(const std::string& name) const override;
IE::Parameter GetMetric(const std::string& name) const override;
std::shared_ptr<IE::RemoteContext> GetContext() const override;
~MultiExecutableNetwork() override;
private:
MultiScheduleContext::Ptr _multiSContext;
};
} // namespace MultiDevicePlugin
src/plugins/auto/multi_schedule.cpp
+285
View File
@@ -0,0 +1,285 @@
// Copyright (C) 2018-2022 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
///////////////////////////////////////////////////////////////////////////////////////////////////
#include "async_infer_request.hpp"
#include "plugin.hpp"
#include "multi_schedule.hpp"
#include "multi_executable_network.hpp"
// ------------------------------MultiSchedule----------------------------
namespace MultiDevicePlugin {
thread_local WorkerInferRequest* MultiSchedule::_thisWorkerInferRequest = nullptr;
// TODO: revert to the plain variable (see header file), when we moved to the next CentOS 8.x in our support matrix
thread_local const char* MultiSchedule::_thisPreferredDeviceName = "";
void MultiSchedule::init(const ScheduleContext::Ptr& sContext) {
_cpuHelpReleaseTime = std::chrono::steady_clock::now();
_multiSContext = std::dynamic_pointer_cast<MultiScheduleContext>(sContext);
for (auto&& networkValue : _multiSContext->_networksPerDevice) {
auto& device = networkValue.first;
auto& network = networkValue.second;
GenerateWorkers(device, network);
}
if (_multiSContext->_networksPerDevice.size() == 1)
_passthroughExeNet = _multiSContext->_networksPerDevice.begin()->second;
}
Pipeline MultiSchedule::GetPipeline(const IInferPtr& syncInferRequest, WorkerInferRequest** workerInferRequest) {
Pipeline pipeline = {
// if the request is coming with device-specific remote blobs make sure it is scheduled to the specific device only:
Stage {
/*TaskExecutor*/ std::make_shared<IE::ImmediateExecutor>(), /*task*/ [this, &syncInferRequest]() {
// by default, no preferred device:
_thisPreferredDeviceName = "";
auto execNetwork = _multiSContext->_executableNetwork.lock();
// if any input is remote (e.g. was set with SetBlob), let' use the corresponding device
for (const auto& it : execNetwork->GetInputsInfo()) {
auto b = syncInferRequest->GetBlob(it.first);
auto r = b->as<IE::RemoteBlob>();
if (r) {
const auto name = r->getDeviceName();
const auto res = std::find_if(
_multiSContext->_devicePrioritiesInitial.cbegin(),
_multiSContext->_devicePrioritiesInitial.cend(),
[&name](const MultiDevicePlugin::DeviceInformation & d) {
return (d.defaultDeviceID.empty() ? d.deviceName : (d.deviceName + "." +
d.defaultDeviceID)) == name;
});
if (_multiSContext->_devicePrioritiesInitial.cend() == res) {
IE_THROW() <<
"None of the devices (for which current MULTI-device configuration was "
"initialized) supports a remote blob created on the device named " << name;
} else {
// it is ok to take the c_str() here (as pointed in the executable_network.hpp we need to use const char*)
// as the original strings are from the "persistent" vector (with the right lifetime)
_thisPreferredDeviceName = res->deviceName.c_str();
break;
}
}
}
}},
// as the scheduling algo may select any device, this stage accepts the scheduling decision (actual workerRequest)
// then sets the device-agnostic blobs to the actual (device-specific) request
Stage {
/*TaskExecutor*/std::dynamic_pointer_cast<IE::ITaskExecutor>(shared_from_this()), /*task*/ [this, &syncInferRequest, workerInferRequest]() {
*workerInferRequest = _thisWorkerInferRequest;
auto multiSyncInferRequest = std::dynamic_pointer_cast<MultiDeviceInferRequest>(syncInferRequest);
multiSyncInferRequest->SetBlobsToAnotherRequest(_thisWorkerInferRequest->_inferRequest);
INFO_RUN([workerInferRequest]() {
(*workerInferRequest)->_startTimes.push_back(std::move(std::chrono::steady_clock::now()));
});
}},
// final task in the pipeline:
Stage {
/*TaskExecutor*/std::make_shared<ThisRequestExecutor>(workerInferRequest), /*task*/ [this, &syncInferRequest, workerInferRequest]() {
if (nullptr != (*workerInferRequest)->_exceptionPtr) {
std::rethrow_exception((*workerInferRequest)->_exceptionPtr);
}
if (_multiSContext->_needPerfCounters) {
auto multiSyncInferRequest = std::dynamic_pointer_cast<MultiDeviceInferRequest>
(syncInferRequest);
multiSyncInferRequest->_perfMap =
(*workerInferRequest)->_inferRequest->GetPerformanceCounts();
}
INFO_RUN([workerInferRequest]() {
(*workerInferRequest)->_endTimes.push_back(std::move(std::chrono::steady_clock::now()));
});
}}
};
return pipeline;
}
void MultiSchedule::GenerateWorkers(const std::string& device,
const SoExecNetwork& executableNetwork) {
auto itNumRequests = std::find_if(_multiSContext->_devicePriorities.cbegin(), _multiSContext->_devicePriorities.cend(),
[&device](const DeviceInformation & d) {
return d.deviceName == device;
});
unsigned int optimalNum = 0;
try {
optimalNum = executableNetwork->GetMetric(METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS)).as<unsigned int>();
} catch (const IE::Exception& iie) {
IE_THROW()
<< "Every device used with the Multi-Device should "
<< "support OPTIMAL_NUMBER_OF_INFER_REQUESTS ExecutableNetwork metric. "
<< "Failed to query the metric for the " << device << " with error:" <<
iie.what();
}
const auto numRequests = (_multiSContext->_devicePriorities.end() == itNumRequests ||
itNumRequests->numRequestsPerDevices == -1) ? optimalNum : itNumRequests->numRequestsPerDevices;
auto& workerRequests = _workerRequests[device];
auto& idleWorkerRequests = _idleWorkerRequests[device];
workerRequests.resize(numRequests);
_inferPipelineTasksDeviceSpecific[device] = std::unique_ptr<IE::ThreadSafeQueue<IE::Task>>(new IE::ThreadSafeQueue<IE::Task>);
auto* idleWorkerRequestsPtr = &(idleWorkerRequests);
idleWorkerRequests.set_capacity(numRequests);
int num = 0;
for (auto&& workerRequest : workerRequests) {
workerRequest._inferRequest = {executableNetwork->CreateInferRequest(), executableNetwork._so};
auto* workerRequestPtr = &workerRequest;
workerRequestPtr->_index = num++;
IE_ASSERT(idleWorkerRequests.try_push(workerRequestPtr) == true);
workerRequest._inferRequest->SetCallback(
[workerRequestPtr, this, device, idleWorkerRequestsPtr](std::exception_ptr exceptionPtr) mutable {
IdleGuard<NotBusyWorkerRequests> idleGuard{workerRequestPtr, *idleWorkerRequestsPtr};
workerRequestPtr->_exceptionPtr = exceptionPtr;
{
auto capturedTask = std::move(workerRequestPtr->_task);
capturedTask();
}
// try to return the request to the idle list (fails if the overall object destruction has began)
if (idleGuard.Release()->try_push(workerRequestPtr)) {
// let's try to pop a task, as we know there is at least one idle request, schedule if succeeded
// if no device-agnostic tasks, let's try pop the device specific task, schedule if succeeded
IE::Task t;
if (_inferPipelineTasks.try_pop(t)) {
ScheduleToWorkerInferRequest(std::move(t));
} else if (_inferPipelineTasksDeviceSpecific[device]->try_pop(t)) {
ScheduleToWorkerInferRequest(std::move(t), device);
}
}
});
}
}
bool MultiSchedule::ScheduleToWorkerInferRequest(IE::Task inferPipelineTask, DeviceName preferred_device) {
std::vector<DeviceInformation> devices;
devices = [&] {
std::lock_guard<std::mutex> lock(_multiSContext->_mutex);
return _multiSContext->_devicePriorities;
}();
for (auto&& device : devices) {
if (!preferred_device.empty() && (device.deviceName != preferred_device)) {
continue;
}
if (RunPipelineTask(inferPipelineTask, _idleWorkerRequests[device.deviceName], preferred_device)) {
return true;
}
}
// no vacant requests this time, storing the task to the respective queue
if (!preferred_device.empty()) {
_inferPipelineTasksDeviceSpecific[preferred_device]->push(std::move(
inferPipelineTask));
} else {
_inferPipelineTasks.push(std::move(inferPipelineTask));
}
return false;
}
bool MultiSchedule::RunPipelineTask(IE::Task& inferPipelineTask,
NotBusyWorkerRequests& idleWorkerRequests,
const DeviceName& preferred_device) {
WorkerInferRequest* workerRequestPtr = nullptr;
if (idleWorkerRequests.try_pop(workerRequestPtr)) {
IdleGuard<NotBusyWorkerRequests> idleGuard{workerRequestPtr, idleWorkerRequests};
_thisWorkerInferRequest = workerRequestPtr;
{
auto capturedTask = std::move(inferPipelineTask);
capturedTask();
}
idleGuard.Release();
return true;
}
return false;
}
void MultiSchedule::run(IE::Task inferPipelineTask) {
ScheduleToWorkerInferRequest(std::move(inferPipelineTask), _thisPreferredDeviceName);
}
MultiSchedule::~MultiSchedule() {
{
std::lock_guard<std::mutex> lock(_multiSContext->_mutex);
_multiSContext->_devicePriorities.clear();
}
/* NOTE: The only threads that use `MultiSchedule` worker infer requests' threads.
* But AsyncInferRequest destructor should wait for all asynchronous tasks by the request
*/
for (auto&& idleWorker : _idleWorkerRequests) {
// stop accepting any idle requests back (for re-scheduling)
idleWorker.second.set_capacity(0);
}
INFO_RUN([this] {
for (auto&& _workerRequest : _workerRequests) {
std::list<Time> reqAllStartTimes;
std::list<Time> reqAllEndTimes;
for (auto& request : _workerRequest.second) {
reqAllStartTimes.splice(reqAllStartTimes.end(), request._startTimes);
reqAllEndTimes.splice(reqAllEndTimes.end(), request._endTimes);
}
unsigned int count = reqAllStartTimes.size();
IE_ASSERT(count == reqAllEndTimes.size());
reqAllStartTimes.sort(std::less<Time>());
reqAllEndTimes.sort(std::less<Time>());
if (_workerRequest.first == "CPU_HELP") {
LOG_INFO("[AUTOPLUGIN]CPU_HELP:infer:%ld", _cpuHelpInferCount + count);
if (_cpuHelpFps > 0.0) {
LOG_INFO("[AUTOPLUGIN]CPU_HELP:fps:%lf", _cpuHelpFps);
} else if (count >= 1) {
std::chrono::duration<double, std::milli> durtation =
reqAllEndTimes.back() - reqAllStartTimes.front();
LOG_INFO("[AUTOPLUGIN]CPU_HELP:fps:%lf", count * 1000 / durtation.count());
}
} else {
LOG_INFO("[AUTOPLUGIN]%s:infer:%ld", _workerRequest.first.c_str(), count);
auto n = reqAllStartTimes.size();
Time time;
while (!reqAllStartTimes.empty()) {
time = reqAllStartTimes.front();
if (time < _cpuHelpReleaseTime) {
reqAllStartTimes.pop_front();
n--;
} else {
break;
}
}
if (n >= 1) {
std::chrono::duration<double, std::milli> durtation =
reqAllEndTimes.back() - time;
LOG_INFO("[AUTOPLUGIN]%s:fps:%lf", _workerRequest.first.c_str(),
n * 1000 / durtation.count());
}
}
}
});
_workerRequests.clear();
}
IInferPtr MultiSchedule::CreateInferRequestImpl(
const std::vector<std::shared_ptr<const ov::Node>>& inputs,
const std::vector<std::shared_ptr<const ov::Node>>& outputs) {
SoInfer request_to_share_blobs_with;
IE::RemoteContext::Ptr ctx = nullptr;
return std::make_shared<MultiDeviceInferRequest>(inputs, outputs, request_to_share_blobs_with);
}
IInferPtr MultiSchedule::CreateInferRequestImpl(IE::InputsDataMap networkInputs,
IE::OutputsDataMap networkOutputs) {
SoInfer request_to_share_blobs_with;
IE::RemoteContext::Ptr ctx = nullptr;
return std::make_shared<MultiDeviceInferRequest>(networkInputs, networkOutputs, request_to_share_blobs_with);
}
IInferPtr MultiSchedule::CreateInferRequest() {
auto execNetwork = std::dynamic_pointer_cast<MultiExecutableNetwork>(
_multiSContext->_executableNetwork.lock());
if (_passthroughExeNet) {
auto res = _passthroughExeNet->CreateInferRequest();
res->setPointerToExecutableNetworkInternal(execNetwork);
return res;
}
IInferPtr syncRequestImpl;
if (_multiSContext->_core && _multiSContext->_core->isNewAPI())
syncRequestImpl = CreateInferRequestImpl(execNetwork->_parameters, execNetwork->_results);
if (!syncRequestImpl)
syncRequestImpl = CreateInferRequestImpl(execNetwork->_networkInputs, execNetwork->_networkOutputs);
syncRequestImpl->setPointerToExecutableNetworkInternal(execNetwork);
return std::make_shared<AsyncInferRequest>(shared_from_this(),
syncRequestImpl,
execNetwork->_callbackExecutor);
}
} // namespace MultiDevicePlugin
src/plugins/auto/multi_schedule.hpp
+65
View File
@@ -0,0 +1,65 @@
// Copyright (C) 2018-2022 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
///////////////////////////////////////////////////////////////////////////////////////////////////
#pragma once
#include "schedule.hpp"
#ifdef MULTIUNITTEST
#define MOCKTESTMACRO virtual
#define MultiDevicePlugin MockMultiDevicePlugin
#else
#define MOCKTESTMACRO
#endif
namespace MultiDevicePlugin {
struct ThisRequestExecutor : public IE::ITaskExecutor {
explicit ThisRequestExecutor(WorkerInferRequest** ptr): _workptrptr{ptr} {}
void run(IE::Task task) override {
(*_workptrptr)->_task = std::move(task);
(*_workptrptr)->_inferRequest->StartAsync();
};
WorkerInferRequest** _workptrptr = nullptr;
};
class MultiSchedule : public Schedule, public IE::ITaskExecutor {
public:
using Ptr = std::shared_ptr<MultiSchedule>;
IInferPtr CreateInferRequest() override;
IInferPtr CreateInferRequestImpl(IE::InputsDataMap networkInputs, IE::OutputsDataMap networkOutputs) override;
IE::IInferRequestInternal::Ptr CreateInferRequestImpl(const std::vector<std::shared_ptr<const ov::Node>>& inputs,
const std::vector<std::shared_ptr<const ov::Node>>& outputs) override;
void run(IE::Task inferTask) override;
void init(const ScheduleContext::Ptr& sContext) override;
Pipeline GetPipeline(const IInferPtr& syncRequestImpl, WorkerInferRequest** WorkerInferRequest) override;
virtual ~MultiSchedule();
public:
static thread_local WorkerInferRequest* _thisWorkerInferRequest;
// have to use the const char* ptr rather than std::string due to a bug in old gcc versions,
// the bug is e.g. manifesting on the old CentOS (and it's 4.8.x gcc) used in our testing
// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=81880
static thread_local const char* _thisPreferredDeviceName;
protected:
virtual void GenerateWorkers(const std::string& device, const IE::SoExecutableNetworkInternal& executableNetwork);
static bool RunPipelineTask(IE::Task& inferPipelineTask, NotBusyWorkerRequests& idleWorkerRequests, const DeviceName& preferred_device);
virtual bool ScheduleToWorkerInferRequest(IE::Task, DeviceName preferred_device = "");
protected:
IE::ThreadSafeQueue<IE::Task> _inferPipelineTasks;
DeviceMap<std::unique_ptr<IE::ThreadSafeQueue<IE::Task>>> _inferPipelineTasksDeviceSpecific;
DeviceMap<NotBusyWorkerRequests> _idleWorkerRequests;
DeviceMap<std::vector<WorkerInferRequest>> _workerRequests;
mutable std::mutex _mutex;
std::atomic_size_t _numRequestsCreated = {0};
MultiScheduleContext::Ptr _multiSContext;
SoExecNetwork _passthroughExeNet;
Time _cpuHelpReleaseTime;
unsigned int _cpuHelpInferCount = 0;
double _cpuHelpFps = 0.0;
};
} // namespace MultiDevicePlugin
src/plugins/auto/plugin.cpp
+35 -17
View File
@@ -21,6 +21,11 @@
#include <ie_algorithm.hpp>
#include <ie_icore.hpp>
#include <ie_ngraph_utils.hpp>
#include "bind_multi_schedule.hpp"
#include "multi_executable_network.hpp"
#include "auto_schedule.hpp"
#include "auto_executable_network.hpp"
#include "itt.hpp"
// ------------------------------MultiDeviceInferencePlugin----------------------------
namespace MultiDevicePlugin {
@@ -223,9 +228,9 @@ InferenceEngine::Parameter MultiDeviceInferencePlugin::GetConfig(const std::stri
}
void MultiDeviceInferencePlugin::SetConfig(const std::map<std::string, std::string> & config) {
AutoContext context;
auto autoSContext = std::make_shared<AutoScheduleContext>();
std::map<std::string, std::string> filterConfig;
CheckConfig(config, context, filterConfig);
CheckConfig(config, autoSContext, filterConfig);
for (auto && kvp : config) {
const auto& name = kvp.first;
_config[name] = kvp.second;
@@ -323,9 +328,9 @@ IExecutableNetworkInternal::Ptr MultiDeviceInferencePlugin::LoadNetworkImpl(cons
// and set filter configure
OV_ITT_SCOPED_TASK(itt::domains::MULTIPlugin, "MultiDeviceInferencePlugin::LoadNetworkImpl::AutoMode");
AutoContext context;
auto autoSContext = std::make_shared<AutoScheduleContext>();
std::map<std::string, std::string> filterConfig;
CheckConfig(fullConfig, context, filterConfig);
CheckConfig(fullConfig, autoSContext, filterConfig);
// filter the device that supports filter configure
auto strDevices = GetDeviceList(fullConfig);
auto metaDevices = ParseMetaDevices(strDevices, fullConfig);
@@ -379,9 +384,15 @@ IExecutableNetworkInternal::Ptr MultiDeviceInferencePlugin::LoadNetworkImpl(cons
strDevices += ((iter + 1) == supportDevices.end()) ? "" : ",";
LOG_INFO("[AUTOPLUGIN]:device:%s, priority:%ld", iter->deviceName.c_str(), iter->devicePriority);
}
return std::make_shared<MultiDeviceExecutableNetwork>(clonedModelPath, clonedNetwork, supportDevices, strDevices,
this, context, context.needPerfCounters);
autoSContext->_modelPath = clonedModelPath;
// clone the network, in case of reshape conflict
autoSContext->_network = clonedNetwork;
autoSContext->_devicePriorities = supportDevices;
autoSContext->_devicePrioritiesInitial = supportDevices;
autoSContext->_strDevices = strDevices;
autoSContext->_plugin = this;
autoSContext->_core = GetCore();
return std::make_shared<AutoExecutableNetwork>(autoSContext, std::make_shared<AutoSchedule>());
}
OV_ITT_SCOPED_TASK(itt::domains::MULTIPlugin, "MultiDeviceInferencePlugin::LoadNetworkImpl:MultiMode");
if (priorities == fullConfig.end()) {
@@ -397,6 +408,9 @@ IExecutableNetworkInternal::Ptr MultiDeviceInferencePlugin::LoadNetworkImpl(cons
std::once_flag readNetworkFlag;
for (auto& p : metaDevices) {
loads.push_back([&]() {
auto tmpiter = fullConfig.find(CONFIG_KEY(ALLOW_AUTO_BATCHING));
if (tmpiter != fullConfig.end())
p.config.insert({tmpiter->first, tmpiter->second});
const auto &deviceName = p.deviceName;
const auto &deviceConfig = p.config;
SoExecutableNetworkInternal exec_net;
@@ -437,10 +451,14 @@ IExecutableNetworkInternal::Ptr MultiDeviceInferencePlugin::LoadNetworkImpl(cons
}
// MULTI can enable the perf counters only if all devices support/enable that
bool enablePerfCounters = num_plugins_supporting_perf_counters == executableNetworkPerDevice.size();
auto impl = std::make_shared<MultiDeviceExecutableNetwork>(executableNetworkPerDevice,
metaDevices,
multiNetworkConfig,
enablePerfCounters);
auto multiSContext = std::make_shared<MultiScheduleContext>();
multiSContext->_devicePriorities = metaDevices;
multiSContext->_devicePrioritiesInitial = metaDevices;
multiSContext->_networksPerDevice = executableNetworkPerDevice;
multiSContext->_config = multiNetworkConfig;
multiSContext->_needPerfCounters = enablePerfCounters;
multiSContext->_core = GetCore();
auto impl = std::make_shared<MultiExecutableNetwork>(multiSContext, std::make_shared<MultiSchedule>());
if (!modelPath.empty()) {
SetExeNetworkInfo(impl,
executableNetworkPerDevice.begin()->second->GetInputsInfo(),
@@ -726,17 +744,17 @@ std::string MultiDeviceInferencePlugin::GetDeviceList(const std::map<std::string
}
void MultiDeviceInferencePlugin::CheckConfig(const std::map<std::string, std::string>& config,
AutoContext& context, std::map<std::string, std::string>& filterConfig) {
AutoScheduleContext::Ptr& context, std::map<std::string, std::string>& filterConfig) {
// TODO need to optimize this code, too much duplicated code
const auto perf_hints_configs = PerfHintsConfig::SupportedKeys();
for (auto&& kvp : config) {
if (kvp.first == ov::enable_profiling) {
if (kvp.second == PluginConfigParams::YES) {
context.needPerfCounters = true;
context->_needPerfCounters = true;
filterConfig.insert({kvp.first, kvp.second});
} else if (kvp.second == PluginConfigParams::NO) {
context.needPerfCounters = false;
context->_needPerfCounters = false;
} else {
IE_THROW() << "Unsupported config value: " << kvp.second
<< " for key: " << kvp.first;
@@ -774,19 +792,19 @@ void MultiDeviceInferencePlugin::CheckConfig(const std::map<std::string, std::st
IE_THROW() << "Unsupported config value: " << kvp.second
<< " for key: " << kvp.first;
}
context.modelPriority = priority;
context->_modelPriority = priority;
} catch(...) {
IE_THROW() << "Unsupported config value: " << kvp.second
<< " for key: " << kvp.first;
}
} else if (kvp.first == ov::hint::allow_auto_batching) {
if (kvp.second == PluginConfigParams::NO) {
context.batchingDisabled = true;
context->_batchingDisabled = true;
continue;
}
} else if (std::find(perf_hints_configs.begin(), perf_hints_configs.end(), kvp.first) != perf_hints_configs.end()) {
PerfHintsConfig::CheckConfigAndValue(kvp);
context.performanceHint = kvp.second;
context->_performanceHint = kvp.second;
} else if (supported_configKeys.end() == std::find(supported_configKeys.begin(), supported_configKeys.end(), kvp.first)) {
IE_THROW() << "Unsupported config key: " << kvp.first;
} else if (kvp.first.find("AUTO_") == 0) {
src/plugins/auto/plugin.hpp
+2 -2
View File
@@ -12,8 +12,8 @@
#include <cpp_interfaces/interface/ie_iplugin_internal.hpp>
#include <cpp_interfaces/interface/ie_internal_plugin_config.hpp>
#include "executable_network.hpp"
#include "utils/log_util.hpp"
#include "common.hpp"
#ifdef MULTIUNITTEST
#define MOCKTESTMACRO virtual
@@ -65,7 +65,7 @@ private:
InferenceEngine::CNNNetwork network,
const std::map<std::string, std::string>& config,
const std::string &networkPrecision = METRIC_VALUE(FP32));
static void CheckConfig(const std::map<std::string, std::string>& config, AutoContext& context,
static void CheckConfig(const std::map<std::string, std::string>& config, AutoScheduleContext::Ptr& context,
std::map<std::string, std::string>& filterConfig);
std::vector<DeviceInformation> FilterDevice(const std::vector<DeviceInformation>& metaDevices,
const std::map<std::string, std::string>& config);
src/plugins/auto/schedule.hpp
+33
View File
@@ -0,0 +1,33 @@
// Copyright (C) 2018-2022 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
///////////////////////////////////////////////////////////////////////////////////////////////////
#pragma once
#include "common.hpp"
#ifdef MULTIUNITTEST
#define MOCKTESTMACRO virtual
#define MultiDevicePlugin MockMultiDevicePlugin
#else
#define MOCKTESTMACRO
#endif
namespace MultiDevicePlugin {
using Stage = std::pair<IE::ITaskExecutor::Ptr, IE::Task>;
using Pipeline = std::vector<Stage>;
class Schedule : public std::enable_shared_from_this<Schedule> {
public:
using Ptr = std::shared_ptr<Schedule>;
virtual IInferPtr CreateInferRequest() = 0;
virtual IInferPtr CreateInferRequestImpl(IE::InputsDataMap networkInputs, IE::OutputsDataMap networkOutputs) = 0;
virtual IInferPtr CreateInferRequestImpl(const std::vector<std::shared_ptr<const ov::Node>>& inputs,
const std::vector<std::shared_ptr<const ov::Node>>& outputs) = 0;
//virtual void release() = 0;
virtual void init(const ScheduleContext::Ptr& context) = 0;
virtual Pipeline GetPipeline(const IInferPtr& syncRequestImpl, WorkerInferRequest** WorkerInferRequest) = 0;
virtual ~Schedule() = default;
};
} // namespace MultiDevicePlugin