IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Unify code path for MULTI and AUTO CTPUT hint (#16349)

Browse Source 
[MULTI] pass through to AUTO with CTPUT hint
After this change
-- MULTI doesn't support setting infer request via CPU(4),GPU(8).
-- MULTI doesn't support CompiledModel::set_property() and ExecutableNetwork::GetConfig().
This commit is contained in:
guozhong wang 2023-04-01 00:40:41 +08:00 committed by GitHub
parent 9a5a8f6abc
commit 341217de99
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
24 changed files with 281 additions and 640 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
src/bindings/c/tests/ie_c_api_test.cpp
View File

@ -954,36 +954,6 @@ TEST_P(ie_c_api_test, ie_exec_network_get_config) {
ie_core_free(&core);
}
TEST_P(ie_c_api_test, ie_exec_network_set_config) {
ie_core_t *core = nullptr;
IE_ASSERT_OK(ie_core_create("", &core));
ASSERT_NE(nullptr, core);
ie_param_t param;
if (ie_core_get_metric(core, "GPU", "AVAILABLE_DEVICES", &param) != IEStatusCode::OK) {
ie_core_free(&core);
GTEST_SKIP();
}
ie_network_t *network = nullptr;
IE_EXPECT_OK(ie_core_read_network(core, xml_file_name.c_str(), bin_file_name.c_str(), &network));
EXPECT_NE(nullptr, network);
const char *device_name = "MULTI:GPU,CPU";
ie_config_t config = {nullptr, nullptr, nullptr};
ie_executable_network_t *exe_network = nullptr;
IE_EXPECT_OK(ie_core_load_network(core, network, device_name, &config, &exe_network));
EXPECT_NE(nullptr, exe_network);
ie_config_t config_param = {"MULTI_DEVICE_PRIORITIES", "GPU,CPU", nullptr};
IE_EXPECT_OK(ie_exec_network_set_config(exe_network, &config_param));
ie_exec_network_free(&exe_network);
ie_network_free(&network);
ie_core_free(&core);
ie_param_free(&param);
}
TEST_P(ie_c_api_test, ie_exec_network_get_metric) {
ie_core_t *core = nullptr;
IE_ASSERT_OK(ie_core_create("", &core));

39
src/bindings/c/tests/ov_compiled_model_test.cpp
View File

@ -121,45 +121,6 @@ TEST_P(ov_compiled_model_test, ov_compiled_model_input_by_name) {
ov_core_free(core);
}
TEST_P(ov_compiled_model_test, set_and_get_property) {
// It seems that all set_property() for CPU plugin are not implement in compiled_model.
auto device_name = "MULTI:GPU,CPU";
ov_core_t* core = nullptr;
OV_EXPECT_OK(ov_core_create(&core));
EXPECT_NE(nullptr, core);
char* info = nullptr;
const char* key_0 = ov_property_key_available_devices;
if (ov_core_get_property(core, "GPU", key_0, &info) != ov_status_e::OK) {
ov_core_free(core);
GTEST_SKIP();
}
ov_model_t* model = nullptr;
OV_EXPECT_OK(ov_core_read_model(core, xml_file_name.c_str(), bin_file_name.c_str(), &model));
EXPECT_NE(nullptr, model);
ov_compiled_model_t* compiled_model = nullptr;
OV_EXPECT_OK(ov_core_compile_model(core, model, device_name, 0, &compiled_model));
EXPECT_NE(nullptr, compiled_model);
const char* key_1 = ov_property_key_device_priorities;
const char* value_1 = "GPU,CPU";
OV_EXPECT_OK(ov_compiled_model_set_property(compiled_model, key_1, value_1));
char* result = nullptr;
OV_EXPECT_OK(ov_compiled_model_get_property(compiled_model, key_1, &result));
EXPECT_STREQ(value_1, result);
ov_free(result);
const char* key_2 = ov_property_key_supported_properties;
OV_EXPECT_OK(ov_compiled_model_get_property(compiled_model, key_2, &result));
ov_free(result);
ov_compiled_model_free(compiled_model);
ov_model_free(model);
ov_core_free(core);
}
TEST_P(ov_compiled_model_test, get_property) {
auto device_name = GetParam();
ov_core_t* core = nullptr;

27
src/plugins/auto/auto_executable_network.cpp
View File

@ -17,6 +17,14 @@ AutoExecutableNetwork::AutoExecutableNetwork(AutoScheduleContext::Ptr& context,
}
std::shared_ptr<IE::RemoteContext> AutoExecutableNetwork::GetContext() const {
if (_autoSchedule->_pCTPUTLoadContext) {
for (size_t i = 0; i < _autoSchedule->_nCTputDeviceNums; i++) {
if (_autoSchedule->_pCTPUTLoadContext[i].isAlready) {
return _autoSchedule->_pCTPUTLoadContext[i].executableNetwork->GetContext();
}
}
return nullptr;
} else {
std::lock_guard<std::mutex> lock(_autoSContext->_fallbackMutex);
if (_autoSchedule->_loadContext[FALLBACKDEVICE].isAlready) {
return _autoSchedule->_loadContext[FALLBACKDEVICE].executableNetwork->GetContext();
@ -24,6 +32,7 @@ std::shared_ptr<IE::RemoteContext> AutoExecutableNetwork::GetContext() const {
_autoSchedule->WaitActualNetworkReady();
return _autoSchedule->_loadContext[ACTUALDEVICE].executableNetwork->GetContext();
}
}
}
void AutoExecutableNetwork::SetConfig(const std::map<std::string, IE::Parameter>
@ -82,11 +91,11 @@ IE::Parameter AutoExecutableNetwork::GetMetric(const std::string& name) const {
all_devices[context.deviceInfo.deviceName] = device_properties;
};
if (_autoSchedule->_pCTPUTLoadContext) {
// need lock for inference failure
std::lock_guard<std::mutex> lock(_autoSContext->_fallbackMutex);
auto load_count = _autoSContext->_devicePriorities.size();
for (size_t i = 0; i < load_count; i++)
for (size_t i = 0; i < _autoSchedule->_nCTputDeviceNums; i++) {
if (_autoSchedule->_pCTPUTLoadContext[i].isAlready) {
get_device_supported_metrics(_autoSchedule->_pCTPUTLoadContext[i]);
}
}
} else {
{
std::lock_guard<std::mutex> lock(_autoSContext->_fallbackMutex);
@ -118,11 +127,13 @@ IE::Parameter AutoExecutableNetwork::GetMetric(const std::string& name) const {
if (_autoSchedule->_pCTPUTLoadContext) {
std::lock_guard<std::mutex> lock(_autoSContext->_fallbackMutex);
unsigned int res = 0u;
auto load_count = _autoSContext->_devicePriorities.size();
for (size_t i = 0; i < load_count; i++) {
for (size_t i = 0; i < _autoSchedule->_nCTputDeviceNums; i++) {
try {
res += (_autoSchedule->_pCTPUTLoadContext[i]).executableNetwork->GetMetric(
METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS)).as<unsigned int>();
if (_autoSchedule->_pCTPUTLoadContext[i].isAlready) {
res += (_autoSchedule->_pCTPUTLoadContext[i])
.executableNetwork->GetMetric(METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS))
.as<unsigned int>();
}
} catch (const IE::Exception& iie) {
IE_THROW()
<< "Every device used in cumulative mode should "

37
src/plugins/auto/auto_schedule.cpp
View File

@ -234,9 +234,7 @@ void AutoSchedule::init(const ScheduleContext::Ptr& sContext) {
bool isCumulative =
(_autoSContext->_performanceHint == IE::PluginConfigParams::CUMULATIVE_THROUGHPUT) ? true : false;
if (isCumulative) {
std::list<DeviceInformation> validDevices =
_autoSContext->_plugin->GetValidDevice(_autoSContext->_devicePriorities,
_loadContext[ACTUALDEVICE].networkPrecision);
const auto& validDevices = _autoSContext->_devicePriorities;
// When the hint is ctput and there is only one device, the single-device logic is used
if (validDevices.size() == 1) {
_loadContext[ACTUALDEVICE].deviceInfo = validDevices.front();
@ -244,14 +242,10 @@ void AutoSchedule::init(const ScheduleContext::Ptr& sContext) {
IE::PluginConfigParams::THROUGHPUT;
} else if (validDevices.size() > 1) {
_loadContext[ACTUALDEVICE].isEnabled = false;
_autoSContext->_devicePriorities.clear();
std::copy(std::begin(validDevices),
std::end(validDevices),
std::back_inserter(_autoSContext->_devicePriorities));
// Total number of devices in CTPUT
auto nCTputDeviceNums = validDevices.size();
_nCTputDeviceNums = validDevices.size();
// Generate contexts for loading each device
_pCTPUTLoadContext.reset(new AutoLoadContext[nCTputDeviceNums]);
_pCTPUTLoadContext.reset(new AutoLoadContext[_nCTputDeviceNums]);
int idx = 0;
DeviceInformation cpuDeviceInformation;
for (auto& device : validDevices) {
@ -272,6 +266,10 @@ void AutoSchedule::init(const ScheduleContext::Ptr& sContext) {
IE::PluginConfigParams::THROUGHPUT;
}
}
if (_autoSContext->_LogTag == "MULTI") {
// MULTI's performance hint always is tput
_autoSContext->_performanceHint = IE::PluginConfigParams::THROUGHPUT;
}
} else {
_loadContext[ACTUALDEVICE].deviceInfo =
_autoSContext->_plugin->SelectDevice(_autoSContext->_devicePriorities,
@ -388,12 +386,12 @@ void AutoSchedule::init(const ScheduleContext::Ptr& sContext) {
std::vector<Task> otherDevicesloads;
std::vector<Task> cpuLoads;
if (_pCTPUTLoadContext) {
for (size_t i = 0; i < _autoSContext->_devicePriorities.size(); i++) {
for (size_t i = 0; i < _nCTputDeviceNums; i++) {
auto* contextPtr = &_pCTPUTLoadContext[i];
auto modelPath = _autoSContext->_modelPath;
auto network = _autoSContext->_network;
_pCTPUTLoadContext[i].task = std::bind(loadDeviceTask, contextPtr, modelPath, network, isCumulative);
if (i == _autoSContext->_devicePriorities.size() - 1 &&
if (i == _nCTputDeviceNums - 1 &&
_pCTPUTLoadContext[i].deviceInfo.deviceName.find("CPU") != std::string::npos) {
cpuLoads.push_back(_pCTPUTLoadContext[i].task);
} else {
@ -518,7 +516,7 @@ void AutoSchedule::init(const ScheduleContext::Ptr& sContext) {
_passthroughExeNet = _loadContext[ACTUALDEVICE].executableNetwork;
}
}
WaitFirstNetworkReady();
_autoSContext->_hwExecutableNetwork = WaitFirstNetworkReady();
}
void AutoSchedule::TryToLoadNetWork(AutoLoadContext& context, const std::string& modelPath, const IE::CNNNetwork& network, bool isCumulative) {
@ -618,7 +616,7 @@ void AutoSchedule::TryToLoadNetWork(AutoLoadContext& context, const std::string&
TryToLoadNetWork(context, modelPath, network, isCumulative);
}
void AutoSchedule::WaitFirstNetworkReady() {
SoExecNetwork AutoSchedule::WaitFirstNetworkReady() {
if (_firstLoadFuture.valid()) {
// wait for the first loading finished
_firstLoadFuture.wait();
@ -626,7 +624,7 @@ void AutoSchedule::WaitFirstNetworkReady() {
// check if there is any device that have loaded network successfully
for (int i = CONTEXTNUM - 2; i >= 0; i--) {
if (_loadContext[i].isEnabled && _loadContext[i].isAlready) {
return;
return _loadContext[i].executableNetwork;
}
}
// the first loading is failed, wait for another loading
@ -635,7 +633,7 @@ void AutoSchedule::WaitFirstNetworkReady() {
_loadContext[i].future.wait();
// check if loading is successful
if (_loadContext[i].isAlready) {
return;
return _loadContext[i].executableNetwork;
}
}
}
@ -646,17 +644,21 @@ void AutoSchedule::WaitFirstNetworkReady() {
}
}
// devices loaded successfully in CTPUT
SoExecNetwork execNetwork;
if (_pCTPUTLoadContext) {
int nLoadSucNums = 0;
for (size_t i = 0; i < _autoSContext->_devicePriorities.size(); i++) {
for (size_t i = 0; i < _nCTputDeviceNums; i++) {
// check if device loaded successfully
if (_pCTPUTLoadContext[i].isAlready) {
if (!execNetwork) {
execNetwork = _pCTPUTLoadContext[i].executableNetwork;
}
nLoadSucNums++;
}
}
// one or more devices loaded successfully
if (nLoadSucNums > 0) {
return;
return execNetwork;
}
}
IE_THROW() << GetLogTag() << "load all devices failed";
@ -784,7 +786,6 @@ IInferPtr AutoSchedule::CreateInferRequest() {
so = _passthroughExeNet._so;
syncRequestImpl->setPointerToSo(so);
} else if (std::static_pointer_cast<MultiDeviceInferRequest>(syncRequestImpl)->GetSharedRequest()) {
// cumulative case, load to MULTI:*
auto sharedMultiRequest = std::static_pointer_cast<MultiDeviceInferRequest>(syncRequestImpl)->GetSharedRequest();
if (sharedMultiRequest._ptr->getPointerToSo())
syncRequestImpl->setPointerToSo(sharedMultiRequest._ptr->getPointerToSo());

10
src/plugins/auto/auto_schedule.hpp
View File

@ -51,6 +51,7 @@ public:
public:
AutoLoadContext _loadContext[CONTEXTNUM];
std::unique_ptr<AutoLoadContext[]> _pCTPUTLoadContext = nullptr;
size_t _nCTputDeviceNums;
protected:
void GenerateWorkers(const std::string& device, const SoExecNetwork& executableNetwork) override;
@ -58,9 +59,15 @@ protected:
static bool RunPipelineTask(IE::Task& inferPipelineTask, NotBusyPriorityWorkerRequests& idleWorkerRequests,
const DeviceName& preferred_device);
DeviceMap<NotBusyPriorityWorkerRequests> _idleWorkerRequests;
AutoScheduleContext::Ptr _autoSContext;
private:
void WaitFirstNetworkReady();
/**
* @brief wait for one of the executable network to finish loading.
* @return An SoPtr object hold an available executable network loaded to HW device.
* @note An exception will be thrown if all loading of network to hw device fails.
*/
SoExecNetwork WaitFirstNetworkReady();
void TryToLoadNetWork(AutoLoadContext& context, const std::string& modelPath, const IE::CNNNetwork& network, bool isCumulative);
bool selectOtherDevice(const std::string& currentDeviceName);
IE::Task releaseActualdeviceTask;
@ -73,7 +80,6 @@ private:
std::promise<void> _firstLoadPromise;
bool _exitFlag = {false};
size_t _cpuHelpInferCount = 0;
AutoScheduleContext::Ptr _autoSContext;
};
} // namespace MultiDevicePlugin

150
src/plugins/auto/bind_multi_schedule.cpp
View File

@ -6,142 +6,42 @@
#include "async_infer_request.hpp"
#include "plugin.hpp"
#include "bind_multi_schedule.hpp"
#include "multi_executable_network.hpp"
// ------------------------------MultiSchedule----------------------------
namespace MultiDevicePlugin {
thread_local IE::IInferRequestInternal* BinderMultiSchedule::_sharedRequest = nullptr;
void BinderMultiSchedule::init(const ScheduleContext::Ptr& sContext) {
MultiSchedule::init(sContext);
AutoSchedule::init(sContext);
LOG_INFO_TAG("enable bind buffer for AUTO");
}
Pipeline BinderMultiSchedule::GetPipeline(const IInferPtr& syncInferRequest, WorkerInferRequest** workerInferRequest) {
Pipeline pipeline = {
// if the request is coming with device-specific remote blobs make sure it is scheduled to the specific device only:
Stage {
/*TaskExecutor*/ std::make_shared<IE::ImmediateExecutor>(), /*task*/ [this, &syncInferRequest, workerInferRequest]() {
// by default, no preferred device:
_thisPreferredDeviceName = "";
auto execNetwork = _multiSContext->_executableNetwork.lock();
// if any input is remote (e.g. was set with SetBlob), let' use the corresponding device
for (const auto& it : execNetwork->GetInputsInfo()) {
auto b = syncInferRequest->GetBlob(it.first);
auto r = b->as<IE::RemoteBlob>();
if (r) {
const auto name = r->getDeviceName();
const auto res = std::find_if(
_multiSContext->_devicePrioritiesInitial.cbegin(),
_multiSContext->_devicePrioritiesInitial.cend(),
[&name](const MultiDevicePlugin::DeviceInformation & d) {
return (d.defaultDeviceID.empty() ? d.deviceName : (d.deviceName + "." +
d.defaultDeviceID)) == name;
Pipeline pipeline;
struct RequestExecutor : ITaskExecutor {
explicit RequestExecutor(InferenceEngine::SoIInferRequestInternal& inferRequest) : _inferRequest(inferRequest) {
_inferRequest->SetCallback([this](std::exception_ptr exceptionPtr) mutable {
_exceptionPtr = exceptionPtr;
auto capturedTask = std::move(_task);
capturedTask();
});
if (_multiSContext->_devicePrioritiesInitial.cend() == res) {
IE_THROW() <<
"None of the devices (for which current MULTI-device configuration was "
"initialized) supports a remote blob created on the device named " << name;
} else {
// it is ok to take the c_str() here (as pointed in the executable_network.hpp we need to use const char*)
// as the original strings are from the "persistent" vector (with the right lifetime)
_thisPreferredDeviceName = res->deviceName.c_str();
break;
}
}
}
_thisWorkerInferRequest = *workerInferRequest;
_sharedRequest = std::dynamic_pointer_cast<MultiDeviceInferRequest>(syncInferRequest)->GetSharedRequest()._ptr.get();
}},
// as the scheduling algo may select any device, this stage accepts the scheduling decision (actual workerRequest)
// then sets the device-agnostic blobs to the actual (device-specific) request
Stage {
/*TaskExecutor*/std::dynamic_pointer_cast<IE::ITaskExecutor>(shared_from_this()), /*task*/ [&syncInferRequest, workerInferRequest]() {
*workerInferRequest = _thisWorkerInferRequest;
auto multiSyncInferRequest = std::dynamic_pointer_cast<MultiDeviceInferRequest>(syncInferRequest);
multiSyncInferRequest->SetBlobsToAnotherRequest(_thisWorkerInferRequest->_inferRequest);
INFO_RUN([workerInferRequest]() {
(*workerInferRequest)->_startTimes.push_back(std::chrono::steady_clock::now());
});
}},
// final task in the pipeline:
Stage {
/*TaskExecutor*/std::make_shared<ThisRequestExecutor>(workerInferRequest), /*task*/ [this, &syncInferRequest, workerInferRequest]() {
if (nullptr != (*workerInferRequest)->_exceptionPtr) {
std::rethrow_exception((*workerInferRequest)->_exceptionPtr);
}
if (_multiSContext->_needPerfCounters) {
auto multiSyncInferRequest = std::dynamic_pointer_cast<MultiDeviceInferRequest>
(syncInferRequest);
multiSyncInferRequest->_scheduledRequest =
(*workerInferRequest)->_inferRequest;
}
INFO_RUN([workerInferRequest]() {
(*workerInferRequest)->_endTimes.push_back(std::chrono::steady_clock::now());
});
}}
void run(InferenceEngine::Task task) override {
_task = std::move(task);
_inferRequest->StartAsync();
};
InferenceEngine::SoIInferRequestInternal& _inferRequest;
std::exception_ptr _exceptionPtr;
InferenceEngine::Task _task;
};
auto requestExecutor = std::make_shared<RequestExecutor>(
std::static_pointer_cast<MultiDeviceInferRequest>(syncInferRequest)->GetSharedRequest());
pipeline.emplace_back(requestExecutor, [requestExecutor] {
if (nullptr != requestExecutor->_exceptionPtr) {
std::rethrow_exception(requestExecutor->_exceptionPtr);
}
});
return pipeline;
}
bool BinderMultiSchedule::ScheduleToWorkerInferRequest(IE::Task inferPipelineTask, DeviceName preferred_device) {
std::vector<DeviceInformation> devices;
devices = [&] {
std::lock_guard<std::mutex> lock(_multiSContext->_mutex);
return _multiSContext->_devicePriorities;
}();
for (auto&& device : devices) {
if (!preferred_device.empty() && (device.deviceName != preferred_device)) {
continue;
}
if (RunPipelineTask(inferPipelineTask, _idleWorkerRequests[device.deviceName], preferred_device)) {
return true;
}
}
// no vacant requests this time, storing the task to the respective queue
if (!preferred_device.empty()) {
_inferPipelineTasksDeviceSpecific[preferred_device]->push(std::move(inferPipelineTask));
} else {
_inferPipelineTasks.push(std::move(inferPipelineTask));
}
return false;
}
bool BinderMultiSchedule::RunPipelineTask(IE::Task& inferPipelineTask,
NotBusyWorkerRequests& idleWorkerRequests,
const DeviceName& preferred_device) {
WorkerInferRequest* workerRequestPtr = nullptr;
WorkerInferRequest* headWorker = nullptr;
bool flag = false;
while (idleWorkerRequests.try_pop(workerRequestPtr)) {
if (flag && workerRequestPtr == headWorker)
break;
if (!flag) {
headWorker = workerRequestPtr;
flag = true;
}
IdleGuard<NotBusyWorkerRequests> idleGuard{workerRequestPtr, idleWorkerRequests};
if (_sharedRequest == workerRequestPtr->_inferRequest._ptr.get()) {
_thisWorkerInferRequest = workerRequestPtr;
{
auto capturedTask = std::move(inferPipelineTask);
capturedTask();
}
idleGuard.Release();
return true;
}
}
return false;
}
void BinderMultiSchedule::run(IE::Task inferPipelineTask) {
if (_thisWorkerInferRequest) {
auto capturedTask = std::move(inferPipelineTask);
capturedTask();
} else {
ScheduleToWorkerInferRequest(std::move(inferPipelineTask), _thisPreferredDeviceName);
}
}
BinderMultiSchedule::~BinderMultiSchedule() {
}
@ -153,7 +53,7 @@ IInferPtr BinderMultiSchedule::CreateInferRequestImpl(
SoInfer request_to_share_blobs_with;
// borrowing device-specific blobs from the underlying requests for the device-agnostic, user-facing requests
// this allows to potentially save on the data-copy later (if the requests are scheduled in the same order)
for (const auto& device : _multiSContext->_devicePrioritiesInitial) {
for (const auto& device : _autoSContext->_devicePrioritiesInitial) {
auto& dev_requests = _workerRequests[device.deviceName];
if ((num - sum) < dev_requests.size()) {
request_to_share_blobs_with = dev_requests.at(num - sum)._inferRequest;
@ -177,7 +77,7 @@ IInferPtr BinderMultiSchedule::CreateInferRequestImpl(IE::InputsDataMap networkI
size_t sum = 0;
// borrowing device-specific blobs from the underlying requests for the device-agnostic, user-facing requests
// this allows to potentially save on the data-copy later (if the requests are scheduled in the same order)
for (const auto& device : _multiSContext->_devicePrioritiesInitial) {
for (const auto& device : _autoSContext->_devicePrioritiesInitial) {
auto& dev_requests = _workerRequests[device.deviceName];
if ((num - sum) < dev_requests.size()) {
request_to_share_blobs_with = dev_requests.at(num - sum)._inferRequest;

12
src/plugins/auto/bind_multi_schedule.hpp
View File

@ -5,7 +5,7 @@
///////////////////////////////////////////////////////////////////////////////////////////////////
#pragma once
#include "multi_schedule.hpp"
#include "auto_schedule.hpp"
#ifdef MULTIUNITTEST
#define MOCKTESTMACRO virtual
@ -15,22 +15,14 @@
#endif
namespace MultiDevicePlugin {
class BinderMultiSchedule : public MultiSchedule {
class BinderMultiSchedule : public AutoSchedule {
public:
using Ptr = std::shared_ptr<BinderMultiSchedule>;
IInferPtr CreateInferRequestImpl(IE::InputsDataMap networkInputs, IE::OutputsDataMap networkOutputs) override;
IE::IInferRequestInternal::Ptr CreateInferRequestImpl(const std::vector<std::shared_ptr<const ov::Node>>& inputs,
const std::vector<std::shared_ptr<const ov::Node>>& outputs) override;
void run(IE::Task inferTask) override;
void init(const ScheduleContext::Ptr& sContext) override;
Pipeline GetPipeline(const IInferPtr& syncRequestImpl, WorkerInferRequest** WorkerInferRequest) override;
virtual ~BinderMultiSchedule();
protected:
static bool RunPipelineTask(IE::Task& inferPipelineTask, NotBusyWorkerRequests& idleWorkerRequests, const DeviceName& preferred_device);
bool ScheduleToWorkerInferRequest(IE::Task, DeviceName preferred_device = "") override;
protected:
thread_local static IE::IInferRequestInternal* _sharedRequest;
};
} // namespace MultiDevicePlugin

2
src/plugins/auto/common.hpp
View File

@ -143,7 +143,6 @@ public:
std::mutex _mutex;
bool _needPerfCounters;
bool _batchingDisabled = {false};
bool _bindBuffer = false;
bool _startupfallback = true;
bool _runtimeFallback = true;
virtual ~MultiScheduleContext() = default;
@ -161,6 +160,7 @@ public:
std::mutex _confMutex;
std::mutex _fallbackMutex;
MultiDeviceInferencePlugin* _plugin;
SoExecNetwork _hwExecutableNetwork;
virtual ~AutoScheduleContext() = default;
};

2
src/plugins/auto/multi_schedule.cpp
View File

@ -312,7 +312,7 @@ IInferPtr MultiSchedule::CreateInferRequest() {
if (!so)
so = _passthroughExeNet._so;
syncRequestImpl->setPointerToSo(so);
} else if (_multiSContext->_bindBuffer) {
} else if (std::static_pointer_cast<MultiDeviceInferRequest>(syncRequestImpl)->GetSharedRequest()) {
auto sharedRequest = std::static_pointer_cast<MultiDeviceInferRequest>(syncRequestImpl)->GetSharedRequest();
if (sharedRequest._ptr->getPointerToSo())
syncRequestImpl->setPointerToSo(sharedRequest._ptr->getPointerToSo());

220
src/plugins/auto/plugin.cpp
View File

@ -351,19 +351,21 @@ IExecutableNetworkInternal::Ptr MultiDeviceInferencePlugin::LoadNetworkImpl(cons
auto loadConfig = _pluginConfig;
// if no perf hint from user with compiled model, or already been set with plugin
// apply latency for AUTO, tput for MULTI
bool isHintSet = _pluginConfig.is_set_by_user(ov::hint::performance_mode) || config.find(ov::hint::performance_mode.name()) != config.end();
if (!isHintSet) {
if (workModeAuto) {
// set performance hint to 'LATENCY' model for AutoExecutable Network.
auto itorConfig = config.find(ov::hint::performance_mode.name());
bool isHintSet = _pluginConfig.is_set_by_user(ov::hint::performance_mode) || itorConfig != config.end();
if (!isHintSet && workModeAuto) {
// NO user sets perfHint, then set perfhint to 'LATENCY' for AutoExecutableNetwork.
loadConfig.set_property(ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY));
} else {
// set performance hint to 'THROUGHPUT' model for MultiExecutable Network.
loadConfig.set_property(ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT));
}
}
// updateFromMap will check config valid
loadConfig.set_user_property(PreProcessConfig(config), workModeAuto);
loadConfig.apply_user_properties();
if (!workModeAuto) {
if (itorConfig != config.end() && itorConfig->second != InferenceEngine::PluginConfigParams::THROUGHPUT) {
LOG_WARNING_TAG("User set perf_hint:%s, but MULTI supports THROUGHPUT only", itorConfig->second.c_str());
}
loadConfig.set_property(ov::hint::performance_mode(ov::hint::PerformanceMode::CUMULATIVE_THROUGHPUT));
}
auto fullProperty = loadConfig.get_full_properties();
// this can be updated when plugin switch to 2.0 API
std::map<std::string, std::string> fullConfig = ConvertToStringMap(fullProperty);
@ -378,6 +380,8 @@ IExecutableNetworkInternal::Ptr MultiDeviceInferencePlugin::LoadNetworkImpl(cons
std::unordered_map<std::string, ov::Any> multiNetworkConfig;
std::vector<DeviceInformation> metaDevices;
auto priorities = loadConfig.get_property(ov::device::priorities);
if (priorities.empty() && !workModeAuto)
IE_THROW() << "KEY_MULTI_DEVICE_PRIORITIES key is not set for " << GetName() << " device";
if (priorities.find("AUTO") != std::string::npos || priorities.find("MULTI") != std::string::npos) {
IE_THROW() << "The device candidate list should not include the meta plugin for " << GetName() << " device";
}
@ -397,12 +401,8 @@ IExecutableNetworkInternal::Ptr MultiDeviceInferencePlugin::LoadNetworkImpl(cons
}
};
// if workMode is AUTO
// only AUTO uses CheckConfig() to check fullConfig's parameters, MULTI does not
if (workModeAuto) {
// check the configure and check if need to set PerfCounters configure to device
// and set filter configure
OV_ITT_SCOPED_TASK(itt::domains::MULTIPlugin, "MultiDeviceInferencePlugin::LoadNetworkImpl::AutoMode");
auto autoSContext = std::make_shared<AutoScheduleContext>();
std::map<std::string, std::string> filterConfig;
@ -414,9 +414,10 @@ IExecutableNetworkInternal::Ptr MultiDeviceInferencePlugin::LoadNetworkImpl(cons
}
autoSContext->_modelPriority = MapPriorityValues(loadConfig.get_property(ov::hint::model_priority));
autoSContext->_batchingDisabled = !(loadConfig.get_property(ov::hint::allow_auto_batching));
// set performanceHint for AutoExecutableNetwork
autoSContext->_performanceHint = loadConfig.get_property(ov::hint::performance_mode.name()).as<std::string>();
// filter the device that supports filter configure
auto metaDevices = ParseMetaDevices(strDevices, fullConfig);
metaDevices = ParseMetaDevices(strDevices, fullConfig);
auto supportDevicesByConfig = FilterDevice(metaDevices, filterConfig);
if (supportDevicesByConfig.empty()) {
IE_THROW() << "There is no device support the configure";
@ -424,15 +425,21 @@ IExecutableNetworkInternal::Ptr MultiDeviceInferencePlugin::LoadNetworkImpl(cons
auto supportDevices = supportDevicesByConfig;
CNNNetwork clonedNetwork;
std::string clonedModelPath = modelPath;
// reset the strDevices to support devices
strDevices = "";
// calling GetValidDevices() to get a prioritized list of devices
bool isCumulative =
(autoSContext->_performanceHint == IE::PluginConfigParams::CUMULATIVE_THROUGHPUT) ? true : false;
std::list<DeviceInformation> devicesWithPriority(supportDevices.begin(), supportDevices.end());
if (modelPath.empty()) {
// if network is valid
LOG_INFO_TAG("load with CNN network");
supportDevices = FilterDeviceByNetwork(supportDevicesByConfig, network);
// clone the network, in case of reshape conflict
clonedNetwork = InferenceEngine::details::cloneNetwork(network);
// clone the network, in case of reshape conflict
} else {
// model path, enable model load with single device situation
if (supportDevices.size() > 1) {
if (supportDevices.size() > 1 && !isCumulative) {
clonedNetwork = GetCore()->ReadNetwork(modelPath, std::string());
// do we really need to disable model path?
clonedModelPath = "";
@ -441,10 +448,9 @@ IExecutableNetworkInternal::Ptr MultiDeviceInferencePlugin::LoadNetworkImpl(cons
LOG_INFO_TAG("load with model path");
}
}
// reset the strDevices to support devices
strDevices = "";
// calling GetValidDevices() to get a prioritized list of devices
auto devicesWithPriority = GetValidDevice(supportDevices, networkPrecision);
if (!isCumulative) {
devicesWithPriority = GetValidDevice(supportDevices, networkPrecision);
}
for (auto iter = devicesWithPriority.begin(); iter != devicesWithPriority.end(); iter++) {
strDevices += iter->deviceName;
strDevices += ",";
@ -475,178 +481,24 @@ IExecutableNetworkInternal::Ptr MultiDeviceInferencePlugin::LoadNetworkImpl(cons
autoSContext->_plugin = this;
autoSContext->_core = GetCore();
autoSContext->_LogTag = _LogTag;
autoSContext->_bindBuffer = loadConfig.get_property(ov::intel_auto::device_bind_buffer);
autoSContext->_startupfallback = loadConfig.get_property(ov::intel_auto::enable_startup_fallback);
autoSContext->_runtimeFallback = loadConfig.get_property(ov::intel_auto::enable_runtime_fallback);
return std::make_shared<AutoExecutableNetwork>(autoSContext, std::make_shared<AutoSchedule>());
}
OV_ITT_SCOPED_TASK(itt::domains::MULTIPlugin, "MultiDeviceInferencePlugin::LoadNetworkImpl:MultiMode");
// if is cumulative, PERFORMANCE_HINT set to THROUGHPUT and _LogTag set to AUTO
auto configIter =
std::find_if(fullConfig.begin(), fullConfig.end(), [](const std::pair<std::string, std::string>& config) {
return (config.first == CONFIG_KEY(PERFORMANCE_HINT));
});
if (configIter != fullConfig.end() && configIter->second == InferenceEngine::PluginConfigParams::CUMULATIVE_THROUGHPUT) {
configIter->second = InferenceEngine::PluginConfigParams::THROUGHPUT;
_LogTag = "AUTO";
LOG_INFO_TAG("CUMULATIVE Call MULTI PERFORMANCE_HINT set to THROUGHPUT");
}
if (priorities.empty()) {
IE_THROW() << "KEY_MULTI_DEVICE_PRIORITIES key is not set for " << GetName() << " device";
} else { // for use case -d MULTI:xPU or -d AUTO:xPU
auto metaDevicesByConfig = ParseMetaDevices(priorities, fullConfig);
metaDevices = modelPath.empty() ? FilterDeviceByNetwork(metaDevicesByConfig, network)
: metaDevicesByConfig;
if (metaDevicesByConfig.size() != metaDevices.size()) {
LOG_DEBUG_TAG("stateful/dynamic model, loaded to single device");
multiNetworkConfig[ov::device::priorities.name()]
= metaDevices[0].deviceName;
} else {
multiNetworkConfig[ov::device::priorities.name()] = priorities;
}
}
auto multiSContext = std::make_shared<MultiScheduleContext>();
DeviceMap<SoExecutableNetworkInternal> executableNetworkPerDevice;
std::mutex load_mutex;
std::vector<Task> loads;
auto loadInferEngTask = [&](DeviceInformation& p) {
auto tmpiter = fullConfig.find(CONFIG_KEY(ALLOW_AUTO_BATCHING));
if (tmpiter != fullConfig.end()) {
if (tmpiter->second == PluginConfigParams::NO) {
LOG_INFO_TAG("set %s=%s", tmpiter->first.c_str(), tmpiter->second.c_str());
multiSContext->_batchingDisabled = true;
}
if (loadConfig.is_set_by_user(ov::hint::allow_auto_batching))
p.config.insert({tmpiter->first, tmpiter->second});
}
if (loadConfig.is_set_by_user(ov::auto_batch_timeout))
insertPropToConfig(ov::auto_batch_timeout.name(), p.deviceName, p.config);
insertPropToConfig(ov::cache_dir.name(), p.deviceName, p.config);
const auto& deviceName = p.deviceName;
const auto& deviceConfig = p.config;
SoExecutableNetworkInternal exec_net;
LOG_DEBUG_TAG("load network to device:%s", deviceName.c_str());
try {
if (modelPath.empty()) {
exec_net = GetCore()->LoadNetwork(network, deviceName, deviceConfig);
} else {
exec_net = GetCore()->LoadNetwork(modelPath, deviceName, deviceConfig);
}
} catch (const IE::Exception& iie) {
if (_LogTag == "AUTO") {
LOG_DEBUG_TAG("Failed to load network to device:%s with error: %s", deviceName.c_str(), iie.what());
return;
} else {
IE_THROW() << "Failed to load network to device: " << deviceName.c_str() << " with error:" <<
iie.what();
}
}
try {
std::string sStreamNums = "";
std::string sThreadNums = "";
if (deviceName.find("CPU") != std::string::npos) {
sStreamNums = exec_net->GetMetric(ov::num_streams.name()).as<std::string>();
sThreadNums = exec_net->GetMetric(ov::inference_num_threads.name()).as<std::string>();
} else if (deviceName.find("GPU") != std::string::npos) {
sStreamNums = exec_net->GetConfig(ov::num_streams.name()).as<std::string>();
sThreadNums = exec_net->GetConfig(ov::compilation_num_threads.name()).as<std::string>();
}
// print CPU or GPU streams num and threads num
if (!sStreamNums.empty() && !sThreadNums.empty()) {
LOG_INFO_TAG("after load network, %s streamNums:%s, %s threadNums:%s",
deviceName.c_str(),
sStreamNums.c_str(),
deviceName.c_str(),
sThreadNums.c_str());
}
} catch (const IE::Exception&) {
LOG_DEBUG_TAG("deviceName:%s cannot get streamNums and threadNums from exec_net", deviceName.c_str());
}
std::unique_lock<std::mutex> lock{load_mutex};
executableNetworkPerDevice.insert({deviceName, exec_net});
multiNetworkConfig.insert(deviceConfig.begin(), deviceConfig.end());
};
// Check if CPU is in device list
auto iterCPU = std::find_if(metaDevices.begin(), metaDevices.end(), [&](DeviceInformation& d) {
return d.deviceName.find("CPU") != std::string::npos;
});
// Load devices other than CPU first
for (auto& p : metaDevices) {
if (iterCPU != metaDevices.end() && p.deviceName == iterCPU->deviceName) {
continue;
}
loads.push_back([&]() {
loadInferEngTask(p);
});
}
auto executor = executorManager()->getIdleCPUStreamsExecutor(
IStreamsExecutor::Config{"MultiDeviceAsyncLoad",
static_cast<int>(std::thread::hardware_concurrency()) /* max possible #streams*/,
0 /*default threads per stream, workaround for ticket 62376*/,
IStreamsExecutor::ThreadBindingType::NONE});
if (loads.size() > 0) {
// Wait for the device to load the network
executor->runAndWait(loads);
loads.clear();
}
// Finally load the CPU
if (iterCPU != metaDevices.end()) {
if (!executableNetworkPerDevice.empty() && iterCPU->config.find(ov::affinity.name()) == iterCPU->config.end()) {
LOG_DEBUG_TAG("set affinity to NUMA and disable hyper thread for CPU");
// If the other devices load successfully and no user set affinity then set NUMA to CPU
iterCPU->config.insert({ov::affinity.name(), ov::affinity(ov::Affinity::NUMA).second.as<std::string>()});
iterCPU->config.insert({CONFIG_KEY_INTERNAL(ENABLE_HYPER_THREAD), CONFIG_VALUE(NO)});
}
loads.push_back([&]() {
loadInferEngTask(*iterCPU);
});
// Wait for CPU to load the network
executor->runAndWait(loads);
}
if (executableNetworkPerDevice.empty())
IE_THROW(NotFound) << "Failed to load network to any device "
<< "that the " << GetName() << " device is initialized to work with";
// checking the perf counters config from the loaded network to respect both device's plugin and load-specific setting
size_t num_plugins_supporting_perf_counters = 0;
for (auto& n : executableNetworkPerDevice) {
try {
num_plugins_supporting_perf_counters +=
n.second->GetConfig(PluginConfigParams::KEY_PERF_COUNT).as<std::string>() ==
PluginConfigParams::YES;
} catch (const IE::Exception&) {
}
}
// MULTI can enable the perf counters only if all devices support/enable that
bool enablePerfCounters = num_plugins_supporting_perf_counters == executableNetworkPerDevice.size();
multiSContext->_devicePriorities = metaDevices;
multiSContext->_devicePrioritiesInitial = metaDevices;
multiSContext->_networksPerDevice = executableNetworkPerDevice;
multiSContext->_config = multiNetworkConfig;
multiSContext->_needPerfCounters = enablePerfCounters;
multiSContext->_core = GetCore();
multiSContext->_LogTag = _LogTag;
IExecutableNetworkInternal::Ptr impl;
auto tmp = loadConfig.get_property(ov::intel_auto::device_bind_buffer);
if (tmp) {
multiSContext->_bindBuffer = true;
impl = std::make_shared<MultiExecutableNetwork>(multiSContext, std::make_shared<BinderMultiSchedule>());
// enable bind only in cumulative_throughput mode
if (loadConfig.get_property(ov::intel_auto::device_bind_buffer) &&
autoSContext->_performanceHint == "CUMULATIVE_THROUGHPUT") {
LOG_INFO_TAG("runtime fallback set to disabled in binder mode");
autoSContext->_runtimeFallback = false;
impl = std::make_shared<AutoExecutableNetwork>(autoSContext, std::make_shared<BinderMultiSchedule>());
} else {
impl = std::make_shared<MultiExecutableNetwork>(multiSContext, std::make_shared<MultiSchedule>());
impl = std::make_shared<AutoExecutableNetwork>(autoSContext, std::make_shared<AutoSchedule>());
}
if (!modelPath.empty()) {
SetExeNetworkInfo(impl,
executableNetworkPerDevice.begin()->second->GetInputsInfo(),
executableNetworkPerDevice.begin()->second->GetOutputsInfo());
impl->setInputs(executableNetworkPerDevice.begin()->second->getInputs());
impl->setOutputs(executableNetworkPerDevice.begin()->second->getOutputs());
autoSContext->_hwExecutableNetwork->GetInputsInfo(),
autoSContext->_hwExecutableNetwork->GetOutputsInfo());
impl->setInputs(autoSContext->_hwExecutableNetwork->getInputs());
impl->setOutputs(autoSContext->_hwExecutableNetwork->getOutputs());
}
return impl;
}

4
src/plugins/intel_cpu/tests/functional/shared_tests_instances/behavior/ov_plugin/core_integration.cpp
View File

@ -411,12 +411,12 @@ INSTANTIATE_TEST_SUITE_P(smoke_HETERO_OVClassLoadNetworkWithSecondaryPropertiesT
// IE Class load and check network with ov::device::properties
INSTANTIATE_TEST_SUITE_P(smoke_CPU_OVClassLoadNetworkAndCheckWithSecondaryPropertiesTest,
OVClassLoadNetworkAndCheckSecondaryPropertiesTest,
::testing::Combine(::testing::Values("CPU", "MULTI:CPU"),
::testing::Combine(::testing::Values("CPU"),
::testing::ValuesIn(configsDeviceProperties)));
INSTANTIATE_TEST_SUITE_P(smoke_CPU_OVClassLoadNetworkAndCheckWithSecondaryPropertiesDoubleTest,
OVClassLoadNetworkAndCheckSecondaryPropertiesTest,
::testing::Combine(::testing::Values("CPU", "MULTI:CPU"),
::testing::Combine(::testing::Values("CPU"),
::testing::ValuesIn(configsDevicePropertiesDouble)));
INSTANTIATE_TEST_SUITE_P(
smoke_OVClassLoadNetworkTest, OVClassLoadNetworkTest,

42
src/plugins/intel_cpu/tests/functional/shared_tests_instances/behavior/ov_plugin/properties_tests.cpp
View File

@ -56,27 +56,13 @@ INSTANTIATE_TEST_SUITE_P(smoke_cpuCompileModelBehaviorTests,
OVSetPropComplieModleGetPropTests::getTestCaseName);
const std::vector<ov::AnyMap> multi_setcore_properties = {
{ov::device::priorities(CommonTestUtils::DEVICE_CPU),
ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT),
ov::hint::num_requests(2),
ov::hint::allow_auto_batching(false),
ov::enable_profiling(false)},
{ov::device::priorities(CommonTestUtils::DEVICE_CPU),
ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY),
ov::hint::num_requests(8),
ov::hint::allow_auto_batching(true),
ov::enable_profiling(true)}};
ov::hint::model_priority(ov::hint::Priority::HIGH)}};
const std::vector<ov::AnyMap> multi_compileModel_properties = {
{ov::device::priorities(CommonTestUtils::DEVICE_CPU),
ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY),
ov::hint::num_requests(10),
ov::hint::allow_auto_batching(true),
ov::enable_profiling(true)},
{ov::device::priorities(CommonTestUtils::DEVICE_CPU),
ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT),
ov::hint::num_requests(2),
ov::hint::allow_auto_batching(false),
ov::enable_profiling(false)}};
ov::hint::model_priority(ov::hint::Priority::MEDIUM)}};
INSTANTIATE_TEST_SUITE_P(smoke_MultiCompileModelBehaviorTests,
OVSetPropComplieModleGetPropTests,
@ -88,36 +74,24 @@ INSTANTIATE_TEST_SUITE_P(smoke_MultiCompileModelBehaviorTests,
const std::vector<ov::AnyMap> auto_setcore_properties = {
{ov::device::priorities(CommonTestUtils::DEVICE_CPU),
ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT),
ov::hint::num_requests(2),
ov::hint::allow_auto_batching(false),
ov::enable_profiling(false)},
ov::hint::model_priority(ov::hint::Priority::HIGH)},
{ov::device::priorities(CommonTestUtils::DEVICE_CPU),
ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY),
ov::hint::num_requests(8),
ov::hint::allow_auto_batching(true),
ov::enable_profiling(true)},
ov::hint::model_priority(ov::hint::Priority::HIGH)},
{ov::device::priorities(CommonTestUtils::DEVICE_CPU),
ov::hint::performance_mode(ov::hint::PerformanceMode::CUMULATIVE_THROUGHPUT),
ov::hint::num_requests(10),
ov::hint::allow_auto_batching(false),
ov::enable_profiling(true)},
ov::hint::model_priority(ov::hint::Priority::HIGH)},
};
const std::vector<ov::AnyMap> auto_compileModel_properties = {
{ov::device::priorities(CommonTestUtils::DEVICE_CPU),
ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY),
ov::hint::num_requests(8),
ov::hint::allow_auto_batching(true),
ov::enable_profiling(true)},
ov::hint::model_priority(ov::hint::Priority::MEDIUM)},
{ov::device::priorities(CommonTestUtils::DEVICE_CPU),
ov::hint::performance_mode(ov::hint::PerformanceMode::CUMULATIVE_THROUGHPUT),
ov::hint::num_requests(10),
ov::hint::allow_auto_batching(false),
ov::enable_profiling(false)},
ov::hint::model_priority(ov::hint::Priority::MEDIUM)},
{ov::device::priorities(CommonTestUtils::DEVICE_CPU),
ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT),
ov::hint::num_requests(2),
ov::hint::allow_auto_batching(true),
ov::enable_profiling(false)}};
ov::hint::model_priority(ov::hint::Priority::MEDIUM)}};
INSTANTIATE_TEST_SUITE_P(smoke_AutoCompileModelBehaviorTests,
OVSetPropComplieModleGetPropTests,
::testing::Combine(::testing::Values(CommonTestUtils::DEVICE_AUTO),

12
src/plugins/intel_cpu/tests/functional/shared_tests_instances/behavior/plugin/configuration_tests.cpp
View File

@ -281,18 +281,12 @@ namespace {
const std::vector<std::map<std::string, std::string>> auto_multi_prop_config = {
{{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES, CommonTestUtils::DEVICE_CPU},
{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT, InferenceEngine::PluginConfigParams::THROUGHPUT},
{InferenceEngine::PluginConfigParams::KEY_EXCLUSIVE_ASYNC_REQUESTS, InferenceEngine::PluginConfigParams::YES},
{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT_NUM_REQUESTS, "2"},
{InferenceEngine::PluginConfigParams::KEY_ALLOW_AUTO_BATCHING, InferenceEngine::PluginConfigParams::NO},
{InferenceEngine::PluginConfigParams::KEY_PERF_COUNT, InferenceEngine::PluginConfigParams::NO}}};
{InferenceEngine::PluginConfigParams::KEY_MODEL_PRIORITY, InferenceEngine::PluginConfigParams::MODEL_PRIORITY_MED}}};
const std::vector<std::map<std::string, std::string>> auto_multi_loadNetWork_config = {
{{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES, CommonTestUtils::DEVICE_CPU},
{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT, InferenceEngine::PluginConfigParams::LATENCY},
{InferenceEngine::PluginConfigParams::KEY_EXCLUSIVE_ASYNC_REQUESTS, InferenceEngine::PluginConfigParams::NO},
{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT_NUM_REQUESTS, "10"},
{InferenceEngine::PluginConfigParams::KEY_ALLOW_AUTO_BATCHING, InferenceEngine::PluginConfigParams::YES},
{InferenceEngine::PluginConfigParams::KEY_PERF_COUNT, InferenceEngine::PluginConfigParams::YES}}};
{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT, InferenceEngine::PluginConfigParams::THROUGHPUT},
{InferenceEngine::PluginConfigParams::KEY_MODEL_PRIORITY, InferenceEngine::PluginConfigParams::MODEL_PRIORITY_HIGH}}};
INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests,
SetPropLoadNetWorkGetPropTests,

4
src/plugins/intel_cpu/tests/functional/shared_tests_instances/skip_tests_config.cpp
View File

@ -93,7 +93,9 @@ std::vector<std::string> disabledTestPatterns() {
R"(.*Behavior.*ExecutableNetworkBaseTest.*canSetConfigToExecNetWithIncorrectConfig.*)",
R"(.*Hetero.*Behavior.*ExecutableNetworkBaseTest.*ExecGraphInfo.*)",
R"(.*Hetero.*Behavior.*ExecutableNetworkBaseTest.*CanCreateTwoExeNetworksAndCheckFunction.*)",
// TODO: CVS-104942
R"(.*(Auto|Multi).*Behavior.*ExecutableNetworkBaseTest.*canLoadCorrectNetworkToGetExecutableAndCheckConfig.*)",
R"(.*(Auto|Multi).*SetPropLoadNetWorkGetPropTests.*)",
// CPU does not support dynamic rank
// Issue: CVS-66778
R"(.*smoke_BehaviorTests.*InferFullyDynamicNetworkWith(S|G)etTensor.*)",

42
src/tests/functional/plugin/gpu/shared_tests_instances/behavior/ov_plugin/properties_tests.cpp
View File

@ -109,25 +109,11 @@ INSTANTIATE_TEST_SUITE_P(smoke_gpuCompileModelBehaviorTests,
const std::vector<ov::AnyMap> multi_setcore_properties = {
{ov::device::priorities(CommonTestUtils::DEVICE_GPU),
ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT),
ov::hint::num_requests(2),
ov::hint::allow_auto_batching(false),
ov::enable_profiling(false)},
{ov::device::priorities(CommonTestUtils::DEVICE_GPU),
ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY),
ov::hint::num_requests(8),
ov::hint::allow_auto_batching(true),
ov::enable_profiling(true)}};
ov::hint::model_priority(ov::hint::Priority::HIGH)}};
const std::vector<ov::AnyMap> multi_compileModel_properties = {
{ov::device::priorities(CommonTestUtils::DEVICE_GPU),
ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY),
ov::hint::num_requests(10),
ov::hint::allow_auto_batching(true),
ov::enable_profiling(true)},
{ov::device::priorities(CommonTestUtils::DEVICE_GPU),
ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT),
ov::hint::num_requests(2),
ov::hint::allow_auto_batching(false),
ov::enable_profiling(false)}};
ov::hint::model_priority(ov::hint::Priority::MEDIUM)}};
INSTANTIATE_TEST_SUITE_P(smoke_MultiCompileModelBehaviorTests,
OVSetPropComplieModleGetPropTests,
@ -139,36 +125,24 @@ INSTANTIATE_TEST_SUITE_P(smoke_MultiCompileModelBehaviorTests,
const std::vector<ov::AnyMap> auto_setcore_properties = {
{ov::device::priorities(CommonTestUtils::DEVICE_GPU),
ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT),
ov::hint::num_requests(2),
ov::hint::allow_auto_batching(false),
ov::enable_profiling(false)},
ov::hint::model_priority(ov::hint::Priority::HIGH)},
{ov::device::priorities(CommonTestUtils::DEVICE_GPU),
ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY),
ov::hint::num_requests(8),
ov::hint::allow_auto_batching(true),
ov::enable_profiling(true)},
ov::hint::model_priority(ov::hint::Priority::HIGH)},
{ov::device::priorities(CommonTestUtils::DEVICE_GPU),
ov::hint::performance_mode(ov::hint::PerformanceMode::CUMULATIVE_THROUGHPUT),
ov::hint::num_requests(10),
ov::hint::allow_auto_batching(false),
ov::enable_profiling(true)},
ov::hint::model_priority(ov::hint::Priority::HIGH)},
};
const std::vector<ov::AnyMap> auto_compileModel_properties = {
{ov::device::priorities(CommonTestUtils::DEVICE_GPU),
ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY),
ov::hint::num_requests(8),
ov::hint::allow_auto_batching(true),
ov::enable_profiling(true)},
ov::hint::model_priority(ov::hint::Priority::MEDIUM)},
{ov::device::priorities(CommonTestUtils::DEVICE_GPU),
ov::hint::performance_mode(ov::hint::PerformanceMode::CUMULATIVE_THROUGHPUT),
ov::hint::num_requests(10),
ov::hint::allow_auto_batching(false),
ov::enable_profiling(false)},
ov::hint::model_priority(ov::hint::Priority::MEDIUM)},
{ov::device::priorities(CommonTestUtils::DEVICE_GPU),
ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT),
ov::hint::num_requests(2),
ov::hint::allow_auto_batching(true),
ov::enable_profiling(false)}};
ov::hint::model_priority(ov::hint::Priority::MEDIUM)}};
INSTANTIATE_TEST_SUITE_P(smoke_AutoCompileModelBehaviorTests,
OVSetPropComplieModleGetPropTests,
::testing::Combine(::testing::Values(CommonTestUtils::DEVICE_AUTO),

16
src/tests/functional/plugin/gpu/shared_tests_instances/behavior/plugin/configuration_tests.cpp
View File

@ -309,22 +309,16 @@ namespace {
{{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES, CommonTestUtils::DEVICE_GPU},
{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT,
InferenceEngine::PluginConfigParams::THROUGHPUT},
{InferenceEngine::PluginConfigParams::KEY_EXCLUSIVE_ASYNC_REQUESTS,
InferenceEngine::PluginConfigParams::YES},
{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT_NUM_REQUESTS, "2"},
{InferenceEngine::PluginConfigParams::KEY_ALLOW_AUTO_BATCHING, InferenceEngine::PluginConfigParams::NO},
{InferenceEngine::PluginConfigParams::KEY_PERF_COUNT, InferenceEngine::PluginConfigParams::NO}}};
{InferenceEngine::PluginConfigParams::KEY_MODEL_PRIORITY,
InferenceEngine::PluginConfigParams::MODEL_PRIORITY_MED}}};
};
auto auto_multi_loadNetWork_config = []() {
return std::vector<std::map<std::string, std::string>>{
{{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES, CommonTestUtils::DEVICE_GPU},
{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT, InferenceEngine::PluginConfigParams::LATENCY},
{InferenceEngine::PluginConfigParams::KEY_EXCLUSIVE_ASYNC_REQUESTS,
InferenceEngine::PluginConfigParams::NO},
{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT_NUM_REQUESTS, "10"},
{InferenceEngine::PluginConfigParams::KEY_ALLOW_AUTO_BATCHING, InferenceEngine::PluginConfigParams::YES},
{InferenceEngine::PluginConfigParams::KEY_PERF_COUNT, InferenceEngine::PluginConfigParams::YES}}};
{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT, InferenceEngine::PluginConfigParams::THROUGHPUT},
{InferenceEngine::PluginConfigParams::KEY_MODEL_PRIORITY,
InferenceEngine::PluginConfigParams::MODEL_PRIORITY_HIGH}}};
};
INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests,

11
src/tests/functional/plugin/gpu/shared_tests_instances/multi/gpu_remote_blob_tests.cpp
View File

@ -106,12 +106,17 @@ TEST_P(MultiDevice_Bind_oversubsciption_test, oversubsciptionOfInferRequest) {
auto device_names_and_support_for_remote_blobs2 = []() {
return std::vector<DevicesNamseAndProperties>{
// another GPU (the test will test its presence), different OCL contexts
// use GPU.0 as reference, expect auto to throw exception on other hardware contexts
#ifdef ENABLE_INTEL_CPU
//{{CPU}, {}}, // stand-alone CPU via MULTI (no GPU), no OCL context
{{CPU}, {ov::intel_auto::device_bind_buffer(true)}}, // stand-alone CPU via MULTI (no GPU), no OCL context
{{"GPU.1", CPU},
{ov::intel_auto::device_bind_buffer(true)}},
{{"GPU.1", CPU},
{ov::intel_auto::device_bind_buffer(false)}},
#endif
{{"GPU.1"}, {}}, // another GPU (the test will test its presence), different OCL contexts
{{"GPU.1"}, {ov::intel_auto::device_bind_buffer(true)}}, // another GPU (the test will test its presence), different OCL contexts
{{"GPU.1"}, {}},
{{"GPU.1"}, {ov::intel_auto::device_bind_buffer(true)}},
};
};

4
src/tests/functional/plugin/gpu/shared_tests_instances/skip_tests_config.cpp
View File

@ -124,5 +124,9 @@ std::vector<std::string> disabledTestPatterns() {
// Looks like the test is targeting CPU plugin and doesn't respect that execution graph may vary from plugin to plugin
R"(.*ExecGraphSerializationTest.*)",
// TODO: support getconfig in auto/multi CVS-104942
// TODO: move auto/multi cases to dedicated unit tests
R"(.*(Auto|Multi).*SetPropLoadNetWorkGetPropTests.*)",
};
}

4
src/tests/functional/plugin/shared/include/behavior/executable_network/exec_network_base.hpp
View File

@ -70,10 +70,6 @@ TEST_P(ExecutableNetworkBaseTest, checkGetMetric) {
TEST_P(ExecutableNetworkBaseTest, canLoadCorrectNetworkToGetExecutableAndCheckConfig) {
auto execNet = ie->LoadNetwork(cnnNet, target_device, configuration);
if (target_device == CommonTestUtils::DEVICE_AUTO) {
// AUTO executable network didn't support to read any config.
GTEST_SKIP();
}
for (const auto& configItem : configuration) {
InferenceEngine::Parameter param;
ASSERT_NO_THROW(param = execNet.GetConfig(configItem.first));

3
src/tests/functional/plugin/shared/include/behavior/ov_plugin/core_integration.hpp
View File

@ -1355,6 +1355,7 @@ TEST_P(OVClassLoadNetworkAndCheckSecondaryPropertiesTest, LoadNetworkAndCheckSec
ASSERT_TRUE(property.count(ov::num_streams.name()));
auto actual = property.at(ov::num_streams.name()).as<int32_t>();
ov::Any value;
//AutoExcutableNetwork GetMetric() does not support key ov::num_streams
OV_ASSERT_NO_THROW(value = model.get_property(ov::num_streams.name()));
int32_t expect = value.as<int32_t>();
ASSERT_EQ(actual, expect);
@ -1382,7 +1383,7 @@ TEST_P(OVClassLoadNetWorkDoNotReturnDefaultHintTest, LoadNetworkDoNotReturnDefau
if (target_device.find("AUTO") != std::string::npos) {
ASSERT_NE(value, ov::hint::PerformanceMode::LATENCY);
} else {
ASSERT_NE(value, ov::hint::PerformanceMode::THROUGHPUT);
ASSERT_EQ(value, ov::hint::PerformanceMode::THROUGHPUT);
}
}

2
src/tests/functional/plugin/shared/src/behavior/ov_plugin/properties_tests.cpp
View File

@ -178,8 +178,6 @@ TEST_P(OVSetPropComplieModleGetPropTests, SetPropertyComplieModelGetProperty) {
ov::CompiledModel exeNetWork;
OV_ASSERT_NO_THROW(exeNetWork = core->compile_model(model, target_device, compileModelProperties));
if (target_device == CommonTestUtils::DEVICE_AUTO)
GTEST_SKIP();
for (const auto& property_item : compileModelProperties) {
Any exeNetProperty;

4
src/tests/functional/plugin/shared/src/behavior/plugin/configuration_tests.cpp
View File

@ -197,9 +197,8 @@ TEST_P(SetPropLoadNetWorkGetPropTests, SetPropLoadNetWorkGetProperty) {
InferenceEngine::ExecutableNetwork exeNetWork;
ASSERT_NO_THROW(exeNetWork = ie->LoadNetwork(cnnNet, target_device, loadNetWorkConfig));
if (target_device == CommonTestUtils::DEVICE_AUTO)
GTEST_SKIP();
//ie's setConfig and LoadNetwork should not affect each other, for config settings
for (const auto& property_item : loadNetWorkConfig) {
InferenceEngine::Parameter exeNetProperty;
ASSERT_NO_THROW(exeNetProperty = exeNetWork.GetConfig(property_item.first));
@ -213,5 +212,4 @@ TEST_P(SetPropLoadNetWorkGetPropTests, SetPropLoadNetWorkGetProperty) {
ASSERT_EQ(property_item.second, property.as<std::string>());
}
}
} // namespace BehaviorTestsDefinitions

63
src/tests/unit/auto/auto_ctput_call_multi.cpp
View File

@ -45,6 +45,11 @@ public:
ov::SoPtr<IExecutableNetworkInternal> cpuMockExeNetwork;
NiceMock<MockIInferencePlugin>* cpuMockIPlugin;
std::shared_ptr<InferenceEngine::IInferencePlugin> cpuMockPlugin;
// mock gpu exeNetwork
std::shared_ptr<NiceMock<MockIExecutableNetworkInternal>> gpuMockIExeNet;
ov::SoPtr<IExecutableNetworkInternal> gpuMockExeNetwork;
NiceMock<MockIInferencePlugin>* gpuMockIPlugin;
std::shared_ptr<InferenceEngine::IInferencePlugin> gpuMockPlugin;
std::shared_ptr<NiceMock<MockIInferRequestInternal>> inferReqInternal;
public:
@ -85,6 +90,16 @@ public:
EXPECT_CALL(*cpuMockIPluginPtr, LoadNetwork(MatcherCast<const CNNNetwork&>(_), _)).Times(1);
cpuMockExeNetwork = ov::SoPtr<InferenceEngine::IExecutableNetworkInternal>(cpuMockPlugin->LoadNetwork(CNNNetwork{}, {}), {});
// prepare gpuMockExeNetwork
gpuMockIExeNet = std::make_shared<NiceMock<MockIExecutableNetworkInternal>>();
auto gpuMockIPluginPtr = std::make_shared<NiceMock<MockIInferencePlugin>>();
ON_CALL(*gpuMockIPluginPtr, LoadNetwork(MatcherCast<const CNNNetwork&>(_), _))
.WillByDefault(Return(gpuMockIExeNet));
gpuMockPlugin = gpuMockIPluginPtr;
// remove annoying ON CALL message
EXPECT_CALL(*gpuMockIPluginPtr, LoadNetwork(MatcherCast<const CNNNetwork&>(_), _)).Times(1);
gpuMockExeNetwork = ov::SoPtr<InferenceEngine::IExecutableNetworkInternal>(gpuMockPlugin->LoadNetwork(CNNNetwork{}, {}), {});
// prepare mockicore and cnnNetwork for loading
core = std::shared_ptr<NiceMock<MockICore>>(new NiceMock<MockICore>());
auto* origin_plugin = new NiceMock<MockMultiPluginForLoadNetworkWithPropertiesTest>();
@ -95,6 +110,9 @@ public:
ON_CALL(*cpuMockIExeNet.get(), CreateInferRequest()).WillByDefault(Return(inferReqInternal));
ON_CALL(*cpuMockIExeNet.get(), GetMetric(StrEq(METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS))))
.WillByDefault(Return("0"));
ON_CALL(*gpuMockIExeNet.get(), CreateInferRequest()).WillByDefault(Return(inferReqInternal));
ON_CALL(*gpuMockIExeNet.get(), GetMetric(StrEq(METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS))))
.WillByDefault(Return("0"));
std::vector<std::string> availableDevs = {"CPU", "GPU"};
ON_CALL(*core, GetAvailableDevices()).WillByDefault(Return(availableDevs));
@ -161,58 +179,53 @@ public:
LoadNetwork(::testing::Matcher<const InferenceEngine::CNNNetwork&>(_),
::testing::Matcher<const std::string&>(StrEq("GPU")),
::testing::Matcher<const std::map<std::string, std::string>&>(_)))
.WillByDefault(Throw(InferenceEngine::GeneralError{""}));
.WillByDefault(Return(gpuMockExeNetwork));
std::shared_ptr<ngraph::Function> simpleNetwork = ngraph::builder::subgraph::makeSingleConv();
ASSERT_NO_THROW(simpleCnnNetwork = InferenceEngine::CNNNetwork(simpleNetwork));
}
};
TEST_P(AutoCTPUTCallMulti, CTPUTDevicesLogicTest) {
TEST_P(AutoCTPUTCallMulti, CTPUTDeviceLoadFailedNoExceptionThrowTest) {
std::vector<std::string> targetDevices;
std::string targetDev;
bool AutoCallMulti;
Config config;
std::tie(AutoCallMulti, targetDevices) = this->GetParam();
std::string loadFailedDevice = targetDevices.size() > 0 ? targetDevices[0] : "";
std::string secondDevice = targetDevices.size() > 1 ? targetDevices[1] : "";
plugin->SetName("MULTI");
for (auto& deviceName : targetDevices) {
targetDev += deviceName;
targetDev += ((deviceName == targetDevices.back()) ? "" : ",");
}
if (AutoCallMulti) {
std::shared_ptr<InferenceEngine::IExecutableNetworkInternal> exeNetwork;
config.insert({{CONFIG_KEY(PERFORMANCE_HINT), InferenceEngine::PluginConfigParams::CUMULATIVE_THROUGHPUT}});
config.insert({InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES, targetDev});
ON_CALL(*core,
LoadNetwork(::testing::Matcher<const InferenceEngine::CNNNetwork&>(_),
::testing::Matcher<const std::string&>(StrEq(loadFailedDevice)),
::testing::Matcher<const std::map<std::string, std::string>&>(_)))
.WillByDefault(Throw(InferenceEngine::GeneralError{""}));
if (loadFailedDevice != CommonTestUtils::DEVICE_CPU) {
EXPECT_CALL(*core,
LoadNetwork(::testing::Matcher<const InferenceEngine::CNNNetwork&>(_),
::testing::Matcher<const std::string&>(CommonTestUtils::DEVICE_CPU),
::testing::Matcher<const std::map<std::string, std::string>&>(_)))
.Times(1);
EXPECT_CALL(*core,
LoadNetwork(::testing::Matcher<const InferenceEngine::CNNNetwork&>(_),
::testing::Matcher<const std::string&>(CommonTestUtils::DEVICE_GPU),
::testing::Matcher<const std::map<std::string, std::string>&>(_)))
.Times(1);
ASSERT_NO_THROW(exeNetwork = plugin->LoadExeNetworkImpl(simpleCnnNetwork, config));
EXPECT_EQ(exeNetwork->GetMetric(ov::execution_devices.name()).as<std::string>(), CommonTestUtils::DEVICE_CPU);
} else {
config.insert({InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES, targetDev});
EXPECT_CALL(*core,
LoadNetwork(::testing::Matcher<const InferenceEngine::CNNNetwork&>(_),
::testing::Matcher<const std::string&>(CommonTestUtils::DEVICE_CPU),
::testing::Matcher<const std::map<std::string, std::string>&>(_)))
.Times(0);
EXPECT_CALL(*core,
LoadNetwork(::testing::Matcher<const InferenceEngine::CNNNetwork&>(_),
::testing::Matcher<const std::string&>(CommonTestUtils::DEVICE_GPU),
::testing::Matcher<const std::map<std::string, std::string>&>(_)))
.Times(1);
EXPECT_THROW(plugin->LoadExeNetworkImpl(simpleCnnNetwork, config), IE::Exception);
}
if (loadFailedDevice != CommonTestUtils::DEVICE_GPU) {
EXPECT_CALL(*core,
LoadNetwork(::testing::Matcher<const InferenceEngine::CNNNetwork&>(_),
::testing::Matcher<const std::string&>(CommonTestUtils::DEVICE_GPU),
::testing::Matcher<const std::map<std::string, std::string>&>(_)))
.Times(1);
}
ASSERT_NO_THROW(exeNetwork = plugin->LoadExeNetworkImpl(simpleCnnNetwork, config));
EXPECT_EQ(exeNetwork->GetMetric(ov::execution_devices.name()).as<std::string>(), secondDevice);
}
const std::vector<ConfigParams> testConfigs = {
ConfigParams{false, {"CPU", "GPU"}},
ConfigParams{false, {"GPU", "CPU"}},
ConfigParams{true, {"CPU", "GPU"}},
ConfigParams{true, {"GPU", "CPU"}},
};

19
src/tests/unit/auto/auto_default_perf_hint_test.cpp
View File

@ -180,13 +180,13 @@ public:
testConfigs.push_back(ConfigParams{
"MULTI:CPU,GPU",
{"CPU", "GPU"},
{{"DEVICE_PROPERTIES", "{GPU:{PERFORMANCE_HINT:LATENCY}}"},
{"MULTI_DEVICE_PRIORITIES", "CPU,GPU"}}}); // CPU: get default_hint:tput GPU: get perf_hint:latency
{{"DEVICE_PROPERTIES", "{GPU:{PERFORMANCE_HINT:THROUGHPUT}}"},
{"MULTI_DEVICE_PRIORITIES", "CPU,GPU"}}}); // CPU: get default_hint:tput GPU: get perf_hint:tput
testConfigs.push_back(ConfigParams{
"MULTI:CPU,GPU",
{"CPU", "GPU"},
{{"DEVICE_PROPERTIES", "{CPU:{PERFORMANCE_HINT:LATENCY},GPU:{PERFORMANCE_HINT:LATENCY}}"},
{"MULTI_DEVICE_PRIORITIES", "CPU,GPU"}}}); // CPU: get perf_hint:lantency GPU: get perf_hint:lantency
{{"DEVICE_PROPERTIES", "{CPU:{PERFORMANCE_HINT:THROUGHPUT},GPU:{PERFORMANCE_HINT:THROUGHPUT}}"},
{"MULTI_DEVICE_PRIORITIES", "CPU,GPU"}}}); // CPU: get perf_hint:tput GPU: get perf_hint:tput
return testConfigs;
}
@ -221,17 +221,17 @@ public:
"MULTI:CPU,GPU",
{"CPU", "GPU"},
{{"CPU", "{ALLOW_AUTO_BATCHING:FALSE}"},
{"MULTI_DEVICE_PRIORITIES", "CPU,GPU"}}}); // CPU: no perf_hint GPU: get default_hint:tput
{"MULTI_DEVICE_PRIORITIES", "CPU,GPU"}}}); // CPU: get default_hint:tput GPU: get default_hint:tput
testConfigs.push_back(ConfigParams{
"MULTI:CPU,GPU",
{"CPU", "GPU"},
{{"DEVICE_PROPERTIES", "GPU:{ALLOW_AUTO_BATCHING:FALSE}}"},
{"MULTI_DEVICE_PRIORITIES", "CPU,GPU"}}}); // CPU: get default_hint:tput GPU: no perf_hint
{"MULTI_DEVICE_PRIORITIES", "CPU,GPU"}}}); // CPU: get default_hint:tput GPU: get default_hint:tput
testConfigs.push_back(ConfigParams{
"MULTI:CPU,GPU",
{"CPU", "GPU"},
{{"DEVICE_PROPERTIES", "CPU:{ALLOW_AUTO_BATCHING:TRUE},GPU:{ALLOW_AUTO_BATCHING:FALSE}}"},
{"MULTI_DEVICE_PRIORITIES", "CPU,GPU"}}}); // CPU: no perf_hint GPU: no perf_hint
{"MULTI_DEVICE_PRIORITIES", "CPU,GPU"}}}); // CPU: no perf_hint GPU: get default_hint:tput
return testConfigs;
}
@ -499,11 +499,6 @@ TEST_P(SecPropAndDefaultPerfHintMockTest, SecPropAndDefaultPerfHintTest) {
// HW default perf_hint
HW_PerfHint = bIsAuto ? "LATENCY" : "THROUGHPUT";
}
auto item = config.find(deviceName);
if (item != config.end() && !isCPUHelper) {
// do not pass default perf_hint to HW
HW_PerfHint = "No PERFORMANCE_HINT";
}
EXPECT_CALL(
*core,