IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[AUTO] Update logic of setting TPUT by default for MULTI and AUTO. (#12279)

Browse Source 
* update logic of setting TPUT by default for MULTI and AUTO.

Signed-off-by: Wang, Yang <yang4.wang@intel.com>

* Set config to MULTI instaed of target device when using -d MULTI:target_device in benchmark app.

Signed-off-by: Wang, Yang <yang4.wang@intel.com>

* Update.

Signed-off-by: Wang, Yang <yang4.wang@intel.com>

* Update logit to check if perfomance hint is set.

Signed-off-by: Wang, Yang <yang4.wang@intel.com>

* Update test cases.

Signed-off-by: Wang, Yang <yang4.wang@intel.com>

* Set the correct _so from passthrough executable network when batch plugin is disabel to fix lifecycle coredump.

Signed-off-by: Wang, Yang <yang4.wang@intel.com>

* Same update for AUTO plugin

Signed-off-by: Wang, Yang <yang4.wang@intel.com>

* Update

Signed-off-by: Wang, Yang <yang4.wang@intel.com>

* Update

Signed-off-by: Wang, Yang <yang4.wang@intel.com>

* Update.

Signed-off-by: Wang, Yang <yang4.wang@intel.com>

Signed-off-by: Wang, Yang <yang4.wang@intel.com>
Co-authored-by: Chen Peter <peter.chen@intel.com>
This commit is contained in:
Wang, Yang 2022-08-26 17:31:22 +08:00 committed by GitHub
parent 19c4bad915
commit 93509147d5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 66 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
samples/cpp/benchmark_app/utils.cpp
View File

@ -109,9 +109,10 @@ std::vector<std::string> parse_devices(const std::string& device_string) {
std::string comma_separated_devices = device_string;
auto colon = comma_separated_devices.find(":");
if (colon != std::string::npos) {
if (comma_separated_devices.substr(0, colon) == "AUTO") {
auto target_device = comma_separated_devices.substr(0, colon);
if (target_device == "AUTO" || target_device == "MULTI") {
std::vector<std::string> result;
result.push_back("AUTO");
result.push_back(target_device);
return result;
}
auto bracket = comma_separated_devices.find("("); // e.g. in BATCH:GPU(4)

11
src/plugins/auto/auto_schedule.cpp
View File

@ -557,10 +557,15 @@ IInferPtr AutoSchedule::CreateInferRequest() {
} catch(...) {
LOG_INFO("query perf hint from passthrough network failed");
}
if (_autoSContext->_batchingDisabled || perfmode != CONFIG_VALUE(THROUGHPUT))
if (_autoSContext->_batchingDisabled || perfmode != CONFIG_VALUE(THROUGHPUT)) {
syncRequestImpl->setPointerToSo(_passthroughExeNet._so);
else
syncRequestImpl->setPointerToSo(_passthroughExeNet._ptr->GetPointerToSo());
} else {
auto so = _passthroughExeNet._ptr->GetPointerToSo();
// Get the _so from passthrough executable network when batch plugin is disable.
if (!so)
so = _passthroughExeNet._so;
syncRequestImpl->setPointerToSo(so);
}
}
return std::make_shared<AsyncInferRequest>(shared_from_this(),
syncRequestImpl,

11
src/plugins/auto/multi_schedule.cpp
View File

@ -311,10 +311,15 @@ IInferPtr MultiSchedule::CreateInferRequest() {
} catch(...) {
LOG_INFO("query perf hint from passthrough network failed");
}
if (_multiSContext->_batchingDisabled || perfmode != CONFIG_VALUE(THROUGHPUT))
if (_multiSContext->_batchingDisabled || perfmode != CONFIG_VALUE(THROUGHPUT)) {
syncRequestImpl->setPointerToSo(_passthroughExeNet._so);
else
syncRequestImpl->setPointerToSo(_passthroughExeNet._ptr->GetPointerToSo());
} else {
auto so = _passthroughExeNet._ptr->GetPointerToSo();
// Get the _so from passthrough executable network when batch plugin is disable.
if (!so)
so = _passthroughExeNet._so;
syncRequestImpl->setPointerToSo(so);
}
}
return std::make_shared<AsyncInferRequest>(shared_from_this(),
syncRequestImpl,

45
src/plugins/auto/plugin.cpp
View File

@ -53,13 +53,6 @@ namespace {
return METRIC_VALUE(FP32);
}
std::map<std::string, std::string> mergeConfigs(std::map<std::string, std::string> config,
const std::map<std::string, std::string> & local) {
for (auto && kvp : local) {
config[kvp.first] = kvp.second;
}
return config;
}
std::vector<std::string> supported_configKeys = []() -> decltype(PerfHintsConfig::SupportedKeys()) {
auto res = PerfHintsConfig::SupportedKeys();
res.push_back(ov::device::priorities.name());
@ -85,10 +78,37 @@ std::vector<DeviceInformation> MultiDeviceInferencePlugin::ParseMetaDevices(cons
// parsing the string and splitting to tokens
std::vector<std::string> devicesWithRequests = _pluginConfig.ParsePrioritiesDevices(priorities);
auto setTputAsDefault = [&](const std::string& targetDevice,
std::map<std::string, std::string>& deviceConfig,
const std::map<std::string, std::string>& mergedConfig) {
// Default value of PERFORMACE_HINT is empty string.
auto iter = mergedConfig.find(PluginConfigParams::KEY_PERFORMANCE_HINT);
if (GetName() == "AUTO" && iter->second.empty() &&
mergedConfig.find(targetDevice) == mergedConfig.end()) {
// setting tput as the default performance mode if no hints setting for AUTO plugin and no properties
// specified for target device.
deviceConfig[PluginConfigParams::KEY_PERFORMANCE_HINT] = PluginConfigParams::THROUGHPUT;
return;
}
// set TPUT for MULTI if no above propertis were set by user
if (GetName() == "MULTI") {
if (!iter->second.empty() || mergedConfig.find(targetDevice) != mergedConfig.end())
return;
for (auto&& kvp : mergedConfig) {
if (kvp.first == ov::affinity || kvp.first == ov::num_streams ||
kvp.first == ov::inference_num_threads) {
return;
}
}
deviceConfig[PluginConfigParams::KEY_PERFORMANCE_HINT] = PluginConfigParams::THROUGHPUT;
}
};
auto getDeviceConfig = [&] (const DeviceName & deviceWithID) {
DeviceIDParser deviceParser(deviceWithID);
std::string deviceName = deviceParser.getDeviceName();
std::map<std::string, std::string> tconfig = mergeConfigs(_config, config);
std::map<std::string, std::string> tconfig = config;
// set device ID if any
std::string deviceIDLocal = deviceParser.getDeviceID();
@ -96,14 +116,7 @@ std::vector<DeviceInformation> MultiDeviceInferencePlugin::ParseMetaDevices(cons
tconfig[PluginConfigParams::KEY_DEVICE_ID] = deviceIDLocal;
}
auto deviceConfig = GetCore()->GetSupportedConfig(deviceName, tconfig);
auto iter = deviceConfig.find(PluginConfigParams::KEY_PERFORMANCE_HINT);
bool flag = (iter == deviceConfig.end()) || iter->second.empty();
if (GetName() == "AUTO" && flag &&
tconfig.find(deviceName) == tconfig.end()) {
// setting tput as the default performance mode if no hints setting for AUTO plugin and no properties
// specified for target device.
deviceConfig[PluginConfigParams::KEY_PERFORMANCE_HINT] = PluginConfigParams::THROUGHPUT;
}
setTputAsDefault(deviceName, deviceConfig, tconfig);
return deviceConfig;
};

20
src/plugins/intel_cpu/tests/functional/shared_tests_instances/behavior/ov_plugin/core_integration.cpp
View File

@ -278,20 +278,30 @@ const std::vector<ov::AnyMap> default_properties = {{ov::device::priorities("CPU
const std::vector<ov::AnyMap> auto_default_properties = {{}};
INSTANTIATE_TEST_SUITE_P(smoke_OVClassLoadNetworkWithDefaultPropertiesTest,
OVClassLoadNetworkWithDefaultPropertiesTest,
::testing::Combine(::testing::Values(CommonTestUtils::DEVICE_AUTO),
::testing::Combine(::testing::Values(CommonTestUtils::DEVICE_AUTO, CommonTestUtils::DEVICE_MULTI),
::testing::ValuesIn(default_properties)));
INSTANTIATE_TEST_SUITE_P(smoke_AUTO_OVClassLoadNetworkWithDefaultPropertiesTest,
OVClassLoadNetworkWithDefaultPropertiesTest,
::testing::Combine(::testing::Values("AUTO:CPU"),
::testing::Combine(::testing::Values("AUTO:CPU", "MULTI:CPU"),
::testing::ValuesIn(auto_default_properties)));
const std::vector<ov::AnyMap> default_incorrect_properties = {
{ov::device::priorities("CPU"), ov::device::properties("CPU", ov::hint::allow_auto_batching(true))},
{ov::device::priorities("CPU"), ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY)},
{ov::device::priorities("CPU"), ov::device::properties("CPU", ov::hint::allow_auto_batching(true))}
};
const std::vector<ov::AnyMap> default_multi_incorrect_properties = {
{ov::device::priorities("CPU"), ov::affinity(ov::Affinity::NONE)},
{ov::device::priorities("CPU"), ov::num_streams(ov::streams::AUTO)},
{ov::device::priorities("CPU"), ov::inference_num_threads(1)}
};
const std::vector<ov::AnyMap> auto_default_incorrect_properties = {{}};
INSTANTIATE_TEST_SUITE_P(smoke_OVClassLoadNetworkWithDefaultIncorrectPropertiesTest,
OVClassLoadNetworkWithDefaultIncorrectPropertiesTest,
::testing::Combine(::testing::Values(CommonTestUtils::DEVICE_AUTO),
::testing::Combine(::testing::Values(CommonTestUtils::DEVICE_AUTO, CommonTestUtils::DEVICE_MULTI),
::testing::ValuesIn(default_incorrect_properties)));
INSTANTIATE_TEST_SUITE_P(smoke_Multi_OVClassLoadNetworkWithDefaultIncorrectPropertiesTest,
OVClassLoadNetworkWithDefaultIncorrectPropertiesTest,
::testing::Combine(::testing::Values(CommonTestUtils::DEVICE_MULTI),
::testing::ValuesIn(default_multi_incorrect_properties)));
} // namespace

6
src/tests/functional/plugin/shared/include/behavior/ov_plugin/core_integration.hpp
View File

@ -1086,7 +1086,7 @@ TEST_P(OVClassLoadNetworkWithDefaultPropertiesTest, LoadNetworkWithDefaultProper
ov::Core ie = createCoreWithTemplate();
ov::CompiledModel model;
OV_ASSERT_NO_THROW(model = ie.compile_model(actualNetwork, target_device, configuration));
ov::hint::PerformanceMode value;
ov::hint::PerformanceMode value = ov::hint::PerformanceMode::UNDEFINED;
OV_ASSERT_NO_THROW(value = model.get_property(ov::hint::performance_mode));
ASSERT_EQ(value, ov::hint::PerformanceMode::THROUGHPUT);
}
@ -1095,9 +1095,9 @@ TEST_P(OVClassLoadNetworkWithDefaultIncorrectPropertiesTest, LoadNetworkWithDefa
ov::Core ie = createCoreWithTemplate();
ov::CompiledModel model;
OV_ASSERT_NO_THROW(model = ie.compile_model(actualNetwork, target_device, configuration));
ov::hint::PerformanceMode value;
ov::hint::PerformanceMode value = ov::hint::PerformanceMode::THROUGHPUT;
OV_ASSERT_NO_THROW(value = model.get_property(ov::hint::performance_mode));
ASSERT_EQ(value, ov::hint::PerformanceMode::UNDEFINED);
ASSERT_NE(value, ov::hint::PerformanceMode::THROUGHPUT);
}
TEST_P(OVClassLoadNetworkTest, LoadNetworkWithInvalidDeviceIDThrows) {