IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[AUTO plugin] Fix benchmark failed to set nstreams on MULTI plugin (#12855)

Browse Source 
* fix benchmark filed to pass on MULTI device when enable num_streams.

Signed-off-by: Wang, Yang <yang4.wang@intel.com>

* Update.

Signed-off-by: Wang, Yang <yang4.wang@intel.com>

* Remove part of test cases because MULTI plguin will not check if the unspported property is valid now.

Signed-off-by: Wang, Yang <yang4.wang@intel.com>

* Remove some incorrect config for MULTI test case since MULTI will pass through those unrecognized config without any exception.

Signed-off-by: Wang, Yang <yang4.wang@intel.com>

* Support MULTI to set nstreams to multi target devices by using ov::device:properties.

Signed-off-by: Wang, Yang <yang4.wang@intel.com>

* Support AUTO to set nstreams with multi target devices bu using ov:device::properties.

Signed-off-by: Wang, Yang <yang4.wang@intel.com>

* Update.

Signed-off-by: Wang, Yang <yang4.wang@intel.com>

* Update.

Signed-off-by: Wang, Yang <yang4.wang@intel.com>

* Update format.

Signed-off-by: Wang, Yang <yang4.wang@intel.com>

Signed-off-by: Wang, Yang <yang4.wang@intel.com>
Co-authored-by: Chen Peter <peter.chen@intel.com>
This commit is contained in:
Wang, Yang
2022-09-27 23:42:40 +08:00
committed by GitHub
parent 2ad15ececc
commit d72e39d8f3
8 changed files with 110 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

88
samples/cpp/benchmark_app/main.cpp
View File

@@ -268,6 +268,24 @@ int main(int argc, char* argv[]) {
}
bool perf_counts = false;
// check if using the virtual device
auto if_auto = std::find(devices.begin(), devices.end(), "AUTO") != devices.end();
auto if_multi = std::find(devices.begin(), devices.end(), "MULTI") != devices.end();
// Remove the hardware devices if AUTO/MULTI appears in the devices list.
if (if_auto || if_multi) {
devices.clear();
std::string virtual_device;
if (if_auto) {
virtual_device = "AUTO";
devices.push_back("AUTO");
}
if (if_multi) {
virtual_device = "MULTI";
devices.push_back("MULTI");
}
parse_value_for_virtual_device(virtual_device, device_nstreams);
parse_value_for_virtual_device(virtual_device, device_infer_precision);
}
// Update config per device according to command line parameters
for (auto& device : devices) {
auto& device_config = config[device];
@@ -320,6 +338,22 @@ int main(int argc, char* argv[]) {
// Use API 2.0 key for streams
key = ov::num_streams.name();
device_config[key] = it_device_nstreams->second;
} else if (device == "MULTI" || device == "AUTO") {
// check if the element contains the hardware device property
auto value_vec = split(it_device_nstreams->second, ' ');
if (value_vec.size() == 1) {
key = ov::num_streams.name();
device_config[key] = it_device_nstreams->second;
} else {
// set device nstreams properties in the AUTO/MULTI plugin
std::stringstream strm(it_device_nstreams->second);
std::map<std::string, std::string> devices_property;
ov::util::Read<std::map<std::string, std::string>>{}(strm, devices_property);
for (auto it : devices_property) {
device_config.insert(
ov::device::properties(it.first, ov::num_streams(std::stoi(it.second))));
}
}
} else {
throw std::logic_error("Device " + device + " doesn't support config key '" + key + "' " +
"and '" + ov::num_streams.name() + "'!" +
@@ -383,41 +417,41 @@ int main(int argc, char* argv[]) {
device_config.emplace(ov::affinity(fix_pin_option(FLAGS_pin)));
}
if (device.find("CPU") != std::string::npos) { // CPU supports few special performance-oriented keys
// limit threading for CPU portion of inference
if (!isFlagSetInCommandLine("pin")) {
auto it_affinity = device_config.find(ov::affinity.name());
if (it_affinity != device_config.end() && (device_name.find("MULTI") != std::string::npos) &&
(device_name.find("GPU") != std::string::npos)) {
slog::warn << "Turn off threads pinning for " << device
<< " device since multi-scenario with GPU device is used." << slog::endl;
it_affinity->second = ov::Affinity::NONE;
}
}
// for CPU execution, more throughput-oriented execution via streams
if (device.find("CPU") != std::string::npos || device.find("GPU") != std::string::npos) {
// CPU supports few special performance-oriented keys
// for CPU and GPU execution, more throughput-oriented execution via streams
setThroughputStreams();
set_infer_precision();
} else if (device.find("GPU") != std::string::npos) {
// for GPU execution, more throughput-oriented execution via streams
setThroughputStreams();
set_infer_precision();
if ((device_name.find("MULTI") != std::string::npos) &&
(device_name.find("CPU") != std::string::npos)) {
slog::warn << "GPU throttling is turned on. Multi-device execution with "
"the CPU + GPU performs best with GPU throttling hint, "
<< "which releases another CPU thread (that is otherwise "
"used by the GPU driver for active polling)."
<< slog::endl;
device_config[GPU_CONFIG_KEY(PLUGIN_THROTTLE)] = "1";
}
} else if (device.find("MYRIAD") != std::string::npos) {
device_config.emplace(ov::log::level(ov::log::Level::WARNING));
setThroughputStreams();
} else if (device.find("GNA") != std::string::npos) {
set_infer_precision();
} else if (device.find("AUTO") != std::string::npos) {
setThroughputStreams();
set_infer_precision();
device_nstreams.erase(device);
} else if (device.find("MULTI") != std::string::npos) {
setThroughputStreams();
set_infer_precision();
if ((device_name.find("GPU") != std::string::npos) && (device_name.find("CPU") != std::string::npos)) {
slog::warn << "GPU throttling is turned on. Multi-device execution with "
"the CPU + GPU performs best with GPU throttling hint, "
<< "which releases another CPU thread (that is otherwise "
"used by the GPU driver for active polling)."
<< slog::endl;
device_config.insert(ov::device::properties("GPU", {{GPU_CONFIG_KEY(PLUGIN_THROTTLE), 1}}));
// limit threading for CPU portion of inference
if (!isFlagSetInCommandLine("pin")) {
auto it_affinity = device_config.find(ov::affinity.name());
if (it_affinity != device_config.end()) {
slog::warn << "Turn off threads pinning for " << device
<< " device since multi-scenario with GPU device is used." << slog::endl;
it_affinity->second = ov::Affinity::NONE;
}
}
}
device_nstreams.erase(device);
}
}

45
samples/cpp/benchmark_app/utils.cpp
View File

@@ -108,12 +108,11 @@ std::vector<float> split_float(const std::string& s, char delim) {
std::vector<std::string> parse_devices(const std::string& device_string) {
std::string comma_separated_devices = device_string;
auto colon = comma_separated_devices.find(":");
std::vector<std::string> result;
if (colon != std::string::npos) {
auto target_device = comma_separated_devices.substr(0, colon);
if (target_device == "AUTO" || target_device == "MULTI") {
std::vector<std::string> result;
result.push_back(target_device);
return result;
}
auto bracket = comma_separated_devices.find("("); // e.g. in BATCH:GPU(4)
comma_separated_devices = comma_separated_devices.substr(colon + 1, bracket - colon - 1);
@@ -122,7 +121,47 @@ std::vector<std::string> parse_devices(const std::string& device_string) {
return std::vector<std::string>();
auto devices = split(comma_separated_devices, ',');
return devices;
result.insert(result.end(), devices.begin(), devices.end());
return result;
}
void parse_value_for_virtual_device(const std::string& device, std::map<std::string, std::string>& values_string) {
auto item_virtual = values_string.find(device);
if (item_virtual != values_string.end() && values_string.size() > 1) {
if (device == "MULTI") {
// Remove the element that the key is virtual device MULTI
// e.g. MULTI:xxx,xxx -nstreams 2 will set nstreams 2 to CPU.
values_string.erase(item_virtual);
} else if (device == "AUTO") {
// Just keep the element that the key is virtual device AUTO
// e.g. AUTO:xxx,xxx -nstreams 2 will trigger exception that AUTO plugin didn't support nstream property.
auto value = item_virtual->second;
values_string.clear();
values_string[device] = value;
return;
}
}
auto iter = values_string.begin();
while (iter != values_string.end()) {
if (iter->first == device) {
iter++;
continue;
}
values_string[device] += iter->first + " " + iter->second + " ";
iter = values_string.erase(iter);
}
if (values_string.find(device) != values_string.end()) {
auto& nstreams = values_string[device];
// Remove the space at the tail.
nstreams.erase(std::find_if(nstreams.rbegin(),
nstreams.rend(),
[](unsigned char ch) {
return !std::isspace(ch);
})
.base(),
nstreams.end());
}
return;
}
std::map<std::string, std::string> parse_value_per_device(const std::vector<std::string>& devices,

1
samples/cpp/benchmark_app/utils.hpp
View File

@@ -58,6 +58,7 @@ std::vector<std::string> parse_devices(const std::string& device_string);
uint32_t device_default_device_duration_in_seconds(const std::string& device);
std::map<std::string, std::string> parse_value_per_device(const std::vector<std::string>& devices,
const std::string& values_string);
void parse_value_for_virtual_device(const std::string& device, std::map<std::string, std::string>& values_string);
std::string get_shape_string(const ov::Shape& shape);
std::string get_shapes_string(const benchmark_app::PartialShapes& shapes);
size_t get_batch_size(const benchmark_app::InputsInfo& inputs_info);

4
src/plugins/auto/plugin.cpp
View File

@@ -267,7 +267,7 @@ InferenceEngine::Parameter MultiDeviceInferencePlugin::GetConfig(const std::stri
}
void MultiDeviceInferencePlugin::SetConfig(const std::map<std::string, std::string> & config) {
_pluginConfig.UpdateFromMap(config, GetName(), true);
_pluginConfig.UpdateFromMap(config, GetName());
}
static const Version version = {{2, 1}, CI_BUILD_NUMBER, "MultiDevicePlugin"};
@@ -368,7 +368,7 @@ IExecutableNetworkInternal::Ptr MultiDeviceInferencePlugin::LoadNetworkImpl(cons
_LogTag = GetName();
auto loadConfig = _pluginConfig;
// updateFromMap will check config valid
loadConfig.UpdateFromMap(config, GetName(), false);
loadConfig.UpdateFromMap(config, GetName());
auto fullConfig = loadConfig._keyConfigMap;
// collect the settings that are applicable to the devices we are loading the network to
std::unordered_map<std::string, InferenceEngine::Parameter> multiNetworkConfig;

4
src/plugins/auto/utils/config.hpp
View File

@@ -28,7 +28,7 @@ struct PluginConfig {
adjustKeyMapValues();
}
void UpdateFromMap(const std::map<std::string, std::string>& config, const std::string& pluginName, bool checkValid = false) {
void UpdateFromMap(const std::map<std::string, std::string>& config, const std::string& pluginName) {
const auto perf_hints_configs = PerfHintsConfig::SupportedKeys();
for (auto&& kvp : config) {
if (kvp.first == ov::enable_profiling) {
@@ -103,7 +103,7 @@ struct PluginConfig {
} else if (kvp.first.find("AUTO_") == 0) {
_passThroughConfig.emplace(kvp.first, kvp.second);
} else {
if (pluginName.find("AUTO") != std::string::npos || checkValid)
if (pluginName.find("AUTO") != std::string::npos)
IE_THROW(NotFound) << "Unsupported property " << kvp.first;
else
_passThroughConfig.emplace(kvp.first, kvp.second);

6
src/plugins/intel_cpu/tests/functional/shared_tests_instances/behavior/plugin/configuration_tests.cpp
View File

@@ -163,12 +163,6 @@ namespace {
{InferenceEngine::PluginConfigParams::KEY_PERF_COUNT, "OFF"}},
{{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES , CommonTestUtils::DEVICE_CPU},
{InferenceEngine::PluginConfigParams::KEY_EXCLUSIVE_ASYNC_REQUESTS, "OFF"}},
{{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES , CommonTestUtils::DEVICE_CPU},
{InferenceEngine::PluginConfigParams::KEY_CPU_THROUGHPUT_STREAMS, "OFF"}},
{{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES , CommonTestUtils::DEVICE_CPU},
{InferenceEngine::PluginConfigParams::KEY_CPU_BIND_THREAD, "OFF"}},
{{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES , CommonTestUtils::DEVICE_CPU},
{InferenceEngine::PluginConfigParams::KEY_DYN_BATCH_LIMIT, "NAN"}}
};
const std::vector<std::map<std::string, std::string>> autoinconfigs = {

3
src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/plugin/configuration_tests.cpp
View File

@@ -96,8 +96,7 @@ INSTANTIATE_TEST_SUITE_P(ie_plugin_Hetero, CorrectConfigTests,
{{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT, InferenceEngine::PluginConfigParams::LATENCY},
{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT_NUM_REQUESTS, "-1"}},
{{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT, InferenceEngine::PluginConfigParams::THROUGHPUT},
{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT_NUM_REQUESTS, "should be int"}},
{{InferenceEngine::PluginConfigParams::KEY_DYN_BATCH_LIMIT, "NAN"}}
{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT_NUM_REQUESTS, "should be int"}}
};
INSTANTIATE_TEST_SUITE_P(ie_plugin, IncorrectConfigTests,

10
src/tests/functional/plugin/gpu/shared_tests_instances/behavior/plugin/configuration_tests.cpp
View File

@@ -38,15 +38,7 @@ namespace {
{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT, InferenceEngine::PluginConfigParams::LATENCY},
{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT_NUM_REQUESTS, "-1"}},
{{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES , CommonTestUtils::DEVICE_GPU},
{InferenceEngine::PluginConfigParams::KEY_PERF_COUNT, "ON"}},
{{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES , CommonTestUtils::DEVICE_GPU},
{InferenceEngine::PluginConfigParams::KEY_CONFIG_FILE, "unknown_file"}},
{{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES , CommonTestUtils::DEVICE_GPU},
{InferenceEngine::PluginConfigParams::KEY_DUMP_KERNELS, "ON"}},
{{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES , CommonTestUtils::DEVICE_GPU},
{InferenceEngine::PluginConfigParams::KEY_TUNING_MODE, "TUNING_UNKNOWN_MODE"}},
{{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES , CommonTestUtils::DEVICE_GPU},
{InferenceEngine::PluginConfigParams::KEY_DEVICE_ID, "DEVICE_UNKNOWN"}}
{InferenceEngine::PluginConfigParams::KEY_PERF_COUNT, "ON"}}
};