diff --git a/samples/cpp/benchmark_app/main.cpp b/samples/cpp/benchmark_app/main.cpp index 5150d7e9499..7d382b2c09c 100644 --- a/samples/cpp/benchmark_app/main.cpp +++ b/samples/cpp/benchmark_app/main.cpp @@ -268,6 +268,24 @@ int main(int argc, char* argv[]) { } bool perf_counts = false; + // check if using the virtual device + auto if_auto = std::find(devices.begin(), devices.end(), "AUTO") != devices.end(); + auto if_multi = std::find(devices.begin(), devices.end(), "MULTI") != devices.end(); + // Remove the hardware devices if AUTO/MULTI appears in the devices list. + if (if_auto || if_multi) { + devices.clear(); + std::string virtual_device; + if (if_auto) { + virtual_device = "AUTO"; + devices.push_back("AUTO"); + } + if (if_multi) { + virtual_device = "MULTI"; + devices.push_back("MULTI"); + } + parse_value_for_virtual_device(virtual_device, device_nstreams); + parse_value_for_virtual_device(virtual_device, device_infer_precision); + } // Update config per device according to command line parameters for (auto& device : devices) { auto& device_config = config[device]; @@ -320,6 +338,22 @@ int main(int argc, char* argv[]) { // Use API 2.0 key for streams key = ov::num_streams.name(); device_config[key] = it_device_nstreams->second; + } else if (device == "MULTI" || device == "AUTO") { + // check if the element contains the hardware device property + auto value_vec = split(it_device_nstreams->second, ' '); + if (value_vec.size() == 1) { + key = ov::num_streams.name(); + device_config[key] = it_device_nstreams->second; + } else { + // set device nstreams properties in the AUTO/MULTI plugin + std::stringstream strm(it_device_nstreams->second); + std::map devices_property; + ov::util::Read>{}(strm, devices_property); + for (auto it : devices_property) { + device_config.insert( + ov::device::properties(it.first, ov::num_streams(std::stoi(it.second)))); + } + } } else { throw std::logic_error("Device " + device + " doesn't support config key '" + key + "' " + "and '" + ov::num_streams.name() + "'!" + @@ -383,41 +417,41 @@ int main(int argc, char* argv[]) { device_config.emplace(ov::affinity(fix_pin_option(FLAGS_pin))); } - if (device.find("CPU") != std::string::npos) { // CPU supports few special performance-oriented keys - // limit threading for CPU portion of inference - if (!isFlagSetInCommandLine("pin")) { - auto it_affinity = device_config.find(ov::affinity.name()); - if (it_affinity != device_config.end() && (device_name.find("MULTI") != std::string::npos) && - (device_name.find("GPU") != std::string::npos)) { - slog::warn << "Turn off threads pinning for " << device - << " device since multi-scenario with GPU device is used." << slog::endl; - it_affinity->second = ov::Affinity::NONE; - } - } - - // for CPU execution, more throughput-oriented execution via streams + if (device.find("CPU") != std::string::npos || device.find("GPU") != std::string::npos) { + // CPU supports few special performance-oriented keys + // for CPU and GPU execution, more throughput-oriented execution via streams setThroughputStreams(); set_infer_precision(); - } else if (device.find("GPU") != std::string::npos) { - // for GPU execution, more throughput-oriented execution via streams - setThroughputStreams(); - set_infer_precision(); - - if ((device_name.find("MULTI") != std::string::npos) && - (device_name.find("CPU") != std::string::npos)) { - slog::warn << "GPU throttling is turned on. Multi-device execution with " - "the CPU + GPU performs best with GPU throttling hint, " - << "which releases another CPU thread (that is otherwise " - "used by the GPU driver for active polling)." - << slog::endl; - device_config[GPU_CONFIG_KEY(PLUGIN_THROTTLE)] = "1"; - } } else if (device.find("MYRIAD") != std::string::npos) { device_config.emplace(ov::log::level(ov::log::Level::WARNING)); setThroughputStreams(); } else if (device.find("GNA") != std::string::npos) { set_infer_precision(); } else if (device.find("AUTO") != std::string::npos) { + setThroughputStreams(); + set_infer_precision(); + device_nstreams.erase(device); + } else if (device.find("MULTI") != std::string::npos) { + setThroughputStreams(); + set_infer_precision(); + if ((device_name.find("GPU") != std::string::npos) && (device_name.find("CPU") != std::string::npos)) { + slog::warn << "GPU throttling is turned on. Multi-device execution with " + "the CPU + GPU performs best with GPU throttling hint, " + << "which releases another CPU thread (that is otherwise " + "used by the GPU driver for active polling)." + << slog::endl; + + device_config.insert(ov::device::properties("GPU", {{GPU_CONFIG_KEY(PLUGIN_THROTTLE), 1}})); + // limit threading for CPU portion of inference + if (!isFlagSetInCommandLine("pin")) { + auto it_affinity = device_config.find(ov::affinity.name()); + if (it_affinity != device_config.end()) { + slog::warn << "Turn off threads pinning for " << device + << " device since multi-scenario with GPU device is used." << slog::endl; + it_affinity->second = ov::Affinity::NONE; + } + } + } device_nstreams.erase(device); } } diff --git a/samples/cpp/benchmark_app/utils.cpp b/samples/cpp/benchmark_app/utils.cpp index 77e8a86786c..ae19ef88ab4 100644 --- a/samples/cpp/benchmark_app/utils.cpp +++ b/samples/cpp/benchmark_app/utils.cpp @@ -108,12 +108,11 @@ std::vector split_float(const std::string& s, char delim) { std::vector parse_devices(const std::string& device_string) { std::string comma_separated_devices = device_string; auto colon = comma_separated_devices.find(":"); + std::vector result; if (colon != std::string::npos) { auto target_device = comma_separated_devices.substr(0, colon); if (target_device == "AUTO" || target_device == "MULTI") { - std::vector result; result.push_back(target_device); - return result; } auto bracket = comma_separated_devices.find("("); // e.g. in BATCH:GPU(4) comma_separated_devices = comma_separated_devices.substr(colon + 1, bracket - colon - 1); @@ -122,7 +121,47 @@ std::vector parse_devices(const std::string& device_string) { return std::vector(); auto devices = split(comma_separated_devices, ','); - return devices; + result.insert(result.end(), devices.begin(), devices.end()); + return result; +} + +void parse_value_for_virtual_device(const std::string& device, std::map& values_string) { + auto item_virtual = values_string.find(device); + if (item_virtual != values_string.end() && values_string.size() > 1) { + if (device == "MULTI") { + // Remove the element that the key is virtual device MULTI + // e.g. MULTI:xxx,xxx -nstreams 2 will set nstreams 2 to CPU. + values_string.erase(item_virtual); + } else if (device == "AUTO") { + // Just keep the element that the key is virtual device AUTO + // e.g. AUTO:xxx,xxx -nstreams 2 will trigger exception that AUTO plugin didn't support nstream property. + auto value = item_virtual->second; + values_string.clear(); + values_string[device] = value; + return; + } + } + auto iter = values_string.begin(); + while (iter != values_string.end()) { + if (iter->first == device) { + iter++; + continue; + } + values_string[device] += iter->first + " " + iter->second + " "; + iter = values_string.erase(iter); + } + if (values_string.find(device) != values_string.end()) { + auto& nstreams = values_string[device]; + // Remove the space at the tail. + nstreams.erase(std::find_if(nstreams.rbegin(), + nstreams.rend(), + [](unsigned char ch) { + return !std::isspace(ch); + }) + .base(), + nstreams.end()); + } + return; } std::map parse_value_per_device(const std::vector& devices, diff --git a/samples/cpp/benchmark_app/utils.hpp b/samples/cpp/benchmark_app/utils.hpp index 6a514eafccf..c1825201bef 100644 --- a/samples/cpp/benchmark_app/utils.hpp +++ b/samples/cpp/benchmark_app/utils.hpp @@ -58,6 +58,7 @@ std::vector parse_devices(const std::string& device_string); uint32_t device_default_device_duration_in_seconds(const std::string& device); std::map parse_value_per_device(const std::vector& devices, const std::string& values_string); +void parse_value_for_virtual_device(const std::string& device, std::map& values_string); std::string get_shape_string(const ov::Shape& shape); std::string get_shapes_string(const benchmark_app::PartialShapes& shapes); size_t get_batch_size(const benchmark_app::InputsInfo& inputs_info); diff --git a/src/plugins/auto/plugin.cpp b/src/plugins/auto/plugin.cpp index 2b768b97010..4f047aff2e0 100644 --- a/src/plugins/auto/plugin.cpp +++ b/src/plugins/auto/plugin.cpp @@ -267,7 +267,7 @@ InferenceEngine::Parameter MultiDeviceInferencePlugin::GetConfig(const std::stri } void MultiDeviceInferencePlugin::SetConfig(const std::map & config) { - _pluginConfig.UpdateFromMap(config, GetName(), true); + _pluginConfig.UpdateFromMap(config, GetName()); } static const Version version = {{2, 1}, CI_BUILD_NUMBER, "MultiDevicePlugin"}; @@ -368,7 +368,7 @@ IExecutableNetworkInternal::Ptr MultiDeviceInferencePlugin::LoadNetworkImpl(cons _LogTag = GetName(); auto loadConfig = _pluginConfig; // updateFromMap will check config valid - loadConfig.UpdateFromMap(config, GetName(), false); + loadConfig.UpdateFromMap(config, GetName()); auto fullConfig = loadConfig._keyConfigMap; // collect the settings that are applicable to the devices we are loading the network to std::unordered_map multiNetworkConfig; diff --git a/src/plugins/auto/utils/config.hpp b/src/plugins/auto/utils/config.hpp index 4909aff82f1..bc8b6737c80 100644 --- a/src/plugins/auto/utils/config.hpp +++ b/src/plugins/auto/utils/config.hpp @@ -28,7 +28,7 @@ struct PluginConfig { adjustKeyMapValues(); } - void UpdateFromMap(const std::map& config, const std::string& pluginName, bool checkValid = false) { + void UpdateFromMap(const std::map& config, const std::string& pluginName) { const auto perf_hints_configs = PerfHintsConfig::SupportedKeys(); for (auto&& kvp : config) { if (kvp.first == ov::enable_profiling) { @@ -103,7 +103,7 @@ struct PluginConfig { } else if (kvp.first.find("AUTO_") == 0) { _passThroughConfig.emplace(kvp.first, kvp.second); } else { - if (pluginName.find("AUTO") != std::string::npos || checkValid) + if (pluginName.find("AUTO") != std::string::npos) IE_THROW(NotFound) << "Unsupported property " << kvp.first; else _passThroughConfig.emplace(kvp.first, kvp.second); diff --git a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/behavior/plugin/configuration_tests.cpp b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/behavior/plugin/configuration_tests.cpp index 31b967b9607..df55baa80f8 100644 --- a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/behavior/plugin/configuration_tests.cpp +++ b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/behavior/plugin/configuration_tests.cpp @@ -163,12 +163,6 @@ namespace { {InferenceEngine::PluginConfigParams::KEY_PERF_COUNT, "OFF"}}, {{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES , CommonTestUtils::DEVICE_CPU}, {InferenceEngine::PluginConfigParams::KEY_EXCLUSIVE_ASYNC_REQUESTS, "OFF"}}, - {{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES , CommonTestUtils::DEVICE_CPU}, - {InferenceEngine::PluginConfigParams::KEY_CPU_THROUGHPUT_STREAMS, "OFF"}}, - {{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES , CommonTestUtils::DEVICE_CPU}, - {InferenceEngine::PluginConfigParams::KEY_CPU_BIND_THREAD, "OFF"}}, - {{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES , CommonTestUtils::DEVICE_CPU}, - {InferenceEngine::PluginConfigParams::KEY_DYN_BATCH_LIMIT, "NAN"}} }; const std::vector> autoinconfigs = { diff --git a/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/plugin/configuration_tests.cpp b/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/plugin/configuration_tests.cpp index feb8844e834..997e11977f8 100644 --- a/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/plugin/configuration_tests.cpp +++ b/src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/plugin/configuration_tests.cpp @@ -96,8 +96,7 @@ INSTANTIATE_TEST_SUITE_P(ie_plugin_Hetero, CorrectConfigTests, {{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT, InferenceEngine::PluginConfigParams::LATENCY}, {InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT_NUM_REQUESTS, "-1"}}, {{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT, InferenceEngine::PluginConfigParams::THROUGHPUT}, - {InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT_NUM_REQUESTS, "should be int"}}, - {{InferenceEngine::PluginConfigParams::KEY_DYN_BATCH_LIMIT, "NAN"}} + {InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT_NUM_REQUESTS, "should be int"}} }; INSTANTIATE_TEST_SUITE_P(ie_plugin, IncorrectConfigTests, diff --git a/src/tests/functional/plugin/gpu/shared_tests_instances/behavior/plugin/configuration_tests.cpp b/src/tests/functional/plugin/gpu/shared_tests_instances/behavior/plugin/configuration_tests.cpp index 7dc9fbf355c..29326da725c 100644 --- a/src/tests/functional/plugin/gpu/shared_tests_instances/behavior/plugin/configuration_tests.cpp +++ b/src/tests/functional/plugin/gpu/shared_tests_instances/behavior/plugin/configuration_tests.cpp @@ -38,15 +38,7 @@ namespace { {InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT, InferenceEngine::PluginConfigParams::LATENCY}, {InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT_NUM_REQUESTS, "-1"}}, {{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES , CommonTestUtils::DEVICE_GPU}, - {InferenceEngine::PluginConfigParams::KEY_PERF_COUNT, "ON"}}, - {{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES , CommonTestUtils::DEVICE_GPU}, - {InferenceEngine::PluginConfigParams::KEY_CONFIG_FILE, "unknown_file"}}, - {{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES , CommonTestUtils::DEVICE_GPU}, - {InferenceEngine::PluginConfigParams::KEY_DUMP_KERNELS, "ON"}}, - {{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES , CommonTestUtils::DEVICE_GPU}, - {InferenceEngine::PluginConfigParams::KEY_TUNING_MODE, "TUNING_UNKNOWN_MODE"}}, - {{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES , CommonTestUtils::DEVICE_GPU}, - {InferenceEngine::PluginConfigParams::KEY_DEVICE_ID, "DEVICE_UNKNOWN"}} + {InferenceEngine::PluginConfigParams::KEY_PERF_COUNT, "ON"}} };