diff --git a/samples/cpp/benchmark_app/README.md b/samples/cpp/benchmark_app/README.md index e6ac6e36ee7..2a50243c797 100644 --- a/samples/cpp/benchmark_app/README.md +++ b/samples/cpp/benchmark_app/README.md @@ -154,6 +154,27 @@ Options: -pcseq Optional. Report latencies for each shape in -data_shape sequence. -dump_config Optional. Path to JSON file to dump IE parameters, which were set by application. -load_config Optional. Path to JSON file to load custom IE parameters. Please note, command line parameters have higher priority then parameters from configuration file. + Example 1: a simple JSON file for HW device with primary properties. + { + "CPU": {"NUM_STREAMS": "3", "PERF_COUNT": "NO"} + } + Example 2: a simple JSON file for meta device(AUTO/MULTI) with HW device properties. + { + "AUTO": { + "PERFORMANCE_HINT": "", + "PERF_COUNT": "NO", + "DEVICE_PROPERTIES": { + "CPU": { + "INFERENCE_PRECISION_HINT": "f32", + "NUM_STREAMS": "3" + }, + "GPU": { + "INFERENCE_PRECISION_HINT": "f32", + "NUM_STREAMS": "5" + } + } + } + } -infer_precision ""Optional. Inference precision -ip Optional. Specifies precision for all input layers of the model. -op Optional. Specifies precision for all output layers of the model. diff --git a/samples/cpp/benchmark_app/benchmark_app.hpp b/samples/cpp/benchmark_app/benchmark_app.hpp index d0913c21137..c717c403938 100644 --- a/samples/cpp/benchmark_app/benchmark_app.hpp +++ b/samples/cpp/benchmark_app/benchmark_app.hpp @@ -170,8 +170,29 @@ static const char use_device_mem_message[] = // @brief message for load config option static const char load_config_message[] = "Optional. Path to JSON file to load custom IE parameters." - " Please note, command line parameters have higher priority then parameters from configuration " - "file."; + " Please note, command line parameters have higher priority then parameters from configuration file.\n" + " Example 1: a simple JSON file for HW device with primary properties.\n" + " {\n" + " \"CPU\": {\"NUM_STREAMS\": \"3\", \"PERF_COUNT\": \"NO\"}\n" + " }\n" + " Example 2: a simple JSON file for meta device(AUTO/MULTI) with HW device " + "properties.\n" + " {\n" + " \"AUTO\": {\n" + " \"PERFORMANCE_HINT\": \"\",\n" + " \"PERF_COUNT\": \"NO\",\n" + " \"DEVICE_PROPERTIES\": {\n" + " \"CPU\": {\n" + " \"INFERENCE_PRECISION_HINT\": \"f32\",\n" + " \"NUM_STREAMS\": \"3\"\n" + " },\n" + " \"GPU\": {\n" + " \"INFERENCE_PRECISION_HINT\": \"f32\",\n" + " \"NUM_STREAMS\": \"5\"\n" + " }\n" + " }\n" + " }\n" + " }\n"; // @brief message for dump config option static const char dump_config_message[] = diff --git a/samples/cpp/benchmark_app/main.cpp b/samples/cpp/benchmark_app/main.cpp index 59d72eefa78..9fcd06087ef 100644 --- a/samples/cpp/benchmark_app/main.cpp +++ b/samples/cpp/benchmark_app/main.cpp @@ -157,6 +157,7 @@ void setDeviceProperty(ov::Core& core, std::string& device, ov::AnyMap& device_config, const std::pair& property, + std::map& is_dev_set_property, const std::pair& config = {}) { auto supported_properties = core.get_property(device, ov::supported_properties); auto supported = [&](const std::string& key) { @@ -172,10 +173,17 @@ void setDeviceProperty(ov::Core& core, if (device_property.first.empty()) return; - if (device_config.find(device) == device_config.end()) { + + if (device_config.find(device) == device_config.end() || // device properties not existed + config.first.empty() && // not setting default value to property + (!FLAGS_load_config.empty() && + is_dev_set_property[device])) { // device properties loaded from file and overwrite is not happened + is_dev_set_property[device] = false; + device_config.erase(device); device_config.insert(ov::device::properties(device, device_property)); } else { auto& properties = device_config[device].as(); + // property present in device properties has higher priority than property set with default value properties.emplace(device_property); } } @@ -230,6 +238,10 @@ int main(int argc, char* argv[]) { // Parse devices auto devices = parse_devices(device_name); + std::map is_dev_set_property = {}; + // initialize flags to ensure ov::device::properties should only be set once per device. + for (auto& dev : devices) + is_dev_set_property[dev] = true; // Parse nstreams per device std::map device_nstreams = parse_value_per_device(devices, FLAGS_nstreams); std::map device_infer_precision = @@ -237,8 +249,10 @@ int main(int argc, char* argv[]) { // Load device config file if specified std::map config; + bool is_load_config = false; if (!FLAGS_load_config.empty()) { load_config(FLAGS_load_config, config); + is_load_config = true; } /** This vector stores paths to the processed images with input names**/ @@ -381,10 +395,17 @@ int main(int argc, char* argv[]) { std::map devices_property; ov::util::Read>{}(strm, devices_property); for (auto it : devices_property) { - if (device_config.find(it.first) == device_config.end()) + if (device_config.find(it.first) == device_config.end() || + (is_load_config && is_dev_set_property[it.first])) { + // Create ov::device::properties with ov::num_stream and + // 1. Insert this ov::device::properties into device config if this + // ov::device::properties isn't existed. Otherwise, + // 2. Replace the existed ov::device::properties within device config. + is_dev_set_property[it.first] = false; + device_config.erase(it.first); device_config.insert( ov::device::properties(it.first, ov::num_streams(std::stoi(it.second)))); - else { + } else { auto& property = device_config[it.first].as(); property.emplace(ov::num_streams(std::stoi(it.second))); } @@ -416,13 +437,14 @@ int main(int argc, char* argv[]) { device_config[key] = ov::streams::AUTO; } else if (device == "MULTI" || device == "AUTO") { // Set nstreams to default value auto if no nstreams specified from cmd line. - std::string key = std::string(getDeviceTypeFromName(device) + "_THROUGHPUT_STREAMS"); for (auto& hwdevice : hardware_devices) { + std::string key = std::string(getDeviceTypeFromName(hwdevice) + "_THROUGHPUT_STREAMS"); auto value = std::string(getDeviceTypeFromName(hwdevice) + "_THROUGHPUT_AUTO"); setDeviceProperty(core, hwdevice, device_config, ov::num_streams(ov::streams::AUTO), + is_dev_set_property, std::make_pair(key, value)); } } @@ -451,10 +473,17 @@ int main(int argc, char* argv[]) { std::map devices_property; ov::util::Read>{}(strm, devices_property); for (auto it : devices_property) { - if (device_config.find(it.first) == device_config.end()) + if (device_config.find(it.first) == device_config.end() || + (is_load_config && is_dev_set_property[it.first])) { + // Create ov::device::properties with ov::inference_precision and + // 1. Insert this ov::device::properties into device config if this + // ov::device::properties isn't existed. Otherwise, + // 2. Replace the existed ov::device::properties within device config. + is_dev_set_property[it.first] = false; + device_config.erase(it.first); device_config.insert( ov::device::properties(it.first, ov::hint::inference_precision(it.second))); - else { + } else { auto& property = device_config[it.first].as(); property.emplace(ov::hint::inference_precision(it.second)); } @@ -491,7 +520,7 @@ int main(int argc, char* argv[]) { // list specified by -d. for (auto& device : hardware_devices) { if (device == "CPU") - setDeviceProperty(core, device, device_config, property); + setDeviceProperty(core, device, device_config, property, is_dev_set_property); } } }; @@ -1249,8 +1278,8 @@ int main(int argc, char* argv[]) { } catch (const ov::Exception&) { } - slog::info << "Count: " << iteration << " iterations" << slog::endl; - slog::info << "Duration: " << double_to_string(totalDuration) << " ms" << slog::endl; + slog::info << "Count: " << iteration << " iterations" << slog::endl; + slog::info << "Duration: " << double_to_string(totalDuration) << " ms" << slog::endl; if (device_name.find("MULTI") == std::string::npos) { slog::info << "Latency:" << slog::endl; @@ -1274,7 +1303,7 @@ int main(int argc, char* argv[]) { } } - slog::info << "Throughput: " << double_to_string(fps) << " FPS" << slog::endl; + slog::info << "Throughput: " << double_to_string(fps) << " FPS" << slog::endl; } catch (const std::exception& ex) { slog::err << ex.what() << slog::endl; diff --git a/samples/cpp/benchmark_app/statistics_report.cpp b/samples/cpp/benchmark_app/statistics_report.cpp index 15e6ca805ba..3c03e570a9a 100644 --- a/samples/cpp/benchmark_app/statistics_report.cpp +++ b/samples/cpp/benchmark_app/statistics_report.cpp @@ -345,13 +345,13 @@ void LatencyMetrics::write_to_stream(std::ostream& stream) const { void LatencyMetrics::write_to_slog() const { std::string percentileStr = (percentile_boundary == 50) - ? " Median: " + ? " Median: " : " " + std::to_string(percentile_boundary) + " percentile: "; slog::info << percentileStr << double_to_string(median_or_percentile) << " ms" << slog::endl; - slog::info << " Average: " << double_to_string(avg) << " ms" << slog::endl; - slog::info << " Min: " << double_to_string(min) << " ms" << slog::endl; - slog::info << " Max: " << double_to_string(max) << " ms" << slog::endl; + slog::info << " Average: " << double_to_string(avg) << " ms" << slog::endl; + slog::info << " Min: " << double_to_string(min) << " ms" << slog::endl; + slog::info << " Max: " << double_to_string(max) << " ms" << slog::endl; } const nlohmann::json LatencyMetrics::to_json() const { diff --git a/samples/cpp/benchmark_app/utils.cpp b/samples/cpp/benchmark_app/utils.cpp index 2b97b15072b..3a704c6d7f0 100644 --- a/samples/cpp/benchmark_app/utils.cpp +++ b/samples/cpp/benchmark_app/utils.cpp @@ -731,10 +731,27 @@ void dump_config(const std::string& filename, const std::map device_properties; for (const auto& option : item.second) { - std::stringstream strm; - option.second.print(strm); - jsonConfig[deviceName][option.first] = strm.str(); + if (option.second.is()) { + // hw device properties + device_properties[option.first] = option.second.as(); + } else { + // primary property + std::stringstream strm; + option.second.print(strm); + auto property_string = strm.str(); + jsonConfig[deviceName][option.first] = property_string; + } + if (!device_properties.empty()) { + for (auto& item : device_properties) { + auto hw_device_name = item.first; + for (auto& property : item.second) { + jsonConfig[deviceName]["DEVICE_PROPERTIES"][hw_device_name][property.first] = + property.second.as(); + } + } + } } } @@ -762,7 +779,17 @@ void load_config(const std::string& filename, std::map& for (const auto& item : jsonConfig.items()) { std::string deviceName = item.key(); for (const auto& option : item.value().items()) { - config[deviceName][option.key()] = option.value().get(); + if (option.key() != "DEVICE_PROPERTIES") { + config[deviceName][option.key()] = option.value().get(); + continue; + } + for (const auto& hw_properties : option.value().items()) { + auto hw_device_name = hw_properties.key(); + std::map hw_device_properties; + for (const auto& property : hw_properties.value().items()) + hw_device_properties[property.key()] = property.value().get(); + config[deviceName][hw_device_name] = hw_device_properties; + } } } }