Benchmark_app: JSON writer for statistics (#9887)

* Refactored statistics output with JSON support

* Detailed/average reports are added

* stylefix

* Update samples/cpp/benchmark_app/statistics_report.hpp

Co-authored-by: Ivan Vikhrev <ivan.vikhrev@intel.com>

* Linux Fixes

* stylefixes

* data_shape field format is changed

* stylefix

Co-authored-by: Ivan Vikhrev <ivan.vikhrev@intel.com>
This commit is contained in:
Fedor Zharinov 2022-02-03 01:47:46 +03:00 committed by GitHub
parent 552454a3f0
commit 9219242dbd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 410 additions and 194 deletions

View File

@ -129,6 +129,10 @@ static const char report_type_message[] =
// @brief message for report_folder option
static const char report_folder_message[] = "Optional. Path to a folder where statistics report is stored.";
// @brief message for json_stats option
static const char json_stats_message[] = "Optional. Enables JSON-based statistics output (by default reporting system "
"will use CSV format). Should be used together with -report_folder option.";
// @brief message for exec_graph_path option
static const char exec_graph_path_message[] =
"Optional. Path to a file where to store executable graph information serialized.";
@ -290,6 +294,9 @@ DEFINE_string(report_type, "", report_type_message);
/// @brief Path to a folder where statistics report is stored
DEFINE_string(report_folder, "", report_folder_message);
/// @brief Enables JSON-based statistics reporting
DEFINE_bool(json_stats, false, json_stats_message);
/// @brief Path to a file where to store executable graph information serialized
DEFINE_string(exec_graph_path, "", exec_graph_path_message);
@ -392,6 +399,7 @@ static void show_usage() {
std::cout << std::endl << " Statistics dumping options:" << std::endl;
std::cout << " -report_type \"<type>\" " << report_type_message << std::endl;
std::cout << " -report_folder " << report_folder_message << std::endl;
std::cout << " -json_stats " << json_stats_message;
std::cout << " -exec_graph_path " << exec_graph_path_message << std::endl;
std::cout << " -pc " << pc_message << std::endl;
std::cout << " -pcseq " << pcseq_message << std::endl;

View File

@ -133,19 +133,22 @@ int main(int argc, char* argv[]) {
gflags::GetAllFlags(&flags);
for (auto& flag : flags) {
if (!flag.is_default) {
command_line_arguments.push_back({flag.name, flag.current_value});
command_line_arguments.emplace_back(flag.name, flag.name, flag.current_value);
}
}
if (!FLAGS_report_type.empty()) {
statistics =
std::make_shared<StatisticsReport>(StatisticsReport::Config{FLAGS_report_type, FLAGS_report_folder});
statistics = FLAGS_json_stats ? std::make_shared<StatisticsReportJSON>(
StatisticsReport::Config{FLAGS_report_type, FLAGS_report_folder})
: std::make_shared<StatisticsReport>(
StatisticsReport::Config{FLAGS_report_type, FLAGS_report_folder});
statistics->add_parameters(StatisticsReport::Category::COMMAND_LINE_PARAMETERS, command_line_arguments);
}
auto isFlagSetInCommandLine = [&command_line_arguments](const std::string& name) {
return (std::find_if(command_line_arguments.begin(),
command_line_arguments.end(),
[name](const std::pair<std::string, std::string>& p) {
return p.first == name;
[name](const StatisticsVariant& p) {
return p.json_name == name;
}) != command_line_arguments.end());
};
@ -392,11 +395,12 @@ int main(int argc, char* argv[]) {
slog::info << "Skipping the step for loading network from file" << slog::endl;
auto startTime = Time::now();
compiledModel = core.compile_model(FLAGS_m, device_name);
auto duration_ms = double_to_string(get_duration_ms_till_now(startTime));
slog::info << "Load network took " << duration_ms << " ms" << slog::endl;
auto duration_ms = get_duration_ms_till_now(startTime);
slog::info << "Load network took " << double_to_string(duration_ms) << " ms" << slog::endl;
if (statistics)
statistics->add_parameters(StatisticsReport::Category::EXECUTION_RESULTS,
{{"load network time (ms)", duration_ms}});
statistics->add_parameters(
StatisticsReport::Category::EXECUTION_RESULTS,
{StatisticsVariant("load network time (ms)", "load_network_time", duration_ms)});
convert_io_names_in_map(inputFiles, compiledModel.inputs());
app_inputs_info = get_inputs_info(FLAGS_shape,
@ -420,11 +424,12 @@ int main(int argc, char* argv[]) {
auto startTime = Time::now();
auto model = core.read_model(FLAGS_m);
auto duration_ms = double_to_string(get_duration_ms_till_now(startTime));
slog::info << "Read network took " << duration_ms << " ms" << slog::endl;
auto duration_ms = get_duration_ms_till_now(startTime);
slog::info << "Read network took " << double_to_string(duration_ms) << " ms" << slog::endl;
if (statistics)
statistics->add_parameters(StatisticsReport::Category::EXECUTION_RESULTS,
{{"read network time (ms)", duration_ms}});
statistics->add_parameters(
StatisticsReport::Category::EXECUTION_RESULTS,
{StatisticsVariant("read network time (ms)", "read_network_time", duration_ms)});
const auto& inputInfo = std::const_pointer_cast<const ov::Model>(model)->inputs();
if (inputInfo.empty()) {
@ -453,11 +458,12 @@ int main(int argc, char* argv[]) {
slog::info << "Reshaping network: " << get_shapes_string(shapes) << slog::endl;
startTime = Time::now();
model->reshape(shapes);
duration_ms = double_to_string(get_duration_ms_till_now(startTime));
slog::info << "Reshape network took " << duration_ms << " ms" << slog::endl;
duration_ms = get_duration_ms_till_now(startTime);
slog::info << "Reshape network took " << double_to_string(duration_ms) << " ms" << slog::endl;
if (statistics)
statistics->add_parameters(StatisticsReport::Category::EXECUTION_RESULTS,
{{"reshape network time (ms)", duration_ms}});
statistics->add_parameters(
StatisticsReport::Category::EXECUTION_RESULTS,
{StatisticsVariant("reshape network time (ms)", "reshape_network_time", duration_ms)});
}
// ----------------- 6. Configuring inputs and outputs
@ -559,11 +565,12 @@ int main(int argc, char* argv[]) {
next_step();
startTime = Time::now();
compiledModel = core.compile_model(model, device_name);
duration_ms = double_to_string(get_duration_ms_till_now(startTime));
slog::info << "Load network took " << duration_ms << " ms" << slog::endl;
duration_ms = get_duration_ms_till_now(startTime);
slog::info << "Load network took " << double_to_string(duration_ms) << " ms" << slog::endl;
if (statistics)
statistics->add_parameters(StatisticsReport::Category::EXECUTION_RESULTS,
{{"load network time (ms)", duration_ms}});
statistics->add_parameters(
StatisticsReport::Category::EXECUTION_RESULTS,
{StatisticsVariant("load network time (ms)", "load_network_time", duration_ms)});
} else {
next_step();
slog::info << "Skipping the step for compiled network" << slog::endl;
@ -583,11 +590,12 @@ int main(int argc, char* argv[]) {
compiledModel = core.import_model(modelStream, device_name, {});
modelStream.close();
auto duration_ms = double_to_string(get_duration_ms_till_now(startTime));
slog::info << "Import network took " << duration_ms << " ms" << slog::endl;
auto duration_ms = get_duration_ms_till_now(startTime);
slog::info << "Import network took " << double_to_string(duration_ms) << " ms" << slog::endl;
if (statistics)
statistics->add_parameters(StatisticsReport::Category::EXECUTION_RESULTS,
{{"import network time (ms)", duration_ms}});
statistics->add_parameters(
StatisticsReport::Category::EXECUTION_RESULTS,
{StatisticsVariant("import network time (ms)", "import_network_time", duration_ms)});
convert_io_names_in_map(inputFiles, compiledModel.inputs());
app_inputs_info = get_inputs_info(FLAGS_shape,
@ -692,24 +700,27 @@ int main(int argc, char* argv[]) {
if (statistics) {
statistics->add_parameters(
StatisticsReport::Category::RUNTIME_CONFIG,
{
{"benchmark mode", inferenceOnly ? "inference only" : "full"},
{"topology", topology_name},
{"target device", device_name},
{"API", FLAGS_api},
{"precision", std::string(type.get_type_name())},
{"batch size", std::to_string(batchSize)},
{"number of iterations", std::to_string(niter)},
{"number of parallel infer requests", std::to_string(nireq)},
{"duration (ms)", std::to_string(get_duration_in_milliseconds(duration_seconds))},
});
StatisticsReport::Parameters(
{StatisticsVariant("benchmark mode", "benchmark_mode", inferenceOnly ? "inference only" : "full"),
StatisticsVariant("topology", "topology", topology_name),
StatisticsVariant("target device", "target_device", device_name),
StatisticsVariant("API", "api", FLAGS_api),
StatisticsVariant("precision", "precision", type.get_type_name()),
StatisticsVariant("batch size", "batch_size", batchSize),
StatisticsVariant("number of iterations", "iterations_num", niter),
StatisticsVariant("number of parallel infer requests", "nireq", nireq),
StatisticsVariant("duration (ms)", "duration", get_duration_in_milliseconds(duration_seconds))}));
for (auto& nstreams : device_nstreams) {
std::stringstream ss;
ss << "number of " << nstreams.first << " streams";
std::string dev_name = nstreams.first;
std::transform(dev_name.begin(), dev_name.end(), dev_name.begin(), [](unsigned char c) {
return c == ' ' ? '_' : std::tolower(c);
});
statistics->add_parameters(StatisticsReport::Category::RUNTIME_CONFIG,
{
{ss.str(), nstreams.second},
});
{StatisticsVariant(ss.str(), dev_name + "_streams_num", nstreams.second)});
}
}
@ -872,12 +883,13 @@ int main(int argc, char* argv[]) {
inferRequestsQueue.wait_all();
auto duration_ms = double_to_string(inferRequestsQueue.get_latencies()[0]);
slog::info << "First inference took " << duration_ms << " ms" << slog::endl;
auto duration_ms = inferRequestsQueue.get_latencies()[0];
slog::info << "First inference took " << double_to_string(duration_ms) << " ms" << slog::endl;
if (statistics) {
statistics->add_parameters(StatisticsReport::Category::EXECUTION_RESULTS,
{{"first inference time (ms)", duration_ms}});
statistics->add_parameters(
StatisticsReport::Category::EXECUTION_RESULTS,
{StatisticsVariant("first inference time (ms)", "first_inference_time", duration_ms)});
}
inferRequestsQueue.reset_times();
@ -967,24 +979,32 @@ int main(int argc, char* argv[]) {
// wait the latest inference executions
inferRequestsQueue.wait_all();
LatencyMetrics generalLatency(inferRequestsQueue.get_latencies());
LatencyMetrics generalLatency(inferRequestsQueue.get_latencies(), "", FLAGS_latency_percentile);
std::vector<LatencyMetrics> groupLatencies = {};
if (FLAGS_pcseq && app_inputs_info.size() > 1) {
for (auto lats : inferRequestsQueue.get_latency_groups()) {
groupLatencies.push_back(LatencyMetrics(lats));
const auto& lat_groups = inferRequestsQueue.get_latency_groups();
for (int i = 0; i < lat_groups.size(); i++) {
const auto& lats = lat_groups[i];
std::string data_shapes_string = "";
for (auto& item : app_inputs_info[i]) {
data_shapes_string += item.first + get_shape_string(item.second.dataShape) + ",";
}
data_shapes_string =
data_shapes_string == "" ? "" : data_shapes_string.substr(0, data_shapes_string.size() - 1);
groupLatencies.emplace_back(lats, data_shapes_string, FLAGS_latency_percentile);
}
}
double totalDuration = inferRequestsQueue.get_duration_in_milliseconds();
double fps = (FLAGS_api == "sync") ? batchSize * 1000.0 / generalLatency.percentile(FLAGS_latency_percentile)
double fps = (FLAGS_api == "sync") ? batchSize * 1000.0 / generalLatency.median_or_percentile
: 1000.0 * processedFramesN / totalDuration;
if (statistics) {
statistics->add_parameters(StatisticsReport::Category::EXECUTION_RESULTS,
{
{"total execution time (ms)", double_to_string(totalDuration)},
{"total number of iterations", std::to_string(iteration)},
});
{StatisticsVariant("total execution time (ms)", "execution_time", totalDuration),
StatisticsVariant("total number of iterations", "iterations_num", iteration)});
if (device_name.find("MULTI") == std::string::npos) {
std::string latency_label;
if (FLAGS_latency_percentile == 50) {
@ -994,60 +1014,21 @@ int main(int argc, char* argv[]) {
}
statistics->add_parameters(
StatisticsReport::Category::EXECUTION_RESULTS,
{
{latency_label, double_to_string(generalLatency.percentile(FLAGS_latency_percentile))},
});
statistics->add_parameters(StatisticsReport::Category::EXECUTION_RESULTS,
{
{"Average latency (ms)", double_to_string(generalLatency.average())},
});
statistics->add_parameters(StatisticsReport::Category::EXECUTION_RESULTS,
{
{"Min latency (ms)", double_to_string(generalLatency.min())},
});
statistics->add_parameters(StatisticsReport::Category::EXECUTION_RESULTS,
{
{"Max latency (ms)", double_to_string(generalLatency.max())},
});
{StatisticsVariant(latency_label, "latency_median", generalLatency.median_or_percentile),
StatisticsVariant("Percentile boundary", "percentile_boundary", FLAGS_latency_percentile),
StatisticsVariant("Average latency (ms)", "latency_avg", generalLatency.avg),
StatisticsVariant("Min latency (ms)", "latency_min", generalLatency.min),
StatisticsVariant("Max latency (ms)", "latency_max", generalLatency.max),
StatisticsVariant("throughput", "throughput", fps)});
if (FLAGS_pcseq && app_inputs_info.size() > 1) {
statistics->add_parameters(StatisticsReport::Category::EXECUTION_RESULTS,
{
{"Latency for each data shape group:", ""},
});
for (size_t i = 0; i < app_inputs_info.size(); ++i) {
std::string data_shapes_string = "";
data_shapes_string += std::to_string(i + 1) + ". ";
for (auto& item : app_inputs_info[i]) {
data_shapes_string += item.first + " : " + get_shape_string(item.second.dataShape) + " ";
}
statistics->add_parameters(StatisticsReport::Category::EXECUTION_RESULTS,
{
{data_shapes_string, ""},
});
for (size_t i = 0; i < groupLatencies.size(); ++i) {
statistics->add_parameters(
StatisticsReport::Category::EXECUTION_RESULTS,
{
{latency_label,
double_to_string(groupLatencies[i].percentile(FLAGS_latency_percentile))},
});
statistics->add_parameters(StatisticsReport::Category::EXECUTION_RESULTS,
{
{"Average (ms)", double_to_string(groupLatencies[i].average())},
});
statistics->add_parameters(StatisticsReport::Category::EXECUTION_RESULTS,
{
{"Min (ms)", double_to_string(groupLatencies[i].min())},
});
statistics->add_parameters(StatisticsReport::Category::EXECUTION_RESULTS,
{
{"Max (ms)", double_to_string(groupLatencies[i].max())},
});
StatisticsReport::Category::EXECUTION_RESULTS_GROUPPED,
{StatisticsVariant("Group Latencies", "group_latencies", groupLatencies[i])});
}
}
}
statistics->add_parameters(StatisticsReport::Category::EXECUTION_RESULTS,
{{"throughput", double_to_string(fps)}});
}
progressBar.finish();
@ -1094,7 +1075,7 @@ int main(int argc, char* argv[]) {
slog::info << "Duration: " << double_to_string(totalDuration) << " ms" << slog::endl;
if (device_name.find("MULTI") == std::string::npos) {
slog::info << "Latency: " << slog::endl;
generalLatency.log_total(FLAGS_latency_percentile);
generalLatency.write_to_slog();
if (FLAGS_pcseq && app_inputs_info.size() > 1) {
slog::info << "Latency for each data shape group:" << slog::endl;
@ -1109,7 +1090,7 @@ int main(int argc, char* argv[]) {
}
slog::info << slog::endl;
groupLatencies[i].log_total(FLAGS_latency_percentile);
groupLatencies[i].write_to_slog();
}
}
}
@ -1120,9 +1101,7 @@ int main(int argc, char* argv[]) {
if (statistics) {
statistics->add_parameters(StatisticsReport::Category::EXECUTION_RESULTS,
{
{"error", ex.what()},
});
{StatisticsVariant("error", "error", ex.what())});
statistics->dump();
}

View File

@ -28,8 +28,8 @@ public:
}
bool is_equal(const AllocatorImpl& other) const override {
auto other_blob_allocator = dynamic_cast<const SharedTensorAllocator*>(&other);
return other_blob_allocator != nullptr && other_blob_allocator == this;
auto other_tensor_allocator = dynamic_cast<const SharedTensorAllocator*>(&other);
return other_tensor_allocator != nullptr && other_tensor_allocator == this;
}
char* get_buffer() {

View File

@ -12,6 +12,8 @@
#include "statistics_report.hpp"
// clang-format on
static const char* status_names[] = {"NOT_RUN", "OPTIMIZED_OUT", "EXECUTED"};
void StatisticsReport::add_parameters(const Category& category, const Parameters& parameters) {
if (_parameters.count(category) == 0)
_parameters[category] = parameters;
@ -24,7 +26,10 @@ void StatisticsReport::dump() {
auto dump_parameters = [&dumper](const Parameters& parameters) {
for (auto& parameter : parameters) {
dumper << parameter.first << parameter.second;
if (parameter.type != StatisticsVariant::METRICS) {
dumper << parameter.csv_name;
}
dumper << parameter.to_string();
dumper.endLine();
}
};
@ -52,10 +57,20 @@ void StatisticsReport::dump() {
dumper.endLine();
}
if (_parameters.count(Category::EXECUTION_RESULTS_GROUPPED)) {
dumper << "Group Latencies";
dumper.endLine();
dumper << "Data shape;Median;Average;Min;Max";
dumper.endLine();
dump_parameters(_parameters.at(Category::EXECUTION_RESULTS_GROUPPED));
dumper.endLine();
}
slog::info << "Statistics report is stored to " << dumper.getFilename() << slog::endl;
}
void StatisticsReport::dump_performance_counters_request(CsvDumper& dumper, const PerformaceCounters& perfCounts) {
void StatisticsReport::dump_performance_counters_request(CsvDumper& dumper, const PerformanceCounters& perfCounts) {
std::chrono::microseconds total = std::chrono::microseconds::zero();
std::chrono::microseconds total_cpu = std::chrono::microseconds::zero();
@ -69,18 +84,9 @@ void StatisticsReport::dump_performance_counters_request(CsvDumper& dumper, cons
for (const auto& layer : perfCounts) {
dumper << layer.node_name; // layer name
switch (layer.status) {
case ov::ProfilingInfo::Status::EXECUTED:
dumper << "EXECUTED";
break;
case ov::ProfilingInfo::Status::NOT_RUN:
dumper << "NOT_RUN";
break;
case ov::ProfilingInfo::Status::OPTIMIZED_OUT:
dumper << "OPTIMIZED_OUT";
break;
}
dumper << ((int)layer.status < (sizeof(status_names) / sizeof(status_names[0]))
? status_names[(int)layer.status]
: "INVALID_STATUS");
dumper << layer.node_type << layer.exec_type;
dumper << std::to_string(layer.real_time.count() / 1000.0) << std::to_string(layer.cpu_time.count() / 1000.0);
total += layer.real_time;
@ -96,7 +102,35 @@ void StatisticsReport::dump_performance_counters_request(CsvDumper& dumper, cons
dumper.endLine();
}
void StatisticsReport::dump_performance_counters(const std::vector<PerformaceCounters>& perfCounts) {
StatisticsReport::PerformanceCounters StatisticsReport::get_average_performance_counters(
const std::vector<PerformanceCounters>& perfCounts) {
StatisticsReport::PerformanceCounters performanceCountersAvg;
// iterate over each processed infer request and handle its PM data
for (size_t i = 0; i < perfCounts.size(); i++) {
// iterate over each layer from sorted vector and add required PM data
// to the per-layer maps
for (const auto& pm : perfCounts[i]) {
int idx = 0;
for (; idx < performanceCountersAvg.size(); idx++) {
if (performanceCountersAvg[idx].node_name == pm.node_name) {
performanceCountersAvg[idx].real_time += pm.real_time;
performanceCountersAvg[idx].cpu_time += pm.cpu_time;
break;
}
}
if (idx == performanceCountersAvg.size()) {
performanceCountersAvg.push_back(pm);
}
}
}
for (auto& pm : performanceCountersAvg) {
pm.real_time /= perfCounts.size();
pm.cpu_time /= perfCounts.size();
}
return performanceCountersAvg;
};
void StatisticsReport::dump_performance_counters(const std::vector<PerformanceCounters>& perfCounts) {
if ((_config.report_type.empty()) || (_config.report_type == noCntReport)) {
slog::info << "Statistics collecting for performance counters was not "
"requested. No reports are dumped."
@ -113,35 +147,180 @@ void StatisticsReport::dump_performance_counters(const std::vector<PerformaceCou
dump_performance_counters_request(dumper, pc);
}
} else if (_config.report_type == averageCntReport) {
auto getAveragePerformanceCounters = [&perfCounts]() {
std::vector<ov::ProfilingInfo> performanceCountersAvg;
// iterate over each processed infer request and handle its PM data
for (size_t i = 0; i < perfCounts.size(); i++) {
// iterate over each layer from sorted vector and add required PM data
// to the per-layer maps
for (const auto& pm : perfCounts[i]) {
int idx = 0;
for (; idx < performanceCountersAvg.size(); idx++) {
if (performanceCountersAvg[idx].node_name == pm.node_name) {
performanceCountersAvg[idx].real_time += pm.real_time;
performanceCountersAvg[idx].cpu_time += pm.cpu_time;
break;
}
}
if (idx == performanceCountersAvg.size()) {
performanceCountersAvg.push_back(pm);
}
}
}
for (auto& pm : performanceCountersAvg) {
pm.real_time /= perfCounts.size();
pm.cpu_time /= perfCounts.size();
}
return performanceCountersAvg;
};
dump_performance_counters_request(dumper, getAveragePerformanceCounters());
dump_performance_counters_request(dumper, get_average_performance_counters(perfCounts));
} else {
throw std::logic_error("PM data can only be collected for average or detailed report types");
}
slog::info << "Performance counters report is stored to " << dumper.getFilename() << slog::endl;
}
void StatisticsReportJSON::dump_parameters(nlohmann::json& js, const StatisticsReport::Parameters& parameters) {
for (auto& parameter : parameters) {
parameter.write_to_json(js);
}
};
void StatisticsReportJSON::dump() {
nlohmann::json js;
std::string name = _config.report_folder + _separator + "benchmark_report.json";
if (_parameters.count(Category::COMMAND_LINE_PARAMETERS)) {
dump_parameters(js["cmd_options"], _parameters.at(Category::COMMAND_LINE_PARAMETERS));
}
if (_parameters.count(Category::RUNTIME_CONFIG)) {
dump_parameters(js["configuration_setup"], _parameters.at(Category::RUNTIME_CONFIG));
}
if (_parameters.count(Category::EXECUTION_RESULTS)) {
dump_parameters(js["execution_results"], _parameters.at(Category::EXECUTION_RESULTS));
}
if (_parameters.count(Category::EXECUTION_RESULTS_GROUPPED)) {
dump_parameters(js["execution_results"], _parameters.at(Category::EXECUTION_RESULTS_GROUPPED));
}
std::ofstream out_stream(name);
out_stream << std::setw(4) << js << std::endl;
slog::info << "Statistics report is stored to " << name << slog::endl;
}
void StatisticsReportJSON::dump_performance_counters(const std::vector<PerformanceCounters>& perfCounts) {
if ((_config.report_type.empty()) || (_config.report_type == noCntReport)) {
slog::info << "Statistics collecting for performance counters was not "
"requested. No reports are dumped."
<< slog::endl;
return;
}
if (perfCounts.empty()) {
slog::info << "Performance counters are empty. No reports are dumped." << slog::endl;
return;
}
nlohmann::json js;
std::string name = _config.report_folder + _separator + "benchmark_" + _config.report_type + "_report.json";
if (_config.report_type == detailedCntReport) {
js["report_type"] = "detailed";
js["detailed_performance"] = nlohmann::json::array();
for (auto& pc : perfCounts) {
js["detailed_performance"].push_back(perf_counters_to_json(pc));
}
} else if (_config.report_type == averageCntReport) {
js["report_type"] = "average";
js["avg_performance"] = perf_counters_to_json(get_average_performance_counters(perfCounts));
} else {
throw std::logic_error("PM data can only be collected for average or detailed report types");
}
std::ofstream out_stream(name);
out_stream << std::setw(4) << js << std::endl;
slog::info << "Performance counters report is stored to " << name << slog::endl;
}
const nlohmann::json StatisticsReportJSON::perf_counters_to_json(
const StatisticsReport::PerformanceCounters& perfCounts) {
std::chrono::microseconds total = std::chrono::microseconds::zero();
std::chrono::microseconds total_cpu = std::chrono::microseconds::zero();
nlohmann::json js;
js["nodes"] = nlohmann::json::array();
for (const auto& layer : perfCounts) {
nlohmann::json item;
item["name"] = layer.node_name; // layer name
item["status"] =
((int)layer.status < (sizeof(status_names) / sizeof(status_names[0])) ? status_names[(int)layer.status]
: "INVALID_STATUS");
item["node_type"] = layer.node_type;
item["exec_type"] = layer.exec_type;
item["real_time"] = layer.real_time.count() / 1000.0;
item["cpu_time"] = layer.cpu_time.count() / 1000.0;
total += layer.real_time;
total_cpu += layer.cpu_time;
js["nodes"].push_back(item);
}
js["total_real_time"] = total.count() / 1000.0;
js["total_cpu_time"] = total_cpu.count() / 1000.0;
return js;
}
void LatencyMetrics::write_to_stream(std::ostream& stream) const {
stream << data_shape << ";" << std::fixed << std::setprecision(2) << median_or_percentile << ";" << avg << ";"
<< min << ";" << max;
}
void LatencyMetrics::write_to_slog() const {
std::string percentileStr = (percentile_boundary == 50)
? "\tMedian: "
: "\t" + std::to_string(percentile_boundary) + " percentile: ";
if (!data_shape.empty()) {
slog::info << "\tData shape: " << data_shape << slog::endl;
}
slog::info << percentileStr << double_to_string(median_or_percentile) << " ms" << slog::endl;
slog::info << "\tAverage: " << double_to_string(avg) << " ms" << slog::endl;
slog::info << "\tMin: " << double_to_string(min) << " ms" << slog::endl;
slog::info << "\tMax: " << double_to_string(max) << " ms" << slog::endl;
}
const nlohmann::json LatencyMetrics::to_json() const {
nlohmann::json stat;
stat["data_shape"] = data_shape;
stat["latency_median"] = median_or_percentile;
stat["latency_average"] = avg;
stat["latency_min"] = min;
stat["latency_max"] = max;
return stat;
}
void LatencyMetrics::fill_data(std::vector<double> latencies, size_t percentile_boundary) {
if (latencies.empty()) {
throw std::logic_error("Latency metrics class expects non-empty vector of latencies at consturction.");
}
std::sort(latencies.begin(), latencies.end());
min = latencies[0];
avg = std::accumulate(latencies.begin(), latencies.end(), 0.0) / latencies.size();
median_or_percentile = latencies[size_t(latencies.size() / 100.0 * percentile_boundary)];
max = latencies.back();
};
std::string StatisticsVariant::to_string() const {
switch (type) {
case INT:
return std::to_string(i_val);
case DOUBLE:
return std::to_string(d_val);
case STRING:
return s_val;
case ULONGLONG:
return std::to_string(ull_val);
case METRICS:
std::ostringstream str;
metrics_val.write_to_stream(str);
return str.str();
}
throw std::invalid_argument("StatisticsVariant::to_string : invalid type is provided");
}
void StatisticsVariant::write_to_json(nlohmann::json& js) const {
switch (type) {
case INT:
js[json_name] = i_val;
break;
case DOUBLE:
js[json_name] = d_val;
break;
case STRING:
js[json_name] = s_val;
break;
case ULONGLONG:
js[json_name] = ull_val;
break;
case METRICS: {
auto& arr = js[json_name];
if (arr.empty()) {
arr = nlohmann::json::array();
}
arr.push_back(metrics_val.to_json());
} break;
default:
throw std::invalid_argument("StatisticsVariant:: json conversion : invalid type is provided");
}
}

View File

@ -5,6 +5,7 @@
#pragma once
#include <map>
#include <nlohmann/json.hpp>
#include <string>
#include <utility>
#include <vector>
@ -25,66 +26,99 @@ static constexpr char detailedCntReport[] = "detailed_counters";
/// @brief Responsible for calculating different latency metrics
class LatencyMetrics {
public:
LatencyMetrics() = delete;
LatencyMetrics() {}
LatencyMetrics(const std::vector<double>& latencies) : latencies(latencies) {
if (latencies.empty()) {
throw std::logic_error("Latency metrics class expects non-empty vector of latencies at consturction.");
}
std::sort(this->latencies.begin(), this->latencies.end());
LatencyMetrics(const std::vector<double>& latencies,
const std::string& data_shape = "",
size_t percentile_boundary = 50)
: data_shape(data_shape),
percentile_boundary(percentile_boundary) {
fill_data(latencies, percentile_boundary);
}
LatencyMetrics(std::vector<double>&& latencies) : latencies(latencies) {
if (latencies.empty()) {
throw std::logic_error("Latency metrics class expects non-empty vector of latencies at consturction.");
}
std::sort(this->latencies.begin(), this->latencies.end());
}
void write_to_stream(std::ostream& stream) const;
void write_to_slog() const;
const nlohmann::json to_json() const;
double min() {
return latencies[0];
}
double average() {
return std::accumulate(latencies.begin(), latencies.end(), 0.0) / latencies.size();
}
double percentile(std::size_t p) {
return latencies[size_t(latencies.size() / 100.0 * p)];
}
double max() {
return latencies.back();
}
void log_total(size_t p) {
std::string percentileStr = (p == 50) ? "\tMedian: " : "\t" + std::to_string(p) + " percentile: ";
slog::info << percentileStr << double_to_string(percentile(p)) << " ms" << slog::endl;
slog::info << "\tAvg: " << double_to_string(average()) << " ms" << slog::endl;
slog::info << "\tMin: " << double_to_string(min()) << " ms" << slog::endl;
slog::info << "\tMax: " << double_to_string(max()) << " ms" << slog::endl;
}
public:
double median_or_percentile = 0;
double avg = 0;
double min = 0;
double max = 0;
std::string data_shape;
private:
std::vector<double> latencies;
void fill_data(std::vector<double> latencies, size_t percentile_boundary);
size_t percentile_boundary = 50;
};
class StatisticsVariant {
public:
enum Type { INT, DOUBLE, STRING, ULONGLONG, METRICS };
StatisticsVariant(std::string csv_name, std::string json_name, int v)
: csv_name(csv_name),
json_name(json_name),
i_val(v),
type(INT) {}
StatisticsVariant(std::string csv_name, std::string json_name, double v)
: csv_name(csv_name),
json_name(json_name),
d_val(v),
type(DOUBLE) {}
StatisticsVariant(std::string csv_name, std::string json_name, const std::string& v)
: csv_name(csv_name),
json_name(json_name),
s_val(v),
type(STRING) {}
StatisticsVariant(std::string csv_name, std::string json_name, unsigned long long v)
: csv_name(csv_name),
json_name(json_name),
ull_val(v),
type(ULONGLONG) {}
StatisticsVariant(std::string csv_name, std::string json_name, uint32_t v)
: csv_name(csv_name),
json_name(json_name),
ull_val(v),
type(ULONGLONG) {}
StatisticsVariant(std::string csv_name, std::string json_name, unsigned long v)
: csv_name(csv_name),
json_name(json_name),
ull_val(v),
type(ULONGLONG) {}
StatisticsVariant(std::string csv_name, std::string json_name, const LatencyMetrics& v)
: csv_name(csv_name),
json_name(json_name),
metrics_val(v),
type(METRICS) {}
~StatisticsVariant() {}
std::string csv_name;
std::string json_name;
int i_val;
double d_val;
unsigned long long ull_val;
std::string s_val;
LatencyMetrics metrics_val;
Type type;
std::string to_string() const;
void write_to_json(nlohmann::json& js) const;
};
/// @brief Responsible for collecting of statistics and dumping to .csv file
class StatisticsReport {
public:
typedef std::vector<ov::ProfilingInfo> PerformaceCounters;
typedef std::vector<std::pair<std::string, std::string>> Parameters;
typedef std::vector<ov::ProfilingInfo> PerformanceCounters;
typedef std::vector<StatisticsVariant> Parameters;
struct Config {
std::string report_type;
std::string report_folder;
};
enum class Category {
COMMAND_LINE_PARAMETERS,
RUNTIME_CONFIG,
EXECUTION_RESULTS,
};
enum class Category { COMMAND_LINE_PARAMETERS, RUNTIME_CONFIG, EXECUTION_RESULTS, EXECUTION_RESULTS_GROUPPED };
explicit StatisticsReport(Config config) : _config(std::move(config)) {
_separator =
@ -103,13 +137,14 @@ public:
void add_parameters(const Category& category, const Parameters& parameters);
void dump();
virtual void dump();
void dump_performance_counters(const std::vector<PerformaceCounters>& perfCounts);
virtual void dump_performance_counters(const std::vector<PerformanceCounters>& perfCounts);
private:
void dump_performance_counters_request(CsvDumper& dumper, const PerformaceCounters& perfCounts);
void dump_performance_counters_request(CsvDumper& dumper, const PerformanceCounters& perfCounts);
protected:
// configuration of current benchmark execution
const Config _config;
@ -118,4 +153,19 @@ private:
// csv separator
std::string _separator;
StatisticsReport::PerformanceCounters get_average_performance_counters(
const std::vector<PerformanceCounters>& perfCounts);
};
class StatisticsReportJSON : public StatisticsReport {
public:
explicit StatisticsReportJSON(Config config) : StatisticsReport(std::move(config)) {}
void dump() override;
void dump_performance_counters(const std::vector<PerformanceCounters>& perfCounts) override;
private:
void dump_parameters(nlohmann::json& js, const StatisticsReport::Parameters& parameters);
const nlohmann::json perf_counters_to_json(const StatisticsReport::PerformanceCounters& perfCounts);
};