Benchmark_app: JSON writer for statistics (#9887)
* Refactored statistics output with JSON support * Detailed/average reports are added * stylefix * Update samples/cpp/benchmark_app/statistics_report.hpp Co-authored-by: Ivan Vikhrev <ivan.vikhrev@intel.com> * Linux Fixes * stylefixes * data_shape field format is changed * stylefix Co-authored-by: Ivan Vikhrev <ivan.vikhrev@intel.com>
This commit is contained in:
parent
552454a3f0
commit
9219242dbd
@ -129,6 +129,10 @@ static const char report_type_message[] =
|
||||
// @brief message for report_folder option
|
||||
static const char report_folder_message[] = "Optional. Path to a folder where statistics report is stored.";
|
||||
|
||||
// @brief message for json_stats option
|
||||
static const char json_stats_message[] = "Optional. Enables JSON-based statistics output (by default reporting system "
|
||||
"will use CSV format). Should be used together with -report_folder option.";
|
||||
|
||||
// @brief message for exec_graph_path option
|
||||
static const char exec_graph_path_message[] =
|
||||
"Optional. Path to a file where to store executable graph information serialized.";
|
||||
@ -290,6 +294,9 @@ DEFINE_string(report_type, "", report_type_message);
|
||||
/// @brief Path to a folder where statistics report is stored
|
||||
DEFINE_string(report_folder, "", report_folder_message);
|
||||
|
||||
/// @brief Enables JSON-based statistics reporting
|
||||
DEFINE_bool(json_stats, false, json_stats_message);
|
||||
|
||||
/// @brief Path to a file where to store executable graph information serialized
|
||||
DEFINE_string(exec_graph_path, "", exec_graph_path_message);
|
||||
|
||||
@ -392,6 +399,7 @@ static void show_usage() {
|
||||
std::cout << std::endl << " Statistics dumping options:" << std::endl;
|
||||
std::cout << " -report_type \"<type>\" " << report_type_message << std::endl;
|
||||
std::cout << " -report_folder " << report_folder_message << std::endl;
|
||||
std::cout << " -json_stats " << json_stats_message;
|
||||
std::cout << " -exec_graph_path " << exec_graph_path_message << std::endl;
|
||||
std::cout << " -pc " << pc_message << std::endl;
|
||||
std::cout << " -pcseq " << pcseq_message << std::endl;
|
||||
|
@ -133,19 +133,22 @@ int main(int argc, char* argv[]) {
|
||||
gflags::GetAllFlags(&flags);
|
||||
for (auto& flag : flags) {
|
||||
if (!flag.is_default) {
|
||||
command_line_arguments.push_back({flag.name, flag.current_value});
|
||||
command_line_arguments.emplace_back(flag.name, flag.name, flag.current_value);
|
||||
}
|
||||
}
|
||||
if (!FLAGS_report_type.empty()) {
|
||||
statistics =
|
||||
std::make_shared<StatisticsReport>(StatisticsReport::Config{FLAGS_report_type, FLAGS_report_folder});
|
||||
statistics = FLAGS_json_stats ? std::make_shared<StatisticsReportJSON>(
|
||||
StatisticsReport::Config{FLAGS_report_type, FLAGS_report_folder})
|
||||
: std::make_shared<StatisticsReport>(
|
||||
StatisticsReport::Config{FLAGS_report_type, FLAGS_report_folder});
|
||||
|
||||
statistics->add_parameters(StatisticsReport::Category::COMMAND_LINE_PARAMETERS, command_line_arguments);
|
||||
}
|
||||
auto isFlagSetInCommandLine = [&command_line_arguments](const std::string& name) {
|
||||
return (std::find_if(command_line_arguments.begin(),
|
||||
command_line_arguments.end(),
|
||||
[name](const std::pair<std::string, std::string>& p) {
|
||||
return p.first == name;
|
||||
[name](const StatisticsVariant& p) {
|
||||
return p.json_name == name;
|
||||
}) != command_line_arguments.end());
|
||||
};
|
||||
|
||||
@ -392,11 +395,12 @@ int main(int argc, char* argv[]) {
|
||||
slog::info << "Skipping the step for loading network from file" << slog::endl;
|
||||
auto startTime = Time::now();
|
||||
compiledModel = core.compile_model(FLAGS_m, device_name);
|
||||
auto duration_ms = double_to_string(get_duration_ms_till_now(startTime));
|
||||
slog::info << "Load network took " << duration_ms << " ms" << slog::endl;
|
||||
auto duration_ms = get_duration_ms_till_now(startTime);
|
||||
slog::info << "Load network took " << double_to_string(duration_ms) << " ms" << slog::endl;
|
||||
if (statistics)
|
||||
statistics->add_parameters(StatisticsReport::Category::EXECUTION_RESULTS,
|
||||
{{"load network time (ms)", duration_ms}});
|
||||
statistics->add_parameters(
|
||||
StatisticsReport::Category::EXECUTION_RESULTS,
|
||||
{StatisticsVariant("load network time (ms)", "load_network_time", duration_ms)});
|
||||
|
||||
convert_io_names_in_map(inputFiles, compiledModel.inputs());
|
||||
app_inputs_info = get_inputs_info(FLAGS_shape,
|
||||
@ -420,11 +424,12 @@ int main(int argc, char* argv[]) {
|
||||
|
||||
auto startTime = Time::now();
|
||||
auto model = core.read_model(FLAGS_m);
|
||||
auto duration_ms = double_to_string(get_duration_ms_till_now(startTime));
|
||||
slog::info << "Read network took " << duration_ms << " ms" << slog::endl;
|
||||
auto duration_ms = get_duration_ms_till_now(startTime);
|
||||
slog::info << "Read network took " << double_to_string(duration_ms) << " ms" << slog::endl;
|
||||
if (statistics)
|
||||
statistics->add_parameters(StatisticsReport::Category::EXECUTION_RESULTS,
|
||||
{{"read network time (ms)", duration_ms}});
|
||||
statistics->add_parameters(
|
||||
StatisticsReport::Category::EXECUTION_RESULTS,
|
||||
{StatisticsVariant("read network time (ms)", "read_network_time", duration_ms)});
|
||||
|
||||
const auto& inputInfo = std::const_pointer_cast<const ov::Model>(model)->inputs();
|
||||
if (inputInfo.empty()) {
|
||||
@ -453,11 +458,12 @@ int main(int argc, char* argv[]) {
|
||||
slog::info << "Reshaping network: " << get_shapes_string(shapes) << slog::endl;
|
||||
startTime = Time::now();
|
||||
model->reshape(shapes);
|
||||
duration_ms = double_to_string(get_duration_ms_till_now(startTime));
|
||||
slog::info << "Reshape network took " << duration_ms << " ms" << slog::endl;
|
||||
duration_ms = get_duration_ms_till_now(startTime);
|
||||
slog::info << "Reshape network took " << double_to_string(duration_ms) << " ms" << slog::endl;
|
||||
if (statistics)
|
||||
statistics->add_parameters(StatisticsReport::Category::EXECUTION_RESULTS,
|
||||
{{"reshape network time (ms)", duration_ms}});
|
||||
statistics->add_parameters(
|
||||
StatisticsReport::Category::EXECUTION_RESULTS,
|
||||
{StatisticsVariant("reshape network time (ms)", "reshape_network_time", duration_ms)});
|
||||
}
|
||||
|
||||
// ----------------- 6. Configuring inputs and outputs
|
||||
@ -559,11 +565,12 @@ int main(int argc, char* argv[]) {
|
||||
next_step();
|
||||
startTime = Time::now();
|
||||
compiledModel = core.compile_model(model, device_name);
|
||||
duration_ms = double_to_string(get_duration_ms_till_now(startTime));
|
||||
slog::info << "Load network took " << duration_ms << " ms" << slog::endl;
|
||||
duration_ms = get_duration_ms_till_now(startTime);
|
||||
slog::info << "Load network took " << double_to_string(duration_ms) << " ms" << slog::endl;
|
||||
if (statistics)
|
||||
statistics->add_parameters(StatisticsReport::Category::EXECUTION_RESULTS,
|
||||
{{"load network time (ms)", duration_ms}});
|
||||
statistics->add_parameters(
|
||||
StatisticsReport::Category::EXECUTION_RESULTS,
|
||||
{StatisticsVariant("load network time (ms)", "load_network_time", duration_ms)});
|
||||
} else {
|
||||
next_step();
|
||||
slog::info << "Skipping the step for compiled network" << slog::endl;
|
||||
@ -583,11 +590,12 @@ int main(int argc, char* argv[]) {
|
||||
compiledModel = core.import_model(modelStream, device_name, {});
|
||||
modelStream.close();
|
||||
|
||||
auto duration_ms = double_to_string(get_duration_ms_till_now(startTime));
|
||||
slog::info << "Import network took " << duration_ms << " ms" << slog::endl;
|
||||
auto duration_ms = get_duration_ms_till_now(startTime);
|
||||
slog::info << "Import network took " << double_to_string(duration_ms) << " ms" << slog::endl;
|
||||
if (statistics)
|
||||
statistics->add_parameters(StatisticsReport::Category::EXECUTION_RESULTS,
|
||||
{{"import network time (ms)", duration_ms}});
|
||||
statistics->add_parameters(
|
||||
StatisticsReport::Category::EXECUTION_RESULTS,
|
||||
{StatisticsVariant("import network time (ms)", "import_network_time", duration_ms)});
|
||||
|
||||
convert_io_names_in_map(inputFiles, compiledModel.inputs());
|
||||
app_inputs_info = get_inputs_info(FLAGS_shape,
|
||||
@ -692,24 +700,27 @@ int main(int argc, char* argv[]) {
|
||||
if (statistics) {
|
||||
statistics->add_parameters(
|
||||
StatisticsReport::Category::RUNTIME_CONFIG,
|
||||
{
|
||||
{"benchmark mode", inferenceOnly ? "inference only" : "full"},
|
||||
{"topology", topology_name},
|
||||
{"target device", device_name},
|
||||
{"API", FLAGS_api},
|
||||
{"precision", std::string(type.get_type_name())},
|
||||
{"batch size", std::to_string(batchSize)},
|
||||
{"number of iterations", std::to_string(niter)},
|
||||
{"number of parallel infer requests", std::to_string(nireq)},
|
||||
{"duration (ms)", std::to_string(get_duration_in_milliseconds(duration_seconds))},
|
||||
});
|
||||
StatisticsReport::Parameters(
|
||||
{StatisticsVariant("benchmark mode", "benchmark_mode", inferenceOnly ? "inference only" : "full"),
|
||||
StatisticsVariant("topology", "topology", topology_name),
|
||||
StatisticsVariant("target device", "target_device", device_name),
|
||||
StatisticsVariant("API", "api", FLAGS_api),
|
||||
StatisticsVariant("precision", "precision", type.get_type_name()),
|
||||
StatisticsVariant("batch size", "batch_size", batchSize),
|
||||
StatisticsVariant("number of iterations", "iterations_num", niter),
|
||||
StatisticsVariant("number of parallel infer requests", "nireq", nireq),
|
||||
StatisticsVariant("duration (ms)", "duration", get_duration_in_milliseconds(duration_seconds))}));
|
||||
for (auto& nstreams : device_nstreams) {
|
||||
std::stringstream ss;
|
||||
ss << "number of " << nstreams.first << " streams";
|
||||
|
||||
std::string dev_name = nstreams.first;
|
||||
std::transform(dev_name.begin(), dev_name.end(), dev_name.begin(), [](unsigned char c) {
|
||||
return c == ' ' ? '_' : std::tolower(c);
|
||||
});
|
||||
|
||||
statistics->add_parameters(StatisticsReport::Category::RUNTIME_CONFIG,
|
||||
{
|
||||
{ss.str(), nstreams.second},
|
||||
});
|
||||
{StatisticsVariant(ss.str(), dev_name + "_streams_num", nstreams.second)});
|
||||
}
|
||||
}
|
||||
|
||||
@ -872,12 +883,13 @@ int main(int argc, char* argv[]) {
|
||||
|
||||
inferRequestsQueue.wait_all();
|
||||
|
||||
auto duration_ms = double_to_string(inferRequestsQueue.get_latencies()[0]);
|
||||
slog::info << "First inference took " << duration_ms << " ms" << slog::endl;
|
||||
auto duration_ms = inferRequestsQueue.get_latencies()[0];
|
||||
slog::info << "First inference took " << double_to_string(duration_ms) << " ms" << slog::endl;
|
||||
|
||||
if (statistics) {
|
||||
statistics->add_parameters(StatisticsReport::Category::EXECUTION_RESULTS,
|
||||
{{"first inference time (ms)", duration_ms}});
|
||||
statistics->add_parameters(
|
||||
StatisticsReport::Category::EXECUTION_RESULTS,
|
||||
{StatisticsVariant("first inference time (ms)", "first_inference_time", duration_ms)});
|
||||
}
|
||||
inferRequestsQueue.reset_times();
|
||||
|
||||
@ -967,24 +979,32 @@ int main(int argc, char* argv[]) {
|
||||
// wait the latest inference executions
|
||||
inferRequestsQueue.wait_all();
|
||||
|
||||
LatencyMetrics generalLatency(inferRequestsQueue.get_latencies());
|
||||
LatencyMetrics generalLatency(inferRequestsQueue.get_latencies(), "", FLAGS_latency_percentile);
|
||||
std::vector<LatencyMetrics> groupLatencies = {};
|
||||
if (FLAGS_pcseq && app_inputs_info.size() > 1) {
|
||||
for (auto lats : inferRequestsQueue.get_latency_groups()) {
|
||||
groupLatencies.push_back(LatencyMetrics(lats));
|
||||
const auto& lat_groups = inferRequestsQueue.get_latency_groups();
|
||||
for (int i = 0; i < lat_groups.size(); i++) {
|
||||
const auto& lats = lat_groups[i];
|
||||
|
||||
std::string data_shapes_string = "";
|
||||
for (auto& item : app_inputs_info[i]) {
|
||||
data_shapes_string += item.first + get_shape_string(item.second.dataShape) + ",";
|
||||
}
|
||||
data_shapes_string =
|
||||
data_shapes_string == "" ? "" : data_shapes_string.substr(0, data_shapes_string.size() - 1);
|
||||
|
||||
groupLatencies.emplace_back(lats, data_shapes_string, FLAGS_latency_percentile);
|
||||
}
|
||||
}
|
||||
|
||||
double totalDuration = inferRequestsQueue.get_duration_in_milliseconds();
|
||||
double fps = (FLAGS_api == "sync") ? batchSize * 1000.0 / generalLatency.percentile(FLAGS_latency_percentile)
|
||||
double fps = (FLAGS_api == "sync") ? batchSize * 1000.0 / generalLatency.median_or_percentile
|
||||
: 1000.0 * processedFramesN / totalDuration;
|
||||
|
||||
if (statistics) {
|
||||
statistics->add_parameters(StatisticsReport::Category::EXECUTION_RESULTS,
|
||||
{
|
||||
{"total execution time (ms)", double_to_string(totalDuration)},
|
||||
{"total number of iterations", std::to_string(iteration)},
|
||||
});
|
||||
{StatisticsVariant("total execution time (ms)", "execution_time", totalDuration),
|
||||
StatisticsVariant("total number of iterations", "iterations_num", iteration)});
|
||||
if (device_name.find("MULTI") == std::string::npos) {
|
||||
std::string latency_label;
|
||||
if (FLAGS_latency_percentile == 50) {
|
||||
@ -994,60 +1014,21 @@ int main(int argc, char* argv[]) {
|
||||
}
|
||||
statistics->add_parameters(
|
||||
StatisticsReport::Category::EXECUTION_RESULTS,
|
||||
{
|
||||
{latency_label, double_to_string(generalLatency.percentile(FLAGS_latency_percentile))},
|
||||
});
|
||||
statistics->add_parameters(StatisticsReport::Category::EXECUTION_RESULTS,
|
||||
{
|
||||
{"Average latency (ms)", double_to_string(generalLatency.average())},
|
||||
});
|
||||
statistics->add_parameters(StatisticsReport::Category::EXECUTION_RESULTS,
|
||||
{
|
||||
{"Min latency (ms)", double_to_string(generalLatency.min())},
|
||||
});
|
||||
statistics->add_parameters(StatisticsReport::Category::EXECUTION_RESULTS,
|
||||
{
|
||||
{"Max latency (ms)", double_to_string(generalLatency.max())},
|
||||
});
|
||||
{StatisticsVariant(latency_label, "latency_median", generalLatency.median_or_percentile),
|
||||
StatisticsVariant("Percentile boundary", "percentile_boundary", FLAGS_latency_percentile),
|
||||
StatisticsVariant("Average latency (ms)", "latency_avg", generalLatency.avg),
|
||||
StatisticsVariant("Min latency (ms)", "latency_min", generalLatency.min),
|
||||
StatisticsVariant("Max latency (ms)", "latency_max", generalLatency.max),
|
||||
StatisticsVariant("throughput", "throughput", fps)});
|
||||
|
||||
if (FLAGS_pcseq && app_inputs_info.size() > 1) {
|
||||
statistics->add_parameters(StatisticsReport::Category::EXECUTION_RESULTS,
|
||||
{
|
||||
{"Latency for each data shape group:", ""},
|
||||
});
|
||||
for (size_t i = 0; i < app_inputs_info.size(); ++i) {
|
||||
std::string data_shapes_string = "";
|
||||
data_shapes_string += std::to_string(i + 1) + ". ";
|
||||
for (auto& item : app_inputs_info[i]) {
|
||||
data_shapes_string += item.first + " : " + get_shape_string(item.second.dataShape) + " ";
|
||||
}
|
||||
statistics->add_parameters(StatisticsReport::Category::EXECUTION_RESULTS,
|
||||
{
|
||||
{data_shapes_string, ""},
|
||||
});
|
||||
for (size_t i = 0; i < groupLatencies.size(); ++i) {
|
||||
statistics->add_parameters(
|
||||
StatisticsReport::Category::EXECUTION_RESULTS,
|
||||
{
|
||||
{latency_label,
|
||||
double_to_string(groupLatencies[i].percentile(FLAGS_latency_percentile))},
|
||||
});
|
||||
statistics->add_parameters(StatisticsReport::Category::EXECUTION_RESULTS,
|
||||
{
|
||||
{"Average (ms)", double_to_string(groupLatencies[i].average())},
|
||||
});
|
||||
statistics->add_parameters(StatisticsReport::Category::EXECUTION_RESULTS,
|
||||
{
|
||||
{"Min (ms)", double_to_string(groupLatencies[i].min())},
|
||||
});
|
||||
statistics->add_parameters(StatisticsReport::Category::EXECUTION_RESULTS,
|
||||
{
|
||||
{"Max (ms)", double_to_string(groupLatencies[i].max())},
|
||||
});
|
||||
StatisticsReport::Category::EXECUTION_RESULTS_GROUPPED,
|
||||
{StatisticsVariant("Group Latencies", "group_latencies", groupLatencies[i])});
|
||||
}
|
||||
}
|
||||
}
|
||||
statistics->add_parameters(StatisticsReport::Category::EXECUTION_RESULTS,
|
||||
{{"throughput", double_to_string(fps)}});
|
||||
}
|
||||
progressBar.finish();
|
||||
|
||||
@ -1094,7 +1075,7 @@ int main(int argc, char* argv[]) {
|
||||
slog::info << "Duration: " << double_to_string(totalDuration) << " ms" << slog::endl;
|
||||
if (device_name.find("MULTI") == std::string::npos) {
|
||||
slog::info << "Latency: " << slog::endl;
|
||||
generalLatency.log_total(FLAGS_latency_percentile);
|
||||
generalLatency.write_to_slog();
|
||||
|
||||
if (FLAGS_pcseq && app_inputs_info.size() > 1) {
|
||||
slog::info << "Latency for each data shape group:" << slog::endl;
|
||||
@ -1109,7 +1090,7 @@ int main(int argc, char* argv[]) {
|
||||
}
|
||||
slog::info << slog::endl;
|
||||
|
||||
groupLatencies[i].log_total(FLAGS_latency_percentile);
|
||||
groupLatencies[i].write_to_slog();
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1120,9 +1101,7 @@ int main(int argc, char* argv[]) {
|
||||
|
||||
if (statistics) {
|
||||
statistics->add_parameters(StatisticsReport::Category::EXECUTION_RESULTS,
|
||||
{
|
||||
{"error", ex.what()},
|
||||
});
|
||||
{StatisticsVariant("error", "error", ex.what())});
|
||||
statistics->dump();
|
||||
}
|
||||
|
||||
|
@ -28,8 +28,8 @@ public:
|
||||
}
|
||||
|
||||
bool is_equal(const AllocatorImpl& other) const override {
|
||||
auto other_blob_allocator = dynamic_cast<const SharedTensorAllocator*>(&other);
|
||||
return other_blob_allocator != nullptr && other_blob_allocator == this;
|
||||
auto other_tensor_allocator = dynamic_cast<const SharedTensorAllocator*>(&other);
|
||||
return other_tensor_allocator != nullptr && other_tensor_allocator == this;
|
||||
}
|
||||
|
||||
char* get_buffer() {
|
||||
|
@ -12,6 +12,8 @@
|
||||
#include "statistics_report.hpp"
|
||||
// clang-format on
|
||||
|
||||
static const char* status_names[] = {"NOT_RUN", "OPTIMIZED_OUT", "EXECUTED"};
|
||||
|
||||
void StatisticsReport::add_parameters(const Category& category, const Parameters& parameters) {
|
||||
if (_parameters.count(category) == 0)
|
||||
_parameters[category] = parameters;
|
||||
@ -24,7 +26,10 @@ void StatisticsReport::dump() {
|
||||
|
||||
auto dump_parameters = [&dumper](const Parameters& parameters) {
|
||||
for (auto& parameter : parameters) {
|
||||
dumper << parameter.first << parameter.second;
|
||||
if (parameter.type != StatisticsVariant::METRICS) {
|
||||
dumper << parameter.csv_name;
|
||||
}
|
||||
dumper << parameter.to_string();
|
||||
dumper.endLine();
|
||||
}
|
||||
};
|
||||
@ -52,10 +57,20 @@ void StatisticsReport::dump() {
|
||||
dumper.endLine();
|
||||
}
|
||||
|
||||
if (_parameters.count(Category::EXECUTION_RESULTS_GROUPPED)) {
|
||||
dumper << "Group Latencies";
|
||||
dumper.endLine();
|
||||
dumper << "Data shape;Median;Average;Min;Max";
|
||||
dumper.endLine();
|
||||
|
||||
dump_parameters(_parameters.at(Category::EXECUTION_RESULTS_GROUPPED));
|
||||
dumper.endLine();
|
||||
}
|
||||
|
||||
slog::info << "Statistics report is stored to " << dumper.getFilename() << slog::endl;
|
||||
}
|
||||
|
||||
void StatisticsReport::dump_performance_counters_request(CsvDumper& dumper, const PerformaceCounters& perfCounts) {
|
||||
void StatisticsReport::dump_performance_counters_request(CsvDumper& dumper, const PerformanceCounters& perfCounts) {
|
||||
std::chrono::microseconds total = std::chrono::microseconds::zero();
|
||||
std::chrono::microseconds total_cpu = std::chrono::microseconds::zero();
|
||||
|
||||
@ -69,18 +84,9 @@ void StatisticsReport::dump_performance_counters_request(CsvDumper& dumper, cons
|
||||
|
||||
for (const auto& layer : perfCounts) {
|
||||
dumper << layer.node_name; // layer name
|
||||
|
||||
switch (layer.status) {
|
||||
case ov::ProfilingInfo::Status::EXECUTED:
|
||||
dumper << "EXECUTED";
|
||||
break;
|
||||
case ov::ProfilingInfo::Status::NOT_RUN:
|
||||
dumper << "NOT_RUN";
|
||||
break;
|
||||
case ov::ProfilingInfo::Status::OPTIMIZED_OUT:
|
||||
dumper << "OPTIMIZED_OUT";
|
||||
break;
|
||||
}
|
||||
dumper << ((int)layer.status < (sizeof(status_names) / sizeof(status_names[0]))
|
||||
? status_names[(int)layer.status]
|
||||
: "INVALID_STATUS");
|
||||
dumper << layer.node_type << layer.exec_type;
|
||||
dumper << std::to_string(layer.real_time.count() / 1000.0) << std::to_string(layer.cpu_time.count() / 1000.0);
|
||||
total += layer.real_time;
|
||||
@ -96,7 +102,35 @@ void StatisticsReport::dump_performance_counters_request(CsvDumper& dumper, cons
|
||||
dumper.endLine();
|
||||
}
|
||||
|
||||
void StatisticsReport::dump_performance_counters(const std::vector<PerformaceCounters>& perfCounts) {
|
||||
StatisticsReport::PerformanceCounters StatisticsReport::get_average_performance_counters(
|
||||
const std::vector<PerformanceCounters>& perfCounts) {
|
||||
StatisticsReport::PerformanceCounters performanceCountersAvg;
|
||||
// iterate over each processed infer request and handle its PM data
|
||||
for (size_t i = 0; i < perfCounts.size(); i++) {
|
||||
// iterate over each layer from sorted vector and add required PM data
|
||||
// to the per-layer maps
|
||||
for (const auto& pm : perfCounts[i]) {
|
||||
int idx = 0;
|
||||
for (; idx < performanceCountersAvg.size(); idx++) {
|
||||
if (performanceCountersAvg[idx].node_name == pm.node_name) {
|
||||
performanceCountersAvg[idx].real_time += pm.real_time;
|
||||
performanceCountersAvg[idx].cpu_time += pm.cpu_time;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (idx == performanceCountersAvg.size()) {
|
||||
performanceCountersAvg.push_back(pm);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (auto& pm : performanceCountersAvg) {
|
||||
pm.real_time /= perfCounts.size();
|
||||
pm.cpu_time /= perfCounts.size();
|
||||
}
|
||||
return performanceCountersAvg;
|
||||
};
|
||||
|
||||
void StatisticsReport::dump_performance_counters(const std::vector<PerformanceCounters>& perfCounts) {
|
||||
if ((_config.report_type.empty()) || (_config.report_type == noCntReport)) {
|
||||
slog::info << "Statistics collecting for performance counters was not "
|
||||
"requested. No reports are dumped."
|
||||
@ -113,35 +147,180 @@ void StatisticsReport::dump_performance_counters(const std::vector<PerformaceCou
|
||||
dump_performance_counters_request(dumper, pc);
|
||||
}
|
||||
} else if (_config.report_type == averageCntReport) {
|
||||
auto getAveragePerformanceCounters = [&perfCounts]() {
|
||||
std::vector<ov::ProfilingInfo> performanceCountersAvg;
|
||||
// iterate over each processed infer request and handle its PM data
|
||||
for (size_t i = 0; i < perfCounts.size(); i++) {
|
||||
// iterate over each layer from sorted vector and add required PM data
|
||||
// to the per-layer maps
|
||||
for (const auto& pm : perfCounts[i]) {
|
||||
int idx = 0;
|
||||
for (; idx < performanceCountersAvg.size(); idx++) {
|
||||
if (performanceCountersAvg[idx].node_name == pm.node_name) {
|
||||
performanceCountersAvg[idx].real_time += pm.real_time;
|
||||
performanceCountersAvg[idx].cpu_time += pm.cpu_time;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (idx == performanceCountersAvg.size()) {
|
||||
performanceCountersAvg.push_back(pm);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (auto& pm : performanceCountersAvg) {
|
||||
pm.real_time /= perfCounts.size();
|
||||
pm.cpu_time /= perfCounts.size();
|
||||
}
|
||||
return performanceCountersAvg;
|
||||
};
|
||||
dump_performance_counters_request(dumper, getAveragePerformanceCounters());
|
||||
dump_performance_counters_request(dumper, get_average_performance_counters(perfCounts));
|
||||
} else {
|
||||
throw std::logic_error("PM data can only be collected for average or detailed report types");
|
||||
}
|
||||
slog::info << "Performance counters report is stored to " << dumper.getFilename() << slog::endl;
|
||||
}
|
||||
|
||||
void StatisticsReportJSON::dump_parameters(nlohmann::json& js, const StatisticsReport::Parameters& parameters) {
|
||||
for (auto& parameter : parameters) {
|
||||
parameter.write_to_json(js);
|
||||
}
|
||||
};
|
||||
|
||||
void StatisticsReportJSON::dump() {
|
||||
nlohmann::json js;
|
||||
std::string name = _config.report_folder + _separator + "benchmark_report.json";
|
||||
|
||||
if (_parameters.count(Category::COMMAND_LINE_PARAMETERS)) {
|
||||
dump_parameters(js["cmd_options"], _parameters.at(Category::COMMAND_LINE_PARAMETERS));
|
||||
}
|
||||
if (_parameters.count(Category::RUNTIME_CONFIG)) {
|
||||
dump_parameters(js["configuration_setup"], _parameters.at(Category::RUNTIME_CONFIG));
|
||||
}
|
||||
if (_parameters.count(Category::EXECUTION_RESULTS)) {
|
||||
dump_parameters(js["execution_results"], _parameters.at(Category::EXECUTION_RESULTS));
|
||||
}
|
||||
if (_parameters.count(Category::EXECUTION_RESULTS_GROUPPED)) {
|
||||
dump_parameters(js["execution_results"], _parameters.at(Category::EXECUTION_RESULTS_GROUPPED));
|
||||
}
|
||||
|
||||
std::ofstream out_stream(name);
|
||||
out_stream << std::setw(4) << js << std::endl;
|
||||
slog::info << "Statistics report is stored to " << name << slog::endl;
|
||||
}
|
||||
|
||||
void StatisticsReportJSON::dump_performance_counters(const std::vector<PerformanceCounters>& perfCounts) {
|
||||
if ((_config.report_type.empty()) || (_config.report_type == noCntReport)) {
|
||||
slog::info << "Statistics collecting for performance counters was not "
|
||||
"requested. No reports are dumped."
|
||||
<< slog::endl;
|
||||
return;
|
||||
}
|
||||
if (perfCounts.empty()) {
|
||||
slog::info << "Performance counters are empty. No reports are dumped." << slog::endl;
|
||||
return;
|
||||
}
|
||||
|
||||
nlohmann::json js;
|
||||
std::string name = _config.report_folder + _separator + "benchmark_" + _config.report_type + "_report.json";
|
||||
|
||||
if (_config.report_type == detailedCntReport) {
|
||||
js["report_type"] = "detailed";
|
||||
js["detailed_performance"] = nlohmann::json::array();
|
||||
for (auto& pc : perfCounts) {
|
||||
js["detailed_performance"].push_back(perf_counters_to_json(pc));
|
||||
}
|
||||
} else if (_config.report_type == averageCntReport) {
|
||||
js["report_type"] = "average";
|
||||
js["avg_performance"] = perf_counters_to_json(get_average_performance_counters(perfCounts));
|
||||
} else {
|
||||
throw std::logic_error("PM data can only be collected for average or detailed report types");
|
||||
}
|
||||
|
||||
std::ofstream out_stream(name);
|
||||
out_stream << std::setw(4) << js << std::endl;
|
||||
slog::info << "Performance counters report is stored to " << name << slog::endl;
|
||||
}
|
||||
|
||||
const nlohmann::json StatisticsReportJSON::perf_counters_to_json(
|
||||
const StatisticsReport::PerformanceCounters& perfCounts) {
|
||||
std::chrono::microseconds total = std::chrono::microseconds::zero();
|
||||
std::chrono::microseconds total_cpu = std::chrono::microseconds::zero();
|
||||
|
||||
nlohmann::json js;
|
||||
js["nodes"] = nlohmann::json::array();
|
||||
for (const auto& layer : perfCounts) {
|
||||
nlohmann::json item;
|
||||
|
||||
item["name"] = layer.node_name; // layer name
|
||||
item["status"] =
|
||||
((int)layer.status < (sizeof(status_names) / sizeof(status_names[0])) ? status_names[(int)layer.status]
|
||||
: "INVALID_STATUS");
|
||||
item["node_type"] = layer.node_type;
|
||||
item["exec_type"] = layer.exec_type;
|
||||
item["real_time"] = layer.real_time.count() / 1000.0;
|
||||
item["cpu_time"] = layer.cpu_time.count() / 1000.0;
|
||||
total += layer.real_time;
|
||||
total_cpu += layer.cpu_time;
|
||||
js["nodes"].push_back(item);
|
||||
}
|
||||
js["total_real_time"] = total.count() / 1000.0;
|
||||
js["total_cpu_time"] = total_cpu.count() / 1000.0;
|
||||
return js;
|
||||
}
|
||||
|
||||
void LatencyMetrics::write_to_stream(std::ostream& stream) const {
|
||||
stream << data_shape << ";" << std::fixed << std::setprecision(2) << median_or_percentile << ";" << avg << ";"
|
||||
<< min << ";" << max;
|
||||
}
|
||||
|
||||
void LatencyMetrics::write_to_slog() const {
|
||||
std::string percentileStr = (percentile_boundary == 50)
|
||||
? "\tMedian: "
|
||||
: "\t" + std::to_string(percentile_boundary) + " percentile: ";
|
||||
if (!data_shape.empty()) {
|
||||
slog::info << "\tData shape: " << data_shape << slog::endl;
|
||||
}
|
||||
slog::info << percentileStr << double_to_string(median_or_percentile) << " ms" << slog::endl;
|
||||
slog::info << "\tAverage: " << double_to_string(avg) << " ms" << slog::endl;
|
||||
slog::info << "\tMin: " << double_to_string(min) << " ms" << slog::endl;
|
||||
slog::info << "\tMax: " << double_to_string(max) << " ms" << slog::endl;
|
||||
}
|
||||
|
||||
const nlohmann::json LatencyMetrics::to_json() const {
|
||||
nlohmann::json stat;
|
||||
stat["data_shape"] = data_shape;
|
||||
stat["latency_median"] = median_or_percentile;
|
||||
stat["latency_average"] = avg;
|
||||
stat["latency_min"] = min;
|
||||
stat["latency_max"] = max;
|
||||
return stat;
|
||||
}
|
||||
|
||||
void LatencyMetrics::fill_data(std::vector<double> latencies, size_t percentile_boundary) {
|
||||
if (latencies.empty()) {
|
||||
throw std::logic_error("Latency metrics class expects non-empty vector of latencies at consturction.");
|
||||
}
|
||||
std::sort(latencies.begin(), latencies.end());
|
||||
min = latencies[0];
|
||||
avg = std::accumulate(latencies.begin(), latencies.end(), 0.0) / latencies.size();
|
||||
median_or_percentile = latencies[size_t(latencies.size() / 100.0 * percentile_boundary)];
|
||||
max = latencies.back();
|
||||
};
|
||||
|
||||
std::string StatisticsVariant::to_string() const {
|
||||
switch (type) {
|
||||
case INT:
|
||||
return std::to_string(i_val);
|
||||
case DOUBLE:
|
||||
return std::to_string(d_val);
|
||||
case STRING:
|
||||
return s_val;
|
||||
case ULONGLONG:
|
||||
return std::to_string(ull_val);
|
||||
case METRICS:
|
||||
std::ostringstream str;
|
||||
metrics_val.write_to_stream(str);
|
||||
return str.str();
|
||||
}
|
||||
throw std::invalid_argument("StatisticsVariant::to_string : invalid type is provided");
|
||||
}
|
||||
|
||||
void StatisticsVariant::write_to_json(nlohmann::json& js) const {
|
||||
switch (type) {
|
||||
case INT:
|
||||
js[json_name] = i_val;
|
||||
break;
|
||||
case DOUBLE:
|
||||
js[json_name] = d_val;
|
||||
break;
|
||||
case STRING:
|
||||
js[json_name] = s_val;
|
||||
break;
|
||||
case ULONGLONG:
|
||||
js[json_name] = ull_val;
|
||||
break;
|
||||
case METRICS: {
|
||||
auto& arr = js[json_name];
|
||||
if (arr.empty()) {
|
||||
arr = nlohmann::json::array();
|
||||
}
|
||||
arr.push_back(metrics_val.to_json());
|
||||
} break;
|
||||
default:
|
||||
throw std::invalid_argument("StatisticsVariant:: json conversion : invalid type is provided");
|
||||
}
|
||||
}
|
||||
|
@ -5,6 +5,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <map>
|
||||
#include <nlohmann/json.hpp>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
@ -25,66 +26,99 @@ static constexpr char detailedCntReport[] = "detailed_counters";
|
||||
/// @brief Responsible for calculating different latency metrics
|
||||
class LatencyMetrics {
|
||||
public:
|
||||
LatencyMetrics() = delete;
|
||||
LatencyMetrics() {}
|
||||
|
||||
LatencyMetrics(const std::vector<double>& latencies) : latencies(latencies) {
|
||||
if (latencies.empty()) {
|
||||
throw std::logic_error("Latency metrics class expects non-empty vector of latencies at consturction.");
|
||||
}
|
||||
std::sort(this->latencies.begin(), this->latencies.end());
|
||||
LatencyMetrics(const std::vector<double>& latencies,
|
||||
const std::string& data_shape = "",
|
||||
size_t percentile_boundary = 50)
|
||||
: data_shape(data_shape),
|
||||
percentile_boundary(percentile_boundary) {
|
||||
fill_data(latencies, percentile_boundary);
|
||||
}
|
||||
|
||||
LatencyMetrics(std::vector<double>&& latencies) : latencies(latencies) {
|
||||
if (latencies.empty()) {
|
||||
throw std::logic_error("Latency metrics class expects non-empty vector of latencies at consturction.");
|
||||
}
|
||||
std::sort(this->latencies.begin(), this->latencies.end());
|
||||
}
|
||||
void write_to_stream(std::ostream& stream) const;
|
||||
void write_to_slog() const;
|
||||
const nlohmann::json to_json() const;
|
||||
|
||||
double min() {
|
||||
return latencies[0];
|
||||
}
|
||||
|
||||
double average() {
|
||||
return std::accumulate(latencies.begin(), latencies.end(), 0.0) / latencies.size();
|
||||
}
|
||||
|
||||
double percentile(std::size_t p) {
|
||||
return latencies[size_t(latencies.size() / 100.0 * p)];
|
||||
}
|
||||
|
||||
double max() {
|
||||
return latencies.back();
|
||||
}
|
||||
|
||||
void log_total(size_t p) {
|
||||
std::string percentileStr = (p == 50) ? "\tMedian: " : "\t" + std::to_string(p) + " percentile: ";
|
||||
slog::info << percentileStr << double_to_string(percentile(p)) << " ms" << slog::endl;
|
||||
slog::info << "\tAvg: " << double_to_string(average()) << " ms" << slog::endl;
|
||||
slog::info << "\tMin: " << double_to_string(min()) << " ms" << slog::endl;
|
||||
slog::info << "\tMax: " << double_to_string(max()) << " ms" << slog::endl;
|
||||
}
|
||||
public:
|
||||
double median_or_percentile = 0;
|
||||
double avg = 0;
|
||||
double min = 0;
|
||||
double max = 0;
|
||||
std::string data_shape;
|
||||
|
||||
private:
|
||||
std::vector<double> latencies;
|
||||
void fill_data(std::vector<double> latencies, size_t percentile_boundary);
|
||||
size_t percentile_boundary = 50;
|
||||
};
|
||||
|
||||
class StatisticsVariant {
|
||||
public:
|
||||
enum Type { INT, DOUBLE, STRING, ULONGLONG, METRICS };
|
||||
|
||||
StatisticsVariant(std::string csv_name, std::string json_name, int v)
|
||||
: csv_name(csv_name),
|
||||
json_name(json_name),
|
||||
i_val(v),
|
||||
type(INT) {}
|
||||
StatisticsVariant(std::string csv_name, std::string json_name, double v)
|
||||
: csv_name(csv_name),
|
||||
json_name(json_name),
|
||||
d_val(v),
|
||||
type(DOUBLE) {}
|
||||
StatisticsVariant(std::string csv_name, std::string json_name, const std::string& v)
|
||||
: csv_name(csv_name),
|
||||
json_name(json_name),
|
||||
s_val(v),
|
||||
type(STRING) {}
|
||||
StatisticsVariant(std::string csv_name, std::string json_name, unsigned long long v)
|
||||
: csv_name(csv_name),
|
||||
json_name(json_name),
|
||||
ull_val(v),
|
||||
type(ULONGLONG) {}
|
||||
StatisticsVariant(std::string csv_name, std::string json_name, uint32_t v)
|
||||
: csv_name(csv_name),
|
||||
json_name(json_name),
|
||||
ull_val(v),
|
||||
type(ULONGLONG) {}
|
||||
StatisticsVariant(std::string csv_name, std::string json_name, unsigned long v)
|
||||
: csv_name(csv_name),
|
||||
json_name(json_name),
|
||||
ull_val(v),
|
||||
type(ULONGLONG) {}
|
||||
StatisticsVariant(std::string csv_name, std::string json_name, const LatencyMetrics& v)
|
||||
: csv_name(csv_name),
|
||||
json_name(json_name),
|
||||
metrics_val(v),
|
||||
type(METRICS) {}
|
||||
|
||||
~StatisticsVariant() {}
|
||||
|
||||
std::string csv_name;
|
||||
std::string json_name;
|
||||
int i_val;
|
||||
double d_val;
|
||||
unsigned long long ull_val;
|
||||
std::string s_val;
|
||||
LatencyMetrics metrics_val;
|
||||
Type type;
|
||||
|
||||
std::string to_string() const;
|
||||
void write_to_json(nlohmann::json& js) const;
|
||||
};
|
||||
|
||||
/// @brief Responsible for collecting of statistics and dumping to .csv file
|
||||
class StatisticsReport {
|
||||
public:
|
||||
typedef std::vector<ov::ProfilingInfo> PerformaceCounters;
|
||||
typedef std::vector<std::pair<std::string, std::string>> Parameters;
|
||||
typedef std::vector<ov::ProfilingInfo> PerformanceCounters;
|
||||
typedef std::vector<StatisticsVariant> Parameters;
|
||||
|
||||
struct Config {
|
||||
std::string report_type;
|
||||
std::string report_folder;
|
||||
};
|
||||
|
||||
enum class Category {
|
||||
COMMAND_LINE_PARAMETERS,
|
||||
RUNTIME_CONFIG,
|
||||
EXECUTION_RESULTS,
|
||||
};
|
||||
enum class Category { COMMAND_LINE_PARAMETERS, RUNTIME_CONFIG, EXECUTION_RESULTS, EXECUTION_RESULTS_GROUPPED };
|
||||
|
||||
explicit StatisticsReport(Config config) : _config(std::move(config)) {
|
||||
_separator =
|
||||
@ -103,13 +137,14 @@ public:
|
||||
|
||||
void add_parameters(const Category& category, const Parameters& parameters);
|
||||
|
||||
void dump();
|
||||
virtual void dump();
|
||||
|
||||
void dump_performance_counters(const std::vector<PerformaceCounters>& perfCounts);
|
||||
virtual void dump_performance_counters(const std::vector<PerformanceCounters>& perfCounts);
|
||||
|
||||
private:
|
||||
void dump_performance_counters_request(CsvDumper& dumper, const PerformaceCounters& perfCounts);
|
||||
void dump_performance_counters_request(CsvDumper& dumper, const PerformanceCounters& perfCounts);
|
||||
|
||||
protected:
|
||||
// configuration of current benchmark execution
|
||||
const Config _config;
|
||||
|
||||
@ -118,4 +153,19 @@ private:
|
||||
|
||||
// csv separator
|
||||
std::string _separator;
|
||||
|
||||
StatisticsReport::PerformanceCounters get_average_performance_counters(
|
||||
const std::vector<PerformanceCounters>& perfCounts);
|
||||
};
|
||||
|
||||
class StatisticsReportJSON : public StatisticsReport {
|
||||
public:
|
||||
explicit StatisticsReportJSON(Config config) : StatisticsReport(std::move(config)) {}
|
||||
|
||||
void dump() override;
|
||||
void dump_performance_counters(const std::vector<PerformanceCounters>& perfCounts) override;
|
||||
|
||||
private:
|
||||
void dump_parameters(nlohmann::json& js, const StatisticsReport::Parameters& parameters);
|
||||
const nlohmann::json perf_counters_to_json(const StatisticsReport::PerformanceCounters& perfCounts);
|
||||
};
|
||||
|
Loading…
Reference in New Issue
Block a user