From 7df8d1ca2d364cf108f47c3d21b6ad7f7bfdc2df Mon Sep 17 00:00:00 2001 From: Sergey Shlyapnikov Date: Tue, 22 Aug 2023 14:37:36 +0400 Subject: [PATCH] [GPU] Add per iteration performance profiling mode (#18637) --- .../include/intel_gpu/graph/network.hpp | 4 +++ .../intel_gpu/runtime/debug_configuration.hpp | 1 + .../include/intel_gpu/runtime/profiling.hpp | 6 +++- .../src/graph/include/primitive_inst.h | 2 +- src/plugins/intel_gpu/src/graph/network.cpp | 34 +++++++++++-------- .../intel_gpu/src/graph/primitive_inst.cpp | 7 +++- .../src/runtime/debug_configuration.cpp | 7 ++-- 7 files changed, 42 insertions(+), 19 deletions(-) diff --git a/src/plugins/intel_gpu/include/intel_gpu/graph/network.hpp b/src/plugins/intel_gpu/include/intel_gpu/graph/network.hpp index 2013a9f4543..acddfd0a462 100644 --- a/src/plugins/intel_gpu/include/intel_gpu/graph/network.hpp +++ b/src/plugins/intel_gpu/include/intel_gpu/graph/network.hpp @@ -249,6 +249,10 @@ public: ShapePredictor& get_shape_predictor() { return *_shape_predictor; } +#ifdef GPU_DEBUG_CONFIG + int64_t get_current_iteration_num() { return iteration; } +#endif + private: using output_chains_map = std::map>>; uint32_t net_id = 0; diff --git a/src/plugins/intel_gpu/include/intel_gpu/runtime/debug_configuration.hpp b/src/plugins/intel_gpu/include/intel_gpu/runtime/debug_configuration.hpp index febf41a557f..a912316e409 100644 --- a/src/plugins/intel_gpu/include/intel_gpu/runtime/debug_configuration.hpp +++ b/src/plugins/intel_gpu/include/intel_gpu/runtime/debug_configuration.hpp @@ -103,6 +103,7 @@ public: int disable_onednn; // Disable onednn for discrete GPU (no effect for integrated GPU) int disable_onednn_opt_post_ops; // Disable onednn optimize post operators std::string dump_profiling_data; // Enables dump of extended performance profiling to specified dir + int dump_profiling_data_per_iter; // Enables dump of extended performance profiling to specified dir for each iteration std::string dump_graphs; // Dump optimized graph std::string dump_sources; // Dump opencl sources std::string dump_layers_path; // Enable dumping intermediate buffers and set the dest path diff --git a/src/plugins/intel_gpu/include/intel_gpu/runtime/profiling.hpp b/src/plugins/intel_gpu/include/intel_gpu/runtime/profiling.hpp index 4d4a1d3121d..0b94329e228 100644 --- a/src/plugins/intel_gpu/include/intel_gpu/runtime/profiling.hpp +++ b/src/plugins/intel_gpu/include/intel_gpu/runtime/profiling.hpp @@ -119,6 +119,7 @@ struct perf_counter_key { std::vector output_layouts; std::string impl_name; pipeline_stage stage; + int64_t iteration_num; bool cache_hit; }; @@ -127,6 +128,7 @@ struct perf_counter_hash { size_t seed = 0; seed = hash_combine(seed, static_cast::type>(k.stage)); seed = hash_combine(seed, static_cast(k.cache_hit)); + seed = hash_combine(seed, k.iteration_num); for (auto& layout : k.network_input_layouts) { for (auto& d : layout.get_shape()) { seed = hash_combine(seed, d); @@ -154,6 +156,7 @@ public: , _obj(obj) , _stage(stage) { GPU_DEBUG_IF(profiling_enabled) { + _per_iter_mode = cldnn::debug_configuration::get_instance()->dump_profiling_data_per_iter != 0; _start = std::chrono::high_resolution_clock::now(); } } @@ -167,7 +170,7 @@ public: auto custom_stage_duration = std::chrono::duration_cast(custom_duration).count(); auto total_duration = custom_stage_duration == 0 ? stage_duration : custom_stage_duration; - _obj.add_profiling_data(_stage, cache_hit, total_duration); + _obj.add_profiling_data(_stage, cache_hit, total_duration, _per_iter_mode); } } void set_cache_hit(bool val = true) { cache_hit = val; } @@ -180,6 +183,7 @@ private: std::chrono::nanoseconds custom_duration = {}; ProfiledObjectType& _obj; instrumentation::pipeline_stage _stage; + bool _per_iter_mode = false; bool cache_hit = false; }; diff --git a/src/plugins/intel_gpu/src/graph/include/primitive_inst.h b/src/plugins/intel_gpu/src/graph/include/primitive_inst.h index 3244ad20b49..ba7ab8f2a0b 100644 --- a/src/plugins/intel_gpu/src/graph/include/primitive_inst.h +++ b/src/plugins/intel_gpu/src/graph/include/primitive_inst.h @@ -248,7 +248,7 @@ public: std::unordered_map> const& primitives); std::string get_implementation_name() const; - void add_profiling_data(instrumentation::pipeline_stage stage, bool cache_hit, int64_t time); + void add_profiling_data(instrumentation::pipeline_stage stage, bool cache_hit, int64_t time, bool per_iter_mode = false); const std::unordered_map>& get_profiling_data() const { return _profiling_data; } const std::unordered_map& get_profiling_info() const { return _profiling_info; } diff --git a/src/plugins/intel_gpu/src/graph/network.cpp b/src/plugins/intel_gpu/src/graph/network.cpp index 29f41c5d7e5..ebfca1cc7f1 100644 --- a/src/plugins/intel_gpu/src/graph/network.cpp +++ b/src/plugins/intel_gpu/src/graph/network.cpp @@ -75,7 +75,9 @@ void dump_perf_data_raw(std::string dump_path, const std::listdump_profiling_data_per_iter != 0; + const std::string perf_raw_csv_header = per_iter_mode ? "prim_id,prim_type,stage,net_in_shapes,in_shapes,out_shapes,impl,iter,time_usec\n" + : "prim_id,prim_type,stage,net_in_shapes,in_shapes,out_shapes,impl,iters,time_usec\n"; std::ofstream of(dump_path); if (of.is_open()) { of << perf_raw_csv_header; @@ -114,8 +116,8 @@ void dump_perf_data_raw(std::string dump_path, const std::list(entry); - auto& num_iters = std::get<1>(entry); - int64_t time_avg = time / num_iters; + auto num_iters = per_iter_mode ? key.iteration_num : std::get<1>(entry); + int64_t time_avg = per_iter_mode ? time : time / num_iters; std::string net_in_l_str = layouts_to_str(key.network_input_layouts); std::string in_l_str = layouts_to_str(key.input_layouts); std::string out_l_str = layouts_to_str(key.output_layouts); @@ -246,7 +248,7 @@ void dump(memory::ptr mem, stream& stream, std::ofstream& file_stream, } } -void log_memory_to_file(memory::ptr mem, stream& stream, std::string layerName, bool dump_raw) { +void log_memory_to_file(memory::ptr mem, layout data_layout, stream& stream, std::string layerName, bool dump_raw) { std::cout << "Dump " << (dump_raw ? "raw " : "") << layerName << std::endl; GPU_DEBUG_GET_INSTANCE(debug_config); std::string filename = debug_config->get_name_for_dump(layerName); @@ -257,21 +259,24 @@ void log_memory_to_file(memory::ptr mem, stream& stream, std::string layerName, return; } - auto mem_dt = mem->get_layout().data_type; + // Reinterpret buffer to represent actual data layout + auto actual_mem = mem->get_engine()->reinterpret_buffer(*mem, data_layout); + + auto mem_dt = actual_mem->get_layout().data_type; if (mem_dt == cldnn::data_types::f32) - dump(mem, stream, file_stream, dump_raw); + dump(actual_mem, stream, file_stream, dump_raw); else if (mem_dt == cldnn::data_types::f16) - dump(mem, stream, file_stream, dump_raw); + dump(actual_mem, stream, file_stream, dump_raw); else if (mem_dt == cldnn::data_types::bin) - dump(mem, stream, file_stream, dump_raw); + dump(actual_mem, stream, file_stream, dump_raw); else if (mem_dt == cldnn::data_types::i64) - dump(mem, stream, file_stream, dump_raw); + dump(actual_mem, stream, file_stream, dump_raw); else if (mem_dt == cldnn::data_types::i32) - dump(mem, stream, file_stream, dump_raw); + dump(actual_mem, stream, file_stream, dump_raw); else if (mem_dt == cldnn::data_types::i8) - dump(mem, stream, file_stream, dump_raw); + dump(actual_mem, stream, file_stream, dump_raw); else if (mem_dt == cldnn::data_types::u8) - dump(mem, stream, file_stream, dump_raw); + dump(actual_mem, stream, file_stream, dump_raw); } void wait_for_the_turn() { @@ -294,7 +299,7 @@ void wait_for_the_turn() { #else void dump_perf_data_raw(std::string, const std::list>&) {} -void log_memory_to_file(memory::ptr, stream&, std::string, bool dump_raw) {} +void log_memory_to_file(memory::ptr, layout, stream&, std::string, bool dump_raw) {} void wait_for_the_turn() {} #endif } // namespace @@ -1337,6 +1342,7 @@ void network::execute_impl(const std::vector& events) { debug_str_for_bin_load += (filename + ","); } else { log_memory_to_file(input_mem, + inst->get_input_layout(i), get_stream(), name, debug_config->dump_layers_raw); @@ -1383,7 +1389,7 @@ void network::execute_impl(const std::vector& events) { debug_str_for_bin_load += (filename + ","); } else { // Text dump - log_memory_to_file(output_mem, get_stream(), name, debug_config->dump_layers_raw); + log_memory_to_file(output_mem, inst->get_output_layout(i), get_stream(), name, debug_config->dump_layers_raw); } } diff --git a/src/plugins/intel_gpu/src/graph/primitive_inst.cpp b/src/plugins/intel_gpu/src/graph/primitive_inst.cpp index 730a62d53be..1a1c856905d 100644 --- a/src/plugins/intel_gpu/src/graph/primitive_inst.cpp +++ b/src/plugins/intel_gpu/src/graph/primitive_inst.cpp @@ -1504,13 +1504,18 @@ bool primitive_inst::is_valid_fusion() const { return true; } -void primitive_inst::add_profiling_data(instrumentation::pipeline_stage stage, bool cache_hit, int64_t time) { +void primitive_inst::add_profiling_data(instrumentation::pipeline_stage stage, bool cache_hit, int64_t time, bool per_iter_mode) { instrumentation::perf_counter_key key { _network.get_input_layouts(), _impl_params->input_layouts, _impl_params->output_layouts, get_implementation_name(), stage, +#ifdef GPU_DEBUG_CONFIG + per_iter_mode ? get_network().get_current_iteration_num() : 0, +#else + 0, +#endif cache_hit }; diff --git a/src/plugins/intel_gpu/src/runtime/debug_configuration.cpp b/src/plugins/intel_gpu/src/runtime/debug_configuration.cpp index 084d0095dd2..eb7324cb7b5 100644 --- a/src/plugins/intel_gpu/src/runtime/debug_configuration.cpp +++ b/src/plugins/intel_gpu/src/runtime/debug_configuration.cpp @@ -112,6 +112,7 @@ static void print_help_messages() { message_list.emplace_back("OV_GPU_DisableOnednn", "Disable onednn for discrete GPU (no effect for integrated GPU)"); message_list.emplace_back("OV_GPU_DisableOnednnOptPostOps", "Disable onednn optimize post operators"); message_list.emplace_back("OV_GPU_DumpProfilingData", "Enables dump of extended profiling information to specified directory." + " Please use OV_GPU_DumpProfilingDataPerIter=1 env variable to collect performance per iteration." " Note: Performance impact may be significant as this option enforces host side sync after each primitive"); message_list.emplace_back("OV_GPU_DumpGraphs", "Dump optimized graph"); message_list.emplace_back("OV_GPU_DumpSources", "Dump opencl sources"); @@ -142,9 +143,9 @@ static void print_help_messages() { message_list.emplace_back("OV_GPU_DisableRuntimeSkipReorder", "Disable runtime skip reorder."); message_list.emplace_back("OV_GPU_DisablePrimitiveFusing", "Disable primitive fusing"); message_list.emplace_back("OV_GPU_DumpIteration", "Dump n-th execution of network, separated by space."); - message_list.emplace_back("OV_GPU_MemPreallocationOptions", "Controls buffer pre-allocation feature. Expects 4 values separated by space in" + message_list.emplace_back("OV_GPU_MemPreallocationOptions", "Controls buffer pre-allocation feature. Expects 4 values separated by space in " "the following order: number of iterations for pre-allocation(int), max size of single iteration in bytes(int), " - "max per-dim allowed diff(int), unconditional buffers preallocation ratio(float). For example for disabling memory" + "max per-dim allowed diff(int), unconditional buffers preallocation ratio(float). For example for disabling memory " "preallocation at all, you can use OV_GPU_MemPreallocationOptions='0 0 0 1.0'"); message_list.emplace_back("OV_GPU_LoadDumpRawBinary", "Specified layers which are loading dumped binary files generated by OV_GPU_DumpLayersRawBinary debug-config." @@ -176,6 +177,7 @@ debug_configuration::debug_configuration() , disable_onednn(0) , disable_onednn_opt_post_ops(0) , dump_profiling_data(std::string("")) + , dump_profiling_data_per_iter(0) , dump_graphs(std::string()) , dump_sources(std::string()) , dump_layers_path(std::string()) @@ -217,6 +219,7 @@ debug_configuration::debug_configuration() get_gpu_debug_env_var("DisableOnednn", disable_onednn); get_gpu_debug_env_var("DisableOnednnOptPostOps", disable_onednn_opt_post_ops); get_gpu_debug_env_var("DumpProfilingData", dump_profiling_data); + get_gpu_debug_env_var("DumpProfilingDataPerIter", dump_profiling_data_per_iter); get_gpu_debug_env_var("DryRunPath", dry_run_path); get_gpu_debug_env_var("DumpRuntimeMemoryPool", dump_runtime_memory_pool); get_gpu_debug_env_var("BaseBatchForMemEstimation", base_batch_for_memory_estimation);