IntenseWebs/openvino
3
0 Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[GPU] Improve dump naming rule in debug feature. (#17275)

Browse Source 
* [GPU] Improve dump naming rule in debug feature.

Now, we can support below dump naming rule.
- Exec_graph name
- Wildcard letter for target names ('*', '?')
- Case-insensitive name searching
- Apply to show loop body primitives.

New introduces OV_GPU_xxx
- OV_GPU_ListLayers = 1 (Show layer names and exit)
- OV_GPU_VerboseColor = 1 (Show verbose with color)

Add file, line, function in log prefix.


Signed-off-by: hyunback <hyunback.kim@intel.com>
This commit is contained in:
hyunback kim authored and GitHub committed 2023-05-09 21:30:50 +09:00
1 parent 0a26372a4f
commit 3b90165581
4 files changed
+69 -20
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/plugins/intel_gpu/include/intel_gpu/runtime/debug_configuration.hpp
+18 -1
View File
@@ -42,6 +42,12 @@ enum class LogLevel : int8_t {
} // namespace ov
#ifdef GPU_DEBUG_CONFIG
#if defined(_WIN32) || defined(_WIN64)
#define SEPARATE '\\'
#else
#define SEPARATE '/'
#endif
#define __FILENAME__ (strrchr(__FILE__, SEPARATE) ? strrchr(__FILE__, SEPARATE) + 1 : __FILE__)
#define GPU_DEBUG_IF(cond) if (cond)
#define GPU_DEBUG_CODE(...) __VA_ARGS__
#define GPU_DEBUG_DEFINE_MEM_LOGGER(stage) \
@@ -52,8 +58,16 @@ enum class LogLevel : int8_t {
#define GPU_DEBUG_PROFILED_STAGE_CACHE_HIT(val) stage_prof.set_cache_hit(val)
#define GPU_DEBUG_LOG_RAW_INT(min_verbose_level) if (cldnn::debug_configuration::get_instance()->verbose >= min_verbose_level) \
std::cout << cldnn::debug_configuration::prefix
((cldnn::debug_configuration::get_instance()->verbose_color == 0) ? GPU_DEBUG_LOG_PREFIX : GPU_DEBUG_LOG_COLOR_PREFIX)
#define GPU_DEBUG_LOG_RAW(min_verbose_level) GPU_DEBUG_LOG_RAW_INT(static_cast<std::underlying_type<ov::intel_gpu::LogLevel>::type>(min_verbose_level))
#define GPU_DEBUG_LOG_PREFIX std::cout << cldnn::debug_configuration::prefix << __FILENAME__ << ":" <<__LINE__ << ":" << __func__ << ": "
#define GPU_DEBUG_LOG_COLOR_PREFIX std::cout << DARK_GRAY << cldnn::debug_configuration::prefix << \
BLUE << __FILENAME__ << ":" << PURPLE << __LINE__ << ":" << CYAN << __func__ << ": " << RESET
#define DARK_GRAY "\033[1;30m"
#define BLUE "\033[1;34m"
#define PURPLE "\033[1;35m"
#define CYAN "\033[1;36m"
#define RESET "\033[0m"
#else
#define GPU_DEBUG_IF(cond) if (0)
#define GPU_DEBUG_CODE(...)
@@ -66,6 +80,7 @@ enum class LogLevel : int8_t {
// Macro below is inserted to avoid unused variable warning when GPU_DEBUG_CONFIG is OFF
#define GPU_DEBUG_GET_INSTANCE(name) auto name = cldnn::debug_configuration::get_instance(); (void)(name);
#define GPU_DEBUG_COUT GPU_DEBUG_LOG_RAW(ov::intel_gpu::LogLevel::DISABLED)
#define GPU_DEBUG_INFO GPU_DEBUG_LOG_RAW(ov::intel_gpu::LogLevel::INFO)
#define GPU_DEBUG_LOG GPU_DEBUG_LOG_RAW(ov::intel_gpu::LogLevel::LOG)
#define GPU_DEBUG_TRACE GPU_DEBUG_LOG_RAW(ov::intel_gpu::LogLevel::TRACE)
@@ -81,6 +96,8 @@ public:
static const char *prefix;
int help; // Print help messages
int verbose; // Verbose execution
int verbose_color; // Print verbose color
int list_layers; // Print list layers
int print_multi_kernel_perf; // Print execution time of each kernel in multi-kernel primitimive
int disable_usm; // Disable usm usage
int disable_onednn; // Disable onednn for discrete GPU (no effect for integrated GPU)
src/plugins/intel_gpu/src/graph/graph_optimizer/reorder_inputs.cpp
+4 -7
View File
@@ -566,7 +566,7 @@ void insert_reorders_in_dir(program& p, const std::map<program_node*, format::ty
in_layout.format = get_target_output_format(lo, fmt_map, predecessor, successor);
out_layout.format = get_target_input_format(lo, fmt_map, successor, predecessor);
GPU_DEBUG_LOG << __func__ << ":" << __LINE__ << ":" << dir_msg(dir) << " " << node->id() << " --> " << next->id() << " ## "
GPU_DEBUG_LOG << dir_msg(dir) << " " << node->id() << " --> " << next->id() << " ## "
<< fmt_to_str(in_layout.format) << " --> " << fmt_to_str(out_layout.format) << std::endl;
if (in_layout.format == format::any || out_layout.format == format::any)
@@ -579,8 +579,7 @@ void insert_reorders_in_dir(program& p, const std::map<program_node*, format::ty
auto reorder = reorder_pair.first;
if (reorder && (in_layout.format != format::any && out_layout.format != format::any)) {
auto& reorder_node = p.get_or_create(reorder);
GPU_DEBUG_LOG << __func__ << ":" << __LINE__ << ":" << dir_msg(dir) << " " << reorder_node.id()
<< " Reorder is added" << std::endl;
GPU_DEBUG_LOG << dir_msg(dir) << " " << reorder_node.id() << " Reorder is added" << std::endl;
p.add_intermediate(reorder_node,
*travel_direction_wrapper<dir>::second(node, next),
*travel_direction_wrapper<dir>::first(node, next),
@@ -615,8 +614,7 @@ void insert_reorders_in_dir<direction_e::backwards>(program& p, const std::map<p
in_layout.format = get_target_output_format(lo, fmt_map, predecessor, successor);
out_layout.format = get_target_input_format(lo, fmt_map, successor, predecessor);
GPU_DEBUG_LOG << __func__ << ":" << __LINE__ << ":" << dir_msg(direction_e::backwards)
<< " " << node->id() << " --> " << next.first->id() << " ## "
GPU_DEBUG_LOG << dir_msg(direction_e::backwards) << " " << node->id() << " --> " << next.first->id() << " ## "
<< fmt_to_str(in_layout.format) << " --> " << fmt_to_str(out_layout.format) << std::endl;
if (in_layout.format == format::any || out_layout.format == format::any)
@@ -629,8 +627,7 @@ void insert_reorders_in_dir<direction_e::backwards>(program& p, const std::map<p
auto reorder = reorder_pair.first;
if (reorder && (in_layout.format != format::any && out_layout.format != format::any)) {
auto& reorder_node = p.get_or_create(reorder);
GPU_DEBUG_LOG << __func__ << ":" << __LINE__ << ":" << dir_msg(direction_e::backwards) << " " << reorder_node.id()
<< " Reorder is added" << std::endl;
GPU_DEBUG_LOG << dir_msg(direction_e::backwards) << " " << reorder_node.id() << " Reorder is added" << std::endl;
p.add_intermediate(reorder_node,
*travel_direction_wrapper<direction_e::backwards>::second(node, next.first),
*travel_direction_wrapper<direction_e::backwards>::first(node, next.first),
src/plugins/intel_gpu/src/graph/network.cpp
+13
View File
@@ -1150,6 +1150,19 @@ void network::execute_impl(const std::vector<event::ptr>& events) {
set_arguments();
GPU_DEBUG_GET_INSTANCE(debug_config);
GPU_DEBUG_IF(debug_config->list_layers == 1) {
for (auto& inst : _exec_order) {
GPU_DEBUG_COUT << inst->id() << std::endl;
if (inst->get_node().is_type<loop>()) {
auto& loop_node = inst->get_node().as<loop>();
auto loop_body_primitives = loop_node.get_body_topology().get_primitives_ids();
for (auto& primitive_id : loop_body_primitives) {
GPU_DEBUG_COUT << "\t" << primitive_id << std::endl;
}
}
}
if (!is_internal()) exit(0);
}
int64_t curr_iter = -1;
#ifdef GPU_DEBUG_CONFIG
GPU_DEBUG_IF(!debug_config->dump_iteration.empty()) {
src/plugins/intel_gpu/src/runtime/debug_configuration.cpp
+34 -12
View File
@@ -4,12 +4,12 @@
#include "intel_gpu/runtime/debug_configuration.hpp"
#include <algorithm>
#include <iostream>
#include <iomanip>
#include <iostream>
#include <memory>
#include <vector>
#include <regex>
#include <sstream>
#include <string>
#include <vector>
namespace cldnn {
const char *debug_configuration::prefix = "GPU_Debug: ";
@@ -18,11 +18,11 @@ const char *debug_configuration::prefix = "GPU_Debug: ";
#ifdef GPU_DEBUG_CONFIG
#define GPU_DEBUG_COUT std::cout << cldnn::debug_configuration::prefix
#define GPU_DEBUG_COUT_ std::cout << cldnn::debug_configuration::prefix
template<typename T>
void print_option(std::string option_name, T option_value) {
GPU_DEBUG_COUT << "Config " << option_name << " = " << option_value << std::endl;
GPU_DEBUG_COUT_ << "Config " << option_name << " = " << option_value << std::endl;
}
static std::string to_upper_case(const std::string& var) {
@@ -105,6 +105,8 @@ static void print_help_messages() {
std::vector<std::pair<std::string, std::string>> message_list;
message_list.emplace_back("OV_GPU_Help", "Print help messages");
message_list.emplace_back("OV_GPU_Verbose", "Verbose execution");
message_list.emplace_back("OV_GPU_VerboseColor", "Print verbose color");
message_list.emplace_back("OV_GPU_ListLayers", "Print layers names");
message_list.emplace_back("OV_GPU_PrintMultiKernelPerf", "Print execution time of each kernel in multi-kernel primitimive");
message_list.emplace_back("OV_GPU_DisableUsm", "Disable usm usage");
message_list.emplace_back("OV_GPU_DisableOnednn", "Disable onednn for discrete GPU (no effect for integrated GPU)");
@@ -114,7 +116,8 @@ static void print_help_messages() {
message_list.emplace_back("OV_GPU_DumpGraphs", "Dump optimized graph");
message_list.emplace_back("OV_GPU_DumpSources", "Dump opencl sources");
message_list.emplace_back("OV_GPU_DumpLayersPath", "Enable dumping intermediate buffers and set the dest path");
message_list.emplace_back("OV_GPU_DumpLayers", "Dump intermediate buffers of specified layers only, separated by space");
message_list.emplace_back("OV_GPU_DumpLayers", "Dump intermediate buffers of specified layers only, separated by space."
" Support case-insensitive and regular expression. For example .*conv.*");
message_list.emplace_back("OV_GPU_DumpLayersResult", "Dump output buffers of result layers only");
message_list.emplace_back("OV_GPU_DumpLayersDstOnly", "Dump only output of layers");
message_list.emplace_back("OV_GPU_DumpLayersLimitBatch", "Limit the size of batch to dump");
@@ -136,9 +139,9 @@ static void print_help_messages() {
});
int name_width = static_cast<int>(max_name_length_item->first.size()) + 2;
GPU_DEBUG_COUT << "Supported environment variables for debugging" << std::endl;
GPU_DEBUG_COUT_ << "Supported environment variables for debugging" << std::endl;
for (auto& p : message_list) {
GPU_DEBUG_COUT << " - " << std::left << std::setw(name_width) << p.first + " " << p.second << std::endl;
GPU_DEBUG_COUT_ << " - " << std::left << std::setw(name_width) << p.first + " " << p.second << std::endl;
}
}
@@ -147,6 +150,8 @@ static void print_help_messages() {
debug_configuration::debug_configuration()
: help(0)
, verbose(0)
, verbose_color(0)
, list_layers(0)
, print_multi_kernel_perf(0)
, disable_usm(0)
, disable_onednn(0)
@@ -166,6 +171,8 @@ debug_configuration::debug_configuration()
#ifdef GPU_DEBUG_CONFIG
get_gpu_debug_env_var("Help", help);
get_common_debug_env_var("Verbose", verbose);
get_gpu_debug_env_var("VerboseColor", verbose_color);
get_gpu_debug_env_var("ListLayers", list_layers);
get_gpu_debug_env_var("PrintMultiKernelPerf", print_multi_kernel_perf);
get_gpu_debug_env_var("DisableUsm", disable_usm);
get_gpu_debug_env_var("DumpGraphs", dump_graphs);
@@ -231,7 +238,7 @@ debug_configuration::debug_configuration()
if (after_proc_str.length() > 0) {
#ifdef _WIN32
GPU_DEBUG_COUT << "Warning: OV_GPU_AfterProc is supported only on linux" << std::endl;
GPU_DEBUG_COUT_ << "Warning: OV_GPU_AfterProc is supported only on linux" << std::endl;
#else
after_proc_str = " " + after_proc_str + " "; // Insert delimiter for easier parsing when used
std::stringstream ss(after_proc_str);
@@ -257,16 +264,31 @@ const debug_configuration *debug_configuration::get_instance() {
#endif
}
bool debug_configuration::is_dumped_layer(const std::string& layerName, bool is_output) const {
bool debug_configuration::is_dumped_layer(const std::string& layer_name, bool is_output) const {
#ifdef GPU_DEBUG_CONFIG
if (is_output == true && dump_layers_result == 1 &&
(layerName.find("constant:") == std::string::npos))
(layer_name.find("constant:") == std::string::npos))
return true;
if (dump_layers.empty() && dump_layers_result == 0)
return true;
auto is_match = [](const std::string& layer_name, const std::string& pattern) -> bool {
auto upper_layer_name = std::string(layer_name.length(), '\0');
std::transform(layer_name.begin(), layer_name.end(), upper_layer_name.begin(), ::toupper);
auto upper_pattern = std::string(pattern.length(), '\0');
std::transform(pattern.begin(), pattern.end(), upper_pattern.begin(), ::toupper);
// Check pattern from exec_graph
size_t pos = upper_layer_name.find(':');
auto upper_exec_graph_name = upper_layer_name.substr(pos + 1, upper_layer_name.size());
if (upper_exec_graph_name.compare(upper_pattern) == 0) {
return true;
}
// Check pattern with regular expression
std::regex re(upper_pattern);
return std::regex_match(upper_layer_name, re);
};
auto iter = std::find_if(dump_layers.begin(), dump_layers.end(), [&](const std::string& dl){
return (layerName.compare(dl) == 0);
return is_match(layer_name, dl);
});
return (iter != dump_layers.end());
#else