Extend blobs dumping with filtering by environment variables (#5113)

* Extend blobs dumping with filtering by environment variables

The idea is to dump blogs without rebuilding

We cannot just use environment variables without compile-time flag
because of security flaws.

Instead, it is expected that developers just always set additional
macro (BLOB_DUMP_PATH) which is not set for the production builds:

export CXXFLAGS="-DBLOB_DUMP_PATH=\\\"mkldnn_dump\\\""

This macro activates blob dump filtering using environment variables.

To prevent unnecessary dumping, blobs are not dumped by default even
if macro is defined.
This commit is contained in:
Egor Duplensky authored and GitHub committed 2021-05-04 14:36:28 +03:00
1 parent d716db4acc
commit 2a580e7c26
11 files changed
+396 -144

No files matched your search

+2
View File
@@ -93,6 +93,8 @@ ie_dependent_option(ENABLE_TBB_RELEASE_ONLY "Only Release TBB libraries are link
ie_option (USE_SYSTEM_PUGIXML "use the system copy of pugixml" OFF)
ie_option (ENABLE_CPU_DEBUG_CAPS "enable CPU debug capabilities at runtime" OFF)
#
# Process featues
#
@@ -12,6 +12,10 @@ if (WIN32)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DNOMINMAX")
endif()
if (ENABLE_CPU_DEBUG_CAPS)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DCPU_DEBUG_CAPS")
endif()
file(GLOB_RECURSE SOURCES
${CMAKE_CURRENT_SOURCE_DIR}/*.cpp)
@@ -36,29 +36,18 @@
#include "precision_utils.h"
#include <ie_plugin_config.hpp>
#include "utils/blob_dump.h"
#include "utils/general_utils.h"
#include "utils/debug_capabilities.h"
#include "utils/node_dumper.h"
/*****************************************************
* Debug capability
* - BLOB_DUMP_PATH : Specify with existing folder name
* to dump intermediate blobs into it
* - PRINT_GRAPH_INFO : Define it to enable printing
* additional information to std output.
*
* @todo Align with CPU_DEBUG_CAPS implementation
*****************************************************/
// #define BLOB_DUMP_PATH "mkldnn_dump"
// #define PRINT_GRAPH_INFO
// #define DUMP_AS_TEXT
// #define DUMP_INTERNAL_BLOBS
#ifdef BLOB_DUMP_PATH
# define DUMP_DIR BLOB_DUMP_PATH
# define ENABLE_DUMP(_x) { _x ;}
#else
# define DUMP_DIR ""
# define ENABLE_DUMP(_x)
#endif
using namespace mkldnn;
using namespace MKLDNNPlugin;
@@ -365,30 +354,15 @@ void MKLDNNGraph::InitGraph() {
if (!config.dumpToDot.empty())
dumpToDotFile(config.dumpToDot + "_init.dot");
#ifndef DUMP_INTERNAL_BLOBS
#ifndef CPU_DEBUG_CAPS
for (auto &graphNode : graphNodes) {
graphNode->cleanup();
}
#endif
#if !defined(NDEBUG) && defined(PRINT_GRAPH_INFO)
for (auto &graphNode : graphNodes) {
std::cout << "name: " << graphNode->getName() << " [ ";
if (graphNode->parentEdges.size() > 0) {
auto prnt_out_desc = graphNode->parentEdges[0].lock()->getOutputDesc();
std::cout << "in: " << prnt_out_desc.getPrecision().name()
<< "/l=" << prnt_out_desc.getLayout()
<< "; ";
}
if (graphNode->childEdges.size() > 0) {
auto chld_in_desc = graphNode->childEdges[0].lock()->getInputDesc();
std::cout << "out: " << chld_in_desc.getPrecision().name()
<< "/l=" << chld_in_desc.getLayout();
}
std::cout << " ]" << std::endl;
}
printGraphInfo();
#endif
ExecuteConstantNodesOnly();
}
@@ -853,6 +827,8 @@ void MKLDNNGraph::Infer(MKLDNNInferRequest* request, int batch) {
mkldnn::stream stream(eng);
ENABLE_CPU_DEBUG_CAP(NodeDumper nd(infer_count));
for (int i = 0; i < graphNodes.size(); i++) {
if (request != nullptr) {
request->ThrowIfCanceled();
@@ -863,13 +839,14 @@ void MKLDNNGraph::Infer(MKLDNNInferRequest* request, int batch) {
if (batch > 0)
graphNodes[i]->setDynamicBatchLim(batch);
ENABLE_DUMP(do_before(DUMP_DIR, graphNodes[i]));
ENABLE_CPU_DEBUG_CAP(nd.dumpInputBlobs(graphNodes[i]));
if (!graphNodes[i]->isConstant()) {
OV_ITT_SCOPED_TASK(itt::domains::MKLDNNPlugin, graphNodes[i]->profiling.execute);
graphNodes[i]->execute(stream);
}
ENABLE_DUMP(do_after(DUMP_DIR, graphNodes[i]));
ENABLE_CPU_DEBUG_CAP(nd.dumpOutputBlobs(graphNodes[i]));
}
if (infer_count != -1) infer_count++;
@@ -1197,106 +1174,6 @@ MKLDNNNodePtr MKLDNNGraph::InsertReorder(MKLDNNEdgePtr edge, std::string layerNa
return newReorder;
}
void MKLDNNGraph::dumpToDotFile(std::string file) const {
std::ofstream dot;
dot.open(file);
if (!dot.is_open()) IE_THROW() << "CPU Plugin cannot create dot file " << file << ".";
dump_graph_as_dot(*this, dot);
dot.close();
}
void MKLDNNGraph::do_before(const std::string &dir, const MKLDNNNodePtr &node) {
auto exec_order = std::to_string(node->execIndex);
std::string nodeName = node->name;
std::replace(nodeName.begin(), nodeName.end(), '\\', '_');
std::replace(nodeName.begin(), nodeName.end(), '/', '_');
std::replace(nodeName.begin(), nodeName.end(), ' ', '_');
std::replace(nodeName.begin(), nodeName.end(), ':', '-');
auto num_ports = node->getSelectedPrimitiveDescriptor()->getConfig().inConfs.size();
for (size_t i = 0; i < num_ports; i++) {
auto prEdge = node->getParentEdgeAt(i);
auto pr = prEdge->getParent();
std::string file_name = nodeName;
if (infer_count != -1) file_name += "_iter" + std::to_string(infer_count);
file_name += "_in" + std::to_string(i) + ".ieb";
if (file_name.size() > 240)
file_name = file_name.substr(file_name.size() - 240);
auto dump_file = dir + "/#" + exec_order + "_" + file_name;
TensorDesc desc = prEdge->getDesc();
if (desc.getPrecision() == Precision::BIN)
continue;
BlobDumper dumper(prEdge->getBlob());
if (pr->ext_scales) dumper.withScales(pr->ext_scales);
#ifdef DUMP_AS_TEXT
dumper.dumpAsTxt(dump_file);
#else
dumper.dump(dump_file);
#endif
}
#ifdef DUMP_INTERNAL_BLOBS
for (size_t i = 0; i < node->internalBlobs.size(); i++) {
const auto& blb = node->internalBlobs[i];
auto dump_file = dir + "/#" + exec_order + "_" + nodeName + "_blb" + std::to_string(i) + ".ieb";
TensorDesc desc = blb->getTensorDesc();
if (desc.getPrecision() == Precision::BIN)
continue;
BlobDumper dumper(blb);
#ifdef DUMP_AS_TEXT
dumper.dumpAsTxt(dump_file);
#else
dumper.dump(dump_file);
#endif
}
#endif
}
void MKLDNNGraph::do_after(const std::string &dir, const MKLDNNNodePtr &node) {
auto exec_order = std::to_string(node->execIndex);
auto nodeName = node->name;
std::replace(nodeName.begin(), nodeName.end(), '\\', '_');
std::replace(nodeName.begin(), nodeName.end(), '/', '_');
std::replace(nodeName.begin(), nodeName.end(), ' ', '_');
std::replace(nodeName.begin(), nodeName.end(), ':', '-');
auto num_ports = node->getSelectedPrimitiveDescriptor()->getConfig().outConfs.size();
for (size_t i = 0; i < num_ports; i++) {
auto childEdge = node->getChildEdgeAt(i);
std::string file_name = nodeName;
if (infer_count != -1) file_name += "_iter" + std::to_string(infer_count);
file_name += "_out" + std::to_string(i) + ".ieb";
if (file_name.size() > 240)
file_name = file_name.substr(file_name.size() - 240);
auto dump_file = dir + "/#" + exec_order + "_" + file_name;
std::cout << "try : " << dump_file << std::endl;
TensorDesc desc = childEdge->getDesc();
if (desc.getPrecision() == Precision::BIN)
continue;
BlobDumper dumper(childEdge->getBlob());
if (node->ext_scales) dumper.withScales(node->ext_scales);
#ifdef DUMP_AS_TEXT
dumper.dumpAsTxt(dump_file);
#else
dumper.dump(dump_file);
#endif
}
}
InferenceEngine::CNNNetwork MKLDNNGraph::dump() const {
return dump_graph_as_ie_ngraph_net(*this);
}
bool MKLDNNGraph::InsertNode(MKLDNNEdgePtr edge, MKLDNNNodePtr node, bool initNode) {
auto oIndex = edge->getOutputNum();
auto iIndex = edge->getInputNum();
@@ -1335,3 +1212,35 @@ bool MKLDNNGraph::InsertNode(MKLDNNNodePtr parent, MKLDNNNodePtr child, MKLDNNNo
graphNodes.push_back(node);
return true;
}
InferenceEngine::CNNNetwork MKLDNNGraph::dump() const {
return dump_graph_as_ie_ngraph_net(*this);
}
void MKLDNNGraph::dumpToDotFile(std::string file) const {
std::ofstream dot;
dot.open(file);
if (!dot.is_open()) IE_THROW() << "CPU Plugin cannot create dot file " << file << ".";
dump_graph_as_dot(*this, dot);
dot.close();
}
void MKLDNNGraph::printGraphInfo() const {
for (auto &graphNode : graphNodes) {
std::cout << "name: " << graphNode->getName() << " [ ";
if (graphNode->parentEdges.size() > 0) {
auto prnt_out_desc = graphNode->parentEdges[0].lock()->getOutputDesc();
std::cout << "in: " << prnt_out_desc.getPrecision().name()
<< "/l=" << prnt_out_desc.getLayout()
<< "; ";
}
if (graphNode->childEdges.size() > 0) {
auto chld_in_desc = graphNode->childEdges[0].lock()->getInputDesc();
std::cout << "out: " << chld_in_desc.getPrecision().name()
<< "/l=" << chld_in_desc.getLayout();
}
std::cout << " ]" << std::endl;
}
}
@@ -206,9 +206,6 @@ protected:
void ExecuteConstantNodesOnly();
void SetOriginalLayerNames();
void do_before(const std::string &dir, const MKLDNNNodePtr &node);
void do_after(const std::string &dir, const MKLDNNNodePtr &node);
friend class MKLDNNInferRequest;
friend class MKLDNNGraphlessInferRequest;
friend InferenceEngine::CNNNetwork dump_graph_as_ie_net(const MKLDNNGraph &graph);
@@ -216,6 +213,8 @@ protected:
private:
void dumpToDotFile(std::string file) const;
void printGraphInfo() const;
struct ParsedLayer {
MKLDNNNodePtr parent;
InferenceEngine::CNNLayerPtr cnnLayer;
@@ -592,6 +592,7 @@ protected:
friend class MKLDNNEdge;
friend class MKLDNNGraph;
friend class MKLDNNGraphOptimizer;
friend class NodeDumper;
bool isUninitTensorDesc(const InferenceEngine::TensorDesc& desc) const;
bool isInitConfig(const InferenceEngine::LayerConfig& config) const;
@@ -0,0 +1,73 @@
# Debug capabilities
Use the following cmake option to enable debug capabilities:
`-DENABLE_CPU_DEBUG_CAPS=ON`
## Blob dumping
Blob dumping is controlled by environment variables (filters).
The variables define conditions of the node which input, output and internal blobs
should be dumped for.
> **NOTE**: Nothing is dumped by default
> **NOTE**: All specified filters should be matched in order blobs to be dumped
Environment variables can be set per execution, for example:
```sh
OV_CPU_BLOB_DUMP_DIR=dump_dir binary ...
```
or for shell session (bash example):
```sh
export OV_CPU_BLOB_DUMP_DIR=dump_dir
binary ...
```
### Specify dump directory
```sh
OV_CPU_BLOB_DUMP_DIR=<directory-name> binary ...
```
Default is *mkldnn_dump*
### Specify dump format
```sh
OV_CPU_BLOB_DUMP_FORMAT=<format> binary ...
```
Options are:
* BIN (default)
* TEXT
### Filter by execution ID
To dump blobs only for node with specified execution IDs:
```sh
OV_CPU_BLOB_DUMP_NODE_EXEC_ID='<space_separated_list_of_ids>' binary ...
```
Example:
```sh
OV_CPU_BLOB_DUMP_NODE_EXEC_ID='1 12 45' binary ...
```
### Filter by type
To dump blobs only for node with specified type:
```sh
OV_CPU_BLOB_DUMP_NODE_TYPE=<type> binary ...
```
Example:
```sh
OV_CPU_BLOB_DUMP_NODE_TYPE=Convolution binary ...
```
> **NOTE**: see **enum Type** in [mkldnn_node.h](../mkldnn_node.h) for list of the types
### Filter by name
To dump blobs only for node with name matching specified regex:
```sh
OV_CPU_BLOB_DUMP_NODE_NAME=<regex> binary ...
```
Example:
```sh
OV_CPU_BLOB_DUMP_NODE_NAME=".+Fused_Add.+" binary ...
```
### Dump all the blobs
```sh
OV_CPU_BLOB_DUMP_NODE_NAME=".+" binary ...
```
@@ -143,7 +143,7 @@ static Blob::Ptr prepare_plain_data(Blob::Ptr blob) {
return pln_blob;
}
void BlobDumper::dump(std::ostream &stream) {
void BlobDumper::dump(std::ostream &stream) const {
if (!_blob)
IE_THROW() << "Dumper cannot dump empty Blob";
@@ -172,7 +172,7 @@ void BlobDumper::dump(std::ostream &stream) {
}
}
void BlobDumper::dumpAsTxt(std::ostream &stream) {
void BlobDumper::dumpAsTxt(std::ostream &stream) const {
if (!_blob)
IE_THROW() << "Dumper cannot dump empty Blob";
@@ -289,7 +289,7 @@ BlobDumper BlobDumper::read(const std::string &file_path) {
return res;
}
void BlobDumper::dump(const std::string &dump_path) {
void BlobDumper::dump(const std::string &dump_path) const {
std::ofstream dump_file;
dump_file.open(dump_path);
if (!dump_file.is_open())
@@ -299,7 +299,7 @@ void BlobDumper::dump(const std::string &dump_path) {
dump_file.close();
}
void BlobDumper::dumpAsTxt(const std::string dump_path) {
void BlobDumper::dumpAsTxt(const std::string& dump_path) const {
std::ofstream dump_file;
dump_file.open(dump_path);
if (!dump_file.is_open())
@@ -32,11 +32,11 @@ public:
static BlobDumper read(const std::string &file_path);
static BlobDumper read(std::istream &stream);
void dump(const std::string &file_path);
void dump(std::ostream &stream);
void dump(const std::string &file_path) const;
void dump(std::ostream &stream) const;
void dumpAsTxt(const std::string file_path);
void dumpAsTxt(std::ostream &stream);
void dumpAsTxt(const std::string &file_path) const;
void dumpAsTxt(std::ostream &stream) const;
BlobDumper& withScales(InferenceEngine::Blob::Ptr scales);
BlobDumper& withoutScales();
@@ -0,0 +1,10 @@
// Copyright (C) 2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#ifdef CPU_DEBUG_CAPS
# define ENABLE_CPU_DEBUG_CAP(_x) _x;
#else
# define ENABLE_CPU_DEBUG_CAP(_x)
#endif
@@ -0,0 +1,195 @@
// Copyright (C) 2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#ifdef CPU_DEBUG_CAPS
#include "node_dumper.h"
#include "mkldnn_node.h"
#include "utils/blob_dump.h"
#include "ie_common.h"
#include <array>
#include <regex>
#include <sstream>
#include <string>
using namespace InferenceEngine;
namespace MKLDNNPlugin {
NodeDumper::NodeDumper(int _count):
count(_count), dumpFormat(DUMP_FORMAT::BIN) {
const char* dumpDirEnv = std::getenv("OV_CPU_BLOB_DUMP_DIR");
if (dumpDirEnv)
dumpDirName = dumpDirEnv;
const char* dumpFormatEnv = std::getenv("OV_CPU_BLOB_DUMP_FORMAT");
if (dumpFormatEnv)
dumpFormat = parseDumpFormat(dumpFormatEnv);
const char* filter = std::getenv("OV_CPU_BLOB_DUMP_NODE_EXEC_ID");
if (filter)
dumpFilters[FILTER::BY_EXEC_ID] = filter;
filter = std::getenv("OV_CPU_BLOB_DUMP_NODE_TYPE");
if (filter)
dumpFilters[FILTER::BY_TYPE] = filter;
filter = std::getenv("OV_CPU_BLOB_DUMP_NODE_NAME");
if (filter)
dumpFilters[FILTER::BY_NAME] = filter;
}
void NodeDumper::dumpInputBlobs(const MKLDNNNodePtr& node) const {
if (!shouldBeDumped(node))
return;
auto exec_order = std::to_string(node->getExecIndex());
std::string nodeName = node->getName();
formatNodeName(nodeName);
auto num_ports = node->getSelectedPrimitiveDescriptor()->getConfig().inConfs.size();
for (size_t i = 0; i < num_ports; i++) {
auto prEdge = node->getParentEdgeAt(i);
auto pr = prEdge->getParent();
std::string file_name = NameFromType(node->getType()) + "_" + nodeName;
if (count != -1)
file_name += "_iter" + std::to_string(count);
file_name += "_in" + std::to_string(i) + ".ieb";
if (file_name.size() > 240)
file_name = file_name.substr(file_name.size() - 240);
auto dump_file = dumpDirName + "/#" + exec_order + "_" + file_name;
std::cout << "Dump before: " << dump_file << std::endl;
TensorDesc desc = prEdge->getDesc();
if (desc.getPrecision() == Precision::BIN)
continue;
BlobDumper dumper(prEdge->getBlob());
if (pr->ext_scales)
dumper.withScales(pr->ext_scales);
dump(dumper, dump_file);
}
dumpInternalBlobs(node);
}
void NodeDumper::dumpOutputBlobs(const MKLDNNNodePtr& node) const {
if (!shouldBeDumped(node))
return;
auto exec_order = std::to_string(node->getExecIndex());
std::string nodeName = node->getName();
formatNodeName(nodeName);
auto num_ports = node->getSelectedPrimitiveDescriptor()->getConfig().outConfs.size();
for (size_t i = 0; i < num_ports; i++) {
auto childEdge = node->getChildEdgeAt(i);
std::string file_name = NameFromType(node->getType()) + "_" + nodeName;
if (count != -1)
file_name += "_iter" + std::to_string(count);
file_name += "_out" + std::to_string(i) + ".ieb";
if (file_name.size() > 240)
file_name = file_name.substr(file_name.size() - 240);
auto dump_file = dumpDirName + "/#" + exec_order + "_" + file_name;
std::cout << "Dump after: " << dump_file << std::endl;
TensorDesc desc = childEdge->getDesc();
if (desc.getPrecision() == Precision::BIN)
continue;
BlobDumper dumper(childEdge->getBlob());
if (node->ext_scales)
dumper.withScales(node->ext_scales);
dump(dumper, dump_file);
}
}
void NodeDumper::dumpInternalBlobs(const MKLDNNNodePtr& node) const {
std::string nodeName = node->getName();
formatNodeName(nodeName);
for (size_t i = 0; i < node->internalBlobs.size(); i++) {
const auto& blb = node->internalBlobs[i];
std::string file_name = NameFromType(node->getType()) + "_" + nodeName + "_blb" + std::to_string(i) + ".ieb";
auto dump_file = dumpDirName + "/#" + std::to_string(node->getExecIndex()) + "_" + file_name;
TensorDesc desc = blb->getTensorDesc();
if (desc.getPrecision() == Precision::BIN)
continue;
BlobDumper dumper(blb);
dump(dumper, dump_file);
}
}
void NodeDumper::dump(const BlobDumper& bd, const std::string& file) const {
switch (dumpFormat) {
case DUMP_FORMAT::BIN: {
bd.dump(file);
break;
}
case DUMP_FORMAT::TEXT: {
bd.dumpAsTxt(file);
break;
}
default:
IE_THROW() << "Unknown dump format";
}
}
bool NodeDumper::shouldBeDumped(const MKLDNNNodePtr& node) const {
if (dumpFilters.empty())
return false;
if (dumpFilters.count(FILTER::BY_EXEC_ID)) { // filter by exec id env set
std::stringstream ss(dumpFilters.at(FILTER::BY_EXEC_ID));
int id;
bool matched = false;
while (ss >> id) {
if (node->getExecIndex() == id) // exec id matches
matched = true;
}
if (!matched)
return false;
}
if (dumpFilters.count(FILTER::BY_TYPE)) { // filter by type env set
if (NameFromType(node->getType()) != dumpFilters.at(FILTER::BY_TYPE)) // type does not match
return false;
}
if (dumpFilters.count(FILTER::BY_NAME)) { // filter by name env set
if (!std::regex_match(node->getName(), std::regex(dumpFilters.at(FILTER::BY_NAME)))) // name does not match
return false;
}
return true;
}
NodeDumper::DUMP_FORMAT NodeDumper::parseDumpFormat(const std::string& format) const {
if (format == "BIN")
return DUMP_FORMAT::BIN;
else if (format == "TEXT")
return DUMP_FORMAT::TEXT;
else
IE_THROW() << "Unknown dump format";
}
void NodeDumper::formatNodeName(std::string& name) const {
std::replace(name.begin(), name.end(), '\\', '_');
std::replace(name.begin(), name.end(), '/', '_');
std::replace(name.begin(), name.end(), ' ', '_');
std::replace(name.begin(), name.end(), ':', '-');
}
} // namespace MKLDNNPlugin
#endif // CPU_DEBUG_CAPS
@@ -0,0 +1,59 @@
// Copyright (C) 2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#ifdef CPU_DEBUG_CAPS
#pragma once
#include "mkldnn_node.h"
#include "utils/blob_dump.h"
#include <unordered_map>
#include <string>
namespace MKLDNNPlugin {
/**
* Blobs are not dumped by default
* Blobs are dumped if node matches all specified env filters
*
* To dump blobs from all the nodes use the following filter:
*
* OV_CPU_BLOB_DUMP_NODE_NAME=.+
*/
class NodeDumper {
public:
NodeDumper(int _count);
void dumpInputBlobs(const MKLDNNNodePtr &node) const;
void dumpOutputBlobs(const MKLDNNNodePtr &node) const;
private:
void dumpInternalBlobs(const MKLDNNNodePtr& node) const;
void dump(const BlobDumper& bd, const std::string& file) const;
bool shouldBeDumped(const MKLDNNNodePtr &node) const;
enum class DUMP_FORMAT {
BIN,
TEXT,
};
DUMP_FORMAT parseDumpFormat(const std::string& format) const;
void formatNodeName(std::string& name) const;
DUMP_FORMAT dumpFormat;
int count;
std::string dumpDirName = "mkldnn_dump";
enum FILTER {
BY_EXEC_ID,
BY_TYPE,
BY_NAME,
COUNT,
};
std::unordered_map<FILTER, std::string> dumpFilters;
};
} // namespace MKLDNNPlugin
#endif // CPU_DEBUG_CAPS