IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Remove Provenance tags (#8101)

Browse Source
This commit is contained in:
Gleb Kazantaev 2021-10-20 10:28:58 +03:00 committed by GitHub
parent 4f203b9a2e
commit 88f56669e2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 16 additions and 875 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ngraph/core/builder/include/ngraph/builder/make_constant.hpp
View File

@ -108,7 +108,7 @@ std::shared_ptr<Node> make_constant(const element::Type& type, const Shape& shap
val = builder::opset1::make_broadcast(val, shape, axes).get_node_shared_ptr();
}
return val->add_provenance_group_members_above({});
return val;
}
/// \brief Create constant filled with double value

9
ngraph/core/builder/src/builder/norm.cpp
View File

@ -45,7 +45,7 @@ shared_ptr<Node> lp_norm(const Output<Node>& value,
// Get outer part of equation: raise values to 1/p_norm exponent.
shared_ptr<Node> inv_p_node = ngraph::opset1::Constant::create(values->get_element_type(), Shape{}, {1.f / p_norm});
return {make_shared<ngraph::opset1::Power>(values, inv_p_node)->add_provenance_group_members_above({value})};
return {make_shared<ngraph::opset1::Power>(values, inv_p_node)};
}
} // namespace opset1
} // namespace detail
@ -61,8 +61,7 @@ shared_ptr<Node> builder::opset1::l0_norm(const Output<Node>& value,
make_shared<ngraph::opset1::Convert>(make_shared<ngraph::opset1::NotEqual>(value, zero_node),
value.get_element_type());
return make_shared<ngraph::opset1::ReduceSum>(non_zero_values, reduction_axes, keep_dims)
->add_provenance_group_members_above({value});
return make_shared<ngraph::opset1::ReduceSum>(non_zero_values, reduction_axes, keep_dims);
}
shared_ptr<Node> builder::opset1::l1_norm(const Output<Node>& value,
@ -74,7 +73,7 @@ shared_ptr<Node> builder::opset1::l1_norm(const Output<Node>& value,
const shared_ptr<Node> bias_node{ngraph::opset1::Constant::create(values->get_element_type(), Shape{}, {bias})};
return make_shared<ngraph::opset1::Add>(values, bias_node)->add_provenance_group_members_above({value});
return make_shared<ngraph::opset1::Add>(values, bias_node);
}
shared_ptr<Node> builder::opset1::l2_norm(const Output<Node>& value,
@ -98,7 +97,7 @@ shared_ptr<Node> builder::opset1::l2_norm(const Output<Node>& value,
default:
result = make_shared<ngraph::opset1::Sqrt>(make_shared<ngraph::opset1::Add>(values, bias_node));
}
return result->add_provenance_group_members_above({value});
return result;
}
shared_ptr<Node> builder::opset1::lp_norm(const Output<Node>& value,

10
ngraph/core/builder/src/builder/reduce_ops.cpp
View File

@ -52,8 +52,7 @@ std::shared_ptr<Node> builder::opset1::mean(const Output<Node>& value, const Axi
elems_number = std::make_shared<ngraph::opset1::Convert>(elems_number, value_elem_type);
}
return std::make_shared<ngraph::opset1::Divide>(value_elems_sum, elems_number)
->add_provenance_group_members_above({value});
return std::make_shared<ngraph::opset1::Divide>(value_elems_sum, elems_number);
}
std::shared_ptr<Node> builder::opset1::mean(const Output<Node>& value,
@ -65,8 +64,7 @@ std::shared_ptr<Node> builder::opset1::mean(const Output<Node>& value,
elems_number = get_num_elements(value, reduction_axes);
elems_number = std::make_shared<ngraph::opset1::Convert>(elems_number, value_elem_type);
return std::make_shared<ngraph::opset1::Divide>(value_elems_sum, elems_number)
->add_provenance_group_members_above({value});
return std::make_shared<ngraph::opset1::Divide>(value_elems_sum, elems_number);
}
std::shared_ptr<Node> builder::opset1::variance(const Output<Node>& value,
@ -93,7 +91,7 @@ std::shared_ptr<Node> builder::opset1::variance(const Output<Node>& value,
const auto Nconst = ngraph::opset1::Constant::create(et, Shape{}, {N});
result = std::make_shared<ngraph::opset1::Divide>(diff, Nconst);
}
return result->add_provenance_group_members_above({value});
return result;
}
std::shared_ptr<Node> builder::opset1::variance(const Output<Node>& value,
@ -119,7 +117,7 @@ std::shared_ptr<Node> builder::opset1::variance(const Output<Node>& value,
}
result = std::make_shared<ngraph::opset1::Divide>(diff, N);
return result->add_provenance_group_members_above({value});
return result;
}
} // namespace builder

10
ngraph/core/builder/src/builder/reshape.cpp
View File

@ -38,8 +38,7 @@ shared_ptr<Node> builder::opset1::reshape(const Output<Node>& value, const Shape
auto out_pattern =
op::Constant::create(element::i64, Shape{shape.size()}, vector<int64_t>(shape.begin(), shape.end()));
return make_shared<ngraph::opset1::Reshape>(value, out_pattern, false)
->add_provenance_group_members_above({value});
return make_shared<ngraph::opset1::Reshape>(value, out_pattern, false);
}
}
@ -47,7 +46,7 @@ shared_ptr<Node> builder::opset1::reorder_axes(const Output<Node>& value, vector
const auto axes_order_const = op::Constant::create(element::i64,
Shape{axes_order.size()},
vector<int64_t>(axes_order.begin(), axes_order.end()));
return make_shared<ngraph::opset1::Transpose>(value, axes_order_const)->add_provenance_group_members_above({value});
return make_shared<ngraph::opset1::Transpose>(value, axes_order_const);
}
shared_ptr<Node> builder::opset1::transpose(const Output<Node>& value) {
@ -65,8 +64,7 @@ shared_ptr<Node> builder::opset1::transpose(const Output<Node>& value) {
const auto reverse_axes_order = std::make_shared<ngraph::opset1::Range>(reshape(start_node, Shape{}), // start
neg_one, // stop (exclusive)
neg_one); // step
return std::make_shared<ngraph::opset1::Transpose>(value, reverse_axes_order)
->add_provenance_group_members_above({value});
return std::make_shared<ngraph::opset1::Transpose>(value, reverse_axes_order);
}
namespace ngraph {
@ -128,7 +126,7 @@ shared_ptr<Node> builder::opset1::flatten(const Output<Node>& value, int axis) {
output_shape =
make_shared<ngraph::opset1::Concat>(OutputVector{first_part_dims_length, remaining_part_length}, 0);
}
return make_shared<ngraph::opset1::Reshape>(value, output_shape, true)->add_provenance_group_members_above({value});
return make_shared<ngraph::opset1::Reshape>(value, output_shape, true);
}
shared_ptr<Node> builder::opset1::expand_dims(const Output<Node>& value, size_t axis) {

17
ngraph/core/include/ngraph/provenance.hpp
View File

@ -1,17 +0,0 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include <cstdlib>
#include "ngraph/env_util.hpp"
#include "ngraph/ngraph_visibility.hpp"
namespace ngraph {
NGRAPH_API
void set_provenance_enabled(bool enabled);
NGRAPH_API
bool get_provenance_enabled();
} // namespace ngraph

1
ngraph/core/include/openvino/core/graph_util.hpp
View File

@ -186,7 +186,6 @@ void replace_node(const std::shared_ptr<Node>& target,
const std::vector<int64_t>& output_order);
/// Replace target.outputs[i] with replacement_values[i] and transfer control dependents and
/// provenance from target to the node(s) in replacement_values.
OPENVINO_API
void replace_node(const std::shared_ptr<Node>& target, const OutputVector& replacement_values);

32
ngraph/core/include/openvino/core/node.hpp
View File

@ -408,36 +408,6 @@ public:
const RTMap& get_rt_info() const {
return m_rt_info;
}
const std::unordered_set<std::string>& get_provenance_tags() const;
void add_provenance_tag(const std::string& tag);
template <typename T>
void add_provenance_tags(T tag_set) {
for (auto tag : tag_set) {
add_provenance_tag(tag);
}
}
/// \brief Adds tag_set to this node and all intermediate nodes above base
void add_provenance_tags_above(const OutputVector& base, const std::unordered_set<std::string>& tag_set);
void remove_provenance_tag(const std::string& tag);
/// \brief Add node to additional nodes that receive tags
void add_provenance_group_member(const std::shared_ptr<Node>& node);
/// \brief Remove node to additional nodes that receive tags
void remove_provenance_group_member(const std::shared_ptr<Node>& node);
/// \brief Replace current_node with replacement_node and transfer tags
void replace_provenance_group_member(const std::shared_ptr<Node>& current_node,
const std::shared_ptr<Node>& replacement_node);
/// \return Provenance group nodes
const std::set<std::shared_ptr<Node>>& get_provenance_group_members() const;
/// \brief Add all nodes between this node and nodes in base as additional nodes to receive
/// provenance tags.
std::shared_ptr<Node> add_provenance_group_members_above(const OutputVector& base);
// to be used when nodes are replaced
void merge_provenance_tags_from(const std::shared_ptr<const Node>& source);
/// Transfer provenance tags to replacement
void transfer_provenance_tags(const std::shared_ptr<Node>& replacement);
/// Get all the nodes that uses the current node
NodeVector get_users(bool check_is_used = false) const;
@ -516,8 +486,6 @@ private:
mutable std::string m_unique_name;
mutable std::atomic_bool m_name_changing{false};
static std::atomic<size_t> m_next_instance_id;
std::unordered_set<std::string> m_provenance_tags;
std::set<std::shared_ptr<Node>> m_provenance_group;
std::deque<descriptor::Input> m_inputs;
std::deque<descriptor::Output> m_outputs;
OPENVINO_SUPPRESS_DEPRECATED_START

30
ngraph/core/src/graph_util.cpp
View File

@ -25,7 +25,6 @@
#include "ngraph/opsets/opset8.hpp"
#include "ngraph/pass/manager.hpp"
#include "ngraph/pass/visualize_tree.hpp"
#include "ngraph/provenance.hpp"
#include "ngraph/rt_info.hpp"
#include "ngraph/util.hpp"
@ -127,27 +126,6 @@ void ov::replace_node(const std::shared_ptr<Node>& target,
// Fix input/output descriptors
NGRAPH_CHECK(target->get_output_size() == replacement->get_output_size());
if (ngraph::get_provenance_enabled()) {
auto common_args = ngraph::find_common_args(target, replacement);
std::set<string> removed_subgraph_tags;
auto set_replacement_prov = [&removed_subgraph_tags](const std::shared_ptr<Node>& node) {
for (const auto& tag : node->get_provenance_tags()) {
removed_subgraph_tags.insert(tag);
}
};
traverse_nodes({target}, set_replacement_prov, common_args);
replacement->add_provenance_tags(removed_subgraph_tags);
auto set_prov_new_nodes = [&removed_subgraph_tags](const std::shared_ptr<Node>& node) {
node->add_provenance_tags(removed_subgraph_tags);
};
traverse_nodes({replacement}, set_prov_new_nodes, common_args);
}
// For each of target's output O with replacement output O_rep:
// For each O's connected downstream input I:
// Change I's connected upstream output to O_rep
@ -177,7 +155,6 @@ void ov::replace_node(const std::shared_ptr<Node>& target, const OutputVector& r
if (replacement_nodes.find(replacement_node) == replacement_nodes.end()) {
replacement_node->add_node_control_dependents(target);
replacement_node->add_node_control_dependencies(target);
target->transfer_provenance_tags(replacement_node);
replacement_nodes.insert(replacement_node);
}
target->output(i).replace(replacement_values.at(i));
@ -269,9 +246,6 @@ std::vector<std::shared_ptr<ngraph::Node>> ngraph::clone_nodes(const std::vector
new_output.get_rt_info() = output_rt_info;
}
for (const auto& tag : node->get_provenance_tags()) {
cloned_node->add_provenance_tag(tag);
}
cloned_node->set_op_annotations(node->get_op_annotations());
node_map[node.get()] = cloned_node;
@ -318,10 +292,6 @@ std::list<std::shared_ptr<ngraph::Node>> ngraph::clone_nodes(const std::vector<s
cloned_node->set_friendly_name(node->get_friendly_name());
auto rt_info = node->get_rt_info();
cloned_node->get_rt_info() = rt_info;
for (const auto& tag : node->get_provenance_tags()) {
cloned_node->add_provenance_tag(tag);
}
cloned_node->set_op_annotations(node->get_op_annotations());
for (const auto& cloned_value : cloned_node->outputs()) {
auto original_value = node_outputs.at(cloned_value.get_index());

123
ngraph/core/src/node.cpp
View File

@ -32,8 +32,6 @@ ov::Node::Node(const Node& node)
m_friendly_name(node.m_friendly_name)
// skip m_unique_name -- will be generated automatically
,
m_provenance_tags(node.m_provenance_tags),
m_provenance_group(node.m_provenance_group),
m_inputs(node.m_inputs) // will be modified in the body
// skip m_outputs -- should be initialized outside
,
@ -51,8 +49,6 @@ ov::Node& ov::Node::operator=(const Node& node) {
this->m_control_dependencies = node.m_control_dependencies;
this->m_instance_id = m_next_instance_id.fetch_add(1);
this->m_friendly_name = node.m_friendly_name;
this->m_provenance_tags = node.m_provenance_tags;
this->m_provenance_group = node.m_provenance_group;
this->m_inputs = node.m_inputs;
this->m_op_annotations = node.m_op_annotations;
this->m_rt_info = node.m_rt_info;
@ -251,125 +247,6 @@ void ov::Node::set_friendly_name(const string& name) {
m_friendly_name = name;
}
void ov::Node::add_provenance_group_member(const shared_ptr<Node>& node) {
m_provenance_group.insert(node);
}
void ov::Node::remove_provenance_group_member(const shared_ptr<Node>& node) {
m_provenance_group.erase(node);
}
void ov::Node::replace_provenance_group_member(const shared_ptr<Node>& current_node,
const shared_ptr<Node>& replacement_node) {
// Catch up with the current state of the group
replacement_node->add_provenance_tags(get_provenance_tags());
if (current_node != nullptr) {
remove_provenance_group_member(current_node);
// Catch up with what was added to the current node
replacement_node->add_provenance_tags(current_node->get_provenance_tags());
}
add_provenance_group_member(replacement_node);
}
const set<shared_ptr<ov::Node>>& ov::Node::get_provenance_group_members() const {
return m_provenance_group;
}
shared_ptr<ov::Node> ov::Node::add_provenance_group_members_above(const OutputVector& base) {
set<Node*> base_set;
for (auto& output : base) {
Node* node = output.get_node();
if (node == this) {
// A builder did nothing
return shared_from_this();
}
base_set.insert(node);
}
vector<Node*> todo;
for (const auto& value : input_values()) {
todo.push_back(value.get_node());
}
while (!todo.empty()) {
Node* node = todo.back();
todo.pop_back();
if (base_set.count(node) > 0) {
continue;
}
add_provenance_group_member(node->shared_from_this());
for (const auto& value : node->input_values()) {
if (m_provenance_group.count(value.get_node_shared_ptr()) == 0) {
todo.push_back(value.get_node());
}
}
base_set.insert(node);
}
return shared_from_this();
}
void ov::Node::add_provenance_tags_above(const OutputVector& base, const std::unordered_set<std::string>& tag_set) {
set<Node*> base_set;
for (auto& output : base) {
base_set.insert(output.get_node());
}
vector<Node*> todo{this};
while (!todo.empty()) {
Node* node = todo.back();
todo.pop_back();
if (base_set.count(node) > 0) {
continue;
}
node->add_provenance_tags(tag_set);
for (const auto& value : node->input_values()) {
todo.push_back(value.get_node());
}
base_set.insert(node);
}
}
const std::unordered_set<std::string>& ov::Node::get_provenance_tags() const {
return m_provenance_tags;
}
void ov::Node::add_provenance_tag(const std::string& tag) {
m_provenance_tags.insert(tag);
for (const auto& node : m_provenance_group) {
node->add_provenance_tag(tag);
}
}
void ov::Node::remove_provenance_tag(const std::string& tag) {
m_provenance_tags.erase(tag);
}
void ov::Node::merge_provenance_tags_from(const std::shared_ptr<const Node>& source) {
for (auto& tag : source->get_provenance_tags()) {
add_provenance_tag(tag);
}
}
void ov::Node::transfer_provenance_tags(const shared_ptr<Node>& replacement) {
NGRAPH_SUPPRESS_DEPRECATED_START
auto common_args = ngraph::find_common_args(shared_from_this(), replacement);
std::set<string> removed_subgraph_tags;
auto set_replacement_prov = [&removed_subgraph_tags](const std::shared_ptr<Node>& node) {
for (const auto& tag : node->get_provenance_tags()) {
removed_subgraph_tags.insert(tag);
}
};
ngraph::traverse_nodes({shared_from_this()}, set_replacement_prov, common_args);
replacement->add_provenance_tags(removed_subgraph_tags);
auto set_prov_new_nodes = [&removed_subgraph_tags](const std::shared_ptr<Node>& node) {
node->add_provenance_tags(removed_subgraph_tags);
};
ngraph::traverse_nodes({replacement}, set_prov_new_nodes, common_args);
NGRAPH_SUPPRESS_DEPRECATED_END
}
ov::Node* ov::Node::get_input_node_ptr(size_t index) const {
NGRAPH_CHECK(index < m_inputs.size(), "index '", index, "' out of range in get_argument(size_t index)");
return m_inputs[index].get_output().get_node().get();

4
ngraph/core/src/op/lrn.cpp
View File

@ -17,9 +17,7 @@ using namespace ngraph;
BWDCMP_RTTI_DEFINITION(op::v0::LRN);
op::LRN::LRN(const Output<Node>& arg, double alpha, double beta, double bias, size_t size)
: LRN(arg, op::v0::Constant::create(element::i64, ov::Shape{1}, {1}), alpha, beta, bias, size) {
add_provenance_group_member(input_value(1).get_node_shared_ptr());
}
: LRN(arg, op::v0::Constant::create(element::i64, ov::Shape{1}, {1}), alpha, beta, bias, size) {}
op::LRN::LRN(const Output<Node>& arg, const Output<Node>& axes, double alpha, double beta, double bias, size_t size)
: Op({arg, axes}),

4
ngraph/core/src/op/util/logical_reduction.cpp
View File

@ -20,9 +20,7 @@ op::util::LogicalReduction::LogicalReduction(const Output<Node>& arg, const Axis
: ReductionBase(
arg,
ngraph::op::Constant::create(element::i64, ov::Shape{reduction_axes.size()}, reduction_axes.to_vector())
->output(0)) {
add_provenance_group_member(input_value(1).get_node_shared_ptr());
}
->output(0)) {}
op::util::LogicalReduction::LogicalReduction(const Output<Node>& arg, const Output<Node>& reduction_axes)
: ReductionBase(arg, reduction_axes) {}

18
ngraph/core/src/provenance.cpp
View File

@ -1,18 +0,0 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "ngraph/provenance.hpp"
#include <cstdlib>
namespace ngraph {
static bool s_provenance_enabled = getenv_bool("NGRAPH_PROVENANCE_ENABLE");
void set_provenance_enabled(bool enabled) {
s_provenance_enabled = enabled;
}
bool get_provenance_enabled() {
return s_provenance_enabled;
}
} // namespace ngraph

1
ngraph/environment_variables.md
View File

@ -9,7 +9,6 @@
| NGRAPH_GRAPH_REWRITE_RERUN_DYNAMIC_CHECK | |
| NGRAPH_GTEST_INFO | |
| NGRAPH_PROFILE_PASS_ENABLE | |
| NGRAPH_PROVENANCE_ENABLE | |
| NGRAPH_VISUALIZE_EDGE_JUMP_DISTANCE | |
| NGRAPH_VISUALIZE_EDGE_LABELS | |
| NGRAPH_VISUALIZE_TRACING_FORMAT | |

41
ngraph/frontend/onnx/frontend/src/core/graph.cpp
View File

@ -14,12 +14,10 @@
#include "exceptions.hpp"
#include "ngraph/log.hpp"
#include "ngraph/node.hpp"
#include "ngraph/provenance.hpp"
#include "onnx_framework_node.hpp"
#include "onnx_import/core/node.hpp"
#include "onnx_import/core/null_node.hpp"
#include "utils/common.hpp"
#include "utils/provenance_tag.hpp"
namespace ngraph {
namespace onnx_import {
@ -47,42 +45,6 @@ static std::string get_op_domain_and_name(const ONNX_NAMESPACE::NodeProto& node_
std::string domain = get_node_domain(node_proto);
return (domain.empty() ? "" : domain + ".") + node_proto.op_type();
}
void add_provenance_tag_to_initializer(const Tensor& tensor, std::shared_ptr<default_opset::Constant> node) {
if (!ngraph::get_provenance_enabled()) {
return;
}
const std::string tag = detail::build_input_provenance_tag(tensor.get_name(), tensor.get_shape());
node->add_provenance_tag(tag);
}
void add_provenance_tag_to_input(const ValueInfo& input, std::shared_ptr<ngraph::Node> node) {
if (!ngraph::get_provenance_enabled()) {
return;
}
const std::string tag = detail::build_input_provenance_tag(input.get_name(), input.get_shape());
node->add_provenance_tag(tag);
}
void add_provenance_tags(const Node& onnx_node, const OutputVector& ng_node_vector) {
if (!ngraph::get_provenance_enabled()) {
return;
}
const auto tag = detail::build_op_provenance_tag(onnx_node);
const auto ng_inputs = onnx_node.get_ng_inputs();
ngraph::traverse_nodes(
as_node_vector(ng_node_vector),
[&tag](std::shared_ptr<ngraph::Node> ng_node) {
ng_node->add_provenance_tag(tag);
},
as_node_vector(ng_inputs));
}
} // namespace detail
Graph::Graph(std::shared_ptr<ONNX_NAMESPACE::ModelProto> model_proto)
@ -116,7 +78,6 @@ Graph::Graph(std::shared_ptr<ONNX_NAMESPACE::ModelProto> model_proto, std::uniqu
}
initializers.emplace(initializer_tensor.name(), tensor);
detail::add_provenance_tag_to_initializer(tensor, ng_constant);
m_cache->emplace_node(initializer_tensor.name(), std::move(ng_constant));
}
}
@ -130,7 +91,6 @@ Graph::Graph(std::shared_ptr<ONNX_NAMESPACE::ModelProto> model_proto, std::uniqu
ValueInfo value_info{input};
auto ng_node = value_info.get_ng_node(m_parameters, initializers);
detail::add_provenance_tag_to_input(value_info, ng_node);
m_cache->emplace_node(input.name(), std::move(ng_node));
}
@ -290,7 +250,6 @@ OutputVector Graph::make_ng_nodes(const Node& onnx_node) const {
std::rethrow_exception(std::current_exception());
}
set_friendly_names(onnx_node, ng_node_vector);
detail::add_provenance_tags(onnx_node, ng_node_vector);
for (std::size_t i{0}; i < onnx_node.get_outputs_size(); ++i) {
auto ng_node = ng_node_vector.at(i);

3
ngraph/frontend/onnx/frontend/src/op/conv_transpose.cpp
View File

@ -124,8 +124,7 @@ Output<ngraph::Node> get_prepared_bias(const Output<ngraph::Node>& bias, const O
std::make_shared<default_opset::Concat>(OutputVector{one_node, C_dim, remaining_bias_shape_ones}, 0);
}
return std::make_shared<default_opset::Reshape>(bias, bias_shape_node, false)
->add_provenance_group_members_above({bias});
return std::make_shared<default_opset::Reshape>(bias, bias_shape_node, false);
}
} // namespace

38
ngraph/frontend/onnx/frontend/src/utils/provenance_tag.cpp
View File

@ -1,38 +0,0 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "utils/provenance_tag.hpp"
#include <functional>
#include <numeric>
#include <sstream>
namespace ngraph {
namespace onnx_import {
namespace detail {
std::string concat_strings(const std::vector<std::reference_wrapper<const std::string>>& strings) {
const auto concat_with_comma = [](const std::string& accumulator,
std::reference_wrapper<const std::string> next_string) {
return accumulator + ", " + next_string.get();
};
return std::accumulate(strings.begin() + 1, strings.end(), strings.begin()->get(), concat_with_comma);
}
std::string build_input_provenance_tag(const std::string& input_name, const PartialShape& shape) {
std::stringstream tag_builder;
tag_builder << "<ONNX Input (" << input_name << ") Shape:" << shape << ">";
return tag_builder.str();
}
std::string build_op_provenance_tag(const Node& onnx_node) {
const auto output_names = concat_strings(onnx_node.get_output_names());
const auto node_name = onnx_node.get_name().empty() ? "" : onnx_node.get_name() + " ";
return std::string{"<ONNX " + onnx_node.op_type() + " (" + node_name + "-> " + output_names + ")>"};
}
} // namespace detail
} // namespace onnx_import
} // namespace ngraph

24
ngraph/frontend/onnx/frontend/src/utils/provenance_tag.hpp
View File

@ -1,24 +0,0 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include <string>
#include <vector>
#include "ngraph/partial_shape.hpp"
#include "onnx_import/core/node.hpp"
namespace ngraph {
namespace onnx_import {
namespace detail {
std::string concat_strings(const std::vector<std::reference_wrapper<const std::string>>& strings);
std::string build_input_provenance_tag(const std::string& input_name, const PartialShape& shape);
std::string build_op_provenance_tag(const Node& onnx_node);
} // namespace detail
} // namespace onnx_import
} // namespace ngraph

1
ngraph/frontend/tensorflow/src/utils.cpp
View File

@ -7,7 +7,6 @@
void ov::frontend::tf::SetTracingInfo(const std::string& op_name, const ov::Output<ov::Node>& ng_node) {
auto node = ng_node.get_node_shared_ptr();
node->set_friendly_name(op_name);
node->add_provenance_tag(op_name);
}
void ov::frontend::tf::TFTensorShapeToNGraphShape(const tensorflow::TensorShapeProto& tf_shape,

2
ngraph/test/CMakeLists.txt
View File

@ -89,7 +89,6 @@ set(SRC
pass/serialization/serialize.cpp
pattern.cpp
preprocess.cpp
provenance.cpp
replace_node.cpp
reshape_opt_kernel.cpp
shape.cpp
@ -554,7 +553,6 @@ if (NGRAPH_ONNX_FRONTEND_ENABLE)
onnx/onnx_import_dyn_shapes.in.cpp
onnx/onnx_import_external_data.in.cpp
onnx/onnx_import_org_openvino.in.cpp
onnx/onnx_import_provenance.in.cpp
onnx/onnx_import_reshape.in.cpp
onnx/onnx_import_rnn.in.cpp
onnx/onnx_import_quant.in.cpp)

91
ngraph/test/onnx/onnx_import_provenance.in.cpp
View File

@ -1,91 +0,0 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "default_opset.hpp"
#include "gtest/gtest.h"
#include "ngraph/file_util.hpp"
#include "ngraph/pass/manager.hpp"
#include "ngraph/provenance.hpp"
#include "onnx_import/onnx.hpp"
#include "util/provenance_enabler.hpp"
#include "util/test_control.hpp"
#include "util/type_prop.hpp"
NGRAPH_SUPPRESS_DEPRECATED_START
using namespace ngraph;
using namespace ngraph::onnx_import;
static std::string s_manifest = "${MANIFEST}";
NGRAPH_TEST(${BACKEND_NAME}, onnx_provenance_tag_text) {
const auto function =
onnx_import::import_onnx_model(file_util::path_join(SERIALIZED_ZOO, "onnx/provenance_tag_add.onnx"));
const auto ng_nodes = function->get_ordered_ops();
for (auto ng_node : ng_nodes) {
for (auto tag : ng_node->get_provenance_tags()) {
EXPECT_HAS_SUBSTRING(tag, "ONNX");
}
}
}
// the NodeToCheck parameter of this template is used to find a node in the whole subgraph
// that a particular unit test is supposed to check against the expected provenance tag
template <typename NodeToCheck>
void test_provenance_tags(const std::shared_ptr<Function> function, const std::string& expected_provenance_tag) {
int node_count = 0;
for (auto ng_node : function->get_ordered_ops()) {
if (as_type_ptr<NodeToCheck>(ng_node)) {
++node_count;
const auto tags = ng_node->get_provenance_tags();
ASSERT_TRUE(tags.size() > 0) << "Node " << ng_node->get_friendly_name()
<< " should have at least one provenance tag.";
EXPECT_TRUE(tags.find(expected_provenance_tag) != tags.end());
}
}
EXPECT_TRUE(node_count > 0) << "Expected type of node doesn't exist in graph.";
}
NGRAPH_TEST(${BACKEND_NAME}, onnx_provenance_only_output) {
test::ProvenanceEnabler provenance_enabler;
// the Add node in the model does not have a name,
// only its output name should be found in the provenance tags
const auto function =
onnx_import::import_onnx_model(file_util::path_join(SERIALIZED_ZOO, "onnx/provenance_only_outputs.onnx"));
test_provenance_tags<default_opset::Add>(function, "<ONNX Add (-> output_of_add)>");
}
NGRAPH_TEST(${BACKEND_NAME}, onnx_provenance_node_name_and_outputs) {
test::ProvenanceEnabler provenance_enabler;
const auto function = onnx_import::import_onnx_model(
file_util::path_join(SERIALIZED_ZOO, "onnx/provenance_node_name_and_outputs.onnx"));
test_provenance_tags<default_opset::Add>(function, "<ONNX Add (Add_node -> output_of_add)>");
}
NGRAPH_TEST(${BACKEND_NAME}, onnx_provenance_multiple_outputs_op) {
test::ProvenanceEnabler provenance_enabler;
const auto function = onnx_import::import_onnx_model(
file_util::path_join(SERIALIZED_ZOO, "onnx/provenance_multiple_outputs_op.onnx"));
test_provenance_tags<default_opset::TopK>(function, "<ONNX TopK (TOPK -> values, indices)>");
}
NGRAPH_TEST(${BACKEND_NAME}, onnx_provenance_tagging_constants) {
test::ProvenanceEnabler provenance_enabler;
const auto function =
onnx_import::import_onnx_model(file_util::path_join(SERIALIZED_ZOO, "onnx/provenance_input_tags.onnx"));
test_provenance_tags<default_opset::Constant>(function, "<ONNX Input (initializer_of_A) Shape:{1}>");
}
NGRAPH_TEST(${BACKEND_NAME}, onnx_provenance_tagging_parameters) {
test::ProvenanceEnabler provenance_enabler;
const auto function =
onnx_import::import_onnx_model(file_util::path_join(SERIALIZED_ZOO, "onnx/provenance_input_tags.onnx"));
test_provenance_tags<default_opset::Parameter>(function, "<ONNX Input (input_B) Shape:{}>");
}

14
ngraph/test/op.cpp
View File

@ -34,20 +34,6 @@ TEST(op, is_parameter) {
EXPECT_FALSE(op::is_parameter(t0));
}
TEST(op, provenance_tag) {
auto node = make_shared<op::Parameter>(element::f32, Shape{1});
auto tag1 = "parameter node";
auto tag2 = "f32 node";
node->add_provenance_tag(tag1);
node->add_provenance_tag(tag2);
node->remove_provenance_tag(tag1);
auto tags = node->get_provenance_tags();
ASSERT_TRUE(tags.find(tag1) == tags.end());
ASSERT_TRUE(tags.find(tag2) != tags.end());
}
TEST(op, opset_multi_thread) {
auto doTest = [&](std::function<const ngraph::OpSet&()> fun) {
std::atomic<const ngraph::OpSet*> opset{nullptr};

389
ngraph/test/provenance.cpp
View File

@ -1,389 +0,0 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "ngraph/provenance.hpp"
#include <fstream>
#include <sstream>
#include <string>
#include <vector>
#include "gmock/gmock.h"
#include "gtest/gtest.h"
#include "ngraph/builder/norm.hpp"
#include "ngraph/graph_util.hpp"
#include "ngraph/ngraph.hpp"
#include "ngraph/pass/manager.hpp"
#include "util/provenance_enabler.hpp"
NGRAPH_SUPPRESS_DEPRECATED_START
using namespace std;
using namespace ngraph;
using ::testing::Return;
using ProvSet = std::unordered_set<std::string>;
TEST(provenance, provenance) {
test::ProvenanceEnabler provenance_enabler;
//
// Before:
//
// A{tag_a} B{tag_b}
// | |
// C{tag_c}
//
// Replacement:
//
// A{tag_a} B{tag_b}
// | |
// C := D{}
//
// After:
//
// A{tag_a} B{tag_b}
// | |
// D{tag_c}
//
// Comment:
// * D is the replacement root, and its insertion kills C. We should not, however, consider
// A and B to be killed, because they are not post-dominated by D until after C is cut out
// of the graph.
//
{
auto x = make_shared<op::Parameter>(element::i32, PartialShape{2, 3, 4});
auto y = make_shared<op::Parameter>(element::i32, PartialShape{2, 3, 4});
auto a = make_shared<op::v1::Add>(x, y);
a->add_provenance_tag("tag_a");
auto b = make_shared<op::v1::Multiply>(y, x);
b->add_provenance_tag("tag_b");
auto c = make_shared<op::v1::Subtract>(a, b);
c->add_provenance_tag("tag_c");
auto f = make_shared<Function>(c, ParameterVector{x, y});
auto new_c = make_shared<op::v1::Subtract>(a, b);
replace_node(c, new_c);
EXPECT_EQ(new_c->get_provenance_tags(), ProvSet{"tag_c"});
}
//
// Before:
//
// A{tag_a} B{tag_b}
// | |
// C{tag_c}
//
// Replacement:
//
//
//
// A{tag_a} B{tag_b}
// | |
// C -> D{tag_d}
//
// After:
//
// A{tag_a} B{tag_b}
// | |
// D{tag_c,tag_d}
//
// Comment:
// * D is the replacement root, and its insertion kills C. We should not, however, consider
// A and B to be killed, because they are not post-dominated by D until after C is cut out
// of the graph.
//
{
auto x = make_shared<op::Parameter>(element::i32, PartialShape{2, 3, 4});
auto y = make_shared<op::Parameter>(element::i32, PartialShape{2, 3, 4});
auto a = make_shared<op::v1::Add>(x, y);
a->add_provenance_tag("tag_a");
auto b = make_shared<op::v1::Multiply>(y, x);
b->add_provenance_tag("tag_b");
auto c = make_shared<op::v1::Subtract>(a, b);
c->add_provenance_tag("tag_c");
auto f = make_shared<Function>(c, ParameterVector{x, y});
auto d = make_shared<op::v1::Subtract>(a, b);
d->add_provenance_tag("tag_d");
replace_node(c, d);
EXPECT_EQ(d->get_provenance_tags(), (ProvSet{"tag_c", "tag_d"}));
}
//
// Before:
//
// A{tag_a} B{tag_b}
// | |
// C{tag_c}
//
// Replacement:
//
// C -> D{tag_d}
//
// After:
//
// D{tag_a,tag_b,tag_c,tag_d}
//
// Comment:
// * D is the replacement root, and its insertion kills A, B, and C.
//
{
auto x = make_shared<op::Parameter>(element::i32, PartialShape{2, 3, 4});
auto y = make_shared<op::Parameter>(element::i32, PartialShape{2, 3, 4});
auto a = make_shared<op::v1::Add>(x, y);
a->add_provenance_tag("tag_a");
auto b = make_shared<op::v1::Multiply>(y, x);
b->add_provenance_tag("tag_b");
auto c = make_shared<op::v1::Subtract>(a, b);
c->add_provenance_tag("tag_c");
auto f = make_shared<Function>(c, ParameterVector{x, y});
auto d = make_zero(element::i32, Shape{2, 3, 4});
d->add_provenance_tag("tag_d");
replace_node(c, d);
EXPECT_EQ(d->get_provenance_tags(), (ProvSet{"tag_a", "tag_b", "tag_c", "tag_d"}));
}
//
// Before:
//
// A{tag_a} B{tag_b}
// | |
// C{tag_c}
//
// Replacement:
//
// C -> D{}
//
// After:
//
// D{tag_a,tag_b,tag_c}
//
// Comment:
// * D is the replacement root, and its insertion kills A, B, and C.
//
{
auto x = make_shared<op::Parameter>(element::i32, PartialShape{2, 3, 4});
auto y = make_shared<op::Parameter>(element::i32, PartialShape{2, 3, 4});
auto a = make_shared<op::v1::Add>(x, y);
a->add_provenance_tag("tag_a");
auto b = make_shared<op::v1::Multiply>(y, x);
b->add_provenance_tag("tag_b");
auto c = make_shared<op::v1::Subtract>(a, b);
c->add_provenance_tag("tag_c");
auto f = make_shared<Function>(c, ParameterVector{x, y});
auto d = make_zero(element::i32, Shape{2, 3, 4});
replace_node(c, d);
EXPECT_EQ(d->get_provenance_tags(), (ProvSet{"tag_a", "tag_b", "tag_c"}));
}
//
// Before:
//
// A{tag_a} B{tag_b}
// | |
// C{tag_c}
//
//
// Replacement:
//
// A{tag_a} B{tag_b}
// | |
// E{} |
// | |
// C -> D{tag_d}
//
//
// After:
//
// A{tag_a} B{tag_b}
// | |
// E{tag_c} |
// | |
// D{tag_c, tag_d}
//
// Comment:
// * D is the replacement root replacing C and creating a new argument node E
//
{
auto x = make_shared<op::Parameter>(element::i32, PartialShape{2, 3, 4});
auto y = make_shared<op::Parameter>(element::i32, PartialShape{2, 3, 4});
auto a = make_shared<op::v1::Add>(x, y);
a->add_provenance_tag("tag_a");
auto b = make_shared<op::v1::Multiply>(y, x);
b->add_provenance_tag("tag_b");
auto c = make_shared<op::v1::Subtract>(a, b);
c->add_provenance_tag("tag_c");
auto f = make_shared<Function>(c, ParameterVector{x, y});
auto e = make_shared<op::v1::Subtract>(a, x);
auto d = make_shared<op::v1::Subtract>(e, b);
d->add_provenance_tag("tag_d");
replace_node(c, d);
EXPECT_EQ(d->get_provenance_tags(), (ProvSet{"tag_c", "tag_d"}));
EXPECT_EQ(e->get_provenance_tags(), (ProvSet{"tag_c"}));
}
//
// Before:
//
// A{tag_a} B{tag_b}
// | |
// C{tag_c}
//
//
// Replacement:
//
// A{tag_a} B{tag_b}
// | |
// E{tag_e} |
// | |
// C -> D{tag_d}
//
//
// After:
//
// A{tag_a} B{tag_b}
// \ /
// E{tag_c, tag_d, tag_e} /
// \ /
// D{tag_c, tag_d}
//
// Comment:
// * D is the replacement root replacing C and creating a new argument node E
//
{
auto x = make_shared<op::Parameter>(element::i32, PartialShape{2, 3, 4});
auto y = make_shared<op::Parameter>(element::i32, PartialShape{2, 3, 4});
auto a = make_shared<op::v1::Add>(x, y);
a->add_provenance_tag("tag_a");
auto b = make_shared<op::v1::Multiply>(y, x);
b->add_provenance_tag("tag_b");
auto c = make_shared<op::v1::Subtract>(a, b);
c->add_provenance_tag("tag_c");
auto f = make_shared<Function>(c, ParameterVector{x, y});
auto e = make_shared<op::v1::Subtract>(a, x);
e->add_provenance_tag("tag_e");
auto d = make_shared<op::v1::Subtract>(e, b);
d->add_provenance_tag("tag_d");
replace_node(c, d);
EXPECT_EQ(d->get_provenance_tags(), (ProvSet{"tag_c", "tag_d"}));
EXPECT_EQ(e->get_provenance_tags(), (ProvSet{"tag_c", "tag_e"}));
}
}
TEST(provenance, add_group_above) {
auto p1 = make_shared<op::Parameter>(element::i32, PartialShape{2, 3, 4});
p1->add_provenance_tag("P1");
auto p2 = make_shared<op::Parameter>(element::i32, PartialShape{2, 3, 4});
p2->add_provenance_tag("P2");
auto a1 = make_shared<op::v1::Add>(p1, p2);
auto m1 = make_shared<op::v1::Multiply>(a1, a1)->add_provenance_group_members_above({p1, p2});
m1->add_provenance_tag("m1");
EXPECT_EQ(p1->get_provenance_tags(), (ProvSet{"P1"}));
EXPECT_EQ(p2->get_provenance_tags(), (ProvSet{"P2"}));
EXPECT_EQ(a1->get_provenance_tags(), (ProvSet{"m1"}));
EXPECT_EQ(m1->get_provenance_tags(), (ProvSet{"m1"}));
}
TEST(provenance, add_tags_above) {
auto x = make_shared<op::Parameter>(element::i32, PartialShape{2, 3, 4});
auto y = make_shared<op::Parameter>(element::i32, PartialShape{2, 3, 4});
auto a = make_shared<op::v1::Add>(x, y);
auto b = make_shared<op::v1::Multiply>(x, y);
auto c = make_shared<op::v1::Subtract>(a, b);
auto d = make_shared<op::Abs>(c);
// Add tags to Subtract and all nodes until Parameters (all above c, until params x, y)
c->add_provenance_tags_above(OutputVector{x, y}, {"tag_above_c - until_params"});
// Add tags to Abs and Subtract (above d, until c inputs)
d->add_provenance_tags_above(c->input_values(), {"tag_above_d - until_c_inputs"});
// Add tags to Abs and all nodes above
d->add_provenance_tags_above(OutputVector{}, {"tag_all_above_d"});
auto x_tags = x->get_provenance_tags();
EXPECT_EQ(x_tags.size(), 1);
EXPECT_TRUE(x_tags.find("tag_all_above_d") != x_tags.end());
auto y_tags = y->get_provenance_tags();
EXPECT_EQ(y_tags.size(), 1);
EXPECT_TRUE(y_tags.find("tag_all_above_d") != y_tags.end());
auto a_tags = a->get_provenance_tags();
EXPECT_EQ(a_tags.size(), 2);
EXPECT_TRUE(a_tags.find("tag_above_c - until_params") != a_tags.end());
EXPECT_FALSE(a_tags.find("tag_above_d - until_c_inputs") != a_tags.end());
EXPECT_TRUE(a_tags.find("tag_all_above_d") != a_tags.end());
auto b_tags = b->get_provenance_tags();
EXPECT_EQ(b_tags.size(), 2);
EXPECT_TRUE(b_tags.find("tag_above_c - until_params") != b_tags.end());
EXPECT_FALSE(b_tags.find("tag_above_d - until_c_inputs") != b_tags.end());
EXPECT_TRUE(b_tags.find("tag_all_above_d") != b_tags.end());
auto c_tags = c->get_provenance_tags();
EXPECT_EQ(c_tags.size(), 3);
EXPECT_TRUE(c_tags.find("tag_above_c - until_params") != c_tags.end());
EXPECT_TRUE(c_tags.find("tag_above_d - until_c_inputs") != c_tags.end());
EXPECT_TRUE(c_tags.find("tag_all_above_d") != c_tags.end());
auto d_tags = d->get_provenance_tags();
EXPECT_EQ(d_tags.size(), 2);
EXPECT_FALSE(d_tags.find("tag_above_c - until_params") != d_tags.end());
EXPECT_TRUE(d_tags.find("tag_above_d - until_c_inputs") != d_tags.end());
EXPECT_TRUE(d_tags.find("tag_all_above_d") != d_tags.end());
}
TEST(provenance, builder) {
auto p1 = make_shared<op::Parameter>(element::i32, PartialShape{2, 3, 4});
p1->add_provenance_tag("P1");
auto norm = builder::opset1::lp_norm(p1, op::Constant::create(element::i64, {}, {0}), 1, 0);
norm->add_provenance_tag("norm");
for (auto node : topological_sort(NodeVector{norm})) {
if (node == p1) {
EXPECT_EQ(node->get_provenance_tags(), (ProvSet{"P1"}));
} else {
EXPECT_EQ(node->get_provenance_tags(), (ProvSet{"norm"}));
}
}
}
TEST(provenance, empty_group) {
auto p1 = make_shared<op::Parameter>(element::i32, PartialShape{2, 3, 4});
p1->add_provenance_tag("P1");
auto abs = make_shared<op::Abs>(p1);
// Make sure group is empty
abs->add_provenance_group_members_above({abs});
abs->add_provenance_tag("abs");
for (auto node : topological_sort(NodeVector{abs})) {
if (node == p1) {
EXPECT_EQ(node->get_provenance_tags(), (ProvSet{"P1"}));
} else {
EXPECT_EQ(node->get_provenance_tags(), (ProvSet{"abs"}));
}
}
}

27
ngraph/test/util/provenance_enabler.hpp
View File

@ -1,27 +0,0 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "ngraph/provenance.hpp"
namespace ngraph {
namespace test {
/// \brief Enable provenance for the duration of a unit test.
///
/// During creation this object activates provenance support, when it's destroyed
/// it returns the provenance support to previous state.
class ProvenanceEnabler {
public:
ProvenanceEnabler() {
saved_enable_state = get_provenance_enabled();
set_provenance_enabled(true);
}
~ProvenanceEnabler() {
set_provenance_enabled(saved_enable_state);
}
private:
bool saved_enable_state;
};
} // namespace test
} // namespace ngraph