ONNX Subgraph - Infer types of inputs from top level Graph nodes (#5277)

This commit is contained in:
Bartosz Sledz authored and GitHub committed 2021-05-12 10:32:34 +02:00
1 parent 932467631e
commit 81f9d78541
15 files changed
+543 -85

No files matched your search

@@ -74,6 +74,10 @@ namespace ngraph
bool has_attribute(const std::string& name) const;
Subgraph get_subgraph_from_attribute(
const std::string& name,
const std::map<std::size_t, std::string>& carried_dependencies_map) const;
template <typename T>
T get_attribute_value(const std::string& name, T default_value) const;
@@ -5,37 +5,57 @@
#include "core/attribute.hpp"
#include "core/graph.hpp"
#include "core/model.hpp"
#include "ngraph/log.hpp"
namespace ngraph
{
namespace onnx_import
{
std::vector<Graph> Attribute::get_graph_array(Model& model) const
{
std::vector<Graph> result;
for (const auto& graph : m_attribute_proto->graphs())
{
result.emplace_back(graph, model);
}
return result;
}
Subgraph Attribute::get_subgraph(const Graph& parent_graph) const
Subgraph Attribute::get_subgraph(
const Graph& parent_graph,
const std::map<std::size_t, std::string>& carried_dependencies_map) const
{
if (m_attribute_proto->type() != ONNX_NAMESPACE::AttributeProto_AttributeType_GRAPH)
{
throw error::attribute::InvalidData{m_attribute_proto->type()};
}
ONNX_NAMESPACE::ModelProto model_proto;
auto model_proto = common::make_unique<ONNX_NAMESPACE::ModelProto>();
const auto& graph = m_attribute_proto->g();
*(model_proto.mutable_graph()) = graph;
model_proto->mutable_graph()->CopyFrom(graph);
const std::size_t subgraph_inputs_count =
static_cast<size_t>(model_proto->mutable_graph()->mutable_input()->size());
// Use the `carried_dependencies_map` to infer the types for the subgraph inputs
for (const auto& carried_dependency : carried_dependencies_map)
{
if (carried_dependency.first >= subgraph_inputs_count)
{
NGRAPH_WARN << "Input with index: '" << carried_dependency.first
<< "' was not found in the subgraph";
}
else
{
const auto& parent_in =
parent_graph.get_ng_node_from_cache(carried_dependency.second);
const auto& carried_type = parent_in.get_element_type();
auto subgraph_in =
model_proto->mutable_graph()->mutable_input(carried_dependency.first);
auto subgraph_in_tensor_type =
subgraph_in->mutable_type()->mutable_tensor_type();
if (!subgraph_in_tensor_type->has_elem_type())
{
subgraph_in_tensor_type->set_elem_type(
onnx_common::ng_to_onnx_data_type(carried_type));
}
}
}
// set opset version and domain from the parent graph
*model_proto.mutable_opset_import() = parent_graph.get_opset_imports();
Model model{model_proto};
return Subgraph{graph, model, parent_graph};
model_proto->mutable_opset_import()->CopyFrom(parent_graph.get_opset_imports());
auto model = common::make_unique<Model>(std::move(model_proto));
return Subgraph{std::move(model), parent_graph};
}
} // namespace onnx_import
@@ -281,7 +281,9 @@ namespace ngraph
float get_float() const { return m_attribute_proto->f(); }
int64_t get_integer() const { return m_attribute_proto->i(); }
const std::string& get_string() const { return m_attribute_proto->s(); }
Subgraph get_subgraph(const Graph& parent_graph) const;
Subgraph get_subgraph(
const Graph& parent_graph,
const std::map<std::size_t, std::string>& carried_dependencies_map) const;
std::vector<Tensor> get_tensor_array() const
{
@@ -306,8 +308,6 @@ namespace ngraph
std::end(m_attribute_proto->strings())};
}
std::vector<Graph> get_graph_array(Model&) const;
/* explicit */ operator ONNX_NAMESPACE::AttributeProto_AttributeType() const
{
return m_attribute_proto->type();
+14 -30
View File
@@ -52,8 +52,8 @@ namespace ngraph
}
} // namespace detail
Graph::Graph(const ONNX_NAMESPACE::GraphProto& graph_proto, Model& model)
: Graph(graph_proto, model, std::unique_ptr<GraphCache>(new GraphCache()))
Graph::Graph(std::unique_ptr<Model>&& model)
: Graph(std::move(model), common::make_unique<GraphCache>())
{
// Remove dangling Parameters
for (auto param_it = m_parameters.begin(); param_it != m_parameters.end();)
@@ -75,27 +75,14 @@ namespace ngraph
param_it++;
}
}
Graph::Graph(const Graph& graph)
: m_parameters(graph.m_parameters)
, m_cache{new GraphCache(*graph.m_cache)}
, m_graph_proto{graph.m_graph_proto}
, m_nodes(graph.m_nodes)
, m_inputs(graph.m_inputs)
, m_outputs(graph.m_outputs)
, m_model{graph.m_model}
{
}
Graph::Graph(const ONNX_NAMESPACE::GraphProto& graph_proto,
Model& model,
std::unique_ptr<GraphCache>&& cache)
: m_cache{std::move(cache)}
, m_graph_proto{&graph_proto}
, m_model{&model}
Graph::Graph(std::unique_ptr<Model>&& model, std::unique_ptr<GraphCache>&& cache)
: m_model{std::move(model)}
, m_cache{std::move(cache)}
{
std::map<std::string, Tensor> initializers;
// Process all initializers in the graph
for (const auto& initializer_tensor : m_graph_proto->initializer())
for (const auto& initializer_tensor : m_model->get_graph().initializer())
{
if (initializer_tensor.has_name())
{
@@ -133,7 +120,7 @@ namespace ngraph
}
// Process all ONNX graph inputs, convert them to nGraph nodes and store in cache
for (const auto& input : m_graph_proto->input())
for (const auto& input : m_model->get_graph().input())
{
m_inputs.emplace_back(input);
@@ -150,7 +137,7 @@ namespace ngraph
}
// Process all graph outputs
for (const auto& output : m_graph_proto->output())
for (const auto& output : m_model->get_graph().output())
{
m_outputs.emplace_back(output);
}
@@ -158,7 +145,7 @@ namespace ngraph
// Verify that ONNX graph contains only nodes of available operator types
std::map<std::string, std::reference_wrapper<const ONNX_NAMESPACE::NodeProto>>
unknown_operators;
for (const auto& node_proto : m_graph_proto->node())
for (const auto& node_proto : m_model->get_graph().node())
{
if (!m_model->is_operator_available(node_proto))
{
@@ -189,7 +176,7 @@ namespace ngraph
detail::to_string(unknown_operators));
// Process ONNX graph nodes, convert to nGraph nodes
for (const auto& node_proto : m_graph_proto->node())
for (const auto& node_proto : m_model->get_graph().node())
{
m_nodes.emplace_back(node_proto, *this);
const Node& node{m_nodes.back()};
@@ -219,7 +206,7 @@ namespace ngraph
OutputVector Graph::get_ng_outputs() const
{
OutputVector results;
for (const auto& output : m_graph_proto->output())
for (const auto& output : m_model->get_graph().output())
{
const auto& ng_output = get_ng_node_from_cache(output.name());
if (!ngraph::op::is_null(ng_output)) // ignore optional outputs
@@ -335,17 +322,14 @@ namespace ngraph
return m_model->get_opset_imports();
}
Subgraph::Subgraph(const ONNX_NAMESPACE::GraphProto& proto,
Model& model,
const Graph& parent_graph)
Subgraph::Subgraph(std::unique_ptr<Model>&& model, const Graph& parent_graph)
: Graph(
proto,
model,
std::move(model),
std::unique_ptr<SubgraphCache>(new SubgraphCache(parent_graph.get_graph_cache())))
{
// find all nodes on edge parent graph-subgraph
// (it means input of node from parent graph, output from subgraph)
for (const auto& node_proto : proto.node())
for (const auto& node_proto : m_model->get_graph().node())
{
int input_index = 0;
for (const auto& in_name : node_proto.input())
+20 -12
View File
@@ -23,8 +23,14 @@ namespace ngraph
class Graph
{
public:
Graph(const ONNX_NAMESPACE::GraphProto& proto, Model& model);
Graph(const Graph& graph);
Graph(std::unique_ptr<Model>&& model);
Graph() = delete;
Graph(const Graph&) = delete;
Graph(Graph&&) = default;
Graph& operator=(const Graph&) = delete;
Graph& operator=(Graph&&) = default;
const std::vector<Node>& get_nodes() const { return m_nodes; }
const std::vector<ValueInfo>& get_inputs() const { return m_inputs; }
const std::vector<ValueInfo>& get_outputs() const { return m_outputs; }
@@ -32,16 +38,14 @@ namespace ngraph
const ParameterVector& get_ng_parameters() const { return m_parameters; }
bool is_node_in_cache(const std::string& name) const;
Output<ngraph::Node> get_ng_node_from_cache(const std::string& name) const;
const std::string& get_name() const { return m_graph_proto->name(); }
const std::string& get_name() const { return m_model->get_graph().name(); }
OutputVector make_ng_nodes(const Node& onnx_node) const;
const GraphCache& get_graph_cache() const;
const OpsetImports& get_opset_imports() const;
virtual ~Graph() = default;
protected:
Graph(const ONNX_NAMESPACE::GraphProto& proto,
Model& model,
std::unique_ptr<GraphCache>&& cache);
Graph(std::unique_ptr<Model>&& model, std::unique_ptr<GraphCache>&& cache);
void set_friendly_names(const Node& onnx_node,
const OutputVector& ng_node_vector) const;
@@ -57,14 +61,13 @@ namespace ngraph
protected:
ParameterVector m_parameters;
std::unique_ptr<Model> m_model;
std::unique_ptr<GraphCache> m_cache;
private:
const ONNX_NAMESPACE::GraphProto* m_graph_proto;
std::vector<Node> m_nodes;
std::vector<ValueInfo> m_inputs;
std::vector<ValueInfo> m_outputs;
Model* m_model;
};
/// \brief Representation of ONNX subgraph. It is used for example by ONNX Loop op.
@@ -75,17 +78,22 @@ namespace ngraph
public:
/// \brief Subgraph a GraphCache class object.
///
/// \param[in] proto The ONNX protobuf graph representation.
/// \param[in] model The ONNX model object.
/// \param[in] parent_graph The reference to the parent graph.
Subgraph(const ONNX_NAMESPACE::GraphProto& proto,
Model& model,
const Graph& parent_graph);
Subgraph(std::unique_ptr<Model>&& model, const Graph& parent_graph);
/// \brief Return outputs which are on the edge the subgraph and the parent graph.
/// \return Vector of edge nodes from parent scope.
const std::vector<Output<ngraph::Node>> get_outputs_from_parent() const;
Subgraph() = delete;
Subgraph(const Subgraph&) = delete;
Subgraph(Subgraph&&) = default;
Subgraph& operator=(const Subgraph&) = delete;
Subgraph& operator=(Subgraph&&) = default;
private:
std::vector<Output<ngraph::Node>> m_outputs_from_parent;
};
@@ -31,8 +31,8 @@ namespace ngraph
throw ngraph_error("Couldn't find operator set's version for domain: " + domain + ".");
}
Model::Model(const ONNX_NAMESPACE::ModelProto& model_proto)
: m_model_proto{&model_proto}
Model::Model(std::unique_ptr<ONNX_NAMESPACE::ModelProto>&& model_proto)
: m_model_proto{std::move(model_proto)}
{
// Walk through the elements of opset_import field and register operator sets
// for each domain. An exception UnknownDomain() will raise if the domain is
@@ -28,10 +28,10 @@ namespace ngraph
{
public:
Model() = delete;
explicit Model(const ONNX_NAMESPACE::ModelProto& model_proto);
explicit Model(std::unique_ptr<ONNX_NAMESPACE::ModelProto>&& model_proto);
Model(const Model&) = default;
Model(Model&&) = default;
Model(const Model&) = delete;
Model(Model&&) = delete;
Model& operator=(const Model&) = delete;
Model& operator=(Model&&) = delete;
@@ -71,7 +71,7 @@ namespace ngraph
void enable_opset_domain(const std::string& domain);
private:
const ONNX_NAMESPACE::ModelProto* m_model_proto;
const std::unique_ptr<ONNX_NAMESPACE::ModelProto> m_model_proto;
std::unordered_map<std::string, OperatorSet> m_opset;
};
+28 -9
View File
@@ -44,6 +44,10 @@ namespace ngraph
bool has_attribute(const std::string& name) const;
Subgraph get_subgraph_from_attribute(
const std::string& name,
const std::map<std::size_t, std::string>& carried_dependencies_map) const;
template <typename T>
T get_attribute_value(const std::string& name, T default_value) const;
@@ -90,6 +94,21 @@ namespace ngraph
return it != std::end(m_attributes);
}
Subgraph Node::Impl::get_subgraph_from_attribute(
const std::string& name,
const std::map<std::size_t, std::string>& carried_dependencies_map) const
{
auto it = std::find_if(
std::begin(m_attributes), std::end(m_attributes), [&](const Attribute& attribute) {
return attribute.get_name() == name;
});
if (it == std::end(m_attributes))
{
throw error::node::UnknownAttribute{this->name(), name};
}
return it->get_subgraph(graph(), carried_dependencies_map);
}
template <typename T>
T Node::Impl::get_attribute_value(const std::string& name, T default_value) const
{
@@ -121,15 +140,8 @@ namespace ngraph
template <>
Subgraph Node::Impl::get_attribute_value(const std::string& name) const
{
auto it = std::find_if(
std::begin(m_attributes), std::end(m_attributes), [&](const Attribute& attribute) {
return attribute.get_name() == name;
});
if (it == std::end(m_attributes))
{
throw error::node::UnknownAttribute{this->name(), name};
}
return it->get_subgraph(graph());
const std::map<std::size_t, std::string> empty_map;
return get_subgraph_from_attribute(name, empty_map);
}
OutputVector Node::Impl::get_ng_nodes(const Node& node) const
@@ -207,6 +219,13 @@ namespace ngraph
return m_pimpl->has_attribute(name);
}
Subgraph Node::get_subgraph_from_attribute(
const std::string& name,
const std::map<std::size_t, std::string>& carried_dependencies_map) const
{
return m_pimpl->get_subgraph_from_attribute(name, carried_dependencies_map);
}
std::vector<std::string> Node::get_attribute_names() const
{
std::vector<std::string> attr_names;
+9 -1
View File
@@ -70,7 +70,15 @@ namespace ngraph
const OutputVector loop_carried_dependencies{std::next(ng_inputs.begin(), 2),
ng_inputs.end()};
const Subgraph& body_graph{node.get_attribute_value<Subgraph>("body")};
std::map<std::size_t, std::string> loop_carried_dependencies_map;
for (std::size_t i = 0; i < loop_carried_dependencies.size(); i++)
{
loop_carried_dependencies_map[i + 2] =
loop_carried_dependencies[i].get_node()->get_friendly_name();
}
const Subgraph& body_graph{
node.get_subgraph_from_attribute("body", loop_carried_dependencies_map)};
auto body_outputs = body_graph.get_ng_outputs();
const auto& body_inputs = body_graph.get_ng_parameters();
@@ -125,6 +125,18 @@ namespace ngraph
const std::shared_ptr<ngraph::Node> input,
const std::set<element::Type> allowed_types = {});
/// \brief Temporary replacement for C++14 std::make_unique.
/// \note details: https://en.cppreference.com/w/cpp/memory/unique_ptr/make_unique
///
/// \param args List of arguments with which an instance of T will be constructed.
///
/// \return std::unique_ptr of an instance of type T
template <typename T, typename... Args>
std::unique_ptr<T> make_unique(Args&&... args)
{
return std::unique_ptr<T>(new T(std::forward<Args>(args)...));
}
} // namespace common
} // namespace onnx_import
} // namespace ngraph
@@ -18,8 +18,10 @@ namespace ngraph
std::shared_ptr<Function>
convert_to_ng_function(const ONNX_NAMESPACE::ModelProto& model_proto)
{
Model model{model_proto};
Graph graph{model_proto.graph(), model};
auto p_model_proto = common::make_unique<ONNX_NAMESPACE::ModelProto>(model_proto);
auto model = common::make_unique<Model>(std::move(p_model_proto));
Graph graph{std::move(model)};
auto function = std::make_shared<Function>(
graph.get_ng_outputs(), graph.get_ng_parameters(), graph.get_name());
for (std::size_t i{0}; i < function->get_output_size(); ++i)
@@ -29,7 +29,6 @@ from tests import (BACKEND_NAME,
xfail_issue_38722,
xfail_issue_38723,
xfail_issue_38724,
xfail_issue_38725,
xfail_issue_38732,
xfail_issue_38734,
xfail_issue_38735,
@@ -299,9 +298,6 @@ tests_expected_to_fail = [
"OnnxBackendNodeModelTest.test_qlinearconv_cpu"),
(xfail_issue_38724,
"OnnxBackendNodeModelTest.test_resize_tf_crop_and_resize_cpu"),
(xfail_issue_38725,
"OnnxBackendNodeModelTest.test_range_int32_type_negative_delta_expanded_cpu",
"OnnxBackendNodeModelTest.test_range_float_type_positive_delta_expanded_cpu"),
(xfail_issue_33512,
"OnnxBackendNodeModelTest.test_einsum_transpose_cpu",
"OnnxBackendNodeModelTest.test_einsum_batch_diagonal_cpu",
@@ -359,7 +355,9 @@ tests_expected_to_fail = [
"OnnxBackendNodeModelTest.test_unsqueeze_two_axes_cpu",
"OnnxBackendNodeModelTest.test_unsqueeze_unsorted_axes_cpu",),
(xfail_issue_44956,
"OnnxBackendNodeModelTest.test_loop11_cpu"),
"OnnxBackendNodeModelTest.test_loop11_cpu",
"OnnxBackendNodeModelTest.test_range_int32_type_negative_delta_expanded_cpu",
"OnnxBackendNodeModelTest.test_range_float_type_positive_delta_expanded_cpu"),
(xfail_issue_44957,
"OnnxBackendNodeModelTest.test_compress_0_cpu",
"OnnxBackendNodeModelTest.test_compress_1_cpu",
@@ -0,0 +1,176 @@
ir_version: 6
producer_name: "nGraph ONNX Importer"
graph {
name: "basic loop"
node {
input: "parent_input"
input: "scale"
name: "mul_node"
op_type: "Mul"
output: "b"
}
node {
input: "parent_input"
input: "b"
name: "parent_add_node"
op_type: "Add"
output: "c"
}
node {
input: "trip_count"
input: "cond_in"
input: "a_init"
output: "a_final"
output: "a_values"
op_type: "Loop"
attribute {
name: "body"
g {
name: "simple add"
node {
input: "b"
input: "a_in"
output: "current_a"
name: "loop_body_add"
op_type: "Add"
}
node {
input: "cond"
output: "cond_out"
name: "cond_identity"
op_type: "Identity"
}
node {
input: "current_a"
output: "a_out"
name: "output_accumulator"
op_type: "Identity"
}
input {
name: "i"
type {
tensor_type {
elem_type: 7
shape {
dim {
dim_value: 1
}
}
}
}
}
input {
name: "cond"
type {
tensor_type {
elem_type: 9
}
}
}
input {
name: "a_in"
}
output {
name: "cond_out"
type {
tensor_type {
elem_type: 9
}
}
}
output {
name: "current_a"
type {
tensor_type {
elem_type: 1
}
}
}
output {
name: "a_out"
type {
tensor_type {
elem_type: 1
}
}
}
}
type: GRAPH
}
}
initializer {
dims: 1
data_type: 7
int64_data: 3
name: "trip_count"
}
initializer {
dims: 1
data_type: 9
int32_data: 00000001
name: "cond_in"
}
initializer {
dims: 1
data_type: 1
float_data: 2
name: "scale"
}
input {
name: "a_init"
type {
tensor_type {
elem_type: 1
shape {
dim {
dim_value: 1
}
dim {
dim_value: 2
}
}
}
}
}
input {
name: "parent_input"
type {
tensor_type {
elem_type: 1
shape {
dim {
dim_value: 1
}
}
}
}
}
output {
name: "a_final"
type {
tensor_type {
elem_type: 1
}
}
}
output {
name: "a_values"
type {
tensor_type {
elem_type: 1
}
}
}
output {
name: "c"
type {
tensor_type {
elem_type: 1
}
}
}
}
opset_import {
version: 11
}
@@ -0,0 +1,184 @@
ir_version: 6
producer_name: "nGraph ONNX Importer"
graph {
name: "basic loop"
node {
input: "trip_count"
input: "cond_in"
input: "a_init"
output: "a_final"
op_type: "Loop"
attribute {
name: "body"
g {
node {
input: "a_in"
input: "b"
output: "current_a"
name: "loop_body_add"
op_type: "Add"
}
node {
input: "i"
input: "threshold"
output: "cond_out"
name: "condition_calc"
op_type: "Less"
}
node {
input: "current_a"
output: "a_out"
name: "output_accumulator"
op_type: "Identity"
}
name: "simple add"
initializer {
dims: 1
dims: 2
data_type: 1
float_data: 1
float_data: 1
name: "b"
}
input {
name: "i"
type {
tensor_type {
elem_type: 7
shape {
dim {
dim_value: 1
}
}
}
}
}
input {
name: "cond"
type {
tensor_type {
elem_type: 9
shape {
dim {
dim_value: 1
}
}
}
}
}
input {
name: "a_in"
type {
tensor_type {
shape {
dim {
dim_value: 1
}
dim {
dim_value: 2
}
}
}
}
}
output {
name: "cond_out"
type {
tensor_type {
elem_type: 9
shape {
dim {
dim_value: 1
}
}
}
}
}
output {
name: "current_a"
type {
tensor_type {
elem_type: 1
shape {
dim {
dim_value: 1
}
dim {
dim_value: 2
}
}
}
}
}
}
type: GRAPH
}
}
initializer {
dims: 1
data_type: 7
int64_data: 5
name: "threshold"
}
input {
name: "trip_count"
type {
tensor_type {
elem_type: 7
shape {
dim {
dim_value: 1
}
}
}
}
}
input {
name: "cond_in"
type {
tensor_type {
elem_type: 9
shape {
dim {
dim_value: 1
}
}
}
}
}
input {
name: "a_init"
type {
tensor_type {
elem_type: 1
shape {
dim {
dim_value: 1
}
dim {
dim_value: 2
}
}
}
}
}
output {
name: "a_final"
type {
tensor_type {
elem_type: 1
shape {
dim {
dim_value: 1
}
dim {
dim_value: 2
}
}
}
}
}
}
opset_import {
version: 11
}
@@ -381,6 +381,49 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_controlflow_loop_2d_trip_count_dynamic)
test_case.run();
}
// ~~~~~~~~SUBGRAPH TYPES INFERENCE:~~~~~~~~
NGRAPH_TEST(${BACKEND_NAME},
onnx_controlflow_loop_2d_infer_types)
{
const auto function = onnx_import::import_onnx_model(file_util::path_join(
SERIALIZED_ZOO,
"onnx/loop/onnx_controlflow_loop_2d_infer_types.prototxt"));
auto test_case = test::TestCase<TestEngine>(function);
// trip count
test_case.add_input<int64_t>({10});
// termination condition
test_case.add_input<bool>({true});
// a_init
test_case.add_input<float>({0.f, 0.f});
test_case.add_expected_output<float>(Shape{1, 2}, {6.f, 6.f});
test_case.run();
}
NGRAPH_TEST(${BACKEND_NAME},
onnx_controlflow_loop_add_node_from_parent_scope_infer_types)
{
const auto function = onnx_import::import_onnx_model(file_util::path_join(
SERIALIZED_ZOO,
"onnx/loop/loop_add_node_from_parent_scope_infer_types.prototxt"));
auto test_case = test::TestCase<TestEngine>(function);
// a_init
test_case.add_input<float>({0.f, 0.f});
// parent_input
test_case.add_input<float>({3.f});
test_case.add_expected_output<float>(Shape{1, 2}, {18.f, 18.f});
test_case.add_expected_output<float>(Shape{3, 1, 2}, {6.f, 6.f, 12.f, 12.f, 18.f, 18.f});
test_case.add_expected_output<float>(Shape{1}, {9.f});
test_case.run();
}
// ~~~~~~~~ADDITIONAL TESTS:~~~~~~~~
NGRAPH_TEST(${BACKEND_NAME}, onnx_controlflow_loop_concat_values)