From f3e7e55968ba4ff9daa3bc257cc39001e2cf3782 Mon Sep 17 00:00:00 2001 From: Roman Kazantsev Date: Wed, 8 Mar 2023 17:29:42 +0400 Subject: [PATCH] [TF FE] Support multioutput body graph nodes (#16142) This is a corner case because body graph nodes have named output ports. This allows to support custom RetinaNet model. Signed-off-by: Kazantsev, Roman --- .../include/openvino/frontend/decoder.hpp | 12 + .../tensorflow/src/decoder_argdef.cpp | 24 +- .../tensorflow/src/decoder_argdef.hpp | 5 + .../tensorflow/src/decoder_proto.cpp | 68 ++++- .../tensorflow/src/decoder_proto.hpp | 10 + .../tensorflow/src/translate_session.cpp | 6 +- .../tests/convert_tricky_models.cpp | 15 + .../models_pbtxt/partitioned_call2.pbtxt | 285 ++++++++++++++++++ .../models_pbtxt/partitioned_call2.py | 21 ++ .../include/helper_ops/internal_operation.hpp | 8 + .../src/decoder_flatbuffer.cpp | 7 + .../tensorflow_lite/src/decoder_flatbuffer.h | 5 + .../tensorflow_lite/src/decoder_map.hpp | 13 + 13 files changed, 456 insertions(+), 23 deletions(-) create mode 100644 src/frontends/tensorflow/tests/test_models/models_pbtxt/partitioned_call2.pbtxt create mode 100644 src/frontends/tensorflow/tests/test_models/models_pbtxt/partitioned_call2.py diff --git a/src/frontends/common/include/openvino/frontend/decoder.hpp b/src/frontends/common/include/openvino/frontend/decoder.hpp index 878bbd7ecc8..02d82116595 100644 --- a/src/frontends/common/include/openvino/frontend/decoder.hpp +++ b/src/frontends/common/include/openvino/frontend/decoder.hpp @@ -50,6 +50,7 @@ public: class FRONTEND_API DecoderBase { public: + using OpTypeByName = std::unordered_map; /// \brief Get attribute value by name /// /// \param name Attribute name @@ -68,6 +69,17 @@ public: std::string& producer_name, size_t& producer_output_port_index) const = 0; + /// \brief Get a producer name and its output port index + /// + /// \param input_port_idx Input port index by which data is consumed + /// \param producer_name A producer name + /// \param producer_output_port_index Output port index from which data is generated + /// \param op_type_by_name Map of operation name to their types + virtual void get_input_node(size_t input_port_idx, + std::string& producer_name, + size_t& producer_output_port_index, + const OpTypeByName& op_type_by_name) const = 0; + /// \brief Get operation type virtual const std::string& get_op_type() const = 0; diff --git a/src/frontends/tensorflow/src/decoder_argdef.cpp b/src/frontends/tensorflow/src/decoder_argdef.cpp index e34f04dd9db..9724a36357b 100644 --- a/src/frontends/tensorflow/src/decoder_argdef.cpp +++ b/src/frontends/tensorflow/src/decoder_argdef.cpp @@ -4,6 +4,7 @@ #include "decoder_argdef.hpp" +#include "decoder_proto.hpp" #include "op_def.pb.h" #include "openvino/frontend/tensorflow/node_context.hpp" #include "openvino/frontend/tensorflow/special_types.hpp" @@ -58,19 +59,16 @@ void DecoderArgDef::get_input_node(size_t input_port_idx, // and output port is 2 FRONT_END_GENERAL_CHECK(m_op_type == "output_arg", "[TensorFlow Frontend] Internal error: get_input_node is supported only for output_arg."); - auto first_colon = m_producer_name.find_first_of(":"); - auto last_colon = m_producer_name.find_last_of(":"); - if (first_colon != std::string::npos && last_colon != std::string::npos) { - producer_name = m_producer_name.substr(0, first_colon); - auto port_id = m_producer_name.substr(last_colon + 1); - FRONT_END_GENERAL_CHECK(!port_id.empty() && std::all_of(port_id.begin(), port_id.end(), ::isdigit), - "Port id is not specified or not a number. Value: ", - port_id); - producer_output_port_index = std::stoi(port_id); - return; - } - producer_name = m_producer_name; - producer_output_port_index = 0; + parse_producer_name(m_producer_name, producer_name, producer_output_port_index, {}); +} + +void DecoderArgDef::get_input_node(size_t input_port_idx, + std::string& producer_name, + size_t& producer_output_port_index, + const OpTypeByName& op_type_by_name) const { + FRONT_END_GENERAL_CHECK(m_op_type == "output_arg", + "[TensorFlow Frontend] Internal error: get_input_node is supported only for output_arg."); + parse_producer_name(m_producer_name, producer_name, producer_output_port_index, op_type_by_name); } ov::Any DecoderArgDef::get_attribute(const std::string& name) const { diff --git a/src/frontends/tensorflow/src/decoder_argdef.hpp b/src/frontends/tensorflow/src/decoder_argdef.hpp index d10d8c6ab0f..5b01025ee4d 100644 --- a/src/frontends/tensorflow/src/decoder_argdef.hpp +++ b/src/frontends/tensorflow/src/decoder_argdef.hpp @@ -38,6 +38,11 @@ public: std::string& producer_name, size_t& producer_output_port_index) const override; + void get_input_node(size_t input_port_idx, + std::string& producer_name, + size_t& producer_output_port_index, + const OpTypeByName& op_type_by_name) const override; + const std::string& get_op_type() const override; const std::string& get_op_name() const override; diff --git a/src/frontends/tensorflow/src/decoder_proto.cpp b/src/frontends/tensorflow/src/decoder_proto.cpp index b875753c493..cc3606a4da2 100644 --- a/src/frontends/tensorflow/src/decoder_proto.cpp +++ b/src/frontends/tensorflow/src/decoder_proto.cpp @@ -285,16 +285,53 @@ size_t DecoderProto::get_input_size() const { return m_node_def->input_size(); } -void DecoderProto::get_input_node(size_t input_port_idx, - std::string& producer_name, - size_t& producer_output_port_index) const { - // Body graph nodes may have two colons `:`, for example, - // producer_name:z:2 means that producer operation name is `producer_name` - // and output port is 2 - std::string producer_port_name = m_node_def->input(static_cast(input_port_idx)); +void parse_producer_name(const std::string& producer_port_name, + std::string& producer_name, + size_t& producer_output_port_index, + const DecoderBase::OpTypeByName& op_type_by_name) { + using OutputPortIdxMax = std::unordered_map; + // create a table of operation type and its output ports + // for which we specify output port indices manually + // it is mainly affects multiple output operations + // extract this information from tensorflow/core/ops/*.cc files + const OutputPortIdxMax output_port_idx_map = { + {"TopK:indices", 1}, + {"TopKV2:indices", 1}, + {"CTCGreedyDecoder:decoded_values", 1}, + {"CTCGreedyDecoder:decoded_shape", 2}, + {"CTCGreedyDecoder:log_probability", 3}, + {"CTCGreedyDecoder:log_probability", 3}, + {"FusedBatchNorm:batch_mean", 1}, + {"FusedBatchNorm:batch_variance", 2}, + {"FusedBatchNormV2:batch_mean", 1}, + {"FusedBatchNormV2:batch_variance", 2}, + {"FusedBatchNormV3:batch_mean", 1}, + {"FusedBatchNormV3:batch_variance", 2}, + }; + // Body graph nodes may have two colons `:` input names, for example, + // `TopKV2Name:indices:0` means that producer operation name is `TopKV2Name` + // the middle name is output port name of the producer `indices` that means + // the second output port of TopKV2 is used. + // The first output port of TopKV2 is described as `TopKV2Name:values:0` auto first_colon = producer_port_name.find_first_of(":"); auto last_colon = producer_port_name.find_last_of(":"); - if (first_colon != std::string::npos && last_colon != std::string::npos) { + if (first_colon != std::string::npos && first_colon < last_colon) { + // we have at least two colons producer_name:output_port_name:port_idx + producer_name = producer_port_name.substr(0, first_colon); + auto port_id = producer_port_name.substr(last_colon + 1); + auto port_name = producer_port_name.substr(first_colon + 1, last_colon - first_colon - 1); + FRONT_END_GENERAL_CHECK(!port_id.empty() && std::all_of(port_id.begin(), port_id.end(), ::isdigit), + "Port id is not specified or not a number. Value: ", + port_id); + producer_output_port_index = std::stoi(port_id); + auto producer_op_type = + (op_type_by_name.count(producer_name) > 0) ? op_type_by_name.at(producer_name) : "Unknown"; + auto producer_key = producer_op_type + ":" + port_name; + producer_output_port_index = output_port_idx_map.count(producer_key) > 0 ? output_port_idx_map.at(producer_key) + : producer_output_port_index; + return; + } else if (first_colon != std::string::npos) { + // just one colon case producer_name = producer_port_name.substr(0, first_colon); auto port_id = producer_port_name.substr(last_colon + 1); FRONT_END_GENERAL_CHECK(!port_id.empty() && std::all_of(port_id.begin(), port_id.end(), ::isdigit), @@ -307,6 +344,21 @@ void DecoderProto::get_input_node(size_t input_port_idx, producer_output_port_index = 0; } +void DecoderProto::get_input_node(size_t input_port_idx, + std::string& producer_name, + size_t& producer_output_port_index) const { + const std::string producer_port_name = m_node_def->input(static_cast(input_port_idx)); + parse_producer_name(producer_port_name, producer_name, producer_output_port_index, {}); +} + +void DecoderProto::get_input_node(size_t input_port_idx, + std::string& producer_name, + size_t& producer_output_port_index, + const OpTypeByName& op_type_by_name) const { + const std::string producer_port_name = m_node_def->input(static_cast(input_port_idx)); + parse_producer_name(producer_port_name, producer_name, producer_output_port_index, op_type_by_name); +} + const std::string& DecoderProto::get_op_type() const { return m_node_def->op(); } diff --git a/src/frontends/tensorflow/src/decoder_proto.hpp b/src/frontends/tensorflow/src/decoder_proto.hpp index 2cdfe12c455..6b277ad54e6 100644 --- a/src/frontends/tensorflow/src/decoder_proto.hpp +++ b/src/frontends/tensorflow/src/decoder_proto.hpp @@ -18,6 +18,11 @@ namespace ov { namespace frontend { namespace tensorflow { +void parse_producer_name(const std::string& producer_port_name, + std::string& producer_name, + size_t& producer_output_port_index, + const DecoderBase::OpTypeByName& op_type_by_name); + class DecoderProto : public ov::frontend::tensorflow::DecoderBase { public: explicit DecoderProto(const ::tensorflow::NodeDef* node_def) : m_node_def(node_def) {} @@ -30,6 +35,11 @@ public: std::string& producer_name, size_t& producer_output_port_index) const override; + void get_input_node(size_t input_port_idx, + std::string& producer_name, + size_t& producer_output_port_index, + const OpTypeByName& op_type_by_name) const override; + const std::string& get_op_type() const override; const std::string& get_op_name() const override; diff --git a/src/frontends/tensorflow/src/translate_session.cpp b/src/frontends/tensorflow/src/translate_session.cpp index 9d983583387..54a9693193e 100644 --- a/src/frontends/tensorflow/src/translate_session.cpp +++ b/src/frontends/tensorflow/src/translate_session.cpp @@ -84,6 +84,7 @@ void TranslateSession::inject_body_model(std::shared_ptr body_model, void TranslateSession::translate_graph(const ov::frontend::InputModel::Ptr& input_model, std::shared_ptr& ov_model) { + DecoderBase::OpTypeByName op_type_by_name; OpMap ng_op_map; ov::ParameterVector params; ov::ResultVector results; @@ -130,6 +131,7 @@ void TranslateSession::translate_graph(const ov::frontend::InputModel::Ptr& inpu for (const auto& operation_place : operation_places) { auto operation_decoder = operation_place->get_decoder(); auto operation_name = operation_place->get_names()[0]; + op_type_by_name[operation_name] = operation_decoder->get_op_type(); // output for parameter nodes has been already generated if (ng_op_map.count(operation_name)) { continue; @@ -151,7 +153,7 @@ void TranslateSession::translate_graph(const ov::frontend::InputModel::Ptr& inpu std::string producer_name; size_t producer_port_idx; try { - operation_decoder->get_input_node(input_port_idx, producer_name, producer_port_idx); + operation_decoder->get_input_node(input_port_idx, producer_name, producer_port_idx, op_type_by_name); } catch (const std::exception&) { FRONT_END_THROW("[ ERROR ] Exception happened when preparing input " + std::to_string(input_port_idx) + " for op '" + operation_decoder->get_op_name() + "', expected input name: '" + @@ -297,7 +299,7 @@ void TranslateSession::translate_graph(const ov::frontend::InputModel::Ptr& inpu std::string producer_name; size_t producer_port_idx; try { - operation_decoder->get_input_node(port_index, producer_name, producer_port_idx); + operation_decoder->get_input_node(port_index, producer_name, producer_port_idx, op_type_by_name); } catch (const std::exception&) { FRONT_END_THROW("[ ERROR ] Exception happened when preparing input " + std::to_string(port_index) + " for op '" + operation_decoder->get_op_name() + "', expected input name: '" + diff --git a/src/frontends/tensorflow/tests/convert_tricky_models.cpp b/src/frontends/tensorflow/tests/convert_tricky_models.cpp index 25ab7fc8045..287c4b52db5 100644 --- a/src/frontends/tensorflow/tests/convert_tricky_models.cpp +++ b/src/frontends/tensorflow/tests/convert_tricky_models.cpp @@ -335,3 +335,18 @@ TEST_F(TransformationTestsF, ModelWithLookupTableOperations) { model_ref = make_shared(OutputVector{add}, ParameterVector{x}); } } + +TEST_F(TransformationTestsF, ModelWithMultioutputBodyGraphNode) { + { model = convert_model("partitioned_call2/partitioned_call2.pb"); } + { + auto x = make_shared(i32, Shape{5}); + auto y = make_shared(i32, Shape{5}); + auto sub = make_shared(x, y); + auto const_three = make_shared(i32, Shape{}, 3); + auto const_ten = make_shared(i32, Shape{}, 10); + auto topk = + make_shared(sub, const_three, -1, op::v1::TopK::Mode::MAX, op::v1::TopK::SortType::SORT_VALUES, i32); + auto add = make_shared(topk->output(1), const_ten); + model_ref = make_shared(OutputVector{add}, ParameterVector{x, y}); + } +} \ No newline at end of file diff --git a/src/frontends/tensorflow/tests/test_models/models_pbtxt/partitioned_call2.pbtxt b/src/frontends/tensorflow/tests/test_models/models_pbtxt/partitioned_call2.pbtxt new file mode 100644 index 00000000000..de7125148be --- /dev/null +++ b/src/frontends/tensorflow/tests/test_models/models_pbtxt/partitioned_call2.pbtxt @@ -0,0 +1,285 @@ +node { + name: "x" + op: "Placeholder" + attr { + key: "_user_specified_name" + value { + s: "x" + } + } + attr { + key: "dtype" + value { + type: DT_INT32 + } + } + attr { + key: "shape" + value { + shape { + dim { + size: 5 + } + } + } + } +} +node { + name: "y" + op: "Placeholder" + attr { + key: "_user_specified_name" + value { + s: "y" + } + } + attr { + key: "dtype" + value { + type: DT_INT32 + } + } + attr { + key: "shape" + value { + shape { + dim { + size: 5 + } + } + } + } +} +node { + name: "sub" + op: "Sub" + input: "x" + input: "y" + attr { + key: "T" + value { + type: DT_INT32 + } + } +} +node { + name: "PartitionedCall" + op: "PartitionedCall" + input: "sub" + attr { + key: "Tin" + value { + list { + type: DT_INT32 + } + } + } + attr { + key: "Tout" + value { + list { + type: DT_INT32 + } + } + } + attr { + key: "_collective_manager_ids" + value { + list { + } + } + } + attr { + key: "_read_only_resource_inputs" + value { + list { + } + } + } + attr { + key: "config" + value { + s: "" + } + } + attr { + key: "config_proto" + value { + s: "\n\007\n\003CPU\020\001\n\007\n\003GPU\020\0002\002J\0008\001\202\001\000" + } + } + attr { + key: "executor_type" + value { + s: "" + } + } + attr { + key: "f" + value { + func { + name: "__inference_second_func_17" + } + } + } +} +node { + name: "Identity" + op: "Identity" + input: "PartitionedCall" + attr { + key: "T" + value { + type: DT_INT32 + } + } +} +library { + function { + signature { + name: "__inference_second_func_17" + input_arg { + name: "x" + type: DT_INT32 + } + output_arg { + name: "identity" + type: DT_INT32 + } + } + node_def { + name: "Const" + op: "Const" + attr { + key: "dtype" + value { + type: DT_INT32 + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_INT32 + tensor_shape { + } + int_val: 3 + } + } + } + experimental_debug_info { + original_node_names: "Const" + } + } + node_def { + name: "TopKV2" + op: "TopKV2" + input: "x" + input: "Const:output:0" + attr { + key: "T" + value { + type: DT_INT32 + } + } + attr { + key: "sorted" + value { + b: true + } + } + experimental_debug_info { + original_node_names: "TopKV2" + } + } + node_def { + name: "Const_1" + op: "Const" + attr { + key: "dtype" + value { + type: DT_INT32 + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_INT32 + tensor_shape { + } + int_val: 10 + } + } + } + experimental_debug_info { + original_node_names: "Const_1" + } + } + node_def { + name: "Add" + op: "AddV2" + input: "TopKV2:indices:0" + input: "Const_1:output:0" + attr { + key: "T" + value { + type: DT_INT32 + } + } + experimental_debug_info { + original_node_names: "Add" + } + } + node_def { + name: "Identity" + op: "Identity" + input: "Add:z:0" + attr { + key: "T" + value { + type: DT_INT32 + } + } + experimental_debug_info { + original_node_names: "Identity" + } + } + ret { + key: "identity" + value: "Identity:output:0" + } + attr { + key: "_construction_context" + value { + s: "kEagerRuntime" + } + } + arg_attr { + value { + attr { + key: "_output_shapes" + value { + list { + shape { + dim { + size: 5 + } + } + } + } + } + attr { + key: "_user_specified_name" + value { + s: "x" + } + } + } + } + } +} +versions { + producer: 808 + min_consumer: 12 +} diff --git a/src/frontends/tensorflow/tests/test_models/models_pbtxt/partitioned_call2.py b/src/frontends/tensorflow/tests/test_models/models_pbtxt/partitioned_call2.py new file mode 100644 index 00000000000..fafd8ac35bd --- /dev/null +++ b/src/frontends/tensorflow/tests/test_models/models_pbtxt/partitioned_call2.py @@ -0,0 +1,21 @@ +# Copyright (C) 2018-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import tensorflow.compat.v1 as tf + + +@tf.function +def second_func(x): + x = tf.raw_ops.TopKV2(input=x, k=tf.constant(3, tf.int32))[1] + x = tf.add(x, tf.constant(10, tf.int32)) + return x + + +@tf.function +def first_func(x, y): + return second_func(x - y) + + +graph_def = first_func.get_concrete_function(tf.constant([1, 2, 3, 4, 5], dtype=tf.int32), + tf.constant([0, 1, 1, 1, 1], dtype=tf.int32)).graph.as_graph_def() +tf.io.write_graph(graph_def, '.', 'partitioned_call2.pbtxt', as_text=True) diff --git a/src/frontends/tensorflow_common/include/helper_ops/internal_operation.hpp b/src/frontends/tensorflow_common/include/helper_ops/internal_operation.hpp index 2a721dd84c3..f882474c79c 100644 --- a/src/frontends/tensorflow_common/include/helper_ops/internal_operation.hpp +++ b/src/frontends/tensorflow_common/include/helper_ops/internal_operation.hpp @@ -35,6 +35,14 @@ public: "Internal error: the get_input_node method of the fake node decoder is invoked."); } + void get_input_node(size_t input_port_idx, + std::string& producer_name, + size_t& producer_output_port_index, + const OpTypeByName& op_type_by_name) const override { + FRONT_END_OP_CONVERSION_CHECK(false, + "Internal error: the get_input_node method of the fake node decoder is invoked."); + } + const std::string& get_op_type() const override { // this method must not throw an exception since it is used by TF FE FrameworkNode constructor return op_type; diff --git a/src/frontends/tensorflow_lite/src/decoder_flatbuffer.cpp b/src/frontends/tensorflow_lite/src/decoder_flatbuffer.cpp index bbd1d6dd0df..d5db1d27141 100644 --- a/src/frontends/tensorflow_lite/src/decoder_flatbuffer.cpp +++ b/src/frontends/tensorflow_lite/src/decoder_flatbuffer.cpp @@ -32,6 +32,13 @@ void DecoderFlatBuffer::get_input_node(size_t input_port_idx, producer_output_port_index = input_tensor_idx; } +void DecoderFlatBuffer::get_input_node(size_t input_port_idx, + std::string& producer_name, + size_t& producer_output_port_index, + const OpTypeByName& op_type_by_name) const { + FRONT_END_NOT_IMPLEMENTED("get_input_node method with op_type_by_name map is not implemented for TFL FE."); +} + const std::string& DecoderFlatBuffer::get_op_type() const { return m_type; } diff --git a/src/frontends/tensorflow_lite/src/decoder_flatbuffer.h b/src/frontends/tensorflow_lite/src/decoder_flatbuffer.h index 7f655622e93..79dfe027c0b 100644 --- a/src/frontends/tensorflow_lite/src/decoder_flatbuffer.h +++ b/src/frontends/tensorflow_lite/src/decoder_flatbuffer.h @@ -55,6 +55,11 @@ public: void get_input_node(size_t input_port_idx, std::string& producer_name, size_t& producer_output_port_index) const override; + void get_input_node(size_t input_port_idx, + std::string& producer_name, + size_t& producer_output_port_index, + const OpTypeByName& op_type_by_name) const override; + std::string get_output_tensor_name(size_t idx) const; element::Type get_output_tensor_type(size_t idx) const; std::string get_input_tensor_name(size_t idx) const; diff --git a/src/frontends/tensorflow_lite/src/decoder_map.hpp b/src/frontends/tensorflow_lite/src/decoder_map.hpp index bc381afb60a..99923b89bef 100644 --- a/src/frontends/tensorflow_lite/src/decoder_map.hpp +++ b/src/frontends/tensorflow_lite/src/decoder_map.hpp @@ -61,6 +61,19 @@ public: m_decoder->get_input_node(input_port_idx, producer_name, producer_output_port_index); } + /// \brief Get a producer name and its output port index + /// + /// \param input_port_idx Input port index by which data is consumed + /// \param producer_name A producer name + /// \param producer_output_port_index Output port index from which data is generated + /// \param op_type_by_name Map of operation name to their types + void get_input_node(size_t input_port_idx, + std::string& producer_name, + size_t& producer_output_port_index, + const OpTypeByName& op_type_by_name) const override { + FRONT_END_NOT_IMPLEMENTED("get_input_node method with op_type_by_name map is not implemented for TFL FE."); + } + /// \brief Get operation type const std::string& get_op_type() const override { if (m_type.empty())