From e7b510376ac01d353d20251e2406c934934334d6 Mon Sep 17 00:00:00 2001 From: Roman Kazantsev Date: Wed, 21 Dec 2022 10:09:25 +0400 Subject: [PATCH] [TF FE][Telemetry] Collect statistics about operations that fails the conversion (#14736) * [TF FE] Add error-cause operation event for telemetry Signed-off-by: Kazantsev, Roman * Revert unneeded changes * Apply code-review feedback: replace test with unreal op, use error_cause * Add test model with unreal Add operation Signed-off-by: Kazantsev, Roman --- src/frontends/tensorflow/src/frontend.cpp | 7 +- src/frontends/tensorflow/tests/telemetry.cpp | 70 +++++++++++++++++++ .../gen_scripts/generate_nonexistent_add.py | 40 +++++++++++ 3 files changed, 115 insertions(+), 2 deletions(-) create mode 100644 src/frontends/tensorflow/tests/test_models/gen_scripts/generate_nonexistent_add.py diff --git a/src/frontends/tensorflow/src/frontend.cpp b/src/frontends/tensorflow/src/frontend.cpp index fcc8e5c27d1..5bfa261b055 100644 --- a/src/frontends/tensorflow/src/frontend.cpp +++ b/src/frontends/tensorflow/src/frontend.cpp @@ -184,13 +184,16 @@ void FrontEnd::translate_graph(const ov::frontend::InputModel::Ptr& model, FRONT_END_OP_CONVERSION_CHECK(translate_map.count(operation_decoder->get_op_type()), "No translator found for " + operation_decoder->get_op_type() + " node."); auto op_fun = &(translate_map[operation_decoder->get_op_type()]); - // NodeContext node_context(ng_inputs, operation_decoder, model_inputs); - // TODO: Check why NodeContextNew doesn't have ngOutputVector ng_inputs input in constructor NodeContext node_context(operation_decoder, ng_inputs); // generate OV node output vector using translator for given operation type ng_outputs = (*op_fun)(node_context); } catch (...) { if (fail_fast) { + // in case of decode, unsupported operation will be converted to FrameworkNode + if (m_telemetry && translate_map.count(operation_decoder->get_op_type()) == 0) { + // send event about which operation is not supported for conversion + m_telemetry->send_event("error_cause", "tf_" + operation_decoder->get_op_type()); + } // re-throw any exception throw; } else { diff --git a/src/frontends/tensorflow/tests/telemetry.cpp b/src/frontends/tensorflow/tests/telemetry.cpp index 111545a3955..88d6eb1f775 100644 --- a/src/frontends/tensorflow/tests/telemetry.cpp +++ b/src/frontends/tensorflow/tests/telemetry.cpp @@ -4,9 +4,15 @@ #include "telemetry.hpp" +#include + +#include "openvino/frontend/extension/telemetry.hpp" #include "tf_utils.hpp" +#include "utils.hpp" using namespace ov::frontend; +using namespace std; +using namespace std::placeholders; using TFTelemetryTest = FrontEndTelemetryTest; @@ -38,3 +44,67 @@ INSTANTIATE_TEST_SUITE_P(TFTelemetryTest, FrontEndTelemetryTest, ::testing::Values(getTestData()), FrontEndTelemetryTest::getTestCaseName); + +TEST(TFTelemetryTest, test_nonexistent_add) { + TelemetryFEParam expected_res; + expected_res.m_expected_events = {{ + std::make_tuple("mo", "op_count", "tf_Placeholder", 1), + std::make_tuple("mo", "op_count", "tf_Const", 1), + std::make_tuple("mo", "op_count", "tf_Relu", 1), + std::make_tuple("mo", "op_count", "tf_Adddd", 1), + std::make_tuple("mo", "op_count", "tf_Mul", 1), + std::make_tuple("mo", "op_count", "tf_NoOp", 1), + // expect error cause event due to operation that fails conversion + std::make_tuple("mo", "error_cause", "tf_Adddd", 1), + }}; + FrontEndManager fem; + FrontEnd::Ptr frontEnd; + InputModel::Ptr inputModel; + ASSERT_NO_THROW(frontEnd = fem.load_by_framework(TF_FE)); + ASSERT_NE(frontEnd, nullptr); + + TelemetryMock m_test_telemetry; + std::string category = "mo"; + auto telemetry_extension = + std::make_shared(category, + std::bind(&TelemetryMock::send_event, &m_test_telemetry, _1, _2, _3, _4), + std::bind(&TelemetryMock::send_error, &m_test_telemetry, _1, _2), + std::bind(&TelemetryMock::send_stack_trace, &m_test_telemetry, _1, _2)); + + m_test_telemetry.clear(); + EXPECT_NO_THROW(frontEnd->add_extension(telemetry_extension)); + + auto model_filename = FrontEndTestUtils::make_model_path(string(TEST_TENSORFLOW_MODELS_DIRNAME) + + string("nonexistent_add/nonexistent_add.pb")); + ASSERT_NO_THROW(inputModel = frontEnd->load(model_filename)); + ASSERT_NE(inputModel, nullptr); + shared_ptr function; + + try { + function = frontEnd->convert(inputModel); + FAIL() << "Non-existent operation Adddd must not be supported by TF FE."; + } catch (const OpConversionFailure& error) { + string error_message = error.what(); + string ref_message = "No translator found for Adddd node."; + ASSERT_TRUE(error_message.find(ref_message) != string::npos); + ASSERT_EQ(function, nullptr); + + // check telemetry data + EXPECT_EQ(m_test_telemetry.m_error_cnt, 0); + EXPECT_EQ(m_test_telemetry.m_event_cnt, 7); + EXPECT_EQ(m_test_telemetry.m_trace_cnt, 0); + bool is_found = false; + for (const auto m_expected_events : expected_res.m_expected_events) { + is_found = false; + is_found = (m_test_telemetry.m_event_cnt == m_expected_events.size()) && + (m_test_telemetry.m_received_events == m_expected_events); + if (is_found) { + break; + } + } + EXPECT_TRUE(is_found) << "Unexpected set of operations received from telemetry."; + m_test_telemetry.clear(); + } catch (...) { + FAIL() << "Conversion of Non-existent operation Adddd failed by wrong reason."; + } +} diff --git a/src/frontends/tensorflow/tests/test_models/gen_scripts/generate_nonexistent_add.py b/src/frontends/tensorflow/tests/test_models/gen_scripts/generate_nonexistent_add.py new file mode 100644 index 00000000000..2f4d2f5d3d5 --- /dev/null +++ b/src/frontends/tensorflow/tests/test_models/gen_scripts/generate_nonexistent_add.py @@ -0,0 +1,40 @@ +# Copyright (C) 2018-2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +# +# unsupported add tensorflow model generator +# + +import numpy as np +import os +import sys +import tensorflow as tf + + +def main(): + tf.compat.v1.reset_default_graph() + + # Create the graph and model + with tf.compat.v1.Session() as sess: + const2 = tf.constant(2.0, dtype=tf.float32) + x = tf.compat.v1.placeholder(dtype=tf.float32, shape=[2, 3], name='x') + relu = tf.nn.relu(x) + add = tf.add(relu, const2, name="add") + tf.multiply(add, relu) + + tf.compat.v1.global_variables_initializer() + tf_net = sess.graph_def + + tf.io.write_graph(tf_net, os.path.join(sys.argv[1], "nonexistent_add"), "nonexistent_add.pb", False) + + with open(os.path.join(sys.argv[1], "nonexistent_add", "nonexistent_add.pb"), mode='rb') as file: + modelContent = file.read() + + modelContent = modelContent.replace(b"AddV2", b"Adddd") + + with open(os.path.join(sys.argv[1], "nonexistent_add", "nonexistent_add.pb"), mode='wb') as file: + file.write(modelContent) + + +if __name__ == "__main__": + main()