[TF FE][Telemetry] Collect statistics about operations that fails the conversion (#14736)

* [TF FE] Add error-cause operation event for telemetry

Signed-off-by: Kazantsev, Roman <roman.kazantsev@intel.com>

* Revert unneeded changes

* Apply code-review feedback: replace test with unreal op, use error_cause

* Add test model with unreal Add operation

Signed-off-by: Kazantsev, Roman <roman.kazantsev@intel.com>

src/frontends/tensorflow/src/frontend.cpp

@@ -184,13 +184,16 @@ void FrontEnd::translate_graph(const ov::frontend::InputModel::Ptr& model,
             FRONT_END_OP_CONVERSION_CHECK(translate_map.count(operation_decoder->get_op_type()),
                                           "No translator found for " + operation_decoder->get_op_type() + " node.");
             auto op_fun = &(translate_map[operation_decoder->get_op_type()]);
-            // NodeContext node_context(ng_inputs, operation_decoder, model_inputs);
-            // TODO: Check why NodeContextNew doesn't have ngOutputVector ng_inputs input in constructor
             NodeContext node_context(operation_decoder, ng_inputs);
             // generate OV node output vector using translator for given operation type
             ng_outputs = (*op_fun)(node_context);
         } catch (...) {
             if (fail_fast) {
+                // in case of decode, unsupported operation will be converted to FrameworkNode
+                if (m_telemetry && translate_map.count(operation_decoder->get_op_type()) == 0) {
+                    // send event about which operation is not supported for conversion
+                    m_telemetry->send_event("error_cause", "tf_" + operation_decoder->get_op_type());
+                }
                 // re-throw any exception
                 throw;
             } else {

src/frontends/tensorflow/tests/telemetry.cpp

@@ -4,9 +4,15 @@
 
 #include "telemetry.hpp"
 
+#include <openvino/frontend/exception.hpp>
+
+#include "openvino/frontend/extension/telemetry.hpp"
 #include "tf_utils.hpp"
+#include "utils.hpp"
 
 using namespace ov::frontend;
+using namespace std;
+using namespace std::placeholders;
 
 using TFTelemetryTest = FrontEndTelemetryTest;
 
@@ -38,3 +44,67 @@ INSTANTIATE_TEST_SUITE_P(TFTelemetryTest,
                          FrontEndTelemetryTest,
                          ::testing::Values(getTestData()),
                          FrontEndTelemetryTest::getTestCaseName);
+
+TEST(TFTelemetryTest, test_nonexistent_add) {
+    TelemetryFEParam expected_res;
+    expected_res.m_expected_events = {{
+        std::make_tuple("mo", "op_count", "tf_Placeholder", 1),
+        std::make_tuple("mo", "op_count", "tf_Const", 1),
+        std::make_tuple("mo", "op_count", "tf_Relu", 1),
+        std::make_tuple("mo", "op_count", "tf_Adddd", 1),
+        std::make_tuple("mo", "op_count", "tf_Mul", 1),
+        std::make_tuple("mo", "op_count", "tf_NoOp", 1),
+        // expect error cause event due to operation that fails conversion
+        std::make_tuple("mo", "error_cause", "tf_Adddd", 1),
+    }};
+    FrontEndManager fem;
+    FrontEnd::Ptr frontEnd;
+    InputModel::Ptr inputModel;
+    ASSERT_NO_THROW(frontEnd = fem.load_by_framework(TF_FE));
+    ASSERT_NE(frontEnd, nullptr);
+
+    TelemetryMock m_test_telemetry;
+    std::string category = "mo";
+    auto telemetry_extension =
+        std::make_shared<TelemetryExtension>(category,
+                                             std::bind(&TelemetryMock::send_event, &m_test_telemetry, _1, _2, _3, _4),
+                                             std::bind(&TelemetryMock::send_error, &m_test_telemetry, _1, _2),
+                                             std::bind(&TelemetryMock::send_stack_trace, &m_test_telemetry, _1, _2));
+
+    m_test_telemetry.clear();
+    EXPECT_NO_THROW(frontEnd->add_extension(telemetry_extension));
+
+    auto model_filename = FrontEndTestUtils::make_model_path(string(TEST_TENSORFLOW_MODELS_DIRNAME) +
+                                                             string("nonexistent_add/nonexistent_add.pb"));
+    ASSERT_NO_THROW(inputModel = frontEnd->load(model_filename));
+    ASSERT_NE(inputModel, nullptr);
+    shared_ptr<ngraph::Function> function;
+
+    try {
+        function = frontEnd->convert(inputModel);
+        FAIL() << "Non-existent operation Adddd must not be supported by TF FE.";
+    } catch (const OpConversionFailure& error) {
+        string error_message = error.what();
+        string ref_message = "No translator found for Adddd node.";
+        ASSERT_TRUE(error_message.find(ref_message) != string::npos);
+        ASSERT_EQ(function, nullptr);
+
+        // check telemetry data
+        EXPECT_EQ(m_test_telemetry.m_error_cnt, 0);
+        EXPECT_EQ(m_test_telemetry.m_event_cnt, 7);
+        EXPECT_EQ(m_test_telemetry.m_trace_cnt, 0);
+        bool is_found = false;
+        for (const auto m_expected_events : expected_res.m_expected_events) {
+            is_found = false;
+            is_found = (m_test_telemetry.m_event_cnt == m_expected_events.size()) &&
+                       (m_test_telemetry.m_received_events == m_expected_events);
+            if (is_found) {
+                break;
+            }
+        }
+        EXPECT_TRUE(is_found) << "Unexpected set of operations received from telemetry.";
+        m_test_telemetry.clear();
+    } catch (...) {
+        FAIL() << "Conversion of Non-existent operation Adddd failed by wrong reason.";
+    }
+}

src/frontends/tensorflow/tests/test_models/gen_scripts/generate_nonexistent_add.py

@@ -0,0 +1,40 @@
+# Copyright (C) 2018-2022 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+#
+# unsupported add tensorflow model generator
+#
+
+import numpy as np
+import os
+import sys
+import tensorflow as tf
+
+
+def main():
+    tf.compat.v1.reset_default_graph()
+
+    # Create the graph and model
+    with tf.compat.v1.Session() as sess:
+        const2 = tf.constant(2.0, dtype=tf.float32)
+        x = tf.compat.v1.placeholder(dtype=tf.float32, shape=[2, 3], name='x')
+        relu = tf.nn.relu(x)    
+        add = tf.add(relu, const2, name="add")
+        tf.multiply(add, relu)
+
+        tf.compat.v1.global_variables_initializer()
+        tf_net = sess.graph_def
+
+    tf.io.write_graph(tf_net, os.path.join(sys.argv[1], "nonexistent_add"), "nonexistent_add.pb", False)
+
+    with open(os.path.join(sys.argv[1], "nonexistent_add", "nonexistent_add.pb"), mode='rb') as file:
+        modelContent = file.read()
+
+    modelContent = modelContent.replace(b"AddV2", b"Adddd")
+
+    with open(os.path.join(sys.argv[1], "nonexistent_add", "nonexistent_add.pb"), mode='wb') as file:
+        file.write(modelContent)
+
+
+if __name__ == "__main__":
+    main()