[conformanceTests] Add key for manage pipeline after crashes (#16123)

* [conformanceTests] Add key for manage pipeline after crashes

* Move crash_handler to funcTestsUtils

src/tests/functional/plugin/conformance/test_runner/README.md

@@ -129,6 +129,7 @@ The target is able to take the following command-line arguments:
 * `--shape_mode` is optional. It allows you to run `static`, `dynamic` , or both scenarios. The default value is an empty string, which allows running both scenarios. Possible values
   are `static`, `dynamic`, ``
 * `--test_timeout` specifies setup timeout for each test in seconds. The default timeout is 900 seconds (15 minutes).
+* `--ignore_crash` Optional. Allow to not terminate the whole run after crash and continue execution from the next test. This is organized with custom crash handler. Please, note, that handler work for test body,  if crash happened on SetUp/TearDown stage, the process will be terminated.
 * All `gtest` command-line parameters
 
 > **NOTE**:

src/tests/functional/plugin/conformance/test_runner/conformance_infra/include/gflag_config.hpp

@@ -46,6 +46,9 @@ static const char extract_body_message[] = "Optional. Allows to count extracted
 static const char shape_mode_message[] = "Optional. Allows to run `static`, `dynamic` or both scenarios. Default value is empty string allows to run both"
                                          " scenarios. Possible values are `static`, `dynamic`, ``";
 static const char test_timeout_message[] = "Optional. Setup timeout for each test in seconds, default timeout 900seconds (15 minutes).";
+static const char ignore_crash_message[] = "Optional. Allow to not terminate the whole run after crash and continue execution from the next test."
+                                           "This is organized with custom crash handler. Please, note, that handler work for test body,"
+                                           "if crash happened on SetUp/TearDown stage, the process will be terminated.";
 static const char reference_cache_dir_message[] = "Optional. Set the directory with reference cache";
 
 
@@ -63,6 +66,7 @@ DEFINE_bool(report_unique_name, false, report_unique_name_message);
 DEFINE_bool(extract_body, false, extract_body_message);
 DEFINE_string(shape_mode, "", shape_mode_message);
 DEFINE_uint32(test_timeout, UINT_MAX, test_timeout_message);
+DEFINE_uint32(ignore_crash, false, ignore_crash_message);
 DEFINE_string(ref_dir, "", reference_cache_dir_message);
 
 /**
@@ -87,6 +91,7 @@ static void showUsage() {
     std::cout << "    --plugin_lib_name                " << output_folder_message << std::endl;
     std::cout << "    --shape_mode  \"<value>\"          " << shape_mode_message << std::endl;
     std::cout << "    --test_timeout  \"<value>\"        " << test_timeout_message << std::endl;
+    std::cout << "    --ignore_crash                     " << ignore_crash_message << std::endl;
     std::cout << "    --ref_dir  \"<paths>\"             " << reference_cache_dir_message << std::endl;
 }
 

src/tests/functional/plugin/conformance/test_runner/conformance_infra/src/main.cpp

@@ -17,7 +17,7 @@
 #include "gflag_config.hpp"
 #include "conformance.hpp"
 
-#include "common_test_utils/crash_handler.hpp"
+#include "functional_test_utils/crash_handler.hpp"
 
 using namespace ov::test::conformance;
 
@@ -63,6 +63,7 @@ int main(int argc, char* argv[]) {
     }
 
     CommonTestUtils::CrashHandler::SetUpTimeout(FLAGS_test_timeout);
+    CommonTestUtils::CrashHandler::SetUpPipelineAfterCrash(FLAGS_ignore_crash);
 
     // ---------------------------Initialization of Gtest env -----------------------------------------------
     ov::test::conformance::targetDevice = FLAGS_device.c_str();

src/tests/functional/plugin/conformance/test_runner/conformance_infra/src/read_ir_test/read_ir.cpp

@@ -13,7 +13,7 @@
 #include "common_test_utils/file_utils.hpp"
 #include "common_test_utils/data_utils.hpp"
 #include "common_test_utils/common_utils.hpp"
-#include "common_test_utils/crash_handler.hpp"
+#include "functional_test_utils/crash_handler.hpp"
 #include "functional_test_utils/summary/op_info.hpp"
 #include "functional_test_utils/skip_tests_config.hpp"
 

src/tests/functional/plugin/shared/include/base/ov_behavior_test_utils.hpp

@@ -18,7 +18,7 @@
 #include "common_test_utils/test_common.hpp"
 #include "common_test_utils/test_constants.hpp"
 #include "common_test_utils/common_utils.hpp"
-#include "common_test_utils/crash_handler.hpp"
+#include "functional_test_utils/crash_handler.hpp"
 #include "common_test_utils/file_utils.hpp"
 
 #include "functional_test_utils/plugin_cache.hpp"

src/tests/functional/plugin/shared/include/behavior/plugin/life_time.hpp

@@ -15,7 +15,7 @@
 #include <common_test_utils/test_constants.hpp>
 #include <cpp/ie_cnn_network.h>
 #include "gtest/gtest.h"
-#include "common_test_utils/crash_handler.hpp"
+#include "functional_test_utils/crash_handler.hpp"
 #include "functional_test_utils/skip_tests_config.hpp"
 #include "functional_test_utils/precision_utils.hpp"
 #include "base/behavior_test_utils.hpp"

src/tests/functional/plugin/shared/include/single_layer_tests/op_impl_check/op_impl_check_compile_model.hpp

@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
-#include "common_test_utils/crash_handler.hpp"
+#include "functional_test_utils/crash_handler.hpp"
 #include "single_layer_tests/op_impl_check/op_impl_check.hpp"
 
 namespace ov {

src/tests/functional/plugin/shared/include/single_layer_tests/op_impl_check/op_impl_check_query_model.hpp

@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
-#include "common_test_utils/crash_handler.hpp"
+#include "functional_test_utils/crash_handler.hpp"
 #include "single_layer_tests/op_impl_check/op_impl_check.hpp"
 
 namespace ov {

src/tests/functional/plugin/shared/src/single_layer_tests/op_impl_check/op_impl_check.cpp

@@ -7,7 +7,7 @@
 #endif
 
 #include "single_layer_tests/op_impl_check/op_impl_check.hpp"
-#include "common_test_utils/crash_handler.hpp"
+#include "functional_test_utils/crash_handler.hpp"
 
 namespace ov {
 namespace test {

src/tests/functional/shared_test_classes/include/shared_test_classes/base/layer_test_utils.hpp

@@ -21,8 +21,8 @@
 #include "common_test_utils/ngraph_test_utils.hpp"
 #include "common_test_utils/common_utils.hpp"
 #include "common_test_utils/test_common.hpp"
-#include "common_test_utils/crash_handler.hpp"
 
+#include "functional_test_utils/crash_handler.hpp"
 #include "functional_test_utils/skip_tests_config.hpp"
 #include "functional_test_utils/plugin_cache.hpp"
 #include "functional_test_utils/blob_utils.hpp"

src/tests/functional/shared_test_classes/src/base/ov_subgraph.cpp

@@ -23,7 +23,7 @@
 #include "ngraph_functions/utils/ngraph_helpers.hpp"
 
 #include "common_test_utils/file_utils.hpp"
-#include "common_test_utils/crash_handler.hpp"
+#include "functional_test_utils/crash_handler.hpp"
 #include "common_test_utils/ov_tensor_utils.hpp"
 #include "functional_test_utils/skip_tests_config.hpp"
 

src/tests/ie_test_utils/functional_test_utils/include/functional_test_utils/crash_handler.hpp

@@ -6,7 +6,7 @@
 
 #include <gtest/gtest.h>
 
-#include "common_utils.hpp"
+#include "common_test_utils/common_utils.hpp"
 
 #include <signal.h>
 #include <setjmp.h>
@@ -20,10 +20,12 @@ enum JMP_STATUS { ok = 0, anyError = 1, alarmErr = 2 };
 class CrashHandler {
 private:
     static unsigned int MAX_TEST_WORK_TIME;
+    static bool IGNORE_CRASH;
 public:
     CrashHandler();
     ~CrashHandler();
     static void SetUpTimeout(unsigned int timeout);
+    static void SetUpPipelineAfterCrash(bool ignore_crash);
     void StartTimer();
 };
 

src/tests/ie_test_utils/functional_test_utils/src/crash_handler.cpp

@@ -2,7 +2,9 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
-#include "crash_handler.hpp"
+#include "functional_test_utils/summary/op_summary.hpp"
+
+#include "functional_test_utils/crash_handler.hpp"
 #include <limits.h>
 
 namespace CommonTestUtils {
@@ -10,6 +12,7 @@ namespace CommonTestUtils {
 // enviroment to restore in case of crash
 jmp_buf env;
 unsigned int CrashHandler::MAX_TEST_WORK_TIME = UINT_MAX;
+bool CrashHandler::IGNORE_CRASH = false;
 
 CrashHandler::CrashHandler() {
     // setup default value for timeout in 15 minutes
@@ -31,6 +34,12 @@ CrashHandler::CrashHandler() {
         signal(SIGALRM, SIG_DFL);
 #endif
 
+        if (!CrashHandler::IGNORE_CRASH) {
+            auto &s = ov::test::utils::OpSummary::getInstance();
+            s.saveReport();
+            std::abort();
+        }
+
 #ifdef _WIN32
         longjmp(env, JMP_STATUS::anyError);
 #else
@@ -84,4 +93,8 @@ void CrashHandler::SetUpTimeout(unsigned int timeout) {
     MAX_TEST_WORK_TIME = timeout;
 }
 
+void CrashHandler::SetUpPipelineAfterCrash(bool ignore_crash) {
+    IGNORE_CRASH = ignore_crash;
+}
+
 }  // namespace CommonTestUtils