[IE TEST] Continue run after crash (#10037)

src/tests/functional/plugin/conformance/test_runner/conformance_infra/src/main.cpp

@@ -59,5 +59,24 @@ int main(int argc, char* argv[]) {
 
     ::testing::InitGoogleTest(&argc, argv);
     ::testing::AddGlobalTestEnvironment(new LayerTestsUtils::TestEnvironment);
+
+    auto exernalSignalHandler = [](int errCode) {
+        std::cerr << "Unexpected application crash with code: " << errCode << std::endl;
+
+        // set default handler for crash
+        signal(SIGINT, SIG_DFL);
+        signal(SIGTERM, SIG_DFL);
+
+        if (errCode == SIGINT || errCode == SIGTERM) {
+            auto& s = LayerTestsUtils::Summary::getInstance();
+            s.saveReport();
+            exit(1);
+        }
+    };
+
+    // killed by extarnal
+    signal(SIGINT, exernalSignalHandler);
+    signal(SIGTERM , exernalSignalHandler);
+
     return RUN_ALL_TESTS();
 }

src/tests/functional/plugin/conformance/test_runner/conformance_infra/src/read_ir_test/read_ir.cpp

@@ -13,11 +13,14 @@
 #include "common_test_utils/file_utils.hpp"
 #include "common_test_utils/data_utils.hpp"
 #include "common_test_utils/common_utils.hpp"
+#include "common_test_utils/crash_handler.hpp"
 #include "functional_test_utils/layer_test_utils/op_info.hpp"
 #include "functional_test_utils/skip_tests_config.hpp"
 
 #include "read_ir_test/read_ir.hpp"
 
+#include <setjmp.h>
+
 namespace ov {
 namespace test {
 namespace subgraph {
@@ -48,17 +51,19 @@ std::string ReadIRTest::getTestCaseName(const testing::TestParamInfo<ReadIRParam
 }
 
 void ReadIRTest::query_model() {
+    // in case of crash jump will be made and work will be continued
+    auto crashHandler = std::unique_ptr<CommonTestUtils::CrashHandler>(new CommonTestUtils::CrashHandler());
+
+    // place to jump in case of a crash
+#ifdef _WIN32
+    if (setjmp(CommonTestUtils::env) == 0) {
+#else
+    if (sigsetjmp(CommonTestUtils::env, 1) == 0) {
+#endif
         if (functionRefs == nullptr) {
             functionRefs = ngraph::clone_function(*function);
             functionRefs->set_friendly_name("refFunction");
         }
-    auto crashHandler = [](int errCode) {
-        auto &s = LayerTestsUtils::Summary::getInstance();
-        s.saveReport();
-        std::cout << "Unexpected application crash!" << std::endl;
-        std::abort();
-    };
-    signal(SIGSEGV, crashHandler);
 
         auto &s = LayerTestsUtils::Summary::getInstance();
         s.setDeviceName(targetDevice);
@@ -75,9 +80,21 @@ void ReadIRTest::query_model() {
         } catch (...) {
             s.updateOPsStats(functionRefs, LayerTestsUtils::PassRate::Statuses::FAILED);
         }
+    } else {
+        IE_THROW() << "Crash happens";
+    }
 }
 
 void ReadIRTest::SetUp() {
+    // in case of crash jump will be made and work will be continued
+    auto crashHandler = std::unique_ptr<CommonTestUtils::CrashHandler>(new CommonTestUtils::CrashHandler());
+
+    // place to jump in case of a crash
+#ifdef _WIN32
+    if (setjmp(CommonTestUtils::env) == 0) {
+#else
+    if (sigsetjmp(CommonTestUtils::env, 1) == 0) {
+#endif
         std::tie(pathToModel, targetDevice, configuration) = this->GetParam();
         function = core->read_model(pathToModel);
         const auto metaFile = CommonTestUtils::replaceExt(pathToModel, "meta");
@@ -141,6 +158,14 @@ void ReadIRTest::SetUp() {
                 }
             }
         }
+        std::vector<ov::Shape> staticShapes;
+        for (const auto param : function->get_parameters()) {
+            if (param->get_partial_shape().is_static()) {
+                staticShapes.push_back(param->get_shape());
+            } else {
+                staticShapes.push_back(param->get_partial_shape().get_max_shape());
+            }
+        }
         std::vector<InputShape> inputShapes;
         for (const auto& param : function -> get_parameters()) {
             if (param->get_partial_shape().is_static()) {
@@ -160,6 +185,9 @@ void ReadIRTest::SetUp() {
             }
         }
         init_input_shapes(inputShapes);
+    } else {
+        IE_THROW() << "Crash happens";
+    }
 }
 
 } // namespace subgraph

src/tests/functional/plugin/shared/include/behavior/plugin/life_time.hpp

@@ -16,6 +16,7 @@
 #include <cpp/ie_cnn_network.h>
 #include "gtest/gtest.h"
 #include "common_test_utils/test_common.hpp"
+#include "common_test_utils/crash_handler.hpp"
 #include "functional_test_utils/skip_tests_config.hpp"
 #include "functional_test_utils/precision_utils.hpp"
 #include <ie_core.hpp>

src/tests/functional/plugin/shared/src/behavior/plugin/life_time.cpp

@@ -8,17 +8,9 @@
 #include <base/behavior_test_utils.hpp>
 #include "behavior/plugin/life_time.hpp"
 
-#ifndef _WIN32
+#include <setjmp.h>
-    #include <signal.h>
-    #include <setjmp.h>
-#endif
 
 namespace BehaviorTestsDefinitions {
-
-#ifndef _WIN32
-    static jmp_buf env;
-#endif
-
     std::string HoldersTest::getTestCaseName(testing::TestParamInfo<HoldersParams> obj) {
         std::string targetDevice;
         std::vector<int> order;
@@ -38,19 +30,6 @@ namespace BehaviorTestsDefinitions {
         SKIP_IF_CURRENT_TEST_IS_DISABLED();
         std::tie(targetDevice, order) = this->GetParam();
         function = ngraph::builder::subgraph::makeConvPoolRelu();
-
-#ifndef _WIN32
-        // configure handling of crash
-        auto crashHandler = [](int errCode) {
-            std::cerr << "Unexpected application crash with code: " << errCode << std::endl;
-            siglongjmp(env, 1);
-        };
-        struct sigaction act;
-        act.sa_handler = crashHandler;
-        sigemptyset(&act.sa_mask);
-        act.sa_flags = 0;
-        sigaction(SIGSEGV, &act, 0);
-#endif
     }
 
     void release_order_test(std::vector<int> order, const std::string &deviceName,
@@ -90,29 +69,35 @@ namespace BehaviorTestsDefinitions {
     }
 
     TEST_P(HoldersTest, Orders) {
+        // in case of crash jump will be made and work will be continued
+        auto crashHandler = std::unique_ptr<CommonTestUtils::CrashHandler>(new CommonTestUtils::CrashHandler());
+
         // Test failed if crash happens
 #ifdef _WIN32
-        EXPECT_NO_THROW(release_order_test(order, targetDevice, function));
+        if (setjmp(CommonTestUtils::env) == 0) {
 #else
-        if (sigsetjmp(env, 1) == 0) {
+        if (sigsetjmp(CommonTestUtils::env, 1) == 0) {
-            release_order_test(order, targetDevice, function);
+#endif
+            EXPECT_NO_THROW(release_order_test(order, targetDevice, function));
         } else {
             IE_THROW() << "Crash happens";
         }
-#endif
     }
 
     TEST_P(HoldersTestImportNetwork, Orders) {
+        // in case of crash jump will be made and work will be continued
+        auto crashHandler = std::unique_ptr<CommonTestUtils::CrashHandler>(new CommonTestUtils::CrashHandler());
+
         // Test failed if crash happens
 #ifdef _WIN32
-        EXPECT_NO_THROW(release_order_test(order, targetDevice, function));
+        if (setjmp(CommonTestUtils::env) == 0) {
 #else
-        if (sigsetjmp(env, 1) == 0) {
+        if (sigsetjmp(CommonTestUtils::env, 1) == 0) {
-            release_order_test(order, targetDevice, function);
+#endif
+            EXPECT_NO_THROW(release_order_test(order, targetDevice, function));
         } else {
             IE_THROW() << "Crash happens";
         }
-#endif
     }
 
     std::string HoldersTestOnImportedNetwork::getTestCaseName(testing::TestParamInfo<std::string> obj) {

src/tests/functional/plugin/shared/src/single_layer_tests/op_impl_check/op_impl_check.cpp

@@ -7,6 +7,7 @@
 #endif
 
 #include "single_layer_tests/op_impl_check/op_impl_check.hpp"
+#include "common_test_utils/crash_handler.hpp"
 
 namespace ov {
 namespace test {
@@ -16,14 +17,16 @@ void OpImplCheckTest::run() {
     if (function == nullptr) {
         GTEST_FAIL() << "Target function is empty!";
     }
-    auto crashHandler = [](int errCode) {
-        auto& s = LayerTestsUtils::Summary::getInstance();
-        s.saveReport();
-        std::cerr << "Unexpected application crash with code: " << errCode << std::endl;
-        std::abort();
-    };
-    signal(SIGSEGV, crashHandler);
 
+    // in case of crash jump will be made and work will be continued
+    auto crashHandler = std::unique_ptr<CommonTestUtils::CrashHandler>(new CommonTestUtils::CrashHandler());
+
+    // place to jump in case of a crash
+#ifdef _WIN32
+    if (setjmp(CommonTestUtils::env) == 0) {
+#else
+    if (sigsetjmp(CommonTestUtils::env, 1) == 0) {
+#endif
         summary.setDeviceName(targetDevice);
         try {
             auto executableNetwork = core->compile_model(function, targetDevice, configuration);
@@ -32,6 +35,9 @@ void OpImplCheckTest::run() {
             summary.updateOPsImplStatus(function, false);
             GTEST_FAIL() << "Error in the LoadNetwork!";
         }
+    } else {
+        IE_THROW() << "Crash happens";
+    }
 }
 
 void OpImplCheckTest::SetUp() {

src/tests/functional/shared_test_classes/include/shared_test_classes/base/layer_test_utils.hpp

@@ -21,6 +21,7 @@
 #include "common_test_utils/ngraph_test_utils.hpp"
 #include "common_test_utils/common_utils.hpp"
 #include "common_test_utils/test_common.hpp"
+#include "common_test_utils/crash_handler.hpp"
 
 #include "functional_test_utils/skip_tests_config.hpp"
 #include "functional_test_utils/plugin_cache.hpp"

src/tests/functional/shared_test_classes/include/shared_test_classes/base/ov_subgraph.hpp

@@ -26,12 +26,6 @@ public:
     virtual void serialize();
     virtual void query_model();
 
-    void TearDown() override {
-        if (!configuration.empty()) {
-            ov::test::utils::PluginCache::get().core().reset();
-        }
-    }
-
 protected:
     virtual void compare(const std::vector<ov::Tensor> &expected,
                          const std::vector<ov::Tensor> &actual);

src/tests/functional/shared_test_classes/src/base/layer_test_utils.cpp

@@ -30,14 +30,16 @@ void LayerTestsCommon::Run() {
         functionRefs = ngraph::clone_function(*function);
         functionRefs->set_friendly_name("refFunction");
     }
-    auto crashHandler = [](int errCode) {
-        auto &s = Summary::getInstance();
-        s.saveReport();
-        std::cout << "Unexpected application crash!" << std::endl;
-        std::abort();
-    };
-    signal(SIGSEGV, crashHandler);
 
+    // in case of crash jump will be made and work will be continued
+    auto crashHandler = std::unique_ptr<CommonTestUtils::CrashHandler>(new CommonTestUtils::CrashHandler());
+
+    // place to jump in case of a crash
+#ifdef _WIN32
+    if (setjmp(CommonTestUtils::env) == 0) {
+#else
+    if (sigsetjmp(CommonTestUtils::env, 1) == 0) {
+#endif
         auto &s = Summary::getInstance();
         s.setDeviceName(targetDevice);
 
@@ -65,6 +67,9 @@ void LayerTestsCommon::Run() {
             s.updateOPsStats(functionRefs, PassRate::Statuses::FAILED);
             GTEST_FATAL_FAILURE_("Unknown failure occurred.");
         }
+    } else {
+        IE_THROW() << "Crash happens";
+    }
 }
 
 void LayerTestsCommon::Serialize(ngraph::pass::Serialize::Version ir_version) {

src/tests/functional/shared_test_classes/src/base/ov_subgraph.cpp

@@ -18,6 +18,7 @@
 #include "ngraph_functions/utils/ngraph_helpers.hpp"
 
 #include "common_test_utils/file_utils.hpp"
+#include "common_test_utils/crash_handler.hpp"
 #include "functional_test_utils/ov_tensor_utils.hpp"
 #include "functional_test_utils/skip_tests_config.hpp"
 
@@ -25,6 +26,8 @@
 #include "shared_test_classes/base/utils/generate_inputs.hpp"
 #include "shared_test_classes/base/utils/compare_results.hpp"
 
+#include <setjmp.h>
+
 namespace ov {
 namespace test {
 
@@ -34,14 +37,15 @@ std::ostream& operator <<(std::ostream& os, const InputShape& inputShape) {
 }
 
 void SubgraphBaseTest::run() {
-    auto crashHandler = [](int errCode) {
+    // in case of crash jump will be made and work will be continued
-        auto& s = LayerTestsUtils::Summary::getInstance();
+    auto crashHandler = std::unique_ptr<CommonTestUtils::CrashHandler>(new CommonTestUtils::CrashHandler());
-        s.saveReport();
-        std::cerr << "Unexpected application crash with code: " << errCode << std::endl;
-        std::abort();
-    };
-    signal(SIGSEGV, crashHandler);
 
+    // place to jump in case of a crash
+#ifdef _WIN32
+    if (setjmp(CommonTestUtils::env) == 0) {
+#else
+    if (sigsetjmp(CommonTestUtils::env, 1) == 0) {
+#endif
         LayerTestsUtils::PassRate::Statuses status = FuncTestUtils::SkipTestsConfig::currentTestIsDisabled()
                                                         ? LayerTestsUtils::PassRate::Statuses::SKIPPED
                                                         : LayerTestsUtils::PassRate::Statuses::CRASHED;
@@ -81,6 +85,9 @@ void SubgraphBaseTest::run() {
         if (status != LayerTestsUtils::PassRate::Statuses::PASSED) {
             GTEST_FATAL_FAILURE_(errorMessage.c_str());
         }
+    } else {
+        IE_THROW() << "Crash happens";
+    }
 }
 
 void SubgraphBaseTest::serialize() {

src/tests/ie_test_utils/common_test_utils/crash_handler.cpp

@@ -0,0 +1,55 @@
+// Copyright (C) 2022 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#include "crash_handler.hpp"
+
+namespace CommonTestUtils {
+
+// enviroment to restore in case of crash
+jmp_buf env;
+
+CrashHandler::CrashHandler() {
+    auto crashHandler = [](int errCode) {
+        std::cerr << "Unexpected application crash with code: " << errCode << std::endl;
+
+        // reset custom signal handler to avoid infinit loop
+        // if for some reasons sigsetjmp will not be available
+        signal(SIGABRT, SIG_DFL);
+        signal(SIGSEGV, SIG_DFL);
+        signal(SIGILL, SIG_DFL);
+#ifndef _WIN32
+        signal(SIGBUS, SIG_DFL);
+        signal(SIGFPE, SIG_DFL);
+#endif
+
+        // goto sigsetjmp
+#ifdef _WIN32
+        longjmp(env, 1);
+#else
+        siglongjmp(env, 1);
+#endif
+    };
+
+    // setup custom handler for signals
+    signal(SIGABRT, crashHandler);
+    signal(SIGSEGV, crashHandler);
+    signal(SIGILL, crashHandler);
+#ifndef _WIN32
+    signal(SIGFPE, crashHandler);
+    signal(SIGBUS, crashHandler);
+#endif
+}
+
+CrashHandler::~CrashHandler() {
+    // reset custom signal handler to avoid infinit loop
+    signal(SIGABRT, SIG_DFL);
+    signal(SIGSEGV, SIG_DFL);
+    signal(SIGILL, SIG_DFL);
+#ifndef _WIN32
+    signal(SIGFPE, SIG_DFL);
+    signal(SIGBUS, SIG_DFL);
+#endif
+}
+
+}  // namespace CommonTestUtils

src/tests/ie_test_utils/common_test_utils/crash_handler.hpp

@@ -0,0 +1,24 @@
+// Copyright (C) 2022 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#pragma once
+
+#include <gtest/gtest.h>
+
+#include "common_utils.hpp"
+
+#include <signal.h>
+#include <setjmp.h>
+
+namespace CommonTestUtils {
+
+extern jmp_buf env;
+
+class CrashHandler {
+public:
+    CrashHandler();
+    ~CrashHandler();
+};
+
+}  // namespace CommonTestUtils