diff --git a/src/tests/functional/plugin/conformance/test_runner/README.md b/src/tests/functional/plugin/conformance/test_runner/README.md index 1e56d86461d..4360b3cbd44 100644 --- a/src/tests/functional/plugin/conformance/test_runner/README.md +++ b/src/tests/functional/plugin/conformance/test_runner/README.md @@ -47,6 +47,7 @@ The target is able to take the following command-line arguments: * `--extract_body` allows to count extracted operation bodies to report. * `--shape_mode` Optional. Allows to run `static`, `dynamic` or both scenarios. Default value is empty string allows to run both scenarios. Possible values are `static`, `dynamic`, `` +* `--test_timeout` Setup timeout for each test in seconds, default timeout 900seconds (15 minutes). * All `gtest` command-line parameters The result of execution is `report.xml` file. It demonstrates tests statistic like pass rate, passed, crashed, skipped failed tests and plugin implementation diff --git a/src/tests/functional/plugin/conformance/test_runner/conformance_infra/include/gflag_config.hpp b/src/tests/functional/plugin/conformance/test_runner/conformance_infra/include/gflag_config.hpp index 836bf0a0c4f..632ba6fd63b 100644 --- a/src/tests/functional/plugin/conformance/test_runner/conformance_infra/include/gflag_config.hpp +++ b/src/tests/functional/plugin/conformance/test_runner/conformance_infra/include/gflag_config.hpp @@ -6,6 +6,7 @@ #include #include +#include namespace ov { namespace test { @@ -35,6 +36,7 @@ static const char config_path_message[] = "Optional. Allows to specify path to f static const char extract_body_message[] = "Optional. Allows to count extracted operation bodies to report. Default value is false."; static const char shape_mode_message[] = "Optional. Allows to run `static`, `dynamic` or both scenarios. Default value is empty string allows to run both" " scenarios. Possible values are `static`, `dynamic`, ``"; +static const char test_timeout_message[] = "Optional. Setup timeout for each test in seconds, default timeout 900seconds (15 minutes)."; DEFINE_bool(h, false, help_message); @@ -50,6 +52,7 @@ DEFINE_bool(extend_report, false, extend_report_config_message); DEFINE_bool(report_unique_name, false, report_unique_name_message); DEFINE_bool(extract_body, false, extract_body_message); DEFINE_string(shape_mode, "", shape_mode_message); +DEFINE_uint32(test_timeout, UINT_MAX, test_timeout_message); /** * @brief This function shows a help message @@ -72,6 +75,7 @@ static void showUsage() { std::cout << " --output_folder \"\" " << output_folder_message << std::endl; std::cout << " --plugin_lib_name " << output_folder_message << std::endl; std::cout << " --shape_mode \"\" " << shape_mode_message << std::endl; + std::cout << " --test_timeout \"\" " << test_timeout_message << std::endl; } } // namespace conformance diff --git a/src/tests/functional/plugin/conformance/test_runner/conformance_infra/src/main.cpp b/src/tests/functional/plugin/conformance/test_runner/conformance_infra/src/main.cpp index 3ec2b46441b..6a9b5e8cbeb 100644 --- a/src/tests/functional/plugin/conformance/test_runner/conformance_infra/src/main.cpp +++ b/src/tests/functional/plugin/conformance/test_runner/conformance_infra/src/main.cpp @@ -17,6 +17,8 @@ #include "gflag_config.hpp" #include "conformance.hpp" +#include "common_test_utils/crash_handler.hpp" + using namespace ov::test::conformance; int main(int argc, char* argv[]) { @@ -56,6 +58,8 @@ int main(int argc, char* argv[]) { throw std::runtime_error("Incorrect value for `--shape_mode`. Should be `dynamic`, `static` or ``. Current value is `" + FLAGS_shape_mode + "`"); } + CommonTestUtils::CrashHandler::SetUpTimeout(FLAGS_test_timeout); + // ---------------------------Initialization of Gtest env ----------------------------------------------- ov::test::conformance::targetDevice = FLAGS_device.c_str(); ov::test::conformance::IRFolderPaths = CommonTestUtils::splitStringByDelimiter(FLAGS_input_folders); diff --git a/src/tests/functional/plugin/conformance/test_runner/conformance_infra/src/read_ir_test/read_ir.cpp b/src/tests/functional/plugin/conformance/test_runner/conformance_infra/src/read_ir_test/read_ir.cpp index 2184e669507..ed95f3f8556 100644 --- a/src/tests/functional/plugin/conformance/test_runner/conformance_infra/src/read_ir_test/read_ir.cpp +++ b/src/tests/functional/plugin/conformance/test_runner/conformance_infra/src/read_ir_test/read_ir.cpp @@ -56,19 +56,21 @@ std::string ReadIRTest::getTestCaseName(const testing::TestParamInfo(new CommonTestUtils::CrashHandler()); + auto &s = LayerTestsUtils::Summary::getInstance(); // place to jump in case of a crash + int jmpRes = 0; #ifdef _WIN32 - if (setjmp(CommonTestUtils::env) == 0) { + jmpRes = setjmp(CommonTestUtils::env); #else - if (sigsetjmp(CommonTestUtils::env, 1) == 0) { + jmpRes = sigsetjmp(CommonTestUtils::env, 1); #endif + if (jmpRes == CommonTestUtils::JMP_STATUS::ok) { + crashHandler->StartTimer(); if (functionRefs == nullptr) { functionRefs = ngraph::clone_function(*function); functionRefs->set_friendly_name("refFunction"); } - - auto &s = LayerTestsUtils::Summary::getInstance(); s.setDeviceName(targetDevice); if (FuncTestUtils::SkipTestsConfig::currentTestIsDisabled()) { @@ -83,7 +85,10 @@ void ReadIRTest::query_model() { } catch (...) { s.updateOPsStats(functionRefs, LayerTestsUtils::PassRate::Statuses::FAILED); } - } else { + } else if (jmpRes == CommonTestUtils::JMP_STATUS::anyError) { + IE_THROW() << "Crash happens"; + } else if (jmpRes == CommonTestUtils::JMP_STATUS::alarmErr) { + s.updateOPsStats(functionRefs, LayerTestsUtils::PassRate::Statuses::HANGED); IE_THROW() << "Crash happens"; } } @@ -93,11 +98,14 @@ void ReadIRTest::SetUp() { auto crashHandler = std::unique_ptr(new CommonTestUtils::CrashHandler()); // place to jump in case of a crash + int jmpRes = 0; #ifdef _WIN32 - if (setjmp(CommonTestUtils::env) == 0) { + jmpRes = setjmp(CommonTestUtils::env); #else - if (sigsetjmp(CommonTestUtils::env, 1) == 0) { + jmpRes = sigsetjmp(CommonTestUtils::env, 1); #endif + if (jmpRes == CommonTestUtils::JMP_STATUS::ok) { + crashHandler->StartTimer(); std::tie(pathToModel, targetDevice, configuration) = this->GetParam(); function = core->read_model(pathToModel); const auto metaFile = CommonTestUtils::replaceExt(pathToModel, "meta"); @@ -213,8 +221,10 @@ void ReadIRTest::SetUp() { } } init_input_shapes(inputShapes); - } else { + } else if (jmpRes == CommonTestUtils::JMP_STATUS::anyError) { IE_THROW() << "Crash happens"; + } else if (jmpRes == CommonTestUtils::JMP_STATUS::alarmErr) { + IE_THROW() << "Hange happens"; } } diff --git a/src/tests/functional/plugin/shared/src/behavior/plugin/life_time.cpp b/src/tests/functional/plugin/shared/src/behavior/plugin/life_time.cpp index 149620d7b46..748e7b7fabc 100644 --- a/src/tests/functional/plugin/shared/src/behavior/plugin/life_time.cpp +++ b/src/tests/functional/plugin/shared/src/behavior/plugin/life_time.cpp @@ -74,9 +74,9 @@ namespace BehaviorTestsDefinitions { // Test failed if crash happens #ifdef _WIN32 - if (setjmp(CommonTestUtils::env) == 0) { + if (setjmp(CommonTestUtils::env) == CommonTestUtils::JMP_STATUS::ok) { #else - if (sigsetjmp(CommonTestUtils::env, 1) == 0) { + if (sigsetjmp(CommonTestUtils::env, 1) == CommonTestUtils::JMP_STATUS::ok) { #endif EXPECT_NO_THROW(release_order_test(order, targetDevice, function)); } else { @@ -90,9 +90,9 @@ namespace BehaviorTestsDefinitions { // Test failed if crash happens #ifdef _WIN32 - if (setjmp(CommonTestUtils::env) == 0) { + if (setjmp(CommonTestUtils::env) == CommonTestUtils::JMP_STATUS::ok) { #else - if (sigsetjmp(CommonTestUtils::env, 1) == 0) { + if (sigsetjmp(CommonTestUtils::env, 1) == CommonTestUtils::JMP_STATUS::ok) { #endif EXPECT_NO_THROW(release_order_test(order, targetDevice, function)); } else { diff --git a/src/tests/functional/plugin/shared/src/single_layer_tests/op_impl_check/op_impl_check.cpp b/src/tests/functional/plugin/shared/src/single_layer_tests/op_impl_check/op_impl_check.cpp index 3a3b8536598..d9d75ae21fd 100644 --- a/src/tests/functional/plugin/shared/src/single_layer_tests/op_impl_check/op_impl_check.cpp +++ b/src/tests/functional/plugin/shared/src/single_layer_tests/op_impl_check/op_impl_check.cpp @@ -22,11 +22,14 @@ void OpImplCheckTest::run() { auto crashHandler = std::unique_ptr(new CommonTestUtils::CrashHandler()); // place to jump in case of a crash + int jmpRes = 0; #ifdef _WIN32 - if (setjmp(CommonTestUtils::env) == 0) { + jmpRes = setjmp(CommonTestUtils::env); #else - if (sigsetjmp(CommonTestUtils::env, 1) == 0) { + jmpRes = sigsetjmp(CommonTestUtils::env, 1); #endif + if (jmpRes == CommonTestUtils::JMP_STATUS::ok) { + crashHandler->StartTimer(); summary.setDeviceName(targetDevice); try { auto executableNetwork = core->compile_model(function, targetDevice, configuration); @@ -35,8 +38,12 @@ void OpImplCheckTest::run() { summary.updateOPsImplStatus(function, false); GTEST_FAIL() << "Error in the LoadNetwork!"; } - } else { + } else if (jmpRes == CommonTestUtils::JMP_STATUS::anyError) { + summary.updateOPsImplStatus(function, false); IE_THROW() << "Crash happens"; + } else if (jmpRes == CommonTestUtils::JMP_STATUS::alarmErr) { + summary.updateOPsImplStatus(function, false); + IE_THROW() << "Hange happens"; } } diff --git a/src/tests/functional/shared_test_classes/src/base/layer_test_utils.cpp b/src/tests/functional/shared_test_classes/src/base/layer_test_utils.cpp index 5d815d69c98..071e703c964 100644 --- a/src/tests/functional/shared_test_classes/src/base/layer_test_utils.cpp +++ b/src/tests/functional/shared_test_classes/src/base/layer_test_utils.cpp @@ -33,23 +33,25 @@ void LayerTestsCommon::Run() { // in case of crash jump will be made and work will be continued auto crashHandler = std::unique_ptr(new CommonTestUtils::CrashHandler()); + auto &s = Summary::getInstance(); + s.setDeviceName(targetDevice); + + if (FuncTestUtils::SkipTestsConfig::currentTestIsDisabled()) { + s.updateOPsStats(functionRefs, PassRate::Statuses::SKIPPED); + GTEST_SKIP() << "Disabled test due to configuration" << std::endl; + } else { + s.updateOPsStats(functionRefs, PassRate::Statuses::CRASHED); + } // place to jump in case of a crash + int jmpRes = 0; #ifdef _WIN32 - if (setjmp(CommonTestUtils::env) == 0) { + jmpRes = setjmp(CommonTestUtils::env); #else - if (sigsetjmp(CommonTestUtils::env, 1) == 0) { + jmpRes = sigsetjmp(CommonTestUtils::env, 1); #endif - auto &s = Summary::getInstance(); - s.setDeviceName(targetDevice); - - if (FuncTestUtils::SkipTestsConfig::currentTestIsDisabled()) { - s.updateOPsStats(functionRefs, PassRate::Statuses::SKIPPED); - GTEST_SKIP() << "Disabled test due to configuration" << std::endl; - } else { - s.updateOPsStats(functionRefs, PassRate::Statuses::CRASHED); - } - + if (jmpRes == CommonTestUtils::JMP_STATUS::ok) { + crashHandler->StartTimer(); try { LoadNetwork(); GenerateInputs(); @@ -67,7 +69,10 @@ void LayerTestsCommon::Run() { s.updateOPsStats(functionRefs, PassRate::Statuses::FAILED); GTEST_FATAL_FAILURE_("Unknown failure occurred."); } - } else { + } else if (jmpRes == CommonTestUtils::JMP_STATUS::anyError) { + IE_THROW() << "Crash happens"; + } else if (jmpRes == CommonTestUtils::JMP_STATUS::alarmErr) { + s.updateOPsStats(functionRefs, PassRate::Statuses::HANGED); IE_THROW() << "Crash happens"; } } diff --git a/src/tests/functional/shared_test_classes/src/base/ov_subgraph.cpp b/src/tests/functional/shared_test_classes/src/base/ov_subgraph.cpp index b30b90cf71c..4afc336f35f 100644 --- a/src/tests/functional/shared_test_classes/src/base/ov_subgraph.cpp +++ b/src/tests/functional/shared_test_classes/src/base/ov_subgraph.cpp @@ -37,25 +37,29 @@ std::ostream& operator <<(std::ostream& os, const InputShape& inputShape) { } void SubgraphBaseTest::run() { + bool isCurrentTestDisabled = FuncTestUtils::SkipTestsConfig::currentTestIsDisabled(); + + LayerTestsUtils::PassRate::Statuses status = isCurrentTestDisabled ? + LayerTestsUtils::PassRate::Statuses::SKIPPED : + LayerTestsUtils::PassRate::Statuses::CRASHED; + summary.setDeviceName(targetDevice); + summary.updateOPsStats(function, status); + + if (isCurrentTestDisabled) + GTEST_SKIP() << "Disabled test due to configuration" << std::endl; + // in case of crash jump will be made and work will be continued auto crashHandler = std::unique_ptr(new CommonTestUtils::CrashHandler()); // place to jump in case of a crash + int jmpRes = 0; #ifdef _WIN32 - if (setjmp(CommonTestUtils::env) == 0) { + jmpRes = setjmp(CommonTestUtils::env); #else - if (sigsetjmp(CommonTestUtils::env, 1) == 0) { + jmpRes = sigsetjmp(CommonTestUtils::env, 1); #endif - bool isCurrentTestDisabled = FuncTestUtils::SkipTestsConfig::currentTestIsDisabled(); - - LayerTestsUtils::PassRate::Statuses status = isCurrentTestDisabled ? - LayerTestsUtils::PassRate::Statuses::SKIPPED : - LayerTestsUtils::PassRate::Statuses::CRASHED; - summary.setDeviceName(targetDevice); - summary.updateOPsStats(function, status); - - if (isCurrentTestDisabled) - GTEST_SKIP() << "Disabled test due to configuration" << std::endl; + if (jmpRes == CommonTestUtils::JMP_STATUS::ok) { + crashHandler->StartTimer(); ASSERT_FALSE(targetStaticShapes.empty() && !function->get_parameters().empty()) << "Target Static Shape is empty!!!"; std::string errorMessage; @@ -89,7 +93,10 @@ void SubgraphBaseTest::run() { if (status != LayerTestsUtils::PassRate::Statuses::PASSED) { GTEST_FATAL_FAILURE_(errorMessage.c_str()); } - } else { + } else if (jmpRes == CommonTestUtils::JMP_STATUS::anyError) { + IE_THROW() << "Crash happens"; + } else if (jmpRes == CommonTestUtils::JMP_STATUS::alarmErr) { + summary.updateOPsStats(function, LayerTestsUtils::PassRate::Statuses::HANGED); IE_THROW() << "Crash happens"; } } diff --git a/src/tests/ie_test_utils/common_test_utils/crash_handler.cpp b/src/tests/ie_test_utils/common_test_utils/crash_handler.cpp index ed3a1e446c4..4372f75a4dc 100644 --- a/src/tests/ie_test_utils/common_test_utils/crash_handler.cpp +++ b/src/tests/ie_test_utils/common_test_utils/crash_handler.cpp @@ -3,13 +3,20 @@ // #include "crash_handler.hpp" +#include namespace CommonTestUtils { // enviroment to restore in case of crash jmp_buf env; +unsigned int CrashHandler::MAX_TEST_WORK_TIME = UINT_MAX; CrashHandler::CrashHandler() { + // setup default value for timeout in 15 minutes + if (MAX_TEST_WORK_TIME == UINT_MAX) { + MAX_TEST_WORK_TIME = 900; + } + auto crashHandler = [](int errCode) { std::cerr << "Unexpected application crash with code: " << errCode << std::endl; @@ -21,13 +28,21 @@ CrashHandler::CrashHandler() { #ifndef _WIN32 signal(SIGBUS, SIG_DFL); signal(SIGFPE, SIG_DFL); + signal(SIGALRM, SIG_DFL); #endif - // goto sigsetjmp #ifdef _WIN32 - longjmp(env, 1); + longjmp(env, JMP_STATUS::anyError); #else - siglongjmp(env, 1); +// reset timeout + alarm(0); + + if (errCode == SIGALRM) { + std::cerr << "Test finished by timeout" << std::endl; + siglongjmp(env, JMP_STATUS::alarmErr); + } else { + siglongjmp(env, JMP_STATUS::anyError); + } #endif }; @@ -38,6 +53,7 @@ CrashHandler::CrashHandler() { #ifndef _WIN32 signal(SIGFPE, crashHandler); signal(SIGBUS, crashHandler); + signal(SIGALRM, crashHandler); #endif } @@ -49,7 +65,23 @@ CrashHandler::~CrashHandler() { #ifndef _WIN32 signal(SIGFPE, SIG_DFL); signal(SIGBUS, SIG_DFL); + signal(SIGALRM, SIG_DFL); +#endif + + // reset timeout +#ifndef _WIN32 + alarm(0); #endif } +void CrashHandler::StartTimer() { +#ifndef _WIN32 + alarm(MAX_TEST_WORK_TIME); +#endif +} + +void CrashHandler::SetUpTimeout(unsigned int timeout) { + MAX_TEST_WORK_TIME = timeout; +} + } // namespace CommonTestUtils \ No newline at end of file diff --git a/src/tests/ie_test_utils/common_test_utils/crash_handler.hpp b/src/tests/ie_test_utils/common_test_utils/crash_handler.hpp index 8e624671951..f06ede67c61 100644 --- a/src/tests/ie_test_utils/common_test_utils/crash_handler.hpp +++ b/src/tests/ie_test_utils/common_test_utils/crash_handler.hpp @@ -15,10 +15,16 @@ namespace CommonTestUtils { extern jmp_buf env; +enum JMP_STATUS { ok = 0, anyError = 1, alarmErr = 2 }; + class CrashHandler { +private: + static unsigned int MAX_TEST_WORK_TIME; public: CrashHandler(); ~CrashHandler(); + static void SetUpTimeout(unsigned int timeout); + void StartTimer(); }; } // namespace CommonTestUtils \ No newline at end of file diff --git a/src/tests/ie_test_utils/functional_test_utils/include/functional_test_utils/layer_test_utils/summary.hpp b/src/tests/ie_test_utils/functional_test_utils/include/functional_test_utils/layer_test_utils/summary.hpp index 54e809d95b1..9737cdf97ec 100644 --- a/src/tests/ie_test_utils/functional_test_utils/include/functional_test_utils/layer_test_utils/summary.hpp +++ b/src/tests/ie_test_utils/functional_test_utils/include/functional_test_utils/layer_test_utils/summary.hpp @@ -30,21 +30,24 @@ struct PassRate { PASSED, FAILED, SKIPPED, - CRASHED + CRASHED, + HANGED }; unsigned long passed = 0; unsigned long failed = 0; unsigned long skipped = 0; unsigned long crashed = 0; + unsigned long hanged = 0; bool isImplemented = false; PassRate() = default; - PassRate(unsigned long p, unsigned long f, unsigned long s, unsigned long c) { + PassRate(unsigned long p, unsigned long f, unsigned long s, unsigned long c, unsigned long h) { passed = p; failed = f; skipped = s; crashed = c; + hanged = h; if (!isImplemented && passed > 0) { isImplemented = true; } @@ -55,10 +58,10 @@ struct PassRate { } float getPassrate() const { - if (passed + failed + crashed == 0) { + if (passed + failed + crashed + hanged == 0) { return 0.f; } else { - return passed * 100.f / (passed + failed + skipped + crashed); + return passed * 100.f / (passed + failed + skipped + crashed + hanged); } } }; @@ -110,7 +113,7 @@ public: #ifdef IE_TEST_DEBUG void saveDebugReport(const char* className, const char* opName, unsigned long passed, unsigned long failed, - unsigned long skipped, unsigned long crashed); + unsigned long skipped, unsigned long crashed, unsigned long hanged); #endif //IE_TEST_DEBUG void saveReport(); diff --git a/src/tests/ie_test_utils/functional_test_utils/layer_tests_summary/summarize.py b/src/tests/ie_test_utils/functional_test_utils/layer_tests_summary/summarize.py index ea87f0d778a..719a377dbe7 100644 --- a/src/tests/ie_test_utils/functional_test_utils/layer_tests_summary/summarize.py +++ b/src/tests/ie_test_utils/functional_test_utils/layer_tests_summary/summarize.py @@ -110,8 +110,11 @@ def collect_statistic(root: ET.Element, is_conformance_mode: bool): trusted_ops[device.tag] = 0 covered_ops[device.tag] = 0 for op in results[device.tag]: + # for correct display of reports without hanged item in report.xml + results[device.tag][op]["hanged"] = results[device.tag][op].get("hanged", 0) op_test_cnt = int(results[device.tag][op]["passed"]) + int(results[device.tag][op]["failed"]) + \ - int(results[device.tag][op]["crashed"]) + int(results[device.tag][op]["skipped"]) + int(results[device.tag][op]["crashed"]) + int(results[device.tag][op]["skipped"]) + \ + int(results[device.tag][op]["hanged"]) if op_test_cnt == 0: continue covered_ops[device.tag] += 1 diff --git a/src/tests/ie_test_utils/functional_test_utils/layer_tests_summary/template/filters.js b/src/tests/ie_test_utils/functional_test_utils/layer_tests_summary/template/filters.js index beb86e1b65f..fee8058a091 100644 --- a/src/tests/ie_test_utils/functional_test_utils/layer_tests_summary/template/filters.js +++ b/src/tests/ie_test_utils/functional_test_utils/layer_tests_summary/template/filters.js @@ -116,7 +116,7 @@ function filterTable() { selector = []; select.forEach(item => { if (item == '100p') { - selector.push('.value:visible[crashed="0"][failed="0"][skipped="0"][value!="---"]'); + selector.push('.value:visible[crashed="0"][failed="0"][skipped="0"][hanged="0"][value!="---"]'); } if (item == '100f') { selector.push('.value:visible[passed="0"][value!="---"]'); @@ -130,6 +130,9 @@ function filterTable() { if (item == 'c') { selector.push('.value:visible[crashed!="0"][value!="---"]'); } + if (item == 'h') { + selector.push('.value:visible[hanged!="0"][value!="---"]'); + } if (item == 's') { selector.push('.value:visible[value!="---"][skipped!="0"]'); } diff --git a/src/tests/ie_test_utils/functional_test_utils/layer_tests_summary/template/report_template.html b/src/tests/ie_test_utils/functional_test_utils/layer_tests_summary/template/report_template.html index 2b572796da3..fef4ad442fb 100644 --- a/src/tests/ie_test_utils/functional_test_utils/layer_tests_summary/template/report_template.html +++ b/src/tests/ie_test_utils/functional_test_utils/layer_tests_summary/template/report_template.html @@ -34,6 +34,7 @@ F:0Failed S:2Skipped C:0Crashed + H:0Hanged
Plugin operation implementation status: @@ -75,6 +76,7 @@ + @@ -137,24 +139,26 @@ - {% if (results[d][op].passed != '0' or results[d][op].failed != '0' or results[d][op].crashed != '0' or results[d][op].skipped) != '0' -%} + {% if (results[d][op].passed != '0' or results[d][op].failed != '0' or results[d][op].crashed != '0' or results[d][op].skipped != '0' or results[d][op].hanged != '0') -%} {{ results[d][op].passrate }} %
{% else -%} ---
{% endif -%}
- {% if (results[d][op].passed != '0' or results[d][op].failed != '0' or results[d][op].crashed != '0' or results[d][op].skipped) != '0' -%} + {% if (results[d][op].passed != '0' or results[d][op].failed != '0' or results[d][op].crashed != '0' or results[d][op].skipped != '0' or results[d][op].hanged != '0') -%} P:{{ results[d][op].passed }} F:{{ results[d][op].failed }} S:{{ results[d][op].skipped }} C:{{ results[d][op].crashed }} + H:{{ results[d][op].hanged }} {% else -%} {% endif -%}
diff --git a/src/tests/ie_test_utils/functional_test_utils/layer_tests_summary/template/style.css b/src/tests/ie_test_utils/functional_test_utils/layer_tests_summary/template/style.css index 9a15630b256..dcd7fd55931 100644 --- a/src/tests/ie_test_utils/functional_test_utils/layer_tests_summary/template/style.css +++ b/src/tests/ie_test_utils/functional_test_utils/layer_tests_summary/template/style.css @@ -29,6 +29,9 @@ body { .dark { background: #8b000040; } +.grey-red { + background: #5e121275; +} .filters { background: #FFF; padding: 5px 10px; diff --git a/src/tests/ie_test_utils/functional_test_utils/src/layer_test_utils/summary.cpp b/src/tests/ie_test_utils/functional_test_utils/src/layer_test_utils/summary.cpp index 03fdfc37c87..5096374ccfd 100644 --- a/src/tests/ie_test_utils/functional_test_utils/src/layer_test_utils/summary.cpp +++ b/src/tests/ie_test_utils/functional_test_utils/src/layer_test_utils/summary.cpp @@ -70,20 +70,27 @@ void Summary::updateOPsStats(const ngraph::NodeTypeInfo &op, const PassRate::Sta case PassRate::CRASHED: passrate.crashed++; break; + case PassRate::HANGED: + passrate.hanged++; + passrate.crashed--; + break; } } else { switch (status) { case PassRate::PASSED: - opsStats[op] = PassRate(1, 0, 0, 0); + opsStats[op] = PassRate(1, 0, 0, 0, 0); break; case PassRate::FAILED: - opsStats[op] = PassRate(0, 1, 0, 0); + opsStats[op] = PassRate(0, 1, 0, 0, 0); break; case PassRate::SKIPPED: - opsStats[op] = PassRate(0, 0, 1, 0); + opsStats[op] = PassRate(0, 0, 1, 0, 0); break; case PassRate::CRASHED: - opsStats[op] = PassRate(0, 0, 0, 1); + opsStats[op] = PassRate(0, 0, 0, 1, 0); + break; + case PassRate::HANGED: + opsStats[op] = PassRate(0, 0, 0, 0, 1); break; } } @@ -96,7 +103,7 @@ void Summary::updateOPsImplStatus(const ngraph::NodeTypeInfo &op, const bool imp it->second.isImplemented = true; } } else { - opsStats[op] = PassRate(0, 0, 0, 0); + opsStats[op] = PassRate(0, 0, 0, 0, 0); opsStats[op].isImplemented = implStatus; } } @@ -129,7 +136,8 @@ std::map Summary::getOpStatisticFromReport() { auto f = std::stoi(child.attribute("failed").value()); auto s = std::stoi(child.attribute("skipped").value()); auto c = std::stoi(child.attribute("crashed").value()); - PassRate obj(p, f, s, c); + auto h = std::stoi(child.attribute("hanged").value()); + PassRate obj(p, f, s, c, h); oldOpsStat.insert({entry, obj}); } return oldOpsStat; @@ -217,11 +225,11 @@ void Summary::updateOPsImplStatus(const std::shared_ptr &funct #ifdef IE_TEST_DEBUG void Summary::saveDebugReport(const char* className, const char* opName, unsigned long passed, unsigned long failed, - unsigned long skipped, unsigned long crashed) { + unsigned long skipped, unsigned long crashed, unsigned long hanged) { std::string outputFilePath = "./part_report.txt"; std::ofstream file; file.open(outputFilePath, std::ios_base::app); - file << className << ' ' << opName << ' ' << passed << ' ' << failed << ' ' << skipped << ' ' << crashed << '\n'; + file << className << ' ' << opName << ' ' << passed << ' ' << failed << ' ' << skipped << ' ' << crashed << ' ' << hanged << '\n'; file.close(); } #endif //IE_TEST_DEBUG @@ -302,6 +310,7 @@ void Summary::saveReport() { entry.append_attribute("failed").set_value(it.second.failed); entry.append_attribute("skipped").set_value(it.second.skipped); entry.append_attribute("crashed").set_value(it.second.crashed); + entry.append_attribute("hanged").set_value(it.second.hanged); entry.append_attribute("passrate").set_value(it.second.getPassrate()); } @@ -316,6 +325,7 @@ void Summary::saveReport() { entry.append_attribute("failed").set_value(item.second.failed); entry.append_attribute("skipped").set_value(item.second.skipped); entry.append_attribute("crashed").set_value(item.second.crashed); + entry.append_attribute("hanged").set_value(item.second.hanged); entry.append_attribute("passrate").set_value(item.second.getPassrate()); } else { entry = currentDeviceNode.child(item.first.c_str()); @@ -324,7 +334,8 @@ void Summary::saveReport() { auto f = std::stoi(entry.attribute("failed").value()) + item.second.failed; auto s = std::stoi(entry.attribute("skipped").value()) + item.second.skipped; auto c = std::stoi(entry.attribute("crashed").value()) + item.second.crashed; - PassRate obj(p, f, s, c); + auto h = std::stoi(entry.attribute("hanged").value()) + item.second.hanged; + PassRate obj(p, f, s, c, h); (implStatus || obj.isImplemented) ? entry.attribute("implemented").set_value(true) @@ -333,6 +344,7 @@ void Summary::saveReport() { entry.attribute("failed").set_value(obj.failed); entry.attribute("skipped").set_value(obj.skipped); entry.attribute("crashed").set_value(obj.crashed); + entry.attribute("hanged").set_value(obj.hanged); entry.attribute("passrate").set_value(obj.getPassrate()); } }