diff --git a/src/tests/functional/plugin/conformance/subgraphs_dumper/include/utils/dynamism.hpp b/src/tests/functional/plugin/conformance/subgraphs_dumper/include/utils/dynamism.hpp new file mode 100644 index 00000000000..a43f759e41a --- /dev/null +++ b/src/tests/functional/plugin/conformance/subgraphs_dumper/include/utils/dynamism.hpp @@ -0,0 +1,43 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "openvino/core/node.hpp" +#include "openvino/core/model.hpp" + +namespace ov { +namespace tools { +namespace subgraph_dumper { + +inline bool is_dynamic_node(const std::shared_ptr& node) { + for (size_t i = 0; i < node->get_input_size(); ++i) { + if (node->get_input_partial_shape(i).is_dynamic()) { + return true; + } + } + for (size_t i = 0; i < node->get_output_size(); ++i) { + if (node->get_output_partial_shape(i).is_dynamic()) { + return true; + } + } + return false; +} + +inline bool is_dynamic_model(const std::shared_ptr& model) { + for (const auto& parameter : model->get_parameters()) { + if (is_dynamic_node(parameter)) { + return true; + } + } + for (const auto& result : model->get_results()) { + if (is_dynamic_node(result)) { + return true; + } + } + return false; +} + +} // namespace subgraph_dumper +} // namespace tools +} // namespace ov diff --git a/src/tests/functional/plugin/conformance/subgraphs_dumper/include/utils/model.hpp b/src/tests/functional/plugin/conformance/subgraphs_dumper/include/utils/model.hpp index bfae8ea874b..786e67fca8d 100644 --- a/src/tests/functional/plugin/conformance/subgraphs_dumper/include/utils/model.hpp +++ b/src/tests/functional/plugin/conformance/subgraphs_dumper/include/utils/model.hpp @@ -18,6 +18,7 @@ #include "cache/cache.hpp" #include "utils/node.hpp" +#include "utils/dynamism.hpp" namespace ov { namespace tools { @@ -75,7 +76,6 @@ std::map> cache_models( void save_model_status_to_file(const std::map>& caching_status, const std::string& output_dir); -bool is_dynamic_model(const std::shared_ptr& model); std::string get_model_type(const std::shared_ptr& model); std::map diff --git a/src/tests/functional/plugin/conformance/subgraphs_dumper/include/utils/node.hpp b/src/tests/functional/plugin/conformance/subgraphs_dumper/include/utils/node.hpp index 928ebd36935..147b73807c0 100644 --- a/src/tests/functional/plugin/conformance/subgraphs_dumper/include/utils/node.hpp +++ b/src/tests/functional/plugin/conformance/subgraphs_dumper/include/utils/node.hpp @@ -6,8 +6,11 @@ #include #include "cache/meta/input_info.hpp" +#include "utils/dynamism.hpp" + #include "functional_test_utils/node_utils.hpp" #include "functional_test_utils/summary/op_info.hpp" + #include "openvino/openvino.hpp" #include "openvino/pass/manager.hpp" #include "openvino/pass/constant_folding.hpp" @@ -45,20 +48,6 @@ std::shared_ptr convert_const_to_param(const std::shared_ // all inputs are defined as parameters and contains detailed info in meta std::shared_ptr generate_model_by_node(const std::shared_ptr& node); -inline bool is_dynamic_node(const std::shared_ptr& node) { - for (size_t i = 0; i < node->get_input_size(); ++i) { - if (node->get_input_partial_shape(i).is_dynamic()) { - return true; - } - } - for (size_t i = 0; i < node->get_output_size(); ++i) { - if (node->get_output_partial_shape(i).is_dynamic()) { - return true; - } - } - return false; -} - inline std::string get_node_type(const std::shared_ptr& node) { if (is_dynamic_node(node)) { return "dynamic"; diff --git a/src/tests/functional/plugin/conformance/subgraphs_dumper/src/utils/model.cpp b/src/tests/functional/plugin/conformance/subgraphs_dumper/src/utils/model.cpp index ef0dd60d0f7..2a9c585bcde 100644 --- a/src/tests/functional/plugin/conformance/subgraphs_dumper/src/utils/model.cpp +++ b/src/tests/functional/plugin/conformance/subgraphs_dumper/src/utils/model.cpp @@ -68,20 +68,6 @@ find_models(const std::vector &dirs, const std::string& regexp) { return { models, { ModelCacheStatus::NOT_READ, not_read_model } }; } -bool is_dynamic_model(const std::shared_ptr& model) { - for (const auto& parameter : model->get_parameters()) { - if (is_dynamic_node(parameter)) { - return true; - } - } - for (const auto& result : model->get_results()) { - if (is_dynamic_node(result)) { - return true; - } - } - return false; -} - std::string get_model_type(const std::shared_ptr& model) { if (is_dynamic_model(model)) { return "dynamic"; diff --git a/src/tests/functional/plugin/conformance/test_runner/op_conformance_runner/CMakeLists.txt b/src/tests/functional/plugin/conformance/test_runner/op_conformance_runner/CMakeLists.txt index a9eb6cec8f3..8fbeea28592 100644 --- a/src/tests/functional/plugin/conformance/test_runner/op_conformance_runner/CMakeLists.txt +++ b/src/tests/functional/plugin/conformance/test_runner/op_conformance_runner/CMakeLists.txt @@ -13,6 +13,7 @@ ov_add_test_target( PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/include" "${OpenVINO_SOURCE_DIR}/src/tests/functional/plugin/conformance/subgraphs_dumper/include/cache/meta/" + "${OpenVINO_SOURCE_DIR}/src/tests/functional/plugin/conformance/subgraphs_dumper/include/utils/" ADD_CPPLINT LINK_LIBRARIES PUBLIC diff --git a/src/tests/functional/plugin/conformance/test_runner/op_conformance_runner/include/read_ir_test/read_ir.hpp b/src/tests/functional/plugin/conformance/test_runner/op_conformance_runner/include/read_ir_test/read_ir.hpp index d8a49d5754b..c62c7943d96 100644 --- a/src/tests/functional/plugin/conformance/test_runner/op_conformance_runner/include/read_ir_test/read_ir.hpp +++ b/src/tests/functional/plugin/conformance/test_runner/op_conformance_runner/include/read_ir_test/read_ir.hpp @@ -27,8 +27,6 @@ class ReadIRTest : public testing::WithParamInterface, virtual public ov::test::SubgraphBaseTest { public: static std::string getTestCaseName(const testing::TestParamInfo &obj); - void query_model() override; - void import_export(); std::vector calculate_refs() override; protected: diff --git a/src/tests/functional/plugin/conformance/test_runner/op_conformance_runner/src/read_ir/read_ir.cpp b/src/tests/functional/plugin/conformance/test_runner/op_conformance_runner/src/read_ir/read_ir.cpp index 308d733574a..954d0fcc0b0 100644 --- a/src/tests/functional/plugin/conformance/test_runner/op_conformance_runner/src/read_ir/read_ir.cpp +++ b/src/tests/functional/plugin/conformance/test_runner/op_conformance_runner/src/read_ir/read_ir.cpp @@ -21,6 +21,7 @@ #include "functional_test_utils/summary/op_info.hpp" #include "functional_test_utils/skip_tests_config.hpp" +#include "dynamism.hpp" #include "input_info.hpp" #include "conformance.hpp" #include "read_ir_test/read_ir.hpp" @@ -116,99 +117,6 @@ std::string ReadIRTest::getTestCaseName(const testing::TestParamInfo(new ov::test::utils::CrashHandler()); - auto &s = ov::test::utils::OpSummary::getInstance(); - - // place to jump in case of a crash - int jmpRes = 0; -#ifdef _WIN32 - jmpRes = setjmp(ov::test::utils::env); -#else - jmpRes = sigsetjmp(ov::test::utils::env, 1); -#endif - if (jmpRes == ov::test::utils::JMP_STATUS::ok) { - crashHandler->StartTimer(); - if (functionRefs == nullptr) { - functionRefs = ngraph::clone_function(*function); - functionRefs->set_friendly_name("refFunction"); - } - s.setDeviceName(targetDevice); - - if (ov::test::utils::current_test_is_disabled()) { - s.updateOPsStats(functionRefs, ov::test::utils::PassRate::Statuses::SKIPPED, rel_influence_coef); - GTEST_SKIP() << "Disabled test due to configuration" << std::endl; - } else { - s.updateOPsStats(functionRefs, ov::test::utils::PassRate::Statuses::CRASHED, rel_influence_coef); - } - try { - SubgraphBaseTest::query_model(); - s.updateOPsStats(functionRefs, ov::test::utils::PassRate::Statuses::PASSED, rel_influence_coef); - } catch (std::exception& err) { - s.updateOPsStats(functionRefs, ov::test::utils::PassRate::Statuses::FAILED, rel_influence_coef); - GTEST_FAIL() << err.what(); - } catch (...) { - s.updateOPsStats(functionRefs, ov::test::utils::PassRate::Statuses::FAILED, rel_influence_coef); - GTEST_FAIL() << "Something is wrong in Query model! Please check"; - } - } else if (jmpRes == ov::test::utils::JMP_STATUS::alarmErr) { - s.updateOPsStats(functionRefs, ov::test::utils::PassRate::Statuses::HANGED, rel_influence_coef); - IE_THROW() << "Crash happens"; - } else if (jmpRes == ov::test::utils::JMP_STATUS::anyError) { - IE_THROW() << "Crash happens"; - } -} - -void ReadIRTest::import_export() { - // in case of crash jump will be made and work will be continued - auto crashHandler = std::unique_ptr(new ov::test::utils::CrashHandler()); - auto &summary = ov::test::utils::OpSummary::getInstance(); - - // place to jump in case of a crash - int jmpRes = 0; -#ifdef _WIN32 - jmpRes = setjmp(ov::test::utils::env); -#else - jmpRes = sigsetjmp(ov::test::utils::env, 1); -#endif - if (jmpRes == ov::test::utils::JMP_STATUS::ok) { - crashHandler->StartTimer(); - summary.setDeviceName(targetDevice); - try { - ov::CompiledModel model = core->compile_model(function, targetDevice, configuration); - - std::stringstream strm; - model.export_model(strm); - - ov::CompiledModel importedModel = core->import_model(strm, targetDevice, configuration); - - auto comparator = FunctionsComparator::with_default() - .enable(FunctionsComparator::ATTRIBUTES) - .enable(FunctionsComparator::NAMES) - .enable(FunctionsComparator::CONST_VALUES); - - auto importedFunction = importedModel.get_runtime_model()->clone(); - auto res = comparator.compare(importedFunction, function); - EXPECT_TRUE(res.valid) << res.message; - - summary.updateOPsImplStatus(function, true); - } catch (const std::exception &e) { - summary.updateOPsImplStatus(function, false); - GTEST_FAIL() << "Exception in the Core::compile_model() method call: " << e.what(); - } catch (...) { - summary.updateOPsImplStatus(function, false); - GTEST_FAIL() << "Error in the Core::query_model() method call!"; - } - } else if (jmpRes == ov::test::utils::JMP_STATUS::anyError) { - summary.updateOPsImplStatus(function, false); - GTEST_FAIL() << "Crash happens"; - } else if (jmpRes == ov::test::utils::JMP_STATUS::alarmErr) { - summary.updateOPsImplStatus(function, false); - GTEST_FAIL() << "Hang happens"; - } -} - uint64_t clip(uint64_t n, uint64_t lower, uint64_t upper) { return std::max(lower, std::min(n, upper)); } @@ -267,21 +175,7 @@ void ReadIRTest::SetUp() { } } - bool hasDynamic = false; - for (const auto& param : function->get_parameters()) { - if (param->get_partial_shape().is_dynamic()) { - hasDynamic = true; - break; - } - } - if (!hasDynamic) { - for (const auto& result : function->get_results()) { - if (result->get_output_partial_shape(0).is_dynamic()) { - hasDynamic = true; - break; - } - } - } + bool hasDynamic = tools::subgraph_dumper::is_dynamic_model(function); #ifdef ENABLE_CONFORMANCE_PGQL // Updating data in runtime. Should be set before possible call of a first GTEST status diff --git a/src/tests/functional/shared_test_classes/include/shared_test_classes/base/ov_subgraph.hpp b/src/tests/functional/shared_test_classes/include/shared_test_classes/base/ov_subgraph.hpp index c76cd8fbc1b..1653f6ffd42 100644 --- a/src/tests/functional/shared_test_classes/include/shared_test_classes/base/ov_subgraph.hpp +++ b/src/tests/functional/shared_test_classes/include/shared_test_classes/base/ov_subgraph.hpp @@ -31,6 +31,7 @@ public: virtual void run(); virtual void serialize(); virtual void query_model(); + virtual void import_export(); protected: virtual void compare(const std::vector& expected, const std::vector& actual); diff --git a/src/tests/functional/shared_test_classes/src/base/ov_subgraph.cpp b/src/tests/functional/shared_test_classes/src/base/ov_subgraph.cpp index aca76c4e9e6..b1c5eb9e983 100644 --- a/src/tests/functional/shared_test_classes/src/base/ov_subgraph.cpp +++ b/src/tests/functional/shared_test_classes/src/base/ov_subgraph.cpp @@ -128,20 +128,122 @@ void SubgraphBaseTest::serialize() { } void SubgraphBaseTest::query_model() { - SKIP_IF_CURRENT_TEST_IS_DISABLED(); + bool isCurrentTestDisabled = ov::test::utils::current_test_is_disabled(); - auto queryNetworkResult = core->query_model(function, targetDevice); - std::set expected; - for (auto&& node : function->get_ops()) { - expected.insert(node->get_friendly_name()); - } + ov::test::utils::PassRate::Statuses status = isCurrentTestDisabled ? + ov::test::utils::PassRate::Statuses::SKIPPED : + ov::test::utils::PassRate::Statuses::CRASHED; + summary.setDeviceName(targetDevice); + summary.updateOPsStats(function, status, rel_influence_coef); - std::set actual; - for (auto&& res : queryNetworkResult) { - actual.insert(res.first); + if (isCurrentTestDisabled) + GTEST_SKIP() << "Disabled test due to configuration" << std::endl; + + // in case of crash jump will be made and work will be continued + auto crashHandler = std::unique_ptr(new ov::test::utils::CrashHandler()); + + // place to jump in case of a crash + int jmpRes = 0; +#ifdef _WIN32 + jmpRes = setjmp(ov::test::utils::env); +#else + jmpRes = sigsetjmp(ov::test::utils::env, 1); +#endif + if (jmpRes == ov::test::utils::JMP_STATUS::ok) { + crashHandler->StartTimer(); + std::string errorMessage; + try { + auto queryNetworkResult = core->query_model(function, targetDevice); + std::set expected; + for (auto&& node : function->get_ops()) { + expected.insert(node->get_friendly_name()); + } + + std::set actual; + for (auto&& res : queryNetworkResult) { + actual.insert(res.first); + } + if (expected != actual) { + IE_THROW() << "Expected and actual are different"; + } + status = ov::test::utils::PassRate::Statuses::PASSED; + } catch (const std::exception& ex) { + status = ov::test::utils::PassRate::Statuses::FAILED; + errorMessage = ex.what(); + } catch (...) { + status = ov::test::utils::PassRate::Statuses::FAILED; + errorMessage = "Unknown failure occurred."; + } + summary.updateOPsStats(function, status, rel_influence_coef); + if (status != ov::test::utils::PassRate::Statuses::PASSED) { + GTEST_FATAL_FAILURE_(errorMessage.c_str()); + } + } else if (jmpRes == ov::test::utils::JMP_STATUS::anyError) { + IE_THROW() << "Crash happens"; + } else if (jmpRes == ov::test::utils::JMP_STATUS::alarmErr) { + summary.updateOPsStats(function, ov::test::utils::PassRate::Statuses::HANGED, rel_influence_coef); + IE_THROW() << "Crash happens"; } - if (expected != actual) { - IE_THROW() << "Expected and actual are different"; +} + +void SubgraphBaseTest::import_export() { + bool isCurrentTestDisabled = ov::test::utils::current_test_is_disabled(); + + ov::test::utils::PassRate::Statuses status = isCurrentTestDisabled ? + ov::test::utils::PassRate::Statuses::SKIPPED : + ov::test::utils::PassRate::Statuses::CRASHED; + summary.setDeviceName(targetDevice); + summary.updateOPsStats(function, status, rel_influence_coef); + + if (isCurrentTestDisabled) + GTEST_SKIP() << "Disabled test due to configuration" << std::endl; + + // in case of crash jump will be made and work will be continued + auto crashHandler = std::unique_ptr(new ov::test::utils::CrashHandler()); + + // place to jump in case of a crash + int jmpRes = 0; +#ifdef _WIN32 + jmpRes = setjmp(ov::test::utils::env); +#else + jmpRes = sigsetjmp(ov::test::utils::env, 1); +#endif + if (jmpRes == ov::test::utils::JMP_STATUS::ok) { + crashHandler->StartTimer(); + std::string errorMessage; + try { + compile_model(); + + std::stringstream strm; + compiledModel.export_model(strm); + ov::CompiledModel importedModel = core->import_model(strm, targetDevice, configuration); + auto importedFunction = importedModel.get_runtime_model()->clone(); + + auto comparator = FunctionsComparator::with_default() + .enable(FunctionsComparator::ATTRIBUTES) + .enable(FunctionsComparator::NAMES) + .enable(FunctionsComparator::CONST_VALUES); + auto res = comparator.compare(importedFunction, function); + if (!res.valid) { + throw std::runtime_error(res.message); + } + status = ov::test::utils::PassRate::Statuses::PASSED; + } catch (const std::exception& ex) { + status = ov::test::utils::PassRate::Statuses::FAILED; + errorMessage = ex.what(); + } catch (...) { + status = ov::test::utils::PassRate::Statuses::FAILED; + errorMessage = "Unknown failure occurred."; + } + summary.updateOPsStats(function, status, rel_influence_coef); + if (status != ov::test::utils::PassRate::Statuses::PASSED) { + GTEST_FATAL_FAILURE_(errorMessage.c_str()); + } + } else if (jmpRes == ov::test::utils::JMP_STATUS::anyError) { + IE_THROW() << "Crash happens"; + } else if (jmpRes == ov::test::utils::JMP_STATUS::alarmErr) { + summary.updateOPsStats(function, ov::test::utils::PassRate::Statuses::HANGED, rel_influence_coef); + IE_THROW() << "Crash happens"; } }