diff --git a/src/tests/functional/plugin/conformance/subgraphs_dumper/src/ops_cache.cpp b/src/tests/functional/plugin/conformance/subgraphs_dumper/src/ops_cache.cpp index 0248428cf8c..94a149ae34d 100644 --- a/src/tests/functional/plugin/conformance/subgraphs_dumper/src/ops_cache.cpp +++ b/src/tests/functional/plugin/conformance/subgraphs_dumper/src/ops_cache.cpp @@ -188,11 +188,20 @@ OPCache::serialize_function(const std::pair, LayerTest auto function = std::make_shared(results, params); // TODO: How to define element type for multi-output ops + std::string op_folder_name = op.first->get_type_info().name; + std::string opset_version = op.first->get_type_info().get_version(); + std::string opset_name = "opset"; + auto pos = opset_version.find(opset_name); + if (pos != std::string::npos) { + op_folder_name += "-" + opset_version.substr(pos + opset_name.size()); + } auto op_el_type = op.first->get_output_element_type(0).get_type_name(); auto current_op_folder = serialization_dir + CommonTestUtils::FileSeparator + (is_dynamic ? "dynamic" : "static") + CommonTestUtils::FileSeparator + - op.first->get_type_info().name + CommonTestUtils::FileSeparator + op_el_type; + op_folder_name + CommonTestUtils::FileSeparator + op_el_type; auto op_name = op.first->get_name(); + op_name = op_name.substr(op_name.find("_") + 1); + std::cout << op_name << " will be serialized to " << current_op_folder << std::endl; if (!CommonTestUtils::directoryExists(current_op_folder)) { CommonTestUtils::createDirectoryRecursive(current_op_folder); diff --git a/src/tests/functional/plugin/conformance/test_runner/conformance_infra/include/conformance.hpp b/src/tests/functional/plugin/conformance/test_runner/conformance_infra/include/conformance.hpp index 382d2f87e51..c174dea7e26 100644 --- a/src/tests/functional/plugin/conformance/test_runner/conformance_infra/include/conformance.hpp +++ b/src/tests/functional/plugin/conformance/test_runner/conformance_infra/include/conformance.hpp @@ -4,8 +4,13 @@ #pragma once -#include "common_test_utils/file_utils.hpp" + #include +#include + +#include "openvino/opsets/opset.hpp" + +#include "common_test_utils/file_utils.hpp" namespace ov { namespace test { @@ -45,8 +50,61 @@ inline ov::AnyMap readPluginConfig(const std::string &configFilePath) { return config; } +static std::map> get_unique_ops() { + std::map> res; + for (const auto& opset_pair : get_available_opsets()) { + std::string opset_name = opset_pair.first; + const OpSet& opset = opset_pair.second(); + for (const auto& op : opset.get_type_info_set()) { + std::string op_version = op.get_version(), opset_name = "opset"; + auto pos = op_version.find(opset_name); + if (pos != std::string::npos) { + op_version = op_version.substr(pos + opset_name.size()); + } + if (res.find(op.name) == res.end()) { + // W/A for existing directories (without op version) + res.insert({op.name, {""}}); + } + res[op.name].insert(op_version); + } + } + return res; +} + +static std::set get_element_type_names() { + std::vector element_types = { ov::element::Type_t::f64, + ov::element::Type_t::f32, + ov::element::Type_t::f16, + ov::element::Type_t::bf16, + ov::element::Type_t::i64, + ov::element::Type_t::i32, + ov::element::Type_t::i16, + ov::element::Type_t::i8, + ov::element::Type_t::i4, + ov::element::Type_t::u64, + ov::element::Type_t::u32, + ov::element::Type_t::u16, + ov::element::Type_t::u8, + ov::element::Type_t::u4, + ov::element::Type_t::u1, + ov::element::Type_t::boolean, + ov::element::Type_t::dynamic + }; + std::set result; + for (const auto& element_type : element_types) { + std::string element_name = element_type.get_type_name(); + std::transform(element_name.begin(), element_name.end(), element_name.begin(), + [](unsigned char symbol){ return std::tolower(symbol); }); + result.insert(element_name); + } + return result; +} + +static auto unique_ops = get_unique_ops(); +static auto element_type_names = get_element_type_names(); + inline std::vector getModelPaths(const std::vector& conformance_ir_paths, - const std::string opName = "Other") { + const std::string& opName = "Other") { // This is required to prevent re-scan folders each call in case there is nothing found static bool listPrepared = false; if (!listPrepared) { @@ -69,22 +127,22 @@ inline std::vector getModelPaths(const std::vector& co std::vector result; if (!opName.empty() && opName != "Other") { - std::string strToFind = CommonTestUtils::FileSeparator + opName + CommonTestUtils::FileSeparator; - auto it = dirList.begin(); - while (it != dirList.end()) { - if (it->find(strToFind) != std::string::npos) { - result.push_back(*it); - it = dirList.erase(it); - } else { - ++it; + for (const auto& op_version : unique_ops[opName]) { + std::string final_op_name = op_version == "" ? opName : opName + "-" + op_version; + std::string strToFind = CommonTestUtils::FileSeparator + final_op_name + CommonTestUtils::FileSeparator; + auto it = dirList.begin(); + while (it != dirList.end()) { + if (it->find(strToFind) != std::string::npos) { + result.push_back(*it); + it = dirList.erase(it); + } else { + ++it; + } } } } else if (opName == "Other") { // For "Undefined" operation name - run all applicable files in "Undefined" handler result.insert(result.end(), dirList.begin(), dirList.end()); - } else { - std::string message = "Operatiion name: " + opName + " is incorrect. Please check the instantiation parameters!"; - throw std::runtime_error(message); } return result; } diff --git a/src/tests/functional/plugin/conformance/test_runner/conformance_infra/src/read_ir_test/read_ir.cpp b/src/tests/functional/plugin/conformance/test_runner/conformance_infra/src/read_ir_test/read_ir.cpp index e23a41c454f..afcd1e42b3c 100644 --- a/src/tests/functional/plugin/conformance/test_runner/conformance_infra/src/read_ir_test/read_ir.cpp +++ b/src/tests/functional/plugin/conformance/test_runner/conformance_infra/src/read_ir_test/read_ir.cpp @@ -17,6 +17,7 @@ #include "functional_test_utils/summary/op_info.hpp" #include "functional_test_utils/skip_tests_config.hpp" +#include "conformance.hpp" #include "read_ir_test/read_ir.hpp" #include @@ -39,11 +40,42 @@ std::string ReadIRTest::getTestCaseName(const testing::TestParamInfo 1) { - result << "PRC=" << *std::next(splittedFilename.rbegin()) << "_"; + std::reverse(splittedFilename.begin(), splittedFilename.end()); + bool is_valid_path_format = true; + + // Check that op is valid + if (splittedFilename.size() > 2) { + auto pos = splittedFilename[2].find('-'); + std::string op_name = splittedFilename[2], op_version = ""; + if (pos != std::string::npos) { + op_name = splittedFilename[2].substr(0, pos); + op_version = splittedFilename[2].substr(pos + 1); + } + if (std::find(ov::test::conformance::unique_ops[op_name].begin(), + ov::test::conformance::unique_ops[op_name].end(), op_version) != ov::test::conformance::unique_ops[op_name].end() && + ov::test::conformance::unique_ops.find(op_name) != ov::test::conformance::unique_ops.end()) { + std::string message = "Op=" + op_name; + if (op_version != "") { + message += "." + op_version; + } + message += "_"; + result << message; + } else { + is_valid_path_format = false; + } } - result << "IR_name=" << splittedFilename.back() << "_"; - result << "TargetDevice=" << deviceName << "_"; + // Check the element_type + if (splittedFilename.size() > 1) { + if (std::find(ov::test::conformance::element_type_names.begin(), + ov::test::conformance::element_type_names.end(), + splittedFilename[1]) != ov::test::conformance::element_type_names.end()) { + result << "Type=" << splittedFilename[1] << "_"; + } else { + is_valid_path_format = false; + } + } + result << "IR=" << (is_valid_path_format ? CommonTestUtils::replaceExt(splittedFilename[0], "") : pathToModel) << "_"; + result << "Device=" << deviceName << "_"; result << "Config=("; auto configItem = config.begin(); while (configItem != config.end()) { diff --git a/src/tests/functional/plugin/conformance/test_runner/op_conformance_runner/src/read_ir/read_ir_compare_with_refs.cpp b/src/tests/functional/plugin/conformance/test_runner/op_conformance_runner/src/read_ir/read_ir_compare_with_refs.cpp index 5bb4ecb6842..bfd9433ef6b 100644 --- a/src/tests/functional/plugin/conformance/test_runner/op_conformance_runner/src/read_ir/read_ir_compare_with_refs.cpp +++ b/src/tests/functional/plugin/conformance/test_runner/op_conformance_runner/src/read_ir/read_ir_compare_with_refs.cpp @@ -16,8 +16,8 @@ using namespace ov::test::subgraph; namespace { -#define _OPENVINO_OP_REG(NAME, NAMESPACE) \ - INSTANTIATE_TEST_SUITE_P(conformance_##NAME, \ +#define _OPENVINO_OP_REG(NAME, NAMESPACE) \ + INSTANTIATE_TEST_SUITE_P(conformance_##NAME, \ ReadIRTest, \ ::testing::Combine(::testing::ValuesIn(getModelPaths(IRFolderPaths, #NAME)), \ ::testing::Values(targetDevice), \ diff --git a/src/tests/ie_test_utils/common_test_utils/file_utils.hpp b/src/tests/ie_test_utils/common_test_utils/file_utils.hpp index 16015aab8aa..8149a31bf31 100644 --- a/src/tests/ie_test_utils/common_test_utils/file_utils.hpp +++ b/src/tests/ie_test_utils/common_test_utils/file_utils.hpp @@ -227,7 +227,11 @@ inline std::string replaceExt(std::string file, const std::string& newExt) { std::string::size_type i = file.rfind('.', file.length()); if (i != std::string::npos) { - file.replace(i + 1, newExt.length(), newExt); + if (newExt == "") { + file = file.substr(0, i); + } else { + file.replace(i + 1, newExt.length(), newExt); + } } return file; } diff --git a/src/tests/ie_test_utils/functional_test_utils/layer_tests_summary/rename_conformance_ir.py b/src/tests/ie_test_utils/functional_test_utils/layer_tests_summary/rename_conformance_ir.py index 0a6d1335991..6950e50fa87 100644 --- a/src/tests/ie_test_utils/functional_test_utils/layer_tests_summary/rename_conformance_ir.py +++ b/src/tests/ie_test_utils/functional_test_utils/layer_tests_summary/rename_conformance_ir.py @@ -94,7 +94,7 @@ def create_hash(in_dir_path: Path): str_to_hash += ET.tostring(ports_info).decode('utf8').replace('\t', '') old_name = model_path - new_name = model_path.name[:model_path.name.find('_') + 1] + str(sha256(str_to_hash.encode('utf-8')).hexdigest()) + new_name = str(sha256(str_to_hash.encode('utf-8')).hexdigest()) model_path.rename(Path(model_path.parent, new_name + XML_EXTENSION)) meta_path.rename(Path(meta_path.parent, new_name + META_EXTENSION))