IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[CONFORMANCE] Change Name of ReadIR tests and op serialization folder by subgraphsDumper (#15671)

Browse Source 
* [CONFORMANCE] Change Name of ReadIR tests and opserialization

* remove  op_name from ir name

* remove extra file
This commit is contained in:
Irina Efode 2023-02-14 12:02:01 +04:00 committed by GitHub
parent b80d05e0e1
commit 3d6474a4a3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 125 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/tests/functional/plugin/conformance/subgraphs_dumper/src/ops_cache.cpp
View File

@ -188,11 +188,20 @@ OPCache::serialize_function(const std::pair<std::shared_ptr<ov::Node>, LayerTest
auto function = std::make_shared<ov::Model>(results, params);
// TODO: How to define element type for multi-output ops
std::string op_folder_name = op.first->get_type_info().name;
std::string opset_version = op.first->get_type_info().get_version();
std::string opset_name = "opset";
auto pos = opset_version.find(opset_name);
if (pos != std::string::npos) {
op_folder_name += "-" + opset_version.substr(pos + opset_name.size());
}
auto op_el_type = op.first->get_output_element_type(0).get_type_name();
auto current_op_folder = serialization_dir + CommonTestUtils::FileSeparator +
(is_dynamic ? "dynamic" : "static") + CommonTestUtils::FileSeparator +
op.first->get_type_info().name + CommonTestUtils::FileSeparator + op_el_type;
op_folder_name + CommonTestUtils::FileSeparator + op_el_type;
auto op_name = op.first->get_name();
op_name = op_name.substr(op_name.find("_") + 1);
std::cout << op_name << " will be serialized to " << current_op_folder << std::endl;
if (!CommonTestUtils::directoryExists(current_op_folder)) {
CommonTestUtils::createDirectoryRecursive(current_op_folder);

84
src/tests/functional/plugin/conformance/test_runner/conformance_infra/include/conformance.hpp
View File

@ -4,8 +4,13 @@
#pragma once
#include "common_test_utils/file_utils.hpp"
#include <list>
#include <algorithm>
#include "openvino/opsets/opset.hpp"
#include "common_test_utils/file_utils.hpp"
namespace ov {
namespace test {
@ -45,8 +50,61 @@ inline ov::AnyMap readPluginConfig(const std::string &configFilePath) {
return config;
}
static std::map<std::string, std::set<std::string>> get_unique_ops() {
std::map<std::string, std::set<std::string>> res;
for (const auto& opset_pair : get_available_opsets()) {
std::string opset_name = opset_pair.first;
const OpSet& opset = opset_pair.second();
for (const auto& op : opset.get_type_info_set()) {
std::string op_version = op.get_version(), opset_name = "opset";
auto pos = op_version.find(opset_name);
if (pos != std::string::npos) {
op_version = op_version.substr(pos + opset_name.size());
}
if (res.find(op.name) == res.end()) {
// W/A for existing directories (without op version)
res.insert({op.name, {""}});
}
res[op.name].insert(op_version);
}
}
return res;
}
static std::set<std::string> get_element_type_names() {
std::vector<ov::element::Type> element_types = { ov::element::Type_t::f64,
ov::element::Type_t::f32,
ov::element::Type_t::f16,
ov::element::Type_t::bf16,
ov::element::Type_t::i64,
ov::element::Type_t::i32,
ov::element::Type_t::i16,
ov::element::Type_t::i8,
ov::element::Type_t::i4,
ov::element::Type_t::u64,
ov::element::Type_t::u32,
ov::element::Type_t::u16,
ov::element::Type_t::u8,
ov::element::Type_t::u4,
ov::element::Type_t::u1,
ov::element::Type_t::boolean,
ov::element::Type_t::dynamic
};
std::set<std::string> result;
for (const auto& element_type : element_types) {
std::string element_name = element_type.get_type_name();
std::transform(element_name.begin(), element_name.end(), element_name.begin(),
[](unsigned char symbol){ return std::tolower(symbol); });
result.insert(element_name);
}
return result;
}
static auto unique_ops = get_unique_ops();
static auto element_type_names = get_element_type_names();
inline std::vector<std::string> getModelPaths(const std::vector<std::string>& conformance_ir_paths,
const std::string opName = "Other") {
const std::string& opName = "Other") {
// This is required to prevent re-scan folders each call in case there is nothing found
static bool listPrepared = false;
if (!listPrepared) {
@ -69,22 +127,22 @@ inline std::vector<std::string> getModelPaths(const std::vector<std::string>& co
std::vector<std::string> result;
if (!opName.empty() && opName != "Other") {
std::string strToFind = CommonTestUtils::FileSeparator + opName + CommonTestUtils::FileSeparator;
auto it = dirList.begin();
while (it != dirList.end()) {
if (it->find(strToFind) != std::string::npos) {
result.push_back(*it);
it = dirList.erase(it);
} else {
++it;
for (const auto& op_version : unique_ops[opName]) {
std::string final_op_name = op_version == "" ? opName : opName + "-" + op_version;
std::string strToFind = CommonTestUtils::FileSeparator + final_op_name + CommonTestUtils::FileSeparator;
auto it = dirList.begin();
while (it != dirList.end()) {
if (it->find(strToFind) != std::string::npos) {
result.push_back(*it);
it = dirList.erase(it);
} else {
++it;
}
}
}
} else if (opName == "Other") {
// For "Undefined" operation name - run all applicable files in "Undefined" handler
result.insert(result.end(), dirList.begin(), dirList.end());
} else {
std::string message = "Operatiion name: " + opName + " is incorrect. Please check the instantiation parameters!";
throw std::runtime_error(message);
}
return result;
}

40
src/tests/functional/plugin/conformance/test_runner/conformance_infra/src/read_ir_test/read_ir.cpp
View File

@ -17,6 +17,7 @@
#include "functional_test_utils/summary/op_info.hpp"
#include "functional_test_utils/skip_tests_config.hpp"
#include "conformance.hpp"
#include "read_ir_test/read_ir.hpp"
#include <setjmp.h>
@ -39,11 +40,42 @@ std::string ReadIRTest::getTestCaseName(const testing::TestParamInfo<ReadIRParam
std::ostringstream result;
auto splittedFilename = CommonTestUtils::splitStringByDelimiter(pathToModel, CommonTestUtils::FileSeparator);
if (splittedFilename.size() > 1) {
result << "PRC=" << *std::next(splittedFilename.rbegin()) << "_";
std::reverse(splittedFilename.begin(), splittedFilename.end());
bool is_valid_path_format = true;
// Check that op is valid
if (splittedFilename.size() > 2) {
auto pos = splittedFilename[2].find('-');
std::string op_name = splittedFilename[2], op_version = "";
if (pos != std::string::npos) {
op_name = splittedFilename[2].substr(0, pos);
op_version = splittedFilename[2].substr(pos + 1);
}
if (std::find(ov::test::conformance::unique_ops[op_name].begin(),
ov::test::conformance::unique_ops[op_name].end(), op_version) != ov::test::conformance::unique_ops[op_name].end() &&
ov::test::conformance::unique_ops.find(op_name) != ov::test::conformance::unique_ops.end()) {
std::string message = "Op=" + op_name;
if (op_version != "") {
message += "." + op_version;
}
message += "_";
result << message;
} else {
is_valid_path_format = false;
}
}
result << "IR_name=" << splittedFilename.back() << "_";
result << "TargetDevice=" << deviceName << "_";
// Check the element_type
if (splittedFilename.size() > 1) {
if (std::find(ov::test::conformance::element_type_names.begin(),
ov::test::conformance::element_type_names.end(),
splittedFilename[1]) != ov::test::conformance::element_type_names.end()) {
result << "Type=" << splittedFilename[1] << "_";
} else {
is_valid_path_format = false;
}
}
result << "IR=" << (is_valid_path_format ? CommonTestUtils::replaceExt(splittedFilename[0], "") : pathToModel) << "_";
result << "Device=" << deviceName << "_";
result << "Config=(";
auto configItem = config.begin();
while (configItem != config.end()) {

4
src/tests/functional/plugin/conformance/test_runner/op_conformance_runner/src/read_ir/read_ir_compare_with_refs.cpp
View File

@ -16,8 +16,8 @@ using namespace ov::test::subgraph;
namespace {
#define _OPENVINO_OP_REG(NAME, NAMESPACE) \
INSTANTIATE_TEST_SUITE_P(conformance_##NAME, \
#define _OPENVINO_OP_REG(NAME, NAMESPACE) \
INSTANTIATE_TEST_SUITE_P(conformance_##NAME, \
ReadIRTest, \
::testing::Combine(::testing::ValuesIn(getModelPaths(IRFolderPaths, #NAME)), \
::testing::Values(targetDevice), \

6
src/tests/ie_test_utils/common_test_utils/file_utils.hpp
View File

@ -227,7 +227,11 @@ inline std::string replaceExt(std::string file, const std::string& newExt) {
std::string::size_type i = file.rfind('.', file.length());
if (i != std::string::npos) {
file.replace(i + 1, newExt.length(), newExt);
if (newExt == "") {
file = file.substr(0, i);
} else {
file.replace(i + 1, newExt.length(), newExt);
}
}
return file;
}

2
src/tests/ie_test_utils/functional_test_utils/layer_tests_summary/rename_conformance_ir.py
View File

@ -94,7 +94,7 @@ def create_hash(in_dir_path: Path):
str_to_hash += ET.tostring(ports_info).decode('utf8').replace('\t', '')
old_name = model_path
new_name = model_path.name[:model_path.name.find('_') + 1] + str(sha256(str_to_hash.encode('utf-8')).hexdigest())
new_name = str(sha256(str_to_hash.encode('utf-8')).hexdigest())
model_path.rename(Path(model_path.parent, new_name + XML_EXTENSION))
meta_path.rename(Path(meta_path.parent, new_name + META_EXTENSION))