IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[conformance] Move query model tests from api to opset (#18242)

Browse Source 
* [conformance] Move query model tests from api to opset

* test summary

* Fix build

* Fix large amount of new tests
This commit is contained in:
Sofya Balandina 2023-07-14 08:53:51 +01:00 committed by GitHub
parent 31f86c83d3
commit 92ecccc1b9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 61 additions and 135 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/plugins/template/src/plugin.cpp
View File

@ -193,6 +193,8 @@ ov::SupportedOpsMap ov::template_plugin::Plugin::query_model(const std::shared_p
#include "openvino/opsets/opset8_tbl.hpp"
#include "openvino/opsets/opset9_tbl.hpp"
#include "openvino/opsets/opset10_tbl.hpp"
#include "openvino/opsets/opset11_tbl.hpp"
#include "openvino/opsets/opset12_tbl.hpp"
// clang-format on
#undef _OPENVINO_OP_REG
return op_super_set.contains_type(node->get_type_info());

18
src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/include/read_ir_test/read_ir_query_model.hpp
View File

@ -1,18 +0,0 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "read_ir_test/read_ir.hpp"
namespace ov {
namespace test {
namespace subgraph {
TEST_P(ReadIRTest, QueryModel) {
query_model();
}
} // namespace subgraph
} // namespace test
} // namespace ov

29
src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/op_impl_check/op_impl_check_query_model.cpp
View File

@ -1,29 +0,0 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "single_layer_tests/op_impl_check/op_impl_check_query_model.hpp"
#include "single_layer_tests/op_impl_check/single_op_graph.hpp"
#include "conformance.hpp"
namespace ov {
namespace test {
namespace conformance {
namespace op {
using namespace ov::test::subgraph;
namespace {
INSTANTIATE_TEST_SUITE_P(conformance_query_model,
OpImplCheckTest,
::testing::Combine(
::testing::ValuesIn(createFunctions()),
::testing::Values(targetDevice),
::testing::Values(pluginConfig)),
OpImplCheckTest::getTestCaseName);
} // namespace
} // namespace op
} // namespace conformance
} // namespace test
} // namespace ov

39
src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/read_ir/read_ir_query_model.cpp
View File

@ -1,39 +0,0 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "read_ir_test/read_ir_query_model.hpp"
#include "conformance.hpp"
namespace ov {
namespace test {
namespace conformance {
namespace op {
using namespace ov::test::subgraph;
namespace {
#define _OPENVINO_OP_REG(NAME, NAMESPACE) \
INSTANTIATE_TEST_SUITE_P(conformance_query_model_##NAME, \
ReadIRTest, \
::testing::Combine(::testing::ValuesIn(getModelPaths(IRFolderPaths, #NAME)), \
::testing::Values(targetDevice), \
::testing::Values(pluginConfig)), \
ReadIRTest::getTestCaseName); \
// It should point on latest opset which contains biggest list of operations
#include <openvino/opsets/opset10_tbl.hpp>
#undef _OPENVINO_OP_REG
INSTANTIATE_TEST_SUITE_P(conformance_query_model_other,
ReadIRTest,
::testing::Combine(::testing::ValuesIn(getModelPaths(IRFolderPaths)),
::testing::Values(targetDevice),
::testing::Values(pluginConfig)),
ReadIRTest::getTestCaseName);
} // namespace
} // namespace op
} // namespace conformance
} // namespace test
} // namespace ov

40
src/tests/functional/plugin/conformance/test_runner/conformance_infra/include/conformance.hpp
View File

@ -21,10 +21,19 @@ extern const char* refCachePath;
extern std::vector<std::string> IRFolderPaths;
extern std::vector<std::string> disabledTests;
extern std::list<std::string> dirList;
// first value - path to model, second - amout of tests with this path
extern std::list<std::pair<std::string, int>> dirListInfo;
extern ov::AnyMap pluginConfig;
enum ShapeMode {
DYNAMIC,
STATIC,
BOTH
};
extern ShapeMode shapeMode;
inline ov::AnyMap readPluginConfig(const std::string &configFilePath) {
if (!CommonTestUtils::fileExists(configFilePath)) {
std::string msg = "Input directory (" + configFilePath + ") doesn't not exist!";
@ -135,8 +144,10 @@ inline std::vector<std::pair<std::string, std::string>> getModelPaths(const std:
} else {
continue;
}
//Save it in a list
dirList.insert(dirList.end(), tmp_buf.begin(), tmp_buf.end());
//Save it in a list, first value - path, second - amout of tests with this path
for (auto& val : tmp_buf) {
dirListInfo.insert(dirListInfo.end(), std::make_pair(val, 0));
}
}
listPrepared = true;
}
@ -146,21 +157,24 @@ inline std::vector<std::pair<std::string, std::string>> getModelPaths(const std:
for (const auto& op_version : unique_ops[opName]) {
std::string final_op_name = op_version == "" ? opName : opName + "-" + op_version;
std::string strToFind = CommonTestUtils::FileSeparator + final_op_name + CommonTestUtils::FileSeparator;
auto it = dirList.begin();
while (it != dirList.end()) {
if (it->find(strToFind) != std::string::npos) {
result.push_back({*it, get_ref_path(*it)});
it = dirList.erase(it);
} else {
++it;
auto it = dirListInfo.begin();
while (it != dirListInfo.end()) {
if (it->first.find(strToFind) != std::string::npos) {
result.push_back({it->first, get_ref_path(it->first)});
it->second++;
}
++it;
}
}
} else if (opName == "Other") {
// For "Undefined" operation name - run all applicable files in "Undefined" handler
// result.insert(result.end(), dirList.begin(), dirList.end());
for (const auto& file : dirList) {
result.push_back({file, get_ref_path(file)});
// result.insert(result.end(), dirListInfo.begin(), dirListInfo.end());
for (auto& file : dirListInfo) {
// if file wasn't used for tests previously we can create test with it
if (file.second == 0) {
result.push_back({file.first, get_ref_path(file.first)});
file.second++;
}
}
}
return result;

5
src/tests/functional/plugin/conformance/test_runner/conformance_infra/src/main.cpp
View File

@ -13,7 +13,6 @@
#include "functional_test_utils/skip_tests_config.hpp"
#include "functional_test_utils/summary/environment.hpp"
#include "read_ir_test/read_ir.hpp"
#include "gflag_config.hpp"
#include "conformance.hpp"
@ -55,9 +54,9 @@ int main(int argc, char* argv[]) {
apiSummary.setDeviceName(FLAGS_device);
}
if (FLAGS_shape_mode == std::string("static")) {
ov::test::subgraph::shapeMode = ov::test::subgraph::ShapeMode::STATIC;
ov::test::conformance::shapeMode = ov::test::conformance::ShapeMode::STATIC;
} else if (FLAGS_shape_mode == std::string("dynamic")) {
ov::test::subgraph::shapeMode = ov::test::subgraph::ShapeMode::DYNAMIC;
ov::test::conformance::shapeMode = ov::test::conformance::ShapeMode::DYNAMIC;
} else if (FLAGS_shape_mode != std::string("")) {
throw std::runtime_error("Incorrect value for `--shape_mode`. Should be `dynamic`, `static` or ``. Current value is `" + FLAGS_shape_mode + "`");
}

2
src/tests/functional/plugin/conformance/test_runner/conformance_infra/src/skip_tests_config.cpp
View File

@ -22,6 +22,8 @@ std::vector<std::string> disabledTests = {};
ov::AnyMap pluginConfig = {};
ShapeMode shapeMode = ov::test::conformance::ShapeMode::BOTH;
} // namespace conformance
} // namespace test
} // namespace ov

0
src/tests/functional/plugin/shared/include/single_layer_tests/op_impl_check/op_impl_check.hpp → src/tests/functional/plugin/conformance/test_runner/op_conformance_runner/include/op_impl_check/op_impl_check.hpp
View File

4
src/tests/functional/plugin/shared/include/single_layer_tests/op_impl_check/op_impl_check_compile_model.hpp → src/tests/functional/plugin/conformance/test_runner/op_conformance_runner/include/op_impl_check/op_impl_check_compile_model.hpp
View File

@ -3,13 +3,13 @@
//
#include "functional_test_utils/crash_handler.hpp"
#include "single_layer_tests/op_impl_check/op_impl_check.hpp"
#include "op_impl_check/op_impl_check.hpp"
namespace ov {
namespace test {
namespace subgraph {
TEST_P(OpImplCheckTest, checkPluginImplementation) {
TEST_P(OpImplCheckTest, checkPluginImplementationCompileModel) {
if (function == nullptr) {
GTEST_FAIL() << "Target model is empty!";
}

4
src/tests/functional/plugin/shared/include/single_layer_tests/op_impl_check/op_impl_check_query_model.hpp → src/tests/functional/plugin/conformance/test_runner/op_conformance_runner/include/op_impl_check/op_impl_check_query_model.hpp
View File

@ -3,13 +3,13 @@
//
#include "functional_test_utils/crash_handler.hpp"
#include "single_layer_tests/op_impl_check/op_impl_check.hpp"
#include "op_impl_check/op_impl_check.hpp"
namespace ov {
namespace test {
namespace subgraph {
TEST_P(OpImplCheckTest, checkPluginImplementation) {
TEST_P(OpImplCheckTest, checkPluginImplementationQueryModel) {
if (function == nullptr) {
GTEST_FAIL() << "Target model is empty!";
}

0
src/tests/functional/plugin/shared/include/single_layer_tests/op_impl_check/single_op_graph.hpp → src/tests/functional/plugin/conformance/test_runner/op_conformance_runner/include/op_impl_check/single_op_graph.hpp
View File

0
src/tests/functional/plugin/conformance/test_runner/conformance_infra/include/read_ir_test/read_ir.hpp → src/tests/functional/plugin/conformance/test_runner/op_conformance_runner/include/read_ir_test/read_ir.hpp
View File

17
src/tests/functional/plugin/conformance/test_runner/op_conformance_runner/include/read_ir_test/read_ir_compare_with_refs.hpp
View File

@ -1,17 +0,0 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "read_ir_test/read_ir.hpp"
namespace ov {
namespace test {
namespace subgraph {
TEST_P(ReadIRTest, ReadIR) {
run();
}
} // namespace subgraph
} // namespace test
} // namespace ov

2
src/tests/functional/plugin/shared/src/single_layer_tests/op_impl_check/op_impl_check.cpp → src/tests/functional/plugin/conformance/test_runner/op_conformance_runner/src/op_impl_check/op_impl_check.cpp
View File

@ -6,7 +6,7 @@
#include <process.h>
#endif
#include "single_layer_tests/op_impl_check/op_impl_check.hpp"
#include "op_impl_check/op_impl_check.hpp"
#include "functional_test_utils/crash_handler.hpp"
namespace ov {

5
src/tests/functional/plugin/conformance/test_runner/op_conformance_runner/src/op_impl_check/op_impl_check_compile_model.cpp → src/tests/functional/plugin/conformance/test_runner/op_conformance_runner/src/op_impl_check/op_impl_check_tests.cpp
View File

@ -2,8 +2,9 @@
// SPDX-License-Identifier: Apache-2.0
//
#include "single_layer_tests/op_impl_check/op_impl_check_compile_model.hpp"
#include "single_layer_tests/op_impl_check/single_op_graph.hpp"
#include "op_impl_check/op_impl_check_compile_model.hpp"
#include "op_impl_check/op_impl_check_query_model.hpp"
#include "op_impl_check/single_op_graph.hpp"
#include "conformance.hpp"
namespace ov {

4
src/tests/functional/plugin/shared/src/single_layer_tests/op_impl_check/single_op_graph.cpp → src/tests/functional/plugin/conformance/test_runner/op_conformance_runner/src/op_impl_check/single_op_graph.cpp
View File

@ -2,8 +2,8 @@
// SPDX-License-Identifier: Apache-2.0
//
#include <single_layer_tests/op_impl_check/op_impl_check.hpp>
#include <single_layer_tests/op_impl_check/single_op_graph.hpp>
#include <op_impl_check/op_impl_check.hpp>
#include <op_impl_check/single_op_graph.hpp>
#include <openvino/pass/constant_folding.hpp>
namespace ov {

11
src/tests/functional/plugin/conformance/test_runner/conformance_infra/src/read_ir_test/read_ir.cpp → src/tests/functional/plugin/conformance/test_runner/op_conformance_runner/src/read_ir/read_ir.cpp
View File

@ -25,12 +25,11 @@
namespace ov {
namespace test {
namespace conformance {
// It is used while files lookup
std::list<std::string> dirList;
// It is used while files lookup, first value - path to model, second - amout of tests with this path
std::list<std::pair<std::string, int>> dirListInfo;
}
namespace subgraph {
ShapeMode shapeMode = ShapeMode::BOTH;
namespace subgraph {
std::string ReadIRTest::getTestCaseName(const testing::TestParamInfo<ReadIRParams> &obj) {
using namespace CommonTestUtils;
@ -228,9 +227,9 @@ void ReadIRTest::SetUp() {
}
}
}
if (hasDynamic && ov::test::subgraph::shapeMode == ov::test::subgraph::ShapeMode::STATIC) {
if (hasDynamic && ov::test::conformance::shapeMode == ov::test::conformance::ShapeMode::STATIC) {
GTEST_SKIP() << "Dynamic cases are skipped according `shape_mode`";
} else if (!hasDynamic && ov::test::subgraph::shapeMode == ov::test::subgraph::ShapeMode::DYNAMIC) {
} else if (!hasDynamic && ov::test::conformance::shapeMode == ov::test::conformance::ShapeMode::DYNAMIC) {
GTEST_SKIP() << "Static cases are skipped according `shape_mode`";
}

11
src/tests/functional/plugin/conformance/test_runner/op_conformance_runner/src/read_ir/read_ir_compare_with_refs.cpp → src/tests/functional/plugin/conformance/test_runner/op_conformance_runner/src/read_ir/read_ir_tests.cpp
View File

@ -4,7 +4,8 @@
#include "common_test_utils/file_utils.hpp"
#include "read_ir_test/read_ir_compare_with_refs.hpp"
#include "read_ir_test/read_ir.hpp"
#include "conformance.hpp"
namespace ov {
@ -16,6 +17,14 @@ using namespace ov::test::subgraph;
namespace {
TEST_P(ReadIRTest, ReadIR) {
run();
}
TEST_P(ReadIRTest, QueryModel) {
query_model();
}
#define _OPENVINO_OP_REG(NAME, NAMESPACE) \
INSTANTIATE_TEST_SUITE_P(conformance_##NAME, \
ReadIRTest, \

3
src/tests/ie_test_utils/functional_test_utils/layer_tests_summary/template/report_template.html
View File

@ -31,6 +31,9 @@
<div>
<span><b>Passrates are based on relative weights each subgraphs! You can check absolute value in `General passrate` row!</b></span>
</div>
<div>
<span><b>Relative Passrate could be from 0 to 200% now! Relative weight of operation is calculated once for current set of models, but there are more then one tests per model. </b></span>
</div>
<div>
<span><b>Status:</b></span>
<span class="green">P:85</span><span>Passed</span>