IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Remove ov::JsonConfigExtension (#14087)

Browse Source 
Ticket: 96278

Co-authored-by: Ilya Lavrenov <ilya.lavrenov@intel.com>
This commit is contained in:
Mateusz Tabaka 2022-11-21 12:06:08 +01:00 committed by GitHub
parent cf7b174bf9
commit 738d7bb09f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
39 changed files with 276 additions and 1039 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.gitmodules vendored
View File

@ -63,7 +63,3 @@
path = thirdparty/json/nlohmann_json
url = https://github.com/nlohmann/json.git
shallow = true
[submodule "thirdparty/json/nlohmann_json_schema_validator"]
path = thirdparty/json/nlohmann_json_schema_validator
url = https://github.com/pboettch/json-schema-validator.git
shallow = true

1
src/bindings/python/src/openvino/frontend/__init__.py
View File

@ -25,7 +25,6 @@ from openvino._pyopenvino import Place
# extensions
from openvino._pyopenvino import DecoderTransformationExtension
from openvino._pyopenvino import JsonConfigExtension
from openvino._pyopenvino import ConversionExtension
from openvino._pyopenvino import OpExtension
from openvino._pyopenvino import ProgressReporterExtension

15
src/bindings/python/src/pyopenvino/frontend/extension.cpp
View File

@ -9,7 +9,6 @@
#include <pybind11/stl.h>
#include <pybind11/stl_bind.h>
#include "extension/json_config.hpp"
#include "openvino/frontend/exception.hpp"
#include "openvino/frontend/extension/conversion.hpp"
#include "openvino/frontend/extension/decoder_transformation.hpp"
@ -46,20 +45,6 @@ void regclass_frontend_DecoderTransformationExtension(py::module m) {
ext(m, "DecoderTransformationExtension", py::dynamic_attr());
}
void regclass_frontend_JsonConfigExtension(py::module m) {
py::class_<ov::frontend::JsonConfigExtension,
std::shared_ptr<ov::frontend::JsonConfigExtension>,
ov::frontend::DecoderTransformationExtension>
ext(m, "JsonConfigExtension", py::dynamic_attr());
ext.doc() = "Extension class to load and process ModelOptimizer JSON config file";
ext.def(py::init([](const py::object& path) {
std::string extension_path = Common::utils::convert_path_to_string(path);
return std::make_shared<ov::frontend::JsonConfigExtension>(extension_path);
}));
}
void regclass_frontend_ConversionExtensionBase(py::module m) {
py::class_<ConversionExtensionBase, ConversionExtensionBase::Ptr, ov::Extension> ext(m,
"ConversionExtensionBase",

1
src/bindings/python/src/pyopenvino/frontend/extension.hpp
View File

@ -10,7 +10,6 @@ namespace py = pybind11;
void regclass_frontend_TelemetryExtension(py::module m);
void regclass_frontend_DecoderTransformationExtension(py::module m);
void regclass_frontend_JsonConfigExtension(py::module m);
void regclass_frontend_ConversionExtension(py::module m);
void regclass_frontend_ConversionExtensionBase(py::module m);
void regclass_frontend_ProgressReporterExtension(py::module m);

1
src/bindings/python/src/pyopenvino/pyopenvino.cpp
View File

@ -235,7 +235,6 @@ PYBIND11_MODULE(_pyopenvino, m) {
// frontend extensions
regclass_frontend_TelemetryExtension(m);
regclass_frontend_DecoderTransformationExtension(m);
regclass_frontend_JsonConfigExtension(m);
regclass_frontend_ConversionExtensionBase(m);
regclass_frontend_ConversionExtension(m);
regclass_frontend_ProgressReporterExtension(m);

2
src/bindings/python/tests/test_frontend/test_frontend_onnx.py
View File

@ -616,7 +616,7 @@ def get_builtin_extensions_path():
win_folder_path = Path(__file__).parent.parent.parent.parent
linux_folder_path = win_folder_path.joinpath("lib")
for lib_path in chain(win_folder_path.glob("*.dll"), linux_folder_path.glob("*.so")):
if "libtest_builtin_extensions_1" in lib_path.name:
if "libtest_builtin_extensions" in lib_path.name:
return str(lib_path)
return ""

4
src/common/offline_transformations/CMakeLists.txt
View File

@ -25,10 +25,8 @@ source_group("include" FILES ${PUBLIC_HEADERS})
add_library(${TARGET_NAME} STATIC ${LIBRARY_SRC} ${PUBLIC_HEADERS})
target_link_libraries(${TARGET_NAME} PRIVATE openvino::runtime::dev openvino::itt openvino::pugixml ngraph::reference
openvino::runtime nlohmann_json_schema_validator)
openvino::runtime)
set_source_files_properties(src/json_extension/json_config.cpp INCLUDE_DIRECTORIES
$<TARGET_PROPERTY:nlohmann_json_schema_validator,INTERFACE_INCLUDE_DIRECTORIES>)
set_source_files_properties(src/generate_mapping_file.cpp INCLUDE_DIRECTORIES
$<TARGET_PROPERTY:openvino::pugixml,INTERFACE_INCLUDE_DIRECTORIES>)

27
src/common/offline_transformations/include/extension/json_config.hpp
View File

@ -1,27 +0,0 @@
// Copyright (C) 2018-2022 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include <fstream>
#include <iostream>
#include <type_traits>
#include "openvino/frontend/extension/decoder_transformation.hpp"
namespace ov {
namespace frontend {
/// \brief JsonConfigExtension reads MO config file and delegate transformation functionality to specified
/// transformation ID specified in the config.
class JsonConfigExtension : public DecoderTransformationExtension {
public:
explicit JsonConfigExtension(const std::string& config_path);
protected:
std::vector<Extension::Ptr> m_loaded_extensions;
std::vector<std::pair<DecoderTransformationExtension::Ptr, std::string>> m_target_extensions;
};
} // namespace frontend
} // namespace ov

54
src/common/offline_transformations/include/extension/json_transformation.hpp
View File

@ -1,54 +0,0 @@
// Copyright (C) 2018-2022 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include <fstream>
#include <iostream>
#include <type_traits>
#include "openvino/core/extension.hpp"
#include "openvino/frontend/extension/decoder_transformation.hpp"
#include "openvino/pass/graph_rewrite.hpp"
#include "openvino/pass/manager.hpp"
#include "openvino/pass/pass.hpp"
namespace ov {
namespace frontend {
/// \brief Describes transformation that JsonConfigExtension can use as a target transformation specified by ID.
/// JsonTransformationExtension passes JSON parsed object and matched points in the graph to JsonTransformationExtension
/// instance, which is derived from DecoderTransformationExtension. DecoderTransformationExtension itself cannot be
/// used for this purpose because we need to pass those additional objects which are the result of JSON parsing and
/// graph matching that JsonTransformationExtension performs.
///
/// This class is left for backward compatibility only. In the future we would like to get rid off this class as well
/// as from JsonConfigExtension, and users will use DecoderTransformationExtension only to match sub-graph and modify it
/// like we do in any other transformations.
///
/// Unlike DecoderTransformationExtension, which is initialized by some ready-to-use transformation code and is not used
/// to derive new classes by regular users, this class is intended to be derived from and it doesn't have convenient
/// ctos to be initialized. So it is intended for more advanced, internal users inside such components like Model
/// Optimizer.
class JsonTransformationExtension : public DecoderTransformationExtension {
public:
explicit JsonTransformationExtension(const std::string& id) : m_id(id) {}
/// \brief The name of the transformation to identify it from JSON file field 'id'
const std::string& id() const {
return m_id;
}
/// \brief Modifies OV Model according to the provided rules.
///
/// \param[in] function The OV Model object.
/// \param[in] replacement_descriptions The rules to modify the model in .json format.
virtual bool transform(const std::shared_ptr<ov::Model>& function,
const std::string& replacement_descriptions) const = 0;
private:
std::string m_id;
};
} // namespace frontend
} // namespace ov

209
src/common/offline_transformations/src/json_extension/json_config.cpp
View File

@ -1,209 +0,0 @@
// Copyright (C) 2018-2022 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "extension/json_config.hpp"
#include "openvino/core/deprecated.hpp"
OPENVINO_SUPPRESS_DEPRECATED_START
#include "nlohmann/json-schema.hpp"
OPENVINO_SUPPRESS_DEPRECATED_END
#include "extension/json_transformation.hpp"
#include "openvino/frontend/extension/decoder_transformation.hpp"
#include "so_extension.hpp"
namespace {
static const nlohmann::json validation_schema =
R"(
{
"definitions": {},
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "Root",
"type": "array",
"default": [],
"items": {
"$id": "#root/items",
"title": "Items",
"type": "object",
"required": [
"id",
"match_kind"
],
"properties": {
"custom_attributes": {
"$id": "#root/items/custom_attributes",
"title": "Custom_attributes",
"type": "object",
"properties": {
}
},
"id": {
"$id": "#root/items/id",
"title": "Id",
"type": "string",
"pattern": "^.*$",
"minLength": 1
},
"inputs": {
"$id": "#root/items/inputs",
"title": "Inputs",
"type": "array",
"default": [],
"items": {
"$id": "#root/items/inputs/items",
"title": "Items",
"type": "array",
"default": [],
"items": {
"$id": "#root/items/inputs/items/items",
"title": "Items",
"type": "object",
"properties": {
"node": {
"$id": "#root/items/inputs/items/items/node",
"title": "Node",
"type": "string",
"default": "",
"pattern": "^.*$"
},
"port": {
"$id": "#root/items/inputs/items/items/port",
"title": "Port",
"type": "integer",
"default": 0
}
},
"required": ["node", "port"]
}
}
},
"instances": {
"$id": "#root/items/instances",
"title": "Instances",
"type": ["array", "object"],
"items": {
"$id": "#root/items/instances/items",
"title": "Items",
"type": "string",
"default": "",
"pattern": "^.*$"
}
},
"match_kind": {
"$id": "#root/items/match_kind",
"title": "Match_kind",
"type": "string",
"enum": ["points", "scope", "general"],
"default": "points",
"pattern": "^.*$"
},
"outputs": {
"$id": "#root/items/outputs",
"title": "Outputs",
"type": "array",
"default": [],
"items": {
"$id": "#root/items/outputs/items",
"title": "Items",
"type": "object",
"properties": {
"node": {
"$id": "#root/items/outputs/items/node",
"title": "Node",
"type": "string",
"default": "",
"pattern": "^.*$"
},
"port": {
"$id": "#root/items/outputs/items/port",
"title": "Port",
"type": "integer",
"default": 0
}
},
"required": ["node", "port"]
}
},
"include_inputs_to_sub_graph": {
"$id": "#root/items/include_inputs_to_sub_graph",
"title": "Include_inputs_to_sub_graph",
"type": "boolean",
"default": false
},
"include_outputs_to_sub_graph": {
"$id": "#root/items/include_outputs_to_sub_graph",
"title": "Include_outputs_to_sub_graph",
"type": "boolean",
"default": false
}
}
}
}
)"_json;
} // namespace
using namespace ov;
using namespace ov::frontend;
JsonConfigExtension::JsonConfigExtension(const std::string& config_path)
: DecoderTransformationExtension([this](std::shared_ptr<ov::Model> f) {
bool res = false;
for (const auto& target_extension : m_target_extensions) {
if (auto extension = std::dynamic_pointer_cast<JsonTransformationExtension>(target_extension.first)) {
res |= extension->transform(f, target_extension.second);
}
}
return res;
}) {
// Load JSON config
nlohmann::json config_json;
std::ifstream config_file(config_path);
config_file >> config_json;
// Validate JSON config
nlohmann::json_schema::json_validator validator;
try {
validator.set_root_schema(validation_schema);
} catch (const std::exception& e) {
OPENVINO_ASSERT(false, "Invalid json schema : ", e.what());
}
try {
validator.validate(config_json);
} catch (const std::exception& e) {
OPENVINO_ASSERT(false, "Json schema validation failed: ", e.what());
}
// Parse JSON Extensions
// Group sections describing transformations by library.
std::unordered_map<std::string, nlohmann::json> lib_to_sections;
for (const auto& section : config_json) {
lib_to_sections[section["library"]].push_back(section);
}
// Load all extensions in each library and select required
for (const auto& it : lib_to_sections) {
const auto& lib = it.first;
const auto& sections = it.second;
auto extensions = ov::detail::load_extensions(lib);
m_loaded_extensions.insert(m_loaded_extensions.end(), extensions.begin(), extensions.end());
for (const auto& ext : extensions) {
auto so_extension = std::dynamic_pointer_cast<ov::detail::SOExtension>(ext);
OPENVINO_ASSERT(so_extension, "Unexpected extension type loaded from shared library.");
auto extension = so_extension->extension();
if (auto json_ext = std::dynamic_pointer_cast<JsonTransformationExtension>(extension)) {
for (const auto& section : sections) {
if (section["id"] == json_ext->id()) {
m_target_extensions.push_back({json_ext, section.dump()});
}
}
}
}
}
}

2
src/frontends/onnx/tests/CMakeLists.txt
View File

@ -80,7 +80,7 @@ set(MULTI_TEST_SRC
onnx_import_with_editor.in.cpp)
set(SRC
conversion.cpp
json_config_extension.cpp
library_extension.cpp
load_from.cpp
onnx_editor.cpp
onnx_import_exceptions.cpp

24
src/frontends/onnx/tests/json_config_extension.cpp
View File

@ -1,24 +0,0 @@
// Copyright (C) 2018-2022 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "json_config.hpp"
#include "onnx_utils.hpp"
using namespace ov::frontend;
using ONNXJsonConfigTest = FrontEndJsonConfigTest;
static JsonConfigFEParam getTestData() {
JsonConfigFEParam res;
res.m_frontEndName = ONNX_FE;
res.m_modelsPath = std::string(TEST_ONNX_MODELS_DIRNAME);
res.m_modelName = "controlflow/loop_2d_add.onnx";
return res;
}
// TODO: 83471
// INSTANTIATE_TEST_SUITE_P(ONNXJsonConfigTest,
// FrontEndJsonConfigTest,
// ::testing::Values(getTestData()),
// FrontEndJsonConfigTest::getTestCaseName);

24
src/frontends/onnx/tests/library_extension.cpp Normal file
View File

@ -0,0 +1,24 @@
// Copyright (C) 2018-2022 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "library_extension.hpp"
#include "onnx_utils.hpp"
using namespace ov::frontend;
using ONNXLibraryExtensionTest = FrontendLibraryExtensionTest;
static FrontendLibraryExtensionTestParams getTestData() {
FrontendLibraryExtensionTestParams params;
params.m_frontEndName = ONNX_FE;
params.m_modelsPath = std::string(TEST_ONNX_MODELS_DIRNAME);
params.m_modelName = "relu.onnx";
return params;
}
INSTANTIATE_TEST_SUITE_P(ONNXLibraryExtensionTest,
FrontendLibraryExtensionTest,
::testing::Values(getTestData()),
FrontendLibraryExtensionTest::getTestCaseName);

23
src/frontends/paddle/tests/json_config_extension.cpp
View File

@ -1,23 +0,0 @@
// Copyright (C) 2018-2022 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "json_config.hpp"
#include "paddle_utils.hpp"
using namespace ov::frontend;
using PaddleJsonConfigTest = FrontEndJsonConfigTest;
static JsonConfigFEParam getTestData() {
JsonConfigFEParam res;
res.m_frontEndName = PADDLE_FE;
res.m_modelsPath = std::string(TEST_PADDLE_MODELS_DIRNAME);
res.m_modelName = "relu/relu.pdmodel";
return res;
}
INSTANTIATE_TEST_SUITE_P(PaddleJsonConfigTest,
FrontEndJsonConfigTest,
::testing::Values(getTestData()),
FrontEndJsonConfigTest::getTestCaseName);

24
src/frontends/paddle/tests/library_extension.cpp Normal file
View File

@ -0,0 +1,24 @@
// Copyright (C) 2018-2022 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "library_extension.hpp"
#include "paddle_utils.hpp"
using namespace ov::frontend;
using PaddleLibraryExtensionTest = FrontendLibraryExtensionTest;
static FrontendLibraryExtensionTestParams getTestData() {
FrontendLibraryExtensionTestParams params;
params.m_frontEndName = PADDLE_FE;
params.m_modelsPath = std::string(TEST_PADDLE_MODELS_DIRNAME);
params.m_modelName = "relu/relu.pdmodel";
return params;
}
INSTANTIATE_TEST_SUITE_P(PaddleLibraryExtensionTest,
FrontendLibraryExtensionTest,
::testing::Values(getTestData()),
FrontendLibraryExtensionTest::getTestCaseName);

23
src/frontends/tensorflow/tests/json_config_extension.cpp
View File

@ -1,23 +0,0 @@
// Copyright (C) 2018-2022 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "json_config.hpp"
#include "tf_utils.hpp"
using namespace ov::frontend;
using TFJsonConfigTest = FrontEndJsonConfigTest;
static JsonConfigFEParam getTestData() {
JsonConfigFEParam res;
res.m_frontEndName = TF_FE;
res.m_modelsPath = std::string(TEST_TENSORFLOW_MODELS_DIRNAME);
res.m_modelName = "2in_2out/2in_2out.pb";
return res;
}
INSTANTIATE_TEST_SUITE_P(TFJsonConfigTest,
FrontEndJsonConfigTest,
::testing::Values(getTestData()),
FrontEndJsonConfigTest::getTestCaseName);

24
src/frontends/tensorflow/tests/library_extension.cpp Normal file
View File

@ -0,0 +1,24 @@
// Copyright (C) 2018-2022 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "library_extension.hpp"
#include "tf_utils.hpp"
using namespace ov::frontend;
using TFLibraryExtensionTest = FrontendLibraryExtensionTest;
static FrontendLibraryExtensionTestParams getTestData() {
FrontendLibraryExtensionTestParams params;
params.m_frontEndName = TF_FE;
params.m_modelsPath = std::string(TEST_TENSORFLOW_MODELS_DIRNAME);
params.m_modelName = "2in_2out/2in_2out.pb";
return params;
}
INSTANTIATE_TEST_SUITE_P(TFLibraryExtensionTest,
FrontendLibraryExtensionTest,
::testing::Values(getTestData()),
FrontendLibraryExtensionTest::getTestCaseName);

10
src/frontends/tests/frontend/shared/CMakeLists.txt
View File

@ -10,8 +10,7 @@ file(GLOB_RECURSE LIBRARY_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/include/*.hpp)
add_library(${TARGET_NAME} STATIC EXCLUDE_FROM_ALL ${LIBRARY_SRC} ${LIBRARY_HEADERS})
add_subdirectory(gtest_main_manifest)
add_subdirectory(test_builtin_extensions_1)
add_subdirectory(test_builtin_extensions_2)
add_subdirectory(test_builtin_extensions)
target_include_directories(${TARGET_NAME} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
target_include_directories(${TARGET_NAME} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/../..)
@ -22,12 +21,9 @@ target_link_libraries(${TARGET_NAME}
commonTestUtils ngraph_test_util openvino::util
openvino::runtime
PRIVATE
cnpy
# TODO: replace with nlohmann_json::nlohmann_json
# but LTO fails for some reason
nlohmann_json_schema_validator)
cnpy)
add_dependencies(${TARGET_NAME} test_builtin_extensions_1 test_builtin_extensions_2)
add_dependencies(${TARGET_NAME} test_builtin_extensions)
target_compile_definitions(${TARGET_NAME}
PRIVATE

48
src/frontends/tests/frontend/shared/include/json_config.hpp
View File

@ -1,48 +0,0 @@
// Copyright (C) 2018-2022 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include <gtest/gtest.h>
#include <extension/json_config.hpp>
#include <openvino/frontend/manager.hpp>
class JsonConfigExtensionWrapper : public ov::frontend::JsonConfigExtension {
public:
explicit JsonConfigExtensionWrapper(const std::string& config_path)
: ov::frontend::JsonConfigExtension(config_path){};
~JsonConfigExtensionWrapper() override = default;
std::vector<Extension::Ptr> get_loaded_extensions() {
return m_loaded_extensions;
};
std::vector<std::pair<DecoderTransformationExtension::Ptr, std::string>> get_target_extensions() {
return m_target_extensions;
}
};
struct JsonConfigFEParam {
std::string m_frontEndName;
std::string m_modelsPath;
std::string m_modelName;
};
class FrontEndJsonConfigTest : public ::testing::TestWithParam<JsonConfigFEParam> {
public:
std::string m_file_name = "test_json.json";
std::ofstream m_output_json;
JsonConfigFEParam m_param;
ov::frontend::FrontEndManager m_fem;
static std::string getTestCaseName(const testing::TestParamInfo<JsonConfigFEParam>& obj);
void SetUp() override;
void TearDown() override;
protected:
void generate_json_config();
void initParamTest();
};

28
src/frontends/tests/frontend/shared/include/library_extension.hpp Normal file
View File

@ -0,0 +1,28 @@
// Copyright (C) 2018-2022 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include <gtest/gtest.h>
#include <openvino/frontend/manager.hpp>
struct FrontendLibraryExtensionTestParams {
std::string m_frontEndName;
std::string m_modelsPath;
std::string m_modelName;
};
class FrontendLibraryExtensionTest : public ::testing::TestWithParam<FrontendLibraryExtensionTestParams> {
public:
FrontendLibraryExtensionTestParams m_param;
ov::frontend::FrontEndManager m_fem;
static std::string getTestCaseName(const testing::TestParamInfo<FrontendLibraryExtensionTestParams>& obj);
void SetUp() override;
protected:
void initParamTest();
};

2
src/frontends/tests/frontend/shared/src/conversion.cpp
View File

@ -78,7 +78,7 @@ TEST_P(FrontEndConversionExtensionTest, TestConversionExtension) {
TEST_P(FrontEndConversionExtensionTest, TestConversionExtensionViaSO) {
auto frontend = m_param.m_frontend;
const auto& lib_path = get_lib_path("test_builtin_extensions_1");
const auto& lib_path = get_lib_path("test_builtin_extensions");
frontend->add_extension(lib_path);
std::shared_ptr<InputModel> input_model;
ASSERT_NO_THROW(input_model = frontend->load(m_param.m_modelName));

197
src/frontends/tests/frontend/shared/src/json_config.cpp
View File

@ -1,197 +0,0 @@
// Copyright (C) 2018-2022 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "json_config.hpp"
#include <common_test_utils/graph_comparator.hpp>
#include <extension/json_config.hpp>
#include <extension/json_transformation.hpp>
#include <nlohmann/json.hpp>
#include <ostream>
#include "common_test_utils/file_utils.hpp"
#include "utils.hpp"
using namespace ov::frontend;
std::string FrontEndJsonConfigTest::getTestCaseName(const testing::TestParamInfo<JsonConfigFEParam>& obj) {
std::string res = obj.param.m_frontEndName + "_" + obj.param.m_modelName;
return FrontEndTestUtils::fileToTestName(res);
}
void FrontEndJsonConfigTest::SetUp() {
FrontEndTestUtils::setupTestEnv();
m_fem = FrontEndManager(); // re-initialize after setting up environment
initParamTest();
}
void FrontEndJsonConfigTest::TearDown() {
m_output_json.close();
std::remove(m_file_name.c_str());
}
void FrontEndJsonConfigTest::initParamTest() {
m_param = GetParam();
m_param.m_modelName = FrontEndTestUtils::make_model_path(m_param.m_modelsPath + m_param.m_modelName);
}
inline std::string get_lib_path(const std::string& lib_name) {
return ov::util::make_plugin_library_name<char>(CommonTestUtils::getExecutableDirectory(),
lib_name + IE_BUILD_POSTFIX);
}
void FrontEndJsonConfigTest::generate_json_config() {
std::string json = R"(
[
{
"custom_attributes": {
"test_attribute": true
},
"id": "buildin_extensions_1::TestExtension1",
"library": "test_builtin_extensions_1",
"match_kind": "scope"
},
{
"custom_attributes": {
"test_attribute": true
},
"id": "buildin_extensions_2::TestExtension1",
"library": "test_builtin_extensions_2",
"match_kind": "scope"
},
{
"custom_attributes": {
"test_attribute": true
},
"id": "buildin_extensions_2::TestExtension2",
"library": "test_builtin_extensions_2",
"match_kind": "scope"
}
]
)";
nlohmann::json config_json = nlohmann::json::parse(json);
// update lib path
for (auto& section : config_json) {
section["library"] = get_lib_path(section["library"]);
}
m_output_json.open(m_file_name);
m_output_json << std::setw(4) << config_json << std::endl;
}
///////////////////////////////////////////////////////////////////
TEST_P(FrontEndJsonConfigTest, testJsonConfig) {
EXPECT_NO_THROW(generate_json_config());
}
TEST_P(FrontEndJsonConfigTest, testAddJsonConfigExtension) {
EXPECT_NO_THROW(generate_json_config());
std::map<std::string, nlohmann::json> reference_map = {
{"buildin_extensions_1::TestExtension1",
nlohmann::json::parse(
R"(
{
"custom_attributes": {
"test_attribute": true
},
"id": "buildin_extensions_1::TestExtension1",
"library": "test_builtin_extensions_1",
"match_kind": "scope"
}
)")},
{"buildin_extensions_2::TestExtension1",
nlohmann::json::parse(
R"(
{
"custom_attributes": {
"test_attribute": true
},
"id": "buildin_extensions_2::TestExtension1",
"library": "test_builtin_extensions_2",
"match_kind": "scope"
}
)")},
{"buildin_extensions_2::TestExtension2",
nlohmann::json::parse(
R"(
{
"custom_attributes": {
"test_attribute": true
},
"id": "buildin_extensions_2::TestExtension2",
"library": "test_builtin_extensions_2",
"match_kind": "scope"
}
)")},
};
for (auto& ref_val : reference_map) {
ref_val.second["library"] = get_lib_path(ref_val.second["library"]);
}
std::shared_ptr<ov::Model> function;
{
ov::frontend::FrontEnd::Ptr m_frontEnd;
ov::frontend::InputModel::Ptr m_inputModel;
m_frontEnd = m_fem.load_by_framework(m_param.m_frontEndName);
auto json_config_ext = std::make_shared<JsonConfigExtensionWrapper>(m_file_name);
m_frontEnd->add_extension(json_config_ext);
auto loaded_ext = json_config_ext->get_loaded_extensions();
auto target_ext = json_config_ext->get_target_extensions();
// the number of Loaded extensions can be more than the number of actually used ones.
EXPECT_EQ(loaded_ext.size(), 9);
EXPECT_EQ(target_ext.size(), 3);
for (const auto& target : target_ext) {
auto transformation = std::dynamic_pointer_cast<JsonTransformationExtension>(target.first);
EXPECT_NE(transformation, nullptr);
EXPECT_EQ(reference_map.at(transformation->id()).dump(), target.second);
}
ASSERT_NO_THROW(m_inputModel = m_frontEnd->load(m_param.m_modelName));
ASSERT_NE(m_inputModel, nullptr);
ASSERT_NO_THROW(function = m_frontEnd->convert(m_inputModel));
ASSERT_NE(function, nullptr);
EXPECT_EQ(function->get_friendly_name(), "TestFunction");
}
}
TEST_P(FrontEndJsonConfigTest, compareFunctions) {
EXPECT_NO_THROW(generate_json_config());
std::shared_ptr<ov::Model> function;
{
ov::frontend::FrontEnd::Ptr m_frontEnd;
ov::frontend::InputModel::Ptr m_inputModel;
m_frontEnd = m_fem.load_by_framework(m_param.m_frontEndName);
auto json_config_ext = std::make_shared<JsonConfigExtensionWrapper>(m_file_name);
m_frontEnd->add_extension(json_config_ext);
ASSERT_NO_THROW(m_inputModel = m_frontEnd->load(m_param.m_modelName));
ASSERT_NE(m_inputModel, nullptr);
ASSERT_NO_THROW(function = m_frontEnd->convert(m_inputModel));
ASSERT_NE(function, nullptr);
}
std::shared_ptr<ov::Model> function_ref;
{
ov::frontend::FrontEnd::Ptr m_frontEnd;
ov::frontend::InputModel::Ptr m_inputModel;
m_frontEnd = m_fem.load_by_framework(m_param.m_frontEndName);
ASSERT_NO_THROW(m_inputModel = m_frontEnd->load(m_param.m_modelName));
ASSERT_NE(m_inputModel, nullptr);
ASSERT_NO_THROW(function_ref = m_frontEnd->convert(m_inputModel));
ASSERT_NE(function, nullptr);
}
compare_functions(function, function_ref);
}

90
src/frontends/tests/frontend/shared/src/library_extension.cpp Normal file
View File

@ -0,0 +1,90 @@
// Copyright (C) 2018-2022 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "library_extension.hpp"
#include <common_test_utils/graph_comparator.hpp>
#include <ostream>
#include "common_test_utils/file_utils.hpp"
#include "utils.hpp"
using namespace ov::frontend;
std::string FrontendLibraryExtensionTest::getTestCaseName(
const testing::TestParamInfo<FrontendLibraryExtensionTestParams>& obj) {
std::string res = obj.param.m_frontEndName + "_" + obj.param.m_modelName;
return FrontEndTestUtils::fileToTestName(res);
}
void FrontendLibraryExtensionTest::SetUp() {
FrontEndTestUtils::setupTestEnv();
m_fem = FrontEndManager(); // re-initialize after setting up environment
initParamTest();
}
void FrontendLibraryExtensionTest::initParamTest() {
m_param = GetParam();
m_param.m_modelName = FrontEndTestUtils::make_model_path(m_param.m_modelsPath + m_param.m_modelName);
}
inline std::string get_lib_path(const std::string& lib_name) {
return ov::util::make_plugin_library_name<char>(CommonTestUtils::getExecutableDirectory(),
lib_name + IE_BUILD_POSTFIX);
}
///////////////////////////////////////////////////////////////////
TEST_P(FrontendLibraryExtensionTest, verifyFunctions) {
std::shared_ptr<ov::Model> function_ref;
{
ov::frontend::FrontEnd::Ptr m_frontEnd;
ov::frontend::InputModel::Ptr m_inputModel;
m_frontEnd = m_fem.load_by_framework(m_param.m_frontEndName);
ASSERT_NO_THROW(m_inputModel = m_frontEnd->load(m_param.m_modelName));
ASSERT_NE(m_inputModel, nullptr);
ASSERT_NO_THROW(function_ref = m_frontEnd->convert(m_inputModel));
ASSERT_NE(function_ref, nullptr);
const auto nodes = function_ref->get_ops();
ASSERT_NE(std::find_if(nodes.begin(),
nodes.end(),
[](const std::shared_ptr<ov::Node>& n) {
return ov::is_type<ov::opset8::Relu>(n);
}),
nodes.end());
}
std::shared_ptr<ov::Model> function;
{
ov::frontend::FrontEnd::Ptr m_frontEnd;
ov::frontend::InputModel::Ptr m_inputModel;
m_frontEnd = m_fem.load_by_framework(m_param.m_frontEndName);
const auto& lib_path = get_lib_path("test_builtin_extensions");
m_frontEnd->add_extension(lib_path);
ASSERT_NO_THROW(m_inputModel = m_frontEnd->load(m_param.m_modelName));
ASSERT_NE(m_inputModel, nullptr);
ASSERT_NO_THROW(function = m_frontEnd->convert(m_inputModel));
ASSERT_NE(function, nullptr);
const auto nodes = function->get_ops();
ASSERT_EQ(std::find_if(nodes.begin(),
nodes.end(),
[](const std::shared_ptr<ov::Node>& n) {
return ov::is_type<ov::opset8::Relu>(n);
}),
nodes.end());
ASSERT_NE(std::find_if(nodes.begin(),
nodes.end(),
[](const std::shared_ptr<ov::Node>& n) {
return ov::is_type<ov::opset8::Swish>(n);
}),
nodes.end());
}
}

4
src/frontends/tests/frontend/shared/test_builtin_extensions_1/CMakeLists.txt → src/frontends/tests/frontend/shared/test_builtin_extensions/CMakeLists.txt
View File

@ -2,7 +2,7 @@
# SPDX-License-Identifier: Apache-2.0
#
set(TARGET_NAME "test_builtin_extensions_1")
set(TARGET_NAME "test_builtin_extensions")
if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /ignore:4217")
@ -13,7 +13,7 @@ endif()
file(GLOB LIBRARY_SRC ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp)
file(GLOB LIBRARY_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/*.hpp)
set(DEPENDENCIES openvino::runtime::dev offline_transformations)
set(DEPENDENCIES openvino::runtime::dev openvino::frontend::common)
if (ENABLE_OV_ONNX_FRONTEND)
list(APPEND DEPENDENCIES openvino::frontend::onnx)

67
src/frontends/tests/frontend/shared/test_builtin_extensions/builtin_extensions.cpp Normal file
View File

@ -0,0 +1,67 @@
// Copyright (C) 2018-2022 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include <openvino/core/extension.hpp>
#include <openvino/frontend/extension/conversion.hpp>
#include <openvino/opsets/opset8.hpp>
#ifdef ENABLE_OV_ONNX_FRONTEND
# include <openvino/frontend/onnx/extension/conversion.hpp>
# define ONNX_EXT \
std::make_shared<ov::frontend::onnx::ConversionExtension>("NewCustomOp_3", CustomTranslatorONNX), \
std::make_shared<ov::frontend::onnx::ConversionExtension>("Relu", ReluToSwishTranslator),
#else
# define ONNX_EXT
#endif
#ifdef ENABLE_OV_PADDLE_FRONTEND
# include <openvino/frontend/paddle/extension/conversion.hpp>
# define PADDLE_EXT \
std::make_shared<ov::frontend::paddle::ConversionExtension>("NewCustomOp_4", CustomTranslatorPaddle), \
std::make_shared<ov::frontend::paddle::ConversionExtension>("relu", ReluToSwishTranslatorPDPD),
#else
# define PADDLE_EXT
#endif
#ifdef ENABLE_OV_TF_FRONTEND
# include <openvino/frontend/tensorflow/extension/conversion.hpp>
# define TF_EXT \
std::make_shared<ov::frontend::tensorflow::ConversionExtension>("NewCustomOp_5", CustomTranslatorTensorflow), \
std::make_shared<ov::frontend::tensorflow::ConversionExtension>("Relu", ReluToSwishTranslator),
#else
# define TF_EXT
#endif
ov::OutputVector CustomTranslatorCommon_1(const ov::frontend::NodeContext& node) {
return ov::OutputVector();
}
std::map<std::string, ov::OutputVector> CustomTranslatorCommon_2(const ov::frontend::NodeContext& node) {
return std::map<std::string, ov::OutputVector>();
}
ov::OutputVector CustomTranslatorTensorflow(const ov::frontend::NodeContext& node) {
return ov::OutputVector();
}
ov::OutputVector CustomTranslatorONNX(const ov::frontend::NodeContext& node) {
return ov::OutputVector();
}
ov::OutputVector ReluToSwishTranslator(const ov::frontend::NodeContext& node) {
return {std::make_shared<ov::opset8::Swish>(node.get_input(0))};
}
std::map<std::string, ov::OutputVector> ReluToSwishTranslatorPDPD(const ov::frontend::NodeContext& node) {
return {{"Out", {std::make_shared<ov::opset8::Swish>(node.get_input("X"))}}};
}
std::map<std::string, ov::OutputVector> CustomTranslatorPaddle(const ov::frontend::NodeContext& node) {
return std::map<std::string, ov::OutputVector>();
}
OPENVINO_CREATE_EXTENSIONS(std::vector<ov::Extension::Ptr>(
{std::make_shared<ov::frontend::ConversionExtension>("NewCustomOp_1", CustomTranslatorCommon_1),
std::make_shared<ov::frontend::ConversionExtension>("NewCustomOp_2", CustomTranslatorCommon_2),
ONNX_EXT PADDLE_EXT TF_EXT}));

39
src/frontends/tests/frontend/shared/test_builtin_extensions_1/builtin_extensions.cpp
View File

@ -1,39 +0,0 @@
// Copyright (C) 2018-2022 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include <openvino/core/extension.hpp>
#include <openvino/frontend/extension/conversion.hpp>
#ifdef ENABLE_OV_ONNX_FRONTEND
# include <openvino/frontend/onnx/extension/conversion.hpp>
# define ONNX_EXT \
std::make_shared<ov::frontend::onnx::ConversionExtension>("NewCustomOp_3", CustomTranslatorONNX), \
std::make_shared<ov::frontend::onnx::ConversionExtension>("Relu", ReluToSwishTranslatorONNX),
#else
# define ONNX_EXT
#endif
#ifdef ENABLE_OV_PADDLE_FRONTEND
# include <openvino/frontend/paddle/extension/conversion.hpp>
# define PADDLE_EXT \
std::make_shared<ov::frontend::paddle::ConversionExtension>("NewCustomOp_4", CustomTranslatorPaddle),
#else
# define PADDLE_EXT
#endif
#ifdef ENABLE_OV_TF_FRONTEND
# include <openvino/frontend/tensorflow/extension/conversion.hpp>
# define TF_EXT \
std::make_shared<ov::frontend::tensorflow::ConversionExtension>("NewCustomOp_5", CustomTranslatorTensorflow)
#else
# define TF_EXT
#endif
#include "test_extension.hpp"
OPENVINO_CREATE_EXTENSIONS(std::vector<ov::Extension::Ptr>(
{std::make_shared<TestExtension1>(),
std::make_shared<ov::frontend::ConversionExtension>("NewCustomOp_1", CustomTranslatorCommon_1),
std::make_shared<ov::frontend::ConversionExtension>("NewCustomOp_2", CustomTranslatorCommon_2),
ONNX_EXT PADDLE_EXT TF_EXT}));

41
src/frontends/tests/frontend/shared/test_builtin_extensions_1/test_extension.cpp
View File

@ -1,41 +0,0 @@
// Copyright (C) 2018-2022 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "test_extension.hpp"
#include <openvino/core/core.hpp>
#include "openvino/opsets/opset8.hpp"
bool TestExtension1::transform(const std::shared_ptr<ov::Model>& function, const std::string& config) const {
function->set_friendly_name("TestFunction");
return true;
}
TestExtension1::TestExtension1() : ov::frontend::JsonTransformationExtension("buildin_extensions_1::TestExtension1") {}
ov::OutputVector CustomTranslatorCommon_1(const ov::frontend::NodeContext& node) {
return ov::OutputVector();
}
std::map<std::string, ov::OutputVector> CustomTranslatorCommon_2(const ov::frontend::NodeContext& node) {
return std::map<std::string, ov::OutputVector>();
}
ov::OutputVector CustomTranslatorTensorflow(const ov::frontend::NodeContext& node) {
return ov::OutputVector();
}
ov::OutputVector CustomTranslatorONNX(const ov::frontend::NodeContext& node) {
return ov::OutputVector();
}
ov::OutputVector ReluToSwishTranslatorONNX(const ov::frontend::NodeContext& node) {
auto swish = std::make_shared<ov::opset8::Swish>(node.get_input(0));
return {swish};
}
std::map<std::string, ov::OutputVector> CustomTranslatorPaddle(const ov::frontend::NodeContext& node) {
return std::map<std::string, ov::OutputVector>();
}

27
src/frontends/tests/frontend/shared/test_builtin_extensions_1/test_extension.hpp
View File

@ -1,27 +0,0 @@
// Copyright (C) 2018-2022 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include <extension/json_transformation.hpp>
#include <openvino/frontend/node_context.hpp>
class TestExtension1 : public ov::frontend::JsonTransformationExtension {
public:
TestExtension1();
bool transform(const std::shared_ptr<ov::Model>& function, const std::string& config) const override;
};
ov::OutputVector CustomTranslatorCommon_1(const ov::frontend::NodeContext& node);
std::map<std::string, ov::OutputVector> CustomTranslatorCommon_2(const ov::frontend::NodeContext& node);
ov::OutputVector CustomTranslatorTensorflow(const ov::frontend::NodeContext& node);
ov::OutputVector CustomTranslatorONNX(const ov::frontend::NodeContext& node);
ov::OutputVector ReluToSwishTranslatorONNX(const ov::frontend::NodeContext& node);
std::map<std::string, ov::OutputVector> CustomTranslatorPaddle(const ov::frontend::NodeContext& node);

24
src/frontends/tests/frontend/shared/test_builtin_extensions_2/CMakeLists.txt
View File

@ -1,24 +0,0 @@
# Copyright (C) 2018-2022 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
#
set(TARGET_NAME "test_builtin_extensions_2")
if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /ignore:4217")
set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} /ignore:4217")
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /ignore:4217")
endif()
file(GLOB_RECURSE LIBRARY_SRC ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp)
file(GLOB_RECURSE LIBRARY_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/*.hpp)
# Create library
add_library(${TARGET_NAME} MODULE ${LIBRARY_SRC} ${LIBRARY_HEADERS})
target_link_libraries(${TARGET_NAME} PRIVATE openvino::runtime::dev offline_transformations)
add_clang_format_target(${TARGET_NAME}_clang FOR_TARGETS ${TARGET_NAME})
install(TARGETS ${TARGET_NAME}
LIBRARY DESTINATION tests COMPONENT tests EXCLUDE_FROM_ALL)

10
src/frontends/tests/frontend/shared/test_builtin_extensions_2/builtin_extensions.cpp
View File

@ -1,10 +0,0 @@
// Copyright (C) 2018-2022 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include <openvino/core/extension.hpp>
#include "test_extensions.hpp"
OPENVINO_CREATE_EXTENSIONS(std::vector<ov::Extension::Ptr>({std::make_shared<TestExtension1>(),
std::make_shared<TestExtension2>()}));

21
src/frontends/tests/frontend/shared/test_builtin_extensions_2/test_extensions.cpp
View File

@ -1,21 +0,0 @@
// Copyright (C) 2018-2022 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "test_extensions.hpp"
#include <openvino/core/core.hpp>
bool TestExtension1::transform(const std::shared_ptr<ov::Model>& function, const std::string& config) const {
function->set_friendly_name("TestFunction");
return true;
}
TestExtension1::TestExtension1() : ov::frontend::JsonTransformationExtension("buildin_extensions_2::TestExtension1") {}
bool TestExtension2::transform(const std::shared_ptr<ov::Model>& function, const std::string& config) const {
function->set_friendly_name("TestFunction");
return true;
}
TestExtension2::TestExtension2() : ov::frontend::JsonTransformationExtension("buildin_extensions_2::TestExtension2") {}

21
src/frontends/tests/frontend/shared/test_builtin_extensions_2/test_extensions.hpp
View File

@ -1,21 +0,0 @@
// Copyright (C) 2018-2022 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include <extension/json_transformation.hpp>
class TestExtension1 : public ov::frontend::JsonTransformationExtension {
public:
TestExtension1();
bool transform(const std::shared_ptr<ov::Model>& function, const std::string& config) const override;
};
class TestExtension2 : public ov::frontend::JsonTransformationExtension {
public:
TestExtension2();
bool transform(const std::shared_ptr<ov::Model>& function, const std::string& config) const override;
};

3
thirdparty/CMakeLists.txt vendored
View File

@ -289,10 +289,9 @@ endif()
add_subdirectory(json)
openvino_developer_export_targets(COMPONENT openvino_common
TARGETS nlohmann_json nlohmann_json_schema_validator)
TARGETS nlohmann_json)
ov_install_static_lib(nlohmann_json ${OV_CPACK_COMP_CORE})
ov_install_static_lib(nlohmann_json_schema_validator ${OV_CPACK_COMP_CORE})
#
# Install

27
thirdparty/json/CMakeLists.txt vendored
View File

@ -5,30 +5,3 @@
# suppress shadowing names warning
set(JSON_SystemInclude ON CACHE BOOL "" FORCE)
add_subdirectory(nlohmann_json EXCLUDE_FROM_ALL)
# disabled due to compiler issues
set(CMAKE_INTERPROCEDURAL_OPTIMIZATION_RELEASE OFF)
set(BUILD_SHARED_LIBS OFF)
set(JSON_VALIDATOR_BUILD_TESTS OFF CACHE BOOL "" FORCE)
set(JSON_VALIDATOR_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE)
add_subdirectory(nlohmann_json_schema_validator EXCLUDE_FROM_ALL)
set_target_properties(nlohmann_json_schema_validator PROPERTIES FOLDER thirdparty)
if(CMAKE_COMPILER_IS_GNUCXX OR OV_COMPILER_IS_CLANG)
target_compile_options(nlohmann_json_schema_validator PRIVATE -Wno-switch -Wno-deprecated-declarations -Wno-range-loop-construct)
elseif(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
target_compile_options(nlohmann_json_schema_validator PRIVATE /wd4996)
endif()
# WA for CentOS with gcc < 4.9: override defitions to remove boost usage
set_property(TARGET nlohmann_json_schema_validator PROPERTY COMPILE_DEFINITIONS "")
set_property(TARGET nlohmann_json_schema_validator PROPERTY INTERFACE_LINK_LIBRARIES nlohmann_json)
if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.9)
target_compile_definitions(nlohmann_json_schema_validator PRIVATE JSON_SCHEMA_NO_REGEX)
target_compile_options(nlohmann_json_schema_validator PRIVATE -Wno-missing-field-initializers -Wno-shadow)
endif()
# WA for nlohmann_json_schema_validator:
# nlohmann_json must be in the same export set of nlohmann_json_schema_validator targets
install(TARGETS nlohmann_json EXPORT nlohmann_json_schema_validatorTargets
ARCHIVE DESTINATION lib EXCLUDE_FROM_ALL)

1
thirdparty/json/nlohmann_json_schema_validator vendored

@ -1 +0,0 @@
Subproject commit 27fc1d094503623dfe39365ba82581507524545c

9
tools/mo/openvino/tools/mo/convert_impl.py
View File

@ -18,8 +18,7 @@ except ImportError:
import openvino.tools.mo.utils.telemetry_stub as tm
from openvino.tools.mo.back.SpecialNodesFinalization import RemoveConstOps, CreateConstNodesReplacement, NormalizeTI
from openvino.tools.mo.moc_frontend.check_config import legacy_transformations_config_used, \
new_extensions_used, new_transformations_config_used
from openvino.tools.mo.moc_frontend.check_config import legacy_transformations_config_used, new_extensions_used
from openvino.tools.mo.moc_frontend.pipeline import moc_pipeline
from openvino.tools.mo.moc_frontend.serialize import moc_emit_ir
from openvino.tools.mo.graph.graph import Graph
@ -47,7 +46,7 @@ from openvino.tools.mo.front.common.partial_infer.utils import mo_array
from openvino.tools.mo.moc_frontend.check_config import legacy_extensions_used
# pylint: disable=no-name-in-module,import-error
from openvino.frontend import FrontEndManager, ProgressReporterExtension, TelemetryExtension, JsonConfigExtension
from openvino.frontend import FrontEndManager, ProgressReporterExtension, TelemetryExtension
from openvino.runtime import PartialShape, Dimension
from openvino.runtime import get_version as get_rt_version
@ -170,8 +169,6 @@ def arguments_post_parsing(argv: argparse.Namespace):
if is_legacy_frontend:
if new_extensions_used(argv):
raise Error('New kind of extensions used on legacy path')
if new_transformations_config_used(argv):
raise Error('New kind of transformations configuration used on legacy path')
if is_tf and not argv.input_model and not argv.saved_model_dir and not argv.input_meta_graph:
raise Error('Path to input model or saved model dir is required: use --input_model, --saved_model_dir or '
@ -392,8 +389,6 @@ def prepare_ir(argv: argparse.Namespace):
raise Error('Legacy extensions are not supported for the new frontend')
if legacy_extensions_used(argv):
raise Error('Legacy transformations configuration is not supported for the new frontend')
if new_transformations_config_used(argv):
moc_front_end.add_extension(JsonConfigExtension(argv.transformations_config))
if new_extensions_used(argv):
for extension in argv.extensions:
moc_front_end.add_extension(extension)

37
tools/mo/openvino/tools/mo/moc_frontend/check_config.py
View File

@ -74,26 +74,6 @@ def new_extensions_used(argv: argparse.Namespace):
return False
def is_new_json_config(json_file_path: str):
with open(json_file_path) as stream:
config_content = json.load(stream)
if len(config_content) == 0: # empty case
return False
if isinstance(config_content, dict): # single transformation
return 'library' in config_content.keys()
# many transformations in single file
library_counter = 0
for transform in config_content:
if any(key == 'library' for key in transform.keys()):
library_counter+=1
if len(config_content) == library_counter: # all transformations has 'library' attribute
return True
elif library_counter == 0: # all transformations are legacy type
return False
else:
raise Error('Mixed types of transformations configurations were used')
def get_transformations_config_path(argv: argparse.Namespace) -> Path:
if hasattr(argv, 'transformations_config') \
and argv.transformations_config is not None and len(argv.transformations_config):
@ -104,23 +84,8 @@ def get_transformations_config_path(argv: argparse.Namespace) -> Path:
return None
def new_transformations_config_used(argv: argparse.Namespace):
path = get_transformations_config_path(argv)
if path != None:
return is_new_json_config(path)
if hasattr(argv, 'transformations_config') \
and argv.transformations_config is not None and not isinstance(argv.transformations_config, str):
return True
return False
def legacy_transformations_config_used(argv: argparse.Namespace):
path = get_transformations_config_path(argv)
if path != None:
return not is_new_json_config(path)
return False
return get_transformations_config_path(argv) != None
def input_freezig_used(argv):

37
tools/mo/unit_tests/mo/extensions_test_actual.py
View File

@ -66,7 +66,7 @@ def get_builtin_extensions_path():
for lib_path in chain(
win_folder_path.glob("*.dll"), linux_folder_path.glob("*.so")
):
if "libtest_builtin_extensions_1" in lib_path.name:
if "libtest_builtin_extensions" in lib_path.name:
return str(lib_path)
return ""
@ -91,34 +91,12 @@ class TestMoFallback(unittest.TestCase):
opset_imports=[onnx.helper.make_opsetid("", 13)],
)
self.models["test_model.onnx"] = model
self.test_config_files = {}
self.test_config_files[
"test_config.json"
] = """[
{
"custom_attributes": {
"test_attribute": true
},
"id": "buildin_extensions_1::TestExtension1",
"library": "library_path",
"match_kind": "scope"
}
]""".replace(
"library_path", get_builtin_extensions_path()
)
for name, model in self.models.items():
onnx.save(model, name)
for name, config in self.test_config_files.items():
with open(name, "w") as f:
f.write(config)
def tearDown(self):
for name in self.models.keys():
os.remove(name)
for name in self.test_config_files.keys():
os.remove(name)
@pytest.mark.skipif(
len(get_builtin_extensions_path()) == 0,
@ -133,16 +111,3 @@ class TestMoFallback(unittest.TestCase):
assert any(op.get_type_name() == "Swish" for op in model.get_ops())
assert all(op.get_type_name() != "Relu" for op in model.get_ops())
@pytest.mark.skipif(
len(get_builtin_extensions_path()) == 0,
reason="The extension library path was not found",
)
def test_conersion_if_transformations_config_is_used(self):
args = base_args_config()
args.input_model = "test_model.onnx"
args.transformations_config = "test_config.json"
graph, model = prepare_ir(args)
assert model.get_friendly_name() == "TestFunction"

110
tools/mo/unit_tests/mo/utils/mo_fallback_test_actual.py
View File

@ -101,39 +101,7 @@ class TestMoFallback(unittest.TestCase):
self.test_config_files = {}
self.test_config_files['fake_config.json'] = '[]' # json format
self.test_config_files['test_config_1.json'] = """[
{
"custom_attributes": {
"test_attribute": true
},
"id": "TransformationName1",
"library": "path_to_library1.so",
"match_kind": "scope"
},
{
"custom_attributes": {
},
"id": "TransfromationName2",
"library": "path_to_library2.so",
"match_kind": "scope"
},
{
"id": "TransfromationName3",
"library": "path_to_library3.so",
"match_kind": "scope"
}
]"""
self.test_config_files['test_config_2.json'] = """{
"custom_attributes": {
"test_attribute": true
},
"id": "TransformationName1",
"library": "path_to_library.so",
"match_kind": "scope"
}"""
self.test_config_files['test_config_3.json'] = """[
self.test_config_files['test_config.json'] = """[
{
"custom_attributes": {
"test_attribute": true
@ -149,26 +117,6 @@ class TestMoFallback(unittest.TestCase):
}
]"""
self.test_config_files['test_config_4.json'] = """[
{
"custom_attributes": {
"test_attribute": true
},
"id": "TransformationName1",
"library": "path_to_library",
"match_kind": "scope"
},
{
"custom_attributes": {
},
"id": "TransfromationName2",
"match_kind": "scope"
},
{
"library": "path_to_library.so"
}
]"""
self.test_config_files['onnx_fe_ext.so'] = 'binary_content'
self.test_config_files['onnx_fe_ext_2.so'] = 'binary_content'
@ -265,6 +213,7 @@ class TestMoFallback(unittest.TestCase):
@generate(*[('fake_config.json' , None, None, 'mo_legacy', 'transformations_config'), # fallback
('test_config.json' , None, None, 'mo_legacy', 'transformations_config'), # fallback
('fake_config.json' , True, None, 'mo_legacy', None),
(None, None, True, 'onnx_frontend', None),
(None, None, None, 'onnx_frontend', None),
@ -276,7 +225,8 @@ class TestMoFallback(unittest.TestCase):
args.input_model = "test_model.onnx"
args.transformations_config = trans_config
prepare_ir(args)
with patch('openvino.tools.mo.utils.class_registration.apply_transform'): # skip applying transforms
prepare_ir(args)
tm.Telemetry.send_event.assert_any_call('mo', 'conversion_method', expected_path)
if fallback_reason:
@ -286,55 +236,6 @@ class TestMoFallback(unittest.TestCase):
tm.Telemetry.send_event.assert_any_call('mo', 'fallback_reason', fallback_reason)
@generate(*[('test_config_1.json', None, None, 'onnx_frontend', None), # 'library' attribute for all transformations
('test_config_2.json', None, None, 'onnx_frontend', None), # 'library' attribute in single transformation
('test_config_3.json', None, None, 'mo_legacy', 'transformations_config'), # 'library' attribute in no transformations
])
def test_fallback_if_new_tranformations_config_specified(self, trans_config, use_legacy, use_new_fe, conversion_method, fallback_reason):
with patch('openvino.tools.mo.convert_impl.get_default_frontends') as default_fe:
default_fe.return_value = get_test_default_frontends()
args = base_args_config(use_legacy, use_new_fe)
args.input_model = "test_model.onnx"
args.transformations_config = trans_config
with patch('openvino.tools.mo.utils.class_registration.apply_transform'): # skip applying transforms
if conversion_method == 'onnx_frontend':
with pytest.raises(RuntimeError): # workaround to use in tests not existed libaries
prepare_ir(args)
else:
prepare_ir(args)
tm.Telemetry.send_event.assert_any_call('mo', 'conversion_method', conversion_method)
if fallback_reason:
tm.Telemetry.send_event.assert_any_call('mo', 'fallback_reason', fallback_reason)
else:
with pytest.raises(AssertionError): # not called
tm.Telemetry.send_event.assert_any_call('mo', 'fallback_reason', fallback_reason)
def test_exception_if_new_trans_config_on_legacy_path(self):
with patch('openvino.tools.mo.convert_impl.get_default_frontends') as default_fe:
default_fe.return_value = get_test_default_frontends()
args = base_args_config(use_legacy_fe=True)
args.input_model = "test_model.onnx"
args.transformations_config = 'test_config_1.json'
with pytest.raises(Error) as ex: # not called
prepare_ir(args)
assert str(ex) == 'New kind of transformations configuration used on legacy path'
def test_exeption_if_mixed_types_of_trans_configs(self):
with patch('openvino.tools.mo.convert_impl.get_default_frontends') as default_fe:
default_fe.return_value = get_test_default_frontends()
args = base_args_config()
args.input_model = "test_model.onnx"
args.transformations_config = 'test_config_4.json'
with pytest.raises(Error):
prepare_ir(args)
@generate(*[(['dir_to_extension'], 'fake_config.json', None, 'mo_legacy', 'extensions, transformations_config'), # fallback
(None, 'fake_config.json', None, 'mo_legacy', 'transformations_config'), # fallback
(['dir_to_extension'], None, None, 'mo_legacy', 'extensions'), # fallback
@ -376,8 +277,7 @@ class TestMoFallback(unittest.TestCase):
tm.Telemetry.send_event.assert_any_call('mo', 'fallback_reason')
@generate(*[(None, None, ['test_config_1.json'], 'paddle_frontend'),
(True, None, None, 'paddle_frontend'),
@generate(*[(True, None, None, 'paddle_frontend'),
(None, None, None, 'paddle_frontend'),
])
def test_no_fallback_if_pdpd(self, use_new_fe, use_legacy, extension, expected_path):