From e53f4cf2f6dbe0da5d7f436ed0a9e54ba3f3b4dd Mon Sep 17 00:00:00 2001 From: Ivan Tikhonov Date: Mon, 6 Dec 2021 17:24:55 +0300 Subject: [PATCH] Squash commits: JsonConfig/JsonTranformation/DecoderTransformation extensions, Unit tests, pybindings --- .gitmodules | 5 +- model-optimizer/mo/main.py | 3 +- .../python/src/openvino/frontend/__init__.py | 2 + .../python/src/pyopenvino/CMakeLists.txt | 3 +- .../src/pyopenvino/frontend/frontend.cpp | 24 ++- .../src/pyopenvino/frontend/frontend.hpp | 2 + src/core/tests/CMakeLists.txt | 3 +- .../decoder_transformation_extension.cpp | 54 +++++ .../frontend/onnx/json_config_extension.cpp | 24 +++ .../paddlepaddle/json_config_extension.cpp | 24 +++ .../test_models/gen_scripts/generate_slice.py | 192 ++++++++--------- src/core/tests/frontend/shared/CMakeLists.txt | 7 +- .../shared/include/json_config_extension.hpp | 44 ++++ .../shared/include/json_files/test.json | 26 +++ .../include/json_files/test_config.json | 26 +++ .../shared/src/json_config_extension.cpp | 194 ++++++++++++++++++ .../tests/frontend/shared/src/telemetry.cpp | 2 +- .../test_builtin_extensions_1/CMakeLists.txt | 20 ++ .../builtin_extensions.cpp | 9 + .../test_extension.cpp | 14 ++ .../test_extension.hpp | 15 ++ .../test_builtin_extensions_2/CMakeLists.txt | 20 ++ .../builtin_extensions.cpp | 10 + .../test_extensions.cpp | 21 ++ .../test_extensions.hpp | 22 ++ .../tensorflow/json_config_extension.cpp | 24 +++ src/frontends/common/CMakeLists.txt | 8 +- .../decoder_transformation_extension.hpp | 56 +++++ .../extensions/json_config_extension.hpp | 34 +++ .../include/common/extensions/json_schema.hpp | 134 ++++++++++++ .../json_transformation_extension.hpp | 52 +++++ .../{ => extensions}/telemetry_extension.hpp | 2 +- .../decoder_transformation_extension.cpp | 47 +++++ .../src/extensions/json_config_extension.cpp | 79 +++++++ .../{ => extensions}/telemetry_extension.cpp | 2 +- .../ir/include/ir_frontend/frontend.hpp | 2 +- .../include/onnx_frontend/frontend.hpp | 3 + .../onnx/frontend/src/core/graph.hpp | 2 +- src/frontends/onnx/frontend/src/editor.hpp | 2 +- src/frontends/onnx/frontend/src/frontend.cpp | 17 +- .../onnx/frontend/src/utils/onnx_internal.hpp | 2 +- .../paddlepaddle_frontend/frontend.hpp | 4 +- .../include/paddlepaddle_frontend/model.hpp | 2 +- src/frontends/paddlepaddle/src/frontend.cpp | 16 ++ .../include/tensorflow_frontend/frontend.hpp | 4 +- src/frontends/tensorflow/src/frontend.cpp | 18 +- src/frontends/tensorflow/src/model.hpp | 2 +- thirdparty/CMakeLists.txt | 12 +- thirdparty/{ => json}/nlohmann_json | 0 .../json/nlohmann_json_schema_validator | 1 + 50 files changed, 1171 insertions(+), 120 deletions(-) create mode 100644 src/core/tests/frontend/decoder_transformation_extension.cpp create mode 100644 src/core/tests/frontend/onnx/json_config_extension.cpp create mode 100644 src/core/tests/frontend/paddlepaddle/json_config_extension.cpp create mode 100644 src/core/tests/frontend/shared/include/json_config_extension.hpp create mode 100644 src/core/tests/frontend/shared/include/json_files/test.json create mode 100644 src/core/tests/frontend/shared/include/json_files/test_config.json create mode 100644 src/core/tests/frontend/shared/src/json_config_extension.cpp create mode 100644 src/core/tests/frontend/shared/test_builtin_extensions_1/CMakeLists.txt create mode 100644 src/core/tests/frontend/shared/test_builtin_extensions_1/builtin_extensions.cpp create mode 100644 src/core/tests/frontend/shared/test_builtin_extensions_1/test_extension.cpp create mode 100644 src/core/tests/frontend/shared/test_builtin_extensions_1/test_extension.hpp create mode 100644 src/core/tests/frontend/shared/test_builtin_extensions_2/CMakeLists.txt create mode 100644 src/core/tests/frontend/shared/test_builtin_extensions_2/builtin_extensions.cpp create mode 100644 src/core/tests/frontend/shared/test_builtin_extensions_2/test_extensions.cpp create mode 100644 src/core/tests/frontend/shared/test_builtin_extensions_2/test_extensions.hpp create mode 100644 src/core/tests/frontend/tensorflow/json_config_extension.cpp create mode 100644 src/frontends/common/include/common/extensions/decoder_transformation_extension.hpp create mode 100644 src/frontends/common/include/common/extensions/json_config_extension.hpp create mode 100644 src/frontends/common/include/common/extensions/json_schema.hpp create mode 100644 src/frontends/common/include/common/extensions/json_transformation_extension.hpp rename src/frontends/common/include/common/{ => extensions}/telemetry_extension.hpp (97%) create mode 100644 src/frontends/common/src/extensions/decoder_transformation_extension.cpp create mode 100644 src/frontends/common/src/extensions/json_config_extension.cpp rename src/frontends/common/src/{ => extensions}/telemetry_extension.cpp (95%) rename thirdparty/{ => json}/nlohmann_json (100%) create mode 160000 thirdparty/json/nlohmann_json_schema_validator diff --git a/.gitmodules b/.gitmodules index 4b624c6f867..05fcb87dade 100644 --- a/.gitmodules +++ b/.gitmodules @@ -60,5 +60,8 @@ path = tools/pot/thirdparty/open_model_zoo url = https://github.com/openvinotoolkit/open_model_zoo.git [submodule "thirdparty/nlohmann_json"] - path = thirdparty/nlohmann_json + path = thirdparty/json/nlohmann_json url = https://github.com/nlohmann/json.git +[submodule "thirdparty/json/nlohmann_json_schema_validator"] + path = thirdparty/json/nlohmann_json_schema_validator + url = https://github.com/pboettch/json-schema-validator.git diff --git a/model-optimizer/mo/main.py b/model-optimizer/mo/main.py index b81d50a3c99..74965c4b48c 100644 --- a/model-optimizer/mo/main.py +++ b/model-optimizer/mo/main.py @@ -45,7 +45,7 @@ from mo.utils.versions_checker import check_requirements # pylint: disable=no-n from mo.utils.telemetry_utils import get_tid # pylint: disable=no-name-in-module,import-error -from openvino.frontend import FrontEndManager, TelemetryExtension +from openvino.frontend import FrontEndManager, TelemetryExtension, JsonConfigExtension def replace_ext(name: str, old: str, new: str): @@ -324,6 +324,7 @@ def prepare_ir(argv): if moc_front_end: t.send_event("mo", "conversion_method", moc_front_end.get_name() + "_frontend") moc_front_end.add_extension(TelemetryExtension("mo", t.send_event, t.send_error, t.send_stack_trace)) + moc_front_end.add_extension(JsonConfigExtension(argv.transformations_config)) ngraph_function = moc_pipeline(argv, moc_front_end) else: t.send_event("mo", "conversion_method", "mo_legacy") diff --git a/src/bindings/python/src/openvino/frontend/__init__.py b/src/bindings/python/src/openvino/frontend/__init__.py index a008675fce0..156b240e00a 100644 --- a/src/bindings/python/src/openvino/frontend/__init__.py +++ b/src/bindings/python/src/openvino/frontend/__init__.py @@ -41,6 +41,8 @@ from openvino.pyopenvino import FrontEnd from openvino.pyopenvino import InputModel from openvino.pyopenvino import Place from openvino.pyopenvino import TelemetryExtension +from openvino.pyopenvino import DecoderTransformationExtension +from openvino.pyopenvino import JsonConfigExtension # exceptions from openvino.pyopenvino import NotImplementedFailure diff --git a/src/bindings/python/src/pyopenvino/CMakeLists.txt b/src/bindings/python/src/pyopenvino/CMakeLists.txt index 36bf8300152..e3f89882a4c 100644 --- a/src/bindings/python/src/pyopenvino/CMakeLists.txt +++ b/src/bindings/python/src/pyopenvino/CMakeLists.txt @@ -57,7 +57,8 @@ else() endif() target_include_directories(${PROJECT_NAME} PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/..") -target_link_libraries(${PROJECT_NAME} PRIVATE openvino::runtime ${OFFLINE_TRANSFORMATIONS_LIB} openvino::frontend::common) +target_link_libraries(${PROJECT_NAME} PRIVATE openvino::runtime ${OFFLINE_TRANSFORMATIONS_LIB} openvino::frontend::common + nlohmann_json) # perform copy if(OpenVINO_SOURCE_DIR) diff --git a/src/bindings/python/src/pyopenvino/frontend/frontend.cpp b/src/bindings/python/src/pyopenvino/frontend/frontend.cpp index 8e3c2d4b10c..775737b2df4 100644 --- a/src/bindings/python/src/pyopenvino/frontend/frontend.cpp +++ b/src/bindings/python/src/pyopenvino/frontend/frontend.cpp @@ -8,7 +8,9 @@ #include #include "common/frontend_exceptions.hpp" -#include "common/telemetry_extension.hpp" +#include "common/extensions/telemetry_extension.hpp" +#include "common/extensions/decoder_transformation_extension.hpp" +#include "common/extensions/json_config_extension.hpp" #include "manager.hpp" #include "pyopenvino/graph/function.hpp" @@ -166,3 +168,23 @@ void regclass_frontend_TelemetryExtension(py::module m) { ext.def("send_stack_trace", &TelemetryExtension::send_stack_trace); } } + +void regclass_frontend_TransformationDecoderExtension(py::module m) { + { + // MatherPass/FunctionPass in python + } +} + +void regclass_frontend_JsonConfigExtension(py::module m) { + { + py::class_, + ov::Extension> ext(m, "JsonConfigExtension", py::dynamic_attr()); + + ext.doc() = "Extension class to load and process ModelOptimizer JSON config file"; + + ext.def(py::init([](const std::string& path) { + return std::make_shared(path); + })); + } +} \ No newline at end of file diff --git a/src/bindings/python/src/pyopenvino/frontend/frontend.hpp b/src/bindings/python/src/pyopenvino/frontend/frontend.hpp index b6df63e9c26..f4f363ba9b9 100644 --- a/src/bindings/python/src/pyopenvino/frontend/frontend.hpp +++ b/src/bindings/python/src/pyopenvino/frontend/frontend.hpp @@ -11,3 +11,5 @@ namespace py = pybind11; void regclass_frontend_FrontEnd(py::module m); void regclass_frontend_Extension(py::module m); void regclass_frontend_TelemetryExtension(py::module m); +void regclass_frontend_TransformationDecoderExtension(py::module m); +void regclass_frontend_JsonConfigExtension(py::module m); diff --git a/src/core/tests/CMakeLists.txt b/src/core/tests/CMakeLists.txt index 23df28547a9..2af1279d164 100644 --- a/src/core/tests/CMakeLists.txt +++ b/src/core/tests/CMakeLists.txt @@ -531,7 +531,8 @@ if (NGRAPH_ONNX_FRONTEND_ENABLE) endif() # SOURCE FOR FRONTEND TESTING -file(GLOB FRONTEND_TESTS_SRC ${CMAKE_CURRENT_SOURCE_DIR}/frontend/frontend_manager.cpp) +file(GLOB FRONTEND_TESTS_SRC ${CMAKE_CURRENT_SOURCE_DIR}/frontend/frontend_manager.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/frontend/decoder_transformation_extension.cpp) list(APPEND SRC ${FRONTEND_TESTS_SRC}) foreach(src IN LISTS SRC MULTI_TEST_SRC) diff --git a/src/core/tests/frontend/decoder_transformation_extension.cpp b/src/core/tests/frontend/decoder_transformation_extension.cpp new file mode 100644 index 00000000000..43b286ba4e1 --- /dev/null +++ b/src/core/tests/frontend/decoder_transformation_extension.cpp @@ -0,0 +1,54 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include +#include +#include "gtest/gtest.h" + +using namespace ov::frontend; + +TEST(DecoderTransformation, MatcherPass) { + bool flag = false; + DecoderTransformationExtension decoder_ext([&](ov::pass::MatcherPass* matcher){ + flag = true; + }); + + ov::pass::Manager manager; + decoder_ext.register_pass(manager); + manager.run_passes(std::make_shared(ov::ResultVector{}, ov::ParameterVector{})); + EXPECT_EQ(flag, true); +} + +TEST(DecoderTransformation, FunctionPass) { + bool flag = false; + DecoderTransformationExtension decoder_ext([&](const std::shared_ptr&){ + flag = true; + return flag; + }); + + ov::pass::Manager manager; + decoder_ext.register_pass(manager); + manager.run_passes(std::make_shared(ov::ResultVector{}, ov::ParameterVector{})); + EXPECT_EQ(flag, true); +} + +TEST(DecoderTransformation, TestPass) { + class TestPass : public ov::pass::FunctionPass { + public: + OPENVINO_RTTI("ov::pass::TestPass"); + TestPass() = default; + TestPass(const TestPass& tp) = default; + bool run_on_function(std::shared_ptr) override { + *m_flag = true; + return *m_flag; + } + std::shared_ptr m_flag = std::make_shared(false); + } test_pass; + DecoderTransformationExtension decoder_ext(test_pass); + + ov::pass::Manager manager; + decoder_ext.register_pass(manager); + manager.run_passes(std::make_shared(ov::ResultVector{}, ov::ParameterVector{})); + EXPECT_EQ(*test_pass.m_flag, true); +} diff --git a/src/core/tests/frontend/onnx/json_config_extension.cpp b/src/core/tests/frontend/onnx/json_config_extension.cpp new file mode 100644 index 00000000000..4becbe935c7 --- /dev/null +++ b/src/core/tests/frontend/onnx/json_config_extension.cpp @@ -0,0 +1,24 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "json_config_extension.hpp" + +#include "onnx_utils.hpp" + +using namespace ov::frontend; + +using ONNXJsonConfigTest = FrontEndJsonConfigTest; + +static JsonConfigFEParam getTestData() { + JsonConfigFEParam res; + res.m_frontEndName = ONNX_FE; + res.m_modelsPath = std::string(TEST_ONNX_MODELS_DIRNAME); + res.m_modelName = "controlflow/loop_2d_add.onnx"; + return res; +} + +INSTANTIATE_TEST_SUITE_P(ONNXJsonConfigTest, + FrontEndJsonConfigTest, + ::testing::Values(getTestData()), + FrontEndJsonConfigTest::getTestCaseName); diff --git a/src/core/tests/frontend/paddlepaddle/json_config_extension.cpp b/src/core/tests/frontend/paddlepaddle/json_config_extension.cpp new file mode 100644 index 00000000000..dffba1ea798 --- /dev/null +++ b/src/core/tests/frontend/paddlepaddle/json_config_extension.cpp @@ -0,0 +1,24 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "json_config_extension.hpp" + +#include "paddle_utils.hpp" + +using namespace ov::frontend; + +using PDPDJsonConfigTest = FrontEndJsonConfigTest; + +static JsonConfigFEParam getTestData() { + JsonConfigFEParam res; + res.m_frontEndName = PADDLE_FE; + res.m_modelsPath = std::string(TEST_PADDLE_MODELS_DIRNAME); + res.m_modelName = "relu/relu.pdmodel"; + return res; +} + +INSTANTIATE_TEST_SUITE_P(PDPDJsonConfigTest, + FrontEndJsonConfigTest, + ::testing::Values(getTestData()), + FrontEndJsonConfigTest::getTestCaseName); diff --git a/src/core/tests/frontend/paddlepaddle/test_models/gen_scripts/generate_slice.py b/src/core/tests/frontend/paddlepaddle/test_models/gen_scripts/generate_slice.py index 8c6b5fd1eeb..743c2a95de2 100644 --- a/src/core/tests/frontend/paddlepaddle/test_models/gen_scripts/generate_slice.py +++ b/src/core/tests/frontend/paddlepaddle/test_models/gen_scripts/generate_slice.py @@ -1,98 +1,98 @@ +# # +# # slice paddle model generator +# # +# import sys +# import os # -# slice paddle model generator +# import numpy as np +# import paddle as pdpd # -import sys -import os - -import numpy as np -import paddle as pdpd - -from save_model import exportModel -from save_model import saveModel - -data_type = 'float32' - -def slice(name : str, x, axes : list, start : list, end : list): - pdpd.enable_static() - - with pdpd.static.program_guard(pdpd.static.Program(), pdpd.static.Program()): - node_x = pdpd.static.data(name='x', shape=x.shape, dtype = data_type) - out = pdpd.fluid.layers.slice(node_x, axes = axes, starts = start, ends = end) - - cpu = pdpd.static.cpu_places(1) - exe = pdpd.static.Executor(cpu[0]) - # startup program will call initializer to initialize the parameters. - exe.run(pdpd.static.default_startup_program()) - - outs = exe.run( - feed={'x': x}, - fetch_list=[out]) - - saveModel(name, exe, feedkeys=['x'], fetchlist=[out], inputs=[x], outputs=[outs[0]], target_dir=sys.argv[1]) - - return outs[0] - - -def slice_dyn(test_shape=[2,8,10,10]): - pdpd.disable_static() - - data = pdpd.rand(shape=test_shape, dtype='float32') - - ''' - slice w/ decrease_axis - ''' - @pdpd.jit.to_static - def test_slice_decrease_axis(x): - return x[0, 1:3, :, 5] - exportModel('slice_decrease_axis', test_slice_decrease_axis, [data], target_dir=sys.argv[1]) # output shape (2, 10) - - ''' - slice w/o decrease_axis - ''' - @pdpd.jit.to_static - def test_slice(x): - return pdpd.slice(x, axes=[0,1,3], starts=[0,1,5], ends=[1,3,6]) - # exportModel('slice_dyn', test_slice, [data], target_dir=sys.argv[1]) # output shape (1, 2, 10, 1) # disable it by default as this kind of test model already there. It's for comparsion only. - - ''' - slice w/ decrease_axis of all dims - ''' - @pdpd.jit.to_static - def test_slice_decrease_axis_all(x): - return x[0, 0, 0, 0] - exportModel('slice_decrease_axis_all', test_slice_decrease_axis_all, [data], target_dir=sys.argv[1]) # output shape (1,) - - ''' - slice w/o decrease_axis of all dims - ''' - @pdpd.jit.to_static - def test_slice_alldim(x): - return pdpd.slice(x, axes=[0,1,2,3], starts=[0,0,0,0], ends=[1,1,1,1]) - # exportModel('slice_alldim', test_slice_alldim, [data], target_dir=sys.argv[1]) # output shape (1, 1, 1, 1) # disable it by default as this kind of test model already there. It's for comparsion only. - -''' -a test case simulating the last reshape2 of ocrnet which accepts slice (with decrease_axes in all dims) as its parents. -''' -def slice_reshape(B=1, C=256, H=16, W=32): - pdpd.disable_static() - - data = pdpd.rand(shape=[B, C, H*W], dtype='float32') - - @pdpd.jit.to_static - def test_model(x): - x2 = pdpd.assign([-1, -1, 16, 32]).astype('int32') - node_reshape = pdpd.reshape(x, [0, 256, x2[2], x2[3]]) - return node_reshape - exportModel('slice_reshape', test_model, [data], target_dir=sys.argv[1]) - -def main(): - x = np.linspace(1, 60, num = 60, dtype=np.int32).reshape(4, 3, 5).astype(data_type) - slice("slice", x, axes=[1, 2], start=(0, 1), end=(-1, 3)) - - x = np.linspace(1, 60, num = 60, dtype=np.int32).reshape(2, 30).astype(data_type) - slice("slice_1d", x, axes=[0], start=[0], end=[1]) - -if __name__ == "__main__": - main() - slice_dyn() - slice_reshape() \ No newline at end of file +# from save_model import exportModel +# from save_model import saveModel +# +# data_type = 'float32' +# +# def slice(name : str, x, axes : list, start : list, end : list): +# pdpd.enable_static() +# +# with pdpd.static.program_guard(pdpd.static.Program(), pdpd.static.Program()): +# node_x = pdpd.static.data(name='x', shape=x.shape, dtype = data_type) +# out = pdpd.fluid.layers.slice(node_x, axes = axes, starts = start, ends = end) +# +# cpu = pdpd.static.cpu_places(1) +# exe = pdpd.static.Executor(cpu[0]) +# # startup program will call initializer to initialize the parameters. +# exe.run(pdpd.static.default_startup_program()) +# +# outs = exe.run( +# feed={'x': x}, +# fetch_list=[out]) +# +# saveModel(name, exe, feedkeys=['x'], fetchlist=[out], inputs=[x], outputs=[outs[0]], target_dir=sys.argv[1]) +# +# return outs[0] +# +# +# def slice_dyn(test_shape=[2,8,10,10]): +# pdpd.disable_static() +# +# data = pdpd.rand(shape=test_shape, dtype='float32') +# +# ''' +# slice w/ decrease_axis +# ''' +# @pdpd.jit.to_static +# def test_slice_decrease_axis(x): +# return x[0, 1:3, :, 5] +# exportModel('slice_decrease_axis', test_slice_decrease_axis, [data], target_dir=sys.argv[1]) # output shape (2, 10) +# +# ''' +# slice w/o decrease_axis +# ''' +# @pdpd.jit.to_static +# def test_slice(x): +# return pdpd.slice(x, axes=[0,1,3], starts=[0,1,5], ends=[1,3,6]) +# # exportModel('slice_dyn', test_slice, [data], target_dir=sys.argv[1]) # output shape (1, 2, 10, 1) # disable it by default as this kind of test model already there. It's for comparsion only. +# +# ''' +# slice w/ decrease_axis of all dims +# ''' +# @pdpd.jit.to_static +# def test_slice_decrease_axis_all(x): +# return x[0, 0, 0, 0] +# exportModel('slice_decrease_axis_all', test_slice_decrease_axis_all, [data], target_dir=sys.argv[1]) # output shape (1,) +# +# ''' +# slice w/o decrease_axis of all dims +# ''' +# @pdpd.jit.to_static +# def test_slice_alldim(x): +# return pdpd.slice(x, axes=[0,1,2,3], starts=[0,0,0,0], ends=[1,1,1,1]) +# # exportModel('slice_alldim', test_slice_alldim, [data], target_dir=sys.argv[1]) # output shape (1, 1, 1, 1) # disable it by default as this kind of test model already there. It's for comparsion only. +# +# ''' +# a test case simulating the last reshape2 of ocrnet which accepts slice (with decrease_axes in all dims) as its parents. +# ''' +# def slice_reshape(B=1, C=256, H=16, W=32): +# pdpd.disable_static() +# +# data = pdpd.rand(shape=[B, C, H*W], dtype='float32') +# +# @pdpd.jit.to_static +# def test_model(x): +# x2 = pdpd.assign([-1, -1, 16, 32]).astype('int32') +# node_reshape = pdpd.reshape(x, [0, 256, x2[2], x2[3]]) +# return node_reshape +# exportModel('slice_reshape', test_model, [data], target_dir=sys.argv[1]) +# +# def main(): +# x = np.linspace(1, 60, num = 60, dtype=np.int32).reshape(4, 3, 5).astype(data_type) +# slice("slice", x, axes=[1, 2], start=(0, 1), end=(-1, 3)) +# +# x = np.linspace(1, 60, num = 60, dtype=np.int32).reshape(2, 30).astype(data_type) +# slice("slice_1d", x, axes=[0], start=[0], end=[1]) +# +# if __name__ == "__main__": +# main() +# slice_dyn() +# slice_reshape() \ No newline at end of file diff --git a/src/core/tests/frontend/shared/CMakeLists.txt b/src/core/tests/frontend/shared/CMakeLists.txt index ffcc6699b5f..b54216bd587 100644 --- a/src/core/tests/frontend/shared/CMakeLists.txt +++ b/src/core/tests/frontend/shared/CMakeLists.txt @@ -7,11 +7,14 @@ set(TARGET_NAME "frontend_shared_test_classes") file(GLOB_RECURSE LIBRARY_SRC ${CMAKE_CURRENT_SOURCE_DIR}/src/*.cpp) file(GLOB_RECURSE LIBRARY_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/include/*.hpp) +add_subdirectory(test_builtin_extensions_1) +add_subdirectory(test_builtin_extensions_2) + add_library(${TARGET_NAME} STATIC ${LIBRARY_SRC} ${LIBRARY_HEADERS}) target_include_directories(${TARGET_NAME} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include) target_include_directories(${TARGET_NAME} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/../..) -target_link_libraries(${TARGET_NAME} PUBLIC frontend_common - ngraph cnpy commonTestUtils ie_backend ngraph_test_util engines_test_util openvino::util) +target_link_libraries(${TARGET_NAME} PUBLIC frontend_common nlohmann_json nlohmann_json_schema_validator + ngraph cnpy commonTestUtils ie_backend ngraph_test_util engines_test_util openvino::util test_builtin_extensions_1 test_builtin_extensions_2) add_clang_format_target(${TARGET_NAME}_clang FOR_TARGETS ${TARGET_NAME}) diff --git a/src/core/tests/frontend/shared/include/json_config_extension.hpp b/src/core/tests/frontend/shared/include/json_config_extension.hpp new file mode 100644 index 00000000000..d308cce06cc --- /dev/null +++ b/src/core/tests/frontend/shared/include/json_config_extension.hpp @@ -0,0 +1,44 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include + +#include +#include + +class JsonConfigExtensionWrapper : public ov::frontend::JsonConfigExtension { +public: + explicit JsonConfigExtensionWrapper(const std::string& config_path) + : ov::frontend::JsonConfigExtension(config_path){}; + ~JsonConfigExtensionWrapper() override{}; + + std::vector get_loaded_extensions() { + return m_loaded_extensions; + }; + + std::vector, nlohmann::json>> get_target_extensions() { + return m_target_extensions; + } +}; + +struct JsonConfigFEParam { + std::string m_frontEndName; + std::string m_modelsPath; + std::string m_modelName; +}; + +class FrontEndJsonConfigTest : public ::testing::TestWithParam { +public: + JsonConfigFEParam m_param; + ov::frontend::FrontEndManager m_fem; + + static std::string getTestCaseName(const testing::TestParamInfo& obj); + + void SetUp() override; + +protected: + void initParamTest(); +}; diff --git a/src/core/tests/frontend/shared/include/json_files/test.json b/src/core/tests/frontend/shared/include/json_files/test.json new file mode 100644 index 00000000000..bdf03a64257 --- /dev/null +++ b/src/core/tests/frontend/shared/include/json_files/test.json @@ -0,0 +1,26 @@ +[ + { + "custom_attributes": { + "test_attribute": true + }, + "id": "buildin_extensions_1::TestExtension1", + "library": "/home/itikhonov/OpenVINO/openvino/bin/intel64/Debug/lib/libtest_builtin_extensions_1.so", + "match_kind": "scope" + }, + { + "custom_attributes": { + "test_attribute": true + }, + "id": "buildin_extensions_2::TestExtension1", + "library": "/home/itikhonov/OpenVINO/openvino/bin/intel64/Debug/lib/libtest_builtin_extensions_2.so", + "match_kind": "scope" + }, + { + "custom_attributes": { + "test_attribute": true + }, + "id": "buildin_extensions_2::TestExtension2", + "library": "/home/itikhonov/OpenVINO/openvino/bin/intel64/Debug/lib/libtest_builtin_extensions_2.so", + "match_kind": "scope" + } +] diff --git a/src/core/tests/frontend/shared/include/json_files/test_config.json b/src/core/tests/frontend/shared/include/json_files/test_config.json new file mode 100644 index 00000000000..a2cb062ec20 --- /dev/null +++ b/src/core/tests/frontend/shared/include/json_files/test_config.json @@ -0,0 +1,26 @@ +[ + { + "custom_attributes": { + "test_attribute": true + }, + "id": "buildin_extensions_1::TestExtension1", + "library": "test_buildin_extensions_1", + "match_kind": "scope" + }, + { + "custom_attributes": { + "test_attribute": true + }, + "id": "buildin_extensions_2::TestExtension1", + "library": "test_buildin_extensions_2", + "match_kind": "scope" + }, + { + "custom_attributes": { + "test_attribute": true + }, + "id": "buildin_extensions_2::TestExtension2", + "library": "test_buildin_extensions_2", + "match_kind": "scope" + } +] diff --git a/src/core/tests/frontend/shared/src/json_config_extension.cpp b/src/core/tests/frontend/shared/src/json_config_extension.cpp new file mode 100644 index 00000000000..b65d3596ac8 --- /dev/null +++ b/src/core/tests/frontend/shared/src/json_config_extension.cpp @@ -0,0 +1,194 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "json_config_extension.hpp" + +#include +#include +#include +#include +#include +#include + +#include "utils.hpp" + +using namespace ov::frontend; + +std::string FrontEndJsonConfigTest::getTestCaseName(const testing::TestParamInfo& obj) { + std::string res = obj.param.m_frontEndName + "_" + obj.param.m_modelName; + return FrontEndTestUtils::fileToTestName(res); +} + +void FrontEndJsonConfigTest::SetUp() { + FrontEndTestUtils::setupTestEnv(); + m_fem = FrontEndManager(); // re-initialize after setting up environment + initParamTest(); +} + +void FrontEndJsonConfigTest::initParamTest() { + m_param = GetParam(); + m_param.m_modelName = FrontEndTestUtils::make_model_path(m_param.m_modelsPath + m_param.m_modelName); +} + +inline std::string get_lib_path(const std::string& lib_name) { + return ov::util::make_plugin_library_name(ov::util::get_ov_lib_path(), lib_name + IE_BUILD_POSTFIX); +} + +std::string generate_json_config() { + std::string json = R"( + [ + { + "custom_attributes": { + "test_attribute": true + }, + "id": "buildin_extensions_1::TestExtension1", + "library": "test_builtin_extensions_1", + "match_kind": "scope" + }, + { + "custom_attributes": { + "test_attribute": true + }, + "id": "buildin_extensions_2::TestExtension1", + "library": "test_builtin_extensions_2", + "match_kind": "scope" + }, + { + "custom_attributes": { + "test_attribute": true + }, + "id": "buildin_extensions_2::TestExtension2", + "library": "test_builtin_extensions_2", + "match_kind": "scope" + } + ] + )"; + nlohmann::json config_json = nlohmann::json::parse(json); + + // Validate JSON config + nlohmann::json_schema::json_validator validator; + validator.set_root_schema(json_schema); + validator.validate(config_json); + + for (auto& section : config_json) { + section["library"] = get_lib_path(section["library"]); + } + std::string path = "/home/itikhonov/OpenVINO/openvino/src/core/tests/frontend/shared/include/json_files/test.json"; + std::ofstream output_json(path); + output_json << std::setw(4) << config_json << std::endl; + return path; +} + +/////////////////////////////////////////////////////////////////// + +TEST_P(FrontEndJsonConfigTest, testJsonConfig) { + EXPECT_NO_THROW(generate_json_config()); +} + +TEST_P(FrontEndJsonConfigTest, testAddJsonConfigExtension) { + auto path_to_json = generate_json_config(); + std::map reference_map = { + {"buildin_extensions_1::TestExtension1", + nlohmann::json::parse( + R"( + { + "custom_attributes": { + "test_attribute": true + }, + "id": "buildin_extensions_1::TestExtension1", + "library": "test_builtin_extensions_1", + "match_kind": "scope" + } + )")}, + {"buildin_extensions_2::TestExtension1", + nlohmann::json::parse( + R"( + { + "custom_attributes": { + "test_attribute": true + }, + "id": "buildin_extensions_2::TestExtension1", + "library": "test_builtin_extensions_2", + "match_kind": "scope" + } + )")}, + {"buildin_extensions_2::TestExtension2", + nlohmann::json::parse( + R"( + { + "custom_attributes": { + "test_attribute": true + }, + "id": "buildin_extensions_2::TestExtension2", + "library": "test_builtin_extensions_2", + "match_kind": "scope" + } + )")}, + }; + for (auto& ref_val : reference_map) { + ref_val.second["library"] = get_lib_path(ref_val.second["library"]); + } + + std::shared_ptr function; + { + ov::frontend::FrontEnd::Ptr m_frontEnd; + ov::frontend::InputModel::Ptr m_inputModel; + m_frontEnd = m_fem.load_by_framework(m_param.m_frontEndName); + + auto json_config_ext = std::make_shared(path_to_json); + m_frontEnd->add_extension(json_config_ext); + + auto loaded_ext = json_config_ext->get_loaded_extensions(); + auto target_ext = json_config_ext->get_target_extensions(); + + EXPECT_EQ(loaded_ext.size(), 3); + EXPECT_EQ(target_ext.size(), 3); + + for (const auto& target : target_ext) { + auto transformation = std::dynamic_pointer_cast(target.first); + EXPECT_NE(transformation, nullptr); + EXPECT_EQ(reference_map.at(transformation->id()), target.second); + } + ASSERT_NO_THROW(m_inputModel = m_frontEnd->load(m_param.m_modelName)); + ASSERT_NE(m_inputModel, nullptr); + + ASSERT_NO_THROW(function = m_frontEnd->convert(m_inputModel)); + ASSERT_NE(function, nullptr); + EXPECT_EQ(function->get_friendly_name(), "TestFunction"); + } +} + +TEST_P(FrontEndJsonConfigTest, compareFunctions) { + auto path_to_json = generate_json_config(); + + std::shared_ptr function; + { + ov::frontend::FrontEnd::Ptr m_frontEnd; + ov::frontend::InputModel::Ptr m_inputModel; + m_frontEnd = m_fem.load_by_framework(m_param.m_frontEndName); + + auto json_config_ext = std::make_shared(path_to_json); + m_frontEnd->add_extension(json_config_ext); + + ASSERT_NO_THROW(m_inputModel = m_frontEnd->load(m_param.m_modelName)); + ASSERT_NE(m_inputModel, nullptr); + + ASSERT_NO_THROW(function = m_frontEnd->convert(m_inputModel)); + ASSERT_NE(function, nullptr); + } + + std::shared_ptr function_ref; + { + ov::frontend::FrontEnd::Ptr m_frontEnd; + ov::frontend::InputModel::Ptr m_inputModel; + m_frontEnd = m_fem.load_by_framework(m_param.m_frontEndName); + + ASSERT_NO_THROW(m_inputModel = m_frontEnd->load(m_param.m_modelName)); + ASSERT_NE(m_inputModel, nullptr); + + ASSERT_NO_THROW(function_ref = m_frontEnd->convert(m_inputModel)); + ASSERT_NE(function, nullptr); + } + compare_functions(function, function_ref); +} diff --git a/src/core/tests/frontend/shared/src/telemetry.cpp b/src/core/tests/frontend/shared/src/telemetry.cpp index 0802f8c8f98..1a417a25cde 100644 --- a/src/core/tests/frontend/shared/src/telemetry.cpp +++ b/src/core/tests/frontend/shared/src/telemetry.cpp @@ -4,7 +4,7 @@ #include "telemetry.hpp" -#include +#include #include "utils.hpp" diff --git a/src/core/tests/frontend/shared/test_builtin_extensions_1/CMakeLists.txt b/src/core/tests/frontend/shared/test_builtin_extensions_1/CMakeLists.txt new file mode 100644 index 00000000000..7ce0a6e376e --- /dev/null +++ b/src/core/tests/frontend/shared/test_builtin_extensions_1/CMakeLists.txt @@ -0,0 +1,20 @@ +# Copyright (C) 2018-2021 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +set(TARGET_NAME "test_builtin_extensions_1") + +file(GLOB_RECURSE LIBRARY_SRC ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp) +file(GLOB_RECURSE LIBRARY_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/*.hpp) + +# Create library +add_library(${TARGET_NAME} SHARED ${LIBRARY_SRC} ${LIBRARY_HEADERS}) + +target_link_libraries(${TARGET_NAME} PRIVATE nlohmann_json inference_engine_transformations frontend_common) + +add_clang_format_target(${TARGET_NAME}_clang FOR_TARGETS ${TARGET_NAME}) + +install(TARGETS ${TARGET_NAME} + RUNTIME DESTINATION tests + COMPONENT tests + EXCLUDE_FROM_ALL) \ No newline at end of file diff --git a/src/core/tests/frontend/shared/test_builtin_extensions_1/builtin_extensions.cpp b/src/core/tests/frontend/shared/test_builtin_extensions_1/builtin_extensions.cpp new file mode 100644 index 00000000000..ada294f7fc2 --- /dev/null +++ b/src/core/tests/frontend/shared/test_builtin_extensions_1/builtin_extensions.cpp @@ -0,0 +1,9 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include + +#include "test_extension.hpp" + +OPENVINO_CREATE_EXTENSIONS(std::vector({std::make_shared()})); diff --git a/src/core/tests/frontend/shared/test_builtin_extensions_1/test_extension.cpp b/src/core/tests/frontend/shared/test_builtin_extensions_1/test_extension.cpp new file mode 100644 index 00000000000..a8cc7de5c28 --- /dev/null +++ b/src/core/tests/frontend/shared/test_builtin_extensions_1/test_extension.cpp @@ -0,0 +1,14 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "test_extension.hpp" + +#include + +bool TestExtension1::transform(std::shared_ptr& function, const nlohmann::json& config) const { + function->set_friendly_name("TestFunction"); + return true; +} + +TestExtension1::TestExtension1() : ov::frontend::JsonTransformationExtension("buildin_extensions_1::TestExtension1") {} diff --git a/src/core/tests/frontend/shared/test_builtin_extensions_1/test_extension.hpp b/src/core/tests/frontend/shared/test_builtin_extensions_1/test_extension.hpp new file mode 100644 index 00000000000..3a985721831 --- /dev/null +++ b/src/core/tests/frontend/shared/test_builtin_extensions_1/test_extension.hpp @@ -0,0 +1,15 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include + +class TestExtension1 : public ov::frontend::JsonTransformationExtension { +public: + TestExtension1(); + + virtual bool transform(std::shared_ptr& function, const nlohmann::json& config) const override; +}; diff --git a/src/core/tests/frontend/shared/test_builtin_extensions_2/CMakeLists.txt b/src/core/tests/frontend/shared/test_builtin_extensions_2/CMakeLists.txt new file mode 100644 index 00000000000..4146e72b4f5 --- /dev/null +++ b/src/core/tests/frontend/shared/test_builtin_extensions_2/CMakeLists.txt @@ -0,0 +1,20 @@ +# Copyright (C) 2018-2021 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +set(TARGET_NAME "test_builtin_extensions_2") + +file(GLOB_RECURSE LIBRARY_SRC ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp) +file(GLOB_RECURSE LIBRARY_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/*.hpp) + +# Create library +add_library(${TARGET_NAME} SHARED ${LIBRARY_SRC} ${LIBRARY_HEADERS}) + +target_link_libraries(${TARGET_NAME} PRIVATE nlohmann_json inference_engine_transformations frontend_common) + +add_clang_format_target(${TARGET_NAME}_clang FOR_TARGETS ${TARGET_NAME}) + +install(TARGETS ${TARGET_NAME} + RUNTIME DESTINATION tests + COMPONENT tests + EXCLUDE_FROM_ALL) \ No newline at end of file diff --git a/src/core/tests/frontend/shared/test_builtin_extensions_2/builtin_extensions.cpp b/src/core/tests/frontend/shared/test_builtin_extensions_2/builtin_extensions.cpp new file mode 100644 index 00000000000..f601200dbff --- /dev/null +++ b/src/core/tests/frontend/shared/test_builtin_extensions_2/builtin_extensions.cpp @@ -0,0 +1,10 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include + +#include "test_extensions.hpp" + +OPENVINO_CREATE_EXTENSIONS(std::vector({std::make_shared(), + std::make_shared()})); diff --git a/src/core/tests/frontend/shared/test_builtin_extensions_2/test_extensions.cpp b/src/core/tests/frontend/shared/test_builtin_extensions_2/test_extensions.cpp new file mode 100644 index 00000000000..b4948e3a791 --- /dev/null +++ b/src/core/tests/frontend/shared/test_builtin_extensions_2/test_extensions.cpp @@ -0,0 +1,21 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "test_extensions.hpp" + +#include + +bool TestExtension1::transform(std::shared_ptr& function, const nlohmann::json& config) const { + function->set_friendly_name("TestFunction"); + return true; +} + +TestExtension1::TestExtension1() : ov::frontend::JsonTransformationExtension("buildin_extensions_2::TestExtension1") {} + +bool TestExtension2::transform(std::shared_ptr& function, const nlohmann::json& config) const { + function->set_friendly_name("TestFunction"); + return true; +} + +TestExtension2::TestExtension2() : ov::frontend::JsonTransformationExtension("buildin_extensions_2::TestExtension2") {} diff --git a/src/core/tests/frontend/shared/test_builtin_extensions_2/test_extensions.hpp b/src/core/tests/frontend/shared/test_builtin_extensions_2/test_extensions.hpp new file mode 100644 index 00000000000..be05ce8b40d --- /dev/null +++ b/src/core/tests/frontend/shared/test_builtin_extensions_2/test_extensions.hpp @@ -0,0 +1,22 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include + +class TestExtension1 : public ov::frontend::JsonTransformationExtension { +public: + TestExtension1(); + + virtual bool transform(std::shared_ptr& function, const nlohmann::json& config) const override; +}; + +class TestExtension2 : public ov::frontend::JsonTransformationExtension { +public: + TestExtension2(); + + virtual bool transform(std::shared_ptr& function, const nlohmann::json& config) const override; +}; diff --git a/src/core/tests/frontend/tensorflow/json_config_extension.cpp b/src/core/tests/frontend/tensorflow/json_config_extension.cpp new file mode 100644 index 00000000000..fc60bd5c16d --- /dev/null +++ b/src/core/tests/frontend/tensorflow/json_config_extension.cpp @@ -0,0 +1,24 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "json_config_extension.hpp" + +#include "tf_utils.hpp" + +using namespace ov::frontend; + +using TFJsonConfigTest = FrontEndJsonConfigTest; + +static JsonConfigFEParam getTestData() { + JsonConfigFEParam res; + res.m_frontEndName = TF_FE; + res.m_modelsPath = std::string(TEST_TENSORFLOW_MODELS_DIRNAME); + res.m_modelName = "2in_2out/2in_2out.pb"; + return res; +} + +INSTANTIATE_TEST_SUITE_P(TFJsonConfigTest, + FrontEndJsonConfigTest, + ::testing::Values(getTestData()), + FrontEndJsonConfigTest::getTestCaseName); diff --git a/src/frontends/common/CMakeLists.txt b/src/frontends/common/CMakeLists.txt index 13b5ff392b1..0aed90f48b1 100644 --- a/src/frontends/common/CMakeLists.txt +++ b/src/frontends/common/CMakeLists.txt @@ -37,7 +37,7 @@ target_include_directories(${TARGET_NAME} "${CMAKE_CURRENT_BINARY_DIR}/src") target_include_directories(${TARGET_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src) -target_link_libraries(${TARGET_NAME} PRIVATE ${CMAKE_DL_LIBS} openvino::util PUBLIC ngraph) +target_link_libraries(${TARGET_NAME} PRIVATE ${CMAKE_DL_LIBS} openvino::util nlohmann_json nlohmann_json_schema_validator PUBLIC ngraph) set_property(SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/src/plugin_loader.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/frontend_manager.cpp @@ -61,7 +61,7 @@ ie_add_api_validator_post_build_step(TARGET ${TARGET_NAME}) if(BUILD_SHARED_LIBS) add_library(${TARGET_NAME}_static STATIC ${LIBRARY_SRC} ${LIBRARY_HEADERS} ${LIBRARY_PUBLIC_HEADERS}) add_library(${TARGET_NAME}::static ALIAS ${TARGET_NAME}_static) - target_link_libraries(${TARGET_NAME}_static PRIVATE ${CMAKE_DL_LIBS} openvino::util PUBLIC ngraph) + target_link_libraries(${TARGET_NAME}_static PRIVATE ${CMAKE_DL_LIBS} openvino::util nlohmann_json nlohmann_json_schema_validator PUBLIC ngraph) target_include_directories(${TARGET_NAME}_static PUBLIC $) target_include_directories(${TARGET_NAME}_static PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src) @@ -71,6 +71,10 @@ else() add_library(${TARGET_NAME}::static ALIAS ${TARGET_NAME}) endif() +# Add include path to so_extension.hpp +set_source_files_properties(${CMAKE_CURRENT_SOURCE_DIR}/src/extensions/json_config_extension.cpp + PROPERTIES INCLUDE_DIRECTORIES "${OpenVINO_SOURCE_DIR}/src/core/src/") + # Installation rules for shared version only set_target_properties(${TARGET_NAME} PROPERTIES EXPORT_NAME frontend::common) diff --git a/src/frontends/common/include/common/extensions/decoder_transformation_extension.hpp b/src/frontends/common/include/common/extensions/decoder_transformation_extension.hpp new file mode 100644 index 00000000000..4ffb92a944f --- /dev/null +++ b/src/frontends/common/include/common/extensions/decoder_transformation_extension.hpp @@ -0,0 +1,56 @@ +// Copyright (C) 2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include +#include + +#include "common/frontend_defs.hpp" +#include "openvino/core/extension.hpp" +#include "openvino/pass/graph_rewrite.hpp" +#include "openvino/pass/manager.hpp" +#include "openvino/pass/pass.hpp" + +namespace ov { +namespace frontend { + +/// \brief Holds a transformation that is applied just after the original model graph is decoded. +/// This class is a holder for transformation. The transformation can be specified as +/// FunctionPass or MathcerPass derivatives or as a function that can be used to build corresponding +/// FunctionPass or MatcherPass object. The type of the extension is determined in the moment of creation by +/// calling corresponding ctor. +class FRONTEND_API DecoderTransformationExtension : public ov::Extension { +public: + DecoderTransformationExtension() = default; + + /// \brief Create a custom functional pass where code of the pass is implemented as a function. + explicit DecoderTransformationExtension(const std::function)>& function_pass); + + /// \brief Create a custom matcher pass where the code of matcher pass initialization is a given function. + explicit DecoderTransformationExtension( + const std::function& matcher_pass_initializer); + + /// \brief Register existing transformation object which will be copied and kept for further registration. + template ::value, bool>::type = true> + explicit DecoderTransformationExtension(const Transformation& transformation) + : m_registration([&](ov::pass::Manager& manager) { + manager.register_pass(transformation); + }) {} + + /// \brief Register pass from this object in a given pass manager object + void register_pass(ov::pass::Manager& manager) const; + +protected: + void set_registration(const std::function& registration) { + m_registration = registration; + } + +private: + std::function m_registration; +}; +} // namespace frontend +} // namespace ov diff --git a/src/frontends/common/include/common/extensions/json_config_extension.hpp b/src/frontends/common/include/common/extensions/json_config_extension.hpp new file mode 100644 index 00000000000..bdbc4a8b3db --- /dev/null +++ b/src/frontends/common/include/common/extensions/json_config_extension.hpp @@ -0,0 +1,34 @@ +// Copyright (C) 2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include +#include + +#include "common/extensions/decoder_transformation_extension.hpp" +#include "common/frontend_defs.hpp" +#include "nlohmann/json.hpp" +#include "openvino/core/extension.hpp" +#include "openvino/pass/graph_rewrite.hpp" +#include "openvino/pass/manager.hpp" +#include "openvino/pass/pass.hpp" + +namespace ov { +namespace frontend { + +/// \brief Reads MO config file and delegate transformation functionality to specified transformation ID +/// specified in the config. +class FRONTEND_API JsonConfigExtension : public DecoderTransformationExtension { +public: + explicit JsonConfigExtension(const std::string& config_path); + ~JsonConfigExtension() override; + +protected: + std::vector m_loaded_extensions; + std::vector, nlohmann::json>> m_target_extensions; +}; +} // namespace frontend +} // namespace ov diff --git a/src/frontends/common/include/common/extensions/json_schema.hpp b/src/frontends/common/include/common/extensions/json_schema.hpp new file mode 100644 index 00000000000..c2d5efcb5c2 --- /dev/null +++ b/src/frontends/common/include/common/extensions/json_schema.hpp @@ -0,0 +1,134 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 + +#pragma once +#include + +static nlohmann::json json_schema = R"( +{ + "definitions": {}, + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Root", + "type": "array", + "default": [], + "items": { + "$id": "#root/items", + "title": "Items", + "type": "object", + "required": [ + "id", + "match_kind" + ], + "properties": { + "custom_attributes": { + "$id": "#root/items/custom_attributes", + "title": "Custom_attributes", + "type": "object", + "properties": { + } + }, + "id": { + "$id": "#root/items/id", + "title": "Id", + "type": "string", + "pattern": "^.*$", + "minLength": 1 + }, + "inputs": { + "$id": "#root/items/inputs", + "title": "Inputs", + "type": "array", + "default": [], + "items": { + "$id": "#root/items/inputs/items", + "title": "Items", + "type": "array", + "default": [], + "items": { + "$id": "#root/items/inputs/items/items", + "title": "Items", + "type": "object", + "properties": { + "node": { + "$id": "#root/items/inputs/items/items/node", + "title": "Node", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "port": { + "$id": "#root/items/inputs/items/items/port", + "title": "Port", + "type": "integer", + "default": 0 + } + }, + "required": ["node", "port"] + } + + } + }, + "instances": { + "$id": "#root/items/instances", + "title": "Instances", + "type": ["array", "object"], + "items": { + "$id": "#root/items/instances/items", + "title": "Items", + "type": "string", + "default": "", + "pattern": "^.*$" + } + }, + "match_kind": { + "$id": "#root/items/match_kind", + "title": "Match_kind", + "type": "string", + "enum": ["points", "scope", "general"], + "default": "points", + "pattern": "^.*$" + }, + "outputs": { + "$id": "#root/items/outputs", + "title": "Outputs", + "type": "array", + "default": [], + "items": { + "$id": "#root/items/outputs/items", + "title": "Items", + "type": "object", + "properties": { + "node": { + "$id": "#root/items/outputs/items/node", + "title": "Node", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "port": { + "$id": "#root/items/outputs/items/port", + "title": "Port", + "type": "integer", + "default": 0 + } + }, + "required": ["node", "port"] + } + + }, + "include_inputs_to_sub_graph": { + "$id": "#root/items/include_inputs_to_sub_graph", + "title": "Include_inputs_to_sub_graph", + "type": "boolean", + "default": false + }, + "include_outputs_to_sub_graph": { + "$id": "#root/items/include_outputs_to_sub_graph", + "title": "Include_outputs_to_sub_graph", + "type": "boolean", + "default": false + } + } + } +} +)"_json; diff --git a/src/frontends/common/include/common/extensions/json_transformation_extension.hpp b/src/frontends/common/include/common/extensions/json_transformation_extension.hpp new file mode 100644 index 00000000000..de691d7f511 --- /dev/null +++ b/src/frontends/common/include/common/extensions/json_transformation_extension.hpp @@ -0,0 +1,52 @@ +// Copyright (C) 2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include +#include + +#include "common/extensions/decoder_transformation_extension.hpp" +#include "common/frontend_defs.hpp" +#include "nlohmann/json.hpp" +#include "openvino/core/extension.hpp" +#include "openvino/pass/graph_rewrite.hpp" +#include "openvino/pass/manager.hpp" +#include "openvino/pass/pass.hpp" + +namespace ov { +namespace frontend { + +/// \brief Describes transformation that JsonConfigExtension can use as a target transformation spedified by ID. +/// JsonTransformationExtension passes JSON parsed object and mathed points in the graph to JsonTransformationExtension +/// instance, which is derived from DecoderTransformationExtension. DecoderTransformationExtension itself cannot be +/// used for this purpose because we need to pass those additional objects which are the result of JSON parsing and +/// graph matching that JsonTransformationExtension performs. +/// +/// This class is left for backward compatibility only. In the future we would like to get rid off this class as well +/// as from JsonConfigExtension, and users will use DecoderTransformationExtension only to match sub-graph and modify it +/// like we do in any other transformations. +/// +/// Unlike DecoderTransformationExtension, which is initialized by some ready-to-use transformation code and is not used +/// to derive new classes by regular users, this class is intended to be derived from and it doesn't have convenient +/// ctos to be initialized. So it is intended for more advanced, internal users inside such components like Model +/// Optimizer. +class FRONTEND_API JsonTransformationExtension : public DecoderTransformationExtension { +public: + explicit JsonTransformationExtension(const std::string& id) : m_id(id) {} + + /// \brief The name of the transformation to identify it from JSON file field 'id' + const std::string& id() const { + return m_id; + } + + virtual bool transform(std::shared_ptr& function, + const nlohmann::json& replacement_descriptions) const = 0; + +private: + std::string m_id; +}; +} // namespace frontend +} // namespace ov diff --git a/src/frontends/common/include/common/telemetry_extension.hpp b/src/frontends/common/include/common/extensions/telemetry_extension.hpp similarity index 97% rename from src/frontends/common/include/common/telemetry_extension.hpp rename to src/frontends/common/include/common/extensions/telemetry_extension.hpp index 914ad41c4bb..cbbf5596311 100644 --- a/src/frontends/common/include/common/telemetry_extension.hpp +++ b/src/frontends/common/include/common/extensions/telemetry_extension.hpp @@ -8,7 +8,7 @@ #include #include -#include "frontend_defs.hpp" +#include "common/frontend_defs.hpp" #include "openvino/core/extension.hpp" #include "openvino/pass/graph_rewrite.hpp" #include "openvino/pass/manager.hpp" diff --git a/src/frontends/common/src/extensions/decoder_transformation_extension.cpp b/src/frontends/common/src/extensions/decoder_transformation_extension.cpp new file mode 100644 index 00000000000..fa1cf647433 --- /dev/null +++ b/src/frontends/common/src/extensions/decoder_transformation_extension.cpp @@ -0,0 +1,47 @@ +// Copyright (C) 2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "common/extensions/decoder_transformation_extension.hpp" + +#include + +using namespace ov; +using namespace ov::frontend; + +/// \brief Helper class to register user function as a FunctionPass +class CustomFunctionPass : public ov::pass::FunctionPass { +public: + explicit CustomFunctionPass(std::function)> pass) : m_pass(std::move(pass)) {} + + bool run_on_function(std::shared_ptr f) override { + return m_pass(f); + } + +private: + std::function)> m_pass; +}; + +/// \brief Helper class to register user matcher pass initialization as a MatcherPass +class CustomMatcherPass : public ov::pass::MatcherPass { +public: + explicit CustomMatcherPass(const std::function& matcher_pass_initializer) { + matcher_pass_initializer(this); + } +}; + +DecoderTransformationExtension::DecoderTransformationExtension( + const std::function)>& function_pass) + : m_registration([&](ov::pass::Manager& manager) { + manager.register_pass(function_pass); + }) {} + +DecoderTransformationExtension::DecoderTransformationExtension( + const std::function& matcher_pass_initializer) + : m_registration([&](ov::pass::Manager& manager) { + manager.register_pass(matcher_pass_initializer); + }) {} + +void DecoderTransformationExtension::register_pass(ov::pass::Manager& manager) const { + m_registration(manager); +} diff --git a/src/frontends/common/src/extensions/json_config_extension.cpp b/src/frontends/common/src/extensions/json_config_extension.cpp new file mode 100644 index 00000000000..fb2cd82a1b7 --- /dev/null +++ b/src/frontends/common/src/extensions/json_config_extension.cpp @@ -0,0 +1,79 @@ +// Copyright (C) 2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "common/extensions/json_config_extension.hpp" +#include "common/extensions/decoder_transformation_extension.hpp" +#include "common/extensions/json_schema.hpp" +#include "common/extensions/json_transformation_extension.hpp" +#include "so_extension.hpp" + +using namespace ov; +using namespace ov::frontend; + +JsonConfigExtension::JsonConfigExtension(const std::string& config_path) + : DecoderTransformationExtension([this](std::shared_ptr f) { + bool res = true; + for (const auto& target_extension : m_target_extensions) { + auto extension = std::dynamic_pointer_cast(target_extension.first); + res &= extension->transform(f, target_extension.second); + } + return res; + }) { + // Load JSON config + nlohmann::json config_json; + std::ifstream config_file(config_path); + config_file >> config_json; + + // Validate JSON config + nlohmann::json_schema::json_validator validator; + try { + validator.set_root_schema(json_schema); + } catch (const std::exception& e) { + OPENVINO_ASSERT(false, "Invalid json schema : ", e.what()); + } + + try { + validator.validate(config_json); + } catch (const std::exception& e) { + OPENVINO_ASSERT(false, "Json schema validation failed: ", e.what()); + } + + // Parse JSON Extensions + + // Group sections describing transformations by library. + std::map lib_to_sections; + for (const auto& section : config_json) { + lib_to_sections[section["library"]].push_back(section); + } + + // Load all extensions in each library and select required + for (const auto& it : lib_to_sections) { + const auto& lib = it.first; + const auto& sections = it.second; + + auto extensions = detail::load_extensions(lib); + m_loaded_extensions.insert(m_loaded_extensions.end(), extensions.begin(), extensions.end()); + for (const auto& ext : extensions) { + auto so_extension = std::dynamic_pointer_cast(ext); + OPENVINO_ASSERT(so_extension, "Unexpected extension type loaded from shared library."); + auto extension = so_extension->extension(); + if (auto json_ext = std::dynamic_pointer_cast(extension)) { + for (const auto& section : sections) { + if (section["id"] == json_ext->id()) { + m_target_extensions.push_back({json_ext, section}); + } + } + } + } + } +} + +JsonConfigExtension::~JsonConfigExtension() { + // Reset is required here prior unload_extensions, because + // there shouldn't be any alive references before the unloading the library. + // Doing it here explicitly to avoid relying on order of class fields definition + /* for (const auto& target_extension : m_target_extensions) { + target_extension.first.reset(); + }*/ +} \ No newline at end of file diff --git a/src/frontends/common/src/telemetry_extension.cpp b/src/frontends/common/src/extensions/telemetry_extension.cpp similarity index 95% rename from src/frontends/common/src/telemetry_extension.cpp rename to src/frontends/common/src/extensions/telemetry_extension.cpp index 134836ac701..ca7ff54b5b0 100644 --- a/src/frontends/common/src/telemetry_extension.cpp +++ b/src/frontends/common/src/extensions/telemetry_extension.cpp @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 // -#include "common/telemetry_extension.hpp" +#include "common/extensions/telemetry_extension.hpp" using namespace ov::frontend; diff --git a/src/frontends/ir/include/ir_frontend/frontend.hpp b/src/frontends/ir/include/ir_frontend/frontend.hpp index f5a85ea86c5..c0e07882636 100644 --- a/src/frontends/ir/include/ir_frontend/frontend.hpp +++ b/src/frontends/ir/include/ir_frontend/frontend.hpp @@ -5,7 +5,7 @@ #pragma once #include "common/frontend.hpp" -#include "common/telemetry_extension.hpp" +#include "common/extensions/telemetry_extension.hpp" #include "openvino/core/variant.hpp" #include "utility.hpp" diff --git a/src/frontends/onnx/frontend/include/onnx_frontend/frontend.hpp b/src/frontends/onnx/frontend/include/onnx_frontend/frontend.hpp index d35f3485daf..b597b9ebe53 100644 --- a/src/frontends/onnx/frontend/include/onnx_frontend/frontend.hpp +++ b/src/frontends/onnx/frontend/include/onnx_frontend/frontend.hpp @@ -4,6 +4,8 @@ #pragma once +#include +#include #include #ifdef OPENVINO_STATIC_LIBRARY @@ -35,6 +37,7 @@ protected: private: std::shared_ptr m_telemetry; + std::vector> m_transformation_extensions; }; } // namespace frontend diff --git a/src/frontends/onnx/frontend/src/core/graph.hpp b/src/frontends/onnx/frontend/src/core/graph.hpp index fd1f145b384..8859ff2d652 100644 --- a/src/frontends/onnx/frontend/src/core/graph.hpp +++ b/src/frontends/onnx/frontend/src/core/graph.hpp @@ -10,7 +10,7 @@ #include #include -#include "common/telemetry_extension.hpp" +#include "common/extensions/telemetry_extension.hpp" #include "core/graph_cache.hpp" #include "core/model.hpp" #include "ngraph/function.hpp" diff --git a/src/frontends/onnx/frontend/src/editor.hpp b/src/frontends/onnx/frontend/src/editor.hpp index 05932132276..2c32fd2698a 100644 --- a/src/frontends/onnx/frontend/src/editor.hpp +++ b/src/frontends/onnx/frontend/src/editor.hpp @@ -4,7 +4,7 @@ #pragma once -#include +#include #include #include #include diff --git a/src/frontends/onnx/frontend/src/frontend.cpp b/src/frontends/onnx/frontend/src/frontend.cpp index db73a7fde03..089548d713a 100644 --- a/src/frontends/onnx/frontend/src/frontend.cpp +++ b/src/frontends/onnx/frontend/src/frontend.cpp @@ -2,8 +2,8 @@ // SPDX-License-Identifier: Apache-2.0 // +#include #include -#include #include #include #include @@ -68,6 +68,19 @@ InputModel::Ptr FrontEndONNX::load_impl(const std::vector FrontEndONNX::convert(InputModel::Ptr model) const { auto model_onnx = std::dynamic_pointer_cast(model); NGRAPH_CHECK(model_onnx != nullptr, "Invalid input model"); + + if (!m_transformation_extensions.empty()) { + auto function = decode(model); + + pass::Manager manager; + for (const auto& transformation : m_transformation_extensions) { + transformation->register_pass(manager); + } + manager.run_passes(function); + convert(function); + return function; + } + return model_onnx->convert(); } @@ -139,5 +152,7 @@ bool FrontEndONNX::supported_impl(const std::vector>& v void FrontEndONNX::add_extension(const std::shared_ptr& extension) { if (auto telemetry = std::dynamic_pointer_cast(extension)) { m_telemetry = telemetry; + } else if (auto transformation = std::dynamic_pointer_cast(extension)) { + m_transformation_extensions.push_back(transformation); } } diff --git a/src/frontends/onnx/frontend/src/utils/onnx_internal.hpp b/src/frontends/onnx/frontend/src/utils/onnx_internal.hpp index 9813e82fa4b..3b69054f4df 100644 --- a/src/frontends/onnx/frontend/src/utils/onnx_internal.hpp +++ b/src/frontends/onnx/frontend/src/utils/onnx_internal.hpp @@ -7,7 +7,7 @@ #include #include -#include "common/telemetry_extension.hpp" +#include "common/extensions/telemetry_extension.hpp" #include "ngraph/function.hpp" namespace ONNX_NAMESPACE { diff --git a/src/frontends/paddlepaddle/include/paddlepaddle_frontend/frontend.hpp b/src/frontends/paddlepaddle/include/paddlepaddle_frontend/frontend.hpp index 2020aa00fee..d03185da08d 100644 --- a/src/frontends/paddlepaddle/include/paddlepaddle_frontend/frontend.hpp +++ b/src/frontends/paddlepaddle/include/paddlepaddle_frontend/frontend.hpp @@ -4,7 +4,8 @@ #pragma once -#include +#include +#include #include #include "exceptions.hpp" @@ -70,6 +71,7 @@ private: std::function(const std::map>&, const std::shared_ptr&)> func); std::shared_ptr m_telemetry; + std::vector> m_transformation_extensions; }; } // namespace frontend diff --git a/src/frontends/paddlepaddle/include/paddlepaddle_frontend/model.hpp b/src/frontends/paddlepaddle/include/paddlepaddle_frontend/model.hpp index 8cc63a5540a..6be52171c46 100644 --- a/src/frontends/paddlepaddle/include/paddlepaddle_frontend/model.hpp +++ b/src/frontends/paddlepaddle/include/paddlepaddle_frontend/model.hpp @@ -4,7 +4,7 @@ #pragma once -#include +#include #include #include "paddlepaddle_frontend/utility.hpp" diff --git a/src/frontends/paddlepaddle/src/frontend.cpp b/src/frontends/paddlepaddle/src/frontend.cpp index e366eb85e90..a63852f6a39 100644 --- a/src/frontends/paddlepaddle/src/frontend.cpp +++ b/src/frontends/paddlepaddle/src/frontend.cpp @@ -279,6 +279,20 @@ InputModel::Ptr FrontEndPDPD::load_impl(const std::vector FrontEndPDPD::convert(InputModel::Ptr model) const { auto pdpd_model = std::dynamic_pointer_cast(model); + FRONT_END_GENERAL_CHECK(pdpd_model != nullptr, "Invalid input model"); + + if (!m_transformation_extensions.empty()) { + auto function = decode(model); + + pass::Manager manager; + for (const auto& transformation : m_transformation_extensions) { + transformation->register_pass(manager); + } + manager.run_passes(function); + convert(function); + return function; + } + std::map CREATORS_MAP = pdpd::get_supported_ops(); auto f = convert_each_node( pdpd_model, @@ -331,6 +345,8 @@ std::string FrontEndPDPD::get_name() const { void FrontEndPDPD::add_extension(const std::shared_ptr& extension) { if (auto telemetry = std::dynamic_pointer_cast(extension)) { m_telemetry = telemetry; + } else if (auto transformation = std::dynamic_pointer_cast(extension)) { + m_transformation_extensions.push_back(transformation); } } diff --git a/src/frontends/tensorflow/include/tensorflow_frontend/frontend.hpp b/src/frontends/tensorflow/include/tensorflow_frontend/frontend.hpp index 2a34b501abb..94e9c6db92e 100644 --- a/src/frontends/tensorflow/include/tensorflow_frontend/frontend.hpp +++ b/src/frontends/tensorflow/include/tensorflow_frontend/frontend.hpp @@ -7,9 +7,10 @@ #include #include +#include "common/extensions/decoder_transformation_extension.hpp" +#include "common/extensions/telemetry_extension.hpp" #include "common/frontend.hpp" #include "common/input_model.hpp" -#include "common/telemetry_extension.hpp" #include "openvino/core/any.hpp" #include "openvino/core/node_vector.hpp" #include "openvino/core/variant.hpp" @@ -84,6 +85,7 @@ private: std::shared_ptr& ng_function) const; std::shared_ptr m_telemetry; + std::vector> m_transformation_extensions; }; } // namespace frontend } // namespace ov diff --git a/src/frontends/tensorflow/src/frontend.cpp b/src/frontends/tensorflow/src/frontend.cpp index 5c6c83fb9d9..27d48b60199 100644 --- a/src/frontends/tensorflow/src/frontend.cpp +++ b/src/frontends/tensorflow/src/frontend.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2018-2021 Intel Corporation + // Copyright (C) 2018-2021 Intel Corporation // SPDX-License-Identifier: Apache-2.0 // @@ -308,6 +308,20 @@ ov::frontend::InputModel::Ptr FrontEndTF::load_impl(const std::vector FrontEndTF::convert(ov::frontend::InputModel::Ptr model) const { auto model_tf = std::dynamic_pointer_cast(model); + FRONT_END_GENERAL_CHECK(model_tf != nullptr, "Invalid input model"); + + if (!m_transformation_extensions.empty()) { + auto function = decode(model); + + pass::Manager manager; + for (const auto& transformation : m_transformation_extensions) { + transformation->register_pass(manager); + } + manager.run_passes(function); + convert(function); + return function; + } + std::shared_ptr f; translate_graph(model_tf, "here_should_be_a_graph_name", true, false, f); normalize(f); @@ -353,5 +367,7 @@ void FrontEndTF::normalize(std::shared_ptr function) const { void FrontEndTF::add_extension(const std::shared_ptr& extension) { if (auto telemetry = std::dynamic_pointer_cast(extension)) { m_telemetry = telemetry; + } else if (auto transformation = std::dynamic_pointer_cast(extension)) { + m_transformation_extensions.push_back(transformation); } } diff --git a/src/frontends/tensorflow/src/model.hpp b/src/frontends/tensorflow/src/model.hpp index d2edc1dc5d3..17481886f1c 100644 --- a/src/frontends/tensorflow/src/model.hpp +++ b/src/frontends/tensorflow/src/model.hpp @@ -4,9 +4,9 @@ #pragma once +#include "common/extensions/telemetry_extension.hpp" #include "common/input_model.hpp" #include "common/place.hpp" -#include "common/telemetry_extension.hpp" #include "tensorflow_frontend/graph_iterator.hpp" namespace ov { diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt index 859b6f29c58..d974ef71cd4 100644 --- a/thirdparty/CMakeLists.txt +++ b/thirdparty/CMakeLists.txt @@ -192,8 +192,16 @@ endif() # nlohmann json # if(ENABLE_SAMPLES OR NGRAPH_PDPD_FRONTEND_ENABLE OR NGRAPH_ONNX_FRONTEND_ENABLE OR NGRAPH_TF_FRONTEND_ENABLE) - add_subdirectory(nlohmann_json EXCLUDE_FROM_ALL) - openvino_developer_export_targets(COMPONENT openvino_common TARGETS nlohmann_json) + function(build_json_libs) + set(JSON_Install ON) + add_subdirectory(json/nlohmann_json EXCLUDE_FROM_ALL) + openvino_developer_export_targets(COMPONENT openvino_common TARGETS nlohmann_json) + + set(BUILD_SHARED_LIBS OFF) + add_subdirectory(json/nlohmann_json_schema_validator EXCLUDE_FROM_ALL) + ov_install_static_lib(nlohmann_json_schema_validator openvino_common) + endfunction() + build_json_libs() endif() # diff --git a/thirdparty/nlohmann_json b/thirdparty/json/nlohmann_json similarity index 100% rename from thirdparty/nlohmann_json rename to thirdparty/json/nlohmann_json diff --git a/thirdparty/json/nlohmann_json_schema_validator b/thirdparty/json/nlohmann_json_schema_validator new file mode 160000 index 00000000000..b1ef8628326 --- /dev/null +++ b/thirdparty/json/nlohmann_json_schema_validator @@ -0,0 +1 @@ +Subproject commit b1ef8628326cf0b53612f12784fd245e5e4382f1