From 70ccd0e91f398ae7ef0cc30cb030beea5c8e1358 Mon Sep 17 00:00:00 2001 From: Mateusz Bencer Date: Thu, 20 Jan 2022 23:34:24 +0100 Subject: [PATCH] Handle special cases during fallback (#9651) --- .../prepare_model/Model_Optimizer_FAQ.md | 3 +- .../src/pyopenvino/frontend/frontend.cpp | 23 +- tools/mo/automation/package_BOM.txt | 1 + tools/mo/openvino/tools/mo/main.py | 28 +- .../tools/mo/moc_frontend/check_config.py | 91 +++++++ .../mo/utils/mo_fallback_test_actual.py | 251 +++++++++++++++--- 6 files changed, 351 insertions(+), 46 deletions(-) create mode 100644 tools/mo/openvino/tools/mo/moc_frontend/check_config.py diff --git a/docs/MO_DG/prepare_model/Model_Optimizer_FAQ.md b/docs/MO_DG/prepare_model/Model_Optimizer_FAQ.md index ef966213a91..5e9bbd8f4c0 100644 --- a/docs/MO_DG/prepare_model/Model_Optimizer_FAQ.md +++ b/docs/MO_DG/prepare_model/Model_Optimizer_FAQ.md @@ -633,5 +633,6 @@ Note that you might have conflicts between previously installed PyPI dependencie For the models in ONNX* format, there are two available paths of IR conversion. The old one is handled by the old Python* implementation, while the new one uses new C++ frontends. Starting from the 2022.1 version, the default IR conversion path for ONNX models is processed using the new ONNX frontend. -Certain features, such as `--extensions` and `--transformations_config`, are not yet supported on the new frontends. +Certain features, such as `--extensions and` `--transformations_config`, are not yet fully supported on the new frontends. +For `--extensions`, the new frontends support only paths to shared libraries (.dll and .so). For `--transformations_config`, they support JSON configurations with defined library fields. The IR conversion falls back to the old path if a user does not select any expected path of conversion explicitly (by `--use_new_frontend` or `--use_legacy_frontend` MO arguments) and unsupported pre-defined scenario is detected on the new frontend path. \ No newline at end of file diff --git a/src/bindings/python/src/pyopenvino/frontend/frontend.cpp b/src/bindings/python/src/pyopenvino/frontend/frontend.cpp index 40f3278c590..4c2bd43a3e5 100644 --- a/src/bindings/python/src/pyopenvino/frontend/frontend.cpp +++ b/src/bindings/python/src/pyopenvino/frontend/frontend.cpp @@ -137,7 +137,28 @@ void regclass_frontend_FrontEnd(py::module m) { )"); fem.def("add_extension", - static_cast& extension)>(&FrontEnd::add_extension)); + static_cast& extension)>(&FrontEnd::add_extension), + R"( + Add extension defined by an object inheriting from Extension + used in order to extend capabilities of Frontend. + + Parameters + ---------- + extension : Extension + Provided extension object. + )"); + + fem.def("add_extension", + static_cast(&FrontEnd::add_extension), + R"( + Add extension defined in external library indicated by a extension_path + used in order to extend capabilities of Frontend. + + Parameters + ---------- + extension_path : str + A path to extension. + )"); fem.def("__repr__", [](const FrontEnd& self) -> std::string { return ""; diff --git a/tools/mo/automation/package_BOM.txt b/tools/mo/automation/package_BOM.txt index 2713419225c..0ed79477230 100644 --- a/tools/mo/automation/package_BOM.txt +++ b/tools/mo/automation/package_BOM.txt @@ -838,6 +838,7 @@ openvino/tools/mo/mo_paddle.py openvino/tools/mo/mo_tf.py openvino/tools/mo/moc_frontend/__init__.py openvino/tools/mo/moc_frontend/analysis.py +openvino/tools/mo/moc_frontend/check_config.py openvino/tools/mo/moc_frontend/extractor.py openvino/tools/mo/moc_frontend/pipeline.py openvino/tools/mo/moc_frontend/serialize.py diff --git a/tools/mo/openvino/tools/mo/main.py b/tools/mo/openvino/tools/mo/main.py index b7393c8f110..c91b1db47aa 100644 --- a/tools/mo/openvino/tools/mo/main.py +++ b/tools/mo/openvino/tools/mo/main.py @@ -10,6 +10,7 @@ import sys import traceback from collections import OrderedDict from copy import deepcopy +import json try: import openvino_telemetry as tm @@ -18,6 +19,8 @@ except ImportError: from openvino.tools.mo.back.SpecialNodesFinalization import RemoveConstOps, CreateConstNodesReplacement, NormalizeTI from openvino.tools.mo.back.ie_ir_ver_2.emitter import append_ir_info +from openvino.tools.mo.moc_frontend.check_config import legacy_extensions_used, legacy_transformations_config_used, \ + new_extensions_used, new_transformations_config_used from openvino.tools.mo.moc_frontend.pipeline import moc_pipeline from openvino.tools.mo.moc_frontend.serialize import moc_emit_ir from openvino.tools.mo.graph.graph import Graph @@ -43,7 +46,7 @@ from openvino.tools.mo.utils.telemetry_utils import get_tid from openvino.tools.mo.front.common.partial_infer.utils import mo_array # pylint: disable=no-name-in-module,import-error -from openvino.frontend import FrontEndManager, ProgressReporterExtension, TelemetryExtension +from openvino.frontend import FrontEndManager, ProgressReporterExtension, TelemetryExtension, JsonConfigExtension def replace_ext(name: str, old: str, new: str): @@ -140,7 +143,12 @@ def arguments_post_parsing(argv: argparse.Namespace): is_tf, is_caffe, is_mxnet, is_kaldi, is_onnx =\ deduce_framework_by_namespace(argv) if not moc_front_end else [False, False, False, False, False] - if not any([is_tf, is_caffe, is_mxnet, is_kaldi, is_onnx]): + if any([is_tf, is_caffe, is_mxnet, is_kaldi, is_onnx]): + if new_extensions_used(argv): + raise Error('New kind of extensions used on legacy path') + if new_transformations_config_used(argv): + raise Error('New kind of transformations configuration used on legacy path') + else: # new frontend used frameworks = ['tf', 'caffe', 'mxnet', 'kaldi', 'onnx'] frameworks = list(set(frameworks + available_moc_front_ends)) if argv.framework not in frameworks: @@ -330,11 +338,8 @@ def check_fallback(argv : argparse.Namespace): if argv.use_new_frontend: return fallback_reasons - fallback_reasons['extensions'] = \ - lambda argv : hasattr(argv, 'extensions') and argv.extensions is not None and len(argv.extensions) > 0 \ - and argv.extensions != import_extensions.default_path() # extensions arg has default value - fallback_reasons['transformations_config'] = \ - lambda argv: hasattr(argv, 'transformations_config') and argv.transformations_config is not None and len(argv.transformations_config) > 0 + fallback_reasons['extensions'] = legacy_extensions_used + fallback_reasons['transformations_config'] = legacy_transformations_config_used reasons = [reason for reason, is_applicable in fallback_reasons.items() if is_applicable(argv)] return reasons @@ -352,6 +357,15 @@ def prepare_ir(argv : argparse.Namespace): t.send_event("mo", "conversion_method", moc_front_end.get_name() + "_frontend") moc_front_end.add_extension(TelemetryExtension("mo", t.send_event, t.send_error, t.send_stack_trace)) moc_front_end.add_extension(ProgressReporterExtension(progress_printer(argv))) + if legacy_transformations_config_used(argv): + raise Error('Legacy extensions are not supported for the new frontend') + if legacy_extensions_used(argv): + raise Error('Legacy transformations configuration is not supported for the new frontend') + if new_transformations_config_used(argv): + moc_front_end.add_extension(JsonConfigExtension(argv.transformations_config)) + if new_extensions_used(argv): + for extension in argv.extensions.split(','): + moc_front_end.add_extension(extension) ngraph_function = moc_pipeline(argv, moc_front_end) return graph, ngraph_function else: # apply fallback diff --git a/tools/mo/openvino/tools/mo/moc_frontend/check_config.py b/tools/mo/openvino/tools/mo/moc_frontend/check_config.py new file mode 100644 index 00000000000..35ad2d77442 --- /dev/null +++ b/tools/mo/openvino/tools/mo/moc_frontend/check_config.py @@ -0,0 +1,91 @@ +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import argparse +import json +from pathlib import Path + +from openvino.tools.mo.utils import import_extensions +from openvino.tools.mo.utils.error import Error + + +def any_extensions_used(argv: argparse.Namespace): + return hasattr(argv, 'extensions') and argv.extensions is not None and len(argv.extensions) > 0 \ + and argv.extensions != import_extensions.default_path() # extensions arg has default value + + +def legacy_extensions_used(argv: argparse.Namespace): + if any_extensions_used(argv): + extensions = argv.extensions.split(',') + legacy_ext_counter = 0 + for extension in extensions: + path = Path(extension) + if not path.is_file(): + legacy_ext_counter += 1 + if legacy_ext_counter == len(extensions): + return True # provided only legacy extensions + elif legacy_ext_counter == 0: + return False # provided only new extensions + else: + raise Error('Using new and legacy extensions in the same time is forbidden') + return False + + +def new_extensions_used(argv: argparse.Namespace): + if any_extensions_used(argv): + extensions = argv.extensions.split(',') + new_ext_counter = 0 + for extension in argv.extensions.split(','): + path = Path(extension) + if path.is_file() and (path.suffix == '.so' or path.suffix == '.dll'): + new_ext_counter += 1 + if new_ext_counter == len(extensions): + return True # provided only new extensions + elif new_ext_counter == 0: + return False # provided only legacy extensions + else: + raise Error('Using new and legacy extensions in the same time is forbidden') + return False + + +def is_new_json_config(json_file_path: str): + with open(json_file_path) as stream: + config_content = json.load(stream) + if len(config_content) == 0: # empty case + return False + if isinstance(config_content, dict): # single transformation + return 'library' in config_content.keys() + # many transformations in single file + library_counter = 0 + for transform in config_content: + if any(key == 'library' for key in transform.keys()): + library_counter+=1 + if len(config_content) == library_counter: # all transformations has 'library' attribute + return True + elif library_counter == 0: # all transformations are legacy type + return False + else: + raise Error('Mixed types of transformations configurations were used') + + +def get_transformations_config_path(argv: argparse.Namespace) -> Path: + if hasattr(argv, 'transformations_config') \ + and argv.transformations_config is not None and len(argv.transformations_config): + path = Path(argv.transformations_config) + if path.is_file(): + return path + return None + + +def new_transformations_config_used(argv: argparse.Namespace): + path = get_transformations_config_path(argv) + if path != None: + return is_new_json_config(path) + return False + + +def legacy_transformations_config_used(argv: argparse.Namespace): + path = get_transformations_config_path(argv) + if path != None: + return not is_new_json_config(path) + return False diff --git a/tools/mo/unit_tests/mo/utils/mo_fallback_test_actual.py b/tools/mo/unit_tests/mo/utils/mo_fallback_test_actual.py index e5aea1ed2a5..d2c2eb3f7aa 100644 --- a/tools/mo/unit_tests/mo/utils/mo_fallback_test_actual.py +++ b/tools/mo/unit_tests/mo/utils/mo_fallback_test_actual.py @@ -2,12 +2,12 @@ # SPDX-License-Identifier: Apache-2.0 import unittest -from unittest.mock import Mock -from unittest.mock import patch +from unittest.mock import patch, Mock import openvino from openvino.tools.mo.main import prepare_ir -from openvino.frontend import FrontEndManager # pylint: disable=no-name-in-module,import-error +from openvino.tools.mo.utils.error import Error +from openvino.frontend import FrontEndManager, FrontEnd # pylint: disable=no-name-in-module,import-error from onnx.helper import make_graph, make_model, make_tensor_value_info import argparse import os @@ -81,6 +81,7 @@ class TestMoFallback(unittest.TestCase): def setUp(self): tm.Telemetry.__init__ = Mock(return_value=None) tm.Telemetry.send_event = Mock() + FrontEnd.add_extension = Mock() self.models = {} add = onnx.helper.make_node("Add", inputs=["in1", "in2"], outputs=["add_out"]) @@ -99,10 +100,83 @@ class TestMoFallback(unittest.TestCase): for name, model in self.models.items(): onnx.save(model, name) - trans_config = 'config.json' - with open(trans_config, 'w') as f: - f.write("[]") # json format - self.trans_config_file = os.path.abspath(trans_config) + self.test_config_files = {} + self.test_config_files['fake_config.json'] = '[]' # json format + + self.test_config_files['test_config_1.json'] = """[ + { + "custom_attributes": { + "test_attribute": true + }, + "id": "TransformationName1", + "library": "path_to_library1.so", + "match_kind": "scope" + }, + { + "custom_attributes": { + }, + "id": "TransfromationName2", + "library": "path_to_library2.so", + "match_kind": "scope" + }, + { + "id": "TransfromationName3", + "library": "path_to_library3.so", + "match_kind": "scope" + } + ]""" + + self.test_config_files['test_config_2.json'] = """{ + "custom_attributes": { + "test_attribute": true + }, + "id": "TransformationName1", + "library": "path_to_library.so", + "match_kind": "scope" + }""" + + self.test_config_files['test_config_3.json'] = """[ + { + "custom_attributes": { + "test_attribute": true + }, + "id": "TransformationName1", + "match_kind": "scope" + }, + { + "custom_attributes": { + }, + "id": "TransfromationName2", + "match_kind": "scope" + } + ]""" + + self.test_config_files['test_config_4.json'] = """[ + { + "custom_attributes": { + "test_attribute": true + }, + "id": "TransformationName1", + "library": "path_to_library", + "match_kind": "scope" + }, + { + "custom_attributes": { + }, + "id": "TransfromationName2", + "match_kind": "scope" + }, + { + "library": "path_to_library.so" + } + ]""" + + self.test_config_files['onnx_fe_ext.so'] = 'binary_content' + self.test_config_files['onnx_fe_ext_2.so'] = 'binary_content' + + for file, content in self.test_config_files.items(): + with open(file, 'w') as f: + f.write(content) self.paddle_dir = "paddle_dir" paddle.enable_static() @@ -122,12 +196,13 @@ class TestMoFallback(unittest.TestCase): def tearDown(self): for name in self.models.keys(): os.remove(name) - os.remove(self.trans_config_file) + for name in self.test_config_files: + os.remove(name) shutil.rmtree(self.paddle_dir) @generate(*[('dir_to_extension', None, None, 'mo_legacy', 'extensions'), # fallback - ('dir_to_extension', None, True, 'onnx_frontend', None), + ('dir_to_extension', None, True, None, None), # exception ('dir_to_extension', True, None, 'mo_legacy', None), ('', True, None, 'mo_legacy', None), ('', None, True, 'onnx_frontend', None), @@ -139,8 +214,98 @@ class TestMoFallback(unittest.TestCase): args = base_args_config(use_legacy, use_new_fe) args.extensions = extension args.input_model = "test_model.onnx" + + if conversion_method: + prepare_ir(args) + tm.Telemetry.send_event.assert_any_call('mo', 'conversion_method', conversion_method) + if fallback_reason: + tm.Telemetry.send_event.assert_any_call('mo', 'fallback_reason', fallback_reason) + else: + with pytest.raises(AssertionError): # not called + tm.Telemetry.send_event.assert_any_call('mo', 'fallback_reason', fallback_reason) + else: + with pytest.raises(Error): # not supported extensions on new path + prepare_ir(args) + + + @generate(*[(None, None, 'onnx_frontend'), + (True, None, None), # exception + (None, True, 'onnx_frontend'), + ]) + def test_fallback_if_new_extension_specified(self, use_legacy, use_new_fe, conversion_method): + with patch('openvino.tools.mo.main.get_default_frontends') as default_fe: + default_fe.return_value = get_test_default_frontends() + args = base_args_config(use_legacy, use_new_fe) + args.extensions = 'onnx_fe_ext.so' + args.input_model = "test_model.onnx" + + if conversion_method: + prepare_ir(args) + tm.Telemetry.send_event.assert_any_call('mo', 'conversion_method', conversion_method) + else: + with pytest.raises(Error): + prepare_ir(args) + + + @generate(*[(None, None, 'onnx_frontend'), + (True, None, None), # exception + (None, True, 'onnx_frontend'), + ]) + def test_fallback_if_two_new_extension_specified(self, use_legacy, use_new_fe, conversion_method): + with patch('openvino.tools.mo.main.get_default_frontends') as default_fe: + default_fe.return_value = get_test_default_frontends() + args = base_args_config(use_legacy, use_new_fe) + args.extensions = 'onnx_fe_ext.so,onnx_fe_ext_2.so' + args.input_model = "test_model.onnx" + + if conversion_method: + prepare_ir(args) + tm.Telemetry.send_event.assert_any_call('mo', 'conversion_method', conversion_method) + else: + with pytest.raises(Error): + prepare_ir(args) + + + @generate(*[('fake_config.json' , None, None, 'mo_legacy', 'transformations_config'), # fallback + ('fake_config.json' , True, None, 'mo_legacy', None), + (None, None, True, 'onnx_frontend', None), + (None, None, None, 'onnx_frontend', None), + ]) + def test_fallback_if_tranformations_config_specified(self, trans_config, use_legacy, use_new_fe, expected_path, fallback_reason): + with patch('openvino.tools.mo.main.get_default_frontends') as default_fe: + default_fe.return_value = get_test_default_frontends() + args = base_args_config(use_legacy, use_new_fe) + args.input_model = "test_model.onnx" + args.transformations_config = trans_config + prepare_ir(args) + tm.Telemetry.send_event.assert_any_call('mo', 'conversion_method', expected_path) + if fallback_reason: + tm.Telemetry.send_event.assert_any_call('mo', 'fallback_reason', fallback_reason) + else: + with pytest.raises(AssertionError): # not called + tm.Telemetry.send_event.assert_any_call('mo', 'fallback_reason', fallback_reason) + + + @generate(*[('test_config_1.json', None, None, 'onnx_frontend', None), # 'library' attribute for all transformations + ('test_config_2.json', None, None, 'onnx_frontend', None), # 'library' attribute in single transformation + ('test_config_3.json', None, None, 'mo_legacy', 'transformations_config'), # 'library' attribute in no transformations + ]) + def test_fallback_if_new_tranformations_config_specified(self, trans_config, use_legacy, use_new_fe, conversion_method, fallback_reason): + with patch('openvino.tools.mo.main.get_default_frontends') as default_fe: + default_fe.return_value = get_test_default_frontends() + args = base_args_config(use_legacy, use_new_fe) + args.input_model = "test_model.onnx" + args.transformations_config = trans_config + + with patch('openvino.tools.mo.utils.class_registration.apply_transform'): # skip applying transforms + if conversion_method == 'onnx_frontend': + with pytest.raises(RuntimeError): # workaround to use in tests not existed libaries + prepare_ir(args) + else: + prepare_ir(args) + tm.Telemetry.send_event.assert_any_call('mo', 'conversion_method', conversion_method) if fallback_reason: tm.Telemetry.send_event.assert_any_call('mo', 'fallback_reason', fallback_reason) @@ -149,40 +314,41 @@ class TestMoFallback(unittest.TestCase): tm.Telemetry.send_event.assert_any_call('mo', 'fallback_reason', fallback_reason) - @generate(*[(True, None, None, 'mo_legacy', 'transformations_config'), # fallback - (True, True, None, 'mo_legacy', None), - (False, None, True, 'onnx_frontend', None), - (False, None, None, 'onnx_frontend', None), - ]) - def test_fallback_if_tranformations_config_specified(self, trans_config_used, use_legacy, use_new_fe, expected_path, fallback_reason): + def test_exception_if_new_trans_config_on_legacy_path(self): with patch('openvino.tools.mo.main.get_default_frontends') as default_fe: default_fe.return_value = get_test_default_frontends() - args = base_args_config(use_legacy, use_new_fe) + args = base_args_config(use_legacy_fe=True) args.input_model = "test_model.onnx" - args.transformations_config = self.trans_config_file if trans_config_used else None + args.transformations_config = 'test_config_1.json' - prepare_ir(args) - - tm.Telemetry.send_event.assert_any_call('mo', 'conversion_method', expected_path) - if fallback_reason: - tm.Telemetry.send_event.assert_any_call('mo', 'fallback_reason', fallback_reason) - else: - with pytest.raises(AssertionError): # not called - tm.Telemetry.send_event.assert_any_call('mo', 'fallback_reason', fallback_reason) + with pytest.raises(Error) as ex: # not called + prepare_ir(args) + assert str(ex) == 'New kind of transformations configuration used on legacy path' - @generate(*[('dir_to_extension', True, None, 'mo_legacy', 'extensions, transformations_config'), # fallback - (None, True, None, 'mo_legacy', 'transformations_config'), # fallback - ('dir_to_extension', False, None, 'mo_legacy', 'extensions'), # fallback - (None, False, True, 'onnx_frontend', None), + def test_exeption_if_mixed_types_of_trans_configs(self): + with patch('openvino.tools.mo.main.get_default_frontends') as default_fe: + default_fe.return_value = get_test_default_frontends() + args = base_args_config() + args.input_model = "test_model.onnx" + args.transformations_config = 'test_config_4.json' + + with pytest.raises(Error): + prepare_ir(args) + + + @generate(*[('dir_to_extension', 'fake_config.json', None, 'mo_legacy', 'extensions, transformations_config'), # fallback + (None, 'fake_config.json', None, 'mo_legacy', 'transformations_config'), # fallback + ('dir_to_extension', None, None, 'mo_legacy', 'extensions'), # fallback + (None, None, True, 'onnx_frontend', None), ]) - def test_fallback_if_both_extension_and_trans_config_specified(self, extension, trans_config_used, use_new_fe, expected_path, fallback_reason): + def test_fallback_if_both_extension_and_trans_config_specified(self, extension, trans_config, use_new_fe, expected_path, fallback_reason): with patch('openvino.tools.mo.main.get_default_frontends') as default_fe: default_fe.return_value = get_test_default_frontends() args = base_args_config(use_new_fe=use_new_fe) args.extensions = extension args.input_model = "test_model.onnx" - args.transformations_config = self.trans_config_file if trans_config_used else None + args.transformations_config = trans_config prepare_ir(args) @@ -194,16 +360,16 @@ class TestMoFallback(unittest.TestCase): tm.Telemetry.send_event.assert_any_call('mo', 'fallback_reason', fallback_reason) - @generate(*[(True, None, None, 'mo_legacy'), - (True, True, None, 'mo_legacy'), - (False, None, True, 'onnx_frontend'), + @generate(*[('fake_config.json', None, None, 'mo_legacy'), + ('fake_config.json', True, None, 'mo_legacy'), + (None, None, True, 'onnx_frontend'), ]) - def test_fallback_if_legacy_set_as_default(self, trans_config_used, use_legacy, use_new_fe, expected_path): + def test_fallback_if_legacy_set_as_default(self, trans_config, use_legacy, use_new_fe, expected_path): with patch('openvino.tools.mo.main.get_default_frontends') as default_fe: default_fe.return_value = {'onnx': 'legacy', 'tf': 'legacy'} args = base_args_config(use_legacy, use_new_fe) args.input_model = "test_model.onnx" - args.transformations_config = self.trans_config_file if trans_config_used else None + args.transformations_config = trans_config prepare_ir(args) @@ -212,7 +378,7 @@ class TestMoFallback(unittest.TestCase): tm.Telemetry.send_event.assert_any_call('mo', 'fallback_reason') - @generate(*[(None, None, 'dir_to_extension', 'paddle_frontend'), + @generate(*[(None, None, 'test_config_1.json', 'paddle_frontend'), (True, None, None, 'paddle_frontend'), (None, None, None, 'paddle_frontend'), ]) @@ -227,3 +393,14 @@ class TestMoFallback(unittest.TestCase): tm.Telemetry.send_event.assert_any_call('mo', 'conversion_method', expected_path) with pytest.raises(AssertionError): # not called tm.Telemetry.send_event.assert_any_call('mo', 'fallback_reason') + + + def test_exception_if_old_extensions_used_for_pdpd(self): + args = base_args_config() + args.framework = 'paddle' + args.extensions = 'dir_to_extension' + args.input_model = 'paddle_dir/relu/relu.pdmodel' + + with pytest.raises(Error) as ex: # not called + prepare_ir(args) + assert str(ex) == 'Legacy transformations configuration is not supported for the new frontend'