Handle special cases during fallback (#9651)

docs/MO_DG/prepare_model/Model_Optimizer_FAQ.md

@@ -633,5 +633,6 @@ Note that you might have conflicts between previously installed PyPI dependencie
 For the models in ONNX* format, there are two available paths of IR conversion. 
 The old one is handled by the old Python* implementation, while the new one uses new C++ frontends. 
 Starting from the 2022.1 version, the default IR conversion path for ONNX models is processed using the new ONNX frontend. 
-Certain features, such as `--extensions` and `--transformations_config`, are not yet supported on the new frontends. 
+Certain features, such as `--extensions and` `--transformations_config`, are not yet fully supported on the new frontends. 
+For `--extensions`, the new frontends support only paths to shared libraries (.dll and .so). For `--transformations_config`, they support JSON configurations with defined library fields. 
 The IR conversion falls back to the old path if a user does not select any expected path of conversion explicitly (by `--use_new_frontend` or `--use_legacy_frontend` MO arguments) and unsupported pre-defined scenario is detected on the new frontend path.

src/bindings/python/src/pyopenvino/frontend/frontend.cpp

@@ -137,7 +137,28 @@ void regclass_frontend_FrontEnd(py::module m) {
             )");
 
     fem.def("add_extension",
-            static_cast<void (FrontEnd::*)(const std::shared_ptr<ov::Extension>& extension)>(&FrontEnd::add_extension));
+            static_cast<void (FrontEnd::*)(const std::shared_ptr<ov::Extension>& extension)>(&FrontEnd::add_extension),
+            R"(
+                Add extension defined by an object inheriting from Extension 
+                used in order to extend capabilities of Frontend.
+
+                Parameters
+                ----------
+                extension : Extension
+                    Provided extension object.
+            )");
+
+    fem.def("add_extension",
+            static_cast<void (FrontEnd::*)(const std::string& extension_path)>(&FrontEnd::add_extension),
+            R"(
+                Add extension defined in external library indicated by a extension_path 
+                used in order to extend capabilities of Frontend.
+
+                Parameters
+                ----------
+                extension_path : str
+                    A path to extension.
+            )");
 
     fem.def("__repr__", [](const FrontEnd& self) -> std::string {
         return "<FrontEnd '" + self.get_name() + "'>";

tools/mo/automation/package_BOM.txt

@@ -838,6 +838,7 @@ openvino/tools/mo/mo_paddle.py
 openvino/tools/mo/mo_tf.py
 openvino/tools/mo/moc_frontend/__init__.py
 openvino/tools/mo/moc_frontend/analysis.py
+openvino/tools/mo/moc_frontend/check_config.py
 openvino/tools/mo/moc_frontend/extractor.py
 openvino/tools/mo/moc_frontend/pipeline.py
 openvino/tools/mo/moc_frontend/serialize.py

tools/mo/openvino/tools/mo/main.py

@@ -10,6 +10,7 @@ import sys
 import traceback
 from collections import OrderedDict
 from copy import deepcopy
+import json
 
 try:
     import openvino_telemetry as tm
@@ -18,6 +19,8 @@ except ImportError:
 
 from openvino.tools.mo.back.SpecialNodesFinalization import RemoveConstOps, CreateConstNodesReplacement, NormalizeTI
 from openvino.tools.mo.back.ie_ir_ver_2.emitter import append_ir_info
+from openvino.tools.mo.moc_frontend.check_config import legacy_extensions_used, legacy_transformations_config_used, \
+    new_extensions_used, new_transformations_config_used
 from openvino.tools.mo.moc_frontend.pipeline import moc_pipeline
 from openvino.tools.mo.moc_frontend.serialize import moc_emit_ir
 from openvino.tools.mo.graph.graph import Graph
@@ -43,7 +46,7 @@ from openvino.tools.mo.utils.telemetry_utils import get_tid
 from openvino.tools.mo.front.common.partial_infer.utils import mo_array
 
 # pylint: disable=no-name-in-module,import-error
-from openvino.frontend import FrontEndManager, ProgressReporterExtension, TelemetryExtension
+from openvino.frontend import FrontEndManager, ProgressReporterExtension, TelemetryExtension, JsonConfigExtension
 
 
 def replace_ext(name: str, old: str, new: str):
@@ -140,7 +143,12 @@ def arguments_post_parsing(argv: argparse.Namespace):
     is_tf, is_caffe, is_mxnet, is_kaldi, is_onnx =\
         deduce_framework_by_namespace(argv) if not moc_front_end else [False, False, False, False, False]
 
-    if not any([is_tf, is_caffe, is_mxnet, is_kaldi, is_onnx]):
+    if any([is_tf, is_caffe, is_mxnet, is_kaldi, is_onnx]):
+        if new_extensions_used(argv):
+            raise Error('New kind of extensions used on legacy path')
+        if new_transformations_config_used(argv):
+            raise Error('New kind of transformations configuration used on legacy path')
+    else: # new frontend used
         frameworks = ['tf', 'caffe', 'mxnet', 'kaldi', 'onnx']
         frameworks = list(set(frameworks + available_moc_front_ends))
         if argv.framework not in frameworks:
@@ -330,11 +338,8 @@ def check_fallback(argv : argparse.Namespace):
     if argv.use_new_frontend:
         return fallback_reasons
 
-    fallback_reasons['extensions'] = \
-        lambda argv : hasattr(argv, 'extensions') and argv.extensions is not None and len(argv.extensions) > 0 \
-            and argv.extensions != import_extensions.default_path() # extensions arg has default value
-    fallback_reasons['transformations_config'] = \
-        lambda argv: hasattr(argv, 'transformations_config') and argv.transformations_config is not None and len(argv.transformations_config) > 0
+    fallback_reasons['extensions'] = legacy_extensions_used
+    fallback_reasons['transformations_config'] = legacy_transformations_config_used
 
     reasons = [reason for reason, is_applicable in fallback_reasons.items() if is_applicable(argv)]
     return reasons
@@ -352,6 +357,15 @@ def prepare_ir(argv : argparse.Namespace):
             t.send_event("mo", "conversion_method", moc_front_end.get_name() + "_frontend")
             moc_front_end.add_extension(TelemetryExtension("mo", t.send_event, t.send_error, t.send_stack_trace))
             moc_front_end.add_extension(ProgressReporterExtension(progress_printer(argv)))
+            if legacy_transformations_config_used(argv):
+                raise Error('Legacy extensions are not supported for the new frontend')
+            if legacy_extensions_used(argv):
+                raise Error('Legacy transformations configuration is not supported for the new frontend')
+            if new_transformations_config_used(argv):
+                moc_front_end.add_extension(JsonConfigExtension(argv.transformations_config))
+            if new_extensions_used(argv):
+                for extension in argv.extensions.split(','):
+                    moc_front_end.add_extension(extension)
             ngraph_function = moc_pipeline(argv, moc_front_end)
             return graph, ngraph_function
         else: # apply fallback

tools/mo/openvino/tools/mo/moc_frontend/check_config.py

@@ -0,0 +1,91 @@
+# Copyright (C) 2022 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+import argparse
+import json
+from pathlib import Path
+
+from openvino.tools.mo.utils import import_extensions
+from openvino.tools.mo.utils.error import Error
+
+
+def any_extensions_used(argv: argparse.Namespace):
+    return hasattr(argv, 'extensions') and argv.extensions is not None and len(argv.extensions) > 0 \
+        and argv.extensions != import_extensions.default_path() # extensions arg has default value
+
+
+def legacy_extensions_used(argv: argparse.Namespace):
+    if any_extensions_used(argv):
+        extensions = argv.extensions.split(',')
+        legacy_ext_counter = 0
+        for extension in extensions:
+            path = Path(extension)
+            if not path.is_file():
+                legacy_ext_counter += 1
+        if legacy_ext_counter == len(extensions):
+            return True # provided only legacy extensions
+        elif legacy_ext_counter == 0:
+            return False # provided only new extensions
+        else:
+            raise Error('Using new and legacy extensions in the same time is forbidden')
+    return False
+
+
+def new_extensions_used(argv: argparse.Namespace):
+    if any_extensions_used(argv):
+        extensions = argv.extensions.split(',')
+        new_ext_counter = 0
+        for extension in argv.extensions.split(','):
+            path = Path(extension)
+            if path.is_file() and (path.suffix == '.so' or path.suffix == '.dll'):
+                new_ext_counter += 1
+        if new_ext_counter == len(extensions):
+            return True # provided only new extensions
+        elif new_ext_counter == 0:
+            return False # provided only legacy extensions
+        else:
+            raise Error('Using new and legacy extensions in the same time is forbidden')
+    return False
+
+
+def is_new_json_config(json_file_path: str):
+    with open(json_file_path) as stream:
+        config_content = json.load(stream)
+        if len(config_content) == 0: # empty case
+            return False
+        if isinstance(config_content, dict): # single transformation
+            return 'library' in config_content.keys()
+        # many transformations in single file
+        library_counter = 0
+        for transform in config_content:
+            if any(key == 'library' for key in transform.keys()):
+                library_counter+=1
+        if len(config_content) == library_counter: # all transformations has 'library' attribute
+            return True
+        elif library_counter == 0: # all transformations are legacy type
+            return False
+        else:
+            raise Error('Mixed types of transformations configurations were used')
+
+
+def get_transformations_config_path(argv: argparse.Namespace) -> Path:
+    if hasattr(argv, 'transformations_config') \
+        and argv.transformations_config is not None and len(argv.transformations_config):
+        path = Path(argv.transformations_config)
+        if path.is_file():
+            return path
+    return None
+
+
+def new_transformations_config_used(argv: argparse.Namespace):
+    path = get_transformations_config_path(argv)
+    if path != None:
+        return is_new_json_config(path)
+    return False
+
+
+def legacy_transformations_config_used(argv: argparse.Namespace):
+    path = get_transformations_config_path(argv)
+    if path != None:
+        return not is_new_json_config(path)
+    return False

tools/mo/unit_tests/mo/utils/mo_fallback_test_actual.py

@@ -2,12 +2,12 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import unittest
-from unittest.mock import Mock
-from unittest.mock import patch
+from unittest.mock import patch, Mock
 
 import openvino
 from openvino.tools.mo.main import prepare_ir
-from openvino.frontend import FrontEndManager # pylint: disable=no-name-in-module,import-error
+from openvino.tools.mo.utils.error import Error
+from openvino.frontend import FrontEndManager, FrontEnd # pylint: disable=no-name-in-module,import-error
 from onnx.helper import make_graph, make_model, make_tensor_value_info
 import argparse
 import os
@@ -81,6 +81,7 @@ class TestMoFallback(unittest.TestCase):
     def setUp(self):
         tm.Telemetry.__init__ = Mock(return_value=None)
         tm.Telemetry.send_event = Mock()
+        FrontEnd.add_extension = Mock()
 
         self.models = {}
         add = onnx.helper.make_node("Add", inputs=["in1", "in2"], outputs=["add_out"])
@@ -99,10 +100,83 @@ class TestMoFallback(unittest.TestCase):
         for name, model in self.models.items():
             onnx.save(model, name)
 
-        trans_config = 'config.json'
-        with open(trans_config, 'w') as f:
-            f.write("[]") # json format
-        self.trans_config_file = os.path.abspath(trans_config)
+        self.test_config_files = {}
+        self.test_config_files['fake_config.json'] = '[]' # json format
+
+        self.test_config_files['test_config_1.json'] = """[
+            {
+            "custom_attributes": {
+            "test_attribute": true
+            },
+            "id": "TransformationName1",
+            "library": "path_to_library1.so",
+            "match_kind": "scope"
+            },
+            { 
+            "custom_attributes": {
+            },
+            "id": "TransfromationName2",
+            "library": "path_to_library2.so",
+            "match_kind": "scope"
+            },
+            {
+            "id": "TransfromationName3",
+            "library": "path_to_library3.so",
+            "match_kind": "scope"
+            }
+        ]"""
+
+        self.test_config_files['test_config_2.json'] = """{
+            "custom_attributes": {
+            "test_attribute": true
+            },
+            "id": "TransformationName1",
+            "library": "path_to_library.so",
+            "match_kind": "scope"
+        }"""
+
+        self.test_config_files['test_config_3.json'] = """[
+            {
+            "custom_attributes": {
+            "test_attribute": true
+            },
+            "id": "TransformationName1",
+            "match_kind": "scope"
+            },
+            { 
+            "custom_attributes": {
+            },
+            "id": "TransfromationName2",
+            "match_kind": "scope"
+            }
+        ]"""
+
+        self.test_config_files['test_config_4.json'] = """[
+        {
+            "custom_attributes": {
+            "test_attribute": true
+            },
+            "id": "TransformationName1",
+            "library": "path_to_library",
+            "match_kind": "scope"
+            },
+            { 
+            "custom_attributes": {
+            },
+            "id": "TransfromationName2",
+            "match_kind": "scope"
+            },
+            {
+            "library": "path_to_library.so"
+            }
+        ]"""
+
+        self.test_config_files['onnx_fe_ext.so'] = 'binary_content'
+        self.test_config_files['onnx_fe_ext_2.so'] = 'binary_content'
+
+        for file, content in self.test_config_files.items():
+            with open(file, 'w') as f:
+                f.write(content)
 
         self.paddle_dir = "paddle_dir"
         paddle.enable_static()
@@ -122,12 +196,13 @@ class TestMoFallback(unittest.TestCase):
     def tearDown(self):
         for name in self.models.keys():
             os.remove(name)
-        os.remove(self.trans_config_file)
+        for name in self.test_config_files:
+            os.remove(name)
         shutil.rmtree(self.paddle_dir)
 
 
     @generate(*[('dir_to_extension', None, None, 'mo_legacy', 'extensions'), # fallback
-                ('dir_to_extension', None, True, 'onnx_frontend', None),
+                ('dir_to_extension', None, True, None, None), # exception
                 ('dir_to_extension', True, None, 'mo_legacy', None),
                 ('', True, None, 'mo_legacy', None),
                 ('', None, True, 'onnx_frontend', None),
@@ -139,6 +214,96 @@ class TestMoFallback(unittest.TestCase):
             args = base_args_config(use_legacy, use_new_fe)
             args.extensions = extension
             args.input_model = "test_model.onnx"
+
+            if conversion_method:
+                prepare_ir(args)
+                tm.Telemetry.send_event.assert_any_call('mo', 'conversion_method', conversion_method)
+                if fallback_reason:
+                    tm.Telemetry.send_event.assert_any_call('mo', 'fallback_reason', fallback_reason)
+                else:
+                    with pytest.raises(AssertionError): # not called
+                        tm.Telemetry.send_event.assert_any_call('mo', 'fallback_reason', fallback_reason)
+            else:
+                with pytest.raises(Error): # not supported extensions on new path
+                    prepare_ir(args)
+
+
+    @generate(*[(None, None, 'onnx_frontend'),
+                (True, None, None), # exception
+                (None, True, 'onnx_frontend'),
+    ])
+    def test_fallback_if_new_extension_specified(self, use_legacy, use_new_fe, conversion_method):
+        with patch('openvino.tools.mo.main.get_default_frontends') as default_fe:
+            default_fe.return_value = get_test_default_frontends()
+            args = base_args_config(use_legacy, use_new_fe)
+            args.extensions = 'onnx_fe_ext.so'
+            args.input_model = "test_model.onnx"
+
+            if conversion_method:
+                prepare_ir(args)
+                tm.Telemetry.send_event.assert_any_call('mo', 'conversion_method', conversion_method)
+            else:
+                with pytest.raises(Error):
+                    prepare_ir(args)
+
+
+    @generate(*[(None, None, 'onnx_frontend'),
+                (True, None, None), # exception
+                (None, True, 'onnx_frontend'),
+    ])
+    def test_fallback_if_two_new_extension_specified(self, use_legacy, use_new_fe, conversion_method):
+        with patch('openvino.tools.mo.main.get_default_frontends') as default_fe:
+            default_fe.return_value = get_test_default_frontends()
+            args = base_args_config(use_legacy, use_new_fe)
+            args.extensions = 'onnx_fe_ext.so,onnx_fe_ext_2.so'
+            args.input_model = "test_model.onnx"
+
+            if conversion_method:
+                prepare_ir(args)
+                tm.Telemetry.send_event.assert_any_call('mo', 'conversion_method', conversion_method)
+            else:
+                with pytest.raises(Error):
+                    prepare_ir(args)
+
+
+    @generate(*[('fake_config.json' , None, None, 'mo_legacy', 'transformations_config'), # fallback
+                ('fake_config.json' , True, None, 'mo_legacy', None),
+                (None, None, True, 'onnx_frontend', None),
+                (None, None, None, 'onnx_frontend', None),
+    ])
+    def test_fallback_if_tranformations_config_specified(self, trans_config, use_legacy, use_new_fe, expected_path, fallback_reason):
+        with patch('openvino.tools.mo.main.get_default_frontends') as default_fe:
+            default_fe.return_value = get_test_default_frontends()
+            args = base_args_config(use_legacy, use_new_fe)
+            args.input_model = "test_model.onnx"
+            args.transformations_config = trans_config
+
+            prepare_ir(args)
+
+            tm.Telemetry.send_event.assert_any_call('mo', 'conversion_method', expected_path)
+            if fallback_reason:
+                tm.Telemetry.send_event.assert_any_call('mo', 'fallback_reason', fallback_reason)
+            else:
+                with pytest.raises(AssertionError): # not called
+                    tm.Telemetry.send_event.assert_any_call('mo', 'fallback_reason', fallback_reason)
+
+
+    @generate(*[('test_config_1.json', None, None, 'onnx_frontend', None), # 'library' attribute for all transformations
+                ('test_config_2.json', None, None, 'onnx_frontend', None), # 'library' attribute in single transformation
+                ('test_config_3.json', None, None, 'mo_legacy', 'transformations_config'), # 'library' attribute in no transformations
+    ])
+    def test_fallback_if_new_tranformations_config_specified(self, trans_config, use_legacy, use_new_fe, conversion_method, fallback_reason):
+        with patch('openvino.tools.mo.main.get_default_frontends') as default_fe:
+            default_fe.return_value = get_test_default_frontends()
+            args = base_args_config(use_legacy, use_new_fe)
+            args.input_model = "test_model.onnx"
+            args.transformations_config = trans_config
+
+            with patch('openvino.tools.mo.utils.class_registration.apply_transform'): # skip applying transforms
+                if conversion_method == 'onnx_frontend':
+                    with pytest.raises(RuntimeError): # workaround to use in tests not existed libaries
+                        prepare_ir(args)
+                else:
                     prepare_ir(args)
 
             tm.Telemetry.send_event.assert_any_call('mo', 'conversion_method', conversion_method)
@@ -149,40 +314,41 @@ class TestMoFallback(unittest.TestCase):
                     tm.Telemetry.send_event.assert_any_call('mo', 'fallback_reason', fallback_reason)
 
 
-    @generate(*[(True, None, None, 'mo_legacy', 'transformations_config'), # fallback
-                (True, True, None, 'mo_legacy', None),
-                (False, None, True, 'onnx_frontend', None),
-                (False, None, None, 'onnx_frontend', None),
-    ])
-    def test_fallback_if_tranformations_config_specified(self, trans_config_used, use_legacy, use_new_fe, expected_path, fallback_reason):
+    def test_exception_if_new_trans_config_on_legacy_path(self):
         with patch('openvino.tools.mo.main.get_default_frontends') as default_fe:
             default_fe.return_value = get_test_default_frontends()
-            args = base_args_config(use_legacy, use_new_fe)
+            args = base_args_config(use_legacy_fe=True)
             args.input_model = "test_model.onnx"
-            args.transformations_config = self.trans_config_file if trans_config_used else None
+            args.transformations_config = 'test_config_1.json'
 
+            with pytest.raises(Error) as ex: # not called
+                prepare_ir(args)
+                assert str(ex) == 'New kind of transformations configuration used on legacy path'
+
+
+    def test_exeption_if_mixed_types_of_trans_configs(self):
+        with patch('openvino.tools.mo.main.get_default_frontends') as default_fe:
+            default_fe.return_value = get_test_default_frontends()
+            args = base_args_config()
+            args.input_model = "test_model.onnx"
+            args.transformations_config = 'test_config_4.json'
+
+            with pytest.raises(Error):
                 prepare_ir(args)
 
-            tm.Telemetry.send_event.assert_any_call('mo', 'conversion_method', expected_path)
-            if fallback_reason:
-                tm.Telemetry.send_event.assert_any_call('mo', 'fallback_reason', fallback_reason)
-            else:
-                with pytest.raises(AssertionError): # not called
-                    tm.Telemetry.send_event.assert_any_call('mo', 'fallback_reason', fallback_reason)
 
-
-    @generate(*[('dir_to_extension', True, None, 'mo_legacy', 'extensions, transformations_config'), # fallback
-                (None, True, None, 'mo_legacy', 'transformations_config'), # fallback
-                ('dir_to_extension', False, None, 'mo_legacy', 'extensions'), # fallback
-                (None, False, True, 'onnx_frontend', None),
+    @generate(*[('dir_to_extension', 'fake_config.json', None, 'mo_legacy', 'extensions, transformations_config'), # fallback
+                (None, 'fake_config.json', None, 'mo_legacy', 'transformations_config'), # fallback
+                ('dir_to_extension', None, None, 'mo_legacy', 'extensions'), # fallback
+                (None, None, True, 'onnx_frontend', None),
     ])
-    def test_fallback_if_both_extension_and_trans_config_specified(self, extension, trans_config_used, use_new_fe, expected_path, fallback_reason):
+    def test_fallback_if_both_extension_and_trans_config_specified(self, extension, trans_config, use_new_fe, expected_path, fallback_reason):
         with patch('openvino.tools.mo.main.get_default_frontends') as default_fe:
             default_fe.return_value = get_test_default_frontends()
             args = base_args_config(use_new_fe=use_new_fe)
             args.extensions = extension
             args.input_model = "test_model.onnx"
-            args.transformations_config = self.trans_config_file if trans_config_used else None
+            args.transformations_config = trans_config
 
             prepare_ir(args)
 
@@ -194,16 +360,16 @@ class TestMoFallback(unittest.TestCase):
                     tm.Telemetry.send_event.assert_any_call('mo', 'fallback_reason', fallback_reason)
 
 
-    @generate(*[(True, None, None, 'mo_legacy'),
-                (True, True, None, 'mo_legacy'),
-                (False, None, True, 'onnx_frontend'),
+    @generate(*[('fake_config.json', None, None, 'mo_legacy'),
+                ('fake_config.json', True, None, 'mo_legacy'),
+                (None, None, True, 'onnx_frontend'),
     ])
-    def test_fallback_if_legacy_set_as_default(self, trans_config_used, use_legacy, use_new_fe, expected_path):
+    def test_fallback_if_legacy_set_as_default(self, trans_config, use_legacy, use_new_fe, expected_path):
         with patch('openvino.tools.mo.main.get_default_frontends') as default_fe:
             default_fe.return_value = {'onnx': 'legacy', 'tf': 'legacy'}
             args = base_args_config(use_legacy, use_new_fe)
             args.input_model = "test_model.onnx"
-            args.transformations_config = self.trans_config_file if trans_config_used else None
+            args.transformations_config = trans_config
 
             prepare_ir(args)
 
@@ -212,7 +378,7 @@ class TestMoFallback(unittest.TestCase):
                 tm.Telemetry.send_event.assert_any_call('mo', 'fallback_reason')
 
 
-    @generate(*[(None, None, 'dir_to_extension', 'paddle_frontend'),
+    @generate(*[(None, None, 'test_config_1.json', 'paddle_frontend'),
                 (True, None, None, 'paddle_frontend'),
                 (None, None, None, 'paddle_frontend'),
     ])
@@ -227,3 +393,14 @@ class TestMoFallback(unittest.TestCase):
         tm.Telemetry.send_event.assert_any_call('mo', 'conversion_method', expected_path)
         with pytest.raises(AssertionError): # not called
             tm.Telemetry.send_event.assert_any_call('mo', 'fallback_reason')
+
+
+    def test_exception_if_old_extensions_used_for_pdpd(self):
+        args = base_args_config()
+        args.framework = 'paddle'
+        args.extensions = 'dir_to_extension'
+        args.input_model = 'paddle_dir/relu/relu.pdmodel'
+
+        with pytest.raises(Error) as ex: # not called
+            prepare_ir(args)
+            assert str(ex) == 'Legacy transformations configuration is not supported for the new frontend'