MO fallback to old path if some features are not supported yet (#9034)

docs/MO_DG/prepare_model/Model_Optimizer_FAQ.md

@@ -626,4 +626,12 @@ The issue "SyntaxError: 'yield' inside list comprehension" might occur during co
 The following workarounds are suggested to resolve this issue:
 1. Use Python 3.6/3.7 to convert MXNet\* models on Windows
 2. Update MXNet: pip install mxnet=1.7.0.post2
-Note that you might have conflicts between previously installed PyPI dependencies.
+Note that you might have conflicts between previously installed PyPI dependencies.m
+
+#### 105. What does the message "The IR preparation was executed by the legacy MO path. ..." mean? <a name="question-105"></a>
+
+For the models in ONNX* format, there are two available paths of IR conversion. 
+The old one is handled by the old Python* implementation, while the new one uses new C++ frontends. 
+Starting from the 2022.1 version, the default IR conversion path for ONNX models is processed using the new ONNX frontend. 
+Certain features, such as `--extensions` and `--transformations_config`, are not yet supported on the new frontends. 
+The IR conversion falls back to the old path if a user does not select any expected path of conversion explicitly (by `--use_new_frontend` or `--use_legacy_frontend` MO arguments) and unsupported pre-defined scenario is detected on the new frontend path.

tools/mo/openvino/tools/mo/main.py

@@ -95,6 +95,15 @@ def print_argv(argv: argparse.Namespace, is_caffe: bool, is_tf: bool, is_mxnet:
     print('\n'.join(lines), flush=True)
 
 
+def get_default_frontends():
+    # Set which frontend to use by default, values should be 'new' or 'legacy'
+    default_frontends = {
+        'onnx': 'legacy',
+        'tf': 'legacy'
+    }
+    return default_frontends
+
+
 def get_moc_frontends(argv: argparse.Namespace):
     fem = argv.feManager
 
@@ -117,13 +126,9 @@ def get_moc_frontends(argv: argparse.Namespace):
     else:
         return None, []
 
-    # Set which frontend to use by default, values should be 'new' or 'legacy'
+    default_frontends = get_default_frontends()
-    frontend_defaults = {
-        'onnx': 'legacy',
-        'tf': 'legacy'
-    }
     # Disable MOC frontend if default is set to legacy and no user override
-    if frontend_defaults.get(moc_front_end.get_name()) == 'legacy' and not use_new_frontend:
+    if default_frontends.get(moc_front_end.get_name()) == 'legacy' and not use_new_frontend:
         moc_front_end = None
 
     return moc_front_end, available_moc_front_ends
@@ -281,13 +286,17 @@ def arguments_post_parsing(argv: argparse.Namespace):
 
     log.debug("Placeholder shapes : {}".format(argv.placeholder_shapes))
 
-    if hasattr(argv, 'extensions') and argv.extensions and argv.extensions != '':
-        extensions = argv.extensions.split(',')
-    else:
-        extensions = None
-
     argv.freeze_placeholder_with_value, argv.input = get_freeze_placeholder_values(argv.input,
                                                                                    argv.freeze_placeholder_with_value)
+
+    load_extensions(argv, is_tf, is_caffe, is_mxnet, is_kaldi, is_onnx)
+
+    return argv
+
+def load_extensions(argv: argparse.Namespace, is_tf: bool, is_caffe: bool, is_mxnet: bool, is_kaldi: bool, is_onnx:bool):
+    extensions = None
+    if hasattr(argv, 'extensions') and argv.extensions and argv.extensions != '':
+        extensions = argv.extensions.split(',')
     if is_tf:
         from openvino.tools.mo.front.tf.register_custom_ops import get_front_classes
         import_extensions.load_dirs(argv.framework, extensions, get_front_classes)
@@ -308,25 +317,54 @@ def arguments_post_parsing(argv: argparse.Namespace):
         from openvino.tools.mo.front.onnx.register_custom_ops import get_front_classes
         import_extensions.load_dirs(argv.framework, extensions, get_front_classes)
 
-    return argv
+
+def check_fallback(argv : argparse.Namespace):
+    fallback_reasons = {}
+
+    # Some frontend such as PDPD does not have legacy path so it has no reasons to fallback
+    if not any(deduce_framework_by_namespace(argv)):
+        return fallback_reasons
+
+    # There is no possibility for fallback if a user strictly wants to use new frontend
+    if argv.use_new_frontend:
+        return fallback_reasons
+
+    fallback_reasons['extensions'] = \
+        lambda argv : hasattr(argv, 'extensions') and argv.extensions is not None and len(argv.extensions) > 0 \
+            and argv.extensions != import_extensions.default_path() # extensions arg has default value
+    fallback_reasons['transformations_config'] = \
+        lambda argv: hasattr(argv, 'transformations_config') and argv.transformations_config is not None and len(argv.transformations_config) > 0
+
+    reasons = [reason for reason, is_applicable in fallback_reasons.items() if is_applicable(argv)]
+    return reasons
 
 
-def prepare_ir(argv):
+def prepare_ir(argv : argparse.Namespace):
     argv = arguments_post_parsing(argv)
-
     t = tm.Telemetry()
     graph = None
     ngraph_function = None
     moc_front_end, available_moc_front_ends = get_moc_frontends(argv)
-
     if moc_front_end:
-        t.send_event("mo", "conversion_method", moc_front_end.get_name() + "_frontend")
+        fallback_reasons = check_fallback(argv)
-        moc_front_end.add_extension(TelemetryExtension("mo", t.send_event, t.send_error, t.send_stack_trace))
+        if len(fallback_reasons) == 0:
-        moc_front_end.add_extension(ProgressReporterExtension(progress_printer(argv)))
+            t.send_event("mo", "conversion_method", moc_front_end.get_name() + "_frontend")
-        ngraph_function = moc_pipeline(argv, moc_front_end)
+            moc_front_end.add_extension(TelemetryExtension("mo", t.send_event, t.send_error, t.send_stack_trace))
-    else:
+            moc_front_end.add_extension(ProgressReporterExtension(progress_printer(argv)))
-        t.send_event("mo", "conversion_method", "mo_legacy")
+            ngraph_function = moc_pipeline(argv, moc_front_end)
-        graph = unified_pipeline(argv)
+            return graph, ngraph_function
+        else: # apply fallback
+            reasons_message = ", ".join(fallback_reasons)
+            load_extensions(argv, *list(deduce_framework_by_namespace(argv)))
+            t.send_event("mo", "fallback_reason", reasons_message)
+            log.warning("The IR preparation was executed by the legacy MO path. "
+                        "This is a fallback scenario applicable only for some specific cases. "
+                       f"The detailed reason why fallback was executed: not supported {reasons_message} were used. "
+                        "You can specify --use_new_frontend flag to force using the Frontend MO path to avoid additional checks. " +
+                        refer_to_faq_msg(105))
+
+    t.send_event("mo", "conversion_method", "mo_legacy")
+    graph = unified_pipeline(argv)
 
     return graph, ngraph_function
 

tools/mo/unit_tests/mo/frontend_ngraph_test.py

@@ -59,3 +59,12 @@ def test_main_test():
 
     status = subprocess.run(args, env=os.environ)
     assert not status.returncode
+
+
+def test_mo_fallback_test():
+    setup_env()
+    args = [sys.executable, '-m', 'pytest',
+            os.path.join(os.path.dirname(__file__), 'utils/mo_fallback_test_actual.py'), '-s']
+
+    status = subprocess.run(args, env=os.environ)
+    assert not status.returncode

tools/mo/unit_tests/mo/utils/mo_fallback_test_actual.py

@@ -0,0 +1,229 @@
+# Copyright (C) 2021 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+import unittest
+from unittest.mock import Mock
+from unittest.mock import patch
+
+import openvino
+from openvino.tools.mo.main import prepare_ir
+from openvino.frontend import FrontEndManager # pylint: disable=no-name-in-module,import-error
+from onnx.helper import make_graph, make_model, make_tensor_value_info
+import argparse
+import os
+import onnx
+import paddle
+import numpy as np
+import shutil
+import pytest
+from generator import generator, generate
+
+try:
+    import openvino_telemetry as tm
+except ImportError:
+    import openvino.tools.mo.utils.telemetry_stub as tm
+
+def base_args_config(use_legacy_fe:bool=None, use_new_fe:bool=None):
+    args = argparse.Namespace()
+    args.feManager = FrontEndManager()
+    args.extensions = None
+    args.use_legacy_frontend = use_legacy_fe
+    args.use_new_frontend = use_new_fe
+    args.framework = 'onnx'
+    args.model_name = None
+    args.input_model = None
+    args.silent = True
+    args.transform=[]
+    args.legacy_ir_generation = False
+    args.scale = None
+    args.output=None
+    args.input=None
+    args.input_shape=None
+    args.batch=None
+    args.mean_values=None
+    args.scale_values=None
+    args.output_dir=os.getcwd()
+    args.freeze_placeholder_with_value = None
+    args.transformations_config = None
+    args.disable_fusing = None
+    args.finegrain_fusing = None
+    args.disable_gfusing = None
+    args.disable_resnet_optimization = None
+    args.enable_concat_optimization = None
+    args.static_shape = None
+    args.disable_weights_compression = None
+    args.reverse_input_channels = None
+    args.data_type = None
+    args.layout = None
+    args.source_layout = None
+    args.target_layout = None
+    return args
+
+
+def get_test_default_frontends():
+    return {
+        'onnx': 'new',
+        'tf': 'legacy'
+    }
+
+
+def save_paddle_model(name, exe, feedkeys:list, fetchlist:list, target_dir:str):
+    model_dir = os.path.join(target_dir, name)
+    if not os.path.exists(model_dir):
+        os.makedirs(model_dir)
+
+    paddle.fluid.io.save_inference_model(model_dir, feedkeys, fetchlist, exe)
+    paddle.fluid.io.save_inference_model(model_dir, feedkeys, fetchlist, exe, model_filename=name+".pdmodel", params_filename=name+".pdiparams")
+
+
+@generator
+class TestMoFallback(unittest.TestCase):
+    def setUp(self):
+        tm.Telemetry.__init__ = Mock(return_value=None)
+        tm.Telemetry.send_event = Mock()
+
+        self.models = {}
+        add = onnx.helper.make_node("Add", inputs=["in1", "in2"], outputs=["add_out"])
+        input_tensors = [
+            make_tensor_value_info("in1", onnx.TensorProto.FLOAT, (2, 2)),
+            make_tensor_value_info("in2", onnx.TensorProto.FLOAT, (2, 2)),
+        ]
+        output_tensors = [
+            make_tensor_value_info("add_out", onnx.TensorProto.FLOAT, (1, 2)),
+        ]
+        graph = make_graph([add], "test_graph", input_tensors, output_tensors)
+        model = make_model(graph, producer_name="MO tests",
+                                                opset_imports=[onnx.helper.make_opsetid("", 13)])
+        self.models["test_model.onnx"] = model
+
+        for name, model in self.models.items():
+            onnx.save(model, name)
+
+        trans_config = 'config.json'
+        with open(trans_config, 'w') as f:
+            f.write("[]") # json format
+        self.trans_config_file = os.path.abspath(trans_config)
+
+        self.paddle_dir = "paddle_dir"
+        paddle.enable_static()
+        if not os.path.exists(self.paddle_dir):
+            os.mkdir(self.paddle_dir)
+        x = np.array([-2, 0, 1]).astype('float32')
+        node_x = paddle.static.data(name='x', shape=x.shape, dtype='float32')
+        out = paddle.nn.functional.relu(node_x)
+
+        cpu = paddle.static.cpu_places(1)
+        exe = paddle.static.Executor(cpu[0])
+        exe.run(paddle.static.default_startup_program())
+
+        save_paddle_model("relu", exe, feedkeys=['x'], fetchlist=[out], target_dir=self.paddle_dir)
+
+
+    def tearDown(self):
+        for name in self.models.keys():
+            os.remove(name)
+        os.remove(self.trans_config_file)
+        shutil.rmtree(self.paddle_dir)
+
+
+    @generate(*[('dir_to_extension', None, None, 'mo_legacy', 'extensions'), # fallback
+                ('dir_to_extension', None, True, 'onnx_frontend', None),
+                ('dir_to_extension', True, None, 'mo_legacy', None),
+                ('', True, None, 'mo_legacy', None),
+                ('', None, True, 'onnx_frontend', None),
+                (None, None, None, 'onnx_frontend', None),
+    ])
+    def test_fallback_if_extension_specified(self, extension, use_legacy, use_new_fe, conversion_method, fallback_reason):
+        with patch('openvino.tools.mo.main.get_default_frontends') as default_fe:
+            default_fe.return_value = get_test_default_frontends()
+            args = base_args_config(use_legacy, use_new_fe)
+            args.extensions = extension
+            args.input_model = "test_model.onnx"
+            prepare_ir(args)
+
+            tm.Telemetry.send_event.assert_any_call('mo', 'conversion_method', conversion_method)
+            if fallback_reason:
+                tm.Telemetry.send_event.assert_any_call('mo', 'fallback_reason', fallback_reason)
+            else:
+                with pytest.raises(AssertionError): # not called
+                    tm.Telemetry.send_event.assert_any_call('mo', 'fallback_reason', fallback_reason)
+
+
+    @generate(*[(True, None, None, 'mo_legacy', 'transformations_config'), # fallback
+                (True, True, None, 'mo_legacy', None),
+                (False, None, True, 'onnx_frontend', None),
+                (False, None, None, 'onnx_frontend', None),
+    ])
+    def test_fallback_if_tranformations_config_specified(self, trans_config_used, use_legacy, use_new_fe, expected_path, fallback_reason):
+        with patch('openvino.tools.mo.main.get_default_frontends') as default_fe:
+            default_fe.return_value = get_test_default_frontends()
+            args = base_args_config(use_legacy, use_new_fe)
+            args.input_model = "test_model.onnx"
+            args.transformations_config = self.trans_config_file if trans_config_used else None
+
+            prepare_ir(args)
+
+            tm.Telemetry.send_event.assert_any_call('mo', 'conversion_method', expected_path)
+            if fallback_reason:
+                tm.Telemetry.send_event.assert_any_call('mo', 'fallback_reason', fallback_reason)
+            else:
+                with pytest.raises(AssertionError): # not called
+                    tm.Telemetry.send_event.assert_any_call('mo', 'fallback_reason', fallback_reason)
+
+
+    @generate(*[('dir_to_extension', True, None, 'mo_legacy', 'extensions, transformations_config'), # fallback
+                (None, True, None, 'mo_legacy', 'transformations_config'), # fallback
+                ('dir_to_extension', False, None, 'mo_legacy', 'extensions'), # fallback
+                (None, False, True, 'onnx_frontend', None),
+    ])
+    def test_fallback_if_both_extension_and_trans_config_specified(self, extension, trans_config_used, use_new_fe, expected_path, fallback_reason):
+        with patch('openvino.tools.mo.main.get_default_frontends') as default_fe:
+            default_fe.return_value = get_test_default_frontends()
+            args = base_args_config(use_new_fe=use_new_fe)
+            args.extensions = extension
+            args.input_model = "test_model.onnx"
+            args.transformations_config = self.trans_config_file if trans_config_used else None
+
+            prepare_ir(args)
+
+            tm.Telemetry.send_event.assert_any_call('mo', 'conversion_method', expected_path)
+            if fallback_reason:
+                tm.Telemetry.send_event.assert_any_call('mo', 'fallback_reason', fallback_reason)
+            else:
+                with pytest.raises(AssertionError): # not called
+                    tm.Telemetry.send_event.assert_any_call('mo', 'fallback_reason', fallback_reason)
+
+
+    @generate(*[(True, None, None, 'mo_legacy'),
+                (True, True, None, 'mo_legacy'),
+                (False, None, True, 'onnx_frontend'),
+    ])
+    def test_fallback_if_legacy_set_as_default(self, trans_config_used, use_legacy, use_new_fe, expected_path):
+        with patch('openvino.tools.mo.main.get_default_frontends') as default_fe:
+            default_fe.return_value = {'onnx': 'legacy', 'tf': 'legacy'}
+            args = base_args_config(use_legacy, use_new_fe)
+            args.input_model = "test_model.onnx"
+            args.transformations_config = self.trans_config_file if trans_config_used else None
+
+            prepare_ir(args)
+
+            tm.Telemetry.send_event.assert_any_call('mo', 'conversion_method', expected_path)
+            with pytest.raises(AssertionError): # not called
+                tm.Telemetry.send_event.assert_any_call('mo', 'fallback_reason')
+
+
+    @generate(*[(None, None, 'dir_to_extension', 'paddle_frontend'),
+                (True, None, None, 'paddle_frontend'),
+                (None, None, None, 'paddle_frontend'),
+    ])
+    def test_no_fallback_if_pdpd(self, use_new_fe, use_legacy, extension, expected_path):
+        args = base_args_config(use_legacy, use_new_fe)
+        args.framework = 'paddle'
+        args.extensions = extension
+        args.input_model = 'paddle_dir/relu/relu.pdmodel'
+
+        prepare_ir(args)
+
+        tm.Telemetry.send_event.assert_any_call('mo', 'conversion_method', expected_path)
+        with pytest.raises(AssertionError): # not called
+            tm.Telemetry.send_event.assert_any_call('mo', 'fallback_reason')