diff --git a/.ci/azure/linux.yml b/.ci/azure/linux.yml index c080b70a80f..4003cb0d818 100644 --- a/.ci/azure/linux.yml +++ b/.ci/azure/linux.yml @@ -524,22 +524,9 @@ jobs: python3 -m pip install -r $(LAYER_TESTS_DIR)/requirements.txt export PYTHONPATH=$(LAYER_TESTS_DIR):$PYTHONPATH export TEST_DEVICE=CPU - $(RUN_PREFIX) python3 -m pytest $(LAYER_TESTS_DIR)/mo_python_api_tests/test_mo_convert_complex_params.py --ir_version=11 --junitxml=./TEST-test_mo_convert_complex_params.xmlTEST - displayName: 'MO Python API Tests - Complex Python params' + $(RUN_PREFIX) python3 -m pytest $(LAYER_TESTS_DIR)/mo_python_api_tests/ --junitxml=./TEST-test_mo_convert.xmlTEST + displayName: 'MO Python API Tests' - - script: | - python3 -m pip install -r $(LAYER_TESTS_DIR)/requirements.txt - export PYTHONPATH=$(LAYER_TESTS_DIR):$PYTHONPATH - export TEST_DEVICE=CPU - $(RUN_PREFIX) python3 -m pytest $(LAYER_TESTS_DIR)/mo_python_api_tests/test_mo_convert_tf.py --ir_version=11 --junitxml=./TEST-test_mo_convert_tf.xmlTEST - displayName: 'MO Python API Tests - Import TF model from memory' - - - script: | - python3 -m pip install -r $(LAYER_TESTS_DIR)/requirements.txt - export PYTHONPATH=$(LAYER_TESTS_DIR):$PYTHONPATH - export TEST_DEVICE=CPU - $(RUN_PREFIX) python3 -m pytest $(LAYER_TESTS_DIR)/mo_python_api_tests/test_mo_convert_pytorch.py --ir_version=11 --junitxml=./TEST-test_mo_convert_pytorch.xmlTEST - displayName: 'MO Python API Tests - Import PyTorch model from memory' - script: | python3 -m pip install -r $(LAYER_TESTS_DIR)/requirements.txt diff --git a/tests/layer_tests/mo_python_api_tests/mo_convert_legacy_extensions_test_actual.py b/tests/layer_tests/mo_python_api_tests/mo_convert_legacy_extensions_test_actual.py new file mode 100644 index 00000000000..5b26a73cc4c --- /dev/null +++ b/tests/layer_tests/mo_python_api_tests/mo_convert_legacy_extensions_test_actual.py @@ -0,0 +1,107 @@ +# Copyright (C) 2018-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import numpy as np +import openvino.runtime as ov +import os +import tempfile +import tensorflow as tf +import unittest + +from openvino.runtime import PartialShape, Model +from openvino.test_utils import compare_functions +from common.utils.common_utils import generate_ir + + +def create_tf_model(): + import tensorflow as tf + + tf.compat.v1.reset_default_graph() + + with tf.compat.v1.Session() as sess: + inp1 = tf.compat.v1.placeholder(tf.float32, [1, 2, 3], 'Input') + inp2 = tf.compat.v1.placeholder(tf.float32, [1, 2, 3], 'Input') + relu = tf.nn.relu(inp1 + inp2, name='Relu') + + output = tf.nn.sigmoid(relu, name='Sigmoid') + + tf.compat.v1.global_variables_initializer() + tf_net = sess.graph_def + return tf_net + + +def create_ref_model_1(): + shape = [1, 2, 3] + shape = PartialShape(shape) + param1 = ov.opset10.parameter(shape) + param2 = ov.opset10.parameter(shape) + add = ov.opset10.add(param1, param2) + relu = ov.opset10.relu(add) + sin = ov.opset10.sin(relu) + sigm = ov.opset10.sigmoid(sin) + parameter_list = [param1, param2] + ref_model = Model([sigm], parameter_list, "test") + return ref_model + + +def create_ref_model_2(): + shape = [1, 2, 3] + shape = PartialShape(shape) + param1 = ov.opset10.parameter(shape) + param2 = ov.opset10.parameter(shape) + add = ov.opset10.add(param1, param2) + relu = ov.opset10.relu(add) + sin = ov.opset10.sin(relu) + sigm = ov.opset10.sigmoid(sin) + tanh = ov.opset10.tanh(sigm) + parameter_list = [param1, param2] + ref_model = Model([tanh], parameter_list, "test") + return ref_model + + +class LegacyExtTest(unittest.TestCase): + test_directory = os.path.dirname(os.path.realpath(__file__)) + def test_legacy_extensions(self): + from openvino.tools.mo import convert_model + with tempfile.TemporaryDirectory(dir=self.test_directory) as tmpdir: + ext_path1 = os.path.join(os.path.dirname(__file__), "test_legacy_exts/test_exts_dir1") + ext_path2 = os.path.join(os.path.dirname(__file__), "test_legacy_exts/test_exts_dir2") + model = create_tf_model() + out_xml = os.path.join(tmpdir, "model.xml") + + # tests for convert_model() + ov_model = convert_model(model, extensions=ext_path1) + flag, msg = compare_functions(ov_model, create_ref_model_1(), False) + assert flag, msg + + ov_model = convert_model(model, extensions=[ext_path1, ext_path2]) + flag, msg = compare_functions(ov_model, create_ref_model_2(), False) + assert flag, msg + + ov_model = convert_model(model, extensions=','.join([ext_path1, ext_path2])) + flag, msg = compare_functions(ov_model, create_ref_model_2(), False) + assert flag, msg + + tf.io.write_graph(model, tmpdir, 'model.pb', False) + inp_model = os.path.join(tmpdir, 'model.pb') + from openvino.runtime import Core + core = Core() + + # tests for MO cli tool + exit_code, stderr = generate_ir(coverage=False, **{"input_model": inp_model, + "extensions": ext_path1, + "output_dir": tmpdir}) + assert not exit_code + + ov_model = core.read_model(os.path.join(tmpdir, "model.xml")) + flag, msg = compare_functions(ov_model, create_ref_model_1(), False) + assert flag, msg + + exit_code, stderr = generate_ir(coverage=False, **{"input_model": inp_model, + "extensions": ','.join([ext_path1, ext_path2]), + "output_dir": tmpdir}) + assert not exit_code + + ov_model = core.read_model(os.path.join(tmpdir, "model.xml")) + flag, msg = compare_functions(ov_model, create_ref_model_2(), False) + assert flag, msg diff --git a/tests/layer_tests/mo_python_api_tests/test_legacy_exts/test_exts_dir1/front/ext1.py b/tests/layer_tests/mo_python_api_tests/test_legacy_exts/test_exts_dir1/front/ext1.py new file mode 100644 index 00000000000..50642ccda0d --- /dev/null +++ b/tests/layer_tests/mo_python_api_tests/test_legacy_exts/test_exts_dir1/front/ext1.py @@ -0,0 +1,15 @@ +# Copyright (C) 2018-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +from openvino.tools.mo.front.common.replacement import FrontReplacementPattern +from openvino.tools.mo.graph.graph import Graph +from openvino.tools.mo.ops.activation_ops import Sin + + +class DummyExt1(FrontReplacementPattern): + enabled = True + + def find_and_replace_pattern(self, graph: Graph): + for node in graph.get_op_nodes(op='ReLU'): + new_node = Sin(graph, {'name': node.soft_get('name') + '/sin'}).create_node() + node.out_port(0).get_connection().insert_node(new_node) diff --git a/tests/layer_tests/mo_python_api_tests/test_legacy_exts/test_exts_dir2/front/ext2.py b/tests/layer_tests/mo_python_api_tests/test_legacy_exts/test_exts_dir2/front/ext2.py new file mode 100644 index 00000000000..366d3ef84a5 --- /dev/null +++ b/tests/layer_tests/mo_python_api_tests/test_legacy_exts/test_exts_dir2/front/ext2.py @@ -0,0 +1,15 @@ +# Copyright (C) 2018-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +from openvino.tools.mo.front.common.replacement import FrontReplacementPattern +from openvino.tools.mo.graph.graph import Graph +from openvino.tools.mo.ops.activation_ops import Tanh + + +class DummyExt2(FrontReplacementPattern): + enabled = True + + def find_and_replace_pattern(self, graph: Graph): + for node in graph.get_op_nodes(op='Sigmoid'): + new_node = Tanh(graph, {'name': node.soft_get('name') + '/tanh'}).create_node() + node.out_port(0).get_connection().insert_node(new_node) diff --git a/tests/layer_tests/mo_python_api_tests/test_mo_convert_extensions.py b/tests/layer_tests/mo_python_api_tests/test_mo_convert_extensions.py index 96b0faba479..fd926c11b3b 100644 --- a/tests/layer_tests/mo_python_api_tests/test_mo_convert_extensions.py +++ b/tests/layer_tests/mo_python_api_tests/test_mo_convert_extensions.py @@ -6,7 +6,6 @@ import numpy as np from common.mo_convert_test_class import CommonMOConvertTest from common.onnx_layer_test_class import save_to_onnx -from unit_tests.utils.graph import build_graph import openvino.runtime as ov from openvino.runtime import PartialShape, Model diff --git a/tests/layer_tests/mo_python_api_tests/test_mo_convert_subprocess_tests.py b/tests/layer_tests/mo_python_api_tests/test_mo_convert_subprocess_tests.py new file mode 100644 index 00000000000..695010cc1dd --- /dev/null +++ b/tests/layer_tests/mo_python_api_tests/test_mo_convert_subprocess_tests.py @@ -0,0 +1,16 @@ +# Copyright (C) 2018-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import os +import subprocess +import sys +import unittest + + +class TestSubprocessMoConvert(unittest.TestCase): + def test_mo_convert(self): + args = [sys.executable, '-m', 'pytest', + os.path.join(os.path.dirname(__file__), 'mo_convert_legacy_extensions_test_actual.py'), '-s'] + + status = subprocess.run(args, env=os.environ) + assert not status.returncode \ No newline at end of file diff --git a/tools/mo/openvino/tools/mo/convert_impl.py b/tools/mo/openvino/tools/mo/convert_impl.py index a0af57459ad..30942457492 100644 --- a/tools/mo/openvino/tools/mo/convert_impl.py +++ b/tools/mo/openvino/tools/mo/convert_impl.py @@ -390,12 +390,6 @@ def prepare_ir(argv: argparse.Namespace): # TODO: remove this workaround once new TensorFlow frontend supports non-frozen formats: checkpoint, MetaGraph, and SavedModel # Now it converts all TensorFlow formats to the frozen .pb format in case new TensorFlow frontend is_tf, _, _, _, _ = deduce_legacy_frontend_by_namespace(argv) - path_to_aux_pb = None - orig_argv_values = {"input_model": argv.input_model, "model_name": argv.model_name} - if not argv.use_legacy_frontend and is_tf: - from openvino.tools.mo.front.tf.loader import convert_to_pb - path_to_aux_pb = convert_to_pb(argv) - argv = arguments_post_parsing(argv) t = tm.Telemetry() graph = None @@ -405,6 +399,11 @@ def prepare_ir(argv: argparse.Namespace): if moc_front_end: fallback_reasons = check_fallback(argv) if len(fallback_reasons) == 0: + path_to_aux_pb = None + orig_argv_values = {"input_model": argv.input_model, "model_name": argv.model_name} + if not argv.use_legacy_frontend and is_tf: + from openvino.tools.mo.front.tf.loader import convert_to_pb + path_to_aux_pb = convert_to_pb(argv) try: t.send_event("mo", "conversion_method", moc_front_end.get_name() + "_frontend") moc_front_end.add_extension(TelemetryExtension("mo", t.send_event, t.send_error, t.send_stack_trace)) diff --git a/tools/mo/openvino/tools/mo/front/tf/loader.py b/tools/mo/openvino/tools/mo/front/tf/loader.py index 6aded155a41..611742cef60 100644 --- a/tools/mo/openvino/tools/mo/front/tf/loader.py +++ b/tools/mo/openvino/tools/mo/front/tf/loader.py @@ -332,7 +332,9 @@ def convert_to_pb(argv: argparse.Namespace): isinstance(argv.input_model, str): return None - user_output_node_names_list = argv.output.split(',') if argv.output else None + user_output_node_names_list = argv.output if argv.output else None + if user_output_node_names_list is not None and not isinstance(user_output_node_names_list, list): + user_output_node_names_list = user_output_node_names_list.split(',') graph_def, _, _, _ = load_tf_graph_def( graph_file_name=argv.input_model, is_binary=not argv.input_model_is_text, diff --git a/tools/mo/openvino/tools/mo/utils/cli_parser.py b/tools/mo/openvino/tools/mo/utils/cli_parser.py index 776022a98ac..836c1a5f003 100644 --- a/tools/mo/openvino/tools/mo/utils/cli_parser.py +++ b/tools/mo/openvino/tools/mo/utils/cli_parser.py @@ -812,6 +812,19 @@ class CanonicalizePathCheckExistenceAction(argparse.Action): setattr(namespace, self.dest, ','.join(list_of_paths)) +class CanonicalizeExtensionsPathCheckExistenceAction(argparse.Action): + """ + Expand user home directory paths and convert relative-paths to absolute and check specified file or directory + existence. + """ + + def __call__(self, parser, namespace, values, option_string=None): + list_of_paths = canonicalize_and_check_paths(values, param_name=option_string, + try_mo_root=False, check_existance=True) + # Extensions paths are needed to be stored as list + setattr(namespace, self.dest, list_of_paths) + + class CanonicalizePathCheckExistenceIfNeededAction(CanonicalizePathCheckExistenceAction): def __call__(self, parser, namespace, values, option_string=None): @@ -1036,7 +1049,7 @@ def get_common_cli_parser(parser: argparse.ArgumentParser = None): help=mo_convert_params_common['extensions'].description.format( mo_convert_params_common['extensions'].possible_types_command_line), default=[import_extensions.default_path()], - action=CanonicalizePathCheckExistenceAction, + action=CanonicalizeExtensionsPathCheckExistenceAction, type=readable_dirs_or_files_or_empty) common_group.add_argument("--batch", "-b", type=check_positive,