diff --git a/tools/mo/openvino/tools/mo/convert_impl.py b/tools/mo/openvino/tools/mo/convert_impl.py index 5366a29d3cb..ab478acbb00 100644 --- a/tools/mo/openvino/tools/mo/convert_impl.py +++ b/tools/mo/openvino/tools/mo/convert_impl.py @@ -47,7 +47,7 @@ from openvino.tools.mo.utils.logger import init_logger, progress_printer from openvino.tools.mo.utils.utils import refer_to_faq_msg from openvino.tools.mo.utils.telemetry_utils import send_params_info, send_framework_info, send_conversion_result, \ get_tid -from openvino.tools.mo.utils.versions_checker import check_requirements, get_environment_setup # pylint: disable=no-name-in-module +from openvino.tools.mo.utils.versions_checker import get_environment_setup # pylint: disable=no-name-in-module from openvino.tools.mo.moc_frontend.check_config import legacy_extensions_used from openvino.tools.mo.moc_frontend.pytorch_frontend_utils import get_pytorch_decoder, convert_pytorch_via_onnx from openvino.tools.mo.moc_frontend.shape_utils import parse_input_shapes, get_static_shape @@ -216,14 +216,6 @@ def arguments_post_parsing(argv: argparse.Namespace): # This is just to check that transform key is valid and transformations are available check_available_transforms(parse_transform(argv.transform)) - # For C++ frontends there are no specific Python installation requirements, check only generic ones - if moc_front_end: - ret_code = check_requirements(silent=argv.silent) - else: - ret_code = check_requirements(framework=argv.framework, silent=argv.silent) - if ret_code: - raise Error('check_requirements exited with return code {}'.format(ret_code)) - if argv.scale and argv.scale_values: raise Error( 'Both --scale and --scale_values are defined. Specify either scale factor or scale values per input ' + diff --git a/tools/mo/openvino/tools/mo/utils/versions_checker.py b/tools/mo/openvino/tools/mo/utils/versions_checker.py index c12c7559e98..df7b21534e9 100644 --- a/tools/mo/openvino/tools/mo/utils/versions_checker.py +++ b/tools/mo/openvino/tools/mo/utils/versions_checker.py @@ -1,27 +1,12 @@ # Copyright (C) 2018-2023 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -import logging as log import os -import re import sys -from distutils.version import LooseVersion -from pathlib import Path # do not print INFO and WARNING messages from TensorFlow os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' -modules = { - "protobuf": "google.protobuf", - "test-generator": "generator", - "openvino-telemetry": "openvino_telemetry" -} -critical_modules = ["networkx", "defusedxml", "numpy"] - -message = "\nDetected not satisfied dependencies:\n" \ - "{}\n" \ - "Please install required versions of components or run pip installation\n{}" - def get_imported_module_version(imported_module): """ @@ -43,172 +28,6 @@ def get_imported_module_version(imported_module): return installed_version -def parse_and_filter_versions_list(required_fw_versions, version_list, env_setup): - """ - Please do not add parameter type annotations (param:type). - Because we import this file while checking Python version. - Python 2.x will fail with no clear message on type annotations. - - Parsing requirements versions for a dependency and filtering out requirements that - satisfy environment setup such as python version. - if environment version (python_version, etc.) is satisfied - :param required_fw_versions: String with fw versions from requirements file - :param version_list: List for append - :param env_setup: a dictionary with environment setup - :return: list of tuples of strings like (name_of_module, sign, version) - - Examples of required_fw_versions: - 'tensorflow>=1.15.2,<2.0; python_version < "3.8"' - 'tensorflow>=2.0' - - Returned object is: - [('tensorflow', '>=', '1.2.0'), ('networkx', '==', '2.1'), ('numpy', None, None)] - """ - - line = required_fw_versions.strip('\n') - line = line.strip(' ') - if line == '': - return version_list - split_requirement = line.split(";") - - # check environment marker - if len(split_requirement) > 1: - env_req = split_requirement[1] - if any([x in split_requirement[1] for x in [' and ', ' or ']]): - log.error("The version checker doesn't support environment marker combination and it will be ignored: {}" - "".format(split_requirement[1]), extra={'is_warning': True}) - return version_list - split_env_req = re.split(r"==|>=|<=|>|<|~=|!=", env_req) - split_env_req = [l.strip(',') for l in split_env_req] - env_marker = split_env_req[0].strip(' ') - if env_marker == 'python_version' and env_marker in env_setup: - installed_python_version = env_setup['python_version'] - env_req_version_list = [] - split_required_versions = re.split(r",", env_req) - for i, l in enumerate(split_required_versions): - for comparison in ['==', '>=', '<=', '<', '>', '~=']: - if comparison in l: - required_version = split_env_req[i + 1].strip(' ').replace("'", "").replace('"', '') - env_req_version_list.append((env_marker, comparison, required_version)) - break - not_satisfied_list = [] - for name, key, required_version in env_req_version_list: - version_check(name, installed_python_version, required_version, - key, not_satisfied_list) - if len(not_satisfied_list) > 0: - # this python_version requirement is not satisfied to required environment - # and requirement for a dependency will be skipped - return version_list - elif env_marker == 'sys_platform' and env_marker in env_setup: - split_env_req[1] = split_env_req[1].strip(' ').replace("'", "").replace('"', '') - if '==' in env_req: - if env_setup['sys_platform'] != split_env_req[1]: - # this sys_platform requirement is not satisfied to required environment - # and requirement for a dependency will be skipped - return version_list - elif '!=' in env_req: - if env_setup['sys_platform'] == split_env_req[1]: - # this sys_platform requirement is not satisfied to required environment - # and requirement for a dependency will be skipped - return version_list - else: - log.error("Error during platform version check, line: {}".format(line)) - else: - log.error("{} is unsupported environment marker and it will be ignored".format(env_marker), - extra={'is_warning': True}) - - # parse a requirement for a dependency - requirement = split_requirement[0] - split_versions_by_conditions = re.split(r"==|>=|<=|>|<|~=", requirement) - split_versions_by_conditions = [l.strip(',').strip(' ') for l in split_versions_by_conditions] - - if len(split_versions_by_conditions) == 0: - return version_list - if len(split_versions_by_conditions) == 1: - version_list.append((split_versions_by_conditions[0], None, None)) - else: - split_required_versions= re.split(r",", requirement) - for i, l in enumerate(split_required_versions): - for comparison in ['==', '>=', '<=', '<', '>', '~=']: - if comparison in l: - version_list.append((split_versions_by_conditions[0], comparison, split_versions_by_conditions[i + 1])) - break - return version_list - - -def get_module_version_list_from_file(file_name, env_setup): - """ - Please do not add parameter type annotations (param:type). - Because we import this file while checking Python version. - Python 2.x will fail with no clear message on type annotations. - - Reads file with requirements - :param file_name: Name of the requirements file - :param env_setup: a dictionary with environment setup elements - :return: list of tuples of strings like (name_of_module, sign, version) - - File content example: - tensorflow>=1.2.0 - networkx==2.1 - numpy - - Returned object is: - [('tensorflow', '>=', '1.2.0'), ('networkx', '==', '2.1'), ('numpy', None, None)] - """ - req_dict = [] - with open(file_name) as f: - for line in f: - # handle comments - line = line.split('#')[0] - - req_dict = parse_and_filter_versions_list(line, req_dict, env_setup) - return req_dict - - -def version_check(name, installed_v, required_v, sign, not_satisfied_v): - """ - Please do not add parameter type annotations (param:type). - Because we import this file while checking Python version. - Python 2.x will fail with no clear message on type annotations. - - Evaluates comparison of installed and required versions according to requirements file of one module. - If installed version does not satisfy requirements appends this module to not_satisfied_v list. - :param name: module name - :param installed_v: installed version of module - :param required_v: required version of module - :param sign: sing for comparison of required and installed versions - :param not_satisfied_v: list of modules with not satisfying versions - """ - if sign is not None: - req_ver = LooseVersion(required_v) - satisfied = False - if sign == '>': - satisfied = installed_v > req_ver - elif sign == '>=': - satisfied = installed_v >= req_ver - elif sign == '<=': - satisfied = installed_v <= req_ver - elif sign == '<': - satisfied = installed_v < req_ver - elif sign == '==': - satisfied = installed_v == req_ver - elif sign == '~=': - req_ver_list = req_ver.vstring.split('.') - if 'post' in req_ver_list[-1]: - assert len(req_ver_list) >= 3, 'Error during {} module version checking: {} {} {}, please check ' \ - 'required version of this module in requirements_*.txt file!'\ - .format(name, installed_v, sign, required_v) - req_ver_list.pop(-1) - idx = len(req_ver_list) - 1 - satisfied = installed_v >= req_ver and (installed_v.split('.')[:idx] == req_ver_list[:idx]) - else: - log.error("Error during version comparison") - else: - satisfied = True - if not satisfied: - not_satisfied_v.append((name, 'installed: {}'.format(installed_v), 'required: {} {}'.format(sign, required_v))) - - def get_environment_setup(framework): """ Get environment setup such as Python version, TensorFlow version @@ -229,87 +48,3 @@ def get_environment_setup(framework): pass env_setup['sys_platform'] = sys.platform return env_setup - - -def check_requirements(framework=None, silent=True): - """ - Please do not add parameter type annotations (param:type). - Because we import this file while checking Python version. - Python 2.x will fail with no clear message on type annotations. - - Checks if installed modules versions satisfy required versions in requirements file - Logs a warning in case of permissible dissatisfaction - Logs an error in cases of critical dissatisfaction - :param framework: framework name - :param silent: determines if it is required to print warning messages - :return: exit code (0 - execution successful, 1 - error) - """ - framework_suffix = "_{}".format(framework) - env_setup = get_environment_setup(framework) - if framework is None: - framework_suffix = "" - elif framework == "tf": - if "tensorflow" in env_setup and env_setup["tensorflow"] < LooseVersion("2.0.0"): - if not silent: - log.error('\t\nSupport of the Model Optimizer tool in TensorFlow 1.x environment is deprecated.' - 'It is highly recommended to use TensorFlow 2.x.\n', - extra={'is_warning': True}) - - file_name = "requirements{}.txt".format(framework_suffix) - - # if MO was run from developer clone requirements are 4 levels out from mo.py - requirements_file = os.path.realpath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, os.pardir, file_name)) - if not Path(requirements_file).is_file(): - # if MO was run from site-packages requirements are at the same level with mo.py file - # in site-packages/openvino/tools/mo/requirements.txt - requirements_file = os.path.realpath(os.path.join(os.path.dirname(__file__), os.pardir, file_name)) - - requirements_list = get_module_version_list_from_file(requirements_file, env_setup) - not_satisfied_versions = [] - exit_code = 0 - for name, key, required_version in requirements_list: - try: - importable_name = modules.get(name, name) - exec("import {}".format(importable_name)) - installed_version = get_imported_module_version(sys.modules[importable_name]) - version_check(name, installed_version, required_version, key, not_satisfied_versions) - exec("del {}".format(importable_name)) - except (AttributeError, ImportError): - # we need to raise error only in cases when import of critical modules is failed - if name in critical_modules: - exit_code = 1 - if key is not None and required_version is not None: - not_satisfied_versions.append((name, 'not installed', 'required: {} {}'.format(key, required_version))) - else: - not_satisfied_versions.append((name, 'not installed', '')) - continue - except Exception as e: - if not silent: - log.error('Error happened while importing {} module. It may happen due to unsatisfied requirements of ' - 'that module. Please run requirements installation script once more.\n' - 'Details on module importing failure: {}'.format(name, e)) - not_satisfied_versions.append((name, 'package error', 'required: {} {}'.format(key, required_version))) - continue - - if len(not_satisfied_versions) != 0: - - if framework == 'tf': - extra = '[tensorflow]' - elif framework == 'tf2': - extra = '[tensorflow2]' - elif framework is None: - extra = '' - else: - extra = '[{}]'.format(framework) - helper_command = "pip install openvino-dev{}".format(extra) - - missed_modules_message = "" - for module in not_satisfied_versions: - missed_modules_message += "\t{}: {}, {}\n".format(module[0], module[1], module[2]) - if exit_code: - if not silent: - log.error(message.format(missed_modules_message, helper_command)) - else: - if not silent: - log.error(message.format(missed_modules_message, helper_command), extra={'is_warning': True}) - return exit_code diff --git a/tools/mo/unit_tests/mo/utils/versions_checker_test.py b/tools/mo/unit_tests/mo/utils/versions_checker_test.py deleted file mode 100644 index 2733f3050de..00000000000 --- a/tools/mo/unit_tests/mo/utils/versions_checker_test.py +++ /dev/null @@ -1,256 +0,0 @@ -# Copyright (C) 2018-2023 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -import unittest -import unittest.mock as mock -from unittest.mock import mock_open - -from openvino.tools.mo.utils.versions_checker import get_module_version_list_from_file, parse_and_filter_versions_list, version_check - - -class TestingVersionsChecker(unittest.TestCase): - @mock.patch('builtins.open', new_callable=mock_open, create=True) - def test_get_module_version_list_from_file(self, mock_open): - mock_open.return_value.__enter__ = mock_open - mock_open.return_value.__iter__ = mock.Mock( - return_value=iter(['mxnet>=1.0.0,<=1.3.1', - 'networkx>=1.11', - 'numpy==1.12.0', - 'defusedxml<=0.5.0', - 'networkx~=1.11'])) - ref_list = [('mxnet', '>=', '1.0.0'), - ('mxnet', '<=', '1.3.1'), - ('networkx', '>=', '1.11'), - ('numpy', '==', '1.12.0'), - ('defusedxml', '<=', '0.5.0'), - ('networkx', '~=', '1.11')] - version_list = get_module_version_list_from_file('mock_file', {}) - self.assertEqual(len(version_list), 6) - for i, version_dict in enumerate(version_list): - self.assertTupleEqual(ref_list[i], version_dict) - - @mock.patch('builtins.open', new_callable=mock_open, create=True) - def test_get_module_version_list_from_file2(self, mock_open): - mock_open.return_value.__enter__ = mock_open - mock_open.return_value.__iter__ = mock.Mock( - return_value=iter(['tensorflow>=1.15.2,<2.0; python_version < "3.8"', - 'tensorflow>=2.0; python_version >= "3.8"', - 'numpy==1.12.0', - 'defusedxml<=0.5.0', - 'networkx~=1.11'])) - ref_list = [('tensorflow', '>=', '1.15.2'), - ('tensorflow', '<', '2.0'), - ('numpy', '==', '1.12.0'), - ('defusedxml', '<=', '0.5.0'), - ('networkx', '~=', '1.11')] - version_list = get_module_version_list_from_file('mock_file', {'python_version': '3.7.0'}) - self.assertEqual(len(version_list), 5) - for i, version_dict in enumerate(version_list): - self.assertTupleEqual(ref_list[i], version_dict) - - @mock.patch('builtins.open', new_callable=mock_open, create=True) - def test_get_module_version_list_from_file3(self, mock_open): - mock_open.return_value.__enter__ = mock_open - mock_open.return_value.__iter__ = mock.Mock( - return_value=iter(['# Commented line', - 'tensorflow>=1.15.2,<2.0; python_version < "3.8"', - 'tensorflow>=2.0; python_version >= "3.8" # Comment after line', - 'numpy==1.12.0', - 'defusedxml<=0.5.0', - 'networkx~=1.11'])) - ref_list = [('tensorflow', '>=', '2.0'), - ('numpy', '==', '1.12.0'), - ('defusedxml', '<=', '0.5.0'), - ('networkx', '~=', '1.11')] - version_list = get_module_version_list_from_file('mock_file', {'python_version': '3.8.1'}) - self.assertEqual(len(version_list), 4) - for i, version_dict in enumerate(version_list): - self.assertTupleEqual(ref_list[i], version_dict) - - @mock.patch('builtins.open', new_callable=mock_open, create=True) - def test_get_module_version_list_from_file_with_fw_name(self, mock_open): - mock_open.return_value.__enter__ = mock_open - mock_open.return_value.__iter__ = mock.Mock( - return_value=iter(['mxnet'])) - ref_list = [('mxnet', None, None)] - version_list = get_module_version_list_from_file('mock_file', {}) - self.assertEqual(len(version_list), 1) - for i, version_dict in enumerate(version_list): - self.assertTupleEqual(ref_list[i], version_dict) - - def test_append_version_list(self): - v1 = 'mxnet>=1.0.0,<=1.3.1' - req_list = [] - parse_and_filter_versions_list(v1, req_list, {}) - ref_list = [('mxnet', '>=', '1.0.0'), - ('mxnet', '<=', '1.3.1')] - for i, v in enumerate(ref_list): - self.assertEqual(v, req_list[i]) - - def test_append_version_list_sys_neg_1(self): - v1 = "mxnet>=1.7.0 ; sys_platform != 'win32'" - req_list = [] - parse_and_filter_versions_list(v1, req_list, {'sys_platform': 'darwin'}) - ref_list = [('mxnet', '>=', '1.7.0')] - for i, v in enumerate(ref_list): - self.assertEqual(v, req_list[i]) - - def test_append_version_list_sys_neg_2(self): - v1 = "mxnet>=1.7.0 ; sys_platform != 'win32'" - req_list = [] - parse_and_filter_versions_list(v1, req_list, {'sys_platform': 'win32'}) - ref_list = [] - for i, v in enumerate(ref_list): - self.assertEqual(v, req_list[i]) - - def test_append_version_list_sys(self): - v1 = "mxnet>=1.7.0 ; sys_platform == 'linux'" - req_list = [] - - parse_and_filter_versions_list(v1, req_list, {'sys_platform': 'linux'}) - ref_list = [('mxnet', '>=', '1.7.0')] - for i, v in enumerate(ref_list): - self.assertEqual(v, req_list[i]) - - def test_append_version_list_sys_double_quotes(self): - v1 = "mxnet>=1.7.0 ; sys_platform == \"linux\"" - req_list = [] - - parse_and_filter_versions_list(v1, req_list, {'sys_platform': 'linux'}) - ref_list = [('mxnet', '>=', '1.7.0')] - for i, v in enumerate(ref_list): - self.assertEqual(v, req_list[i]) - - def test_append_version_list_py_ver_single_quotes(self): - v1 = "mxnet>=1.7.0 ; python_version < '3.8'" - req_list = [] - - parse_and_filter_versions_list(v1, req_list, {'python_version': '3.7.1'}) - ref_list = [('mxnet', '>=', '1.7.0')] - for i, v in enumerate(ref_list): - self.assertEqual(v, req_list[i]) - - def test_append_version_list_sys_python_ver_1(self): - v1 = "mxnet>=1.7.0 ; sys_platform == 'linux' or python_version >= \"3.8\"" - req_list = [] - parse_and_filter_versions_list(v1, req_list, {'python_version': '3.8.1', 'sys_platform': 'linux'}) - ref_list = [] - for i, v in enumerate(ref_list): - self.assertEqual(v, req_list[i]) - - def test_append_version_list_sys_python_ver_2(self): - v1 = "mxnet>=1.7.0 ; sys_platform == 'linux' and python_version >= \"3.8\"" - req_list = [] - parse_and_filter_versions_list(v1, req_list, {'python_version': '3.7.1', 'sys_platform': 'linux'}) - ref_list = [] - for i, v in enumerate(ref_list): - self.assertEqual(v, req_list[i]) - - def test_version_check_equal(self): - modules_versions_list = [('module_1', '==', '2.0', '2.0'), - ('module_2', '==', '2.0', '2.0.1'), - ] - - ref_list = [('module_2', 'installed: 2.0.1', 'required: == 2.0'), - ] - - not_satisfied_versions = [] - - for name, key, required_version, installed_version in modules_versions_list: - version_check(name, installed_version, required_version, key, not_satisfied_versions) - self.assertEqual(not_satisfied_versions, ref_list) - - def test_version_check_less_equal(self): - modules_versions_list = [('module_1', '>=', '1.12.0', '1.09.2'), - ('module_2', '>=', '1.12.0', '1.12.0'), - ('module_3', '>=', '1.12.0', '1.12.1'), - ('module_4', '>=', '1.12.0', '1.20.0'), - ] - - ref_list = [('module_1', 'installed: 1.09.2', 'required: >= 1.12.0'), - ] - - not_satisfied_versions = [] - - for name, key, required_version, installed_version in modules_versions_list: - version_check(name, installed_version, required_version, key, not_satisfied_versions) - self.assertEqual(not_satisfied_versions, ref_list) - - def test_version_check_greater_equal(self): - modules_versions_list = [('module_1', '>=', '1.12.0', '1.09.2'), - ('module_2', '>=', '1.12.0', '1.12.0'), - ('module_3', '>=', '1.12.0', '1.12.1'), - ('module_4', '>=', '1.12.0', '1.20.0'), - ('module_5', '>=', '1.12.0.post2', '1.12.0.post1'), - ] - - ref_list = [('module_1', 'installed: 1.09.2', 'required: >= 1.12.0'), - ('module_5', 'installed: 1.12.0.post1', 'required: >= 1.12.0.post2') - ] - - not_satisfied_versions = [] - - for name, key, required_version, installed_version in modules_versions_list: - version_check(name, installed_version, required_version, key, not_satisfied_versions) - self.assertEqual(not_satisfied_versions, ref_list) - - def test_version_check_less(self): - modules_versions_list = [('module_1', '<', '1.11', '1.01'), - ('module_2', '<', '1.11', '1.10.1'), - ('module_3', '<', '1.11', '1.11'), - ('module_4', '<', '1.11', '1.20'), - ('module_5', '<', '1.11', '1.10.post2'), - ] - - ref_list = [('module_3', 'installed: 1.11', 'required: < 1.11'), - ('module_4', 'installed: 1.20', 'required: < 1.11'), - ] - - not_satisfied_versions = [] - - for name, key, required_version, installed_version in modules_versions_list: - version_check(name, installed_version, required_version, key, not_satisfied_versions) - self.assertEqual(not_satisfied_versions, ref_list) - - def test_version_check_greater(self): - modules_versions_list = [('module_1', '>', '1.11', '1.01'), - ('module_2', '>', '1.11', '1.11'), - ('module_3', '>', '1.11', '1.11.1'), - ('module_4', '>', '1.11', '1.20'), - ] - - ref_list = [('module_1', 'installed: 1.01', 'required: > 1.11'), - ('module_2', 'installed: 1.11', 'required: > 1.11'), - ] - - not_satisfied_versions = [] - - for name, key, required_version, installed_version in modules_versions_list: - version_check(name, installed_version, required_version, key, not_satisfied_versions) - self.assertEqual(not_satisfied_versions, ref_list) - - def test_version_check_compatible(self): - modules_versions_list = [('module_1', '~=', '1.2.3', '1.2.3'), - ('module_2', '~=', '1.2.3', '1.2.3b4'), - ('module_3', '~=', '1.2.3', '1.2.4'), - ('module_4', '~=', '1.2.3', '1.3.0'), - ('module_5', '~=', '1.2.3', '1.2.2'), - ('module_6', '~=', '1.2.3', '2.2.2'), - ('module_7', '~=', '1.2.post2', '1.3'), - ('module_8', '~=', '1.2.post2', '1.2'), - ('module_9', '~=', '1.2.post2', '1.2.post1'), - ('module_9', '~=', '1.2.post2', '1.2.post4') - ] - - ref_list = [('module_4', 'installed: 1.3.0', 'required: ~= 1.2.3'), - ('module_5', 'installed: 1.2.2', 'required: ~= 1.2.3'), - ('module_6', 'installed: 2.2.2', 'required: ~= 1.2.3'), - ('module_8', 'installed: 1.2', 'required: ~= 1.2.post2'), - ('module_9', 'installed: 1.2.post1', 'required: ~= 1.2.post2') - ] - - not_satisfied_versions = [] - - for name, key, required_version, installed_version in modules_versions_list: - version_check(name, installed_version, required_version, key, not_satisfied_versions) - self.assertEqual(not_satisfied_versions, ref_list)