Removed checks of unsatisfied dependencies in MO (#16991)

* Fixed dependencies check, made unsatisfied dependencies show only in case of error.

* Small fix.

* Test correction.

* Small test correction.

* Temporarily added debug print.

* Debug output.

* Debug output.

* Debug output.

* Test fix.

* Removed debug output.

* Small fix.

* Moved tests to check_info_messages_test.py

* Remove dependies checks from MO.

* Small corrections.
This commit is contained in:
Anastasiia Pnevskaia authored and GitHub committed 2023-05-08 16:54:21 +04:00
1 parent 585c33dfa6
commit fd9d501306
3 files changed
+1 -530

No files matched your search

+1 -9
View File
@@ -47,7 +47,7 @@ from openvino.tools.mo.utils.logger import init_logger, progress_printer
from openvino.tools.mo.utils.utils import refer_to_faq_msg
from openvino.tools.mo.utils.telemetry_utils import send_params_info, send_framework_info, send_conversion_result, \
get_tid
from openvino.tools.mo.utils.versions_checker import check_requirements, get_environment_setup # pylint: disable=no-name-in-module
from openvino.tools.mo.utils.versions_checker import get_environment_setup # pylint: disable=no-name-in-module
from openvino.tools.mo.moc_frontend.check_config import legacy_extensions_used
from openvino.tools.mo.moc_frontend.pytorch_frontend_utils import get_pytorch_decoder, convert_pytorch_via_onnx
from openvino.tools.mo.moc_frontend.shape_utils import parse_input_shapes, get_static_shape
@@ -216,14 +216,6 @@ def arguments_post_parsing(argv: argparse.Namespace):
# This is just to check that transform key is valid and transformations are available
check_available_transforms(parse_transform(argv.transform))
# For C++ frontends there are no specific Python installation requirements, check only generic ones
if moc_front_end:
ret_code = check_requirements(silent=argv.silent)
else:
ret_code = check_requirements(framework=argv.framework, silent=argv.silent)
if ret_code:
raise Error('check_requirements exited with return code {}'.format(ret_code))
if argv.scale and argv.scale_values:
raise Error(
'Both --scale and --scale_values are defined. Specify either scale factor or scale values per input ' +
@@ -1,27 +1,12 @@
# Copyright (C) 2018-2023 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
import logging as log
import os
import re
import sys
from distutils.version import LooseVersion
from pathlib import Path
# do not print INFO and WARNING messages from TensorFlow
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
modules = {
"protobuf": "google.protobuf",
"test-generator": "generator",
"openvino-telemetry": "openvino_telemetry"
}
critical_modules = ["networkx", "defusedxml", "numpy"]
message = "\nDetected not satisfied dependencies:\n" \
"{}\n" \
"Please install required versions of components or run pip installation\n{}"
def get_imported_module_version(imported_module):
"""
@@ -43,172 +28,6 @@ def get_imported_module_version(imported_module):
return installed_version
def parse_and_filter_versions_list(required_fw_versions, version_list, env_setup):
"""
Please do not add parameter type annotations (param:type).
Because we import this file while checking Python version.
Python 2.x will fail with no clear message on type annotations.
Parsing requirements versions for a dependency and filtering out requirements that
satisfy environment setup such as python version.
if environment version (python_version, etc.) is satisfied
:param required_fw_versions: String with fw versions from requirements file
:param version_list: List for append
:param env_setup: a dictionary with environment setup
:return: list of tuples of strings like (name_of_module, sign, version)
Examples of required_fw_versions:
'tensorflow>=1.15.2,<2.0; python_version < "3.8"'
'tensorflow>=2.0'
Returned object is:
[('tensorflow', '>=', '1.2.0'), ('networkx', '==', '2.1'), ('numpy', None, None)]
"""
line = required_fw_versions.strip('\n')
line = line.strip(' ')
if line == '':
return version_list
split_requirement = line.split(";")
# check environment marker
if len(split_requirement) > 1:
env_req = split_requirement[1]
if any([x in split_requirement[1] for x in [' and ', ' or ']]):
log.error("The version checker doesn't support environment marker combination and it will be ignored: {}"
"".format(split_requirement[1]), extra={'is_warning': True})
return version_list
split_env_req = re.split(r"==|>=|<=|>|<|~=|!=", env_req)
split_env_req = [l.strip(',') for l in split_env_req]
env_marker = split_env_req[0].strip(' ')
if env_marker == 'python_version' and env_marker in env_setup:
installed_python_version = env_setup['python_version']
env_req_version_list = []
split_required_versions = re.split(r",", env_req)
for i, l in enumerate(split_required_versions):
for comparison in ['==', '>=', '<=', '<', '>', '~=']:
if comparison in l:
required_version = split_env_req[i + 1].strip(' ').replace("'", "").replace('"', '')
env_req_version_list.append((env_marker, comparison, required_version))
break
not_satisfied_list = []
for name, key, required_version in env_req_version_list:
version_check(name, installed_python_version, required_version,
key, not_satisfied_list)
if len(not_satisfied_list) > 0:
# this python_version requirement is not satisfied to required environment
# and requirement for a dependency will be skipped
return version_list
elif env_marker == 'sys_platform' and env_marker in env_setup:
split_env_req[1] = split_env_req[1].strip(' ').replace("'", "").replace('"', '')
if '==' in env_req:
if env_setup['sys_platform'] != split_env_req[1]:
# this sys_platform requirement is not satisfied to required environment
# and requirement for a dependency will be skipped
return version_list
elif '!=' in env_req:
if env_setup['sys_platform'] == split_env_req[1]:
# this sys_platform requirement is not satisfied to required environment
# and requirement for a dependency will be skipped
return version_list
else:
log.error("Error during platform version check, line: {}".format(line))
else:
log.error("{} is unsupported environment marker and it will be ignored".format(env_marker),
extra={'is_warning': True})
# parse a requirement for a dependency
requirement = split_requirement[0]
split_versions_by_conditions = re.split(r"==|>=|<=|>|<|~=", requirement)
split_versions_by_conditions = [l.strip(',').strip(' ') for l in split_versions_by_conditions]
if len(split_versions_by_conditions) == 0:
return version_list
if len(split_versions_by_conditions) == 1:
version_list.append((split_versions_by_conditions[0], None, None))
else:
split_required_versions= re.split(r",", requirement)
for i, l in enumerate(split_required_versions):
for comparison in ['==', '>=', '<=', '<', '>', '~=']:
if comparison in l:
version_list.append((split_versions_by_conditions[0], comparison, split_versions_by_conditions[i + 1]))
break
return version_list
def get_module_version_list_from_file(file_name, env_setup):
"""
Please do not add parameter type annotations (param:type).
Because we import this file while checking Python version.
Python 2.x will fail with no clear message on type annotations.
Reads file with requirements
:param file_name: Name of the requirements file
:param env_setup: a dictionary with environment setup elements
:return: list of tuples of strings like (name_of_module, sign, version)
File content example:
tensorflow>=1.2.0
networkx==2.1
numpy
Returned object is:
[('tensorflow', '>=', '1.2.0'), ('networkx', '==', '2.1'), ('numpy', None, None)]
"""
req_dict = []
with open(file_name) as f:
for line in f:
# handle comments
line = line.split('#')[0]
req_dict = parse_and_filter_versions_list(line, req_dict, env_setup)
return req_dict
def version_check(name, installed_v, required_v, sign, not_satisfied_v):
"""
Please do not add parameter type annotations (param:type).
Because we import this file while checking Python version.
Python 2.x will fail with no clear message on type annotations.
Evaluates comparison of installed and required versions according to requirements file of one module.
If installed version does not satisfy requirements appends this module to not_satisfied_v list.
:param name: module name
:param installed_v: installed version of module
:param required_v: required version of module
:param sign: sing for comparison of required and installed versions
:param not_satisfied_v: list of modules with not satisfying versions
"""
if sign is not None:
req_ver = LooseVersion(required_v)
satisfied = False
if sign == '>':
satisfied = installed_v > req_ver
elif sign == '>=':
satisfied = installed_v >= req_ver
elif sign == '<=':
satisfied = installed_v <= req_ver
elif sign == '<':
satisfied = installed_v < req_ver
elif sign == '==':
satisfied = installed_v == req_ver
elif sign == '~=':
req_ver_list = req_ver.vstring.split('.')
if 'post' in req_ver_list[-1]:
assert len(req_ver_list) >= 3, 'Error during {} module version checking: {} {} {}, please check ' \
'required version of this module in requirements_*.txt file!'\
.format(name, installed_v, sign, required_v)
req_ver_list.pop(-1)
idx = len(req_ver_list) - 1
satisfied = installed_v >= req_ver and (installed_v.split('.')[:idx] == req_ver_list[:idx])
else:
log.error("Error during version comparison")
else:
satisfied = True
if not satisfied:
not_satisfied_v.append((name, 'installed: {}'.format(installed_v), 'required: {} {}'.format(sign, required_v)))
def get_environment_setup(framework):
"""
Get environment setup such as Python version, TensorFlow version
@@ -229,87 +48,3 @@ def get_environment_setup(framework):
pass
env_setup['sys_platform'] = sys.platform
return env_setup
def check_requirements(framework=None, silent=True):
"""
Please do not add parameter type annotations (param:type).
Because we import this file while checking Python version.
Python 2.x will fail with no clear message on type annotations.
Checks if installed modules versions satisfy required versions in requirements file
Logs a warning in case of permissible dissatisfaction
Logs an error in cases of critical dissatisfaction
:param framework: framework name
:param silent: determines if it is required to print warning messages
:return: exit code (0 - execution successful, 1 - error)
"""
framework_suffix = "_{}".format(framework)
env_setup = get_environment_setup(framework)
if framework is None:
framework_suffix = ""
elif framework == "tf":
if "tensorflow" in env_setup and env_setup["tensorflow"] < LooseVersion("2.0.0"):
if not silent:
log.error('\t\nSupport of the Model Optimizer tool in TensorFlow 1.x environment is deprecated.'
'It is highly recommended to use TensorFlow 2.x.\n',
extra={'is_warning': True})
file_name = "requirements{}.txt".format(framework_suffix)
# if MO was run from developer clone requirements are 4 levels out from mo.py
requirements_file = os.path.realpath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, os.pardir, file_name))
if not Path(requirements_file).is_file():
# if MO was run from site-packages requirements are at the same level with mo.py file
# in site-packages/openvino/tools/mo/requirements.txt
requirements_file = os.path.realpath(os.path.join(os.path.dirname(__file__), os.pardir, file_name))
requirements_list = get_module_version_list_from_file(requirements_file, env_setup)
not_satisfied_versions = []
exit_code = 0
for name, key, required_version in requirements_list:
try:
importable_name = modules.get(name, name)
exec("import {}".format(importable_name))
installed_version = get_imported_module_version(sys.modules[importable_name])
version_check(name, installed_version, required_version, key, not_satisfied_versions)
exec("del {}".format(importable_name))
except (AttributeError, ImportError):
# we need to raise error only in cases when import of critical modules is failed
if name in critical_modules:
exit_code = 1
if key is not None and required_version is not None:
not_satisfied_versions.append((name, 'not installed', 'required: {} {}'.format(key, required_version)))
else:
not_satisfied_versions.append((name, 'not installed', ''))
continue
except Exception as e:
if not silent:
log.error('Error happened while importing {} module. It may happen due to unsatisfied requirements of '
'that module. Please run requirements installation script once more.\n'
'Details on module importing failure: {}'.format(name, e))
not_satisfied_versions.append((name, 'package error', 'required: {} {}'.format(key, required_version)))
continue
if len(not_satisfied_versions) != 0:
if framework == 'tf':
extra = '[tensorflow]'
elif framework == 'tf2':
extra = '[tensorflow2]'
elif framework is None:
extra = ''
else:
extra = '[{}]'.format(framework)
helper_command = "pip install openvino-dev{}".format(extra)
missed_modules_message = ""
for module in not_satisfied_versions:
missed_modules_message += "\t{}: {}, {}\n".format(module[0], module[1], module[2])
if exit_code:
if not silent:
log.error(message.format(missed_modules_message, helper_command))
else:
if not silent:
log.error(message.format(missed_modules_message, helper_command), extra={'is_warning': True})
return exit_code
@@ -1,256 +0,0 @@
# Copyright (C) 2018-2023 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
import unittest
import unittest.mock as mock
from unittest.mock import mock_open
from openvino.tools.mo.utils.versions_checker import get_module_version_list_from_file, parse_and_filter_versions_list, version_check
class TestingVersionsChecker(unittest.TestCase):
@mock.patch('builtins.open', new_callable=mock_open, create=True)
def test_get_module_version_list_from_file(self, mock_open):
mock_open.return_value.__enter__ = mock_open
mock_open.return_value.__iter__ = mock.Mock(
return_value=iter(['mxnet>=1.0.0,<=1.3.1',
'networkx>=1.11',
'numpy==1.12.0',
'defusedxml<=0.5.0',
'networkx~=1.11']))
ref_list = [('mxnet', '>=', '1.0.0'),
('mxnet', '<=', '1.3.1'),
('networkx', '>=', '1.11'),
('numpy', '==', '1.12.0'),
('defusedxml', '<=', '0.5.0'),
('networkx', '~=', '1.11')]
version_list = get_module_version_list_from_file('mock_file', {})
self.assertEqual(len(version_list), 6)
for i, version_dict in enumerate(version_list):
self.assertTupleEqual(ref_list[i], version_dict)
@mock.patch('builtins.open', new_callable=mock_open, create=True)
def test_get_module_version_list_from_file2(self, mock_open):
mock_open.return_value.__enter__ = mock_open
mock_open.return_value.__iter__ = mock.Mock(
return_value=iter(['tensorflow>=1.15.2,<2.0; python_version < "3.8"',
'tensorflow>=2.0; python_version >= "3.8"',
'numpy==1.12.0',
'defusedxml<=0.5.0',
'networkx~=1.11']))
ref_list = [('tensorflow', '>=', '1.15.2'),
('tensorflow', '<', '2.0'),
('numpy', '==', '1.12.0'),
('defusedxml', '<=', '0.5.0'),
('networkx', '~=', '1.11')]
version_list = get_module_version_list_from_file('mock_file', {'python_version': '3.7.0'})
self.assertEqual(len(version_list), 5)
for i, version_dict in enumerate(version_list):
self.assertTupleEqual(ref_list[i], version_dict)
@mock.patch('builtins.open', new_callable=mock_open, create=True)
def test_get_module_version_list_from_file3(self, mock_open):
mock_open.return_value.__enter__ = mock_open
mock_open.return_value.__iter__ = mock.Mock(
return_value=iter(['# Commented line',
'tensorflow>=1.15.2,<2.0; python_version < "3.8"',
'tensorflow>=2.0; python_version >= "3.8" # Comment after line',
'numpy==1.12.0',
'defusedxml<=0.5.0',
'networkx~=1.11']))
ref_list = [('tensorflow', '>=', '2.0'),
('numpy', '==', '1.12.0'),
('defusedxml', '<=', '0.5.0'),
('networkx', '~=', '1.11')]
version_list = get_module_version_list_from_file('mock_file', {'python_version': '3.8.1'})
self.assertEqual(len(version_list), 4)
for i, version_dict in enumerate(version_list):
self.assertTupleEqual(ref_list[i], version_dict)
@mock.patch('builtins.open', new_callable=mock_open, create=True)
def test_get_module_version_list_from_file_with_fw_name(self, mock_open):
mock_open.return_value.__enter__ = mock_open
mock_open.return_value.__iter__ = mock.Mock(
return_value=iter(['mxnet']))
ref_list = [('mxnet', None, None)]
version_list = get_module_version_list_from_file('mock_file', {})
self.assertEqual(len(version_list), 1)
for i, version_dict in enumerate(version_list):
self.assertTupleEqual(ref_list[i], version_dict)
def test_append_version_list(self):
v1 = 'mxnet>=1.0.0,<=1.3.1'
req_list = []
parse_and_filter_versions_list(v1, req_list, {})
ref_list = [('mxnet', '>=', '1.0.0'),
('mxnet', '<=', '1.3.1')]
for i, v in enumerate(ref_list):
self.assertEqual(v, req_list[i])
def test_append_version_list_sys_neg_1(self):
v1 = "mxnet>=1.7.0 ; sys_platform != 'win32'"
req_list = []
parse_and_filter_versions_list(v1, req_list, {'sys_platform': 'darwin'})
ref_list = [('mxnet', '>=', '1.7.0')]
for i, v in enumerate(ref_list):
self.assertEqual(v, req_list[i])
def test_append_version_list_sys_neg_2(self):
v1 = "mxnet>=1.7.0 ; sys_platform != 'win32'"
req_list = []
parse_and_filter_versions_list(v1, req_list, {'sys_platform': 'win32'})
ref_list = []
for i, v in enumerate(ref_list):
self.assertEqual(v, req_list[i])
def test_append_version_list_sys(self):
v1 = "mxnet>=1.7.0 ; sys_platform == 'linux'"
req_list = []
parse_and_filter_versions_list(v1, req_list, {'sys_platform': 'linux'})
ref_list = [('mxnet', '>=', '1.7.0')]
for i, v in enumerate(ref_list):
self.assertEqual(v, req_list[i])
def test_append_version_list_sys_double_quotes(self):
v1 = "mxnet>=1.7.0 ; sys_platform == \"linux\""
req_list = []
parse_and_filter_versions_list(v1, req_list, {'sys_platform': 'linux'})
ref_list = [('mxnet', '>=', '1.7.0')]
for i, v in enumerate(ref_list):
self.assertEqual(v, req_list[i])
def test_append_version_list_py_ver_single_quotes(self):
v1 = "mxnet>=1.7.0 ; python_version < '3.8'"
req_list = []
parse_and_filter_versions_list(v1, req_list, {'python_version': '3.7.1'})
ref_list = [('mxnet', '>=', '1.7.0')]
for i, v in enumerate(ref_list):
self.assertEqual(v, req_list[i])
def test_append_version_list_sys_python_ver_1(self):
v1 = "mxnet>=1.7.0 ; sys_platform == 'linux' or python_version >= \"3.8\""
req_list = []
parse_and_filter_versions_list(v1, req_list, {'python_version': '3.8.1', 'sys_platform': 'linux'})
ref_list = []
for i, v in enumerate(ref_list):
self.assertEqual(v, req_list[i])
def test_append_version_list_sys_python_ver_2(self):
v1 = "mxnet>=1.7.0 ; sys_platform == 'linux' and python_version >= \"3.8\""
req_list = []
parse_and_filter_versions_list(v1, req_list, {'python_version': '3.7.1', 'sys_platform': 'linux'})
ref_list = []
for i, v in enumerate(ref_list):
self.assertEqual(v, req_list[i])
def test_version_check_equal(self):
modules_versions_list = [('module_1', '==', '2.0', '2.0'),
('module_2', '==', '2.0', '2.0.1'),
]
ref_list = [('module_2', 'installed: 2.0.1', 'required: == 2.0'),
]
not_satisfied_versions = []
for name, key, required_version, installed_version in modules_versions_list:
version_check(name, installed_version, required_version, key, not_satisfied_versions)
self.assertEqual(not_satisfied_versions, ref_list)
def test_version_check_less_equal(self):
modules_versions_list = [('module_1', '>=', '1.12.0', '1.09.2'),
('module_2', '>=', '1.12.0', '1.12.0'),
('module_3', '>=', '1.12.0', '1.12.1'),
('module_4', '>=', '1.12.0', '1.20.0'),
]
ref_list = [('module_1', 'installed: 1.09.2', 'required: >= 1.12.0'),
]
not_satisfied_versions = []
for name, key, required_version, installed_version in modules_versions_list:
version_check(name, installed_version, required_version, key, not_satisfied_versions)
self.assertEqual(not_satisfied_versions, ref_list)
def test_version_check_greater_equal(self):
modules_versions_list = [('module_1', '>=', '1.12.0', '1.09.2'),
('module_2', '>=', '1.12.0', '1.12.0'),
('module_3', '>=', '1.12.0', '1.12.1'),
('module_4', '>=', '1.12.0', '1.20.0'),
('module_5', '>=', '1.12.0.post2', '1.12.0.post1'),
]
ref_list = [('module_1', 'installed: 1.09.2', 'required: >= 1.12.0'),
('module_5', 'installed: 1.12.0.post1', 'required: >= 1.12.0.post2')
]
not_satisfied_versions = []
for name, key, required_version, installed_version in modules_versions_list:
version_check(name, installed_version, required_version, key, not_satisfied_versions)
self.assertEqual(not_satisfied_versions, ref_list)
def test_version_check_less(self):
modules_versions_list = [('module_1', '<', '1.11', '1.01'),
('module_2', '<', '1.11', '1.10.1'),
('module_3', '<', '1.11', '1.11'),
('module_4', '<', '1.11', '1.20'),
('module_5', '<', '1.11', '1.10.post2'),
]
ref_list = [('module_3', 'installed: 1.11', 'required: < 1.11'),
('module_4', 'installed: 1.20', 'required: < 1.11'),
]
not_satisfied_versions = []
for name, key, required_version, installed_version in modules_versions_list:
version_check(name, installed_version, required_version, key, not_satisfied_versions)
self.assertEqual(not_satisfied_versions, ref_list)
def test_version_check_greater(self):
modules_versions_list = [('module_1', '>', '1.11', '1.01'),
('module_2', '>', '1.11', '1.11'),
('module_3', '>', '1.11', '1.11.1'),
('module_4', '>', '1.11', '1.20'),
]
ref_list = [('module_1', 'installed: 1.01', 'required: > 1.11'),
('module_2', 'installed: 1.11', 'required: > 1.11'),
]
not_satisfied_versions = []
for name, key, required_version, installed_version in modules_versions_list:
version_check(name, installed_version, required_version, key, not_satisfied_versions)
self.assertEqual(not_satisfied_versions, ref_list)
def test_version_check_compatible(self):
modules_versions_list = [('module_1', '~=', '1.2.3', '1.2.3'),
('module_2', '~=', '1.2.3', '1.2.3b4'),
('module_3', '~=', '1.2.3', '1.2.4'),
('module_4', '~=', '1.2.3', '1.3.0'),
('module_5', '~=', '1.2.3', '1.2.2'),
('module_6', '~=', '1.2.3', '2.2.2'),
('module_7', '~=', '1.2.post2', '1.3'),
('module_8', '~=', '1.2.post2', '1.2'),
('module_9', '~=', '1.2.post2', '1.2.post1'),
('module_9', '~=', '1.2.post2', '1.2.post4')
]
ref_list = [('module_4', 'installed: 1.3.0', 'required: ~= 1.2.3'),
('module_5', 'installed: 1.2.2', 'required: ~= 1.2.3'),
('module_6', 'installed: 2.2.2', 'required: ~= 1.2.3'),
('module_8', 'installed: 1.2', 'required: ~= 1.2.post2'),
('module_9', 'installed: 1.2.post1', 'required: ~= 1.2.post2')
]
not_satisfied_versions = []
for name, key, required_version, installed_version in modules_versions_list:
version_check(name, installed_version, required_version, key, not_satisfied_versions)
self.assertEqual(not_satisfied_versions, ref_list)