diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 137d5131cdc..74661a77af5 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -6,7 +6,7 @@ updates: # Python product dependencies # - # Python API requirements + # Python API, Frontends - package-ecosystem: pip directory: "/src/bindings/python/" schedule: @@ -17,12 +17,33 @@ updates: assignees: - "jiwaszki" - "p-wysocki" + - "akuporos" + - "rkazants" + - "ceciliapeng2011" + - "meiyang-intel" + - "mbencer" + - "tomdol" + - "jane-intel" + versioning-strategy: increase-if-necessary + + # Tests + - package-ecosystem: pip + directory: "/tests" + schedule: + interval: "daily" + time: "09:00" + timezone: "Poland" + open-pull-requests-limit: 3 + assignees: + - "jiwaszki" + - "p-wysocki" + - "akuporos" - "rkazants" versioning-strategy: increase-if-necessary - # Model Optimizer requirements + # Model Optimizer, openvino_dev and Benchmark tool - package-ecosystem: pip - directory: "/tools/mo" + directory: "/tools" schedule: interval: "daily" time: "09:00" @@ -33,6 +54,8 @@ updates: - "andrei-kochin" - "jiwaszki" - "p-wysocki" + - "akuporos" + - "Wovchena" allow: - dependency-name: "*" dependency-type: "production" @@ -51,89 +74,10 @@ updates: - "KodiaqQ" - "jiwaszki" - "p-wysocki" + - "akuporos" - "rkazants" versioning-strategy: increase-if-necessary - # benchmark_tool requirements - - package-ecosystem: pip - directory: "/tools/benchmark_tool" - schedule: - interval: "daily" - time: "09:00" - timezone: "Asia/Dubai" - open-pull-requests-limit: 3 - assignees: - - "Wovchena" - - "jiwaszki" - - "p-wysocki" - - "rkazants" - versioning-strategy: increase-if-necessary - - # - # Tests requirements for frontends - # - - # PaddlePaddle FE tests requirements - - package-ecosystem: pip - directory: "/src/frontends/paddle/tests/" - schedule: - interval: "daily" - time: "09:00" - timezone: "Asia/Shanghai" - open-pull-requests-limit: 3 - assignees: - - "ceciliapeng2011" - - "meiyang-intel" - - "jiwaszki" - - "p-wysocki" - - "rkazants" - versioning-strategy: increase-if-necessary - - # ONNX FE tests requirements - - package-ecosystem: pip - directory: "/src/frontends/onnx/tests/" - schedule: - interval: "daily" - time: "09:00" - timezone: "Poland" - open-pull-requests-limit: 3 - assignees: - - "mbencer" - - "tomdol" - - "jiwaszki" - - "p-wysocki" - - "rkazants" - versioning-strategy: increase-if-necessary - - # TensorFlow FE tests requirements - - package-ecosystem: pip - directory: "/src/frontends/tensorflow/tests/" - schedule: - interval: "daily" - time: "09:00" - timezone: "Asia/Dubai" - open-pull-requests-limit: 3 - assignees: - - "rkazants" - - "jiwaszki" - - "p-wysocki" - versioning-strategy: increase-if-necessary - - # TensorFlow Lite FE tests requirements - - package-ecosystem: pip - directory: "/src/frontends/tensorflow_lite/tests/" - schedule: - interval: "daily" - time: "09:00" - timezone: "Asia/Dubai" - open-pull-requests-limit: 3 - assignees: - - "jane-intel" - - "rkazants" - - "jiwaszki" - - "p-wysocki" - versioning-strategy: increase-if-necessary - # # Python Samples # @@ -149,6 +93,7 @@ updates: - "Wovchena" - "jiwaszki" - "p-wysocki" + - "akuporos" - "rkazants" versioning-strategy: increase-if-necessary @@ -163,6 +108,7 @@ updates: - "Wovchena" - "jiwaszki" - "p-wysocki" + - "akuporos" - "rkazants" versioning-strategy: increase-if-necessary @@ -177,6 +123,7 @@ updates: - "Wovchena" - "jiwaszki" - "p-wysocki" + - "akuporos" - "rkazants" versioning-strategy: increase-if-necessary @@ -191,6 +138,7 @@ updates: - "Wovchena" - "jiwaszki" - "p-wysocki" + - "akuporos" - "rkazants" versioning-strategy: increase-if-necessary diff --git a/cmake/developer_package/check_python_requirements.py b/cmake/developer_package/check_python_requirements.py new file mode 100644 index 00000000000..8db944ee6e4 --- /dev/null +++ b/cmake/developer_package/check_python_requirements.py @@ -0,0 +1,52 @@ +import pkg_resources +import re +import os + + +def check_python_requirements(requirements_path: str) -> None: + """ + Checks if the requirements defined in `requirements_path` are installed + in the active Python environment, while also taking constraints.txt files + into account. + """ + + constraints = {} + constraints_path = None + requirements = [] + + # read requirements and find constraints file + with open(requirements_path) as f: + raw_requirements = f.readlines() + for line in raw_requirements: + if line.startswith("-c"): + constraints_path = os.path.join(os.path.dirname(requirements_path), line.split(' ')[1][:-1]) + + # read constraints if they exist + if constraints_path: + with open(constraints_path) as f: + raw_constraints = f.readlines() + for line in raw_constraints: + if line.startswith("#") or line=="\n": + continue + line = line.replace("\n", "") + package, delimiter, constraint = re.split("(~|=|<|>|;)", line, maxsplit=1) + if constraints.get(package) is None: + constraints[package] = [delimiter + constraint] + else: + constraints[package].extend([delimiter + constraint]) + for line in raw_requirements: + if line.startswith(("#", "-c")): + continue + line = line.replace("\n", "") + if re.search("\W", line): + requirements.append(line) + else: + constraint = constraints.get(line) + if constraint: + for marker in constraint: + requirements.append(line+marker) + else: + requirements.append(line) + else: + requirements = raw_requirements + pkg_resources.require(requirements) diff --git a/cmake/developer_package/python_requirements.cmake b/cmake/developer_package/python_requirements.cmake index d187669d6ff..3f5dcd6bd0d 100644 --- a/cmake/developer_package/python_requirements.cmake +++ b/cmake/developer_package/python_requirements.cmake @@ -97,7 +97,11 @@ function(ov_check_pip_packages) if(PYTHONINTERP_FOUND) execute_process( - COMMAND ${PYTHON_EXECUTABLE} -c "import pkg_resources ; pkg_resources.require(open('${ARG_REQUIREMENTS_FILE}', mode='r'))" + COMMAND ${PYTHON_EXECUTABLE} -c " +from check_python_requirements import check_python_requirements ; +check_python_requirements('${ARG_REQUIREMENTS_FILE}') ; + " + WORKING_DIRECTORY "${IEDevScripts_DIR}" RESULT_VARIABLE EXIT_CODE OUTPUT_VARIABLE OUTPUT_TEXT ERROR_VARIABLE ERROR_TEXT) diff --git a/src/bindings/python/constraints.txt b/src/bindings/python/constraints.txt new file mode 100644 index 00000000000..8586435dc59 --- /dev/null +++ b/src/bindings/python/constraints.txt @@ -0,0 +1,22 @@ +# used in multiple components +onnx==1.13.1 # Python bindings, ONNX Frontend + +# pytest +pytest==7.2.0; python_version >= '3.10' +pytest>=5.0,<=7.0.1; python_version < '3.10' +pytest-dependency==0.5.1 +pytest-forked; platform_system != "Windows" +pytest-html==1.19.0 +pytest-timeout==2.0.1 + +# Python bindings +py>=1.9.0 +pygments>=2.8.1 +setuptools>=53.0.0 +wheel>=0.38.1 + +# Frontends +docopt~=0.6.2 +paddlepaddle==2.4.2 +tensorflow>=1.15.5,<2.13.0 +six~=1.16.0 \ No newline at end of file diff --git a/src/bindings/python/requirements_test.txt b/src/bindings/python/requirements_test.txt index d8fc77f469c..e056fa13852 100644 --- a/src/bindings/python/requirements_test.txt +++ b/src/bindings/python/requirements_test.txt @@ -1,3 +1,7 @@ +-c ./constraints.txt +numpy>=1.16.6,<=1.23.4 +protobuf~=3.20.3 +onnx bandit black flake8 @@ -14,8 +18,7 @@ flake8-expression-complexity flake8-print flake8-pytest-style flake8-rst-docstrings -# Force version of pygments from flake8-rst-docstrings -pygments>=2.8.1 +pygments flake8-string-format flake8-variables-names flake8_builtins @@ -24,20 +27,16 @@ flake8_commas flake8_pep3101 flake8_quotes mypy -onnx==1.13.1 Pep8-naming pydocstyle -pytest-forked; platform_system != "Windows" +pytest-forked pytest-xdist -pytest>=5.0,<=7.0.1; python_version < '3.10' -pytest==7.2.1; python_version >= '3.10' -pytest-html==1.19.0 -py>=1.9.0 +pytest-html +pytest +py radon retrying tox types-pkg_resources -wheel>=0.38.1 -protobuf~=3.20.3 -numpy>=1.16.6,<=1.23.4 -singledispatchmethod; python_version<'3.8' +wheel +singledispatchmethod diff --git a/src/frontends/onnx/tests/requirements.txt b/src/frontends/onnx/tests/requirements.txt index e8e66c157fb..e3d6ea192ad 100644 --- a/src/frontends/onnx/tests/requirements.txt +++ b/src/frontends/onnx/tests/requirements.txt @@ -1,4 +1,5 @@ # ONNX - generate test models -docopt~=0.6.2 -onnx==1.13.1 -protobuf>=3.18.1,<4.0.0 \ No newline at end of file +-c ../../../bindings/python/constraints.txt +protobuf>=3.18.1,<4.0.0 +docopt +onnx diff --git a/src/frontends/paddle/tests/requirements.txt b/src/frontends/paddle/tests/requirements.txt index 8ad03b94a79..8263f791981 100644 --- a/src/frontends/paddle/tests/requirements.txt +++ b/src/frontends/paddle/tests/requirements.txt @@ -1,5 +1,6 @@ # PaddlePaddle - generate test models -paddlepaddle==2.4.2 +-c ../../../../src/bindings/python/constraints.txt numpy>=1.16.6,<1.25.0 -six~=1.16.0 protobuf>=3.18.1,<4.0.0 +six +paddlepaddle diff --git a/src/frontends/tensorflow/tests/requirements.txt b/src/frontends/tensorflow/tests/requirements.txt index 398e7ce8868..2f8b23acc09 100644 --- a/src/frontends/tensorflow/tests/requirements.txt +++ b/src/frontends/tensorflow/tests/requirements.txt @@ -1,2 +1,3 @@ +-c ../../../bindings/python/constraints.txt numpy>=1.16.6,<=1.23.4 -tensorflow>=1.15.5,<2.13.0 +tensorflow diff --git a/src/frontends/tensorflow_lite/tests/requirements.txt b/src/frontends/tensorflow_lite/tests/requirements.txt index 49b433c132e..85cc64cbcc4 100644 --- a/src/frontends/tensorflow_lite/tests/requirements.txt +++ b/src/frontends/tensorflow_lite/tests/requirements.txt @@ -1,2 +1,3 @@ +-c ../../../bindings/python/constraints.txt numpy>=1.16.6,<1.25.0 -tensorflow>=1.15.5,<2.13.0 +tensorflow diff --git a/tests/conditional_compilation/requirements.txt b/tests/conditional_compilation/requirements.txt index e7a9a3e564b..e8518d0d389 100644 --- a/tests/conditional_compilation/requirements.txt +++ b/tests/conditional_compilation/requirements.txt @@ -1,6 +1,7 @@ -pytest>=6.2.1 -pytest-dependency==0.5.1 -py>=1.9.0 -PyYAML>=5.4.1 +-c ../constraints.txt numpy>=1.16.6 -pytest-html>=1.19.0 \ No newline at end of file +pytest +pytest-dependency +pytest-html +py +PyYAML diff --git a/tests/constraints.txt b/tests/constraints.txt new file mode 100644 index 00000000000..219731dc417 --- /dev/null +++ b/tests/constraints.txt @@ -0,0 +1,21 @@ +attrs==22.1.0 +distro==1.8.0 +h5py>=3.1.0 +Jinja2>=2.11.2 +pandas>=1.3.5 +pymongo>=3.12.0 +PyYAML>=5.4.1 +scipy>=1.8; python_version >= '3.8' +scipy~=1.7; python_version == '3.7' +wheel>=0.38.1 +defusedxml>=0.7.1 +fastjsonschema~=2.15.1 +test-generator==0.1.1 +requests>=2.25.1 +opencv-python>=4.5 +py>=1.9.0 +pytest==7.2.0; python_version >= '3.10' +pytest>=5.0,<=7.0.1; python_version < '3.10' +pytest-dependency==0.5.1 +pytest-html==1.19.0 +pytest-timeout==2.0.1 diff --git a/tests/layer_tests/requirements.txt b/tests/layer_tests/requirements.txt index 1fa23c01b70..6138c71a61e 100644 --- a/tests/layer_tests/requirements.txt +++ b/tests/layer_tests/requirements.txt @@ -1,4 +1,5 @@ -requests>=2.25.1 +-c ../constraints.txt +requests numpy>=1.19.2 torch torchvision diff --git a/tests/memory_tests/test_runner/requirements.txt b/tests/memory_tests/test_runner/requirements.txt index e65c5a97cbb..db36fb72f32 100644 --- a/tests/memory_tests/test_runner/requirements.txt +++ b/tests/memory_tests/test_runner/requirements.txt @@ -1,10 +1,10 @@ -pytest>=5.0,<=7.0.1; python_version < '3.10' -pytest==7.2.0; python_version >= '3.10' -py>=1.9.0 -PyYAML>=5.4.1 -jsonschema==3.2.0 -distro==1.8.0 +-c ../../constraints.txt +pytest +py +PyYAML +pymongo +jsonschema +distro +pytest-html +pytest-timeout numpy>=1.19.2 -pymongo>=3.12.0 -pytest-html>=1.19.0 -pytest-timeout==2.0.1 diff --git a/tests/samples_tests/CMakeLists.txt b/tests/samples_tests/CMakeLists.txt index 7279ea123ad..d656709b459 100644 --- a/tests/samples_tests/CMakeLists.txt +++ b/tests/samples_tests/CMakeLists.txt @@ -6,4 +6,10 @@ cmake_minimum_required(VERSION 3.13) project(samples_tests) -install(DIRECTORY smoke_tests/ DESTINATION tests/smoke_tests COMPONENT tests EXCLUDE_FROM_ALL) +install(DIRECTORY smoke_tests/ DESTINATION tests/smoke_tests COMPONENT tests EXCLUDE_FROM_ALL PATTERN "requirements.txt" EXCLUDE) +install(FILES ../constraints.txt DESTINATION tests COMPONENT tests EXCLUDE_FROM_ALL) + +file(READ smoke_tests/requirements.txt REQUIREMENTS_REPO) +string(REPLACE "-c ../../constraints.txt" "-c ../constraints.txt" REQUIREMENTS_TMP ${REQUIREMENTS_REPO}) +file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/smoke_tests/requirements.txt ${REQUIREMENTS_TMP}) +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/smoke_tests/requirements.txt DESTINATION tests/smoke_tests COMPONENT tests EXCLUDE_FROM_ALL) diff --git a/tests/samples_tests/smoke_tests/requirements.txt b/tests/samples_tests/smoke_tests/requirements.txt index a92c0a4fca4..2181461187e 100644 --- a/tests/samples_tests/smoke_tests/requirements.txt +++ b/tests/samples_tests/smoke_tests/requirements.txt @@ -1,10 +1,11 @@ +-c ../../constraints.txt requests pyyaml wheel test-generator numpy pytest -py>=1.9.0 +py scikit-build opencv-python progress diff --git a/tests/samples_tests/smoke_tests/test_benchmark_app.py b/tests/samples_tests/smoke_tests/test_benchmark_app.py index fa33d4a48d0..bfe9f6230a1 100644 --- a/tests/samples_tests/smoke_tests/test_benchmark_app.py +++ b/tests/samples_tests/smoke_tests/test_benchmark_app.py @@ -51,10 +51,12 @@ class TestBenchmarkApp(SamplesCommonTestClass): super().setup_class() @pytest.mark.parametrize("param", test_data_fp32_async) + @pytest.mark.skip("Ticket: 106850") def test_benchmark_app_sample_fp32_async(self, param): _check_output(self, param) @pytest.mark.parametrize("param", test_data_fp32_sync) + @pytest.mark.skip("Ticket: 106850") def test_benchmark_app_fp32_sync(self, param): _check_output(self, param) diff --git a/tests/stress_tests/scripts/requirements.txt b/tests/stress_tests/scripts/requirements.txt index f8913753397..da7d6a5a51c 100644 --- a/tests/stress_tests/scripts/requirements.txt +++ b/tests/stress_tests/scripts/requirements.txt @@ -1,9 +1,9 @@ -pymongo>=3.12.0 -Jinja2>=2.11.2 -PyYAML>=5.4.1 -fastjsonschema~=2.15.1 -pandas>=1.3.5 -h5py>=3.1.0 -scipy~=1.7; python_version == '3.7' -scipy>=1.8; python_version >= '3.8' -defusedxml>=0.7.1 +-c ../../constraints.txt +pymongo +Jinja2 +PyYAML +fastjsonschema +pandas +h5py +scipy +defusedxml diff --git a/tests/time_tests/scripts/requirements.txt b/tests/time_tests/scripts/requirements.txt index 373be65de35..268e1cbd18c 100644 --- a/tests/time_tests/scripts/requirements.txt +++ b/tests/time_tests/scripts/requirements.txt @@ -1 +1,2 @@ -PyYAML>=5.4.1 \ No newline at end of file +-c ../../constraints.txt +PyYAML \ No newline at end of file diff --git a/tests/time_tests/test_runner/requirements.txt b/tests/time_tests/test_runner/requirements.txt index 049b37efb6e..4ade7443081 100644 --- a/tests/time_tests/test_runner/requirements.txt +++ b/tests/time_tests/test_runner/requirements.txt @@ -1,11 +1,11 @@ -pytest>=5.0,<=7.0.1; python_version < '3.10' -pytest==7.2.0; python_version >= '3.10' -py>=1.9.0 -attrs==22.1.0 -PyYAML>=5.4.1 -jsonschema==3.2.0 -distro==1.8.0 +-c ../../constraints.txt numpy>=1.19.2 -pymongo>=3.12.0 -pytest-html>=1.19.0 -pytest-timeout==2.0.1 +pytest +py +attrs +PyYAML +jsonschema +distro +pymongo +pytest-html +pytest-timeout diff --git a/tools/benchmark_tool/requirements.txt b/tools/benchmark_tool/requirements.txt index 24a1329aa18..690121c4223 100644 --- a/tools/benchmark_tool/requirements.txt +++ b/tools/benchmark_tool/requirements.txt @@ -1,2 +1,3 @@ +-c ../constraints.txt numpy>=1.16.6 -opencv-python>=4.5 +opencv-python diff --git a/tools/benchmark_tool/setup.py b/tools/benchmark_tool/setup.py index 7a51dceb7a3..4b5d3145e61 100644 --- a/tools/benchmark_tool/setup.py +++ b/tools/benchmark_tool/setup.py @@ -9,17 +9,78 @@ Use this script to create a wheel with OpenVINO™ Python* tools: $ python setup.py sdist bdist_wheel """ import pkg_resources +import re from setuptools import setup, find_packages +from pathlib import Path +from typing import List + with open('README.md', 'r', encoding='utf-8') as f: long_description = f.read() -with open('requirements.txt') as requirements_txt: - reqs = [ - str(requirement) - for requirement - in pkg_resources.parse_requirements(requirements_txt) - ] + +def read_constraints(path: str='../constraints.txt') -> Dict[str, List[str]]: + """ + Read a constraints.txt file and return a dict + of {package_name: [required_version_1, required_version_2]}. + The dict values are a list because a package can be mentioned + multiple times, for example: + mxnet~=1.2.0; sys_platform == 'win32' + mxnet>=1.7.0; sys_platform != 'win32' + """ + constraints = {} + with open(Path(__file__).resolve().parent / path) as f: + raw_constraints = f.readlines() + for line in raw_constraints: + # skip comments + if line.startswith('#'): + continue + line = line.replace('\n', '') + # read constraints for that package + package, delimiter, constraint = re.split('(~|=|<|>|;)', line, maxsplit=1) + # if there is no entry for that package, add it + if constraints.get(package) is None: + constraints[package] = [delimiter + constraint] + # else add another entry for that package + else: + constraints[package].extend([delimiter + constraint]) + return constraints + + +def read_requirements(path: str) -> List[str]: + """ + Read a requirements.txt file and return a list + of requirements. Three cases are supported, the + list corresponds to priority: + 1. version specified in requirements.txt + 2. version specified in constraints.txt + 3. version unbound + """ + requirements = [] + constraints = read_constraints() + with open(Path(__file__).resolve().parent / path) as f: + raw_requirements = f.readlines() + for line in raw_requirements: + # skip comments and constraints link + if line.startswith(('#', '-c')): + continue + # get rid of newlines + line = line.replace('\n', '') + # if version is specified (non-word chars present) + if re.search('\W', line): + requirements.append(line) + # else get version from constraints + else: + constraint = constraints.get(line) + # if version found in constraints.txt + if constraint: + for marker in constraint: + requirements.append(line+marker) + # else version is unbound + else: + requirements.append(line) + return requirements + setup( name='benchmark_tool', @@ -40,6 +101,6 @@ setup( 'Operating System :: OS Independent', ], packages=find_packages(), - install_requires=reqs, + install_requires=read_requirements('requirements.txt'), python_requires='>=3.7', ) diff --git a/tools/constraints.txt b/tools/constraints.txt new file mode 100644 index 00000000000..988689cf0f8 --- /dev/null +++ b/tools/constraints.txt @@ -0,0 +1,24 @@ +# EXCEPTIONS +# some package versions need to be specified in respective requirements.txt +# files because the version differs between them: +# tensorflow, numpy +mxnet~=1.2.0; sys_platform == 'win32' +mxnet>=1.7.0.post2,<=1.9.1; sys_platform != 'win32' +onnx>=1.8.1,<=1.13.1 +networkx<=2.8.8 +pytest>=6.2.4; python_version < '3.10' +pytest==7.2.0; python_version >= '3.10' +protobuf>=3.18.1,<4.0.0 +defusedxml>=0.7.1 +requests>=2.25.1 +fastjsonschema>=2.15.1,<2.17 +coverage>=4.4.2,<=7.0.5 +astroid>=2.9.0 +pylint>=2.7.0 +pyenchant>=3.0.0 +test-generator==0.1.1 +py>=1.9.0 +urllib3>=1.26.4 +importlib-metadata; python_version < "3.8" and sys_platform == 'win32' +openvino-telemetry>=2022.1.0 +opencv-python>=4.5 \ No newline at end of file diff --git a/tools/mo/requirements.txt b/tools/mo/requirements.txt index 36cb4f32eb7..c6e9e00c350 100644 --- a/tools/mo/requirements.txt +++ b/tools/mo/requirements.txt @@ -1,6 +1,6 @@ +-c ../constraints.txt numpy>=1.16.6 -networkx~=2.5; python_version <= "3.6" -networkx<=2.8.8; python_version > "3.6" -defusedxml>=0.7.1 -importlib-metadata; python_version < "3.8" and sys_platform == 'win32' -openvino-telemetry>=2022.1.0 \ No newline at end of file +networkx +defusedxml +importlib-metadata +openvino-telemetry \ No newline at end of file diff --git a/tools/mo/requirements_caffe.txt b/tools/mo/requirements_caffe.txt index 93a2e760ac5..3d8b0480a43 100644 --- a/tools/mo/requirements_caffe.txt +++ b/tools/mo/requirements_caffe.txt @@ -1,7 +1,7 @@ -networkx~=2.5; python_version <= "3.6" -networkx<=2.8.8; python_version > "3.6" +-c ../constraints.txt numpy>=1.16.6,<1.25.0 -protobuf>=3.18.1,<4.0.0 -defusedxml>=0.7.1 -requests>=2.25.1 -fastjsonschema>=2.15.1,<2.17 +networkx +protobuf +defusedxml +requests +fastjsonschema \ No newline at end of file diff --git a/tools/mo/requirements_dev.txt b/tools/mo/requirements_dev.txt index 542e2e86cb9..2dff4a760c7 100644 --- a/tools/mo/requirements_dev.txt +++ b/tools/mo/requirements_dev.txt @@ -1,11 +1,11 @@ -coverage>=4.4.2,<=7.0.5 -astroid>=2.9.0 -pylint>=2.7.0 -pyenchant>=3.0.0 -test-generator==0.1.1 -defusedxml>=0.5.0 -requests>=2.20.0 -pytest>=6.2.4; python_version < '3.10' -pytest==7.2.0; python_version >= '3.10' -py>=1.9.0 -fastjsonschema>=2.15.1,<2.17 +-c ../constraints.txt +coverage +astroid +pylint +pyenchant +test-generator +defusedxml +requests +pytest +py +fastjsonschema \ No newline at end of file diff --git a/tools/mo/requirements_kaldi.txt b/tools/mo/requirements_kaldi.txt index 1973fc24acd..6ef5aeaf13f 100644 --- a/tools/mo/requirements_kaldi.txt +++ b/tools/mo/requirements_kaldi.txt @@ -1,6 +1,6 @@ -networkx~=2.5; python_version <= "3.6" -networkx<=2.8.8; python_version > "3.6" +-c ../constraints.txt numpy>=1.16.6,<1.25.0 -defusedxml>=0.7.1 -requests>=2.25.1 -fastjsonschema>=2.15.1,<2.17 +networkx +defusedxml +requests +fastjsonschema \ No newline at end of file diff --git a/tools/mo/requirements_mxnet.txt b/tools/mo/requirements_mxnet.txt index f9ba6982ff6..8df37622cc2 100644 --- a/tools/mo/requirements_mxnet.txt +++ b/tools/mo/requirements_mxnet.txt @@ -1,9 +1,8 @@ -mxnet~=1.2.0; sys_platform == 'win32' -mxnet>=1.7.0.post2,<=1.9.1; sys_platform != 'win32' -networkx~=2.5; python_version <= "3.6" -networkx<=2.8.8; python_version > "3.6" +-c ../constraints.txt numpy>=1.16.6,<1.25.0 -defusedxml>=0.7.1 -urllib3>=1.26.4 -requests>=2.25.1 -fastjsonschema>=2.15.1,<2.17 +mxnet +networkx +defusedxml +urllib3 +requests +fastjsonschema \ No newline at end of file diff --git a/tools/mo/requirements_onnx.txt b/tools/mo/requirements_onnx.txt index a7223c85951..178e18ac85e 100644 --- a/tools/mo/requirements_onnx.txt +++ b/tools/mo/requirements_onnx.txt @@ -1,8 +1,8 @@ -onnx>=1.8.1,<=1.13.1 -networkx~=2.5; python_version <= "3.6" -networkx<=2.8.8; python_version > "3.6" +-c ../constraints.txt numpy>=1.16.6,<1.25.0 -defusedxml>=0.7.1 -requests>=2.25.1 -fastjsonschema>=2.15.1,<2.17 -protobuf>=3.18.1,<4.0.0 +onnx +networkx +defusedxml +requests +fastjsonschema +protobuf \ No newline at end of file diff --git a/tools/mo/requirements_tf.txt b/tools/mo/requirements_tf.txt index 4c8329c8192..a6ab68f6564 100644 --- a/tools/mo/requirements_tf.txt +++ b/tools/mo/requirements_tf.txt @@ -1,7 +1,7 @@ -numpy>=1.16.6,<1.25.0 +-c ../constraints.txt tensorflow>=1.15.5,<2.13.0 -networkx~=2.5; python_version <= "3.6" -networkx<=2.8.8; python_version > "3.6" -defusedxml>=0.7.1 -requests>=2.25.1 -fastjsonschema>=2.15.1,<2.17 \ No newline at end of file +numpy>=1.16.6,<1.25.0 +networkx +defusedxml +requests +fastjsonschema diff --git a/tools/mo/requirements_tf2.txt b/tools/mo/requirements_tf2.txt index b77a4ae3365..562e77e06f2 100644 --- a/tools/mo/requirements_tf2.txt +++ b/tools/mo/requirements_tf2.txt @@ -1,7 +1,7 @@ -numpy>=1.16.6,<1.25.0 +-c ../constraints.txt tensorflow>=2.5,<2.13.0 -networkx~=2.5; python_version <= "3.6" -networkx<=2.8.8; python_version > "3.6" -defusedxml>=0.7.1 -requests>=2.25.1 -fastjsonschema>=2.15.1,<2.17 \ No newline at end of file +numpy>=1.16.6,<1.25.0 +networkx +defusedxml +requests +fastjsonschema diff --git a/tools/mo/setup.py b/tools/mo/setup.py index 3c3d01bfe88..d670e2c7cec 100644 --- a/tools/mo/setup.py +++ b/tools/mo/setup.py @@ -11,7 +11,6 @@ $ python setup.py sdist bdist_wheel import os import re -import sys from pathlib import Path from shutil import copyfile, copy @@ -19,12 +18,73 @@ from setuptools import setup, find_namespace_packages from setuptools.command.build_py import build_py from setuptools.command.install import install +from typing import Dict, List + prefix = 'openvino/tools/mo/' SETUP_DIR = Path(__file__).resolve().parent / Path(prefix) -def read_text(path): - return (Path(__file__).resolve().parent / path).read_text() +def read_constraints(path: str='../constraints.txt') -> Dict[str, List[str]]: + """ + Read a constraints.txt file and return a dict + of {package_name: [required_version_1, required_version_2]}. + The dict values are a list because a package can be mentioned + multiple times, for example: + mxnet~=1.2.0; sys_platform == 'win32' + mxnet>=1.7.0; sys_platform != 'win32' + """ + constraints = {} + with open(Path(__file__).resolve().parent / path) as f: + raw_constraints = f.readlines() + for line in raw_constraints: + # skip comments + if line.startswith('#'): + continue + line = line.replace('\n', '') + # read constraints for that package + package, delimiter, constraint = re.split('(~|=|<|>|;)', line, maxsplit=1) + # if there is no entry for that package, add it + if constraints.get(package) is None: + constraints[package] = [delimiter + constraint] + # else add another entry for that package + else: + constraints[package].extend([delimiter + constraint]) + return constraints + + +def read_requirements(path: str) -> List[str]: + """ + Read a requirements.txt file and return a list + of requirements. Three cases are supported, the + list corresponds to priority: + 1. version specified in requirements.txt + 2. version specified in constraints.txt + 3. version unbound + """ + requirements = [] + constraints = read_constraints() + with open(Path(__file__).resolve().parent / path) as f: + raw_requirements = f.readlines() + for line in raw_requirements: + # skip comments and constraints link + if line.startswith(('#', '-c')): + continue + # get rid of newlines + line = line.replace('\n', '') + # if version is specified (non-word chars present) + if re.search('\W', line): + requirements.append(line) + # else get version from constraints + else: + constraint = constraints.get(line) + # if version found in constraints.txt + if constraint: + for marker in constraint: + requirements.append(line+marker) + # else version is unbound + else: + requirements.append(line) + return requirements # Detect all the framework specific requirements_*.txt files. @@ -91,17 +151,17 @@ setup( 'openvino.tools.mo.front.caffe': ['CustomLayersMapping.xml*'] }, extras_require={ - 'caffe': read_text('requirements_caffe.txt'), - 'kaldi': read_text('requirements_kaldi.txt'), - 'mxnet': read_text('requirements_mxnet.txt'), - 'onnx': read_text('requirements_onnx.txt'), - 'tensorflow': read_text('requirements_tf.txt'), - 'tensorflow2': read_text('requirements_tf2.txt'), + 'caffe': read_requirements('requirements_caffe.txt'), + 'kaldi': read_requirements('requirements_kaldi.txt'), + 'mxnet': read_requirements('requirements_mxnet.txt'), + 'onnx': read_requirements('requirements_onnx.txt'), + 'tensorflow': read_requirements('requirements_tf.txt'), + 'tensorflow2': read_requirements('requirements_tf2.txt'), }, classifiers=[ "Programming Language :: Python :: 3", "License :: OSI Approved :: Apache Software License", "Operating System :: OS Independent", ], - install_requires=read_text('requirements.txt'), + install_requires=read_requirements('requirements.txt'), ) diff --git a/tools/openvino_dev/requirements.txt b/tools/openvino_dev/requirements.txt index e69de29bb2d..5153bff24e3 100644 --- a/tools/openvino_dev/requirements.txt +++ b/tools/openvino_dev/requirements.txt @@ -0,0 +1 @@ +-c ../constraints.txt \ No newline at end of file diff --git a/tools/openvino_dev/setup.py b/tools/openvino_dev/setup.py index e6f84abd86f..081c1e5e1d7 100644 --- a/tools/openvino_dev/setup.py +++ b/tools/openvino_dev/setup.py @@ -12,6 +12,7 @@ import sys import platform import subprocess # nosec import shutil +import re from distutils import log from distutils.command.build import build from distutils.command.clean import clean @@ -20,6 +21,7 @@ from fnmatch import fnmatchcase import pkg_resources from setuptools.command.install import install from setuptools import setup, find_namespace_packages +from typing import Dict, List PYTHON_VERSION = f'python{sys.version_info.major}.{sys.version_info.minor}' SCRIPT_DIR = Path(__file__).resolve().parents[0] @@ -198,12 +200,68 @@ def get_description(desc_file_path): description = fstream.read() return description -with (SCRIPT_DIR / 'requirements.txt').open() as requirements: - install_reqs = [ - str(requirement) - for requirement - in pkg_resources.parse_requirements(requirements) - ] + +def read_constraints(path: str='../constraints.txt') -> Dict[str, List[str]]: + """ + Read a constraints.txt file and return a dict + of {package_name: [required_version_1, required_version_2]}. + The dict values are a list because a package can be mentioned + multiple times, for example: + mxnet~=1.2.0; sys_platform == 'win32' + mxnet>=1.7.0; sys_platform != 'win32' + """ + constraints = {} + with open(Path(__file__).resolve().parent / path) as f: + raw_constraints = f.readlines() + for line in raw_constraints: + # skip comments + if line.startswith('#'): + continue + line = line.replace('\n', '') + # read constraints for that package + package, delimiter, constraint = re.split('(~|=|<|>|;)', line, maxsplit=1) + # if there is no entry for that package, add it + if constraints.get(package) is None: + constraints[package] = [delimiter + constraint] + # else add another entry for that package + else: + constraints[package].extend([delimiter + constraint]) + return constraints + + +def read_requirements(path: str) -> List[str]: + """ + Read a requirements.txt file and return a list + of requirements. Three cases are supported, the + list corresponds to priority: + 1. version specified in requirements.txt + 2. version specified in constraints.txt + 3. version unbound + """ + requirements = [] + constraints = read_constraints() + with open(Path(__file__).resolve().parent / path) as f: + raw_requirements = f.readlines() + for line in raw_requirements: + # skip comments and constraints link + if line.startswith(('#', '-c')): + continue + # get rid of newlines + line = line.replace('\n', '') + # if version is specified (non-word chars present) + if re.search('\W', line): + requirements.append(line) + # else get version from constraints + else: + constraint = constraints.get(line) + # if version found in constraints.txt + if constraint: + for marker in constraint: + requirements.append(line+marker) + # else version is unbound + else: + requirements.append(line) + return requirements def concat_files(output_file, input_files): @@ -214,7 +272,6 @@ def concat_files(output_file, input_files): outfile.write(content) return output_file - description_md = SCRIPT_DIR.parents[1] / 'docs' / 'install_guides' / 'pypi-openvino-dev.md' md_files = [description_md, SCRIPT_DIR.parents[1] / 'docs' / 'install_guides' / 'pre-release-note.md'] docs_url = 'https://docs.openvino.ai/latest/index.html' @@ -248,7 +305,7 @@ setup( entry_points = { 'console_scripts': [], }, - install_requires=install_reqs, + install_requires=read_requirements(SCRIPT_DIR / 'requirements.txt'), packages=find_namespace_packages(where=str(SRC_DIR)), package_dir={'': str(SRC_DIR)}, )