IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[PyOV] Simplify requirement files (#15343)

Browse Source 
* Partial progress

* Finish v1

* Cleanup

* Remove useless files

* Fix path to pdpd

* Fix onnx path

* Minor change

* Rework MO

* Minor change

* Remove some costraints

* Add MO constraints

* Update gitignore for MO

* Minor change

* Apply tech sync discussion

* Cleanup

* CR comment

* Debug ONNX FE

* simplify ONNX FE

* Update cmake

* Hardcode ONNX requirement

* Add dependency resolver to cmake

* Add constraints for openvino/tests

* Add missing pytest-html

* Fix -c path

* Revert debug changes to path

* Add cmake to copy constraints.txt

* Update dependabot

* Remove slash

* Remove cmake

* Debug prints

* Minor changes

* Move reqs check to separate file

* Add requirements parser to benchmark_tool

* Fix smoke tests constraints

* Minor fixes

* Minor change

* My fixes were apparently wrong

* Debug - self.executable_path

* Debug - add singledispatch to tests and tools

* Debug - print IE_APP_PATHs

* Revert "Debug - print IE_APP_PATHs"

This reverts commit 67ccb6d3f5.

* Revert "Debug - add singledispatch to tests and tools"

This reverts commit 3b945931e2.

* Revert "Debug - self.executable_path"

This reverts commit 3aa724eff6.

* update dependabot

* update dependabot

* Skip benchmark_app tests

* Use CMAKE_CURRENT_BINARY_DIR in cmake

* Remove debug prints

* minor change

---------

Signed-off-by: p-wysocki <przemyslaw.wysocki@intel.com>
This commit is contained in:
Przemyslaw Wysocki 2023-03-29 12:27:27 +02:00 committed by GitHub
parent 988a8dd6a9
commit 591c3e61c5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
33 changed files with 485 additions and 220 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

114
.github/dependabot.yml vendored
View File

@ -6,7 +6,7 @@ updates:
# Python product dependencies
#
# Python API requirements
# Python API, Frontends
- package-ecosystem: pip
directory: "/src/bindings/python/"
schedule:
@ -17,12 +17,33 @@ updates:
assignees:
- "jiwaszki"
- "p-wysocki"
- "akuporos"
- "rkazants"
- "ceciliapeng2011"
- "meiyang-intel"
- "mbencer"
- "tomdol"
- "jane-intel"
versioning-strategy: increase-if-necessary
# Tests
- package-ecosystem: pip
directory: "/tests"
schedule:
interval: "daily"
time: "09:00"
timezone: "Poland"
open-pull-requests-limit: 3
assignees:
- "jiwaszki"
- "p-wysocki"
- "akuporos"
- "rkazants"
versioning-strategy: increase-if-necessary
# Model Optimizer requirements
# Model Optimizer, openvino_dev and Benchmark tool
- package-ecosystem: pip
directory: "/tools/mo"
directory: "/tools"
schedule:
interval: "daily"
time: "09:00"
@ -33,6 +54,8 @@ updates:
- "andrei-kochin"
- "jiwaszki"
- "p-wysocki"
- "akuporos"
- "Wovchena"
allow:
- dependency-name: "*"
dependency-type: "production"
@ -51,89 +74,10 @@ updates:
- "KodiaqQ"
- "jiwaszki"
- "p-wysocki"
- "akuporos"
- "rkazants"
versioning-strategy: increase-if-necessary
# benchmark_tool requirements
- package-ecosystem: pip
directory: "/tools/benchmark_tool"
schedule:
interval: "daily"
time: "09:00"
timezone: "Asia/Dubai"
open-pull-requests-limit: 3
assignees:
- "Wovchena"
- "jiwaszki"
- "p-wysocki"
- "rkazants"
versioning-strategy: increase-if-necessary
#
# Tests requirements for frontends
#
# PaddlePaddle FE tests requirements
- package-ecosystem: pip
directory: "/src/frontends/paddle/tests/"
schedule:
interval: "daily"
time: "09:00"
timezone: "Asia/Shanghai"
open-pull-requests-limit: 3
assignees:
- "ceciliapeng2011"
- "meiyang-intel"
- "jiwaszki"
- "p-wysocki"
- "rkazants"
versioning-strategy: increase-if-necessary
# ONNX FE tests requirements
- package-ecosystem: pip
directory: "/src/frontends/onnx/tests/"
schedule:
interval: "daily"
time: "09:00"
timezone: "Poland"
open-pull-requests-limit: 3
assignees:
- "mbencer"
- "tomdol"
- "jiwaszki"
- "p-wysocki"
- "rkazants"
versioning-strategy: increase-if-necessary
# TensorFlow FE tests requirements
- package-ecosystem: pip
directory: "/src/frontends/tensorflow/tests/"
schedule:
interval: "daily"
time: "09:00"
timezone: "Asia/Dubai"
open-pull-requests-limit: 3
assignees:
- "rkazants"
- "jiwaszki"
- "p-wysocki"
versioning-strategy: increase-if-necessary
# TensorFlow Lite FE tests requirements
- package-ecosystem: pip
directory: "/src/frontends/tensorflow_lite/tests/"
schedule:
interval: "daily"
time: "09:00"
timezone: "Asia/Dubai"
open-pull-requests-limit: 3
assignees:
- "jane-intel"
- "rkazants"
- "jiwaszki"
- "p-wysocki"
versioning-strategy: increase-if-necessary
#
# Python Samples
#
@ -149,6 +93,7 @@ updates:
- "Wovchena"
- "jiwaszki"
- "p-wysocki"
- "akuporos"
- "rkazants"
versioning-strategy: increase-if-necessary
@ -163,6 +108,7 @@ updates:
- "Wovchena"
- "jiwaszki"
- "p-wysocki"
- "akuporos"
- "rkazants"
versioning-strategy: increase-if-necessary
@ -177,6 +123,7 @@ updates:
- "Wovchena"
- "jiwaszki"
- "p-wysocki"
- "akuporos"
- "rkazants"
versioning-strategy: increase-if-necessary
@ -191,6 +138,7 @@ updates:
- "Wovchena"
- "jiwaszki"
- "p-wysocki"
- "akuporos"
- "rkazants"
versioning-strategy: increase-if-necessary

52
cmake/developer_package/check_python_requirements.py Normal file
View File

@ -0,0 +1,52 @@
import pkg_resources
import re
import os
def check_python_requirements(requirements_path: str) -> None:
"""
Checks if the requirements defined in `requirements_path` are installed
in the active Python environment, while also taking constraints.txt files
into account.
"""
constraints = {}
constraints_path = None
requirements = []
# read requirements and find constraints file
with open(requirements_path) as f:
raw_requirements = f.readlines()
for line in raw_requirements:
if line.startswith("-c"):
constraints_path = os.path.join(os.path.dirname(requirements_path), line.split(' ')[1][:-1])
# read constraints if they exist
if constraints_path:
with open(constraints_path) as f:
raw_constraints = f.readlines()
for line in raw_constraints:
if line.startswith("#") or line=="\n":
continue
line = line.replace("\n", "")
package, delimiter, constraint = re.split("(~|=|<|>|;)", line, maxsplit=1)
if constraints.get(package) is None:
constraints[package] = [delimiter + constraint]
else:
constraints[package].extend([delimiter + constraint])
for line in raw_requirements:
if line.startswith(("#", "-c")):
continue
line = line.replace("\n", "")
if re.search("\W", line):
requirements.append(line)
else:
constraint = constraints.get(line)
if constraint:
for marker in constraint:
requirements.append(line+marker)
else:
requirements.append(line)
else:
requirements = raw_requirements
pkg_resources.require(requirements)

6
cmake/developer_package/python_requirements.cmake
View File

@ -97,7 +97,11 @@ function(ov_check_pip_packages)
if(PYTHONINTERP_FOUND)
execute_process(
COMMAND ${PYTHON_EXECUTABLE} -c "import pkg_resources ; pkg_resources.require(open('${ARG_REQUIREMENTS_FILE}', mode='r'))"
COMMAND ${PYTHON_EXECUTABLE} -c "
from check_python_requirements import check_python_requirements ;
check_python_requirements('${ARG_REQUIREMENTS_FILE}') ;
"
WORKING_DIRECTORY "${IEDevScripts_DIR}"
RESULT_VARIABLE EXIT_CODE
OUTPUT_VARIABLE OUTPUT_TEXT
ERROR_VARIABLE ERROR_TEXT)

22
src/bindings/python/constraints.txt Normal file
View File

@ -0,0 +1,22 @@
# used in multiple components
onnx==1.13.1 # Python bindings, ONNX Frontend
# pytest
pytest==7.2.0; python_version >= '3.10'
pytest>=5.0,<=7.0.1; python_version < '3.10'
pytest-dependency==0.5.1
pytest-forked; platform_system != "Windows"
pytest-html==1.19.0
pytest-timeout==2.0.1
# Python bindings
py>=1.9.0
pygments>=2.8.1
setuptools>=53.0.0
wheel>=0.38.1
# Frontends
docopt~=0.6.2
paddlepaddle==2.4.2
tensorflow>=1.15.5,<2.13.0
six~=1.16.0

23
src/bindings/python/requirements_test.txt
View File

@ -1,3 +1,7 @@
-c ./constraints.txt
numpy>=1.16.6,<=1.23.4
protobuf~=3.20.3
onnx
bandit
black
flake8
@ -14,8 +18,7 @@ flake8-expression-complexity
flake8-print
flake8-pytest-style
flake8-rst-docstrings
# Force version of pygments from flake8-rst-docstrings
pygments>=2.8.1
pygments
flake8-string-format
flake8-variables-names
flake8_builtins
@ -24,20 +27,16 @@ flake8_commas
flake8_pep3101
flake8_quotes
mypy
onnx==1.13.1
Pep8-naming
pydocstyle
pytest-forked; platform_system != "Windows"
pytest-forked
pytest-xdist
pytest>=5.0,<=7.0.1; python_version < '3.10'
pytest==7.2.1; python_version >= '3.10'
pytest-html==1.19.0
py>=1.9.0
pytest-html
pytest
py
radon
retrying
tox
types-pkg_resources
wheel>=0.38.1
protobuf~=3.20.3
numpy>=1.16.6,<=1.23.4
singledispatchmethod; python_version<'3.8'
wheel
singledispatchmethod

7
src/frontends/onnx/tests/requirements.txt
View File

@ -1,4 +1,5 @@
# ONNX - generate test models
docopt~=0.6.2
onnx==1.13.1
protobuf>=3.18.1,<4.0.0
-c ../../../bindings/python/constraints.txt
protobuf>=3.18.1,<4.0.0
docopt
onnx

5
src/frontends/paddle/tests/requirements.txt
View File

@ -1,5 +1,6 @@
# PaddlePaddle - generate test models
paddlepaddle==2.4.2
-c ../../../../src/bindings/python/constraints.txt
numpy>=1.16.6,<1.25.0
six~=1.16.0
protobuf>=3.18.1,<4.0.0
six
paddlepaddle

3
src/frontends/tensorflow/tests/requirements.txt
View File

@ -1,2 +1,3 @@
-c ../../../bindings/python/constraints.txt
numpy>=1.16.6,<=1.23.4
tensorflow>=1.15.5,<2.13.0
tensorflow

3
src/frontends/tensorflow_lite/tests/requirements.txt
View File

@ -1,2 +1,3 @@
-c ../../../bindings/python/constraints.txt
numpy>=1.16.6,<1.25.0
tensorflow>=1.15.5,<2.13.0
tensorflow

11
tests/conditional_compilation/requirements.txt
View File

@ -1,6 +1,7 @@
pytest>=6.2.1
pytest-dependency==0.5.1
py>=1.9.0
PyYAML>=5.4.1
-c ../constraints.txt
numpy>=1.16.6
pytest-html>=1.19.0
pytest
pytest-dependency
pytest-html
py
PyYAML

21
tests/constraints.txt Normal file
View File

@ -0,0 +1,21 @@
attrs==22.1.0
distro==1.8.0
h5py>=3.1.0
Jinja2>=2.11.2
pandas>=1.3.5
pymongo>=3.12.0
PyYAML>=5.4.1
scipy>=1.8; python_version >= '3.8'
scipy~=1.7; python_version == '3.7'
wheel>=0.38.1
defusedxml>=0.7.1
fastjsonschema~=2.15.1
test-generator==0.1.1
requests>=2.25.1
opencv-python>=4.5
py>=1.9.0
pytest==7.2.0; python_version >= '3.10'
pytest>=5.0,<=7.0.1; python_version < '3.10'
pytest-dependency==0.5.1
pytest-html==1.19.0
pytest-timeout==2.0.1

3
tests/layer_tests/requirements.txt
View File

@ -1,4 +1,5 @@
requests>=2.25.1
-c ../constraints.txt
requests
numpy>=1.19.2
torch
torchvision

18
tests/memory_tests/test_runner/requirements.txt
View File

@ -1,10 +1,10 @@
pytest>=5.0,<=7.0.1; python_version < '3.10'
pytest==7.2.0; python_version >= '3.10'
py>=1.9.0
PyYAML>=5.4.1
jsonschema==3.2.0
distro==1.8.0
-c ../../constraints.txt
pytest
py
PyYAML
pymongo
jsonschema
distro
pytest-html
pytest-timeout
numpy>=1.19.2
pymongo>=3.12.0
pytest-html>=1.19.0
pytest-timeout==2.0.1

8
tests/samples_tests/CMakeLists.txt
View File

@ -6,4 +6,10 @@ cmake_minimum_required(VERSION 3.13)
project(samples_tests)
install(DIRECTORY smoke_tests/ DESTINATION tests/smoke_tests COMPONENT tests EXCLUDE_FROM_ALL)
install(DIRECTORY smoke_tests/ DESTINATION tests/smoke_tests COMPONENT tests EXCLUDE_FROM_ALL PATTERN "requirements.txt" EXCLUDE)
install(FILES ../constraints.txt DESTINATION tests COMPONENT tests EXCLUDE_FROM_ALL)
file(READ smoke_tests/requirements.txt REQUIREMENTS_REPO)
string(REPLACE "-c ../../constraints.txt" "-c ../constraints.txt" REQUIREMENTS_TMP ${REQUIREMENTS_REPO})
file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/smoke_tests/requirements.txt ${REQUIREMENTS_TMP})
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/smoke_tests/requirements.txt DESTINATION tests/smoke_tests COMPONENT tests EXCLUDE_FROM_ALL)

3
tests/samples_tests/smoke_tests/requirements.txt
View File

@ -1,10 +1,11 @@
-c ../../constraints.txt
requests
pyyaml
wheel
test-generator
numpy
pytest
py>=1.9.0
py
scikit-build
opencv-python
progress

2
tests/samples_tests/smoke_tests/test_benchmark_app.py
View File

@ -51,10 +51,12 @@ class TestBenchmarkApp(SamplesCommonTestClass):
super().setup_class()
@pytest.mark.parametrize("param", test_data_fp32_async)
@pytest.mark.skip("Ticket: 106850")
def test_benchmark_app_sample_fp32_async(self, param):
_check_output(self, param)
@pytest.mark.parametrize("param", test_data_fp32_sync)
@pytest.mark.skip("Ticket: 106850")
def test_benchmark_app_fp32_sync(self, param):
_check_output(self, param)

18
tests/stress_tests/scripts/requirements.txt
View File

@ -1,9 +1,9 @@
pymongo>=3.12.0
Jinja2>=2.11.2
PyYAML>=5.4.1
fastjsonschema~=2.15.1
pandas>=1.3.5
h5py>=3.1.0
scipy~=1.7; python_version == '3.7'
scipy>=1.8; python_version >= '3.8'
defusedxml>=0.7.1
-c ../../constraints.txt
pymongo
Jinja2
PyYAML
fastjsonschema
pandas
h5py
scipy
defusedxml

3
tests/time_tests/scripts/requirements.txt
View File

@ -1 +1,2 @@
PyYAML>=5.4.1
-c ../../constraints.txt
PyYAML

20
tests/time_tests/test_runner/requirements.txt
View File

@ -1,11 +1,11 @@
pytest>=5.0,<=7.0.1; python_version < '3.10'
pytest==7.2.0; python_version >= '3.10'
py>=1.9.0
attrs==22.1.0
PyYAML>=5.4.1
jsonschema==3.2.0
distro==1.8.0
-c ../../constraints.txt
numpy>=1.19.2
pymongo>=3.12.0
pytest-html>=1.19.0
pytest-timeout==2.0.1
pytest
py
attrs
PyYAML
jsonschema
distro
pymongo
pytest-html
pytest-timeout

3
tools/benchmark_tool/requirements.txt
View File

@ -1,2 +1,3 @@
-c ../constraints.txt
numpy>=1.16.6
opencv-python>=4.5
opencv-python

75
tools/benchmark_tool/setup.py
View File

@ -9,17 +9,78 @@ Use this script to create a wheel with OpenVINO™ Python* tools:
$ python setup.py sdist bdist_wheel
"""
import pkg_resources
import re
from setuptools import setup, find_packages
from pathlib import Path
from typing import List
with open('README.md', 'r', encoding='utf-8') as f:
long_description = f.read()
with open('requirements.txt') as requirements_txt:
reqs = [
str(requirement)
for requirement
in pkg_resources.parse_requirements(requirements_txt)
]
def read_constraints(path: str='../constraints.txt') -> Dict[str, List[str]]:
"""
Read a constraints.txt file and return a dict
of {package_name: [required_version_1, required_version_2]}.
The dict values are a list because a package can be mentioned
multiple times, for example:
mxnet~=1.2.0; sys_platform == 'win32'
mxnet>=1.7.0; sys_platform != 'win32'
"""
constraints = {}
with open(Path(__file__).resolve().parent / path) as f:
raw_constraints = f.readlines()
for line in raw_constraints:
# skip comments
if line.startswith('#'):
continue
line = line.replace('\n', '')
# read constraints for that package
package, delimiter, constraint = re.split('(~|=|<|>|;)', line, maxsplit=1)
# if there is no entry for that package, add it
if constraints.get(package) is None:
constraints[package] = [delimiter + constraint]
# else add another entry for that package
else:
constraints[package].extend([delimiter + constraint])
return constraints
def read_requirements(path: str) -> List[str]:
"""
Read a requirements.txt file and return a list
of requirements. Three cases are supported, the
list corresponds to priority:
1. version specified in requirements.txt
2. version specified in constraints.txt
3. version unbound
"""
requirements = []
constraints = read_constraints()
with open(Path(__file__).resolve().parent / path) as f:
raw_requirements = f.readlines()
for line in raw_requirements:
# skip comments and constraints link
if line.startswith(('#', '-c')):
continue
# get rid of newlines
line = line.replace('\n', '')
# if version is specified (non-word chars present)
if re.search('\W', line):
requirements.append(line)
# else get version from constraints
else:
constraint = constraints.get(line)
# if version found in constraints.txt
if constraint:
for marker in constraint:
requirements.append(line+marker)
# else version is unbound
else:
requirements.append(line)
return requirements
setup(
name='benchmark_tool',
@ -40,6 +101,6 @@ setup(
'Operating System :: OS Independent',
],
packages=find_packages(),
install_requires=reqs,
install_requires=read_requirements('requirements.txt'),
python_requires='>=3.7',
)

24
tools/constraints.txt Normal file
View File

@ -0,0 +1,24 @@
# EXCEPTIONS
# some package versions need to be specified in respective requirements.txt
# files because the version differs between them:
# tensorflow, numpy
mxnet~=1.2.0; sys_platform == 'win32'
mxnet>=1.7.0.post2,<=1.9.1; sys_platform != 'win32'
onnx>=1.8.1,<=1.13.1
networkx<=2.8.8
pytest>=6.2.4; python_version < '3.10'
pytest==7.2.0; python_version >= '3.10'
protobuf>=3.18.1,<4.0.0
defusedxml>=0.7.1
requests>=2.25.1
fastjsonschema>=2.15.1,<2.17
coverage>=4.4.2,<=7.0.5
astroid>=2.9.0
pylint>=2.7.0
pyenchant>=3.0.0
test-generator==0.1.1
py>=1.9.0
urllib3>=1.26.4
importlib-metadata; python_version < "3.8" and sys_platform == 'win32'
openvino-telemetry>=2022.1.0
opencv-python>=4.5

10
tools/mo/requirements.txt
View File

@ -1,6 +1,6 @@
-c ../constraints.txt
numpy>=1.16.6
networkx~=2.5; python_version <= "3.6"
networkx<=2.8.8; python_version > "3.6"
defusedxml>=0.7.1
importlib-metadata; python_version < "3.8" and sys_platform == 'win32'
openvino-telemetry>=2022.1.0
networkx
defusedxml
importlib-metadata
openvino-telemetry

12
tools/mo/requirements_caffe.txt
View File

@ -1,7 +1,7 @@
networkx~=2.5; python_version <= "3.6"
networkx<=2.8.8; python_version > "3.6"
-c ../constraints.txt
numpy>=1.16.6,<1.25.0
protobuf>=3.18.1,<4.0.0
defusedxml>=0.7.1
requests>=2.25.1
fastjsonschema>=2.15.1,<2.17
networkx
protobuf
defusedxml
requests
fastjsonschema

22
tools/mo/requirements_dev.txt
View File

@ -1,11 +1,11 @@
coverage>=4.4.2,<=7.0.5
astroid>=2.9.0
pylint>=2.7.0
pyenchant>=3.0.0
test-generator==0.1.1
defusedxml>=0.5.0
requests>=2.20.0
pytest>=6.2.4; python_version < '3.10'
pytest==7.2.0; python_version >= '3.10'
py>=1.9.0
fastjsonschema>=2.15.1,<2.17
-c ../constraints.txt
coverage
astroid
pylint
pyenchant
test-generator
defusedxml
requests
pytest
py
fastjsonschema

10
tools/mo/requirements_kaldi.txt
View File

@ -1,6 +1,6 @@
networkx~=2.5; python_version <= "3.6"
networkx<=2.8.8; python_version > "3.6"
-c ../constraints.txt
numpy>=1.16.6,<1.25.0
defusedxml>=0.7.1
requests>=2.25.1
fastjsonschema>=2.15.1,<2.17
networkx
defusedxml
requests
fastjsonschema

15
tools/mo/requirements_mxnet.txt
View File

@ -1,9 +1,8 @@
mxnet~=1.2.0; sys_platform == 'win32'
mxnet>=1.7.0.post2,<=1.9.1; sys_platform != 'win32'
networkx~=2.5; python_version <= "3.6"
networkx<=2.8.8; python_version > "3.6"
-c ../constraints.txt
numpy>=1.16.6,<1.25.0
defusedxml>=0.7.1
urllib3>=1.26.4
requests>=2.25.1
fastjsonschema>=2.15.1,<2.17
mxnet
networkx
defusedxml
urllib3
requests
fastjsonschema

14
tools/mo/requirements_onnx.txt
View File

@ -1,8 +1,8 @@
onnx>=1.8.1,<=1.13.1
networkx~=2.5; python_version <= "3.6"
networkx<=2.8.8; python_version > "3.6"
-c ../constraints.txt
numpy>=1.16.6,<1.25.0
defusedxml>=0.7.1
requests>=2.25.1
fastjsonschema>=2.15.1,<2.17
protobuf>=3.18.1,<4.0.0
onnx
networkx
defusedxml
requests
fastjsonschema
protobuf

12
tools/mo/requirements_tf.txt
View File

@ -1,7 +1,7 @@
numpy>=1.16.6,<1.25.0
-c ../constraints.txt
tensorflow>=1.15.5,<2.13.0
networkx~=2.5; python_version <= "3.6"
networkx<=2.8.8; python_version > "3.6"
defusedxml>=0.7.1
requests>=2.25.1
fastjsonschema>=2.15.1,<2.17
numpy>=1.16.6,<1.25.0
networkx
defusedxml
requests
fastjsonschema

12
tools/mo/requirements_tf2.txt
View File

@ -1,7 +1,7 @@
numpy>=1.16.6,<1.25.0
-c ../constraints.txt
tensorflow>=2.5,<2.13.0
networkx~=2.5; python_version <= "3.6"
networkx<=2.8.8; python_version > "3.6"
defusedxml>=0.7.1
requests>=2.25.1
fastjsonschema>=2.15.1,<2.17
numpy>=1.16.6,<1.25.0
networkx
defusedxml
requests
fastjsonschema

80
tools/mo/setup.py
View File

@ -11,7 +11,6 @@ $ python setup.py sdist bdist_wheel
import os
import re
import sys
from pathlib import Path
from shutil import copyfile, copy
@ -19,12 +18,73 @@ from setuptools import setup, find_namespace_packages
from setuptools.command.build_py import build_py
from setuptools.command.install import install
from typing import Dict, List
prefix = 'openvino/tools/mo/'
SETUP_DIR = Path(__file__).resolve().parent / Path(prefix)
def read_text(path):
return (Path(__file__).resolve().parent / path).read_text()
def read_constraints(path: str='../constraints.txt') -> Dict[str, List[str]]:
"""
Read a constraints.txt file and return a dict
of {package_name: [required_version_1, required_version_2]}.
The dict values are a list because a package can be mentioned
multiple times, for example:
mxnet~=1.2.0; sys_platform == 'win32'
mxnet>=1.7.0; sys_platform != 'win32'
"""
constraints = {}
with open(Path(__file__).resolve().parent / path) as f:
raw_constraints = f.readlines()
for line in raw_constraints:
# skip comments
if line.startswith('#'):
continue
line = line.replace('\n', '')
# read constraints for that package
package, delimiter, constraint = re.split('(~|=|<|>|;)', line, maxsplit=1)
# if there is no entry for that package, add it
if constraints.get(package) is None:
constraints[package] = [delimiter + constraint]
# else add another entry for that package
else:
constraints[package].extend([delimiter + constraint])
return constraints
def read_requirements(path: str) -> List[str]:
"""
Read a requirements.txt file and return a list
of requirements. Three cases are supported, the
list corresponds to priority:
1. version specified in requirements.txt
2. version specified in constraints.txt
3. version unbound
"""
requirements = []
constraints = read_constraints()
with open(Path(__file__).resolve().parent / path) as f:
raw_requirements = f.readlines()
for line in raw_requirements:
# skip comments and constraints link
if line.startswith(('#', '-c')):
continue
# get rid of newlines
line = line.replace('\n', '')
# if version is specified (non-word chars present)
if re.search('\W', line):
requirements.append(line)
# else get version from constraints
else:
constraint = constraints.get(line)
# if version found in constraints.txt
if constraint:
for marker in constraint:
requirements.append(line+marker)
# else version is unbound
else:
requirements.append(line)
return requirements
# Detect all the framework specific requirements_*.txt files.
@ -91,17 +151,17 @@ setup(
'openvino.tools.mo.front.caffe': ['CustomLayersMapping.xml*']
},
extras_require={
'caffe': read_text('requirements_caffe.txt'),
'kaldi': read_text('requirements_kaldi.txt'),
'mxnet': read_text('requirements_mxnet.txt'),
'onnx': read_text('requirements_onnx.txt'),
'tensorflow': read_text('requirements_tf.txt'),
'tensorflow2': read_text('requirements_tf2.txt'),
'caffe': read_requirements('requirements_caffe.txt'),
'kaldi': read_requirements('requirements_kaldi.txt'),
'mxnet': read_requirements('requirements_mxnet.txt'),
'onnx': read_requirements('requirements_onnx.txt'),
'tensorflow': read_requirements('requirements_tf.txt'),
'tensorflow2': read_requirements('requirements_tf2.txt'),
},
classifiers=[
"Programming Language :: Python :: 3",
"License :: OSI Approved :: Apache Software License",
"Operating System :: OS Independent",
],
install_requires=read_text('requirements.txt'),
install_requires=read_requirements('requirements.txt'),
)

1
tools/openvino_dev/requirements.txt
View File

@ -0,0 +1 @@
-c ../constraints.txt

73
tools/openvino_dev/setup.py
View File

@ -12,6 +12,7 @@ import sys
import platform
import subprocess # nosec
import shutil
import re
from distutils import log
from distutils.command.build import build
from distutils.command.clean import clean
@ -20,6 +21,7 @@ from fnmatch import fnmatchcase
import pkg_resources
from setuptools.command.install import install
from setuptools import setup, find_namespace_packages
from typing import Dict, List
PYTHON_VERSION = f'python{sys.version_info.major}.{sys.version_info.minor}'
SCRIPT_DIR = Path(__file__).resolve().parents[0]
@ -198,12 +200,68 @@ def get_description(desc_file_path):
description = fstream.read()
return description
with (SCRIPT_DIR / 'requirements.txt').open() as requirements:
install_reqs = [
str(requirement)
for requirement
in pkg_resources.parse_requirements(requirements)
]
def read_constraints(path: str='../constraints.txt') -> Dict[str, List[str]]:
"""
Read a constraints.txt file and return a dict
of {package_name: [required_version_1, required_version_2]}.
The dict values are a list because a package can be mentioned
multiple times, for example:
mxnet~=1.2.0; sys_platform == 'win32'
mxnet>=1.7.0; sys_platform != 'win32'
"""
constraints = {}
with open(Path(__file__).resolve().parent / path) as f:
raw_constraints = f.readlines()
for line in raw_constraints:
# skip comments
if line.startswith('#'):
continue
line = line.replace('\n', '')
# read constraints for that package
package, delimiter, constraint = re.split('(~|=|<|>|;)', line, maxsplit=1)
# if there is no entry for that package, add it
if constraints.get(package) is None:
constraints[package] = [delimiter + constraint]
# else add another entry for that package
else:
constraints[package].extend([delimiter + constraint])
return constraints
def read_requirements(path: str) -> List[str]:
"""
Read a requirements.txt file and return a list
of requirements. Three cases are supported, the
list corresponds to priority:
1. version specified in requirements.txt
2. version specified in constraints.txt
3. version unbound
"""
requirements = []
constraints = read_constraints()
with open(Path(__file__).resolve().parent / path) as f:
raw_requirements = f.readlines()
for line in raw_requirements:
# skip comments and constraints link
if line.startswith(('#', '-c')):
continue
# get rid of newlines
line = line.replace('\n', '')
# if version is specified (non-word chars present)
if re.search('\W', line):
requirements.append(line)
# else get version from constraints
else:
constraint = constraints.get(line)
# if version found in constraints.txt
if constraint:
for marker in constraint:
requirements.append(line+marker)
# else version is unbound
else:
requirements.append(line)
return requirements
def concat_files(output_file, input_files):
@ -214,7 +272,6 @@ def concat_files(output_file, input_files):
outfile.write(content)
return output_file
description_md = SCRIPT_DIR.parents[1] / 'docs' / 'install_guides' / 'pypi-openvino-dev.md'
md_files = [description_md, SCRIPT_DIR.parents[1] / 'docs' / 'install_guides' / 'pre-release-note.md']
docs_url = 'https://docs.openvino.ai/latest/index.html'
@ -248,7 +305,7 @@ setup(
entry_points = {
'console_scripts': [],
},
install_requires=install_reqs,
install_requires=read_requirements(SCRIPT_DIR / 'requirements.txt'),
packages=find_namespace_packages(where=str(SRC_DIR)),
package_dir={'': str(SRC_DIR)},
)