IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add openvino-dev wheel build from opensource (#8174)

Browse Source 
* Add openvino-dev wheel build from opensource

* fix path to OMZ tools

* fix for working directory

* change order

* pull out requirements to requirements.txt for snyk scans

* fix patterns

* fix patterns

* fix AttributeError: : 'PosixPath' object has no attribute 'startswith'
This commit is contained in:
Sergey Lyubimtsev 2021-11-09 14:37:17 +03:00 committed by GitHub
parent 1c3d7cea83
commit 6c3e68a8d1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 299 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
tools/CMakeLists.txt
View File

@ -63,4 +63,9 @@ if(ENABLE_PYTHON)
COMPONENT python_tools_${PYTHON_VERSION})
ie_cpack(python_tools python_tools_${PYTHON_VERSION})
if(ENABLE_WHEEL)
add_subdirectory(openvino_dev)
endif()
endif()

22
tools/openvino_dev/CMakeLists.txt Normal file
View File

@ -0,0 +1,22 @@
# Copyright (C) 2018-2021 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
#
set(WHEEL_VERSION "${IE_VERSION}" CACHE STRING "Version of this release" FORCE)
set(WHEEL_BUILD "${IE_VERSION_BUILD}" CACHE STRING "Build number of this release" FORCE)
set(SETUP_PY "${CMAKE_CURRENT_SOURCE_DIR}/setup.py")
set(openvino_wheel_name "openvino_dev-${WHEEL_VERSION}-${WHEEL_BUILD}-py3-none-any.whl")
set(openvino_wheels_output_dir "${CMAKE_BINARY_DIR}/wheels")
set(openvino_wheel_path "${openvino_wheels_output_dir}/${openvino_wheel_name}")
add_custom_command(OUTPUT ${openvino_wheel_path}
COMMAND ${CMAKE_COMMAND} -E env OPENVINO_VERSION=${WHEEL_VERSION}
${PYTHON_EXECUTABLE} ${SETUP_PY} clean bdist_wheel
--dist-dir ${openvino_wheels_output_dir}
--build=${WHEEL_BUILD}
WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
COMMENT "Building Python wheel ${openvino_wheel_name}"
VERBATIM)
add_custom_target(openvino_dev_wheel ALL DEPENDS ${openvino_wheel_path})

28
tools/openvino_dev/requirements.txt Normal file
View File

@ -0,0 +1,28 @@
defusedxml>=0.7.1
scipy~=1.5.4
jstyleson~=0.0.2
numpy>=1.16.6,<1.20
addict>=2.4.0
pandas~=1.1.5
hyperopt~=0.1.2
networkx~=2.5
tqdm>=4.54.1
texttable~=1.6.3
py-cpuinfo>=7.0.0
PyYAML>=5.4.1
pillow>=8.1.2
scikit-image>=0.17.2
scikit-learn>=0.24.1
yamlloader>=0.5
shapely>=1.7.1
nibabel>=3.2.1
pydicom>=2.1.2
sentencepiece>=0.1.95
tokenizers>=0.10.1
editdistance>=0.5.3
parasail>=1.2.4
fast-ctc-decode>=0.2.5
rawpy>=0.16.0
nltk>=3.5
opencv-python==4.5.*
progress>=1.5

21
tools/openvino_dev/setup.cfg Normal file
View File

@ -0,0 +1,21 @@
[options]
setup_requires =
wheel
[options.extras_require]
[options.package_data]
* = *
[options.entry_points]
console_scripts =
[metadata]
license_files =
readme*
*LICENSE*
*license*
*third-party-programs*
*EULA*
../../licensing/*
../../LICENSE

217
tools/openvino_dev/setup.py Normal file
View File

@ -0,0 +1,217 @@
#!/usr/bin/env python3
# Copyright (C) 2018-2021 Intel Corporation SPDX-License-Identifier: Apache-2.0
""" Use this script to create a openvino-dev wheel package:
$ python3 setup.py bdist_wheel
"""
# pylint: disable-msg=line-too-long
import os
import sys
import platform
import subprocess # nosec
import shutil
from distutils import log
from distutils.command.build import build
from distutils.command.clean import clean
from pathlib import Path
from fnmatch import fnmatchcase
import pkg_resources
from setuptools.command.install import install
from setuptools import setup, find_namespace_packages
PYTHON_VERSION = f'python{sys.version_info.major}.{sys.version_info.minor}'
SCRIPT_DIR = Path(__file__).resolve().parents[0]
OPENVINO_DIR = Path(__file__).resolve().parents[2]
SRC_DIR = SCRIPT_DIR / 'src'
PKG_INSTALL_CFG = {
'openvino-mo': {
'src_dir': OPENVINO_DIR / 'model-optimizer',
'black_list': ['*unit_tests*'],
'prefix': 'mo',
'extract_entry_points': True,
'extract_requirements': True,
'extract_extras': True,
},
'benchmark_tool': {
'src_dir': OPENVINO_DIR / 'tools' / 'benchmark_tool',
'black_list': [],
'prefix': 'benchmark_tool',
'extract_entry_points': True,
},
"accuracy_checker": {
'src_dir': OPENVINO_DIR / 'tools' / 'pot' / 'thirdparty' / 'open_model_zoo' / 'tools' / 'accuracy_checker', # noqa:E501
'black_list': ['*tests*'],
'prefix': 'accuracy_checker',
'extract_entry_points': True,
},
"omz_tools": {
'src_dir': OPENVINO_DIR / 'tools' / 'pot' / 'thirdparty' / 'open_model_zoo' / 'tools' / 'model_tools', # noqa:E501
'black_list': [],
'prefix': 'omz_tools',
'extract_requirements': True,
'extract_entry_points': True,
'extract_extras': True,
},
"pot": {
'src_dir': OPENVINO_DIR / 'tools' / 'pot',
'black_list': ['*tests*'],
'prefix': 'pot',
'extract_entry_points': True,
},
}
def ignore_patterns(*patterns):
"""
Filter names by given patterns
"""
return lambda name: any(fnmatchcase(name, pat=pat) for pat in patterns)
class CustomBuild(build):
"""Custom implementation of build"""
def run(self):
# pylint: disable-msg=too-many-locals
self.announce('Installing packages', level=log.INFO)
for cmp, cmp_data in PKG_INSTALL_CFG.items():
self.announce(f'Processing package: {cmp}', level=log.INFO)
subprocess.call([sys.executable, 'setup.py', 'install',
'--root', str(SCRIPT_DIR),
'--prefix', str(cmp_data.get("prefix"))],
cwd=str(cmp_data.get('src_dir')))
# grab installed modules
lib_dir = 'lib/site-packages' if platform.system() == 'Windows' else f'lib/{PYTHON_VERSION}/site-packages'
src = SCRIPT_DIR / cmp_data.get('prefix') / lib_dir
egg_info = list(src.glob('**/*.egg-info'))
if egg_info:
def raw_req(req):
req.marker = None
return str(req)
distributions = pkg_resources.find_distributions(str(Path(egg_info[0]).parent))
for dist in distributions:
self.announce(f'Distribution: {dist.egg_name()}', level=log.INFO)
# load install_requires list
install_requires = list(sorted(map(raw_req, dist.requires())))
self.announce(f'Install requires: {install_requires}', level=log.INFO)
if cmp_data.get("extract_requirements"):
self.distribution.install_requires.extend(install_requires)
# load extras_require
if cmp_data.get("extract_extras"):
for extra in dist.extras:
if extra not in self.distribution.extras_require:
self.distribution.extras_require[extra] = []
extras_require = set(map(raw_req, dist.requires((extra,))))
self.announce(f'Extras: {extra}:{extras_require}', level=log.INFO)
self.distribution.extras_require[extra].extend(extras_require)
# extract console scripts
if cmp_data.get("extract_entry_points"):
for console_scripts in dist.get_entry_map('console_scripts'):
self.announce(f'Entry point: {console_scripts}', level=log.INFO)
entry = dist.get_entry_info('console_scripts', console_scripts)
self.distribution.entry_points['console_scripts'].append(str(entry))
# copy modules to the build directory
dst = Path(self.build_lib)
black_list = cmp_data.get('black_list')
exclude = ignore_patterns('*ez_setup*', '*__pycache__*', '*.egg-info*', *black_list)
for path in src.glob('**/*'):
if path.is_dir() or exclude(str(path)):
continue
path_rel = path.relative_to(src)
(dst / path_rel.parent).mkdir(exist_ok=True, parents=True)
shutil.copyfile(path, dst / path_rel)
# add dependecy on runtime package
runtime_req = [f'openvino=={self.distribution.get_version()}']
self.distribution.install_requires.extend(runtime_req)
self.announce(f'{self.distribution.install_requires}', level=log.DEBUG)
self.announce(f'{self.distribution.extras_require}', level=log.DEBUG)
self.announce(f'{self.distribution.entry_points}', level=log.DEBUG)
class CustomInstall(install):
"""Enable build_clib during the installation"""
def run(self):
self.run_command('build')
install.run(self)
class CustomClean(clean):
"""Clean up staging directories"""
def clean(self, install_cfg):
"""Clean components staging directories"""
for comp, comp_data in install_cfg.items():
install_prefix = comp_data.get('prefix')
self.announce(f'Cleaning {comp}: {install_prefix}', level=log.INFO)
if os.path.exists(install_prefix):
shutil.rmtree(install_prefix)
def run(self):
self.clean(PKG_INSTALL_CFG)
for pattern in './build ./dist **/*.pyc **/*.tgz **/*.egg-info'.split(' '):
paths = SCRIPT_DIR.glob(pattern)
for path in paths:
if path.is_file() and path.exists():
path = path.parent
self.announce(f'Cleaning: {path}', level=log.INFO)
shutil.rmtree(path)
clean.run(self)
def get_description(desc_file_path):
"""read description from README.md"""
with open(desc_file_path, 'r', encoding='utf-8') as fstream:
description = fstream.read()
return description
with (SCRIPT_DIR / 'requirements.txt').open() as requirements:
install_reqs = [
str(requirement)
for requirement
in pkg_resources.parse_requirements(requirements)
]
setup(
name='openvino-dev',
version=os.getenv('OPENVINO_VERSION', '0.0.0'),
author='Intel® Corporation',
license='OSI Approved :: Apache Software License',
author_email='openvino_pushbot@intel.com',
url='https://docs.openvinotoolkit.org/latest/index.html',
download_url='https://github.com/openvinotoolkit/openvino/tags',
description='OpenVINO™ Developer Package',
long_description=get_description(SCRIPT_DIR.parents[1] / 'docs/install_guides/pypi-openvino-dev.md'),
long_description_content_type='text/markdown',
classifiers=[
'Programming Language :: Python :: 3',
'OSI Approved :: Apache Software License',
'Operating System :: OS Independent',
],
cmdclass={
'build': CustomBuild,
'install': CustomInstall,
'clean': CustomClean,
},
entry_points = {
'console_scripts': [],
},
install_requires=install_reqs,
packages=find_namespace_packages(where=str(SRC_DIR)),
package_dir={'': str(SRC_DIR)},
)

3
tools/openvino_dev/src/openvino/__init__.py Normal file
View File

@ -0,0 +1,3 @@
# Copyright (C) 2020-2021 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
__path__ = __import__('pkgutil').extend_path(__path__, __name__)

3
tools/openvino_dev/src/openvino/tools/__init__.py Normal file
View File

@ -0,0 +1,3 @@
# Copyright (C) 2020-2021 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
__path__ = __import__('pkgutil').extend_path(__path__, __name__)