Removed duplicated files from OpenVINO wheel (#13725)

This commit is contained in:
Ilya Lavrenov 2022-11-02 07:58:58 +04:00 committed by GitHub
parent 592e250714
commit c40d225b6f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 240 additions and 81 deletions

View File

@ -147,6 +147,9 @@ jobs:
export BUILD_OPENCV=$(BUILD_OPENCV) export BUILD_OPENCV=$(BUILD_OPENCV)
export INSTALL_OPENVINO=$(INSTALL_OPENVINO) export INSTALL_OPENVINO=$(INSTALL_OPENVINO)
$(OPENVINO_CONTRIB_REPO_DIR)/modules/arm_plugin/scripts/install_build_dependencies.sh $(OPENVINO_CONTRIB_REPO_DIR)/modules/arm_plugin/scripts/install_build_dependencies.sh
python3 -m pip install --upgrade pip
python3 -m pip install -r $(OPENVINO_REPO_DIR)/src/bindings/python/src/compatibility/openvino/requirements.txt
python3 -m pip install -r $(OPENVINO_REPO_DIR)/src/bindings/python/wheel/requirements-dev.txt
workingDirectory: $(BUILD_OPENVINO) workingDirectory: $(BUILD_OPENVINO)
displayName: 'Install dependencies' displayName: 'Install dependencies'
@ -161,7 +164,7 @@ jobs:
-DPYTHON_INCLUDE_DIRS=$(INSTALL_PYTHON)/include/python3.8 -DPYTHON_INCLUDE_DIRS=$(INSTALL_PYTHON)/include/python3.8
-DPYTHON_LIBRARY=$(INSTALL_PYTHON)/lib/libpython3.8.so -DPYTHON_LIBRARY=$(INSTALL_PYTHON)/lib/libpython3.8.so
-DENABLE_PYTHON=ON -DENABLE_PYTHON=ON
-DPYTHON_MODULE_EXTENSION=".so" -DPYTHON_MODULE_EXTENSION=".cpython-38-x86_64-linux-gnu.so"
-DENABLE_TESTS=ON -DENABLE_TESTS=ON
-DENABLE_DATA=OFF -DENABLE_DATA=OFF
-DCMAKE_EXE_LINKER_FLAGS=-Wl,-rpath-link,$(INSTALL_OPENCV)/lib -DCMAKE_EXE_LINKER_FLAGS=-Wl,-rpath-link,$(INSTALL_OPENCV)/lib

View File

@ -93,6 +93,30 @@ if(ENABLE_TESTS)
cpack_add_component(tests HIDDEN) cpack_add_component(tests HIDDEN)
endif() endif()
#
# ov_install_with_name(<FILE> <COMPONENT>)
#
# if <FILE> is a symlink, we resolve it, but install file with a name of symlink
#
function(ov_install_with_name file component)
if((APPLE AND file MATCHES "^[^\.]+\.[0-9]+${CMAKE_SHARED_LIBRARY_SUFFIX}$") OR
(file MATCHES "^.*\.${CMAKE_SHARED_LIBRARY_SUFFIX}\.[0-9]+$"))
if(IS_SYMLINK "${file}")
get_filename_component(actual_name "${file}" NAME)
get_filename_component(file "${file}" REALPATH)
set(install_rename RENAME "${actual_name}")
endif()
install(FILES "${file}"
DESTINATION runtime/3rdparty/${component}/lib
COMPONENT ${component}
EXCLUDE_FROM_ALL
${install_rename})
set("${component}_INSTALLED" ON PARENT_SCOPE)
endif()
endfunction()
# #
# List of public OpenVINO components # List of public OpenVINO components
# #

View File

@ -87,7 +87,7 @@ macro(ov_rpm_specific_settings)
# ASL 2.0 # Apache Software License 2.0 # ASL 2.0 # Apache Software License 2.0
set(CPACK_RPM_PACKAGE_LICENSE "ASL 2.0") set(CPACK_RPM_PACKAGE_LICENSE "ASL 2.0")
# group # group
set(CPACK_RPM_PACKAGE_GROUP "Developent/Libraries") set(CPACK_RPM_PACKAGE_GROUP "Development/Libraries")
# changelog file # changelog file
# TODO: fix "error: bad date in %changelog" # TODO: fix "error: bad date in %changelog"
# set(CPACK_RPM_CHANGELOG_FILE "${OpenVINO_SOURCE_DIR}/cmake/developer_package/packaging/changelog-rpm") # set(CPACK_RPM_CHANGELOG_FILE "${OpenVINO_SOURCE_DIR}/cmake/developer_package/packaging/changelog-rpm")

View File

@ -103,7 +103,7 @@ macro(ov_parse_ci_build_number)
string(REGEX REPLACE "^20" "" OpenVINO_SOVERSION "${OpenVINO_SOVERSION}") string(REGEX REPLACE "^20" "" OpenVINO_SOVERSION "${OpenVINO_SOVERSION}")
set(OpenVINO_VERSION "${OpenVINO_VERSION_MAJOR}.${OpenVINO_VERSION_MINOR}.${OpenVINO_VERSION_PATCH}") set(OpenVINO_VERSION "${OpenVINO_VERSION_MAJOR}.${OpenVINO_VERSION_MINOR}.${OpenVINO_VERSION_PATCH}")
if(ENABLE_LIBRARY_VERSIONING) if(ENABLE_LIBRARY_VERSIONING)
set(OpenVINO_VERSION_SUFFIX ".${OpenVINO_VERSION}") set(OpenVINO_VERSION_SUFFIX ".${OpenVINO_SOVERSION}")
else() else()
set(OpenVINO_VERSION_SUFFIX "") set(OpenVINO_VERSION_SUFFIX "")
endif() endif()

View File

@ -49,6 +49,7 @@ if [ -f /etc/lsb-release ]; then
unzip \ unzip \
shellcheck \ shellcheck \
patchelf \ patchelf \
fdupes \
lintian \ lintian \
file \ file \
gzip \ gzip \

View File

@ -6,8 +6,12 @@
# Common vars used by setup.py # Common vars used by setup.py
# #
set(pyversion python${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR}) if(DEFINED ENV{CI_BUILD_DEV_TAG} AND NOT "$ENV{CI_BUILD_DEV_TAG}" STREQUAL "")
set(WHEEL_VERSION "${OpenVINO_VERSION}" CACHE STRING "Version of this release" FORCE) set(WHEEL_VERSION "${OpenVINO_VERSION}.$ENV{CI_BUILD_DEV_TAG}" CACHE STRING "Version of this release" FORCE)
set(wheel_pre_release ON)
else()
set(WHEEL_VERSION ${OpenVINO_VERSION} CACHE STRING "Version of this release" FORCE)
endif()
set(WHEEL_BUILD "${OpenVINO_VERSION_BUILD}" CACHE STRING "Build number of this release" FORCE) set(WHEEL_BUILD "${OpenVINO_VERSION_BUILD}" CACHE STRING "Build number of this release" FORCE)
set(PY_PACKAGES_DIR ${OV_CPACK_PYTHONDIR}) set(PY_PACKAGES_DIR ${OV_CPACK_PYTHONDIR})
@ -27,6 +31,30 @@ foreach(_target ie_api constants _pyngraph openvino_c pyopenvino ov_plugins ov_f
endif() endif()
endforeach() endforeach()
file(GLOB_RECURSE compat_ngraph_py_files ${OpenVINOPython_SOURCE_DIR}/src/compatibility/*.py)
file(GLOB_RECURSE openvino_py_files ${OpenVINOPython_SOURCE_DIR}/src/openvino/*.py)
list(APPEND openvino_wheel_deps
${openvino_py_files}
${compat_ngraph_py_files}
"${CMAKE_CURRENT_SOURCE_DIR}/setup.py"
"${OpenVINOPython_SOURCE_DIR}/requirements.txt"
"${OpenVINOPython_SOURCE_DIR}/wheel/readme.txt"
"${OpenVINO_SOURCE_DIR}/LICENSE"
"${OpenVINO_SOURCE_DIR}/licensing/onednn_third-party-programs.txt"
"${OpenVINO_SOURCE_DIR}/licensing/runtime-third-party-programs.txt"
"${OpenVINO_SOURCE_DIR}/licensing/tbb_third-party-programs.txt"
"${OpenVINO_SOURCE_DIR}/docs/install_guides/pypi-openvino-rt.md")
if(wheel_pre_release)
list(APPEND openvino_wheel_deps
"${OpenVINO_SOURCE_DIR}/docs/install_guides/pre-release-note.md")
endif()
#
# Define proper package name
#
execute_process(COMMAND ${PYTHON_EXECUTABLE} -c "import wheel.vendored.packaging.tags as tags ; print(f'{tags.interpreter_name()}{tags.interpreter_version()}')" execute_process(COMMAND ${PYTHON_EXECUTABLE} -c "import wheel.vendored.packaging.tags as tags ; print(f'{tags.interpreter_name()}{tags.interpreter_version()}')"
OUTPUT_VARIABLE PYTHON_TAG OUTPUT_STRIP_TRAILING_WHITESPACE) OUTPUT_VARIABLE PYTHON_TAG OUTPUT_STRIP_TRAILING_WHITESPACE)
execute_process(COMMAND ${PYTHON_EXECUTABLE} -c "import wheel.bdist_wheel ; print(f'{wheel.bdist_wheel.get_abi_tag()}')" execute_process(COMMAND ${PYTHON_EXECUTABLE} -c "import wheel.bdist_wheel ; print(f'{wheel.bdist_wheel.get_abi_tag()}')"
@ -34,9 +62,14 @@ execute_process(COMMAND ${PYTHON_EXECUTABLE} -c "import wheel.bdist_wheel ; prin
execute_process(COMMAND ${PYTHON_EXECUTABLE} -c "import wheel.vendored.packaging.tags as tags ; print(f'{next(tags._platform_tags())}')" execute_process(COMMAND ${PYTHON_EXECUTABLE} -c "import wheel.vendored.packaging.tags as tags ; print(f'{next(tags._platform_tags())}')"
OUTPUT_VARIABLE PLATFORM_TAG OUTPUT_STRIP_TRAILING_WHITESPACE) OUTPUT_VARIABLE PLATFORM_TAG OUTPUT_STRIP_TRAILING_WHITESPACE)
# defines wheel architecture part of `PLATFORM_TAG`
macro(_ov_platform_arch) macro(_ov_platform_arch)
if(AARCH64) if(AARCH64)
set(_arch "aarch64") if(APPLE)
set(_arch "arm64")
else()
set(_arch "aarch64")
endif()
elseif(ARM) elseif(ARM)
set(_arch "armvl7") set(_arch "armvl7")
elseif(X86_64) elseif(X86_64)
@ -61,7 +94,7 @@ if(APPLE AND DEFINED CMAKE_OSX_DEPLOYMENT_TARGET)
if(_arch AND _macos_min_version) if(_arch AND _macos_min_version)
set(PLATFORM_TAG "macosx_${_macos_min_version}_${_arch}") set(PLATFORM_TAG "macosx_${_macos_min_version}_${_arch}")
endif() endif()
elseif(LINUX AND OFF) elseif(LINUX)
_ov_platform_arch() _ov_platform_arch()
execute_process(COMMAND ldd --version execute_process(COMMAND ldd --version
@ -75,7 +108,12 @@ elseif(LINUX AND OFF)
# common pattern manylinux_<libc major>_<libc minor>_<arch> # common pattern manylinux_<libc major>_<libc minor>_<arch>
if(_libc_major AND _libc_minor AND _arch) if(_libc_major AND _libc_minor AND _arch)
set(PLATFORM_TAG "manylinux_${_libc_major}_${_libc_minor}_${_arch}") if(CMAKE_CROSSCOMPILING)
# TODO: think which proper tag is needed for arm / aarch64
set(PLATFORM_TAG "linux_${_arch}")
else()
set(PLATFORM_TAG "manylinux_${_libc_major}_${_libc_minor}_${_arch}")
endif()
endif() endif()
endif() endif()
@ -87,31 +125,48 @@ set(openvino_wheel_path "${openvino_wheels_output_dir}/${openvino_wheel_name}")
# create target for openvino.wheel # create target for openvino.wheel
# #
set(wheel_env ${CMAKE_COMMAND} -E env
WHEEL_VERSION=${WHEEL_VERSION}
WHEEL_BUILD=${WHEEL_BUILD}
OPENVINO_BUILD_DIR=${OpenVINO_BINARY_DIR}
OPENVINO_PYTHON_BUILD_DIR=${OpenVINOPython_BINARY_DIR}
OV_RUNTIME_LIBS_DIR=${OV_WHEEL_RUNTIMEDIR}
TBB_LIBS_DIR=${TBB_LIBS_DIR}
PUGIXML_LIBS_DIR=${PUGIXML_LIBS_DIR}
PY_PACKAGES_DIR=${PY_PACKAGES_DIR})
add_custom_command(OUTPUT ${openvino_wheel_path} add_custom_command(OUTPUT ${openvino_wheel_path}
COMMAND ${CMAKE_COMMAND} -E copy_directory "${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_CURRENT_BINARY_DIR}" COMMAND ${CMAKE_COMMAND} -E copy_directory "${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_CURRENT_BINARY_DIR}"
COMMAND ${CMAKE_COMMAND} -E copy_directory "${OpenVINO_SOURCE_DIR}/licensing" "${CMAKE_BINARY_DIR}/licensing" COMMAND ${CMAKE_COMMAND} -E copy_directory "${OpenVINO_SOURCE_DIR}/licensing" "${CMAKE_BINARY_DIR}/licensing"
COMMAND ${CMAKE_COMMAND} -E remove_directory "${CMAKE_CURRENT_BINARY_DIR}/site-packages" COMMAND ${wheel_env}
COMMAND ${CMAKE_COMMAND} -E env ${PYTHON_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/setup.py" bdist_wheel
WHEEL_VERSION=${WHEEL_VERSION}
WHEEL_BUILD=${WHEEL_BUILD}
OPENVINO_BUILD_DIR=${OpenVINO_BINARY_DIR}
OPENVINO_PYTHON_BUILD_DIR=${OpenVINOPython_BINARY_DIR}
OV_RUNTIME_LIBS_DIR=${OV_WHEEL_RUNTIMEDIR}
TBB_LIBS_DIR=${TBB_LIBS_DIR}
PUGIXML_LIBS_DIR=${PUGIXML_LIBS_DIR}
PY_PACKAGES_DIR=${PY_PACKAGES_DIR}
${PYTHON_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/setup.py" clean bdist_wheel
--dist-dir ${openvino_wheels_output_dir} --dist-dir ${openvino_wheels_output_dir}
--build-number=${WHEEL_BUILD} --build-number=${WHEEL_BUILD}
--plat-name=${PLATFORM_TAG} --plat-name=${PLATFORM_TAG}
--quiet --quiet
COMMAND ${CMAKE_COMMAND} -E remove_directory "${CMAKE_CURRENT_BINARY_DIR}/site-packages"
COMMAND ${wheel_env}
${PYTHON_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/setup.py" clean
DEPENDS ${openvino_wheel_deps} DEPENDS ${openvino_wheel_deps}
"${CMAKE_CURRENT_SOURCE_DIR}/setup.py"
WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}"
COMMENT "Building Python wheel ${openvino_wheel_name}" COMMENT "Building Python wheel ${openvino_wheel_name}"
VERBATIM) VERBATIM)
add_custom_target(ie_wheel ALL DEPENDS ${openvino_wheel_path}) set(fdupes_report ${CMAKE_CURRENT_BINARY_DIR}/fdupes_report.txt)
add_custom_command(OUTPUT "${fdupes_report}"
COMMAND ${CMAKE_COMMAND}
-D PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE}
-D WORKING_DIRECTORY=${CMAKE_CURRENT_BINARY_DIR}
-D WHEEL_VERSION=${WHEEL_VERSION}
-D PACKAGE_FILE=${openvino_wheel_path}
-D REPORT_FILE=${fdupes_report}
-P "${CMAKE_CURRENT_SOURCE_DIR}/fdupes_check.cmake"
DEPENDS "${openvino_wheel_path}"
WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}"
COMMENT "Run 'fdupes' checks for wheel ${openvino_wheel_name}"
VERBATIM)
add_custom_target(ie_wheel ALL DEPENDS ${openvino_wheel_path} ${fdupes_report})
# install # install

View File

@ -0,0 +1,54 @@
# Copyright (C) 2018-2022 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
#
foreach(var PYTHON_EXECUTABLE WORKING_DIRECTORY REPORT_FILE WHEEL_VERSION PACKAGE_FILE)
if(NOT DEFINED ${var})
message(FATAL_ERROR "Variable ${var} is not defined")
endif()
endforeach()
# find programs
find_program(fdupes_PROGRAM NAMES fdupes DOC "Path to fdupes")
if(NOT fdupes_PROGRAM)
message(WARNING "Failed to find 'fdupes' tool, use 'sudo apt-get install fdupes' to install it")
return()
endif()
# execute
get_filename_component(wheel_name "${PACKAGE_FILE}" NAME)
execute_process(COMMAND ${PYTHON_EXECUTABLE} -m wheel unpack ${PACKAGE_FILE}
WORKING_DIRECTORY ${WORKING_DIRECTORY}
OUTPUT_VARIABLE output_message
ERROR_VARIABLE error_message
RESULT_VARIABLE exit_code
OUTPUT_STRIP_TRAILING_WHITESPACE)
if(NOT exit_code EQUAL 0)
message(FATAL_ERROR "Failed to unpack wheel package")
endif()
set(WORKING_DIRECTORY "${WORKING_DIRECTORY}/openvino-${WHEEL_VERSION}")
if(NOT EXISTS "${WORKING_DIRECTORY}")
message(FATAL_ERROR "Failed to find ${WORKING_DIRECTORY}")
endif()
execute_process(COMMAND ${fdupes_PROGRAM} -f -r "${WORKING_DIRECTORY}"
OUTPUT_VARIABLE output_message
ERROR_VARIABLE error_message
RESULT_VARIABLE exit_code
OUTPUT_STRIP_TRAILING_WHITESPACE)
# remove unpacked directory
file(REMOVE_RECURSE "${WORKING_DIRECTORY}")
# write output
file(WRITE "${REPORT_FILE}" "${output_message}")
if(output_message)
message(FATAL_ERROR "${output_message}")
endif()

View File

@ -1,2 +1,3 @@
setuptools>=53.0.0 setuptools>=53.0.0
wheel>=0.36.2 wheel>=0.36.2
patchelf; sys_platform == 'linux'

View File

@ -60,32 +60,37 @@ LIB_INSTALL_CFG = {
}, },
"hetero_plugin": { "hetero_plugin": {
"name": "hetero", "name": "hetero",
"prefix": "libs.core", "prefix": "libs.hetero",
"install_dir": OV_RUNTIME_LIBS_DIR, "install_dir": OV_RUNTIME_LIBS_DIR,
"rpath": LIBS_RPATH,
"binary_dir": OPENVINO_BUILD_DIR, "binary_dir": OPENVINO_BUILD_DIR,
}, },
"gpu_plugin": { "gpu_plugin": {
"name": "gpu", "name": "gpu",
"prefix": "libs.core", "prefix": "libs.gpu",
"install_dir": OV_RUNTIME_LIBS_DIR, "install_dir": OV_RUNTIME_LIBS_DIR,
"rpath": LIBS_RPATH,
"binary_dir": OPENVINO_BUILD_DIR, "binary_dir": OPENVINO_BUILD_DIR,
}, },
"cpu_plugin": { "cpu_plugin": {
"name": "cpu", "name": "cpu",
"prefix": "libs.core", "prefix": "libs.cpu",
"install_dir": OV_RUNTIME_LIBS_DIR, "install_dir": OV_RUNTIME_LIBS_DIR,
"rpath": LIBS_RPATH,
"binary_dir": OPENVINO_BUILD_DIR, "binary_dir": OPENVINO_BUILD_DIR,
}, },
"multi_plugin": { "multi_plugin": {
"name": "multi", "name": "multi",
"prefix": "libs.core", "prefix": "libs.multi",
"install_dir": OV_RUNTIME_LIBS_DIR, "install_dir": OV_RUNTIME_LIBS_DIR,
"rpath": LIBS_RPATH,
"binary_dir": OPENVINO_BUILD_DIR, "binary_dir": OPENVINO_BUILD_DIR,
}, },
"batch_plugin": { "batch_plugin": {
"name": "batch", "name": "batch",
"prefix": "libs.core", "prefix": "libs.batch",
"install_dir": OV_RUNTIME_LIBS_DIR, "install_dir": OV_RUNTIME_LIBS_DIR,
"rpath": LIBS_RPATH,
"binary_dir": OPENVINO_BUILD_DIR, "binary_dir": OPENVINO_BUILD_DIR,
}, },
"tbb_libs": { "tbb_libs": {
@ -102,20 +107,23 @@ LIB_INSTALL_CFG = {
}, },
"ir_libs": { "ir_libs": {
"name": "ir", "name": "ir",
"prefix": "libs.core", "prefix": "libs.ir",
"install_dir": OV_RUNTIME_LIBS_DIR, "install_dir": OV_RUNTIME_LIBS_DIR,
"rpath": LIBS_RPATH,
"binary_dir": OPENVINO_BUILD_DIR, "binary_dir": OPENVINO_BUILD_DIR,
}, },
"paddle_libs": { "paddle_libs": {
"name": "paddle", "name": "paddle",
"prefix": "libs.core", "prefix": "libs.paddle",
"install_dir": OV_RUNTIME_LIBS_DIR, "install_dir": OV_RUNTIME_LIBS_DIR,
"rpath": LIBS_RPATH,
"binary_dir": OPENVINO_BUILD_DIR, "binary_dir": OPENVINO_BUILD_DIR,
}, },
"onnx_libs": { "onnx_libs": {
"name": "onnx", "name": "onnx",
"prefix": "libs.core", "prefix": "libs.onnx",
"install_dir": OV_RUNTIME_LIBS_DIR, "install_dir": OV_RUNTIME_LIBS_DIR,
"rpath": LIBS_RPATH,
"binary_dir": OPENVINO_BUILD_DIR, "binary_dir": OPENVINO_BUILD_DIR,
}, },
# uncomment once TF FE will be used in MO # uncomment once TF FE will be used in MO
@ -206,6 +214,7 @@ class CustomBuild(build):
self.jobs = multiprocessing.cpu_count() self.jobs = multiprocessing.cpu_count()
plat_specifier = ".{0}-{1}.{2}".format(self.plat_name, *sys.version_info[:2]) plat_specifier = ".{0}-{1}.{2}".format(self.plat_name, *sys.version_info[:2])
self.build_temp = os.path.join(self.build_base, "temp" + plat_specifier, self.config) self.build_temp = os.path.join(self.build_base, "temp" + plat_specifier, self.config)
self.announce(f"Create build directory: {self.build_temp}", level=3)
# if setup.py is directly called use CMake to build product # if setup.py is directly called use CMake to build product
if OPENVINO_BUILD_DIR == ".": if OPENVINO_BUILD_DIR == ".":
@ -215,7 +224,11 @@ class CustomBuild(build):
self.spawn(["cmake", "-S" + str(openvino_root_dir), self.spawn(["cmake", "-S" + str(openvino_root_dir),
"-B" + self.build_temp, "-B" + self.build_temp,
"-DCMAKE_BUILD_TYPE={type}".format(type=self.config), "-DCMAKE_BUILD_TYPE={type}".format(type=self.config),
"-DENABLE_PYTHON=ON"]) "-DENABLE_PYTHON=ON",
"-DENABLE_NCC_STYLE=OFF",
"-DENABLE_CPPLINT=OFF",
"-DENABLE_TEMPLATE=OFF",
"-DENABLE_SAMPLES=OFF"])
self.announce("Building binaries", level=3) self.announce("Building binaries", level=3)
self.spawn(["cmake", "--build", self.build_temp, self.spawn(["cmake", "--build", self.build_temp,
@ -243,7 +256,7 @@ class PrepareLibs(build_clib):
def run(self): def run(self):
self.configure(LIB_INSTALL_CFG) self.configure(LIB_INSTALL_CFG)
self.configure(PY_INSTALL_CFG) self.configure(PY_INSTALL_CFG)
self.generate_package(get_dir_list(LIB_INSTALL_CFG)) self.generate_package(get_install_dirs_list(LIB_INSTALL_CFG))
def configure(self, install_cfg): def configure(self, install_cfg):
"""Collect prebuilt libraries. Install them to the temp directories, set rpath.""" """Collect prebuilt libraries. Install them to the temp directories, set rpath."""
@ -252,17 +265,18 @@ class PrepareLibs(build_clib):
install_dir = comp_data.get("install_dir") install_dir = comp_data.get("install_dir")
binary_dir = comp_data.get("binary_dir") binary_dir = comp_data.get("binary_dir")
if install_dir and not os.path.isabs(install_dir): if install_dir and not os.path.isabs(install_dir):
install_dir = os.path.join(install_prefix, install_dir)
self.announce(f"Installing {comp}", level=3) self.announce(f"Installing {comp}", level=3)
self.spawn(["cmake", "--install", binary_dir, self.spawn(["cmake", "--install", binary_dir,
"--prefix", install_prefix, "--prefix", install_prefix,
"--config", "Release", "--config", "Release",
"--strip", "--strip",
"--component", comp_data.get("name")]) "--component", comp_data.get("name")])
install_dir = os.path.join(install_prefix, install_dir)
# set rpath if applicable # set rpath if applicable
if sys.platform != "win32" and comp_data.get("rpath"): if sys.platform != "win32" and comp_data.get("rpath"):
for path in filter( for path in filter(
lambda x: any(item in ([".so"] if sys.platform == "linux" else [".dylib", ".so"]) for item in x.suffixes), Path(install_dir).glob("*"), lambda x: any(item in ([".so"] if sys.platform == "linux" else [".dylib", ".so"])
for item in x.suffixes), Path(install_dir).glob("*"),
): ):
set_rpath(comp_data["rpath"], os.path.realpath(path)) set_rpath(comp_data["rpath"], os.path.realpath(path))
@ -274,12 +288,38 @@ class PrepareLibs(build_clib):
for src_dir in src_dirs: for src_dir in src_dirs:
local_base_dir = Path(src_dir) local_base_dir = Path(src_dir)
# skip symlinks of higher level like libX.so or libX.dylib
for symlink in local_base_dir.rglob("*"):
if symlink.is_symlink():
file_name = os.readlink(symlink)
if not os.path.isabs(file_name):
file_name = os.path.join(os.path.dirname(symlink), file_name)
if Path(file_name).is_symlink():
self.announce(f"Unlink symlink {symlink}, use {file_name} instead", level=3)
os.unlink(symlink)
# transform libX.so.Y / libX.Y.dylib symlinks to real files
for symlink in local_base_dir.rglob("*"):
if symlink.is_symlink():
file_name = os.readlink(symlink)
if not os.path.isabs(file_name):
file_name = os.path.join(os.path.dirname(symlink), file_name)
os.unlink(symlink)
os.rename(file_name, symlink)
self.announce(f"Resolved symlink {symlink} as {file_name}", level=3)
# copy so / dylib files to WHEEL_LIBS_INSTALL_DIR
for file_path in local_base_dir.rglob("*"): for file_path in local_base_dir.rglob("*"):
file_name = os.path.basename(file_path) file_name = os.path.basename(file_path)
if file_path.is_symlink():
sys.exit(f"Wheel package content must not contain symlinks {file_path}")
if file_path.is_file() and not any(file_name.endswith(ext) for ext in blacklist): if file_path.is_file() and not any(file_name.endswith(ext) for ext in blacklist):
dst_file = os.path.join(package_dir, os.path.relpath(file_path, local_base_dir)) dst_file = os.path.join(package_dir, os.path.relpath(file_path, local_base_dir))
os.makedirs(os.path.dirname(dst_file), exist_ok=True) os.makedirs(os.path.dirname(dst_file), exist_ok=True)
copyfile(file_path, dst_file) copyfile(file_path, dst_file)
self.announce(f"Copy {file_path} to {dst_file}", level=3)
if Path(package_dir).exists(): if Path(package_dir).exists():
self.announce(f"Adding {WHEEL_LIBS_PACKAGE} package", level=3) self.announce(f"Adding {WHEEL_LIBS_PACKAGE} package", level=3)
@ -294,7 +334,7 @@ class CopyExt(build_ext):
if len(self.extensions) == 1: if len(self.extensions) == 1:
self.run_command("build_clib") self.run_command("build_clib")
self.extensions = [] self.extensions = []
self.extensions = find_prebuilt_extensions(get_dir_list(PY_INSTALL_CFG)) self.extensions = find_prebuilt_extensions(get_install_dirs_list(PY_INSTALL_CFG))
for extension in self.extensions: for extension in self.extensions:
if not isinstance(extension, PrebuiltExtension): if not isinstance(extension, PrebuiltExtension):
raise DistutilsSetupError(f"copy_ext can accept PrebuiltExtension only, but got {extension.name}") raise DistutilsSetupError(f"copy_ext can accept PrebuiltExtension only, but got {extension.name}")
@ -304,11 +344,9 @@ class CopyExt(build_ext):
# setting relative path to find dlls # setting relative path to find dlls
if sys.platform != "win32": if sys.platform != "win32":
rpath = os.path.relpath(get_package_dir(PY_INSTALL_CFG), os.path.dirname(src)) rpath = os.path.relpath(get_package_dir(PY_INSTALL_CFG), os.path.dirname(src))
if sys.platform == "linux": rpath = os.path.join(LIBS_RPATH, rpath, WHEEL_LIBS_INSTALL_DIR)
rpath = os.path.join("$ORIGIN", rpath, WHEEL_LIBS_INSTALL_DIR)
elif sys.platform == "darwin":
rpath = os.path.join("@loader_path", rpath, WHEEL_LIBS_INSTALL_DIR)
set_rpath(rpath, os.path.realpath(src)) set_rpath(rpath, os.path.realpath(src))
copy_file(src, dst, verbose=self.verbose, dry_run=self.dry_run) copy_file(src, dst, verbose=self.verbose, dry_run=self.dry_run)
@ -413,7 +451,7 @@ def find_prebuilt_extensions(search_dirs):
ext_pattern = "**/*.so" ext_pattern = "**/*.so"
for base_dir in search_dirs: for base_dir in search_dirs:
for path in Path(base_dir).glob(ext_pattern): for path in Path(base_dir).glob(ext_pattern):
if path.match("openvino/libs/*"): if path.match("openvino/libs/*") or path.match(f"openvino/libs/openvino-{OPENVINO_VERSION}/*"):
continue continue
relpath = path.relative_to(base_dir) relpath = path.relative_to(base_dir)
if relpath.parent != ".": if relpath.parent != ".":
@ -442,7 +480,7 @@ def get_dependencies(requirements_file_path):
return dependencies return dependencies
def get_dir_list(install_cfg): def get_install_dirs_list(install_cfg):
"""Collect all available directories with libs or python packages.""" """Collect all available directories with libs or python packages."""
dirs = [] dirs = []
for comp_info in install_cfg.values(): for comp_info in install_cfg.values():
@ -459,7 +497,7 @@ def get_dir_list(install_cfg):
def get_package_dir(install_cfg): def get_package_dir(install_cfg):
"""Get python package path based on config. All the packages should be located in one directory.""" """Get python package path based on config. All the packages should be located in one directory."""
py_package_path = "" py_package_path = ""
dirs = get_dir_list(install_cfg) dirs = get_install_dirs_list(install_cfg)
if len(dirs) != 0: if len(dirs) != 0:
# setup.py support only one package directory, all modules should be located there # setup.py support only one package directory, all modules should be located there
py_package_path = dirs[0] py_package_path = dirs[0]
@ -467,6 +505,7 @@ def get_package_dir(install_cfg):
def concat_files(output_file, input_files): def concat_files(output_file, input_files):
"""Concatenates multuple input files to a single output file."""
with open(output_file, "w", encoding="utf-8") as outfile: with open(output_file, "w", encoding="utf-8") as outfile:
for filename in input_files: for filename in input_files:
with open(filename, "r", encoding="utf-8") as infile: with open(filename, "r", encoding="utf-8") as infile:
@ -479,6 +518,7 @@ platforms = ["linux", "win32", "darwin"]
if not any(pl in sys.platform for pl in platforms): if not any(pl in sys.platform for pl in platforms):
sys.exit(f"Unsupported platform: {sys.platform}, expected: linux, win32, darwin") sys.exit(f"Unsupported platform: {sys.platform}, expected: linux, win32, darwin")
OPENVINO_VERSION = WHEEL_VERSION = os.getenv("WHEEL_VERSION", "0.0.0")
# copy license file into the build directory # copy license file into the build directory
package_license = os.getenv("WHEEL_LICENSE", SCRIPT_DIR.parents[3] / "LICENSE") package_license = os.getenv("WHEEL_LICENSE", SCRIPT_DIR.parents[3] / "LICENSE")
if os.path.exists(package_license): if os.path.exists(package_license):
@ -487,7 +527,7 @@ if os.path.exists(package_license):
packages = find_namespace_packages(get_package_dir(PY_INSTALL_CFG)) packages = find_namespace_packages(get_package_dir(PY_INSTALL_CFG))
package_data: typing.Dict[str, list] = {} package_data: typing.Dict[str, list] = {}
pkg_name = os.getenv("WHEEL_PACKAGE_NAME", "openvino") pkg_name = os.getenv("WHEEL_PACKAGE_NAME", "openvino")
ext_modules = find_prebuilt_extensions(get_dir_list(PY_INSTALL_CFG)) if pkg_name == "openvino" else [] ext_modules = find_prebuilt_extensions(get_install_dirs_list(PY_INSTALL_CFG)) if pkg_name == "openvino" else []
description_md = SCRIPT_DIR.parents[3] / "docs" / "install_guides" / "pypi-openvino-rt.md" description_md = SCRIPT_DIR.parents[3] / "docs" / "install_guides" / "pypi-openvino-rt.md"
md_files = [description_md, SCRIPT_DIR.parents[3] / "docs" / "install_guides" / "pre-release-note.md"] md_files = [description_md, SCRIPT_DIR.parents[3] / "docs" / "install_guides" / "pre-release-note.md"]
@ -498,10 +538,10 @@ if (os.getenv("CI_BUILD_DEV_TAG")):
output.parent.mkdir(exist_ok=True) output.parent.mkdir(exist_ok=True)
description_md = concat_files(output, md_files) description_md = concat_files(output, md_files)
docs_url = "https://docs.openvino.ai/nightly/index.html" docs_url = "https://docs.openvino.ai/nightly/index.html"
OPENVINO_VERSION = WHEEL_VERSION[0:8]
setup( setup(
version=os.getenv("WHEEL_VERSION", "0.0.0"), version=WHEEL_VERSION,
build=os.getenv("WHEEL_BUILD", "000"), build=os.getenv("WHEEL_BUILD", "000"),
author_email=os.getenv("WHEEL_AUTHOR_EMAIL", "openvino_pushbot@intel.com"), author_email=os.getenv("WHEEL_AUTHOR_EMAIL", "openvino_pushbot@intel.com"),
name=pkg_name, name=pkg_name,

View File

@ -109,21 +109,18 @@ if(THREADING MATCHES "^(TBB|TBB_AUTO)$" AND
# depending on the TBB, tbb_lib_location can be in form: # depending on the TBB, tbb_lib_location can be in form:
# - libtbb.so.x.y # - libtbb.so.x.y
# - libtbb.so.x # - libtbb.so.x
# We need to install such files # We need to install such only libtbb.so.x files
get_filename_component(name_we "${tbb_lib_location}" NAME_WE) get_filename_component(name_we "${tbb_lib_location}" NAME_WE)
get_filename_component(dir "${tbb_lib_location}" DIRECTORY) get_filename_component(dir "${tbb_lib_location}" DIRECTORY)
# grab all tbb files matching pattern # grab all tbb files matching pattern
file(GLOB tbb_files "${dir}/${name_we}.*") file(GLOB tbb_files "${dir}/${name_we}.*")
# since the setup.py for pip installs tbb component
# explicitly, it's OK to put EXCLUDE_FROM_ALL to such component
# to ignore from IRC / apt / yum distribution;
# but they will be present in .wheel
foreach(tbb_file IN LISTS tbb_files) foreach(tbb_file IN LISTS tbb_files)
if(tbb_file MATCHES "^.*\.${CMAKE_SHARED_LIBRARY_SUFFIX}(\.[0-9]+)*$") ov_install_with_name("${tbb_file}" tbb)
# since the setup.py for pip installs tbb component
# explicitly, it's OK to put EXCLUDE_FROM_ALL to such component
# to ignore from IRC / apt / yum / brew distribution;
# but they will be present in .wheel
install(FILES "${tbb_file}"
DESTINATION runtime/3rdparty/tbb/lib
COMPONENT tbb EXCLUDE_FROM_ALL)
endif()
endforeach() endforeach()
endforeach() endforeach()

View File

@ -104,28 +104,18 @@ if(ENABLE_SYSTEM_PUGIXML)
# grab all tbb files matching pattern # grab all tbb files matching pattern
file(GLOB pugixml_files "${pugixml_dir}/${name_we}.*") file(GLOB pugixml_files "${pugixml_dir}/${name_we}.*")
foreach(pugixml_file IN LISTS pugixml_files) foreach(pugixml_file IN LISTS pugixml_files)
if(pugixml_file MATCHES "^.*\.${CMAKE_SHARED_LIBRARY_SUFFIX}(\.[0-9]+)*$") ov_install_with_name("${pugixml_file}" pugixml)
list(APPEND pugixml_libs "${pugixml_file}")
endif()
endforeach() endforeach()
elseif(target_type STREQUAL "INTERFACE_LIBRARY") elseif(target_type STREQUAL "INTERFACE_LIBRARY")
get_target_property(pugixml_loc ${pugixml_target} INTERFACE_LINK_LIBRARIES) get_target_property(pugixml_loc ${pugixml_target} INTERFACE_LINK_LIBRARIES)
file(GLOB pugixml_libs_all "${pugixml_loc}.*") file(GLOB pugixml_libs "${pugixml_loc}.*")
foreach(pugixml_lib IN LISTS pugixml_libs_all) foreach(pugixml_lib IN LISTS pugixml_libs)
if(pugixml_lib MATCHES "^${pugixml_loc}(\.[0-9]+)+$") ov_install_with_name("${pugixml_lib}" pugixml)
list(APPEND pugixml_libs "${pugixml_lib}")
endif()
endforeach() endforeach()
endif() endif()
# if dynamic libpugixml.so.1 and libpugixml.so.1.X are found # if dynamic libpugixml.so.1 and libpugixml.so.1.X are found
if(pugixml_libs) if(NOT pugixml_INSTALLED AND CPACK_GENERATOR MATCHES "^(DEB|RPM)$")
ie_cpack_add_component(pugixml HIDDEN)
install(FILES ${pugixml_libs}
DESTINATION runtime/3rdparty/pugixml/lib
COMPONENT pugixml
EXCLUDE_FROM_ALL)
elseif(CPACK_GENERATOR MATCHES "^(DEB|RPM)$")
message(FATAL_ERROR "Debian | RPM package build requires shared Pugixml library") message(FATAL_ERROR "Debian | RPM package build requires shared Pugixml library")
endif() endif()

View File

@ -28,12 +28,6 @@ install(DIRECTORY deployment_manager
ie_cpack_add_component(${OV_CPACK_COMP_DEV_REQ_FILES} HIDDEN) ie_cpack_add_component(${OV_CPACK_COMP_DEV_REQ_FILES} HIDDEN)
if(DEFINED ENV{CI_BUILD_DEV_TAG} AND NOT $ENV{CI_BUILD_DEV_TAG} STREQUAL "")
set(WHEEL_VERSION "${OpenVINO_VERSION}.$ENV{CI_BUILD_DEV_TAG}")
else()
set(WHEEL_VERSION ${OpenVINO_VERSION})
endif()
set(REQUIREMENTS_IN "${CMAKE_CURRENT_SOURCE_DIR}/requirements_dev.in") set(REQUIREMENTS_IN "${CMAKE_CURRENT_SOURCE_DIR}/requirements_dev.in")
set(EXTRAS_LIST _ caffe kaldi mxnet onnx pytorch tensorflow tensorflow2) set(EXTRAS_LIST _ caffe kaldi mxnet onnx pytorch tensorflow tensorflow2)

View File

@ -2,8 +2,6 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# #
set(WHEEL_VERSION "${OpenVINO_VERSION}" CACHE STRING "Version of this release" FORCE)
set(WHEEL_BUILD "${OpenVINO_VERSION_BUILD}" CACHE STRING "Build number of this release" FORCE)
set(SETUP_PY "${CMAKE_CURRENT_SOURCE_DIR}/setup.py") set(SETUP_PY "${CMAKE_CURRENT_SOURCE_DIR}/setup.py")
set(openvino_wheel_name "openvino_dev-${WHEEL_VERSION}-${WHEEL_BUILD}-py3-none-any.whl") set(openvino_wheel_name "openvino_dev-${WHEEL_VERSION}-${WHEEL_BUILD}-py3-none-any.whl")
@ -13,10 +11,12 @@ set(openvino_wheel_path "${openvino_wheels_output_dir}/${openvino_wheel_name}")
add_custom_command(OUTPUT ${openvino_wheel_path} add_custom_command(OUTPUT ${openvino_wheel_path}
COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/setup.cfg" "${CMAKE_CURRENT_BINARY_DIR}" COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/setup.cfg" "${CMAKE_CURRENT_BINARY_DIR}"
COMMAND ${CMAKE_COMMAND} -E env OPENVINO_VERSION=${WHEEL_VERSION} COMMAND ${CMAKE_COMMAND} -E env OPENVINO_VERSION=${WHEEL_VERSION}
${PYTHON_EXECUTABLE} ${SETUP_PY} clean bdist_wheel ${PYTHON_EXECUTABLE} ${SETUP_PY} bdist_wheel
--dist-dir ${openvino_wheels_output_dir} --dist-dir ${openvino_wheels_output_dir}
--build=${WHEEL_BUILD} --build=${WHEEL_BUILD}
--quiet --quiet
COMMAND ${CMAKE_COMMAND} -E env OPENVINO_VERSION=${WHEEL_VERSION}
${PYTHON_EXECUTABLE} ${SETUP_PY} clean
DEPENDS ie_wheel DEPENDS ie_wheel
WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}"
COMMENT "Building Python wheel ${openvino_wheel_name}" COMMENT "Building Python wheel ${openvino_wheel_name}"