From c40d225b6f96d16d9b60ac2aba70cac08c7bacd1 Mon Sep 17 00:00:00 2001 From: Ilya Lavrenov Date: Wed, 2 Nov 2022 07:58:58 +0400 Subject: [PATCH] Removed duplicated files from OpenVINO wheel (#13725) --- .ci/azure/linux_arm64.yml | 5 +- .../packaging/packaging.cmake | 24 +++++ cmake/developer_package/packaging/rpm.cmake | 2 +- cmake/developer_package/version.cmake | 2 +- install_build_dependencies.sh | 1 + src/bindings/python/wheel/CMakeLists.txt | 91 +++++++++++++++---- src/bindings/python/wheel/fdupes_check.cmake | 54 +++++++++++ .../python/wheel/requirements-dev.txt | 1 + src/bindings/python/wheel/setup.py | 86 +++++++++++++----- src/cmake/install_tbb.cmake | 17 ++-- thirdparty/CMakeLists.txt | 20 +--- tools/CMakeLists.txt | 6 -- tools/openvino_dev/CMakeLists.txt | 12 +-- 13 files changed, 240 insertions(+), 81 deletions(-) create mode 100644 src/bindings/python/wheel/fdupes_check.cmake diff --git a/.ci/azure/linux_arm64.yml b/.ci/azure/linux_arm64.yml index 992bfa0918b..3026bd0e53d 100644 --- a/.ci/azure/linux_arm64.yml +++ b/.ci/azure/linux_arm64.yml @@ -147,6 +147,9 @@ jobs: export BUILD_OPENCV=$(BUILD_OPENCV) export INSTALL_OPENVINO=$(INSTALL_OPENVINO) $(OPENVINO_CONTRIB_REPO_DIR)/modules/arm_plugin/scripts/install_build_dependencies.sh + python3 -m pip install --upgrade pip + python3 -m pip install -r $(OPENVINO_REPO_DIR)/src/bindings/python/src/compatibility/openvino/requirements.txt + python3 -m pip install -r $(OPENVINO_REPO_DIR)/src/bindings/python/wheel/requirements-dev.txt workingDirectory: $(BUILD_OPENVINO) displayName: 'Install dependencies' @@ -161,7 +164,7 @@ jobs: -DPYTHON_INCLUDE_DIRS=$(INSTALL_PYTHON)/include/python3.8 -DPYTHON_LIBRARY=$(INSTALL_PYTHON)/lib/libpython3.8.so -DENABLE_PYTHON=ON - -DPYTHON_MODULE_EXTENSION=".so" + -DPYTHON_MODULE_EXTENSION=".cpython-38-x86_64-linux-gnu.so" -DENABLE_TESTS=ON -DENABLE_DATA=OFF -DCMAKE_EXE_LINKER_FLAGS=-Wl,-rpath-link,$(INSTALL_OPENCV)/lib diff --git a/cmake/developer_package/packaging/packaging.cmake b/cmake/developer_package/packaging/packaging.cmake index d22a38986b4..3af4ac6e8c6 100644 --- a/cmake/developer_package/packaging/packaging.cmake +++ b/cmake/developer_package/packaging/packaging.cmake @@ -93,6 +93,30 @@ if(ENABLE_TESTS) cpack_add_component(tests HIDDEN) endif() +# +# ov_install_with_name( ) +# +# if is a symlink, we resolve it, but install file with a name of symlink +# +function(ov_install_with_name file component) + if((APPLE AND file MATCHES "^[^\.]+\.[0-9]+${CMAKE_SHARED_LIBRARY_SUFFIX}$") OR + (file MATCHES "^.*\.${CMAKE_SHARED_LIBRARY_SUFFIX}\.[0-9]+$")) + if(IS_SYMLINK "${file}") + get_filename_component(actual_name "${file}" NAME) + get_filename_component(file "${file}" REALPATH) + set(install_rename RENAME "${actual_name}") + endif() + + install(FILES "${file}" + DESTINATION runtime/3rdparty/${component}/lib + COMPONENT ${component} + EXCLUDE_FROM_ALL + ${install_rename}) + + set("${component}_INSTALLED" ON PARENT_SCOPE) + endif() +endfunction() + # # List of public OpenVINO components # diff --git a/cmake/developer_package/packaging/rpm.cmake b/cmake/developer_package/packaging/rpm.cmake index 16ead4e872f..c7a57f29d2b 100644 --- a/cmake/developer_package/packaging/rpm.cmake +++ b/cmake/developer_package/packaging/rpm.cmake @@ -87,7 +87,7 @@ macro(ov_rpm_specific_settings) # ASL 2.0 # Apache Software License 2.0 set(CPACK_RPM_PACKAGE_LICENSE "ASL 2.0") # group - set(CPACK_RPM_PACKAGE_GROUP "Developent/Libraries") + set(CPACK_RPM_PACKAGE_GROUP "Development/Libraries") # changelog file # TODO: fix "error: bad date in %changelog" # set(CPACK_RPM_CHANGELOG_FILE "${OpenVINO_SOURCE_DIR}/cmake/developer_package/packaging/changelog-rpm") diff --git a/cmake/developer_package/version.cmake b/cmake/developer_package/version.cmake index 6d79fbbf03c..c6fb129bc8c 100644 --- a/cmake/developer_package/version.cmake +++ b/cmake/developer_package/version.cmake @@ -103,7 +103,7 @@ macro(ov_parse_ci_build_number) string(REGEX REPLACE "^20" "" OpenVINO_SOVERSION "${OpenVINO_SOVERSION}") set(OpenVINO_VERSION "${OpenVINO_VERSION_MAJOR}.${OpenVINO_VERSION_MINOR}.${OpenVINO_VERSION_PATCH}") if(ENABLE_LIBRARY_VERSIONING) - set(OpenVINO_VERSION_SUFFIX ".${OpenVINO_VERSION}") + set(OpenVINO_VERSION_SUFFIX ".${OpenVINO_SOVERSION}") else() set(OpenVINO_VERSION_SUFFIX "") endif() diff --git a/install_build_dependencies.sh b/install_build_dependencies.sh index 7e3138a1efa..8d81c82dd53 100755 --- a/install_build_dependencies.sh +++ b/install_build_dependencies.sh @@ -49,6 +49,7 @@ if [ -f /etc/lsb-release ]; then unzip \ shellcheck \ patchelf \ + fdupes \ lintian \ file \ gzip \ diff --git a/src/bindings/python/wheel/CMakeLists.txt b/src/bindings/python/wheel/CMakeLists.txt index 096c0a52c3d..9d4e385393d 100644 --- a/src/bindings/python/wheel/CMakeLists.txt +++ b/src/bindings/python/wheel/CMakeLists.txt @@ -6,8 +6,12 @@ # Common vars used by setup.py # -set(pyversion python${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR}) -set(WHEEL_VERSION "${OpenVINO_VERSION}" CACHE STRING "Version of this release" FORCE) +if(DEFINED ENV{CI_BUILD_DEV_TAG} AND NOT "$ENV{CI_BUILD_DEV_TAG}" STREQUAL "") + set(WHEEL_VERSION "${OpenVINO_VERSION}.$ENV{CI_BUILD_DEV_TAG}" CACHE STRING "Version of this release" FORCE) + set(wheel_pre_release ON) +else() + set(WHEEL_VERSION ${OpenVINO_VERSION} CACHE STRING "Version of this release" FORCE) +endif() set(WHEEL_BUILD "${OpenVINO_VERSION_BUILD}" CACHE STRING "Build number of this release" FORCE) set(PY_PACKAGES_DIR ${OV_CPACK_PYTHONDIR}) @@ -27,6 +31,30 @@ foreach(_target ie_api constants _pyngraph openvino_c pyopenvino ov_plugins ov_f endif() endforeach() +file(GLOB_RECURSE compat_ngraph_py_files ${OpenVINOPython_SOURCE_DIR}/src/compatibility/*.py) +file(GLOB_RECURSE openvino_py_files ${OpenVINOPython_SOURCE_DIR}/src/openvino/*.py) + +list(APPEND openvino_wheel_deps + ${openvino_py_files} + ${compat_ngraph_py_files} + "${CMAKE_CURRENT_SOURCE_DIR}/setup.py" + "${OpenVINOPython_SOURCE_DIR}/requirements.txt" + "${OpenVINOPython_SOURCE_DIR}/wheel/readme.txt" + "${OpenVINO_SOURCE_DIR}/LICENSE" + "${OpenVINO_SOURCE_DIR}/licensing/onednn_third-party-programs.txt" + "${OpenVINO_SOURCE_DIR}/licensing/runtime-third-party-programs.txt" + "${OpenVINO_SOURCE_DIR}/licensing/tbb_third-party-programs.txt" + "${OpenVINO_SOURCE_DIR}/docs/install_guides/pypi-openvino-rt.md") + +if(wheel_pre_release) + list(APPEND openvino_wheel_deps + "${OpenVINO_SOURCE_DIR}/docs/install_guides/pre-release-note.md") +endif() + +# +# Define proper package name +# + execute_process(COMMAND ${PYTHON_EXECUTABLE} -c "import wheel.vendored.packaging.tags as tags ; print(f'{tags.interpreter_name()}{tags.interpreter_version()}')" OUTPUT_VARIABLE PYTHON_TAG OUTPUT_STRIP_TRAILING_WHITESPACE) execute_process(COMMAND ${PYTHON_EXECUTABLE} -c "import wheel.bdist_wheel ; print(f'{wheel.bdist_wheel.get_abi_tag()}')" @@ -34,9 +62,14 @@ execute_process(COMMAND ${PYTHON_EXECUTABLE} -c "import wheel.bdist_wheel ; prin execute_process(COMMAND ${PYTHON_EXECUTABLE} -c "import wheel.vendored.packaging.tags as tags ; print(f'{next(tags._platform_tags())}')" OUTPUT_VARIABLE PLATFORM_TAG OUTPUT_STRIP_TRAILING_WHITESPACE) +# defines wheel architecture part of `PLATFORM_TAG` macro(_ov_platform_arch) if(AARCH64) - set(_arch "aarch64") + if(APPLE) + set(_arch "arm64") + else() + set(_arch "aarch64") + endif() elseif(ARM) set(_arch "armvl7") elseif(X86_64) @@ -61,7 +94,7 @@ if(APPLE AND DEFINED CMAKE_OSX_DEPLOYMENT_TARGET) if(_arch AND _macos_min_version) set(PLATFORM_TAG "macosx_${_macos_min_version}_${_arch}") endif() -elseif(LINUX AND OFF) +elseif(LINUX) _ov_platform_arch() execute_process(COMMAND ldd --version @@ -75,7 +108,12 @@ elseif(LINUX AND OFF) # common pattern manylinux___ if(_libc_major AND _libc_minor AND _arch) - set(PLATFORM_TAG "manylinux_${_libc_major}_${_libc_minor}_${_arch}") + if(CMAKE_CROSSCOMPILING) + # TODO: think which proper tag is needed for arm / aarch64 + set(PLATFORM_TAG "linux_${_arch}") + else() + set(PLATFORM_TAG "manylinux_${_libc_major}_${_libc_minor}_${_arch}") + endif() endif() endif() @@ -87,31 +125,48 @@ set(openvino_wheel_path "${openvino_wheels_output_dir}/${openvino_wheel_name}") # create target for openvino.wheel # +set(wheel_env ${CMAKE_COMMAND} -E env + WHEEL_VERSION=${WHEEL_VERSION} + WHEEL_BUILD=${WHEEL_BUILD} + OPENVINO_BUILD_DIR=${OpenVINO_BINARY_DIR} + OPENVINO_PYTHON_BUILD_DIR=${OpenVINOPython_BINARY_DIR} + OV_RUNTIME_LIBS_DIR=${OV_WHEEL_RUNTIMEDIR} + TBB_LIBS_DIR=${TBB_LIBS_DIR} + PUGIXML_LIBS_DIR=${PUGIXML_LIBS_DIR} + PY_PACKAGES_DIR=${PY_PACKAGES_DIR}) + add_custom_command(OUTPUT ${openvino_wheel_path} COMMAND ${CMAKE_COMMAND} -E copy_directory "${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_CURRENT_BINARY_DIR}" COMMAND ${CMAKE_COMMAND} -E copy_directory "${OpenVINO_SOURCE_DIR}/licensing" "${CMAKE_BINARY_DIR}/licensing" - COMMAND ${CMAKE_COMMAND} -E remove_directory "${CMAKE_CURRENT_BINARY_DIR}/site-packages" - COMMAND ${CMAKE_COMMAND} -E env - WHEEL_VERSION=${WHEEL_VERSION} - WHEEL_BUILD=${WHEEL_BUILD} - OPENVINO_BUILD_DIR=${OpenVINO_BINARY_DIR} - OPENVINO_PYTHON_BUILD_DIR=${OpenVINOPython_BINARY_DIR} - OV_RUNTIME_LIBS_DIR=${OV_WHEEL_RUNTIMEDIR} - TBB_LIBS_DIR=${TBB_LIBS_DIR} - PUGIXML_LIBS_DIR=${PUGIXML_LIBS_DIR} - PY_PACKAGES_DIR=${PY_PACKAGES_DIR} - ${PYTHON_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/setup.py" clean bdist_wheel + COMMAND ${wheel_env} + ${PYTHON_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/setup.py" bdist_wheel --dist-dir ${openvino_wheels_output_dir} --build-number=${WHEEL_BUILD} --plat-name=${PLATFORM_TAG} --quiet + COMMAND ${CMAKE_COMMAND} -E remove_directory "${CMAKE_CURRENT_BINARY_DIR}/site-packages" + COMMAND ${wheel_env} + ${PYTHON_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/setup.py" clean DEPENDS ${openvino_wheel_deps} - "${CMAKE_CURRENT_SOURCE_DIR}/setup.py" WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" COMMENT "Building Python wheel ${openvino_wheel_name}" VERBATIM) -add_custom_target(ie_wheel ALL DEPENDS ${openvino_wheel_path}) +set(fdupes_report ${CMAKE_CURRENT_BINARY_DIR}/fdupes_report.txt) +add_custom_command(OUTPUT "${fdupes_report}" + COMMAND ${CMAKE_COMMAND} + -D PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE} + -D WORKING_DIRECTORY=${CMAKE_CURRENT_BINARY_DIR} + -D WHEEL_VERSION=${WHEEL_VERSION} + -D PACKAGE_FILE=${openvino_wheel_path} + -D REPORT_FILE=${fdupes_report} + -P "${CMAKE_CURRENT_SOURCE_DIR}/fdupes_check.cmake" + DEPENDS "${openvino_wheel_path}" + WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" + COMMENT "Run 'fdupes' checks for wheel ${openvino_wheel_name}" + VERBATIM) + +add_custom_target(ie_wheel ALL DEPENDS ${openvino_wheel_path} ${fdupes_report}) # install diff --git a/src/bindings/python/wheel/fdupes_check.cmake b/src/bindings/python/wheel/fdupes_check.cmake new file mode 100644 index 00000000000..b0c1d19cad7 --- /dev/null +++ b/src/bindings/python/wheel/fdupes_check.cmake @@ -0,0 +1,54 @@ +# Copyright (C) 2018-2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +foreach(var PYTHON_EXECUTABLE WORKING_DIRECTORY REPORT_FILE WHEEL_VERSION PACKAGE_FILE) + if(NOT DEFINED ${var}) + message(FATAL_ERROR "Variable ${var} is not defined") + endif() +endforeach() + +# find programs + +find_program(fdupes_PROGRAM NAMES fdupes DOC "Path to fdupes") +if(NOT fdupes_PROGRAM) + message(WARNING "Failed to find 'fdupes' tool, use 'sudo apt-get install fdupes' to install it") + return() +endif() + +# execute + +get_filename_component(wheel_name "${PACKAGE_FILE}" NAME) + +execute_process(COMMAND ${PYTHON_EXECUTABLE} -m wheel unpack ${PACKAGE_FILE} + WORKING_DIRECTORY ${WORKING_DIRECTORY} + OUTPUT_VARIABLE output_message + ERROR_VARIABLE error_message + RESULT_VARIABLE exit_code + OUTPUT_STRIP_TRAILING_WHITESPACE) + +if(NOT exit_code EQUAL 0) + message(FATAL_ERROR "Failed to unpack wheel package") +endif() + +set(WORKING_DIRECTORY "${WORKING_DIRECTORY}/openvino-${WHEEL_VERSION}") +if(NOT EXISTS "${WORKING_DIRECTORY}") + message(FATAL_ERROR "Failed to find ${WORKING_DIRECTORY}") +endif() + +execute_process(COMMAND ${fdupes_PROGRAM} -f -r "${WORKING_DIRECTORY}" + OUTPUT_VARIABLE output_message + ERROR_VARIABLE error_message + RESULT_VARIABLE exit_code + OUTPUT_STRIP_TRAILING_WHITESPACE) + +# remove unpacked directory +file(REMOVE_RECURSE "${WORKING_DIRECTORY}") + +# write output + +file(WRITE "${REPORT_FILE}" "${output_message}") + +if(output_message) + message(FATAL_ERROR "${output_message}") +endif() diff --git a/src/bindings/python/wheel/requirements-dev.txt b/src/bindings/python/wheel/requirements-dev.txt index bbc7e165437..17a84582fec 100644 --- a/src/bindings/python/wheel/requirements-dev.txt +++ b/src/bindings/python/wheel/requirements-dev.txt @@ -1,2 +1,3 @@ setuptools>=53.0.0 wheel>=0.36.2 +patchelf; sys_platform == 'linux' diff --git a/src/bindings/python/wheel/setup.py b/src/bindings/python/wheel/setup.py index a5a861752e8..72cfe69854e 100644 --- a/src/bindings/python/wheel/setup.py +++ b/src/bindings/python/wheel/setup.py @@ -60,32 +60,37 @@ LIB_INSTALL_CFG = { }, "hetero_plugin": { "name": "hetero", - "prefix": "libs.core", + "prefix": "libs.hetero", "install_dir": OV_RUNTIME_LIBS_DIR, + "rpath": LIBS_RPATH, "binary_dir": OPENVINO_BUILD_DIR, }, "gpu_plugin": { "name": "gpu", - "prefix": "libs.core", + "prefix": "libs.gpu", "install_dir": OV_RUNTIME_LIBS_DIR, + "rpath": LIBS_RPATH, "binary_dir": OPENVINO_BUILD_DIR, }, "cpu_plugin": { "name": "cpu", - "prefix": "libs.core", + "prefix": "libs.cpu", "install_dir": OV_RUNTIME_LIBS_DIR, + "rpath": LIBS_RPATH, "binary_dir": OPENVINO_BUILD_DIR, }, "multi_plugin": { "name": "multi", - "prefix": "libs.core", + "prefix": "libs.multi", "install_dir": OV_RUNTIME_LIBS_DIR, + "rpath": LIBS_RPATH, "binary_dir": OPENVINO_BUILD_DIR, }, "batch_plugin": { "name": "batch", - "prefix": "libs.core", + "prefix": "libs.batch", "install_dir": OV_RUNTIME_LIBS_DIR, + "rpath": LIBS_RPATH, "binary_dir": OPENVINO_BUILD_DIR, }, "tbb_libs": { @@ -102,20 +107,23 @@ LIB_INSTALL_CFG = { }, "ir_libs": { "name": "ir", - "prefix": "libs.core", + "prefix": "libs.ir", "install_dir": OV_RUNTIME_LIBS_DIR, + "rpath": LIBS_RPATH, "binary_dir": OPENVINO_BUILD_DIR, }, "paddle_libs": { "name": "paddle", - "prefix": "libs.core", + "prefix": "libs.paddle", "install_dir": OV_RUNTIME_LIBS_DIR, + "rpath": LIBS_RPATH, "binary_dir": OPENVINO_BUILD_DIR, }, "onnx_libs": { "name": "onnx", - "prefix": "libs.core", + "prefix": "libs.onnx", "install_dir": OV_RUNTIME_LIBS_DIR, + "rpath": LIBS_RPATH, "binary_dir": OPENVINO_BUILD_DIR, }, # uncomment once TF FE will be used in MO @@ -206,6 +214,7 @@ class CustomBuild(build): self.jobs = multiprocessing.cpu_count() plat_specifier = ".{0}-{1}.{2}".format(self.plat_name, *sys.version_info[:2]) self.build_temp = os.path.join(self.build_base, "temp" + plat_specifier, self.config) + self.announce(f"Create build directory: {self.build_temp}", level=3) # if setup.py is directly called use CMake to build product if OPENVINO_BUILD_DIR == ".": @@ -215,7 +224,11 @@ class CustomBuild(build): self.spawn(["cmake", "-S" + str(openvino_root_dir), "-B" + self.build_temp, "-DCMAKE_BUILD_TYPE={type}".format(type=self.config), - "-DENABLE_PYTHON=ON"]) + "-DENABLE_PYTHON=ON", + "-DENABLE_NCC_STYLE=OFF", + "-DENABLE_CPPLINT=OFF", + "-DENABLE_TEMPLATE=OFF", + "-DENABLE_SAMPLES=OFF"]) self.announce("Building binaries", level=3) self.spawn(["cmake", "--build", self.build_temp, @@ -243,7 +256,7 @@ class PrepareLibs(build_clib): def run(self): self.configure(LIB_INSTALL_CFG) self.configure(PY_INSTALL_CFG) - self.generate_package(get_dir_list(LIB_INSTALL_CFG)) + self.generate_package(get_install_dirs_list(LIB_INSTALL_CFG)) def configure(self, install_cfg): """Collect prebuilt libraries. Install them to the temp directories, set rpath.""" @@ -252,17 +265,18 @@ class PrepareLibs(build_clib): install_dir = comp_data.get("install_dir") binary_dir = comp_data.get("binary_dir") if install_dir and not os.path.isabs(install_dir): - install_dir = os.path.join(install_prefix, install_dir) self.announce(f"Installing {comp}", level=3) self.spawn(["cmake", "--install", binary_dir, "--prefix", install_prefix, "--config", "Release", "--strip", "--component", comp_data.get("name")]) + install_dir = os.path.join(install_prefix, install_dir) # set rpath if applicable if sys.platform != "win32" and comp_data.get("rpath"): for path in filter( - lambda x: any(item in ([".so"] if sys.platform == "linux" else [".dylib", ".so"]) for item in x.suffixes), Path(install_dir).glob("*"), + lambda x: any(item in ([".so"] if sys.platform == "linux" else [".dylib", ".so"]) + for item in x.suffixes), Path(install_dir).glob("*"), ): set_rpath(comp_data["rpath"], os.path.realpath(path)) @@ -274,12 +288,38 @@ class PrepareLibs(build_clib): for src_dir in src_dirs: local_base_dir = Path(src_dir) + + # skip symlinks of higher level like libX.so or libX.dylib + for symlink in local_base_dir.rglob("*"): + if symlink.is_symlink(): + file_name = os.readlink(symlink) + if not os.path.isabs(file_name): + file_name = os.path.join(os.path.dirname(symlink), file_name) + if Path(file_name).is_symlink(): + self.announce(f"Unlink symlink {symlink}, use {file_name} instead", level=3) + os.unlink(symlink) + + # transform libX.so.Y / libX.Y.dylib symlinks to real files + for symlink in local_base_dir.rglob("*"): + if symlink.is_symlink(): + file_name = os.readlink(symlink) + if not os.path.isabs(file_name): + file_name = os.path.join(os.path.dirname(symlink), file_name) + + os.unlink(symlink) + os.rename(file_name, symlink) + self.announce(f"Resolved symlink {symlink} as {file_name}", level=3) + + # copy so / dylib files to WHEEL_LIBS_INSTALL_DIR for file_path in local_base_dir.rglob("*"): file_name = os.path.basename(file_path) + if file_path.is_symlink(): + sys.exit(f"Wheel package content must not contain symlinks {file_path}") if file_path.is_file() and not any(file_name.endswith(ext) for ext in blacklist): dst_file = os.path.join(package_dir, os.path.relpath(file_path, local_base_dir)) os.makedirs(os.path.dirname(dst_file), exist_ok=True) copyfile(file_path, dst_file) + self.announce(f"Copy {file_path} to {dst_file}", level=3) if Path(package_dir).exists(): self.announce(f"Adding {WHEEL_LIBS_PACKAGE} package", level=3) @@ -294,7 +334,7 @@ class CopyExt(build_ext): if len(self.extensions) == 1: self.run_command("build_clib") self.extensions = [] - self.extensions = find_prebuilt_extensions(get_dir_list(PY_INSTALL_CFG)) + self.extensions = find_prebuilt_extensions(get_install_dirs_list(PY_INSTALL_CFG)) for extension in self.extensions: if not isinstance(extension, PrebuiltExtension): raise DistutilsSetupError(f"copy_ext can accept PrebuiltExtension only, but got {extension.name}") @@ -304,11 +344,9 @@ class CopyExt(build_ext): # setting relative path to find dlls if sys.platform != "win32": rpath = os.path.relpath(get_package_dir(PY_INSTALL_CFG), os.path.dirname(src)) - if sys.platform == "linux": - rpath = os.path.join("$ORIGIN", rpath, WHEEL_LIBS_INSTALL_DIR) - elif sys.platform == "darwin": - rpath = os.path.join("@loader_path", rpath, WHEEL_LIBS_INSTALL_DIR) + rpath = os.path.join(LIBS_RPATH, rpath, WHEEL_LIBS_INSTALL_DIR) set_rpath(rpath, os.path.realpath(src)) + copy_file(src, dst, verbose=self.verbose, dry_run=self.dry_run) @@ -413,7 +451,7 @@ def find_prebuilt_extensions(search_dirs): ext_pattern = "**/*.so" for base_dir in search_dirs: for path in Path(base_dir).glob(ext_pattern): - if path.match("openvino/libs/*"): + if path.match("openvino/libs/*") or path.match(f"openvino/libs/openvino-{OPENVINO_VERSION}/*"): continue relpath = path.relative_to(base_dir) if relpath.parent != ".": @@ -442,7 +480,7 @@ def get_dependencies(requirements_file_path): return dependencies -def get_dir_list(install_cfg): +def get_install_dirs_list(install_cfg): """Collect all available directories with libs or python packages.""" dirs = [] for comp_info in install_cfg.values(): @@ -459,7 +497,7 @@ def get_dir_list(install_cfg): def get_package_dir(install_cfg): """Get python package path based on config. All the packages should be located in one directory.""" py_package_path = "" - dirs = get_dir_list(install_cfg) + dirs = get_install_dirs_list(install_cfg) if len(dirs) != 0: # setup.py support only one package directory, all modules should be located there py_package_path = dirs[0] @@ -467,6 +505,7 @@ def get_package_dir(install_cfg): def concat_files(output_file, input_files): + """Concatenates multuple input files to a single output file.""" with open(output_file, "w", encoding="utf-8") as outfile: for filename in input_files: with open(filename, "r", encoding="utf-8") as infile: @@ -479,6 +518,7 @@ platforms = ["linux", "win32", "darwin"] if not any(pl in sys.platform for pl in platforms): sys.exit(f"Unsupported platform: {sys.platform}, expected: linux, win32, darwin") +OPENVINO_VERSION = WHEEL_VERSION = os.getenv("WHEEL_VERSION", "0.0.0") # copy license file into the build directory package_license = os.getenv("WHEEL_LICENSE", SCRIPT_DIR.parents[3] / "LICENSE") if os.path.exists(package_license): @@ -487,7 +527,7 @@ if os.path.exists(package_license): packages = find_namespace_packages(get_package_dir(PY_INSTALL_CFG)) package_data: typing.Dict[str, list] = {} pkg_name = os.getenv("WHEEL_PACKAGE_NAME", "openvino") -ext_modules = find_prebuilt_extensions(get_dir_list(PY_INSTALL_CFG)) if pkg_name == "openvino" else [] +ext_modules = find_prebuilt_extensions(get_install_dirs_list(PY_INSTALL_CFG)) if pkg_name == "openvino" else [] description_md = SCRIPT_DIR.parents[3] / "docs" / "install_guides" / "pypi-openvino-rt.md" md_files = [description_md, SCRIPT_DIR.parents[3] / "docs" / "install_guides" / "pre-release-note.md"] @@ -498,10 +538,10 @@ if (os.getenv("CI_BUILD_DEV_TAG")): output.parent.mkdir(exist_ok=True) description_md = concat_files(output, md_files) docs_url = "https://docs.openvino.ai/nightly/index.html" - + OPENVINO_VERSION = WHEEL_VERSION[0:8] setup( - version=os.getenv("WHEEL_VERSION", "0.0.0"), + version=WHEEL_VERSION, build=os.getenv("WHEEL_BUILD", "000"), author_email=os.getenv("WHEEL_AUTHOR_EMAIL", "openvino_pushbot@intel.com"), name=pkg_name, diff --git a/src/cmake/install_tbb.cmake b/src/cmake/install_tbb.cmake index 8a0ace9aaeb..dd7feb11a2b 100644 --- a/src/cmake/install_tbb.cmake +++ b/src/cmake/install_tbb.cmake @@ -109,21 +109,18 @@ if(THREADING MATCHES "^(TBB|TBB_AUTO)$" AND # depending on the TBB, tbb_lib_location can be in form: # - libtbb.so.x.y # - libtbb.so.x - # We need to install such files + # We need to install such only libtbb.so.x files get_filename_component(name_we "${tbb_lib_location}" NAME_WE) get_filename_component(dir "${tbb_lib_location}" DIRECTORY) # grab all tbb files matching pattern file(GLOB tbb_files "${dir}/${name_we}.*") + + # since the setup.py for pip installs tbb component + # explicitly, it's OK to put EXCLUDE_FROM_ALL to such component + # to ignore from IRC / apt / yum distribution; + # but they will be present in .wheel foreach(tbb_file IN LISTS tbb_files) - if(tbb_file MATCHES "^.*\.${CMAKE_SHARED_LIBRARY_SUFFIX}(\.[0-9]+)*$") - # since the setup.py for pip installs tbb component - # explicitly, it's OK to put EXCLUDE_FROM_ALL to such component - # to ignore from IRC / apt / yum / brew distribution; - # but they will be present in .wheel - install(FILES "${tbb_file}" - DESTINATION runtime/3rdparty/tbb/lib - COMPONENT tbb EXCLUDE_FROM_ALL) - endif() + ov_install_with_name("${tbb_file}" tbb) endforeach() endforeach() diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt index b833da5c96a..53bae1c40b5 100644 --- a/thirdparty/CMakeLists.txt +++ b/thirdparty/CMakeLists.txt @@ -104,28 +104,18 @@ if(ENABLE_SYSTEM_PUGIXML) # grab all tbb files matching pattern file(GLOB pugixml_files "${pugixml_dir}/${name_we}.*") foreach(pugixml_file IN LISTS pugixml_files) - if(pugixml_file MATCHES "^.*\.${CMAKE_SHARED_LIBRARY_SUFFIX}(\.[0-9]+)*$") - list(APPEND pugixml_libs "${pugixml_file}") - endif() + ov_install_with_name("${pugixml_file}" pugixml) endforeach() elseif(target_type STREQUAL "INTERFACE_LIBRARY") get_target_property(pugixml_loc ${pugixml_target} INTERFACE_LINK_LIBRARIES) - file(GLOB pugixml_libs_all "${pugixml_loc}.*") - foreach(pugixml_lib IN LISTS pugixml_libs_all) - if(pugixml_lib MATCHES "^${pugixml_loc}(\.[0-9]+)+$") - list(APPEND pugixml_libs "${pugixml_lib}") - endif() + file(GLOB pugixml_libs "${pugixml_loc}.*") + foreach(pugixml_lib IN LISTS pugixml_libs) + ov_install_with_name("${pugixml_lib}" pugixml) endforeach() endif() # if dynamic libpugixml.so.1 and libpugixml.so.1.X are found - if(pugixml_libs) - ie_cpack_add_component(pugixml HIDDEN) - install(FILES ${pugixml_libs} - DESTINATION runtime/3rdparty/pugixml/lib - COMPONENT pugixml - EXCLUDE_FROM_ALL) - elseif(CPACK_GENERATOR MATCHES "^(DEB|RPM)$") + if(NOT pugixml_INSTALLED AND CPACK_GENERATOR MATCHES "^(DEB|RPM)$") message(FATAL_ERROR "Debian | RPM package build requires shared Pugixml library") endif() diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 73a99c24dc9..024a5475d4c 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -28,12 +28,6 @@ install(DIRECTORY deployment_manager ie_cpack_add_component(${OV_CPACK_COMP_DEV_REQ_FILES} HIDDEN) -if(DEFINED ENV{CI_BUILD_DEV_TAG} AND NOT $ENV{CI_BUILD_DEV_TAG} STREQUAL "") - set(WHEEL_VERSION "${OpenVINO_VERSION}.$ENV{CI_BUILD_DEV_TAG}") -else() - set(WHEEL_VERSION ${OpenVINO_VERSION}) -endif() - set(REQUIREMENTS_IN "${CMAKE_CURRENT_SOURCE_DIR}/requirements_dev.in") set(EXTRAS_LIST _ caffe kaldi mxnet onnx pytorch tensorflow tensorflow2) diff --git a/tools/openvino_dev/CMakeLists.txt b/tools/openvino_dev/CMakeLists.txt index e6e420d56fa..a6dcdee5eeb 100644 --- a/tools/openvino_dev/CMakeLists.txt +++ b/tools/openvino_dev/CMakeLists.txt @@ -2,8 +2,6 @@ # SPDX-License-Identifier: Apache-2.0 # -set(WHEEL_VERSION "${OpenVINO_VERSION}" CACHE STRING "Version of this release" FORCE) -set(WHEEL_BUILD "${OpenVINO_VERSION_BUILD}" CACHE STRING "Build number of this release" FORCE) set(SETUP_PY "${CMAKE_CURRENT_SOURCE_DIR}/setup.py") set(openvino_wheel_name "openvino_dev-${WHEEL_VERSION}-${WHEEL_BUILD}-py3-none-any.whl") @@ -13,10 +11,12 @@ set(openvino_wheel_path "${openvino_wheels_output_dir}/${openvino_wheel_name}") add_custom_command(OUTPUT ${openvino_wheel_path} COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/setup.cfg" "${CMAKE_CURRENT_BINARY_DIR}" COMMAND ${CMAKE_COMMAND} -E env OPENVINO_VERSION=${WHEEL_VERSION} - ${PYTHON_EXECUTABLE} ${SETUP_PY} clean bdist_wheel - --dist-dir ${openvino_wheels_output_dir} - --build=${WHEEL_BUILD} - --quiet + ${PYTHON_EXECUTABLE} ${SETUP_PY} bdist_wheel + --dist-dir ${openvino_wheels_output_dir} + --build=${WHEEL_BUILD} + --quiet + COMMAND ${CMAKE_COMMAND} -E env OPENVINO_VERSION=${WHEEL_VERSION} + ${PYTHON_EXECUTABLE} ${SETUP_PY} clean DEPENDS ie_wheel WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" COMMENT "Building Python wheel ${openvino_wheel_name}"