diff --git a/.ci/azure/linux_arm64.yml b/.ci/azure/linux_arm64.yml index f562b6b7764..7ba72f6e9ae 100644 --- a/.ci/azure/linux_arm64.yml +++ b/.ci/azure/linux_arm64.yml @@ -203,6 +203,7 @@ jobs: -DARM_COMPUTE_SCONS_JOBS=$(NUM_PROC) \ -DCMAKE_INSTALL_PREFIX=$(INSTALL_OPENVINO) \ -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) \ + -DENABLE_PYTHON_PACKAGING=ON \ -S $(OPENVINO_REPO_DIR) \ -B $(BUILD_OPENVINO) source $(BUILD_OPENVINO)/deactivate_conanbuild.sh diff --git a/cmake/developer_package/packaging/common-libraries.cmake b/cmake/developer_package/packaging/common-libraries.cmake index 89ce26ed272..c8cde9746ea 100644 --- a/cmake/developer_package/packaging/common-libraries.cmake +++ b/cmake/developer_package/packaging/common-libraries.cmake @@ -36,6 +36,7 @@ macro(ov_common_libraries_cpack_set_dirs) ov_get_pyversion(pyversion) if(pyversion) + # should not be used in production; only by setup.py install set(OV_CPACK_PYTHONDIR ${CMAKE_INSTALL_LIBDIR}/${pyversion}/site-packages) endif() @@ -107,6 +108,10 @@ macro(ov_define_component_include_rules) set(OV_CPACK_COMP_PYTHON_OPENVINO_EXCLUDE_ALL EXCLUDE_FROM_ALL) set(OV_CPACK_COMP_PYTHON_IE_API_EXCLUDE_ALL ${OV_CPACK_COMP_PYTHON_OPENVINO_EXCLUDE_ALL}) set(OV_CPACK_COMP_PYTHON_NGRAPH_EXCLUDE_ALL ${OV_CPACK_COMP_PYTHON_OPENVINO_EXCLUDE_ALL}) + # we don't pack artifacts of setup.py install, because it's called explicitly in conda / brew + # or not used at all like in cases with conan / vcpkg + set(OV_CPACK_COMP_PYTHON_OPENVINO_PACKAGE_EXCLUDE_ALL ${OV_CPACK_COMP_PYTHON_OPENVINO_EXCLUDE_ALL}) + # we don't need wheels in package, it's used installed only in open source distribution set(OV_CPACK_COMP_PYTHON_WHEELS_EXCLUDE_ALL EXCLUDE_FROM_ALL) # tools set(OV_CPACK_COMP_CORE_TOOLS_EXCLUDE_ALL EXCLUDE_FROM_ALL) diff --git a/cmake/developer_package/packaging/debian/debian.cmake b/cmake/developer_package/packaging/debian/debian.cmake index ccfa2b34f8a..9adcc83919f 100644 --- a/cmake/developer_package/packaging/debian/debian.cmake +++ b/cmake/developer_package/packaging/debian/debian.cmake @@ -100,14 +100,16 @@ macro(ov_define_component_include_rules) endif() # python if(ENABLE_PYTHON_PACKAGING) - unset(OV_CPACK_COMP_PYTHON_OPENVINO_EXCLUDE_ALL) - set(OV_CPACK_COMP_PYTHON_IE_API_EXCLUDE_ALL ${OV_CPACK_COMP_PYTHON_OPENVINO_EXCLUDE_ALL}) - set(OV_CPACK_COMP_PYTHON_NGRAPH_EXCLUDE_ALL ${OV_CPACK_COMP_PYTHON_OPENVINO_EXCLUDE_ALL}) + # pack artifacts of setup.py install + unset(OV_CPACK_COMP_PYTHON_OPENVINO_PACKAGE_EXCLUDE_ALL) else() - set(OV_CPACK_COMP_PYTHON_OPENVINO_EXCLUDE_ALL EXCLUDE_FROM_ALL) - set(OV_CPACK_COMP_PYTHON_IE_API_EXCLUDE_ALL ${OV_CPACK_COMP_PYTHON_OPENVINO_EXCLUDE_ALL}) - set(OV_CPACK_COMP_PYTHON_NGRAPH_EXCLUDE_ALL ${OV_CPACK_COMP_PYTHON_OPENVINO_EXCLUDE_ALL}) + set(OV_CPACK_COMP_PYTHON_OPENVINO_PACKAGE_EXCLUDE_ALL EXCLUDE_FROM_ALL) endif() + # we don't pack python components itself, we pack artifacts of setup.py install + set(OV_CPACK_COMP_PYTHON_OPENVINO_EXCLUDE_ALL EXCLUDE_FROM_ALL) + set(OV_CPACK_COMP_PYTHON_IE_API_EXCLUDE_ALL ${OV_CPACK_COMP_PYTHON_OPENVINO_EXCLUDE_ALL}) + set(OV_CPACK_COMP_PYTHON_NGRAPH_EXCLUDE_ALL ${OV_CPACK_COMP_PYTHON_OPENVINO_EXCLUDE_ALL}) + # we don't need wheels in Debian packages set(OV_CPACK_COMP_PYTHON_WHEELS_EXCLUDE_ALL EXCLUDE_FROM_ALL) # tools set(OV_CPACK_COMP_CORE_TOOLS_EXCLUDE_ALL EXCLUDE_FROM_ALL) @@ -182,9 +184,9 @@ endmacro() ov_debian_specific_settings() # needed to override cmake auto generated files -set(def_postinst "${OpenVINO_BINARY_DIR}/_CPack_Packages/postinst") -set(def_postrm "${OpenVINO_BINARY_DIR}/_CPack_Packages/postrm") -set(def_triggers "${OpenVINO_BINARY_DIR}/_CPack_Packages/triggers") +set(def_postinst "${CMAKE_CURRENT_BINARY_DIR}/_CPack_Packages/postinst") +set(def_postrm "${CMAKE_CURRENT_BINARY_DIR}/_CPack_Packages/postrm") +set(def_triggers "${CMAKE_CURRENT_BINARY_DIR}/_CPack_Packages/triggers") set(triggers_content "activate-noawait ldconfig\n\n") set(post_content "#!/bin/sh\n\nset -e;\nset -e\n\n") @@ -320,7 +322,7 @@ macro(ov_debian_add_latest_component comp) set(upper_case "${ucomp}_LATEST") set(CPACK_COMPONENT_${upper_case}_DESCRIPTION "${CPACK_COMPONENT_${ucomp}_DESCRIPTION}") - set(CPACK_COMPONENT_${upper_case}_ARCHITECTURE "all") + set(CPACK_DEBIAN_${upper_case}_PACKAGE_ARCHITECTURE "all") set(CPACK_COMPONENT_${upper_case}_DEPENDS "${comp}") set(${comp_name}_copyright "generic") diff --git a/cmake/developer_package/packaging/nsis.cmake b/cmake/developer_package/packaging/nsis.cmake index 0b8338eaaee..77239fee00b 100644 --- a/cmake/developer_package/packaging/nsis.cmake +++ b/cmake/developer_package/packaging/nsis.cmake @@ -66,6 +66,7 @@ macro(ov_define_component_include_rules) unset(OV_CPACK_COMP_PYTHON_NGRAPH_EXCLUDE_ALL) unset(OV_CPACK_COMP_PYTHON_OPENVINO_EXCLUDE_ALL) set(OV_CPACK_COMP_PYTHON_WHEELS_EXCLUDE_ALL EXCLUDE_FROM_ALL) + set(OV_CPACK_COMP_PYTHON_OPENVINO_PACKAGE_EXCLUDE_ALL EXCLUDE_FROM_ALL) # tools unset(OV_CPACK_COMP_CORE_TOOLS_EXCLUDE_ALL) unset(OV_CPACK_COMP_OPENVINO_DEV_REQ_FILES_EXCLUDE_ALL) diff --git a/cmake/developer_package/packaging/packaging.cmake b/cmake/developer_package/packaging/packaging.cmake index 86c95ae1c82..79beb1cd99a 100644 --- a/cmake/developer_package/packaging/packaging.cmake +++ b/cmake/developer_package/packaging/packaging.cmake @@ -148,6 +148,7 @@ macro(ov_define_component_names) set(OV_CPACK_COMP_PYTHON_IE_API "pyie") set(OV_CPACK_COMP_PYTHON_NGRAPH "pyngraph") set(OV_CPACK_COMP_PYTHON_OPENVINO "pyopenvino") + set(OV_CPACK_COMP_PYTHON_OPENVINO_PACKAGE "pyopenvino_package") set(OV_CPACK_COMP_PYTHON_WHEELS "python_wheels") # tools set(OV_CPACK_COMP_CORE_TOOLS "core_tools") @@ -160,6 +161,7 @@ endmacro() ov_define_component_names() +# default components for case when CPACK_GENERATOR is not set (i.e. default open source user) macro(ov_define_component_include_rules) # core components unset(OV_CPACK_COMP_CORE_EXCLUDE_ALL) @@ -177,6 +179,9 @@ macro(ov_define_component_include_rules) unset(OV_CPACK_COMP_PYTHON_NGRAPH_EXCLUDE_ALL) unset(OV_CPACK_COMP_PYTHON_OPENVINO_EXCLUDE_ALL) unset(OV_CPACK_COMP_PYTHON_WHEELS_EXCLUDE_ALL) + # TODO: think about python entry points + # maybe we can create entry points without python interpreter and use it in debian / rpm as well? + set(OV_CPACK_COMP_PYTHON_OPENVINO_PACKAGE_EXCLUDE_ALL EXCLUDE_FROM_ALL) # tools unset(OV_CPACK_COMP_CORE_TOOLS_EXCLUDE_ALL) set(OV_CPACK_COMP_OPENVINO_DEV_REQ_FILES_EXCLUDE_ALL EXCLUDE_FROM_ALL) diff --git a/cmake/developer_package/packaging/rpm/rpm.cmake b/cmake/developer_package/packaging/rpm/rpm.cmake index 875dc86710b..00dc0b76f46 100644 --- a/cmake/developer_package/packaging/rpm/rpm.cmake +++ b/cmake/developer_package/packaging/rpm/rpm.cmake @@ -96,14 +96,16 @@ macro(ov_define_component_include_rules) endif() # python if(ENABLE_PYTHON_PACKAGING) - unset(OV_CPACK_COMP_PYTHON_OPENVINO_EXCLUDE_ALL) - set(OV_CPACK_COMP_PYTHON_IE_API_EXCLUDE_ALL ${OV_CPACK_COMP_PYTHON_OPENVINO_EXCLUDE_ALL}) - set(OV_CPACK_COMP_PYTHON_NGRAPH_EXCLUDE_ALL ${OV_CPACK_COMP_PYTHON_OPENVINO_EXCLUDE_ALL}) + # pack artifacts of setup.py install + unset(OV_CPACK_COMP_PYTHON_OPENVINO_PACKAGE_EXCLUDE_ALL) else() - set(OV_CPACK_COMP_PYTHON_OPENVINO_EXCLUDE_ALL EXCLUDE_FROM_ALL) - set(OV_CPACK_COMP_PYTHON_IE_API_EXCLUDE_ALL ${OV_CPACK_COMP_PYTHON_OPENVINO_EXCLUDE_ALL}) - set(OV_CPACK_COMP_PYTHON_NGRAPH_EXCLUDE_ALL ${OV_CPACK_COMP_PYTHON_OPENVINO_EXCLUDE_ALL}) + set(OV_CPACK_COMP_PYTHON_OPENVINO_PACKAGE_EXCLUDE_ALL EXCLUDE_FROM_ALL) endif() + # we don't pack python components itself, we pack artifacts of setup.py install + set(OV_CPACK_COMP_PYTHON_OPENVINO_EXCLUDE_ALL EXCLUDE_FROM_ALL) + set(OV_CPACK_COMP_PYTHON_IE_API_EXCLUDE_ALL ${OV_CPACK_COMP_PYTHON_OPENVINO_EXCLUDE_ALL}) + set(OV_CPACK_COMP_PYTHON_NGRAPH_EXCLUDE_ALL ${OV_CPACK_COMP_PYTHON_OPENVINO_EXCLUDE_ALL}) + # we don't need wheels in RPM packages set(OV_CPACK_COMP_PYTHON_WHEELS_EXCLUDE_ALL EXCLUDE_FROM_ALL) # tools set(OV_CPACK_COMP_CORE_TOOLS_EXCLUDE_ALL EXCLUDE_FROM_ALL) diff --git a/cmake/features.cmake b/cmake/features.cmake index 546fc721938..3d2e926f7ac 100644 --- a/cmake/features.cmake +++ b/cmake/features.cmake @@ -186,7 +186,7 @@ ie_dependent_option (ENABLE_SYSTEM_SNAPPY "Enables use of system version of Snap # temporary option until we enable this by default when review python API distribution ie_dependent_option (ENABLE_PYTHON_PACKAGING "Enables packaging of Python API in APT / YUM" OFF - "ENABLE_PYTHON;CPACK_GENERATOR STREQUAL RPM OR CPACK_GENERATOR STREQUAL DEB" OFF) + "ENABLE_PYTHON;UNIX" OFF) ie_option(ENABLE_OPENVINO_DEBUG "Enable output for OPENVINO_DEBUG statements" OFF) diff --git a/cmake/packaging/debian.cmake b/cmake/packaging/debian.cmake index 7302127f3e3..c5684c0df00 100644 --- a/cmake/packaging/debian.cmake +++ b/cmake/packaging/debian.cmake @@ -48,10 +48,12 @@ macro(ov_cpack_settings) # filter out some components, which are not needed to be wrapped to .deb package if(NOT OV_CPACK_COMP_${UPPER_COMP}_EXCLUDE_ALL AND # skip OpenVINO Python API (pattern in form of "_python${PYTHON_VERSION_MAJOR}${PYTHON_VERSION_MINOR}") - (NOT item MATCHES "^${OV_CPACK_COMP_PYTHON_OPENVINO}_python.*" OR ENABLE_PYTHON_PACKAGING) AND + NOT item MATCHES "^${OV_CPACK_COMP_PYTHON_OPENVINO}_python.*" AND + # because in case of .deb package, pyopenvino_package_python${PYTHON_VERSION_MAJOR}${PYTHON_VERSION_MINOR} is installed + (NOT item MATCHES "^${OV_CPACK_COMP_PYTHON_OPENVINO_PACKAGE}_python.*" OR ENABLE_PYTHON_PACKAGING) AND # see ticket # 82605 NOT item STREQUAL "gna" AND - # don't install Intel OpenMP during debian + # don't install Intel OpenMP NOT item STREQUAL "omp" AND # even for case of system TBB we have installation rules for wheels packages # so, need to skip this explicitly @@ -290,7 +292,8 @@ macro(ov_cpack_settings) # set(CPACK_COMPONENT_CORE_DEV_DESCRIPTION "Intel(R) Distribution of OpenVINO(TM) Toolkit C / C++ Development files") - set(CPACK_COMPONENT_CORE_DEV_DEPENDS "${OV_CPACK_COMP_CORE};${frontends}") + set(CPACK_COMPONENT_CORE_DEV_DEPENDS "${OV_CPACK_COMP_CORE}") + list(APPEND CPACK_COMPONENT_CORE_DEV_DEPENDS ${frontends}) set(CPACK_DEBIAN_CORE_DEV_PACKAGE_NAME "libopenvino-dev-${cpack_name_ver}") ov_debian_generate_conflicts("${OV_CPACK_COMP_CORE_DEV}" ${conflicting_versions}) set(${OV_CPACK_COMP_CORE_DEV}_copyright "generic") @@ -301,17 +304,17 @@ macro(ov_cpack_settings) if(ENABLE_PYTHON_PACKAGING) ov_get_pyversion(pyversion) - set(python_component "${OV_CPACK_COMP_PYTHON_OPENVINO}_${pyversion}") + set(python_component "${OV_CPACK_COMP_PYTHON_OPENVINO_PACKAGE}_${pyversion}") string(TOUPPER "${pyversion}" pyversion) - set(CPACK_COMPONENT_PYOPENVINO_${pyversion}_DESCRIPTION "OpenVINO Python API") - set(CPACK_COMPONENT_PYOPENVINO_${pyversion}_DEPENDS "${OV_CPACK_COMP_CORE}") - list(APPEND CPACK_COMPONENT_PYOPENVINO_${pyversion}_DEPENDS ${installed_plugins}) - list(APPEND CPACK_COMPONENT_PYOPENVINO_${pyversion}_DEPENDS ${frontends}) + set(CPACK_COMPONENT_PYOPENVINO_PACKAGE_${pyversion}_DESCRIPTION "OpenVINO Python API") + set(CPACK_COMPONENT_PYOPENVINO_PACKAGE_${pyversion}_DEPENDS "${OV_CPACK_COMP_CORE}") + list(APPEND CPACK_COMPONENT_PYOPENVINO_PACKAGE_${pyversion}_DEPENDS ${installed_plugins}) + list(APPEND CPACK_COMPONENT_PYOPENVINO_PACKAGE_${pyversion}_DEPENDS ${frontends}) - set(CPACK_DEBIAN_PYOPENVINO_${pyversion}_PACKAGE_NAME "python3-openvino") - set(python_package "${CPACK_DEBIAN_PYOPENVINO_${pyversion}_PACKAGE_NAME} (= ${cpack_full_ver})") - set(CPACK_DEBIAN_PYOPENVINO_${pyversion}_PACKAGE_DEPENDS "python3, python3-numpy") + set(CPACK_DEBIAN_PYOPENVINO_PACKAGE_${pyversion}_PACKAGE_NAME "python3-openvino") + set(python_package "${CPACK_DEBIAN_PYOPENVINO_PACKAGE_${pyversion}_PACKAGE_NAME} (= ${cpack_full_ver})") + set(CPACK_DEBIAN_PYOPENVINO_PACKAGE_${pyversion}_PACKAGE_DEPENDS "python3, python3-numpy") # we can have a single python installed, so we need to generate conflicts for all other versions ov_debian_generate_conflicts(${python_component} ${conflicting_versions}) @@ -324,6 +327,10 @@ macro(ov_cpack_settings) "executable-not-elf-or-script" # all directories "non-standard-dir-perm" + # usr/bin/benchmark_app + "binary-without-manpage" + # usr/bin/benchmark_app + "non-standard-executable-perm" # all python files "non-standard-file-perm") set(${python_component}_copyright "generic") diff --git a/cmake/packaging/rpm.cmake b/cmake/packaging/rpm.cmake index 4c9bbf71254..8155aac88b1 100644 --- a/cmake/packaging/rpm.cmake +++ b/cmake/packaging/rpm.cmake @@ -31,13 +31,15 @@ macro(ov_cpack_settings) unset(CPACK_COMPONENTS_ALL) foreach(item IN LISTS cpack_components_all) string(TOUPPER ${item} UPPER_COMP) - # filter out some components, which are not needed to be wrapped to .deb package + # filter out some components, which are not needed to be wrapped to .rpm package if(NOT OV_CPACK_COMP_${UPPER_COMP}_EXCLUDE_ALL AND # skip OpenVINO Python API (pattern in form of "_python${PYTHON_VERSION_MAJOR}${PYTHON_VERSION_MINOR}") - (NOT item MATCHES "^${OV_CPACK_COMP_PYTHON_OPENVINO}_python.*" OR ENABLE_PYTHON_PACKAGING) AND + NOT item MATCHES "^${OV_CPACK_COMP_PYTHON_OPENVINO}_python.*" AND + # because in case of .rpm package, pyopenvino_package_python${PYTHON_VERSION_MAJOR}${PYTHON_VERSION_MINOR} is installed + (NOT item MATCHES "^${OV_CPACK_COMP_PYTHON_OPENVINO_PACKAGE}_python.*" OR ENABLE_PYTHON_PACKAGING) AND # see ticket # 82605 NOT item STREQUAL "gna" AND - # don't install Intel OpenMP during rpm + # don't install Intel OpenMP NOT item STREQUAL "omp" AND # even for case of system TBB we have installation rules for wheels packages # so, need to skip this explicitly @@ -266,14 +268,14 @@ macro(ov_cpack_settings) if(ENABLE_PYTHON_PACKAGING) ov_get_pyversion(pyversion) - set(python_component "${OV_CPACK_COMP_PYTHON_OPENVINO}_${pyversion}") + set(python_component "${OV_CPACK_COMP_PYTHON_OPENVINO_PACKAGE}_${pyversion}") string(TOUPPER "${pyversion}" pyversion_upper) - set(CPACK_COMPONENT_PYOPENVINO_${pyversion_upper}_DESCRIPTION "OpenVINO Python API") - set(CPACK_RPM_PYOPENVINO_${pyversion_upper}_PACKAGE_REQUIRES + set(CPACK_COMPONENT_PYOPENVINO_PACKAGE_${pyversion_upper}_DESCRIPTION "OpenVINO Python API") + set(CPACK_RPM_PYOPENVINO_PACKAGE_${pyversion_upper}_PACKAGE_REQUIRES "${core_package}, ${frontend_packages}, ${plugin_packages}, python3, python3-numpy") - set(CPACK_RPM_PYOPENVINO_${pyversion_upper}_PACKAGE_NAME "python3-openvino") - set(python_package "${CPACK_RPM_PYOPENVINO_${pyversion_upper}_PACKAGE_NAME} = ${cpack_full_ver}") + set(CPACK_RPM_PYOPENVINO_PACKAGE_${pyversion_upper}_PACKAGE_NAME "python3-openvino") + set(python_package "${CPACK_RPM_PYOPENVINO_PACKAGE_${pyversion_upper}_PACKAGE_NAME} = ${cpack_full_ver}") set(${python_component}_copyright "generic") # we can have a single python installed, so we need to generate conflicts for all other versions diff --git a/cmake/templates/OpenVINODeveloperPackageConfig.cmake.in b/cmake/templates/OpenVINODeveloperPackageConfig.cmake.in index 64dae5d7c28..c6decaf418e 100644 --- a/cmake/templates/OpenVINODeveloperPackageConfig.cmake.in +++ b/cmake/templates/OpenVINODeveloperPackageConfig.cmake.in @@ -15,7 +15,7 @@ list(APPEND ov_options CMAKE_CXX_COMPILER_LAUNCHER CMAKE_C_COMPILER_LAUNCHER CMAKE_CXX_LINKER_LAUNCHER CMAKE_C_LINKER_LAUNCHER CMAKE_BUILD_TYPE CMAKE_SKIP_RPATH CMAKE_INSTALL_PREFIX CMAKE_OSX_ARCHITECTURES CMAKE_OSX_DEPLOYMENT_TARGET - CMAKE_CONFIGURATION_TYPES CMAKE_DEFAULT_BUILD_TYPE) + CPACK_GENERATOR) file(TO_CMAKE_PATH "${CMAKE_CURRENT_LIST_DIR}" cache_path) message(STATUS "The following CMake options are exported from OpenVINO Developer package") diff --git a/conanfile.txt b/conanfile.txt index 3ee3460ab57..33413b613db 100644 --- a/conanfile.txt +++ b/conanfile.txt @@ -5,9 +5,9 @@ pugixml/[>=1.10] protobuf/[>=3.20.3] ittapi/[>=3.23.0] zlib/[>=1.2.8] -opencl-icd-loader/2022.09.30 -opencl-clhpp-headers/2022.09.30 -opencl-headers/2022.09.30 +opencl-icd-loader/2023.04.17 +opencl-clhpp-headers/2023.04.17 +opencl-headers/2023.04.17 xbyak/[>=6.62] snappy/[>=1.1.7] gflags/2.2.2 @@ -24,8 +24,6 @@ flatbuffers/[>=22.9.24] [options] protobuf/*:lite=True -onetbb/*:tbbmalloc=True -onetbb/*:tbbproxy=True flatbuffers/*:header_only=True [generators] diff --git a/src/bindings/python/CMakeLists.txt b/src/bindings/python/CMakeLists.txt index f05013af026..93fd65992e2 100644 --- a/src/bindings/python/CMakeLists.txt +++ b/src/bindings/python/CMakeLists.txt @@ -160,8 +160,15 @@ if(LINUX) endif() endif() -if(CMAKE_VERSION VERSION_LESS 3.15) +if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.15) + set(SETUP_PY_REQUIREMENTS_FOUND ON) +else() message(${message_mode} "Cmake version 3.15 and higher is required to build 'openvino' wheel. Provided version ${CMAKE_VERSION}") + set(SETUP_PY_REQUIREMENTS_FOUND OFF) +endif() + +if(NOT SETUP_PY_REQUIREMENTS_FOUND) + # setup.py requirements are importnant to build wheel set(ENABLE_WHEEL_DEFAULT OFF) endif() @@ -177,6 +184,10 @@ if(NOT ENABLE_PYTHON) endif() endif() +if(NOT SETUP_PY_REQUIREMENTS_FOUND AND ENABLE_PYTHON_PACKAGING) + message(FATAL_ERROR "Python cannot be packaged, because setup.py requirements are not satisfied (cmake version >= 3.15 is required, provided ${CMAKE_VERSION})") +endif() + # # Build the code # @@ -196,10 +207,150 @@ else() message(WARNING "NOTE: Python API for OpenVINO 1.0 is disabled") endif() +# +# Packaging +# + +macro(ov_define_setup_py_packaging_vars) + # PYTHON_VERSION_MAJOR and PYTHON_VERSION_MINOR are defined inside pybind11 + set(pyversion python${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR}) + + # define version (syncronize with tools/openvino_dev/CMakeLists.txt) + if(DEFINED ENV{CI_BUILD_DEV_TAG} AND NOT "$ENV{CI_BUILD_DEV_TAG}" STREQUAL "") + set(WHEEL_VERSION "${OpenVINO_VERSION}.$ENV{CI_BUILD_DEV_TAG}" CACHE STRING "Version of this release" FORCE) + set(wheel_pre_release ON) + else() + set(WHEEL_VERSION ${OpenVINO_VERSION} CACHE STRING "Version of this release" FORCE) + endif() + set(WHEEL_BUILD "${OpenVINO_VERSION_BUILD}" CACHE STRING "Build number of this release" FORCE) + + # Common vars used by setup.py + set(PY_PACKAGES_DIR ${OV_CPACK_PYTHONDIR}) + set(TBB_LIBS_DIR runtime/3rdparty/tbb/lib) + if(WIN32) + set(TBB_LIBS_DIR runtime/3rdparty/tbb/bin) + endif() + set(PUGIXML_LIBS_DIR runtime/3rdparty/pugixml/lib) + + # define setup.py running environment + set(setup_py_env ${CMAKE_COMMAND} -E env + # for cross-compilation + SETUPTOOLS_EXT_SUFFIX=${PYTHON_MODULE_EXTENSION} + # versions + WHEEL_VERSION=${WHEEL_VERSION} + WHEEL_BUILD=${WHEEL_BUILD} + # build locations + OPENVINO_BINARY_DIR=${OpenVINO_BINARY_DIR} + OPENVINO_PYTHON_BINARY_DIR=${OpenVINOPython_BINARY_DIR} + # variables to reflect cpack locations + OV_RUNTIME_LIBS_DIR=${OV_WHEEL_RUNTIMEDIR} + TBB_LIBS_DIR=${TBB_LIBS_DIR} + PUGIXML_LIBS_DIR=${PUGIXML_LIBS_DIR} + PY_PACKAGES_DIR=${PY_PACKAGES_DIR}) +endmacro() + +macro(ov_define_setup_py_dependencies) + foreach(_target + # Python API dependencies + _pyngraph pyopenvino py_ov_frontends + # legacy Python API 1.0 dependencies (remove before 2024.0 release) + ie_api constants + # plugins + ov_plugins + # frontends + ov_frontends) + if(TARGET ${_target}) + list(APPEND ov_setup_py_deps ${_target}) + endif() + endforeach() + + file(GLOB_RECURSE compat_ngraph_py_files ${OpenVINOPython_SOURCE_DIR}/src/compatibility/*.py) + file(GLOB_RECURSE openvino_py_files ${OpenVINOPython_SOURCE_DIR}/src/openvino/*.py) + + list(APPEND ov_setup_py_deps + ${openvino_py_files} + ${compat_ngraph_py_files} + "${CMAKE_CURRENT_SOURCE_DIR}/wheel/setup.py" + "${OpenVINOPython_SOURCE_DIR}/requirements.txt" + "${OpenVINOPython_SOURCE_DIR}/wheel/readme.txt" + "${OpenVINO_SOURCE_DIR}/LICENSE" + "${OpenVINO_SOURCE_DIR}/licensing/onednn_third-party-programs.txt" + "${OpenVINO_SOURCE_DIR}/licensing/runtime-third-party-programs.txt" + "${OpenVINO_SOURCE_DIR}/licensing/tbb_third-party-programs.txt" + "${OpenVINO_SOURCE_DIR}/docs/install_guides/pypi-openvino-rt.md") + + if(wheel_pre_release) + list(APPEND ov_setup_py_deps + "${OpenVINO_SOURCE_DIR}/docs/install_guides/pre-release-note.md") + endif() +endmacro() + +ov_define_setup_py_packaging_vars() +ov_define_setup_py_dependencies() + if(ENABLE_WHEEL) add_subdirectory(wheel) endif() +# +# Target, which creates python install tree for cases of DEB | RPM packages +# + +if(ENABLE_PYTHON_PACKAGING) + # site-packages depending on package type + set(python_xy "${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR}") + if(CPACK_GENERATOR STREQUAL "DEB") + set(python_versioned_folder "python${PYTHON_VERSION_MAJOR}") + set(ov_site_packages "dist-packages") + elseif(CPACK_GENERATOR STREQUAL "RPM") + set(python_versioned_folder "python${python_xy}") + set(ov_site_packages "site-packages") + endif() + + set(python_package_prefix "${CMAKE_CURRENT_BINARY_DIR}/install_${pyversion}") + set(install_lib "${python_package_prefix}/lib/${python_versioned_folder}/${ov_site_packages}") + set(meta_info_subdir "openvino-${OpenVINO_VERSION}-py${python_xy}.egg-info") + set(meta_info_file "${install_lib}/${meta_info_subdir}/PKG-INFO") + + add_custom_command(OUTPUT ${meta_info_file} + COMMAND ${CMAKE_COMMAND} -E remove_directory + "${python_package_prefix}" + COMMAND ${setup_py_env} + "${PYTHON_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/wheel/setup.py" + --no-user-cfg + --quiet + build + --python-extensions-only + --executable "/usr/bin/python3" + build_ext + --skip-rpath + install + --no-compile + --prefix "${python_package_prefix}" + --install-lib "${install_lib}" + --install-scripts "${python_package_prefix}/bin" + # --single-version-externally-managed + # --record=installed.txt + WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" + DEPENDS ${ov_setup_py_deps} + COMMENT "Create python package with ${meta_info_subdir} folder") + + add_custom_target(_python_api_package ALL DEPENDS ${meta_info_file}) + + # install python package, which will be later packed into DEB | RPM + ov_cpack_add_component(${OV_CPACK_COMP_PYTHON_NGRAPH}_package_${pyversion} HIDDEN) + + install(DIRECTORY ${python_package_prefix}/ + DESTINATION ${CMAKE_INSTALL_PREFIX} + COMPONENT ${OV_CPACK_COMP_PYTHON_OPENVINO_PACKAGE}_${pyversion} + ${OV_CPACK_COMP_PYTHON_OPENVINO_PACKAGE_EXCLUDE_ALL} + USE_SOURCE_PERMISSIONS) +endif() + +# +# Tests +# + if(ENABLE_TESTS) add_subdirectory(tests/mock/mock_py_frontend) add_subdirectory(tests/mock/pyngraph_fe_mock_api) diff --git a/src/bindings/python/wheel/CMakeLists.txt b/src/bindings/python/wheel/CMakeLists.txt index a214c255b23..3311380557e 100644 --- a/src/bindings/python/wheel/CMakeLists.txt +++ b/src/bindings/python/wheel/CMakeLists.txt @@ -2,58 +2,6 @@ # SPDX-License-Identifier: Apache-2.0 # -# define version (syncronize with tools/openvino_dev/CMakeLists.txt) - -if(DEFINED ENV{CI_BUILD_DEV_TAG} AND NOT "$ENV{CI_BUILD_DEV_TAG}" STREQUAL "") - set(WHEEL_VERSION "${OpenVINO_VERSION}.$ENV{CI_BUILD_DEV_TAG}" CACHE STRING "Version of this release" FORCE) - set(wheel_pre_release ON) -else() - set(WHEEL_VERSION ${OpenVINO_VERSION} CACHE STRING "Version of this release" FORCE) -endif() -set(WHEEL_BUILD "${OpenVINO_VERSION_BUILD}" CACHE STRING "Build number of this release" FORCE) - - -# -# Common vars used by setup.py -# - -set(PY_PACKAGES_DIR ${OV_CPACK_PYTHONDIR}) -set(TBB_LIBS_DIR runtime/3rdparty/tbb/lib) -if(WIN32) - set(TBB_LIBS_DIR runtime/3rdparty/tbb/bin) -endif() -set(PUGIXML_LIBS_DIR runtime/3rdparty/pugixml/lib) - -# -# Dependencies -# - -foreach(_target ie_api constants _pyngraph openvino_c pyopenvino ov_plugins ov_frontends py_ov_frontends) - if(TARGET ${_target}) - list(APPEND openvino_wheel_deps ${_target}) - endif() -endforeach() - -file(GLOB_RECURSE compat_ngraph_py_files ${OpenVINOPython_SOURCE_DIR}/src/compatibility/*.py) -file(GLOB_RECURSE openvino_py_files ${OpenVINOPython_SOURCE_DIR}/src/openvino/*.py) - -list(APPEND openvino_wheel_deps - ${openvino_py_files} - ${compat_ngraph_py_files} - "${CMAKE_CURRENT_SOURCE_DIR}/setup.py" - "${OpenVINOPython_SOURCE_DIR}/requirements.txt" - "${OpenVINOPython_SOURCE_DIR}/wheel/readme.txt" - "${OpenVINO_SOURCE_DIR}/LICENSE" - "${OpenVINO_SOURCE_DIR}/licensing/onednn_third-party-programs.txt" - "${OpenVINO_SOURCE_DIR}/licensing/runtime-third-party-programs.txt" - "${OpenVINO_SOURCE_DIR}/licensing/tbb_third-party-programs.txt" - "${OpenVINO_SOURCE_DIR}/docs/install_guides/pypi-openvino-rt.md") - -if(wheel_pre_release) - list(APPEND openvino_wheel_deps - "${OpenVINO_SOURCE_DIR}/docs/install_guides/pre-release-note.md") -endif() - # # Define proper package name # @@ -138,33 +86,38 @@ set(openvino_wheel_path "${openvino_wheels_output_dir}/${openvino_wheel_name}") # create target for openvino.wheel # -set(wheel_env ${CMAKE_COMMAND} -E env - SETUPTOOLS_EXT_SUFFIX=${PYTHON_MODULE_EXTENSION} - WHEEL_VERSION=${WHEEL_VERSION} - WHEEL_BUILD=${WHEEL_BUILD} - OPENVINO_BUILD_DIR=${OpenVINO_BINARY_DIR} - OPENVINO_PYTHON_BUILD_DIR=${OpenVINOPython_BINARY_DIR} - OV_RUNTIME_LIBS_DIR=${OV_WHEEL_RUNTIMEDIR} - TBB_LIBS_DIR=${TBB_LIBS_DIR} - PUGIXML_LIBS_DIR=${PUGIXML_LIBS_DIR} - PY_PACKAGES_DIR=${PY_PACKAGES_DIR}) +execute_process(COMMAND ${PYTHON_EXECUTABLE} -m pip --version + OUTPUT_VARIABLE pip_version OUTPUT_STRIP_TRAILING_WHITESPACE) -add_custom_command(OUTPUT ${openvino_wheel_path} - COMMAND ${CMAKE_COMMAND} -E copy_directory "${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_CURRENT_BINARY_DIR}" - COMMAND ${CMAKE_COMMAND} -E copy_directory "${OpenVINO_SOURCE_DIR}/licensing" "${CMAKE_BINARY_DIR}/licensing" - COMMAND ${wheel_env} +string(REGEX MATCH "pip[ ]+([\\.0-9]*)" pip_version "${pip_version}") +set(pip_version ${CMAKE_MATCH_1}) + +if(pip_version VERSION_GREATER_EQUAL 22.0) + set(wheel_build_command + ${PYTHON_EXECUTABLE} -m pip wheel + --no-deps + --wheel-dir ${openvino_wheels_output_dir} + # --verbose + --build-option --build-number=${WHEEL_BUILD} + --build-option --plat-name=${PLATFORM_TAG} + "${CMAKE_CURRENT_SOURCE_DIR}") +else() + set(wheel_build_command ${PYTHON_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/setup.py" --quiet --no-user-cfg bdist_wheel --dist-dir ${openvino_wheels_output_dir} --build-number=${WHEEL_BUILD} - --plat-name=${PLATFORM_TAG} - COMMAND ${CMAKE_COMMAND} -E remove_directory "${CMAKE_CURRENT_BINARY_DIR}/site-packages" - COMMAND ${wheel_env} - ${PYTHON_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/setup.py" clean - DEPENDS ${openvino_wheel_deps} - WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" + --plat-name=${PLATFORM_TAG}) +endif() + +add_custom_command(OUTPUT ${openvino_wheel_path} + COMMAND ${setup_py_env} + ${wheel_build_command} + COMMAND ${CMAKE_COMMAND} -E remove_directory "${CMAKE_CURRENT_SOURCE_DIR}/build_${pyversion}" + DEPENDS ${ov_setup_py_deps} + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" COMMENT "Building Python wheel ${openvino_wheel_name}" VERBATIM) @@ -176,6 +129,7 @@ add_custom_command(OUTPUT "${fdupes_report}" -D WHEEL_VERSION=${WHEEL_VERSION} -D PACKAGE_FILE=${openvino_wheel_path} -D REPORT_FILE=${fdupes_report} + -D CMAKE_SHARED_LIBRARY_SUFFIX=${CMAKE_SHARED_LIBRARY_SUFFIX} -P "${CMAKE_CURRENT_SOURCE_DIR}/fdupes_check.cmake" DEPENDS "${openvino_wheel_path}" WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" diff --git a/src/bindings/python/wheel/fdupes_check.cmake b/src/bindings/python/wheel/fdupes_check.cmake index 278de0fe83d..2709b1fdfbf 100644 --- a/src/bindings/python/wheel/fdupes_check.cmake +++ b/src/bindings/python/wheel/fdupes_check.cmake @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 # -foreach(var PYTHON_EXECUTABLE WORKING_DIRECTORY REPORT_FILE WHEEL_VERSION PACKAGE_FILE) +foreach(var PYTHON_EXECUTABLE WORKING_DIRECTORY REPORT_FILE WHEEL_VERSION PACKAGE_FILE CMAKE_SHARED_LIBRARY_SUFFIX) if(NOT DEFINED ${var}) message(FATAL_ERROR "Variable ${var} is not defined") endif() @@ -37,7 +37,7 @@ if(NOT EXISTS "${WORKING_DIRECTORY}") endif() execute_process(COMMAND ${fdupes_PROGRAM} -f -r "${WORKING_DIRECTORY}" - OUTPUT_VARIABLE output_message + OUTPUT_VARIABLE duplicated_files ERROR_VARIABLE error_message RESULT_VARIABLE exit_code OUTPUT_STRIP_TRAILING_WHITESPACE) @@ -45,10 +45,18 @@ execute_process(COMMAND ${fdupes_PROGRAM} -f -r "${WORKING_DIRECTORY}" # remove unpacked directory file(REMOVE_RECURSE "${WORKING_DIRECTORY}") +# filtering of 'duplicated_files' + +foreach(duplicated_file IN LISTS duplicated_files) + if(duplicated_file MATCHES ".*${CMAKE_SHARED_LIBRARY_SUFFIX}.*") + set(duplicated_libraries "${duplicated_file}\n${duplicated_libraries}") + endif() +endforeach() + # write output -file(WRITE "${REPORT_FILE}" "${output_message}") +file(WRITE "${REPORT_FILE}" "${duplicated_libraries}") -if(output_message) - message(FATAL_ERROR "${output_message}") +if(duplicated_libraries) + message(FATAL_ERROR "${duplicated_libraries}") endif() diff --git a/src/bindings/python/wheel/setup.cfg b/src/bindings/python/wheel/setup.cfg index c6893c93c42..1cae6c7f3f1 100644 --- a/src/bindings/python/wheel/setup.cfg +++ b/src/bindings/python/wheel/setup.cfg @@ -1,11 +1,7 @@ [metadata] license_files = - readme* - *LICENSE* - *license* - *third-party-programs* + readme.txt ../../../../licensing/runtime-third-party-programs.txt ../../../../licensing/tbb_third-party-programs.txt ../../../../licensing/onednn_third-party-programs.txt ../../../../LICENSE - diff --git a/src/bindings/python/wheel/setup.py b/src/bindings/python/wheel/setup.py index dce072a8c46..e8427dea31e 100644 --- a/src/bindings/python/wheel/setup.py +++ b/src/bindings/python/wheel/setup.py @@ -43,9 +43,10 @@ elif machine == "aarch64" or machine == "arm64" or machine == "ARM64": # The following variables can be defined in environment or .env file SCRIPT_DIR = Path(__file__).resolve().parents[0] WORKING_DIR = Path.cwd() +BUILD_BASE = f"{WORKING_DIR}/build_{PYTHON_VERSION}" OPENVINO_SOURCE_DIR = SCRIPT_DIR.parents[3] -OPENVINO_BUILD_DIR = os.getenv("OPENVINO_BUILD_DIR") -OPENVINO_PYTHON_BUILD_DIR = os.getenv("OPENVINO_PYTHON_BUILD_DIR", OPENVINO_BUILD_DIR) +OPENVINO_BINARY_DIR = os.getenv("OPENVINO_BINARY_DIR") +OPENVINO_PYTHON_BINARY_DIR = os.getenv("OPENVINO_PYTHON_BINARY_DIR", "python_build") OV_RUNTIME_LIBS_DIR = os.getenv("OV_RUNTIME_LIBS_DIR", f"runtime/{LIBS_DIR}/{ARCH}/{CONFIG}") TBB_LIBS_DIR = os.getenv("TBB_LIBS_DIR", f"runtime/3rdparty/tbb/{LIBS_DIR}") PUGIXML_LIBS_DIR = os.getenv("PUGIXML_LIBS_DIR", f"runtime/3rdparty/pugixml/{LIBS_DIR}") @@ -55,122 +56,149 @@ LIBS_RPATH = "$ORIGIN" if sys.platform == "linux" else "@loader_path" LIB_INSTALL_CFG = { "ie_libs": { "name": "core", - "prefix": f"{WORKING_DIR}/build/libs.core", + "prefix": f"{BUILD_BASE}/libs.core", "install_dir": OV_RUNTIME_LIBS_DIR, "rpath": LIBS_RPATH, - "binary_dir": OPENVINO_BUILD_DIR, + "binary_dir": OPENVINO_BINARY_DIR, }, "hetero_plugin": { "name": "hetero", - "prefix": f"{WORKING_DIR}/build/libs.hetero", + "prefix": f"{BUILD_BASE}/libs.hetero", "install_dir": OV_RUNTIME_LIBS_DIR, "rpath": LIBS_RPATH, - "binary_dir": OPENVINO_BUILD_DIR, + "binary_dir": OPENVINO_BINARY_DIR, }, "gpu_plugin": { "name": "gpu", - "prefix": f"{WORKING_DIR}/build/libs.gpu", + "prefix": f"{BUILD_BASE}/libs.gpu", "install_dir": OV_RUNTIME_LIBS_DIR, "rpath": LIBS_RPATH, - "binary_dir": OPENVINO_BUILD_DIR, + "binary_dir": OPENVINO_BINARY_DIR, }, "cpu_plugin": { "name": "cpu", - "prefix": f"{WORKING_DIR}/build/libs.cpu", + "prefix": f"{BUILD_BASE}/libs.cpu", "install_dir": OV_RUNTIME_LIBS_DIR, "rpath": LIBS_RPATH, - "binary_dir": OPENVINO_BUILD_DIR, + "binary_dir": OPENVINO_BINARY_DIR, }, "multi_plugin": { "name": "multi", - "prefix": f"{WORKING_DIR}/build/libs.multi", + "prefix": f"{BUILD_BASE}/libs.multi", "install_dir": OV_RUNTIME_LIBS_DIR, "rpath": LIBS_RPATH, - "binary_dir": OPENVINO_BUILD_DIR, + "binary_dir": OPENVINO_BINARY_DIR, }, "batch_plugin": { "name": "batch", - "prefix": f"{WORKING_DIR}/build/libs.batch", + "prefix": f"{BUILD_BASE}/libs.batch", "install_dir": OV_RUNTIME_LIBS_DIR, "rpath": LIBS_RPATH, - "binary_dir": OPENVINO_BUILD_DIR, + "binary_dir": OPENVINO_BINARY_DIR, }, "tbb_libs": { "name": "tbb", - "prefix": f"{WORKING_DIR}/build/libs.tbb", + "prefix": f"{BUILD_BASE}/libs.tbb", "install_dir": TBB_LIBS_DIR, "rpath": LIBS_RPATH, - "binary_dir": OPENVINO_BUILD_DIR, + "binary_dir": OPENVINO_BINARY_DIR, }, "pugixml_libs": { "name": "pugixml", - "prefix": f"{WORKING_DIR}/build/libs.pugixml", + "prefix": f"{BUILD_BASE}/libs.pugixml", "install_dir": PUGIXML_LIBS_DIR, - "binary_dir": OPENVINO_BUILD_DIR, + "binary_dir": OPENVINO_BINARY_DIR, }, "ir_libs": { "name": "ir", - "prefix": f"{WORKING_DIR}/build/libs.ir", + "prefix": f"{BUILD_BASE}/libs.ir", "install_dir": OV_RUNTIME_LIBS_DIR, "rpath": LIBS_RPATH, - "binary_dir": OPENVINO_BUILD_DIR, + "binary_dir": OPENVINO_BINARY_DIR, }, "paddle_libs": { "name": "paddle", - "prefix": f"{WORKING_DIR}/build/libs.paddle", + "prefix": f"{BUILD_BASE}/libs.paddle", "install_dir": OV_RUNTIME_LIBS_DIR, "rpath": LIBS_RPATH, - "binary_dir": OPENVINO_BUILD_DIR, + "binary_dir": OPENVINO_BINARY_DIR, }, "pytorch_libs": { "name": "pytorch", - "prefix": f"{WORKING_DIR}/build/libs.pytorch", + "prefix": f"{BUILD_BASE}/libs.pytorch", "install_dir": OV_RUNTIME_LIBS_DIR, "rpath": LIBS_RPATH, - "binary_dir": OPENVINO_BUILD_DIR, + "binary_dir": OPENVINO_BINARY_DIR, }, "onnx_libs": { "name": "onnx", - "prefix": f"{WORKING_DIR}/build/libs.onnx", + "prefix": f"{BUILD_BASE}/libs.onnx", "install_dir": OV_RUNTIME_LIBS_DIR, "rpath": LIBS_RPATH, - "binary_dir": OPENVINO_BUILD_DIR, + "binary_dir": OPENVINO_BINARY_DIR, }, "tensorflow_libs": { "name": "tensorflow", - "prefix": f"{WORKING_DIR}/build/libs.tensorflow", + "prefix": f"{BUILD_BASE}/libs.tensorflow", "install_dir": OV_RUNTIME_LIBS_DIR, "rpath": LIBS_RPATH, - "binary_dir": OPENVINO_BUILD_DIR, + "binary_dir": OPENVINO_BINARY_DIR, }, "tensorflow_lite_libs": { "name": "tensorflow_lite", - "prefix": f"{WORKING_DIR}/build/libs.tensorflow_lite", + "prefix": f"{BUILD_BASE}/libs.tensorflow_lite", "install_dir": OV_RUNTIME_LIBS_DIR, "rpath": LIBS_RPATH, - "binary_dir": OPENVINO_BUILD_DIR, + "binary_dir": OPENVINO_BINARY_DIR, }, } PY_INSTALL_CFG = { "pyie": { "name": f"pyie_{PYTHON_VERSION}", - "prefix": f"{WORKING_DIR}/build/site-packages", + "prefix": f"{BUILD_BASE}/site-packages", + "source_dir": f"{OPENVINO_SOURCE_DIR}/src/bindings/python", "install_dir": PY_PACKAGES_DIR, - "binary_dir": OPENVINO_PYTHON_BUILD_DIR, + "binary_dir": OPENVINO_PYTHON_BINARY_DIR, }, "pyngraph": { "name": f"pyngraph_{PYTHON_VERSION}", - "prefix": f"{WORKING_DIR}/build/site-packages", + "prefix": f"{BUILD_BASE}/site-packages", + "source_dir": f"{OPENVINO_SOURCE_DIR}/src/bindings/python", "install_dir": PY_PACKAGES_DIR, - "binary_dir": OPENVINO_PYTHON_BUILD_DIR, + "binary_dir": OPENVINO_PYTHON_BINARY_DIR, }, "pyopenvino": { "name": f"pyopenvino_{PYTHON_VERSION}", - "prefix": f"{WORKING_DIR}/build/site-packages", + "prefix": f"{BUILD_BASE}/site-packages", + "source_dir": f"{OPENVINO_SOURCE_DIR}/src/bindings/python", "install_dir": PY_PACKAGES_DIR, - "binary_dir": OPENVINO_PYTHON_BUILD_DIR, + "binary_dir": OPENVINO_PYTHON_BINARY_DIR, }, + "benchmark_app": { + "entry_point": { + "console_scripts": [ + "benchmark_app = openvino.tools.benchmark.main:main", + ], + }, + "name": f"pyopenvino_{PYTHON_VERSION}", + "prefix": f"{BUILD_BASE}/site-packages", + "source_dir": f"{OPENVINO_SOURCE_DIR}/tools/benchmark_tool", + "install_dir": PY_PACKAGES_DIR, + "binary_dir": "benchmark_app", + }, + # "model_optimizer": { # noqa: E731 + # "entry_point": { # noqa: E731 + # "console_scripts": [ # noqa: E731 + # "mo = openvino.tools.mo.main:main", # noqa: E731 + # ], # noqa: E731 + # }, # noqa: E731 + # "name": f"pyopenvino_{PYTHON_VERSION}", # noqa: E731 + # "prefix": f"{BUILD_BASE}/site-packages", # noqa: E731 + # "source_dir": f"{OPENVINO_SOURCE_DIR}/tools/mo", # noqa: E731 + # "install_dir": PY_PACKAGES_DIR, # noqa: E731 + # "binary_dir": "model_optimizer", # noqa: E731 + # }, # noqa: E731 } @@ -187,80 +215,88 @@ class PrebuiltExtension(Extension): class CustomBuild(build): """Custom implementation of build_clib.""" - cmake_build_types = ["Release", "Debug", "RelWithDebInfo", "MinSizeRel"] - user_options = [ - ("config=", None, "Build configuration [{types}].".format(types="|".join(cmake_build_types))), + user_options = build.user_options + [ ("jobs=", None, "Specifies the number of jobs to use with make."), ("cmake-args=", None, "Additional options to be passed to CMake."), + ("python-extensions-only", None, "Install Python extensions without C++ libraries."), ] + boolean_options = build.boolean_options + ['python-extensions-only'] def initialize_options(self): """Set default values for all the options that this command supports.""" super().initialize_options() - self.build_base = "build" - self.config = None + self.build_base = BUILD_BASE self.jobs = None self.cmake_args = None + self.python_extensions_only = False def finalize_options(self): """Set final values for all the options that this command supports.""" super().finalize_options() - if not self.config: - if self.debug: - self.config = "Debug" - else: - self.announce("Set default value for CMAKE_BUILD_TYPE = Release.", level=4) - self.config = "Release" - else: - build_types = [item.lower() for item in self.cmake_build_types] - try: - i = build_types.index(str(self.config).lower()) - self.config = self.cmake_build_types[i] - self.debug = True if "Debug" == self.config else False - except ValueError: - self.announce("Unsupported CMAKE_BUILD_TYPE value: " + self.config, level=4) - self.announce("Supported values: {types}".format(types=", ".join(self.cmake_build_types)), level=4) - sys.exit(1) if self.jobs is None and os.getenv("MAX_JOBS") is not None: self.jobs = os.getenv("MAX_JOBS") self.jobs = multiprocessing.cpu_count() if self.jobs is None else int(self.jobs) - def run(self): - global OPENVINO_BUILD_DIR - self.jobs = multiprocessing.cpu_count() + if self.cmake_args is None: + self.cmake_args = "" + + def cmake_build_and_install(self, install_cfg): + """Runs cmake (configure, build and install) if artfiacts are not already built / installed.""" plat_specifier = ".{0}-{1}.{2}".format(self.plat_name, *sys.version_info[:2]) - self.build_temp = os.path.join(self.build_base, "temp" + plat_specifier, self.config) + self.build_temp = os.path.join(self.build_base, "temp" + plat_specifier) self.announce(f"Create build directory: {self.build_temp}", level=3) - # if setup.py is directly called use CMake to build product - if OPENVINO_BUILD_DIR == ".": - # set path to the root of OpenVINO CMakeList file - self.announce(f"Configuring cmake project: {OPENVINO_SOURCE_DIR}", level=3) - self.spawn(["cmake", "-S" + str(OPENVINO_SOURCE_DIR), - "-B" + self.build_temp, - self.cmake_args, - "-DCMAKE_BUILD_TYPE={type}".format(type=self.config), - "-DENABLE_PYTHON=ON", - "-DBUILD_SHARED_LIBS=ON" - "-DENABLE_WHEEL=OFF", - "-DENABLE_NCC_STYLE=OFF", - "-DENABLE_CPPLINT=OFF", - "-DENABLE_TEMPLATE=OFF", - "-DENABLE_COMPILE_TOOL=OFF" - "-DENABLE_SAMPLES=OFF"]) + # build some components which have not been built yet + for comp, comp_data in install_cfg.items(): + cpack_comp_name = comp_data.get("name") + source_dir = comp_data.get("source_dir") + binary_dir = comp_data.get("binary_dir") + install_dir = comp_data.get("install_dir") + prefix = comp_data.get("prefix") - self.announce("Building binaries", level=3) - self.spawn(["cmake", "--build", self.build_temp, - "--config", self.config, - "--parallel", str(self.jobs)]) - OPENVINO_BUILD_DIR = self.build_temp - # perform installation - self.run_command("build_clib") + # perform installation steps if we are not given a full path + if not os.path.isabs(install_dir): + # install_dir is just a sub-dir after install prefix, let's make a full path + comp_data["install_dir"] = os.path.join(prefix, install_dir) - build.run(self) + # even perform a build in case of binary directory does not exist + binary_dir = binary_dir if os.path.isabs(binary_dir) else os.path.join(self.build_temp, binary_dir) + if not os.path.exists(binary_dir): + binary_dir = os.path.join(self.build_temp, binary_dir) + self.announce(f"Configuring {comp} cmake project", level=3) + self.spawn(["cmake", f"-DOpenVINODeveloperPackage_DIR={OPENVINO_BINARY_DIR}", + "-DCMAKE_BUILD_TYPE=Release", + "-DENABLE_WHEEL=OFF", + self.cmake_args, + "-S", source_dir, + "-B", binary_dir]) - # Copy extra package_data content filtered by 'copy_package_data' + self.announce(f"Building {comp} project", level=3) + self.spawn(["cmake", "--build", binary_dir, + "--config", "Release", + "--parallel", str(self.jobs)]) + + self.announce(f"Installing {comp}", level=3) + self.spawn(["cmake", "--install", binary_dir, + "--prefix", prefix, + "--config", "Release", + "--strip", + "--component", cpack_comp_name]) + + def run(self): + # build and install clib into temporary directories + if not self.python_extensions_only: + self.cmake_build_and_install(LIB_INSTALL_CFG) + + # install python code into a temporary directory (site-packages) + self.cmake_build_and_install(PY_INSTALL_CFG) + + # install clibs into a temporary directory (site-packages) + if not self.python_extensions_only: + self.run_command("build_clib") + + # Copy extra package_data content filtered by 'find_packages' exclude = ignore_patterns("*ez_setup*", "*__pycache__*", "*.egg-info*") src, dst = Path(PACKAGE_DIR), Path(self.build_lib) for path in src.glob("**/*"): @@ -272,29 +308,26 @@ class CustomBuild(build): class PrepareLibs(build_clib): - """Install prebuilt libraries.""" + """Prepares prebuilt libraries. + + The steps include: resolve symlinks for versioned shared libraries, set RPATHs for clibs + """ def run(self): - self.configure(LIB_INSTALL_CFG) - self.configure(PY_INSTALL_CFG) + """Run build_clib command.""" + # remove symlink to avoid copying it, set RPATH + self.post_install(LIB_INSTALL_CFG) + # copy clib to package data (to WHEEL_LIBS_INSTALL_DIR) self.copy_package_data(get_install_dirs_list(LIB_INSTALL_CFG)) - def configure(self, install_cfg): - """Collect prebuilt libraries. Install them to the temp directories, set rpath.""" + def post_install(self, install_cfg): + """Install prebuilt libraries to the temp directories, set rpath.""" for comp, comp_data in install_cfg.items(): - install_prefix = comp_data.get("prefix") install_dir = comp_data.get("install_dir") - binary_dir = comp_data.get("binary_dir") - # perform installation steps if we are not given a full path - if not os.path.isabs(install_dir): - self.announce(f"Installing {comp}", level=3) - self.spawn(["cmake", "--install", binary_dir, - "--prefix", install_prefix, - "--config", "Release", - "--strip", - "--component", comp_data.get("name")]) - install_dir = os.path.join(install_prefix, install_dir) + # we need to resolve symlinks before setting rpath to avoid doing it multiple times + install_dir_path = Path(install_dir) + self.resolve_symlinks(install_dir_path) # set rpath if applicable if sys.platform != "win32" and comp_data.get("rpath"): @@ -305,11 +338,12 @@ class PrepareLibs(build_clib): for path in filter( lambda x: any(item in ([".so"] if sys.platform == "linux" else [".dylib", ".so"]) - for item in x.suffixes), Path(install_dir).glob("*"), + for item in x.suffixes), install_dir_path.glob("*"), ): set_rpath(comp_data["rpath"], os.path.realpath(path)) def get_reallink(self, link_file): + """Get the real file path of a soft link.""" real_name = link_file while True: real_name = os.readlink(real_name) @@ -366,9 +400,6 @@ class PrepareLibs(build_clib): os.makedirs(package_clibs_dir, exist_ok=True) for src_dir in src_dirs: - local_base_dir = Path(src_dir) - self.resolve_symlinks(local_base_dir) - # additional blacklist filter, just to fix cmake install issues blacklist_patterns = [ # static libraries and PBD files "^.*\\.a$", "^.*\\.lib$", "^.*\\.pdb$", @@ -378,7 +409,7 @@ class PrepareLibs(build_clib): "^.*\\.la$"] # copy so / dylib files to WHEEL_LIBS_INSTALL_DIR (clibs) inside python package - for file_path in local_base_dir.rglob("*"): + for file_path in Path(src_dir).rglob("*"): file_name = os.path.basename(file_path) if file_path.is_symlink(): # sanity check for self.resolve_symlinks @@ -402,10 +433,18 @@ class PrepareLibs(build_clib): class CopyExt(build_ext): """Copy extension files to the build directory.""" + user_options = [ + ("skip-rpath", None, "Skips RPATH for Python extensions."), + ] + boolean_options = ['skip_rpath'] + + def initialize_options(self): + """Set default values for all the options that this command supports.""" + super().initialize_options() + self.skip_rpath = False + def run(self): if len(self.extensions) == 1: - # when python3 setup.py build_ext is called, while build_clib is not called before - self.run_command("build_clib") self.extensions = find_prebuilt_extensions(get_install_dirs_list(PY_INSTALL_CFG)) for extension in self.extensions: @@ -415,7 +454,7 @@ class CopyExt(build_ext): dst = self.get_ext_fullpath(extension.name) os.makedirs(os.path.dirname(dst), exist_ok=True) # setting relative RPATH to found dlls - if sys.platform != "win32": + if sys.platform != "win32" and not self.skip_rpath: rpath = os.path.relpath(get_package_dir(PY_INSTALL_CFG), os.path.dirname(src)) rpath = os.path.join(LIBS_RPATH, rpath, WHEEL_LIBS_INSTALL_DIR) set_rpath(rpath, os.path.realpath(src)) @@ -464,52 +503,52 @@ def is_tool(name): def remove_rpath(file_path): - """Remove rpath from binaries. + """Remove rpath from macOS binaries. :param file_path: binary path :type file_path: pathlib.Path """ - if sys.platform == "darwin": - cmd = ( - f"otool -l {file_path} " # noqa: P103 - f"| grep LC_RPATH -A3 " - f"| grep -o 'path.*' " - f"| cut -d ' ' -f2 " - f"| xargs -I{{}} install_name_tool -delete_rpath {{}} {file_path}" - ) - if os.WEXITSTATUS(os.system(cmd)) != 0: # nosec - sys.exit(f"Could not remove rpath for {file_path}") - else: + if not sys.platform == "darwin": sys.exit(f"Unsupported platform: {sys.platform}") + cmd = ( + f"otool -l {file_path} " # noqa: P103 + f"| grep LC_RPATH -A3 " + f"| grep -o 'path.*' " + f"| cut -d ' ' -f2 " + f"| xargs -I{{}} install_name_tool -delete_rpath {{}} {file_path}" + ) + if os.WEXITSTATUS(os.system(cmd)) != 0: # nosec + sys.exit(f"Could not remove rpath for {file_path}") -def set_rpath(rpath, executable): - """Setting rpath for linux and macOS libraries.""" - print(f"Setting rpath {rpath} for {executable}") # noqa: T001, T201 + +def set_rpath(rpath, binary): + """Setting rpath for Linux and macOS binaries.""" + print(f"Setting rpath {rpath} for {binary}") # noqa: T001, T201 cmd = [] rpath_tool = "" if sys.platform == "linux": - with open(os.path.realpath(executable), "rb") as file: + with open(os.path.realpath(binary), "rb") as file: if file.read(1) != b"\x7f": - log.warn(f"WARNING: {executable}: missed ELF header") + log.warn(f"WARNING: {binary}: missed ELF header") return rpath_tool = "patchelf" - cmd = [rpath_tool, "--set-rpath", rpath, executable] + cmd = [rpath_tool, "--set-rpath", rpath, binary] elif sys.platform == "darwin": rpath_tool = "install_name_tool" - cmd = [rpath_tool, "-add_rpath", rpath, executable] + cmd = [rpath_tool, "-add_rpath", rpath, binary] else: sys.exit(f"Unsupported platform: {sys.platform}") - if is_tool(rpath_tool): - if sys.platform == "darwin": - remove_rpath(executable) - ret_info = subprocess.run(cmd, check=True, shell=False) # nosec - if ret_info.returncode != 0: - sys.exit(f"Could not set rpath: {rpath} for {executable}") - else: - sys.exit(f"Could not found {rpath_tool} on the system, " f"please make sure that this tool is installed") + if not is_tool(rpath_tool): + sys.exit(f"Could not find {rpath_tool} on the system, " f"please make sure that this tool is installed") + + if sys.platform == "darwin": + remove_rpath(binary) + ret_info = subprocess.run(cmd, check=True, shell=False) # nosec + if ret_info.returncode != 0: + sys.exit(f"Could not set rpath: {rpath} for {binary}") def find_prebuilt_extensions(search_dirs): @@ -575,6 +614,18 @@ def get_package_dir(install_cfg): return py_package_path +def find_entry_points(install_cfg): + """Creates a list of entry points for OpenVINO runtime package.""" + entry_points = { + "console_scripts": [], + } + for comp_info in install_cfg.values(): + empty_point = comp_info.get("entry_point") + if empty_point is not None: + entry_points["console_scripts"].append(empty_point["console_scripts"]) + return entry_points + + def concat_files(input_files, output_file): """Concatenates multuple input files to a single output file.""" with open(output_file, "w", encoding="utf-8") as outfile: @@ -590,6 +641,7 @@ PACKAGE_DIR = get_package_dir(PY_INSTALL_CFG) packages = find_namespace_packages(PACKAGE_DIR) package_data: typing.Dict[str, list] = {} ext_modules = find_prebuilt_extensions(get_install_dirs_list(PY_INSTALL_CFG)) +entry_points = find_entry_points(PY_INSTALL_CFG) long_description_md = OPENVINO_SOURCE_DIR / "docs" / "install_guides" / "pypi-openvino-rt.md" md_files = [long_description_md, OPENVINO_SOURCE_DIR / "docs" / "install_guides" / "pre-release-note.md"] @@ -615,6 +667,7 @@ setup( long_description_content_type="text/markdown", download_url="https://github.com/openvinotoolkit/openvino/releases", url=docs_url, + entry_points=entry_points, cmdclass={ "build": CustomBuild, "install": CustomInstall, diff --git a/src/plugins/intel_cpu/thirdparty/ACLConfig.cmake b/src/plugins/intel_cpu/thirdparty/ACLConfig.cmake index ff23d1ca1e7..2c0b7d12ee3 100644 --- a/src/plugins/intel_cpu/thirdparty/ACLConfig.cmake +++ b/src/plugins/intel_cpu/thirdparty/ACLConfig.cmake @@ -326,9 +326,14 @@ elseif(NOT TARGET arm_compute::arm_compute) endif() if(ENABLE_LTO) - if((CMAKE_COMPILER_IS_GNUCXX OR OV_COMPILER_IS_CLANG) AND NOT CMAKE_CROSSCOMPILING) - set(extra_cxx_flags "${extra_cxx_flags} -flto=thin") - set(extra_link_flags "${extra_link_flags} -flto=thin") + if(NOT CMAKE_CROSSCOMPILING) + if(CMAKE_COMPILER_IS_GNUCXX) + set(extra_cxx_flags "${extra_cxx_flags} -flto -fno-fat-lto-objects") + set(extra_link_flags "${extra_link_flags} -flto -fno-fat-lto-objects") + elseif(OV_COMPILER_IS_CLANG) + set(extra_cxx_flags "${extra_cxx_flags} -flto=thin") + set(extra_link_flags "${extra_link_flags} -flto=thin") + endif() endif() endif() diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 2fe73bfd125..89a14833d31 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -36,6 +36,12 @@ add_subdirectory(mo) configure_file("${CMAKE_CURRENT_SOURCE_DIR}/pot/openvino/tools/pot/version.txt.in" "${CMAKE_CURRENT_SOURCE_DIR}/pot/openvino/tools/pot/version.txt" @ONLY) +# Benchmark Tool + +if(ENABLE_PYTHON) + add_subdirectory(benchmark_tool) +endif() + # wheel openvino-dev add_subdirectory(openvino_dev) diff --git a/tools/benchmark_tool/CMakeLists.txt b/tools/benchmark_tool/CMakeLists.txt new file mode 100644 index 00000000000..be66a9e7c24 --- /dev/null +++ b/tools/benchmark_tool/CMakeLists.txt @@ -0,0 +1,36 @@ +# Copyright (C) 2018-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +cmake_minimum_required (VERSION 3.13) + +project(OpenVINOBenchmarkTool) + +# +# Packages & settings +# + +if(NOT DEFINED OpenVINO_SOURCE_DIR) + get_filename_component(OpenVINO_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../.." REALPATH) +endif() + +if(NOT IEDevScripts_FOUND) + find_package(IEDevScripts REQUIRED + PATHS "${OpenVINO_SOURCE_DIR}/cmake/developer_package" + NO_CMAKE_FIND_ROOT_PATH + NO_DEFAULT_PATH) +endif() + +# +# Installation rules +# + +ov_get_pyversion(pyversion) +ov_cpack_add_component(${OV_CPACK_COMP_PYTHON_OPENVINO}_${pyversion} + HIDDEN) + +install(DIRECTORY ${OpenVINOBenchmarkTool_SOURCE_DIR}/openvino + DESTINATION ${OV_CPACK_PYTHONDIR} + COMPONENT ${OV_CPACK_COMP_PYTHON_OPENVINO}_${pyversion} + ${OV_CPACK_COMP_PYTHON_OPENVINO_EXCLUDE_ALL} + USE_SOURCE_PERMISSIONS)