From 4c92ffa563d787e49e815f181350517c54c12dba Mon Sep 17 00:00:00 2001 From: Ilya Lavrenov Date: Tue, 23 May 2023 00:31:35 +0400 Subject: [PATCH] Build wheel arm64 packages using cross-compilation (#17635) --- .ci/azure/linux_arm64.yml | 44 +++++++++++++++++-- src/bindings/python/CMakeLists.txt | 4 ++ .../intel_gpu/thirdparty/CMakeLists.txt | 15 +++++++ .../mock_mo_python_api/CMakeLists.txt | 4 ++ 4 files changed, 63 insertions(+), 4 deletions(-) diff --git a/.ci/azure/linux_arm64.yml b/.ci/azure/linux_arm64.yml index 0ffe9655cc0..6312d7a6e9f 100644 --- a/.ci/azure/linux_arm64.yml +++ b/.ci/azure/linux_arm64.yml @@ -56,7 +56,8 @@ jobs: TMP_DIR: /mnt/tmp OPENVINO_CCACHE_DIR: $(SHARE_DIR)/ccache/master/linux_arm64 LD_LIBRARY_PATH: $(Agent.ToolsDirectory)/Python/$(OV_PYTHON_VERSION)/x64/lib - OV_PYTHON_VERSION: 3.11.2 # Full version of Python its required for LD_LIBRARY_PATH. More details https://github.com/microsoft/azure-pipelines-tool-lib/blob/master/docs/overview.md#tool-cache + OV_PYTHON_VERSION_MAJOR_MINOR: 3.11 + OV_PYTHON_VERSION: $(OV_PYTHON_VERSION_MAJOR_MINOR).2 # Full version of Python its required for LD_LIBRARY_PATH. More details https://github.com/microsoft/azure-pipelines-tool-lib/blob/master/docs/overview.md#tool-cache steps: - task: UsePythonVersion@0 @@ -114,15 +115,44 @@ jobs: python3 -m pip install --upgrade pip python3 -m pip install -r $(OPENVINO_REPO_DIR)/src/bindings/python/requirements.txt python3 -m pip install -r $(OPENVINO_REPO_DIR)/src/bindings/python/wheel/requirements-dev.txt + python3 -m pip install -r $(OPENVINO_REPO_DIR)/src/bindings/python/src/compatibility/openvino/requirements-dev.txt # install dependencies needed to build CPU plugin for ARM sudo -E apt --assume-yes install scons gcc-10-aarch64-linux-gnu g++-10-aarch64-linux-gnu # generic dependencies sudo -E apt --assume-yes install cmake ccache ninja-build unzip - displayName: 'Install dependencies' + displayName: 'Install build dependencies' + + - script: | + set -e + echo deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ focal main restricted > arm64-sources.list + echo deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ focal-updates main restricted >> arm64-sources.list + echo deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ focal universe >> arm64-sources.list + echo deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ focal-updates universe >> arm64-sources.list + echo deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ focal multiverse >> arm64-sources.list + echo deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ focal-updates multiverse >> arm64-sources.list + echo deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ focal-backports main restricted universe multiverse >> arm64-sources.list + echo deb [arch=amd64] http://security.ubuntu.com/ubuntu/ focal-security main restricted >> arm64-sources.list + echo deb [arch=amd64] http://security.ubuntu.com/ubuntu/ focal-security universe >> arm64-sources.list + echo deb [arch=amd64] http://security.ubuntu.com/ubuntu/ focal-security multiverse >> arm64-sources.list + echo deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ focal main >> arm64-sources.list + echo deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ focal universe >> arm64-sources.list + echo deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ focal-updates main >> arm64-sources.list + echo deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ focal-security main >> arm64-sources.list + sudo mv arm64-sources.list /etc/apt/sources.list.d/ + sudo -E dpkg --add-architecture arm64 + sudo -E apt-get update -o Dir::Etc::sourcelist=/etc/apt/sources.list.d/arm64-sources.list + sudo -E apt-get install -y --no-install-recommends \ + libpython3-dev:arm64 \ + ocl-icd-opencl-dev:arm64 \ + opencl-headers:arm64 + displayName: 'Install arm64 libraries' - script: | git submodule update --init -- $(OPENVINO_REPO_DIR)/src/plugins git submodule update --init -- $(OPENVINO_REPO_DIR)/thirdparty/gtest + git submodule update --init -- $(OPENVINO_REPO_DIR)/thirdparty/ocl/cl_headers + git submodule update --init -- $(OPENVINO_REPO_DIR)/thirdparty/ocl/clhpp_headers + git submodule update --init -- $(OPENVINO_REPO_DIR)/thirdparty/open_model_zoo displayName: 'Init submodules for non Conan dependencies' - script: | @@ -158,8 +188,14 @@ jobs: -DCMAKE_VERBOSE_MAKEFILE=ON \ -DBUILD_SHARED_LIBS=ON \ -DCMAKE_COMPILE_WARNING_AS_ERROR=ON \ - -DENABLE_CPPLINT=OFF \ - -DENABLE_PYTHON=OFF \ + -DENABLE_CPPLINT=ON \ + -DENABLE_INTEL_GPU=ON \ + -DENABLE_PYTHON=ON \ + -DENABLE_WHEEL=ON \ + -DPYBIND11_PYTHONLIBS_OVERWRITE=OFF \ + -DPYTHON_MODULE_EXTENSION=$(aarch64-linux-gnu-python3-config --extension-suffix) \ + -DPYTHON_LIBRARY=/usr/lib/aarch64-linux-gnu/libc-2.31.so \ + -DPYTHON_INCLUDE_DIR=$(Agent.ToolsDirectory)/Python/$(OV_PYTHON_VERSION)/x64/include/python$(OV_PYTHON_VERSION_MAJOR_MINOR) \ -DENABLE_TESTS=ON \ -DENABLE_DATA=OFF \ -DENABLE_SYSTEM_TBB=ON \ diff --git a/src/bindings/python/CMakeLists.txt b/src/bindings/python/CMakeLists.txt index 6602571c292..8ac8a6429ea 100644 --- a/src/bindings/python/CMakeLists.txt +++ b/src/bindings/python/CMakeLists.txt @@ -41,6 +41,10 @@ function(ov_check_python_build_conditions) else() set(pybind11_min_version 2.8.0) endif() + if(CMAKE_CROSSCOMPILING) + # since this version pybind11 has PYBIND11_PYTHONLIBS_OVERWRITE option + set(pybind11_min_version 2.10.1) + endif() function(_ov_find_python_libs_new) set(pybind11_tools_dir "${OpenVINOPython_SOURCE_DIR}/thirdparty/pybind11/tools") diff --git a/src/plugins/intel_gpu/thirdparty/CMakeLists.txt b/src/plugins/intel_gpu/thirdparty/CMakeLists.txt index cec5ca58fcd..788f3ca350d 100644 --- a/src/plugins/intel_gpu/thirdparty/CMakeLists.txt +++ b/src/plugins/intel_gpu/thirdparty/CMakeLists.txt @@ -23,6 +23,20 @@ if(ENABLE_ONEDNN_FOR_GPU) set(ONEDNN_ENABLED_PRIMITIVES "CONCAT;CONVOLUTION;DECONVOLUTION;INNER_PRODUCT;MATMUL;REORDER;POOLING;REDUCTION") set(ONEDNN_ENABLED_ISA "XEHP;XEHPG;XEHPC") + if(X86_64) + set(ONEDNN_TARGET_ARCH "X64" CACHE STRING "" FORCE) + elseif(X86) + set(ONEDNN_TARGET_ARCH "X86" CACHE STRING "" FORCE) + elseif(RISCV64) + set(ONEDNN_TARGET_ARCH "RV64" CACHE STRING "" FORCE) + elseif(ARM) + set(ONEDNN_TARGET_ARCH "ARM" CACHE STRING "" FORCE) + elseif(AARCH64) + set(ONEDNN_TARGET_ARCH "AARCH64" CACHE STRING "" FORCE) + else() + message(WARNING "Intel GPU plugin unsupported architecture: ${CMAKE_SYSTEM_PROCESSOR}") + endif() + if(CMAKE_COMPILER_IS_GNUCXX OR OV_COMPILER_IS_CLANG) ie_add_compiler_flags(-Wno-undef) ie_add_compiler_flags(-Wno-missing-declarations) @@ -87,6 +101,7 @@ if(ENABLE_ONEDNN_FOR_GPU) "-DCMAKE_C_FLAGS=${CMAKE_C_FLAGS}" "-DCMAKE_INTERPROCEDURAL_OPTIMIZATION_RELEASE=${ENABLE_LTO}" "-DCMAKE_POLICY_DEFAULT_CMP0069=NEW" + "-DDNNL_TARGET_ARCH=${ONEDNN_TARGET_ARCH}" "-DDNNL_CPU_RUNTIME=NONE" "-DDNNL_GPU_RUNTIME=OCL" "-DDNNL_LIBRARY_NAME=onednn_gpu" diff --git a/tools/mo/unit_tests/mock_mo_frontend/mock_mo_python_api/CMakeLists.txt b/tools/mo/unit_tests/mock_mo_frontend/mock_mo_python_api/CMakeLists.txt index 78b1ba08d1b..a5e0daa0a02 100644 --- a/tools/mo/unit_tests/mock_mo_frontend/mock_mo_python_api/CMakeLists.txt +++ b/tools/mo/unit_tests/mock_mo_frontend/mock_mo_python_api/CMakeLists.txt @@ -33,6 +33,10 @@ if(PYTHON_VERSION_MINOR GREATER_EQUAL 11) else() set(pybind11_min_version 2.8.0) endif() +if(CMAKE_CROSSCOMPILING) + # since this version pybind11 has PYBIND11_PYTHONLIBS_OVERWRITE + set(pybind11_min_version 2.10.1) +endif() find_package(pybind11 ${pybind11_min_version} QUIET)