Debian packages on GHA (#20033)

* Refactored Linix GHA pipeline * Move ONNX tests to Unit * Install tests together with openvino * Added checks for GNA usage in docs python snippets * Install openssl for working pip * Disabled onnx_dyn_shapes_expand_1_dyn_shape ONNX test * exclude dyn shapes from onnx * Migrated samples tests * Fixed ONNX tests * caches for PT tests dependencies * Build Contrib separately from main OpenVINO * More fixes * Added python install for samples stage * Installed manifests for ONNX FE tests * Try to avoid libgl1 dependency in tests * Clean-up * Fixed issue with manifests in ONNX tests * Try to build debian packages in GHA * Fixed debian packages build for multi-config generators
2023-09-26 01:54:41 +04:00 · 2023-09-26 01:54:41 +04:00 · 5616c26d19
commit 5616c26d19
parent d6d6888d7a
6 changed files with 122 additions and 38 deletions
--- a/.github/workflows/linux.yml
+++ b/.github/workflows/linux.yml
@ -24,7 +24,7 @@ on:
      - master

 concurrency:
-  group: ${{ github.ref || github.run_id }}-linux
+  group: ${{ github.head_ref || github.run_id }}-linux
  cancel-in-progress: true

 jobs:
@ -96,7 +96,6 @@ jobs:
        run: |
          # For Python API: build and wheel packaging
          python3 -m pip install -r ${OPENVINO_REPO}/src/bindings/python/wheel/requirements-dev.txt
-          python3 -m pip install -r ${OPENVINO_REPO}/src/bindings/python/requirements.txt
          python3 -m pip install -r ${OPENVINO_REPO}/src/bindings/python/src/compatibility/openvino/requirements-dev.txt

          # For running ONNX frontend unit tests
@ -130,14 +129,12 @@ jobs:
            -DENABLE_SYSTEM_TBB=ON \
            -DENABLE_SYSTEM_OPENCL=ON \
            -DENABLE_SYSTEM_PUGIXML=ON \
-            -DBUILD_nvidia_plugin=OFF \
-            -DCUSTOM_OPERATIONS="calculate_grid;complex_mul;fft;grid_sample;sparse_conv;sparse_conv_transpose" \
-            -DOPENVINO_EXTRA_MODULES=${OPENVINO_CONTRIB_REPO}/modules \
            -DCMAKE_VERBOSE_MAKEFILE=ON \
-            -DCMAKE_BUILD_TYPE=${{ env.CMAKE_BUILD_TYPE }} \
+            -DCPACK_GENERATOR=TGZ \
            -DCMAKE_COMPILE_WARNING_AS_ERROR=ON \
            -DCMAKE_CXX_COMPILER_LAUNCHER=${{ env.CMAKE_CXX_COMPILER_LAUNCHER }} \
            -DCMAKE_C_COMPILER_LAUNCHER=${{ env.CMAKE_C_COMPILER_LAUNCHER }} \
+            -DCMAKE_MINIMUM_REQUIRED_VERSION=3.20 \
            -S ${OPENVINO_REPO} \
            -B ${BUILD_DIR}

@ -147,19 +144,6 @@ jobs:
      - name: Cmake build - OpenVINO
        run: cmake --build ${BUILD_DIR} --parallel --config ${{ env.CMAKE_BUILD_TYPE }}

-      - name: Cmake & Build - OpenVINO Contrib
-        if: ${{ 'false' }}
-        run: |
-          cmake \
-            -G "${{ env.CMAKE_GENERATOR }}" \
-            -DBUILD_nvidia_plugin=OFF \
-            -DCUSTOM_OPERATIONS="calculate_grid;complex_mul;fft;grid_sample;sparse_conv;sparse_conv_transpose" \
-            -DOpenVINODeveloperPackage_DIR=${BUILD_DIR} \
-            -S ${OPENVINO_CONTRIB_REPO}/modules \
-            -B ${OPENVINO_CONTRIB_BUILD_DIR}
-
-          cmake --build ${OPENVINO_CONTRIB_BUILD_DIR} --parallel --config ${{ env.CMAKE_BUILD_TYPE }}
-
      - name: Show ccache stats
        run: ccache --show-stats

@ -179,6 +163,30 @@ jobs:
            tar -czvf ${BUILD_DIR}/openvino_tests.tar.gz *
          popd

+      - name: Build Debian packages
+        run: |
+          /usr/bin/python3.8 -m pip install -U pip
+          /usr/bin/python3.8 -m pip install -r ${OPENVINO_REPO}/src/bindings/python/wheel/requirements-dev.txt
+          /usr/bin/python3.8 -m pip install -r ${OPENVINO_REPO}/src/bindings/python/src/compatibility/openvino/requirements-dev.txt
+          cmake -UPYTHON* \
+                -DCPACK_GENERATOR=DEB \
+                -DENABLE_PYTHON_PACKAGING=ON \
+                -DPYTHON_EXECUTABLE=/usr/bin/python3.8 \
+                -DENABLE_TESTS=OFF \
+                -S ${OPENVINO_REPO} \
+                -B ${BUILD_DIR}
+          cmake --build ${BUILD_DIR} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} --target package
+
+      - name: Cmake & Build - OpenVINO Contrib
+        run: |
+          cmake \
+            -DBUILD_nvidia_plugin=OFF \
+            -DCUSTOM_OPERATIONS="calculate_grid;complex_mul;fft;grid_sample;sparse_conv;sparse_conv_transpose" \
+            -DOPENVINO_EXTRA_MODULES=${OPENVINO_CONTRIB_REPO}/modules \
+            -S ${OPENVINO_REPO} \
+            -B ${BUILD_DIR}
+          cmake --build ${BUILD_DIR} --parallel --config ${{ env.CMAKE_BUILD_TYPE }}
+
      #
      # Upload build artifacts
      #
@ -191,6 +199,14 @@ jobs:
          path: ${{ env.BUILD_DIR }}/openvino_package.tar.gz
          if-no-files-found: 'error'

+      - name: Upload openvino debian packages
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: openvino_debian_packages
+          path: ${{ env.BUILD_DIR }}/*.deb
+          if-no-files-found: 'error'
+
      - name: Upload openvino tests package
        if: ${{ always() }}
        uses: actions/upload-artifact@v3
@ -199,6 +215,57 @@ jobs:
          path: ${{ env.BUILD_DIR }}/openvino_tests.tar.gz
          if-no-files-found: 'error'

+  Debian_Packages:
+    needs: Build
+    defaults:
+      run:
+        shell: bash
+    runs-on: ubuntu-20.04
+    container:
+      image: ubuntu:20.04
+    env:
+      DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input
+      DEBIAN_PACKAGES_DIR: /__w/openvino/packages/
+
+    steps:
+      - name: Create Directories
+        run: mkdir -p ${DEBIAN_PACKAGES_DIR}
+
+      - name: Download OpenVINO debian packages
+        uses: actions/download-artifact@v3
+        with:
+          name: openvino_debian_packages
+          path: ${{ env.DEBIAN_PACKAGES_DIR }}
+
+      - name: Install debian packages & check conflicts
+        run: |
+          apt-get update -y
+          # Install debian packages from previous release
+          apt-get install --no-install-recommends -y gnupg wget ca-certificates
+          wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
+          apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
+          echo "deb https://apt.repos.intel.com/openvino/2023 ubuntu20 main" | tee /etc/apt/sources.list.d/intel-openvino-2023.list
+          apt-get update -y
+          apt-get install -y openvino
+          # install our local one and make sure the conflicts are resolved
+          apt-get install --no-install-recommends -y dpkg-dev
+          dpkg-scanpackages . /dev/null | gzip -9c > Packages.gz
+          echo "deb [trusted=yes] file:${DEBIAN_PACKAGES_DIR} ./" | tee /etc/apt/sources.list.d/openvino-local.list
+          apt-get update -y
+          apt-get install openvino -y
+        working-directory: ${{ env.DEBIAN_PACKAGES_DIR }}
+
+      - name: Test debian packages
+        run: |
+          /usr/share/openvino/samples/cpp/build_samples.sh
+          /usr/share/openvino/samples/c/build_samples.sh
+          ~/openvino_cpp_samples_build/intel64/Release/hello_query_device
+          python3 /usr/share/openvino/samples/python/hello_query_device/hello_query_device.py
+          python3 -c 'from openvino import Core; print(Core().available_devices)'
+          python3 -c 'from openvino.frontend import FrontEndManager; assert len(FrontEndManager().get_available_front_ends()) == 6'
+          benchmark_app --help
+          ovc --help
+
  Samples:
    needs: Build
    defaults:
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@ -2,22 +2,26 @@
 # SPDX-License-Identifier: Apache-2.0
 #

-if(DEFINED BUILD_SHARED_LIBS AND NOT BUILD_SHARED_LIBS)
-    # 3.17: 'target_link_libraries' does not work correctly when called from
-    # different directory where 'add_library' is called: CMake generates
-    # incorrect OpenVINOConfig.cmake in this case
-    # 3.18: add_library cannot create ALIAS for non-GLOBAL targets
-    cmake_minimum_required(VERSION 3.18)
-else()
-    if(CPACK_GENERATOR STREQUAL "DEB")
-        # we have to use CPACK_DEBIAN_PACKAGE_SHLIBDEPS_PRIVATE_DIRS variable
-        cmake_minimum_required(VERSION 3.20)
+if(NOT DEFINED CMAKE_MINIMUM_REQUIRED_VERSION)
+    if(DEFINED BUILD_SHARED_LIBS AND NOT BUILD_SHARED_LIBS)
+        # 3.17: 'target_link_libraries' does not work correctly when called from
+        # different directory where 'add_library' is called: CMake generates
+        # incorrect OpenVINOConfig.cmake in this case
+        # 3.18: add_library cannot create ALIAS for non-GLOBAL targets
+        set(CMAKE_MINIMUM_REQUIRED_VERSION 3.18)
    else()
-        # default choice
-        cmake_minimum_required(VERSION 3.13)
+        if(CPACK_GENERATOR STREQUAL "DEB")
+            # we have to use CPACK_DEBIAN_PACKAGE_SHLIBDEPS_PRIVATE_DIRS variable
+            set(CMAKE_MINIMUM_REQUIRED_VERSION 3.20)
+        else()
+            # default choice
+            set(CMAKE_MINIMUM_REQUIRED_VERSION 3.13)
+        endif()
    endif()
 endif()

+cmake_minimum_required(VERSION ${CMAKE_MINIMUM_REQUIRED_VERSION})
+
 if(POLICY CMP0091)
    cmake_policy(SET CMP0091 NEW) # Enables use of MSVC_RUNTIME_LIBRARY
 endif()
--- a/cmake/developer_package/features.cmake
+++ b/cmake/developer_package/features.cmake
@ -5,6 +5,9 @@
 include(options)
 include(target_flags)

+set (CPACK_GENERATOR "TGZ" CACHE STRING "Cpack generator for OpenVINO")
+list (APPEND IE_OPTIONS CPACK_GENERATOR)
+
 # FIXME: there are compiler failures with LTO and Cross-Compile toolchains. Disabling for now, but
 #        this must be addressed in a proper way
 ie_dependent_option (ENABLE_LTO "Enable Link Time Optimization" OFF
--- a/cmake/developer_package/packaging/debian/debian.cmake
+++ b/cmake/developer_package/packaging/debian/debian.cmake
@ -21,6 +21,10 @@ macro(ov_debian_cpack_set_dirs)
            set(OV_CPACK_RUNTIMEDIR "${OV_CPACK_RUNTIMEDIR}/aarch64-linux-gnu")
        elseif(X86)
            set(OV_CPACK_RUNTIMEDIR "${OV_CPACK_RUNTIMEDIR}/i386-linux-gnu")
+        elseif(X86_64)
+            set(OV_CPACK_RUNTIMEDIR "${OV_CPACK_RUNTIMEDIR}/x86_64-linux-gnu")
+        elseif(RISCV64)
+            set(OV_CPACK_RUNTIMEDIR "${OV_CPACK_RUNTIMEDIR}/riscv64-linux-gnu")
        endif()
    endif()
    set(OV_CPACK_LIBRARYDIR ${OV_CPACK_RUNTIMEDIR})
@ -144,7 +148,14 @@ macro(ov_debian_specific_settings)
    # CPACK_COMPONENT_<UCOMP>_DEPENDS variables

    if(DEFINED CMAKE_LIBRARY_OUTPUT_DIRECTORY)
-        set(CPACK_DEBIAN_PACKAGE_SHLIBDEPS_PRIVATE_DIRS "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}")
+        if(OV_GENERATOR_MULTI_CONFIG)
+            # $<CONFIG> generator expression does not work in this place, have to add all possible configs
+            foreach(config IN LISTS CMAKE_CONFIGURATION_TYPES)
+                list(APPEND CPACK_DEBIAN_PACKAGE_SHLIBDEPS_PRIVATE_DIRS "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/${config}")
+            endforeach()
+        else()
+            set(CPACK_DEBIAN_PACKAGE_SHLIBDEPS_PRIVATE_DIRS "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}")
+        endif()
    else()
        message(FATAL_ERROR "CMAKE_LIBRARY_OUTPUT_DIRECTORY is empty")
    endif()
@ -169,6 +180,10 @@ macro(ov_debian_specific_settings)
            set(CPACK_DEBIAN_PACKAGE_ARCHITECTURE armhf)
        elseif(x86)
            set(CPACK_DEBIAN_PACKAGE_ARCHITECTURE i386)
+        elseif(X86_64)
+            set(CPACK_DEBIAN_PACKAGE_ARCHITECTURE x86_64)
+        elseif(RISCV64)
+            set(CPACK_DEBIAN_PACKAGE_ARCHITECTURE riscv64)
        endif()
    endif()

--- a/cmake/developer_package/packaging/packaging.cmake
+++ b/cmake/developer_package/packaging/packaging.cmake
@ -128,12 +128,6 @@ endmacro()

 ov_define_component_names()

-if(NOT DEFINED CPACK_GENERATOR)
-    set(CPACK_GENERATOR "TGZ")
-elseif(NOT CPACK_GENERATOR)
-    message(FATAL_ERROR "CPACK_GENERATOR cannot contain an empty value")
-endif()
-
 #
 # Include generator specific configuration file:
 # 1. Overrides directories set by ov_<debian | rpm | archive | common_libraries>_cpack_set_dirs()
--- a/cmake/features.cmake
+++ b/cmake/features.cmake
@ -5,6 +5,7 @@
 #
 # Common cmake options
 #
+
 ov_option (ENABLE_PROXY "Proxy plugin for OpenVINO Runtime" ON)

 ie_dependent_option (ENABLE_INTEL_CPU "CPU plugin for OpenVINO Runtime" ON "RISCV64 OR X86 OR X86_64 OR AARCH64 OR ARM" OFF)