Merge Linux CC + static build + clang compiler (#20243)

* Merge Linux CC + static build + clang compiler

* Improvements

* Fixes

.github/workflows/fedora.yml

@@ -178,9 +178,6 @@ jobs:
       RPM_PACKAGES_DIR: /__w/openvino/packages/
 
     steps:
-      - name: Create Directories
-        run: mkdir -p ${RPM_PACKAGES_DIR}
-
       - name: Download OpenVINO RPM packages
         uses: actions/download-artifact@v3
         with:

.github/workflows/linux.yml

@@ -90,6 +90,7 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: ${{ env.PYTHON_VERSION }}
+
       - name: Setup pip cache dir
         run: |
           PIP_VER=$(python3 -c "import pip; print(pip.__version__)")
@@ -232,9 +233,6 @@ jobs:
       DEBIAN_PACKAGES_DIR: /__w/openvino/packages/
 
     steps:
-      - name: Create Directories
-        run: mkdir -p ${DEBIAN_PACKAGES_DIR}
-
       - name: Download OpenVINO debian packages
         uses: actions/download-artifact@v3
         with:
@@ -290,13 +288,6 @@ jobs:
       BUILD_DIR: /__w/openvino/openvino/build
 
     steps:
-      - name: Create Directories
-        run: mkdir -p ${INSTALL_DIR} ${INSTALL_TEST_DIR}
-
-      #
-      # Initialize OpenVINO
-      #
-
       - name: Download OpenVINO package
         uses: actions/download-artifact@v3
         with:
@@ -324,6 +315,7 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: ${{ env.PYTHON_VERSION }}
+
       - name: Setup pip cache dir
         run: |
           PIP_VER=$(python3 -c "import pip; print(pip.__version__)")
@@ -615,13 +607,6 @@ jobs:
       INSTALL_TEST_DIR: /__w/openvino/openvino/install/tests
 
     steps:
-      - name: Create Directories
-        run: mkdir -p ${INSTALL_DIR} ${INSTALL_TEST_DIR}
-
-      #
-      # Initialize OpenVINO
-      #
-
       - name: Download OpenVINO package
         uses: actions/download-artifact@v3
         with:
@@ -812,9 +797,6 @@ jobs:
       LAYER_TESTS_INSTALL_DIR: /__w/openvino/openvino/install/tests/layer_tests
 
     steps:
-      - name: Create Directories
-        run: mkdir -p ${INSTALL_DIR} ${INSTALL_TEST_DIR}
-
       - name: Install git
         run: |
           apt update
@@ -832,6 +814,7 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: ${{ env.PYTHON_VERSION }}
+
       - name: Setup pip cache dir
         run: |
           PIP_VER=$(python3 -c "import pip; print(pip.__version__)")
@@ -1033,9 +1016,6 @@ jobs:
       PARALLEL_TEST_CACHE: /__w/openvino/openvino/install/tests/test_cache.lst
 
     steps:
-      - name: Create Directories
-        run: mkdir -p ${INSTALL_DIR} ${INSTALL_TEST_DIR}
-
       - name: Download OpenVINO package
         uses: actions/download-artifact@v3
         with:
@@ -1121,15 +1101,13 @@ jobs:
       MODEL_HUB_TESTS_INSTALL_DIR: ${{ github.workspace }}/install/tests/model_hub_tests
 
     steps:
-      - name: Create Directories
-        run: mkdir -p ${INSTALL_DIR} ${INSTALL_TEST_DIR}
-
       - name: Install 'actions/setup-python@v4' dependencies
         run: apt-get update && apt-get install -y libssl1.1 ca-certificates
 
       - uses: actions/setup-python@v4
         with:
           python-version: ${{ env.PYTHON_VERSION }}
+
       - name: Setup pip cache dir
         run: |
           PIP_VER=$(python3 -c "import pip; print(pip.__version__)")
@@ -1209,9 +1187,6 @@ jobs:
           # install 'g++' to build 'detectron2' and 'natten' wheels
           sudo apt-get install --assume-yes --no-install-recommends g++ git ca-certificates
 
-      - name: Create Directories
-        run: mkdir -p ${INSTALL_DIR} ${INSTALL_TEST_DIR}
-
       - name: Download OpenVINO package
         uses: actions/download-artifact@v3
         with:

.github/workflows/linux_conditional_compilation.yml

@@ -1,4 +1,4 @@
-name: Linux Conditional Compilation (Ubuntu 22.04, Python 3.11)
+name: Linux Static CC (Ubuntu 22.04, Python 3.11, Clang)
 on:
   workflow_dispatch:
   pull_request:
@@ -25,6 +25,10 @@ concurrency:
   group: ${{ github.event_name == 'push' && github.run_id || github.ref }}-linux-cc
   cancel-in-progress: true
 
+env:
+  PIP_CACHE_PATH: /mount/caches/pip/linux
+  PYTHON_VERSION: '3.11'
+
 jobs:
   Build:
     defaults:
@@ -43,13 +47,15 @@ jobs:
       CMAKE_C_COMPILER_LAUNCHER: ccache
       GITHUB_WORKSPACE: '/__w/openvino/openvino'
       OPENVINO_REPO: /__w/openvino/openvino/openvino
+      INSTALL_DIR: /__w/openvino/openvino/openvino_install
       BUILD_DIR: /__w/openvino/openvino/openvino_build
+      SELECTIVE_BUILD_STAT_DIR: /__w/openvino/openvino/selective_build_stat
       MODELS_PATH: /__w/openvino/openvino/testdata
-      CCACHE_DIR: /mount/caches/ccache/ubuntu22_x86_64_cc_Release
+      CCACHE_DIR: /mount/caches/ccache/ubuntu22_x86_64_itt_clang_Release
       CCACHE_TEMPDIR: /__w/openvino/openvino/ccache_temp
-      CCACHE_MAXSIZE: 50G
-    steps:
+      CCACHE_MAXSIZE: 20G
 
+    steps:
       - name: Install git
         run: |
           apt-get update
@@ -76,44 +82,265 @@ jobs:
         run: |
           bash ${OPENVINO_REPO}/install_build_dependencies.sh
 
+          # use clang as a default compiler
+          apt --assume-yes install clang
+          update-alternatives --install /usr/bin/cc cc /usr/bin/clang 100
+          update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++ 100
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Setup pip cache dir
+        run: |
+          PIP_VER=$(python3 -c "import pip; print(pip.__version__)")
+          echo "Using pip version: ${PIP_VER}"
+          echo "PIP_CACHE_DIR=${PIP_CACHE_PATH}/${PIP_VER}" >> $GITHUB_ENV
+
+      - name: Install python dependencies
+        run: |
+          # For running ONNX frontend unit tests
+          python3 -m pip install --force-reinstall -r ${OPENVINO_REPO}/src/frontends/onnx/tests/requirements.txt
+
+          # For running TensorFlow frontend unit tests
+          python3 -m pip install -r ${OPENVINO_REPO}/src/frontends/tensorflow/tests/requirements.txt
+
+          # For running TensorFlow Lite frontend unit tests
+          python3 -m pip install -r ${OPENVINO_REPO}/src/frontends/tensorflow_lite/tests/requirements.txt
+
+          # For running Paddle frontend unit tests
+          python3 -m pip install -r ${OPENVINO_REPO}/src/frontends/paddle/tests/requirements.txt
+
       #
       # Build
       #
 
-      - name: CMake configure CC COLLECT
+      - name: CMake configure - CC COLLECT
         run: |
           cmake \
             -G "${{ env.CMAKE_GENERATOR }}" \
+            -DBUILD_SHARED_LIBS=OFF \
+            -DENABLE_TESTS=ON \
             -DENABLE_CPPLINT=OFF \
-            -DENABLE_GAPI_PREPROCESSING=OFF \
-            -DCMAKE_VERBOSE_MAKEFILE=ON \
+            -DENABLE_NCC_STYLE=OFF \
+            -DENABLE_INTEL_GNA=OFF \
             -DCMAKE_COMPILE_WARNING_AS_ERROR=OFF \
-            -DENABLE_FASTER_BUILD=ON \
             -DENABLE_PROFILING_ITT=ON \
             -DSELECTIVE_BUILD=COLLECT \
+            -DCMAKE_C_COMPILER_LAUNCHER=${{ env.CMAKE_C_COMPILER_LAUNCHER }} \
+            -DCMAKE_CXX_COMPILER_LAUNCHER=${{ env.CMAKE_CXX_COMPILER_LAUNCHER }} \
             -S ${OPENVINO_REPO} \
             -B ${BUILD_DIR}
 
-      - name: Build CC COLLECT
-        run: cmake --build ${BUILD_DIR} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} --target openvino_intel_cpu_plugin openvino_ir_frontend benchmark_app sea_itt_lib
+      - name: Cmake build - CC COLLECT
+        run: |
+          cmake --build ${BUILD_DIR} --parallel 8 --config ${{ env.CMAKE_BUILD_TYPE }}
+          cmake --build ${BUILD_DIR} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} --target sea_itt_lib
 
       - name: Show ccache stats
         run: ccache --show-stats
 
-      - name: Code usage analysis
+      - name: Cmake install - OpenVINO
+        run: cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} -P ${BUILD_DIR}/cmake_install.cmake
+
+      - name: Build C++ samples - OpenVINO build tree
+        run: |
+          cmake -G "${{ env.CMAKE_GENERATOR }}" -DOpenVINO_DIR=${BUILD_DIR} -S ${INSTALL_DIR}/samples/cpp -B ${BUILD_DIR}/cpp_samples
+          cmake --build ${BUILD_DIR}/cpp_samples --parallel --config ${{ env.CMAKE_BUILD_TYPE }} --target hello_query_device
+
+      - name: Build C samples - OpenVINO install tree
+        run: ${INSTALL_DIR}/samples/c/build_samples.sh -i ${INSTALL_DIR} -b ${BUILD_DIR}/c_samples
+
+      - name: Ctest - OpenVINO unit tests
+        run: ctest -C ${{ env.CMAKE_BUILD_TYPE }} --test-dir ${BUILD_DIR} -V -L UNIT
+
+      - name: Perform code tracing via ITT collector
         run: |
           python3 ${OPENVINO_REPO}/thirdparty/itt_collector/runtool/sea_runtool.py \
-            --bindir ${OPENVINO_REPO}/bin/intel64/Release -o ${BUILD_DIR}/itt_stat ! \
+            --bindir ${OPENVINO_REPO}/bin/intel64/Release -o ${SELECTIVE_BUILD_STAT_DIR}/itt_stat ! \
           ${OPENVINO_REPO}/bin/intel64/Release/benchmark_app -niter 1 -nireq 1 \
             -m ${MODELS_PATH}/models/test_model/test_model_fp32.xml -d CPU
 
-      - name: CMake configure with CC ON
-        run: cmake -DSELECTIVE_BUILD=ON -DSELECTIVE_BUILD_STAT=${BUILD_DIR}/*.csv -S ${OPENVINO_REPO} -B ${BUILD_DIR}
-
-      - name: Build with CC ON
-        run: cmake --build ${BUILD_DIR} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} --target openvino_intel_cpu_plugin openvino_ir_frontend
-
-      - name: Use OpenVINO after CC
+      - name: Pack Artifacts
         run: |
-          ${OPENVINO_REPO}/bin/intel64/Release/benchmark_app -niter 1 -nireq 1 \
-            -m ${MODELS_PATH}/models/test_model/test_model_fp32.xml -d CPU
+          pushd ${SELECTIVE_BUILD_STAT_DIR}
+            tar -czvf ${BUILD_DIR}/openvino_selective_build_stat.tar.gz *
+          popd
+
+          pushd ${OPENVINO_REPO}
+            tar -czvf ${BUILD_DIR}/openvino_tests.tar.gz \
+              bin/intel64/Release/ov_cpu_func_tests \
+              src/tests/test_utils/functional_test_utils/layer_tests_summary/* \
+              scripts/install_dependencies/*
+          popd
+
+      - name: Upload selective build statistics package
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: openvino_selective_build_stat
+          path: ${{ env.BUILD_DIR }}/openvino_selective_build_stat.tar.gz
+          if-no-files-found: 'error'
+
+      - name: Upload OpenVINO tests package
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: openvino_tests
+          path: ${{ env.BUILD_DIR }}/openvino_tests.tar.gz
+          if-no-files-found: 'error'
+
+  CC_Build:
+    name: Conditional Compilation
+    needs: Build
+    defaults:
+      run:
+        shell: bash
+    runs-on: aks-linux-16-cores
+    container:
+      image: openvinogithubactions.azurecr.io/dockerhub/ubuntu:22.04
+      volumes:
+        - /mount/caches:/mount/caches
+    env:
+      DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input
+      CMAKE_CXX_COMPILER_LAUNCHER: ccache
+      CMAKE_C_COMPILER_LAUNCHER: ccache
+      GITHUB_WORKSPACE: '/__w/openvino/openvino'
+      OPENVINO_REPO: /__w/openvino/openvino/openvino
+      BUILD_DIR: /__w/openvino/openvino/openvino_build
+      SELECTIVE_BUILD_STAT_DIR: /__w/openvino/openvino/selective_build_stat
+      MODELS_PATH: /__w/openvino/openvino/testdata
+      CCACHE_DIR: /mount/caches/ccache/ubuntu22_x86_64_cc_Release
+      CCACHE_TEMPDIR: /__w/openvino/openvino/ccache_temp
+      CCACHE_MAXSIZE: 20G
+
+    steps:
+      - name: Install git
+        run: apt-get update && apt-get install --assume-yes --no-install-recommends git ca-certificates git-lfs
+
+      - name: Clone OpenVINO
+        uses: actions/checkout@v4
+        with:
+          path: ${{ env.OPENVINO_REPO }}
+          submodules: 'true'
+
+      - name: Clone test models
+        uses: actions/checkout@v4
+        with:
+          repository: 'openvinotoolkit/testdata'
+          path: ${{ env.MODELS_PATH }}
+          lfs: 'true'
+
+      - name: Download selective build statistics package
+        uses: actions/download-artifact@v3
+        with:
+          name: openvino_selective_build_stat
+          path: ${{ env.SELECTIVE_BUILD_STAT_DIR }}
+
+      - name: Extract selective build statistics package
+        run: tar -xvzf ${SELECTIVE_BUILD_STAT_DIR}/openvino_selective_build_stat.tar.gz -C ${SELECTIVE_BUILD_STAT_DIR}
+
+      #
+      # Dependencies
+      #
+
+      - name: Install build dependencies
+        run: bash ${OPENVINO_REPO}/install_build_dependencies.sh
+
+      #
+      # Build
+      #
+
+      - name: CMake configure - CC ON
+        run: |
+          cmake \
+            -DBUILD_SHARED_LIBS=OFF \
+            -DENABLE_CPPLINT=OFF \
+            -DSELECTIVE_BUILD=ON \
+            -DENABLE_LTO=OFF \
+            -DENABLE_TEMPLATE=OFF \
+            -DENABLE_INTEL_GPU=OFF \
+            -DENABLE_INTEL_GNA=OFF \
+            -DENABLE_OV_TF_FRONTEND=OFF \
+            -DENABLE_OV_TF_LITE_FRONTEND=OFF \
+            -DENABLE_OV_PADDLE_FRONTEND=OFF \
+            -DENABLE_OV_PYTORCH_FRONTEND=OFF \
+            -DENABLE_OV_ONNX_FRONTEND=OFF \
+            -DSELECTIVE_BUILD_STAT=${SELECTIVE_BUILD_STAT_DIR}/*.csv \
+            -DCMAKE_C_COMPILER_LAUNCHER=${{ env.CMAKE_C_COMPILER_LAUNCHER }} \
+            -DCMAKE_CXX_COMPILER_LAUNCHER=${{ env.CMAKE_CXX_COMPILER_LAUNCHER }} \
+            -S ${OPENVINO_REPO} \
+            -B ${BUILD_DIR}
+
+      - name: Cmake build - CC ON
+        run: cmake --build ${BUILD_DIR} --parallel 8 --target benchmark_app
+
+      - name: Show ccache stats
+        run: ccache --show-stats
+
+      - name: Run with CC-ed runtime
+        run: ${OPENVINO_REPO}/bin/intel64/Release/benchmark_app -niter 1 -nireq 1 -m ${MODELS_PATH}/models/test_model/test_model_fp32.xml -d CPU
+
+  CPU_Functional_Tests:
+    name: CPU functional tests
+    needs: Build
+    defaults:
+      run:
+        shell: bash
+    runs-on: aks-linux-8-cores
+    container:
+      image: openvinogithubactions.azurecr.io/dockerhub/ubuntu:22.04
+    env:
+      DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input
+      INSTALL_TEST_DIR: /__w/openvino/openvino/install/tests
+      PARALLEL_TEST_SCRIPT: /__w/openvino/openvino/install/tests/src/tests/test_utils/functional_test_utils/layer_tests_summary/run_parallel.py
+      PARALLEL_TEST_CACHE: /__w/openvino/openvino/install/tests/test_cache.lst
+
+    steps:
+      - name: Download OpenVINO tests package
+        uses: actions/download-artifact@v3
+        with:
+          name: openvino_tests
+          path: ${{ env.INSTALL_TEST_DIR }}
+
+      - name: Extract OpenVINO packages
+        run: tar -xvzf ${INSTALL_TEST_DIR}/openvino_tests.tar.gz -C ${INSTALL_TEST_DIR}
+
+      - name: Install OpenVINO dependencies
+        run: bash ${INSTALL_TEST_DIR}/scripts/install_dependencies/install_openvino_dependencies.sh -c=core -c=gpu -y
+
+      - name: Install 'actions/setup-python@v4' dependencies
+        run: apt-get install -y libssl3
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Install python dependencies for run_parallel.py
+        run: python3 -m pip install -r ${INSTALL_TEST_DIR}/src/tests/test_utils/functional_test_utils/layer_tests_summary/requirements.txt
+
+      - name: Restore tests execution time
+        uses: actions/cache/restore@v3
+        with:
+          path: ${{ env.PARALLEL_TEST_CACHE }}
+          key: ${{ runner.os }}-tests-functional-cpu-stamp-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-tests-functional-cpu-stamp
+
+      - name: Intel CPU plugin func tests (parallel)
+        run: python3 ${PARALLEL_TEST_SCRIPT} -e ${INSTALL_TEST_DIR}/bin/intel64/Release/ov_cpu_func_tests -c ${PARALLEL_TEST_CACHE} -w ${INSTALL_TEST_DIR} -s suite -rf 0 -- --gtest_print_time=1 --gtest_filter=*smoke*
+        timeout-minutes: 40
+
+      - name: Upload Test Results
+        uses: actions/upload-artifact@v3
+        if: ${{ !cancelled() }}
+        with:
+          name: test-results-functional-cpu
+          path: |
+            ${{ env.INSTALL_TEST_DIR }}/TEST*.xml
+            ${{ env.INSTALL_TEST_DIR }}/logs/failed/*.log
+            ${{ env.INSTALL_TEST_DIR }}/logs/crashed/*.log
+            ${{ env.INSTALL_TEST_DIR }}/logs/hanged/*.log
+            ${{ env.INSTALL_TEST_DIR }}/logs/interapted/*.log
+            ${{ env.INSTALL_TEST_DIR }}/logs/disabled_tests.log
+          if-no-files-found: 'error'

.github/workflows/linux_cuda.yml

@@ -115,7 +115,7 @@ jobs:
             -DENABLE_INTEL_GPU=OFF \
             -DENABLE_INTEL_GNA=OFF \
             -DENABLE_OV_TF_FRONTEND=OFF \
-            -DENABLE_OV_TF_LITE=OFF \
+            -DENABLE_OV_TF_LITE_FRONTEND=OFF \
             -DENABLE_OV_PADDLE_FRONTEND=OFF \
             -DENABLE_OV_PYTORCH_FRONTEND=OFF \
             -DENABLE_OV_ONNX_FRONTEND=OFF \