[GHA] Azure pip cache (#20029)

* switched PyTorch_Models * fixed root issue * upgrade pip * set bash environments * install pip deps * added git installation * added g++ deps * added python-dev package * get particular python-dev package * fixed package name * removed python-dev * cython dep * debug * fixed inlude path * test cache * set home dir * changed disk space calculation * crosspipeline pip cache * added tag for source builded deps * immutable deps * use commit id instead of tag * Update .github/workflows/linux.yml Co-authored-by: Ilya Lavrenov <ilya.lavrenov@intel.com> * fixed logs upload condition * removed pip upgrade * changed detectotron version * Update tests/model_hub_tests/torch_tests/requirements_secondary.txt Co-authored-by: Ilya Lavrenov <ilya.lavrenov@intel.com> * changed wheel installation way * reset onednn * added cleanup workload * cleanup test * fixed reqs install * test cache depending on pip version * syntax fix * env overwriting test * test env output * fixed set env step * move pip cahc initialization to step * changed pip ver regexp * set custom pip cahce for each step * cleanup code and enabled test cleanup job * test cleanup 2 * speedup removal * check existed dir * Enabled cache for samples job * removed debug job * changed way how to get pip ver * improved pip version extration * revert omz * switched PyTorch_Models * fixed root issue * upgrade pip * set bash environments * install pip deps * added git installation * added g++ deps * added python-dev package * get particular python-dev package * fixed package name * removed python-dev * cython dep * debug * fixed inlude path * test cache * set home dir * changed disk space calculation * crosspipeline pip cache * added tag for source builded deps * immutable deps * use commit id instead of tag * Update .github/workflows/linux.yml Co-authored-by: Ilya Lavrenov <ilya.lavrenov@intel.com> * fixed logs upload condition * removed pip upgrade * changed detectotron version * Update tests/model_hub_tests/torch_tests/requirements_secondary.txt Co-authored-by: Ilya Lavrenov <ilya.lavrenov@intel.com> * changed wheel installation way * added cleanup workload * reset onednn * cleanup test * fixed reqs install * test cache depending on pip version * syntax fix * env overwriting test * test env output * fixed set env step * move pip cahc initialization to step * changed pip ver regexp * set custom pip cahce for each step * cleanup code and enabled test cleanup job * test cleanup 2 * speedup removal * check existed dir * Enabled cache for samples job * removed debug job * changed way how to get pip ver * improved pip version extration * revert omz * review fixes * remove cache files using atime * mtime->atime --------- Co-authored-by: Ilya Lavrenov <ilya.lavrenov@intel.com>
2023-09-28 21:21:32 +03:00 · 2023-09-28 21:21:32 +03:00 · f85a276782
commit f85a276782
parent 84d98d8bf7
3 changed files with 111 additions and 42 deletions
--- a/.github/workflows/cleanup_pip_cache.yml
+++ b/.github/workflows/cleanup_pip_cache.yml
@ -0,0 +1,31 @@
+name: Cleanup PIP caches
+on:
+  schedule:
+    # at 00:00 on the 1st day of every month
+    - cron: '0 0 1 * *'
+
+jobs:
+  Cleanup_PIP_Caches:
+    runs-on: aks-linux-2-cores
+    container:
+      image: openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04
+      volumes:
+        - /mount/caches:/mount/caches
+    env:
+      PIP_CACHE_PATH: /mount/caches/pip
+
+    steps:
+      - name: Pre-Collecting Cache Info
+        run: |
+          echo "Cache info: "
+          du -h -d2 ${PIP_CACHE_PATH}
+
+      - name: Cleanup cache
+        run: |
+          echo "Delete cache files if they have not been used in over 30 days"
+          [ ! -z "${PIP_CACHE_PATH}" ] && find ${PIP_CACHE_PATH} ! -type d -atime +30 -delete
+
+      - name: Post-Collecting Cache Info
+        run: |
+          echo "Cache info: "
+          du -h -d2 ${PIP_CACHE_PATH}
--- a/.github/workflows/linux.yml
+++ b/.github/workflows/linux.yml
@ -29,6 +29,10 @@ concurrency:
  group: ${{ github.event_name == 'push' && github.run_id || github.ref }}-linux
  cancel-in-progress: true

+env:
+  PIP_CACHE_PATH: /mount/caches/pip/linux
+  PYTHON_VERSION: '3.11'
+
 jobs:
  Build:
    defaults:
@ -55,6 +59,7 @@ jobs:
      CCACHE_DIR: /mount/caches/ccache/ubuntu20_x86_64_Release
      CCACHE_TEMPDIR: /__w/openvino/openvino/ccache_temp
      CCACHE_MAXSIZE: 50G
+
    steps:
      - name: Install git
        run: |
@ -92,7 +97,12 @@ jobs:

      - uses: actions/setup-python@v4
        with:
-          python-version: '3.11'
+          python-version: ${{ env.PYTHON_VERSION }}
+      - name: Setup pip cache dir
+        run: |
+          PIP_VER=$(python3 -c "import pip; print(pip.__version__)")
+          echo "Using pip version: ${PIP_VER}"
+          echo "PIP_CACHE_DIR=${PIP_CACHE_PATH}/${PIP_VER}" >> $GITHUB_ENV

      - name: Install python dependencies
        run: |
@ -116,9 +126,8 @@ jobs:
      # Build
      #

-      - name: Setup ccache
-        run: |
-          mkdir -p $CCACHE_DIR
+      - name: Setup ccache dir
+        run: mkdir -p ${CCACHE_DIR}

      - name: CMake configure - OpenVINO
        run: |
@ -321,7 +330,12 @@ jobs:

      - uses: actions/setup-python@v4
        with:
-          python-version: '3.11'
+          python-version: ${{ env.PYTHON_VERSION }}
+      - name: Setup pip cache dir
+        run: |
+          PIP_VER=$(python3 -c "import pip; print(pip.__version__)")
+          echo "Using pip version: ${PIP_VER}"
+          echo "PIP_CACHE_DIR=${PIP_CACHE_PATH}/${PIP_VER}" >> $GITHUB_ENV

      - name: Install OpenVINO dependencies
        run: ${INSTALL_DIR}/install_dependencies/install_openvino_dependencies.sh -c=core -c=dev -y
@ -367,7 +381,7 @@ jobs:

      - name: Upload Test Results
        uses: actions/upload-artifact@v3
-        if: ${{ always() }}
+        if: ${{ !cancelled() }}
        with:
          name: test-results-cpp
          path: ${{ env.INSTALL_TEST_DIR }}/TEST*.xml
@ -560,7 +574,7 @@ jobs:

      - name: Upload Test Results
        uses: actions/upload-artifact@v3
-        if: ${{ always() }}
+        if: ${{ !cancelled() }}
        with:
          name: test-results-cpp
          path: ${{ env.INSTALL_TEST_DIR }}/TEST*.xml
@ -574,6 +588,8 @@ jobs:
    runs-on: aks-linux-4-cores
    container:
      image: openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04
+      volumes:
+        - /mount/caches:/mount/caches
    env:
      OPENVINO_REPO: /__w/openvino/openvino/openvino
      INSTALL_DIR: /__w/openvino/openvino/install
@ -600,7 +616,12 @@ jobs:

      - uses: actions/setup-python@v4
        with:
-          python-version: '3.11'
+          python-version: ${{ env.PYTHON_VERSION }}
+      - name: Setup pip cache dir
+        run: |
+          PIP_VER=$(python3 -c "import pip; print(pip.__version__)")
+          echo "Using pip version: ${PIP_VER}"
+          echo "PIP_CACHE_DIR=${PIP_CACHE_PATH}/${PIP_VER}" >> $GITHUB_ENV

      - name: Install Python API tests dependencies
        run: |
@ -774,7 +795,7 @@ jobs:

      - name: Upload Test Results
        uses: actions/upload-artifact@v3
-        if: ${{ always() }}
+        if: ${{ !cancelled() }}
        with:
          name: test-results-python
          path: |
@ -829,7 +850,7 @@ jobs:

      - uses: actions/setup-python@v4
        with:
-          python-version: '3.11'
+          python-version: ${{ env.PYTHON_VERSION }}

      - name: Install python dependencies for run_parallel.py
        run: python3 -m pip install -r ${INSTALL_TEST_DIR}/functional_test_utils/requirements.txt
@ -857,7 +878,7 @@ jobs:

      - name: Upload Test Results
        uses: actions/upload-artifact@v3
-        if: ${{ always() }}
+        if: ${{ !cancelled() }}
        with:
          name: test-results-functional-cpu
          path: |
@ -893,7 +914,12 @@ jobs:

      - uses: actions/setup-python@v4
        with:
-          python-version: '3.11'
+          python-version: ${{ env.PYTHON_VERSION }}
+      - name: Setup pip cache dir
+        run: |
+          PIP_VER=$(python3 -c "import pip; print(pip.__version__)")
+          echo "Using pip version: ${PIP_VER}"
+          echo "PIP_CACHE_DIR=${PIP_CACHE_PATH}/${PIP_VER}" >> $GITHUB_ENV

      - name: Download OpenVINO package
        uses: actions/download-artifact@v3
@ -934,7 +960,7 @@ jobs:

      - name: Upload Test Results
        uses: actions/upload-artifact@v3
-        if: ${{ always() }}
+        if: ${{ !cancelled() }}
        with:
          name: test-results-tensorflow-hub-models
          path: |
@ -946,27 +972,30 @@ jobs:
    defaults:
      run:
        shell: bash
-    runs-on: ${{ github.event_name == 'schedule' && 'ubuntu-20.04-8-cores' || 'ubuntu-20.04'}}
+    runs-on: ${{ github.event_name == 'schedule' && 'aks-linux-16-cores' || 'aks-linux-8-cores'}}
+    container:
+      image: openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04
+      volumes:
+        - /mount/caches:/mount/caches
    env:
      INSTALL_DIR: ${{ github.workspace }}/install
      INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests
      MODEL_HUB_TESTS_INSTALL_DIR: ${{ github.workspace }}/install/tests/model_hub_tests

    steps:
-      - name: Maximize build space
-        run:  |
-          sudo rm -rf /usr/local/lib/android # will release about 10 GB if you don't need Android
-          sudo rm -rf /usr/share/dotnet # will release about 20GB if you don't need .NET
-          sudo rm -rf /opt/ghc
-          echo "Available storage:"
-          df -h
+      - name: Check sudo
+        run: if [ "$(id -u)" -eq 0 ]; then apt update && apt --assume-yes install sudo; fi
+
+      - name: Install dependencies
+        run: |
+          # libssl1.1 - 'python3 -m pip' in self-hosted runner
+          sudo apt install --assume-yes --no-install-recommends libssl1.1
+          # install git (required to build pip deps from the sources)
+          # install 'g++' to build 'detectron2' and 'natten' wheels
+          sudo apt-get install --assume-yes --no-install-recommends g++ git ca-certificates

      - name: Create Directories
-        run: mkdir -p ${{ env.INSTALL_DIR }} ${{ env.INSTALL_TEST_DIR }}
-
-      - uses: actions/setup-python@v4
-        with:
-          python-version: '3.11'
+        run: mkdir -p ${INSTALL_DIR} ${INSTALL_TEST_DIR}

      - name: Download OpenVINO package
        uses: actions/download-artifact@v3
@ -982,29 +1011,38 @@ jobs:

      - name: Extract OpenVINO packages
        run: |
-          pushd ${{ env.INSTALL_DIR }}
-            tar -xzf openvino_package.tar.gz -C ${{ env.INSTALL_DIR }} && rm openvino_package.tar.gz || exit 1
+          pushd ${INSTALL_DIR}
+            tar -xzf openvino_package.tar.gz -C ${INSTALL_DIR} && rm openvino_package.tar.gz || exit 1
          popd
-          pushd ${{ env.INSTALL_TEST_DIR }}
-            tar -xzf openvino_tests.tar.gz -C ${{ env.INSTALL_DIR }} && rm openvino_tests.tar.gz || exit 1
+          pushd ${INSTALL_TEST_DIR}
+            tar -xzf openvino_tests.tar.gz -C ${INSTALL_DIR} && rm openvino_tests.tar.gz || exit 1
          popd
-      - name: Install Python wheels
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+      - name: Setup pip cache dir
        run: |
-          python3 -m pip install openvino --find-links=${{ env.INSTALL_DIR }}/tools
+          PIP_VER=$(python3 -c "import pip; print(pip.__version__)")
+          echo "Using pip version: ${PIP_VER}"
+          echo "PIP_CACHE_DIR=${PIP_CACHE_PATH}/${PIP_VER}" >> $GITHUB_ENV
+
+      - name: Install OpenVINO Python wheels
+        run: python3 -m pip install ${INSTALL_DIR}/tools/openvino-*

      - name: Install PyTorch tests requirements
        run: |
-          python3 -m pip install -r ${{ env.MODEL_HUB_TESTS_INSTALL_DIR }}/torch_tests/requirements.txt
-          python3 -m pip install -r ${{ env.MODEL_HUB_TESTS_INSTALL_DIR }}/torch_tests/requirements_secondary.txt
-          python3 -m pip cache purge
+          python3 -m pip install -r ${MODEL_HUB_TESTS_INSTALL_DIR}/torch_tests/requirements.txt
+          python3 -m pip install -r ${MODEL_HUB_TESTS_INSTALL_DIR}/torch_tests/requirements_secondary.txt
          echo "Available storage:"
          df -h
-          du -h -d0 ~/.cache ~/*
+        env:
+          CPLUS_INCLUDE_PATH: ${{ env.Python_ROOT_DIR }}/include/python${{ env.PYTHON_VERSION }}

      - name: PyTorch Models Tests
        run: |
-          export PYTHONPATH=${{ env.MODEL_HUB_TESTS_INSTALL_DIR }}:$PYTHONPATH
-          python3 -m pytest ${{ env.MODEL_HUB_TESTS_INSTALL_DIR }}/torch_tests/ -m ${{ env.TYPE }} --html=${{ env.INSTALL_TEST_DIR }}/TEST-torch_model_tests.html --self-contained-html -v
+          export PYTHONPATH=${MODEL_HUB_TESTS_INSTALL_DIR}:$PYTHONPATH
+          python3 -m pytest ${MODEL_HUB_TESTS_INSTALL_DIR}/torch_tests/ -m ${TYPE} --html=${INSTALL_TEST_DIR}/TEST-torch_model_tests.html --self-contained-html -v
        env:
          TYPE: ${{ github.event_name == 'schedule' && 'nightly' || 'precommit'}}
          TEST_DEVICE: CPU
@ -1013,13 +1051,12 @@ jobs:
        run: |
          echo "Available storage:"
          df -h
-          du -h -d0 ~/.cache ~/*

      - name: Upload Test Results
        uses: actions/upload-artifact@v3
-        if: ${{ always() }}
+        if: ${{ !cancelled() }}
        with:
          name: test-results-torch-models
          path: |
            ${{ env.INSTALL_TEST_DIR }}/TEST*.html
-          if-no-files-found: 'error'
+          if-no-files-found: 'error'
--- a/tests/model_hub_tests/torch_tests/requirements_secondary.txt
+++ b/tests/model_hub_tests/torch_tests/requirements_secondary.txt
@ -1,4 +1,5 @@
 -c ../../constraints.txt
 # This file contains requirements dependednt from modules in requirements.txt
-git+https://github.com/facebookresearch/detectron2.git 
+# get immutable commit to be able to use pip cache
+git+https://github.com/facebookresearch/detectron2.git@1a4df4d954089534daebbd7a0a45f08ece87056e
 natten