[GHA] Smart CI for more pipelines (#21307)

* Add Smart CI for more pipelines. Refactor logic for workflow skip

Since different workflows may have different requirements for skipping them, I suggest to move these requirements to parameters instead of hardcoding them

* Use patterns for skipping pipeline for conformance-only

* Remove path filters for Android

* Return mistakenly deleted param

* Propagate params to Python script

* Add missing outputs mapping

* Return push trigger

* Skip CC CPU func when CPU is not affected

* Fix variable name

.github/actions/smart-ci/action.yml

@@ -32,6 +32,14 @@ inputs:
     description: "Path to labeler configuration file"
     required: false
     default: ".github/labeler.yml"
+  skip_when_only_listed_labels_set:
+    description: "Comma-separated list of labels. If PR has only these labels set, 
+                  return indicator that CI can be skipped"
+    required: false
+  skip_when_only_listed_files_changed:
+    description: "Comma-separated list of patterns (fnmatch-style). If PR has only matching files changed, 
+                  return indicator that CI can be skipped"
+    required: false
 
 outputs:
   all_components:
@@ -40,6 +48,9 @@ outputs:
   affected_components:
     description: "Affected components to run validation for and their validation scope"
     value: ${{ steps.smart_ci.outputs.affected_components }}
+  skip_workflow:
+    description: "Whether the workflow should be run with Smart CI rules applied or skipped completely"
+    value: ${{ steps.smart_ci.outputs.skip_workflow }}
 
 runs:
   using: "composite"
@@ -80,7 +91,9 @@ runs:
           -p "${{ inputs.component_pattern }}" \
           -c "${{ inputs.components_config }}" \
           -m "${{ inputs.components_config_schema }}" \
-          -l "${{ inputs.labeler_config }}"
+          -l "${{ inputs.labeler_config }}" \
+          --skip-when-only-listed-labels-set "${{ inputs.skip_when_only_listed_labels_set }}" \
+          --skip-when-only-listed-files-changed "${{ inputs.skip_when_only_listed_files_changed }}"
       shell: bash
       env:
         GITHUB_TOKEN: ${{ inputs.repo_token }}

.github/actions/smart-ci/smart_ci.py

@@ -7,6 +7,7 @@ import jsonschema
 import logging
 from pathlib import Path
 from ghapi.all import GhApi
+from fnmatch import fnmatch
 
 
 class ComponentConfig:
@@ -121,6 +122,12 @@ def parse_args():
                         help='Path to the schema file for components config')
     parser.add_argument('-l', '--labeler-config', default='.github/labeler.yml',
                         help='Path to PR labeler config file')
+    parser.add_argument('--skip-when-only-listed-labels-set',
+                        help="Comma-separated list of labels. If PR has only these labels set, "
+                             "return indicator that CI can be skipped")
+    parser.add_argument('--skip-when-only-listed-files-changed',
+                        help="Comma-separated list of patterns (fnmatch-style). If PR has only matching files changed, "
+                             "return indicator that CI can be skipped")
     args = parser.parse_args()
     return args
 
@@ -189,25 +196,25 @@ def main():
     cfg = ComponentConfig(components_config, schema, all_possible_components)
     affected_components = cfg.get_affected_components(changed_component_names)
 
-    # We don't need to run workflow if changes were only in documentation
+    skip_workflow = False
-    # This is the case when we have no product labels set except "docs"
-    # or only if md files were matched (the latter covers cases where *.md is outside docs directory)
-    only_docs_changes = False
     if args.pr and not run_full_scope:
-        # We don't want to add helper labels to labeler config for now, so handling it manually
+        if args.skip_when_only_listed_labels_set:
-        docs_label_only = changed_component_names - {'docs'} == set()
+            excepted_labels = set(args.skip_when_only_listed_labels_set.split(','))
-        if docs_label_only:
+            excepted_labels_only = changed_component_names - excepted_labels == set()
-            only_docs_changes = True
+            skip_workflow = excepted_labels_only
-        else:
+
+        if not skip_workflow and args.skip_when_only_listed_files_changed:
             # To avoid spending extra API requests running step below only if necessary
             changed_files = gh_api.pulls.list_files(args.pr)
-            doc_suffixes = ['.md', '.rst', '.png', '.jpg', '.svg']
+            patterns = set(args.skip_when_only_listed_files_changed.split(','))
-            only_docs_changes = all([Path(f.filename).suffix in doc_suffixes for f in changed_files])
-            logger.debug(f"doc files only: {only_docs_changes}")
 
-    if only_docs_changes:
+            matched_files_only = all(any(fnmatch(f.filename, pattern) for pattern in patterns) for f in changed_files)
-        logger.info(f"Changes are documentation only, workflow may be skipped")
+            logger.debug(f"matched files only: {matched_files_only}")
-        affected_components['docs_only'] = ComponentConfig.FullScope
+            skip_workflow = matched_files_only
+
+    if skip_workflow:
+        logger.info(f"All changes are marked for skip, workflow may be skipped")
+        set_github_output("skip_workflow", str(skip_workflow))
 
     # Syntactic sugar for easier use in GHA pipeline
     affected_components_output = {name: {s: True for s in scope} for name, scope in affected_components.items()}

.github/workflows/android_arm64.yml

@@ -2,21 +2,7 @@ name: Android ARM64 with vcpkg
 on:
   workflow_dispatch:
   pull_request:
-    paths-ignore:
-      - '**/docs/**'
-      - 'docs/**'
-      - '**/**.md'
-      - '**.md'
-      - '**/layer_tests_summary/**'
-      - '**/conformance/**'
   push:
-    paths-ignore:
-      - '**/docs/**'
-      - 'docs/**'
-      - '**/**.md'
-      - '**.md'
-      - '**/layer_tests_summary/**'
-      - '**/conformance/**'
     branches:
       - master
 
@@ -26,7 +12,31 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  Smart_CI:
+    runs-on: ubuntu-latest
+    outputs:
+      affected_components: "${{ steps.smart_ci.outputs.affected_components }}"
+      skip_workflow: "${{ steps.smart_ci.outputs.skip_workflow }}"
+    steps:
+      - name: checkout action
+        uses: actions/checkout@v4
+        with:
+          sparse-checkout: .github/actions/smart-ci
+
+      - name: Get affected components
+        id: smart_ci
+        uses: ./.github/actions/smart-ci
+        with:
+          repository: ${{ github.repository }}
+          pr: ${{ github.event.number }}
+          commit_sha: ${{ github.sha }}
+          component_pattern: "category: (.*)"
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          skip_when_only_listed_labels_set: 'docs'
+          skip_when_only_listed_files_changed: '*.md,*.rst,*.png,*.jpg,*.svg,*/layer_tests_summary/*,*/conformance/*'
+
   Build:
+    needs: Smart_CI
     timeout-minutes: 150
     defaults:
       run:
@@ -52,6 +62,7 @@ jobs:
       VCPKG_DEFAULT_BINARY_CACHE: '/mount/caches/ccache/android_arm64/vcpkg_cache'
       VCPKG_FORCE_SYSTEM_BINARIES: '1'
       SCCACHE_AZURE_KEY_PREFIX: android_arm64
+    if: "!needs.smart_ci.outputs.skip_workflow"
     steps:
       - name: Install git
         run: apt-get update && apt-get install --assume-yes --no-install-recommends git ca-certificates

.github/workflows/fedora.yml

@@ -2,21 +2,7 @@ name: Fedora (RHEL), Python 3.9
 on:
   workflow_dispatch:
   pull_request:
-    paths-ignore:
-      - '**/docs/**'
-      - 'docs/**'
-      - '**/**.md'
-      - '**.md'
-      - '**/layer_tests_summary/**'
-      - '**/conformance/**'
   push:
-    paths-ignore:
-      - '**/docs/**'
-      - 'docs/**'
-      - '**/**.md'
-      - '**.md'
-      - '**/layer_tests_summary/**'
-      - '**/conformance/**'
     branches:
       - master
       - 'releases/**'
@@ -27,7 +13,31 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  Smart_CI:
+    runs-on: ubuntu-latest
+    outputs:
+      affected_components: "${{ steps.smart_ci.outputs.affected_components }}"
+      skip_workflow: "${{ steps.smart_ci.outputs.skip_workflow }}"
+    steps:
+      - name: checkout action
+        uses: actions/checkout@v4
+        with:
+          sparse-checkout: .github/actions/smart-ci
+
+      - name: Get affected components
+        id: smart_ci
+        uses: ./.github/actions/smart-ci
+        with:
+          repository: ${{ github.repository }}
+          pr: ${{ github.event.number }}
+          commit_sha: ${{ github.sha }}
+          component_pattern: "category: (.*)"
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          skip_when_only_listed_labels_set: 'docs'
+          skip_when_only_listed_files_changed: '*.md,*.rst,*.png,*.jpg,*.svg,*/layer_tests_summary/*,*/conformance/*'
+
   Build:
+    needs: Smart_CI
     timeout-minutes: 150
     defaults:
       run:
@@ -49,6 +59,7 @@ jobs:
       INSTALL_TEST_DIR: /__w/openvino/openvino/tests_install
       BUILD_DIR: /__w/openvino/openvino/openvino_build
       SCCACHE_AZURE_KEY_PREFIX: fedora33_x86_64_Release
+    if: "!needs.smart_ci.outputs.skip_workflow"
     steps:
       - name: Install git
         run: yum update -y && yum install -y git

.github/workflows/linux.yml

@@ -24,6 +24,7 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       affected_components: "${{ steps.smart_ci.outputs.affected_components }}"
+      skip_workflow: "${{ steps.smart_ci.outputs.skip_workflow }}"
     steps:
       - name: checkout action
         uses: actions/checkout@v4
@@ -39,11 +40,8 @@ jobs:
           commit_sha: ${{ github.sha }}
           component_pattern: "category: (.*)"
           repo_token: ${{ secrets.GITHUB_TOKEN }}
-
+          skip_when_only_listed_labels_set: 'docs'
-      - name: Show affected components
+          skip_when_only_listed_files_changed: '*.md,*.rst,*.png,*.jpg,*.svg'
-        run: |
-          echo "${{ toJSON(steps.smart_ci.outputs.affected_components) }}"
-        shell: bash
 
   Build:
     needs: Smart_CI
@@ -72,7 +70,7 @@ jobs:
       BUILD_DIR: /__w/openvino/openvino/openvino_build
       SCCACHE_AZURE_KEY_PREFIX: ubuntu20_x86_64_Release
       ONNX_RUNTIME_UTILS: /__w/openvino/openvino/openvino/.ci/azure/ci_utils/onnxruntime
-    if: "!fromJSON(needs.smart_ci.outputs.affected_components).docs_only"
+    if: "!needs.smart_ci.outputs.skip_workflow"
 
     steps:
       - name: Install git

.github/workflows/linux_conditional_compilation.yml

@@ -2,21 +2,7 @@ name: Linux Static CC (Ubuntu 22.04, Python 3.11, Clang)
 on:
   workflow_dispatch:
   pull_request:
-    paths-ignore:
-      - '**/docs/**'
-      - 'docs/**'
-      - '**/**.md'
-      - '**.md'
-      - '**/layer_tests_summary/**'
-      - '**/conformance/**'
   push:
-    paths-ignore:
-      - '**/docs/**'
-      - 'docs/**'
-      - '**/**.md'
-      - '**.md'
-      - '**/layer_tests_summary/**'
-      - '**/conformance/**'
     branches:
       - master
 
@@ -30,7 +16,31 @@ env:
   PYTHON_VERSION: '3.11'
 
 jobs:
+  Smart_CI:
+    runs-on: ubuntu-latest
+    outputs:
+      affected_components: "${{ steps.smart_ci.outputs.affected_components }}"
+      skip_workflow: "${{ steps.smart_ci.outputs.skip_workflow }}"
+    steps:
+      - name: checkout action
+        uses: actions/checkout@v4
+        with:
+          sparse-checkout: .github/actions/smart-ci
+
+      - name: Get affected components
+        id: smart_ci
+        uses: ./.github/actions/smart-ci
+        with:
+          repository: ${{ github.repository }}
+          pr: ${{ github.event.number }}
+          commit_sha: ${{ github.sha }}
+          component_pattern: "category: (.*)"
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          skip_when_only_listed_labels_set: 'docs'
+          skip_when_only_listed_files_changed: '*.md,*.rst,*.png,*.jpg,*.svg,*/layer_tests_summary/*,*/conformance/*'
+
   Build:
+    needs: Smart_CI
     timeout-minutes: 150
     defaults:
       run:
@@ -54,6 +64,7 @@ jobs:
       SELECTIVE_BUILD_STAT_DIR: /__w/openvino/openvino/selective_build_stat
       MODELS_PATH: /__w/openvino/openvino/testdata
       SCCACHE_AZURE_KEY_PREFIX: ubuntu22_x86_64_itt_clang_Release
+    if: "!needs.smart_ci.outputs.skip_workflow"
 
     steps:
       - name: Install git
@@ -299,7 +310,7 @@ jobs:
 
   CPU_Functional_Tests:
     name: CPU functional tests
-    needs: Build
+    needs: [Build, Smart_CI]
     timeout-minutes: 25
     defaults:
       run:
@@ -313,6 +324,7 @@ jobs:
       INSTALL_TEST_DIR: /__w/openvino/openvino/install/tests
       PARALLEL_TEST_SCRIPT: /__w/openvino/openvino/install/tests/src/tests/test_utils/functional_test_utils/layer_tests_summary/run_parallel.py
       PARALLEL_TEST_CACHE: /__w/openvino/openvino/install/tests/test_cache.lst
+    if: fromJSON(needs.smart_ci.outputs.affected_components).CPU.test
 
     steps:
       - name: Download OpenVINO tests package

.github/workflows/linux_riscv.yml

@@ -5,21 +5,7 @@ on:
     - cron: '0 0 * * 3,6'
   workflow_dispatch:
   pull_request:
-    paths-ignore:
-      - '**/docs/**'
-      - 'docs/**'
-      - '**/**.md'
-      - '**.md'
-      - '**/layer_tests_summary/**'
-      - '**/conformance/**'
   push:
-    paths-ignore:
-      - '**/docs/**'
-      - 'docs/**'
-      - '**/**.md'
-      - '**.md'
-      - '**/layer_tests_summary/**'
-      - '**/conformance/**'
     branches:
       - master
       - 'releases/**'
@@ -30,7 +16,31 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  Smart_CI:
+    runs-on: ubuntu-latest
+    outputs:
+      affected_components: "${{ steps.smart_ci.outputs.affected_components }}"
+      skip_workflow: "${{ steps.smart_ci.outputs.skip_workflow }}"
+    steps:
+      - name: checkout action
+        uses: actions/checkout@v4
+        with:
+          sparse-checkout: .github/actions/smart-ci
+
+      - name: Get affected components
+        id: smart_ci
+        uses: ./.github/actions/smart-ci
+        with:
+          repository: ${{ github.repository }}
+          pr: ${{ github.event.number }}
+          commit_sha: ${{ github.sha }}
+          component_pattern: "category: (.*)"
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          skip_when_only_listed_labels_set: 'docs'
+          skip_when_only_listed_files_changed: '*.md,*.rst,*.png,*.jpg,*.svg,*/layer_tests_summary/*,*/conformance/*'
+
   Build:
+    needs: Smart_CI
     timeout-minutes: 150
     defaults:
       run:
@@ -52,6 +62,7 @@ jobs:
       CCACHE_DIR: /mount/caches/ccache/ubuntu22_riscv64_Release
       CCACHE_TEMPDIR: /__w/openvino/openvino/ccache_temp
       CCACHE_MAXSIZE: 50G
+    if: "!needs.smart_ci.outputs.skip_workflow"
     steps:
       - name: Install git
         run: apt-get update && apt-get install --assume-yes --no-install-recommends git ca-certificates