mirror of
https://github.com/LibreQoE/LibreQoS.git
synced 2026-10-24 22:07:40 -05:00
CI: gate Python and Rust jobs on changed paths
Both workflows ran their full pipelines on every push, including docs-only and opposite-language changes, which wasted runner time and reported failures unrelated to the change under test. Add a shared detect-changes workflow that diffs the pushed or PR range against per-group patterns and exposes a matched output. The heavy jobs run only when their group changed, or when detection itself fails, so a broken detector cannot silently skip checks. Tag pushes and unknown events fall back to running everything. Include .github/scripts/test-changed-files.sh, a manual harness for the detection logic that follows the repo's existing shell self-tests.
This commit is contained in:
Herbert Wolverson committed
1 parent
2a6a9a1ac4
commit
4577b9b5b1
5 files changed
+394
No files matched your search
Executable
+132
@@ -0,0 +1,132 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Decide whether files relevant to a CI job changed in the current run.
|
||||
#
|
||||
# Usage: changed-files.sh <python|rust>
|
||||
#
|
||||
# Reads the standard GITHUB_* environment plus EVENT_BEFORE and DEFAULT_BRANCH
|
||||
# (supplied by the calling workflow) and writes "matched=true|false" to
|
||||
# $GITHUB_OUTPUT. Matching files are listed on stdout for the workflow log.
|
||||
# If the change range cannot be determined, the group is reported as matched
|
||||
# so that a CI job is never silently skipped.
|
||||
set -euo pipefail
|
||||
|
||||
group="${1:-}"
|
||||
case "$group" in
|
||||
python)
|
||||
patterns=(
|
||||
'*.py'
|
||||
'requirements*.txt'
|
||||
'.github/workflows/lint_python.yml'
|
||||
'.github/workflows/detect-changes.yml'
|
||||
'.github/scripts/*'
|
||||
)
|
||||
;;
|
||||
rust)
|
||||
# The Rust workspace plus the build scripts the Rust workflow executes.
|
||||
# Files that are only copied into the .deb (Python sources, service
|
||||
# examples) do not trigger this group.
|
||||
patterns=(
|
||||
'src/rust/*'
|
||||
'src/build_dpkg.sh'
|
||||
'src/build_rust.sh'
|
||||
'src/VERSION_STRING'
|
||||
'.github/workflows/rust.yml'
|
||||
'.github/workflows/detect-changes.yml'
|
||||
'.github/scripts/*'
|
||||
)
|
||||
;;
|
||||
*)
|
||||
echo "usage: $0 <python|rust>" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
|
||||
# Report every tracked file as changed; used when no change range is available.
|
||||
all_tracked_files() {
|
||||
echo "Treating every tracked file as changed." >&2
|
||||
git -c core.quotePath=false ls-files
|
||||
}
|
||||
|
||||
# Files touched by this run, one per line.
|
||||
changed_files() {
|
||||
local base_ref=""
|
||||
local merge_base
|
||||
|
||||
# Tag pushes have no meaningful change range; run everything.
|
||||
if [[ "${GITHUB_REF:-}" == refs/tags/* ]]; then
|
||||
all_tracked_files
|
||||
return
|
||||
fi
|
||||
|
||||
case "${GITHUB_EVENT_NAME:-}" in
|
||||
pull_request)
|
||||
# Pull requests check out a merge commit; diff against the merge base
|
||||
# so files that only changed on the base branch are not reported.
|
||||
if [[ -n "${GITHUB_BASE_REF:-}" ]]; then
|
||||
base_ref="origin/${GITHUB_BASE_REF}"
|
||||
fi
|
||||
;;
|
||||
push)
|
||||
# github.event.before is the previously pushed commit; a new branch
|
||||
# pushes the all-zero object id and falls back to the default branch.
|
||||
if [[ -n "${EVENT_BEFORE:-}" && ! "${EVENT_BEFORE}" =~ ^0+$ ]] &&
|
||||
git rev-parse --verify --quiet "${EVENT_BEFORE}^{commit}" >/dev/null; then
|
||||
git -c core.quotePath=false diff --name-only "${EVENT_BEFORE}" "${GITHUB_SHA:-HEAD}"
|
||||
return
|
||||
fi
|
||||
if [[ -n "${DEFAULT_BRANCH:-}" ]]; then
|
||||
base_ref="origin/${DEFAULT_BRANCH}"
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
all_tracked_files
|
||||
return
|
||||
;;
|
||||
esac
|
||||
|
||||
if [[ -n "$base_ref" ]] && git rev-parse --verify --quiet "$base_ref" >/dev/null; then
|
||||
merge_base="$(git merge-base "$base_ref" HEAD)"
|
||||
git -c core.quotePath=false diff --name-only "$merge_base" HEAD
|
||||
return
|
||||
fi
|
||||
|
||||
all_tracked_files
|
||||
}
|
||||
|
||||
# Does a changed file belong to the requested group?
|
||||
matches_group() {
|
||||
local pattern
|
||||
for pattern in "${patterns[@]}"; do
|
||||
if [[ "$1" == $pattern ]]; then
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
if ! changed="$(changed_files)"; then
|
||||
changed="$(all_tracked_files)"
|
||||
fi
|
||||
|
||||
matching_files=()
|
||||
while IFS= read -r file; do
|
||||
if [[ -n "$file" ]] && matches_group "$file"; then
|
||||
matching_files+=("$file")
|
||||
fi
|
||||
done <<< "$changed"
|
||||
|
||||
if (( ${#matching_files[@]} > 0 )); then
|
||||
echo "Change group '$group': matching files changed:"
|
||||
printf ' %s\n' "${matching_files[@]}"
|
||||
else
|
||||
echo "Change group '$group': no matching files changed."
|
||||
fi
|
||||
|
||||
if [[ -n "${GITHUB_OUTPUT:-}" ]]; then
|
||||
if (( ${#matching_files[@]} > 0 )); then
|
||||
echo "matched=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "matched=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
fi
|
||||
Executable
+209
@@ -0,0 +1,209 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Self-test for changed-files.sh. Run it after editing the detection logic or
|
||||
# the pattern lists:
|
||||
#
|
||||
# ./.github/scripts/test-changed-files.sh
|
||||
#
|
||||
set -euo pipefail
|
||||
|
||||
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
detect="$script_dir/changed-files.sh"
|
||||
workdir="$(mktemp -d)"
|
||||
trap 'rm -rf "$workdir"' EXIT
|
||||
|
||||
checks=0
|
||||
failures=0
|
||||
|
||||
check() {
|
||||
local label="$1" expect="$2" group="$3"
|
||||
local out="$workdir/out.txt" log got
|
||||
rm -f "$out"
|
||||
checks=$((checks + 1))
|
||||
if ! log="$(GITHUB_OUTPUT="$out" bash "$detect" "$group" 2>&1)"; then
|
||||
echo "FAILED [$label] $group: script exited non-zero"
|
||||
echo "$log" | sed 's/^/ | /'
|
||||
failures=$((failures + 1))
|
||||
return
|
||||
fi
|
||||
got="$(grep -E '^matched=' "$out" | cut -d= -f2)"
|
||||
if [[ "$got" == "$expect" ]]; then
|
||||
echo "ok [$label] $group -> matched=$got"
|
||||
else
|
||||
echo "FAILED [$label] $group: expected $expect, got '$got'"
|
||||
echo "$log" | sed 's/^/ | /'
|
||||
failures=$((failures + 1))
|
||||
fi
|
||||
}
|
||||
|
||||
cd "$workdir"
|
||||
git init -q --bare origin.git
|
||||
git clone -q origin.git work
|
||||
cd work
|
||||
git config user.email ci@test.invalid
|
||||
git config user.name "CI Test"
|
||||
git symbolic-ref HEAD refs/heads/develop
|
||||
|
||||
mkdir -p src/rust/lqosd .github/workflows .github/scripts
|
||||
printf 'base\n' > README.md
|
||||
printf 'shaper\n' > src/LibreQoS.py
|
||||
printf 'fn main() {}\n' > src/rust/lqosd/main.rs
|
||||
printf '2.0\n' > src/VERSION_STRING
|
||||
printf 'binpacking\n' > requirements.txt
|
||||
printf 'pytest\n' > requirements_dev.txt
|
||||
printf '# package\n' > src/build_dpkg.sh
|
||||
printf '# build rust\n' > src/build_rust.sh
|
||||
printf 'name: Rust\n' > .github/workflows/rust.yml
|
||||
printf 'name: lint_python\n' > .github/workflows/lint_python.yml
|
||||
printf 'name: detect-changes\n' > .github/workflows/detect-changes.yml
|
||||
printf '# helper\n' > .github/scripts/other.sh
|
||||
git add -A
|
||||
git commit -qm base
|
||||
git push -q origin develop
|
||||
|
||||
new_branch() {
|
||||
git checkout -q -b "$1" develop
|
||||
}
|
||||
|
||||
commit_file() {
|
||||
printf '%s\n' "$2" > "$1"
|
||||
git add -A
|
||||
git commit -qm "change $1"
|
||||
}
|
||||
|
||||
export GITHUB_REF=refs/heads/test
|
||||
export GITHUB_EVENT_NAME=push
|
||||
export DEFAULT_BRANCH=develop
|
||||
export GITHUB_BASE_REF=""
|
||||
|
||||
# --- push: one file per group ---
|
||||
new_branch python-change
|
||||
commit_file src/LibreQoS.py '# python change'
|
||||
export GITHUB_SHA="$(git rev-parse HEAD)" EVENT_BEFORE="$(git rev-parse HEAD~1)"
|
||||
check "push/python" true python
|
||||
check "push/python" false rust
|
||||
|
||||
new_branch rust-change
|
||||
commit_file src/rust/lqosd/main.rs '// rust change'
|
||||
export GITHUB_SHA="$(git rev-parse HEAD)" EVENT_BEFORE="$(git rev-parse HEAD~1)"
|
||||
check "push/rust" true rust
|
||||
check "push/rust" false python
|
||||
|
||||
new_branch docs-change
|
||||
commit_file README.md 'more docs'
|
||||
export GITHUB_SHA="$(git rev-parse HEAD)" EVENT_BEFORE="$(git rev-parse HEAD~1)"
|
||||
check "push/docs" false python
|
||||
check "push/docs" false rust
|
||||
|
||||
# --- push: pattern coverage ---
|
||||
new_branch requirements-change
|
||||
commit_file requirements_dev.txt 'coverage'
|
||||
export GITHUB_SHA="$(git rev-parse HEAD)" EVENT_BEFORE="$(git rev-parse HEAD~1)"
|
||||
check "push/requirements" true python
|
||||
check "push/requirements" false rust
|
||||
|
||||
new_branch version-change
|
||||
commit_file src/VERSION_STRING '2.1'
|
||||
export GITHUB_SHA="$(git rev-parse HEAD)" EVENT_BEFORE="$(git rev-parse HEAD~1)"
|
||||
check "push/version" true rust
|
||||
check "push/version" false python
|
||||
|
||||
new_branch build-script-change
|
||||
commit_file src/build_rust.sh '# tweak'
|
||||
export GITHUB_SHA="$(git rev-parse HEAD)" EVENT_BEFORE="$(git rev-parse HEAD~1)"
|
||||
check "push/build-script" true rust
|
||||
check "push/build-script" false python
|
||||
|
||||
new_branch package-script-change
|
||||
commit_file src/build_dpkg.sh '# tweak'
|
||||
export GITHUB_SHA="$(git rev-parse HEAD)" EVENT_BEFORE="$(git rev-parse HEAD~1)"
|
||||
check "push/package-script" true rust
|
||||
check "push/package-script" false python
|
||||
|
||||
new_branch rust-workflow-change
|
||||
commit_file .github/workflows/rust.yml 'name: Rust # tweak'
|
||||
export GITHUB_SHA="$(git rev-parse HEAD)" EVENT_BEFORE="$(git rev-parse HEAD~1)"
|
||||
check "push/rust-workflow" true rust
|
||||
check "push/rust-workflow" false python
|
||||
|
||||
new_branch python-workflow-change
|
||||
commit_file .github/workflows/lint_python.yml 'name: lint_python # tweak'
|
||||
export GITHUB_SHA="$(git rev-parse HEAD)" EVENT_BEFORE="$(git rev-parse HEAD~1)"
|
||||
check "push/python-workflow" true python
|
||||
check "push/python-workflow" false rust
|
||||
|
||||
new_branch shared-workflow-change
|
||||
commit_file .github/workflows/detect-changes.yml 'name: detect-changes # tweak'
|
||||
export GITHUB_SHA="$(git rev-parse HEAD)" EVENT_BEFORE="$(git rev-parse HEAD~1)"
|
||||
check "push/shared-workflow" true python
|
||||
check "push/shared-workflow" true rust
|
||||
|
||||
new_branch shared-script-change
|
||||
commit_file .github/scripts/other.sh '# tweak'
|
||||
export GITHUB_SHA="$(git rev-parse HEAD)" EVENT_BEFORE="$(git rev-parse HEAD~1)"
|
||||
check "push/shared-script" true python
|
||||
check "push/shared-script" true rust
|
||||
|
||||
# --- push: new branch with the all-zero before ---
|
||||
new_branch new-branch
|
||||
commit_file src/LibreQoS.py '# new branch change'
|
||||
export GITHUB_SHA="$(git rev-parse HEAD)"
|
||||
export EVENT_BEFORE=0000000000000000000000000000000000000000
|
||||
check "push/new-branch" true python
|
||||
check "push/new-branch" false rust
|
||||
|
||||
# --- push: before no longer exists (force push) ---
|
||||
export EVENT_BEFORE=1111111111111111111111111111111111111111
|
||||
check "push/gone-before" true python
|
||||
check "push/gone-before" false rust
|
||||
|
||||
# --- push: no before at all, falls back to the default branch ---
|
||||
export EVENT_BEFORE=""
|
||||
check "push/no-before" true python
|
||||
check "push/no-before" false rust
|
||||
|
||||
# --- push: tag on an already-pushed commit ---
|
||||
git checkout -q develop
|
||||
export GITHUB_SHA="$(git rev-parse HEAD)"
|
||||
export EVENT_BEFORE=0000000000000000000000000000000000000000
|
||||
export GITHUB_REF=refs/tags/v2.0.0
|
||||
check "push/tag" true python
|
||||
check "push/tag" true rust
|
||||
export GITHUB_REF=refs/heads/test
|
||||
|
||||
# --- push: non-ASCII filename ---
|
||||
new_branch unicode-change
|
||||
commit_file "src/café.py" '# unicode'
|
||||
export GITHUB_SHA="$(git rev-parse HEAD)" EVENT_BEFORE="$(git rev-parse HEAD~1)"
|
||||
check "push/unicode" true python
|
||||
check "push/unicode" false rust
|
||||
|
||||
# --- pull_request: only the branch's own changes count ---
|
||||
new_branch pr-branch
|
||||
commit_file src/LibreQoS.py '# pr change'
|
||||
git checkout -q develop
|
||||
commit_file src/rust/lqosd/main.rs '// develop advances'
|
||||
git push -q origin develop
|
||||
git checkout -q pr-branch
|
||||
export GITHUB_EVENT_NAME=pull_request
|
||||
export GITHUB_BASE_REF=develop
|
||||
export GITHUB_SHA="$(git rev-parse HEAD)"
|
||||
check "pr/python-only" true python
|
||||
check "pr/python-only" false rust
|
||||
|
||||
export GITHUB_BASE_REF=does-not-exist
|
||||
check "pr/unknown-base" true python
|
||||
check "pr/unknown-base" true rust
|
||||
|
||||
# --- unknown event: everything counts as changed ---
|
||||
export GITHUB_EVENT_NAME=workflow_dispatch
|
||||
export GITHUB_BASE_REF=""
|
||||
check "workflow_dispatch" true python
|
||||
check "workflow_dispatch" true rust
|
||||
|
||||
echo
|
||||
if (( failures > 0 )); then
|
||||
echo "$checks checks, $failures failures"
|
||||
exit 1
|
||||
fi
|
||||
echo "$checks checks, 0 failures"
|
||||
@@ -0,0 +1,31 @@
|
||||
name: detect-changes
|
||||
|
||||
# Shared change detection for lint_python.yml and rust.yml. Callers gate their
|
||||
# heavy jobs on the `matched` output; see .github/scripts/changed-files.sh.
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
group:
|
||||
description: File group to detect (python or rust)
|
||||
required: true
|
||||
type: string
|
||||
outputs:
|
||||
matched:
|
||||
description: "true when files in the group changed"
|
||||
value: ${{ jobs.detect.outputs.matched }}
|
||||
|
||||
jobs:
|
||||
detect:
|
||||
runs-on: lq_run
|
||||
outputs:
|
||||
matched: ${{ steps.filter.outputs.matched }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Detect ${{ inputs.group }} changes
|
||||
id: filter
|
||||
env:
|
||||
EVENT_BEFORE: ${{ github.event.before }}
|
||||
DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
|
||||
run: bash .github/scripts/changed-files.sh "${{ inputs.group }}"
|
||||
@@ -1,7 +1,17 @@
|
||||
name: lint_python
|
||||
on: [pull_request, push]
|
||||
jobs:
|
||||
changes:
|
||||
uses: ./.github/workflows/detect-changes.yml
|
||||
with:
|
||||
group: python
|
||||
|
||||
# Python checks only run when Python sources or dependencies changed.
|
||||
# Skipped jobs report success, so required status checks stay compatible;
|
||||
# if change detection fails, the checks run anyway.
|
||||
lint_python:
|
||||
needs: changes
|
||||
if: needs.changes.result != 'success' || needs.changes.outputs.matched == 'true'
|
||||
runs-on: lq_run
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
@@ -3,7 +3,17 @@ name: Rust
|
||||
on: [push]
|
||||
|
||||
jobs:
|
||||
changes:
|
||||
uses: ./.github/workflows/detect-changes.yml
|
||||
with:
|
||||
group: rust
|
||||
|
||||
# The Rust build and audit only run when the Rust workspace or its build
|
||||
# scripts changed. Skipped jobs report success; if change detection fails,
|
||||
# they run anyway.
|
||||
build:
|
||||
needs: changes
|
||||
if: needs.changes.result != 'success' || needs.changes.outputs.matched == 'true'
|
||||
runs-on: [ lq_run ]
|
||||
|
||||
steps:
|
||||
@@ -41,6 +51,8 @@ jobs:
|
||||
path: src/dist/libreqos_${{ env.VERSION }}.-1_amd64.deb
|
||||
|
||||
audit:
|
||||
needs: changes
|
||||
if: needs.changes.result != 'success' || needs.changes.outputs.matched == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
|
||||
Reference in new issue
Block a user