From 0aba1761714661b8576d4676c61c319e76bfac1b Mon Sep 17 00:00:00 2001
From: dundargoc <33953936+dundargoc@users.noreply.github.com>
Date: Fri, 4 Nov 2022 13:26:12 +0100
Subject: [PATCH] ci: skip tests if build fails (#20908)

It's currently difficult to pinpoint the cause of a failure since all
tests are run even if the build steps fail. But since the build failed
the test will almost always fail as well as it's dependent on a
successful build, leading to many steps being marked as a failure even
though the real problem was the build step. Even worse, the default
behavior of GitHub Actions is to only automatically show the last failed
step, which is misleading if the build process fails since it'll show
the logs of the failing test step.

An easy solution would be to abort all subsequent steps if any steps
fail. This isn't optimal however, as we want all lint and test failures
to show on a single run instead of prematurely aborting on a single test
step.

We can solve both problems by dividing each job into two phases: the
build/installation phase and the test/lint phase, with a checkmark step
in between. The strategy is simple: if any step before the checkmark
step fails (the build phase), then abort all following steps. If any
step after the checkmark fails (the test phase), then show that test as
failed but continue running all tests.
---
 .github/workflows/ci.yml | 43 ++++++++++++++++++++++++++++++----------
 1 file changed, 32 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index c386930073..732e3bb8c1 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -95,6 +95,11 @@ jobs:
         run: ./ci/before_script.sh
 
       - if: "!cancelled()"
+        name: Determine if run should be aborted
+        id: abort_job
+        run: echo "status=${{ job.status }}" >> $GITHUB_OUTPUT
+
+      - if: success() || failure() && steps.abort_job.outputs.status == 'success'
         name: lintstylua
         uses: JohnnyMorganz/stylua-action@v1
         with:
@@ -102,20 +107,20 @@ jobs:
           version: latest
           args: --check runtime/
 
-      - if: "!cancelled()"
+      - if: success() || failure() && steps.abort_job.outputs.status == 'success'
         name: lintlua
         run: make lintlua
 
-      - if: "!cancelled()"
+      - if: success() || failure() && steps.abort_job.outputs.status == 'success'
         name: lintsh
         run: make lintsh
 
-      - if: "!cancelled()"
+      - if: success() || failure() && steps.abort_job.outputs.status == 'success'
         name: uncrustify
         run: |
           ${{ env.CACHE_UNCRUSTIFY }} -c ./src/uncrustify.cfg -q --replace --no-backup $(find ./src/nvim -name "*.[ch]")
 
-      - if: "!cancelled()"
+      - if: success() || failure() && steps.abort_job.outputs.status == 'success'
         name: suggester / uncrustify
         uses: reviewdog/action-suggester@v1
         with:
@@ -123,7 +128,7 @@ jobs:
           tool_name: uncrustify
           cleanup: false
 
-      - if: "!cancelled()"
+      - if: success() || failure() && steps.abort_job.outputs.status == 'success'
         name: check uncrustify
         run: |
           git diff --color --exit-code
@@ -192,10 +197,15 @@ jobs:
         run: ./ci/run_tests.sh build_nvim
 
       - if: "!cancelled()"
+        name: Determine if run should be aborted
+        id: abort_job
+        run: echo "status=${{ job.status }}" >> $GITHUB_OUTPUT
+
+      - if: success() || failure() && steps.abort_job.outputs.status == 'success'
         name: lintc
         run: make lintc
 
-      - if: "!cancelled()"
+      - if: success() || failure() && steps.abort_job.outputs.status == 'success'
         name: check-single-includes
         run: make check-single-includes
 
@@ -292,19 +302,24 @@ jobs:
       - name: Build
         run: ./ci/run_tests.sh build_nvim
 
-      - if: matrix.flavor != 'tsan' && matrix.flavor != 'functionaltest-lua' && !cancelled()
+      - if: "!cancelled()"
+        name: Determine if run should be aborted
+        id: abort_job
+        run: echo "status=${{ job.status }}" >> $GITHUB_OUTPUT
+
+      - if: matrix.flavor != 'tsan' && matrix.flavor != 'functionaltest-lua' && (success() || failure() && steps.abort_job.outputs.status == 'success')
         name: Unittests
         run: ./ci/run_tests.sh unittests
 
-      - if: matrix.flavor != 'tsan' && !cancelled()
+      - if: matrix.flavor != 'tsan' && (success() || failure() && steps.abort_job.outputs.status == 'success')
         name: Functionaltests
         run: ./ci/run_tests.sh functionaltests
 
-      - if: "!cancelled()"
+      - if: success() || failure() && steps.abort_job.outputs.status == 'success'
         name: Oldtests
         run: ./ci/run_tests.sh oldtests
 
-      - if: "!cancelled()"
+      - if: success() || failure() && steps.abort_job.outputs.status == 'success'
         name: Install nvim
         run: ./ci/run_tests.sh install_nvim
 
@@ -336,9 +351,15 @@ jobs:
         run: .\ci\build.ps1 -EnsureTestDeps
 
       - if: "!cancelled()"
+        name: Determine if run should be aborted
+        id: abort_job
+        run: |
+          "status=${{ job.status }}" >> $env:GITHUB_OUTPUT
+
+      - if: success() || failure() && steps.abort_job.outputs.status == 'success'
         name: Run tests
         run: .\ci\build.ps1 -Test
 
-      - if: "!cancelled()"
+      - if: success() || failure() && steps.abort_job.outputs.status == 'success'
         name: Run old tests
         run: .\ci\build.ps1 -TestOld