IntenseWebs/discourse
3
0
Fork 0
mirror of https://github.com/discourse/discourse.git synced 2024-11-24 09:50:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

DEV: Only retry and log flaky tests on the main branch (#24889)

Browse Source 
Why this change?

Pull requests can introduce flaky tests into the mix and we do not want
to be hiing that during the pull request process. While this does mean
builds for PR will be less stable than the `main` branch without
retries, we do not foresee this to be a problem long term since the
monitoring of flaky tests on the `main` branch will mean that the number
of flaky tests will eventually be reduced.

What does this change do?

1. Introduce the `DISCOURSE_TURBO_RSPEC_RETRY_AND_LOG_FLAKY_TESTS` env
   variable which will initialize `TurboTest::Runner` with the `retry_and_log_flaky_tests`
   kwarg set to true when set.

2. Change the tests workflow run to set `DISCOURSE_TURBO_RSPEC_RETRY_AND_LOG_FLAKY_TESTS` only when
   the build type is `backend` or `system` and the `github.ref_name` is
   `main`.
This commit is contained in:
Alan Guo Xiang Tan 2023-12-14 09:41:30 +08:00 committed by GitHub
parent 00209f03e6
commit 2a1952d9ba
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 8 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
.github/workflows/tests.yml vendored
View File

@ -36,6 +36,7 @@ jobs:
CAPYBARA_DEFAULT_MAX_WAIT_TIME: 10
MINIO_RUNNER_LOG_LEVEL: DEBUG
EMBER_VERSION: ${{ (matrix.updated_ember && '5') || '3' }}
DISCOURSE_TURBO_RSPEC_RETRY_AND_LOG_FLAKY_TESTS: ${{ (matrix.build_type == 'system' || matrix.build_type == 'backend') && github.ref_name == 'main' }}
strategy:
fail-fast: false
@ -218,11 +219,11 @@ jobs:
- name: Core RSpec
if: matrix.build_type == 'backend' && matrix.target == 'core'
run: bin/turbo_rspec --use-runtime-info --verbose --format documentation --retry-and-log-flaky-tests
run: bin/turbo_rspec --use-runtime-info --verbose --format documentation
- name: Plugin RSpec
if: matrix.build_type == 'backend' && matrix.target == 'plugins'
run: bin/rake plugin:turbo_spec['*','--verbose --format documentation --use-runtime-info --retry-and-log-flaky-tests']
run: bin/rake plugin:turbo_spec['*','--verbose --format documentation --use-runtime-info']
- name: Plugin QUnit
if: matrix.build_type == 'frontend' && matrix.target == 'plugins'
@ -250,25 +251,25 @@ jobs:
- name: Core System Tests
if: matrix.build_type == 'system' && matrix.target == 'core'
run: RAILS_ENABLE_TEST_LOG=1 RAILS_TEST_LOG_LEVEL=error PARALLEL_TEST_PROCESSORS=4 bin/turbo_rspec --use-runtime-info --profile=50 --verbose --format documentation --retry-and-log-flaky-tests spec/system
run: RAILS_ENABLE_TEST_LOG=1 RAILS_TEST_LOG_LEVEL=error PARALLEL_TEST_PROCESSORS=4 bin/turbo_rspec --use-runtime-info --profile=50 --verbose --format documentation spec/system
- name: Plugin System Tests
if: matrix.build_type == 'system' && matrix.target == 'plugins'
run: |
GLOBIGNORE="plugins/chat/*";
LOAD_PLUGINS=1 RAILS_ENABLE_TEST_LOG=1 RAILS_TEST_LOG_LEVEL=error PARALLEL_TEST_PROCESSORS=4 bin/turbo_rspec --use-runtime-info --profile=50 --verbose --format documentation --retry-and-log-flaky-tests plugins/*/spec/system
LOAD_PLUGINS=1 RAILS_ENABLE_TEST_LOG=1 RAILS_TEST_LOG_LEVEL=error PARALLEL_TEST_PROCESSORS=4 bin/turbo_rspec --use-runtime-info --profile=50 --verbose --format documentation plugins/*/spec/system
shell: bash
timeout-minutes: 30
- name: Chat System Tests
if: matrix.build_type == 'system' && matrix.target == 'chat'
run: LOAD_PLUGINS=1 RAILS_ENABLE_TEST_LOG=1 RAILS_TEST_LOG_LEVEL=error PARALLEL_TEST_PROCESSORS=4 bin/turbo_rspec --use-runtime-info --profile=50 --verbose --format documentation --retry-and-log-flaky-tests plugins/chat/spec/system
run: LOAD_PLUGINS=1 RAILS_ENABLE_TEST_LOG=1 RAILS_TEST_LOG_LEVEL=error PARALLEL_TEST_PROCESSORS=4 bin/turbo_rspec --use-runtime-info --profile=50 --verbose --format documentation plugins/chat/spec/system
timeout-minutes: 30
- name: Theme System Tests
if: matrix.build_type == 'system' && matrix.target == 'themes'
run: |
RAILS_ENABLE_TEST_LOG=1 RAILS_TEST_LOG_LEVEL=error PARALLEL_TEST_PROCESSORS=4 bin/turbo_rspec --profile=50 --verbose --format documentation --retry-and-log-flaky-tests tmp/themes/*/spec/system
RAILS_ENABLE_TEST_LOG=1 RAILS_TEST_LOG_LEVEL=error PARALLEL_TEST_PROCESSORS=4 bin/turbo_rspec --profile=50 --verbose --format documentation tmp/themes/*/spec/system
shell: bash
timeout-minutes: 30

2
bin/turbo_rspec
View File

@ -16,7 +16,7 @@ profile_print_slowest_examples_count = 10
use_runtime_info = nil
enable_system_tests = nil
disable_system_tests = nil
retry_and_log_flaky_tests = false
retry_and_log_flaky_tests = !!ENV["DISCOURSE_TURBO_RSPEC_RETRY_AND_LOG_FLAKY_TESTS"]
OptionParser
.new do |opts|