From 2a1952d9bae990effe7e27db4186a6e1ec56d6df Mon Sep 17 00:00:00 2001 From: Alan Guo Xiang Tan Date: Thu, 14 Dec 2023 09:41:30 +0800 Subject: [PATCH] DEV: Only retry and log flaky tests on the main branch (#24889) Why this change? Pull requests can introduce flaky tests into the mix and we do not want to be hiing that during the pull request process. While this does mean builds for PR will be less stable than the `main` branch without retries, we do not foresee this to be a problem long term since the monitoring of flaky tests on the `main` branch will mean that the number of flaky tests will eventually be reduced. What does this change do? 1. Introduce the `DISCOURSE_TURBO_RSPEC_RETRY_AND_LOG_FLAKY_TESTS` env variable which will initialize `TurboTest::Runner` with the `retry_and_log_flaky_tests` kwarg set to true when set. 2. Change the tests workflow run to set `DISCOURSE_TURBO_RSPEC_RETRY_AND_LOG_FLAKY_TESTS` only when the build type is `backend` or `system` and the `github.ref_name` is `main`. --- .github/workflows/tests.yml | 13 +++++++------ bin/turbo_rspec | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8f5cff61ba2..79935f97165 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -36,6 +36,7 @@ jobs: CAPYBARA_DEFAULT_MAX_WAIT_TIME: 10 MINIO_RUNNER_LOG_LEVEL: DEBUG EMBER_VERSION: ${{ (matrix.updated_ember && '5') || '3' }} + DISCOURSE_TURBO_RSPEC_RETRY_AND_LOG_FLAKY_TESTS: ${{ (matrix.build_type == 'system' || matrix.build_type == 'backend') && github.ref_name == 'main' }} strategy: fail-fast: false @@ -218,11 +219,11 @@ jobs: - name: Core RSpec if: matrix.build_type == 'backend' && matrix.target == 'core' - run: bin/turbo_rspec --use-runtime-info --verbose --format documentation --retry-and-log-flaky-tests + run: bin/turbo_rspec --use-runtime-info --verbose --format documentation - name: Plugin RSpec if: matrix.build_type == 'backend' && matrix.target == 'plugins' - run: bin/rake plugin:turbo_spec['*','--verbose --format documentation --use-runtime-info --retry-and-log-flaky-tests'] + run: bin/rake plugin:turbo_spec['*','--verbose --format documentation --use-runtime-info'] - name: Plugin QUnit if: matrix.build_type == 'frontend' && matrix.target == 'plugins' @@ -250,25 +251,25 @@ jobs: - name: Core System Tests if: matrix.build_type == 'system' && matrix.target == 'core' - run: RAILS_ENABLE_TEST_LOG=1 RAILS_TEST_LOG_LEVEL=error PARALLEL_TEST_PROCESSORS=4 bin/turbo_rspec --use-runtime-info --profile=50 --verbose --format documentation --retry-and-log-flaky-tests spec/system + run: RAILS_ENABLE_TEST_LOG=1 RAILS_TEST_LOG_LEVEL=error PARALLEL_TEST_PROCESSORS=4 bin/turbo_rspec --use-runtime-info --profile=50 --verbose --format documentation spec/system - name: Plugin System Tests if: matrix.build_type == 'system' && matrix.target == 'plugins' run: | GLOBIGNORE="plugins/chat/*"; - LOAD_PLUGINS=1 RAILS_ENABLE_TEST_LOG=1 RAILS_TEST_LOG_LEVEL=error PARALLEL_TEST_PROCESSORS=4 bin/turbo_rspec --use-runtime-info --profile=50 --verbose --format documentation --retry-and-log-flaky-tests plugins/*/spec/system + LOAD_PLUGINS=1 RAILS_ENABLE_TEST_LOG=1 RAILS_TEST_LOG_LEVEL=error PARALLEL_TEST_PROCESSORS=4 bin/turbo_rspec --use-runtime-info --profile=50 --verbose --format documentation plugins/*/spec/system shell: bash timeout-minutes: 30 - name: Chat System Tests if: matrix.build_type == 'system' && matrix.target == 'chat' - run: LOAD_PLUGINS=1 RAILS_ENABLE_TEST_LOG=1 RAILS_TEST_LOG_LEVEL=error PARALLEL_TEST_PROCESSORS=4 bin/turbo_rspec --use-runtime-info --profile=50 --verbose --format documentation --retry-and-log-flaky-tests plugins/chat/spec/system + run: LOAD_PLUGINS=1 RAILS_ENABLE_TEST_LOG=1 RAILS_TEST_LOG_LEVEL=error PARALLEL_TEST_PROCESSORS=4 bin/turbo_rspec --use-runtime-info --profile=50 --verbose --format documentation plugins/chat/spec/system timeout-minutes: 30 - name: Theme System Tests if: matrix.build_type == 'system' && matrix.target == 'themes' run: | - RAILS_ENABLE_TEST_LOG=1 RAILS_TEST_LOG_LEVEL=error PARALLEL_TEST_PROCESSORS=4 bin/turbo_rspec --profile=50 --verbose --format documentation --retry-and-log-flaky-tests tmp/themes/*/spec/system + RAILS_ENABLE_TEST_LOG=1 RAILS_TEST_LOG_LEVEL=error PARALLEL_TEST_PROCESSORS=4 bin/turbo_rspec --profile=50 --verbose --format documentation tmp/themes/*/spec/system shell: bash timeout-minutes: 30 diff --git a/bin/turbo_rspec b/bin/turbo_rspec index 910d77aec0c..348e17ecf5c 100755 --- a/bin/turbo_rspec +++ b/bin/turbo_rspec @@ -16,7 +16,7 @@ profile_print_slowest_examples_count = 10 use_runtime_info = nil enable_system_tests = nil disable_system_tests = nil -retry_and_log_flaky_tests = false +retry_and_log_flaky_tests = !!ENV["DISCOURSE_TURBO_RSPEC_RETRY_AND_LOG_FLAKY_TESTS"] OptionParser .new do |opts|