diff --git a/.circleci/bazel.common.rc b/.circleci/bazel.common.rc index f70a957aea..ab84c0ed80 100644 --- a/.circleci/bazel.common.rc +++ b/.circleci/bazel.common.rc @@ -13,3 +13,7 @@ test --flaky_test_attempts=2 # More details on failures build --verbose_failures=true + +# For saucelabs tests we don't want to enable flaky test attempts. Karma has its own integrated +# retry mechanism and we do not want to retry unnecessarily if Karma already tried multiple times. +test:saucelabs --flaky_test_attempts=1 diff --git a/.circleci/config.yml b/.circleci/config.yml index 36eef246d8..6b0f0557f0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -330,7 +330,7 @@ jobs: # [BUILD.bazel](https://github.com/angular/angular/blob/ef44f51d5/BUILD.bazel#L66-L92)). # # NOTE: This is currently limited to master builds only. See the `default_workflow` configuration. - test_saucelabs_bazel: + saucelabs_view_engine: executor: name: default-executor # In order to avoid the bottleneck of having a slow host machine, we acquire a better @@ -344,9 +344,9 @@ jobs: - setup_bazel_rbe - run: name: Run Bazel tests in saucelabs - # All web tests are contained within a single //:test_web_all target for Saucelabs - # as running each set of tests as a separate target will attempt to acquire too - # many browsers on Saucelabs (7 per target currently) and some tests will always + # All web tests are contained within a single //:saucelabs_unit_tests_poc target + # for Saucelabs as running each set of tests as a separate target will attempt to acquire + # too many browsers on Saucelabs (7 per target currently) and some tests will always # fail to acquire browsers. For example: # 14 02 2019 19:52:33.170:WARN [launcher]: chrome beta on SauceLabs have not captured in 180000 ms, killing. # //packages/forms/test:web_test_sauce TIMEOUT in 315.0s @@ -355,11 +355,34 @@ jobs: --tunnel-id angular-${CIRCLE_BUILD_NUM}-${CIRCLE_NODE_INDEX} \ --username $SAUCE_USERNAME \ --key $(echo $SAUCE_ACCESS_KEY | rev) \ - yarn bazel test //:test_web_all + yarn bazel test //:saucelabs_unit_tests_poc --config=saucelabs no_output_timeout: 20m - notify_webhook_on_fail: webhook_url_env_var: SLACK_DEV_INFRA_CI_FAILURES_WEBHOOK_URL + saucelabs_ivy: + executor: + name: default-executor + # In order to avoid the bottleneck of having a slow host machine, we acquire a better + # container for this job. This is necessary because we launch a lot of browsers concurrently + # and therefore the tunnel and Karma need to process a lot of file requests and tests. + resource_class: xlarge + steps: + - custom_attach_workspace + - init_environment + - setup_circleci_bazel_config + - setup_bazel_rbe + - run: + name: Run Bazel tests on Saucelabs + # Runs the //:saucelabs_tests target with Saucelabs and Ivy. + command: | + ./scripts/saucelabs/run-bazel-via-tunnel.sh \ + --tunnel-id angular-${CIRCLE_BUILD_NUM}-${CIRCLE_NODE_INDEX} \ + --username $SAUCE_USERNAME \ + --key $(echo $SAUCE_ACCESS_KEY | rev) \ + -- yarn bazel test //:saucelabs_unit_tests --config=ivy --config=saucelabs + no_output_timeout: 20m + test_aio: # Needed because the AIO tests and the PWA score test depend on Chrome being available. executor: browsers-executor @@ -819,11 +842,14 @@ workflows: - legacy-unit-tests-saucelabs: requires: - setup - - test_saucelabs_bazel: + - saucelabs_ivy: + requires: + - test_ivy_aot + - saucelabs_view_engine: # This job is currently a PoC and a subset of `legacy-unit-tests-saucelabs`. Running on # master only to avoid wasting resources. - # - # TODO: Run this job on all branches (including PRs) as soon as it is not a PoC. + # TODO: Run this job on all branches (including PRs) as soon as it is not a PoC and + # we can remove the legacy saucelabs job. <<: *only_on_master requires: - setup @@ -876,6 +902,7 @@ workflows: - test - test_ivy_aot - integration_test + - saucelabs_ivy # Only publish if `aio`/`docs` tests using the locally built Angular packages pass - test_aio_local - test_aio_local_viewengine diff --git a/BUILD.bazel b/BUILD.bazel index c031b302ef..3aacc15542 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -47,17 +47,28 @@ filegroup( ], ) -# To run a karma_web_test target locally on SauceLabs: -# 1) have SAUCE_USERNAME, SAUCE_ACCESS_KEY (and optionally a SAUCE_TUNNEL_IDENTIFIER) set in your environment -# 2) open a sauce connection with `./scripts/saucelabs/start-tunnel.sh` -# NOTE: start-tunnel.sh uses `node_modules/sauce-connect` which is current linux specific: -# "sauce-connect": "https://saucelabs.com/downloads/sc-4.5.3-linux.tar.gz". -# On OSX or Windows you'll need to use the appropriate sauce-connect binary. -# 3) run target with `yarn bazel test --config=saucelabs ` -# NOTE: --config=saucelabs is required as it makes the SAUCE_XXX environment variables available to -# the action. See /.bazelrc. +# To run a karma_web_test target manually, run the "./scripts/saucelabs/run-bazel-via-tunnel.sh" +# script. Note: If you are on MacOS or Windows, you need to manually start the Saucelabs tunnel +# because the script only supports the linux Saucelabs tunnel binary. We combine all tests into +# a single "karma_web_test" target because running them as separate targets in parallel can result +# in to too many browsers being acquired at the same time. This will then result in tests being +# flaky. This target runs in CI with Saucelabs and Ivy. karma_web_test( - name = "test_web_all", + name = "saucelabs_unit_tests", + tags = [ + "local", + "manual", + "saucelabs", + ], + deps = [ + "//packages/core/test/acceptance:acceptance_lib", + ], +) + +karma_web_test( + # This target runs in CI with View Engine as a Saucelabs and Bazel proof-of-concept. It's a + # subset of the legacy saucelabs tests. + name = "saucelabs_unit_tests_poc", tags = [ "local", "manual", @@ -65,7 +76,7 @@ karma_web_test( ], deps = [ # We combine all tests into a single karma_web_test target - # as running them as seperate targets in parallel leads to too many + # as running them as separate targets in parallel leads to too many # browsers being acquired at once in SauceLabs and the tests flake out # TODO: this is an example subset of tests below, add all remaining angular tests "//packages/common/http/test:test_lib", diff --git a/packages/common/http/test/BUILD.bazel b/packages/common/http/test/BUILD.bazel index ec114437a3..65eadfbb96 100644 --- a/packages/common/http/test/BUILD.bazel +++ b/packages/common/http/test/BUILD.bazel @@ -6,7 +6,7 @@ ts_library( srcs = glob( ["**/*.ts"], ), - # Visible to //:test_web_all target + # Visible to //:saucelabs_unit_tests_poc target visibility = ["//:__pkg__"], deps = [ "//packages/common/http", diff --git a/packages/common/http/testing/test/BUILD.bazel b/packages/common/http/testing/test/BUILD.bazel index a0a42f8be2..d9de470171 100644 --- a/packages/common/http/testing/test/BUILD.bazel +++ b/packages/common/http/testing/test/BUILD.bazel @@ -6,7 +6,7 @@ ts_library( srcs = glob( ["**/*.ts"], ), - # Visible to //:test_web_all target + # Visible to //:saucelabs_unit_tests_poc target visibility = ["//:__pkg__"], deps = [ "//packages/common/http", diff --git a/packages/common/test/BUILD.bazel b/packages/common/test/BUILD.bazel index f444bbf315..87e8552355 100644 --- a/packages/common/test/BUILD.bazel +++ b/packages/common/test/BUILD.bazel @@ -6,7 +6,7 @@ ts_library( srcs = glob( ["**/*.ts"], ), - # Visible to //:test_web_all target + # Visible to //:saucelabs_unit_tests_poc target visibility = ["//:__pkg__"], deps = [ "//packages/common", diff --git a/packages/core/test/BUILD.bazel b/packages/core/test/BUILD.bazel index d5e8207789..ab2ad8f7fa 100644 --- a/packages/core/test/BUILD.bazel +++ b/packages/core/test/BUILD.bazel @@ -11,7 +11,7 @@ ts_library( "**/*_node_only_spec.ts", ], ), - # Visible to //:test_web_all target + # Visible to //:saucelabs_unit_tests_poc target visibility = ["//:__pkg__"], deps = [ "//packages/animations", diff --git a/packages/core/test/acceptance/BUILD.bazel b/packages/core/test/acceptance/BUILD.bazel index 9d2c5d7274..cc87ecc34a 100644 --- a/packages/core/test/acceptance/BUILD.bazel +++ b/packages/core/test/acceptance/BUILD.bazel @@ -8,6 +8,8 @@ ts_library( srcs = glob( ["**/*.ts"], ), + # Visible to //:saucelabs_unit_tests + visibility = ["//:__pkg__"], deps = [ "//packages/animations", "//packages/animations/browser", diff --git a/packages/forms/test/BUILD.bazel b/packages/forms/test/BUILD.bazel index 1bf1a831e5..679a2af5c0 100644 --- a/packages/forms/test/BUILD.bazel +++ b/packages/forms/test/BUILD.bazel @@ -4,7 +4,7 @@ ts_library( name = "test_lib", testonly = True, srcs = glob(["**/*.ts"]), - # Visible to //:test_web_all target + # Visible to //:saucelabs_unit_tests_poc target visibility = ["//:__pkg__"], deps = [ "//packages/common", diff --git a/packages/http/test/BUILD.bazel b/packages/http/test/BUILD.bazel index 01d68b79c2..f7d5d2bc1d 100644 --- a/packages/http/test/BUILD.bazel +++ b/packages/http/test/BUILD.bazel @@ -4,7 +4,7 @@ ts_library( name = "test_lib", testonly = True, srcs = glob(["**/*.ts"]), - # Visible to //:test_web_all target + # Visible to //:saucelabs_unit_tests_poc target visibility = ["//:__pkg__"], deps = [ "//packages/common", diff --git a/packages/router/test/BUILD.bazel b/packages/router/test/BUILD.bazel index d1eab8b6cf..30e62191e2 100644 --- a/packages/router/test/BUILD.bazel +++ b/packages/router/test/BUILD.bazel @@ -4,7 +4,7 @@ ts_library( name = "test_lib", testonly = True, srcs = glob(["**/*.ts"]), - # Visible to //:test_web_all target + # Visible to //:saucelabs_unit_tests_poc target visibility = ["//:__pkg__"], deps = [ "//packages/common", diff --git a/packages/zone.js/test/karma_test.bzl b/packages/zone.js/test/karma_test.bzl index 49a451f425..8a96a9e232 100644 --- a/packages/zone.js/test/karma_test.bzl +++ b/packages/zone.js/test/karma_test.bzl @@ -127,7 +127,7 @@ def karma_test(name, env_srcs, env_deps, env_entry_point, test_srcs, test_deps, ":assets/import.html", ], tags = ["zone_karma_test"], - # Visible to //:test_web_all target + # Visible to //:saucelabs_unit_tests_poc target visibility = ["//:__pkg__"], runtime_deps = [ "@npm//karma-browserstack-launcher", diff --git a/scripts/saucelabs/run-bazel-via-tunnel.sh b/scripts/saucelabs/run-bazel-via-tunnel.sh index 349a5772a9..1d359a3443 100755 --- a/scripts/saucelabs/run-bazel-via-tunnel.sh +++ b/scripts/saucelabs/run-bazel-via-tunnel.sh @@ -158,9 +158,9 @@ set +e cd $GIT_ROOT_DIR && \ # Run bazel command with saucelabs specific environment variables passed to the action # The KARMA_WEB_TEST_MODE and SAUCE_TUNNEL_IDENTIFIER environment variables provide - # envirnment variables to be read in the karma configuration file to set correct + # environment variables to be read in the karma configuration file to set correct # configurations for karma saucelabs and browser configs. - # Usage of these envirnment variables can be seen in this repo in + # Usage of these environment variables can be seen in this repo in # /karma-js.conf.js and /browser-providers.conf.js eval "$USER_COMMAND --define=KARMA_WEB_TEST_MODE=SL_REQUIRED \ --action_env=SAUCE_USERNAME=$SAUCE_USERNAME \