From f89d9f384ecaea61b149da8e52eec4873be2ae83 Mon Sep 17 00:00:00 2001 From: Greg Magolan Date: Sun, 23 Feb 2020 17:35:46 -0800 Subject: [PATCH] test: saucelab targets for all karma tests (#35516) PR Close #35516 --- .circleci/config.yml | 41 ++++++++++++++--------- BUILD.bazel | 75 ------------------------------------------ tools/defaults.bzl | 77 ++++++++++++++++++++++++++------------------ 3 files changed, 72 insertions(+), 121 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a0b37f686e..7fa68872ad 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -343,10 +343,6 @@ jobs: path: dist/bin/packages/core/test/bundling/todo/bundle.min.js.br destination: core/todo/bundle.br - # This job is currently a PoC for running tests on SauceLabs via bazel. It runs a subset of the - # tests in `legacy-unit-tests-saucelabs` (see - # [BUILD.bazel](https://github.com/angular/angular/blob/ef44f51d5/BUILD.bazel#L66-L92)). - # # NOTE: This is currently limited to master builds only. See the `default_workflow` configuration. saucelabs_view_engine: executor: @@ -360,16 +356,18 @@ jobs: - init_environment - init_saucelabs_environment - run: - name: Run Bazel tests on Saucelabs + name: Run Bazel tests on Saucelabs with ViewEngine # See /tools/saucelabs/README.md for more info command: | yarn bazel run //tools/saucelabs:sauce_service_setup - yarn bazel test //:saucelabs_unit_tests_poc_suite --config=saucelabs + TESTS=$(./node_modules/.bin/bazel query --output label '(kind(karma_web_test, ...) intersect attr("tags", "saucelabs", ...)) except attr("tags", "ivy-only", ...) except attr("tags", "fixme-saucelabs", ...)') + yarn bazel test --config=saucelabs ${TESTS} yarn bazel run //tools/saucelabs:sauce_service_stop - no_output_timeout: 20m + no_output_timeout: 40m - notify_webhook_on_fail: webhook_url_env_var: SLACK_DEV_INFRA_CI_FAILURES_WEBHOOK_URL + # NOTE: This is currently limited to master builds only. See the `default_workflow` configuration. saucelabs_ivy: executor: name: default-executor @@ -382,13 +380,16 @@ jobs: - init_environment - init_saucelabs_environment - run: - name: Run Bazel tests on Saucelabs + name: Run Bazel tests on Saucelabs with Ivy # See /tools/saucelabs/README.md for more info command: | yarn bazel run //tools/saucelabs:sauce_service_setup - yarn bazel test //:saucelabs_unit_tests --config=saucelabs --config=ivy + TESTS=$(./node_modules/.bin/bazel query --output label '(kind(karma_web_test, ...) intersect attr("tags", "saucelabs", ...)) except attr("tags", "no-ivy-aot", ...) except attr("tags", "fixme-saucelabs", ...)') + yarn bazel test --config=saucelabs --config=ivy ${TESTS} yarn bazel run //tools/saucelabs:sauce_service_stop - no_output_timeout: 20m + no_output_timeout: 40m + - notify_webhook_on_fail: + webhook_url_env_var: SLACK_DEV_INFRA_CI_FAILURES_WEBHOOK_URL test_aio: executor: default-executor @@ -840,13 +841,22 @@ workflows: requires: - setup - saucelabs_ivy: + # Testing saucelabs via Bazel currently taking longer than the legacy saucelabs job as it + # each karma_web_test target is provisioning and tearing down browsers which is adding + # a lot of overhead. Running on master only to avoid wasting resources and slowing down + # CI for PRs. + # TODO: Run this job on all branches (including PRs) once karma_web_test targets can + # share provisioned browsers and we can remove the legacy saucelabs job. + <<: *only_on_master requires: - - test_ivy_aot + - setup - saucelabs_view_engine: - # This job is currently a PoC and a subset of `legacy-unit-tests-saucelabs`. Running on - # master only to avoid wasting resources. - # TODO: Run this job on all branches (including PRs) as soon as it is not a PoC and - # we can remove the legacy saucelabs job. + # Testing saucelabs via Bazel currently taking longer than the legacy saucelabs job as it + # each karma_web_test target is provisioning and tearing down browsers which is adding + # a lot of overhead. Running on master only to avoid wasting resources and slowing down + # CI for PRs. + # TODO: Run this job on all branches (including PRs) once karma_web_test targets can + # share provisioned browsers and we can remove the legacy saucelabs job. <<: *only_on_master requires: - setup @@ -899,6 +909,7 @@ workflows: - test - test_ivy_aot - integration_test + - saucelabs_view_engine - saucelabs_ivy # Only publish if `aio`/`docs` tests using the locally built Angular packages pass - test_aio_local diff --git a/BUILD.bazel b/BUILD.bazel index 79be349a96..0a599d8bfa 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -1,7 +1,5 @@ package(default_visibility = ["//visibility:public"]) -load("//tools:defaults.bzl", "karma_web_test") - exports_files([ "LICENSE", "protractor-perf.conf.js", @@ -46,76 +44,3 @@ filegroup( "@npm//:node_modules/angular-mocks-1.6/angular-mocks.js", ], ) - -# To run manually: -# Setup your SAUCE_USERNAME, SAUCE_ACCESS_KEY & SAUCE_TUNNEL_IDENTIFIER. -# If on OSX, also set SAUCE_CONNECT to the path of your `sc` binary. -# environment variables and run: -# ``` -# yarn bazel run //tools/saucelabs:sauce_service_setup -# yarn bazel test //:saucelabs_unit_tests --config=saucelabs --config=ivy -# ``` -# See /tools/saucelabs/README.md for more info on karma Saucelabs tests under Bazel. -karma_web_test( - name = "saucelabs_unit_tests", - # Default timeout is moderate (5min). This causes the test to be terminated while - # Saucelabs browsers keep running. Ultimately resulting in failing tests and browsers - # unnecessarily being acquired. Our specified Saucelabs idle timeout is 10min, so we use - # Bazel's long timeout (15min). This ensures that Karma can shut down properly. - timeout = "long", - karma = "//tools/saucelabs:karma-saucelabs", - tags = [ - "manual", - "no-remote-exec", - "saucelabs", - ], - deps = [ - "//packages/core/test/acceptance:acceptance_lib", - ], -) - -SAUCE_TEST_SUITE_TARGETS = [ - "packages/common/http/test:test_lib", - "packages/common/http/testing/test:test_lib", - "packages/common/test:test_lib", - "packages/core/test:test_lib", - "packages/forms/test:test_lib", - "packages/http/test:test_lib", -] - -[ - # These target runs in CI with View Engine as a Saucelabs and Bazel proof-of-concept. It's a - # subset of the legacy saucelabs tests. - karma_web_test( - name = "saucelabs_unit_tests_poc_%s" % test.replace("/", "_").replace(":", "_").replace(".", "_"), - # Default timeout is moderate (5min). This causes the test to be terminated while - # Saucelabs browsers keep running. Ultimately resulting in failing tests and browsers - # unnecessarily being acquired. Our specified Saucelabs idle timeout is 10min, so we use - # Bazel's long timeout (15min). This ensures that Karma can shut down properly. - timeout = "long", - karma = "//tools/saucelabs:karma-saucelabs", - tags = [ - "exclusive", - "manual", - "no-remote-exec", - "saucelabs", - ], - deps = ["//%s" % test], - ) - for test in SAUCE_TEST_SUITE_TARGETS -] - -# To run manually: -# Setup your SAUCE_USERNAME, SAUCE_ACCESS_KEY & SAUCE_TUNNEL_IDENTIFIER. -# If on OSX, also set SAUCE_CONNECT to the path of your `sc` binary. -# environment variables and run: -# ``` -# yarn bazel run //tools/saucelabs:sauce_service_setup -# yarn bazel test //:saucelabs_unit_tests_poc_suite --config=saucelabs -# ``` -# See /tools/saucelabs/README.md for more info on karma Saucelabs tests under Bazel. -test_suite( - name = "saucelabs_unit_tests_poc_suite", - tags = ["manual"], - tests = ["//:saucelabs_unit_tests_poc_%s" % test.replace("/", "_").replace(":", "_").replace(".", "_") for test in SAUCE_TEST_SUITE_TARGETS], -) diff --git a/tools/defaults.bzl b/tools/defaults.bzl index 4cfdcf125d..d9255a9f41 100644 --- a/tools/defaults.bzl +++ b/tools/defaults.bzl @@ -214,54 +214,69 @@ def pkg_npm(name, substitutions = {}, **kwargs): visibility = visibility, ) -def karma_web_test(bootstrap = [], deps = [], data = [], runtime_deps = [], **kwargs): - """Default values for karma_web_test""" - if not bootstrap: - bootstrap = ["//:web_test_bootstrap_scripts"] - local_deps = [ +def karma_web_test_suite(name, **kwargs): + """Default values for karma_web_test_suite""" + + # Default value for bootstrap + bootstrap = kwargs.pop("bootstrap", [ + "//:web_test_bootstrap_scripts", + ]) + + # Add common deps + deps = kwargs.pop("deps", []) + [ "@npm//karma-browserstack-launcher", "@npm//karma-sauce-launcher", "@npm//:node_modules/tslib/tslib.js", "//tools/rxjs:rxjs_umd_modules", "//packages/zone.js:npm_package", - ] + deps - local_runtime_deps = [ - "//tools/testing:browser", - ] + runtime_deps + ] + # Add common runtime deps + runtime_deps = kwargs.pop("runtime_deps", []) + [ + "//tools/testing:browser", + ] + + data = kwargs.pop("data", []) + tags = kwargs.pop("tags", []) + + _karma_web_test_suite( + name = name, + runtime_deps = runtime_deps, + bootstrap = bootstrap, + deps = deps, + browsers = ["//tools/browsers:chromium"], + data = data, + tags = tags, + **kwargs + ) + + # Add a saucelabs target for these karma tests _karma_web_test( - runtime_deps = local_runtime_deps, + name = "saucelabs_%s" % name, + # Default timeout is moderate (5min). This causes the test to be terminated while + # Saucelabs browsers keep running. Ultimately resulting in failing tests and browsers + # unnecessarily being acquired. Our specified Saucelabs idle timeout is 10min, so we use + # Bazel's long timeout (15min). This ensures that Karma can shut down properly. + timeout = "long", + runtime_deps = runtime_deps, bootstrap = bootstrap, config_file = "//:karma-js.conf.js", - deps = local_deps, + deps = deps, data = data + [ "//:browser-providers.conf.js", "//tools:jasmine-seed-generator.js", ], + karma = "//tools/saucelabs:karma-saucelabs", + tags = tags + [ + "exclusive", + "manual", + "no-remote-exec", + "saucelabs", + ], configuration_env_vars = ["KARMA_WEB_TEST_MODE"], **kwargs ) -def karma_web_test_suite(bootstrap = [], deps = [], runtime_deps = [], **kwargs): - """Default values for karma_web_test_suite""" - if not bootstrap: - bootstrap = ["//:web_test_bootstrap_scripts"] - local_deps = [ - "@npm//:node_modules/tslib/tslib.js", - "//tools/rxjs:rxjs_umd_modules", - ] + deps - local_runtime_deps = [ - "//tools/testing:browser", - ] + runtime_deps - - _karma_web_test_suite( - runtime_deps = local_runtime_deps, - bootstrap = bootstrap, - deps = local_deps, - browsers = ["//tools/browsers:chromium"], - **kwargs - ) - def protractor_web_test_suite(**kwargs): """Default values for protractor_web_test_suite"""