test: saucelab targets for all karma tests (#35516)

PR Close #35516
This commit is contained in:
Greg Magolan 2020-02-23 17:35:46 -08:00 committed by Miško Hevery
parent 3400af6888
commit f89d9f384e
3 changed files with 72 additions and 121 deletions

View File

@ -343,10 +343,6 @@ jobs:
path: dist/bin/packages/core/test/bundling/todo/bundle.min.js.br
destination: core/todo/bundle.br
# This job is currently a PoC for running tests on SauceLabs via bazel. It runs a subset of the
# tests in `legacy-unit-tests-saucelabs` (see
# [BUILD.bazel](https://github.com/angular/angular/blob/ef44f51d5/BUILD.bazel#L66-L92)).
#
# NOTE: This is currently limited to master builds only. See the `default_workflow` configuration.
saucelabs_view_engine:
executor:
@ -360,16 +356,18 @@ jobs:
- init_environment
- init_saucelabs_environment
- run:
name: Run Bazel tests on Saucelabs
name: Run Bazel tests on Saucelabs with ViewEngine
# See /tools/saucelabs/README.md for more info
command: |
yarn bazel run //tools/saucelabs:sauce_service_setup
yarn bazel test //:saucelabs_unit_tests_poc_suite --config=saucelabs
TESTS=$(./node_modules/.bin/bazel query --output label '(kind(karma_web_test, ...) intersect attr("tags", "saucelabs", ...)) except attr("tags", "ivy-only", ...) except attr("tags", "fixme-saucelabs", ...)')
yarn bazel test --config=saucelabs ${TESTS}
yarn bazel run //tools/saucelabs:sauce_service_stop
no_output_timeout: 20m
no_output_timeout: 40m
- notify_webhook_on_fail:
webhook_url_env_var: SLACK_DEV_INFRA_CI_FAILURES_WEBHOOK_URL
# NOTE: This is currently limited to master builds only. See the `default_workflow` configuration.
saucelabs_ivy:
executor:
name: default-executor
@ -382,13 +380,16 @@ jobs:
- init_environment
- init_saucelabs_environment
- run:
name: Run Bazel tests on Saucelabs
name: Run Bazel tests on Saucelabs with Ivy
# See /tools/saucelabs/README.md for more info
command: |
yarn bazel run //tools/saucelabs:sauce_service_setup
yarn bazel test //:saucelabs_unit_tests --config=saucelabs --config=ivy
TESTS=$(./node_modules/.bin/bazel query --output label '(kind(karma_web_test, ...) intersect attr("tags", "saucelabs", ...)) except attr("tags", "no-ivy-aot", ...) except attr("tags", "fixme-saucelabs", ...)')
yarn bazel test --config=saucelabs --config=ivy ${TESTS}
yarn bazel run //tools/saucelabs:sauce_service_stop
no_output_timeout: 20m
no_output_timeout: 40m
- notify_webhook_on_fail:
webhook_url_env_var: SLACK_DEV_INFRA_CI_FAILURES_WEBHOOK_URL
test_aio:
executor: default-executor
@ -840,13 +841,22 @@ workflows:
requires:
- setup
- saucelabs_ivy:
# Testing saucelabs via Bazel currently taking longer than the legacy saucelabs job as it
# each karma_web_test target is provisioning and tearing down browsers which is adding
# a lot of overhead. Running on master only to avoid wasting resources and slowing down
# CI for PRs.
# TODO: Run this job on all branches (including PRs) once karma_web_test targets can
# share provisioned browsers and we can remove the legacy saucelabs job.
<<: *only_on_master
requires:
- test_ivy_aot
- setup
- saucelabs_view_engine:
# This job is currently a PoC and a subset of `legacy-unit-tests-saucelabs`. Running on
# master only to avoid wasting resources.
# TODO: Run this job on all branches (including PRs) as soon as it is not a PoC and
# we can remove the legacy saucelabs job.
# Testing saucelabs via Bazel currently taking longer than the legacy saucelabs job as it
# each karma_web_test target is provisioning and tearing down browsers which is adding
# a lot of overhead. Running on master only to avoid wasting resources and slowing down
# CI for PRs.
# TODO: Run this job on all branches (including PRs) once karma_web_test targets can
# share provisioned browsers and we can remove the legacy saucelabs job.
<<: *only_on_master
requires:
- setup
@ -899,6 +909,7 @@ workflows:
- test
- test_ivy_aot
- integration_test
- saucelabs_view_engine
- saucelabs_ivy
# Only publish if `aio`/`docs` tests using the locally built Angular packages pass
- test_aio_local

View File

@ -1,7 +1,5 @@
package(default_visibility = ["//visibility:public"])
load("//tools:defaults.bzl", "karma_web_test")
exports_files([
"LICENSE",
"protractor-perf.conf.js",
@ -46,76 +44,3 @@ filegroup(
"@npm//:node_modules/angular-mocks-1.6/angular-mocks.js",
],
)
# To run manually:
# Setup your SAUCE_USERNAME, SAUCE_ACCESS_KEY & SAUCE_TUNNEL_IDENTIFIER.
# If on OSX, also set SAUCE_CONNECT to the path of your `sc` binary.
# environment variables and run:
# ```
# yarn bazel run //tools/saucelabs:sauce_service_setup
# yarn bazel test //:saucelabs_unit_tests --config=saucelabs --config=ivy
# ```
# See /tools/saucelabs/README.md for more info on karma Saucelabs tests under Bazel.
karma_web_test(
name = "saucelabs_unit_tests",
# Default timeout is moderate (5min). This causes the test to be terminated while
# Saucelabs browsers keep running. Ultimately resulting in failing tests and browsers
# unnecessarily being acquired. Our specified Saucelabs idle timeout is 10min, so we use
# Bazel's long timeout (15min). This ensures that Karma can shut down properly.
timeout = "long",
karma = "//tools/saucelabs:karma-saucelabs",
tags = [
"manual",
"no-remote-exec",
"saucelabs",
],
deps = [
"//packages/core/test/acceptance:acceptance_lib",
],
)
SAUCE_TEST_SUITE_TARGETS = [
"packages/common/http/test:test_lib",
"packages/common/http/testing/test:test_lib",
"packages/common/test:test_lib",
"packages/core/test:test_lib",
"packages/forms/test:test_lib",
"packages/http/test:test_lib",
]
[
# These target runs in CI with View Engine as a Saucelabs and Bazel proof-of-concept. It's a
# subset of the legacy saucelabs tests.
karma_web_test(
name = "saucelabs_unit_tests_poc_%s" % test.replace("/", "_").replace(":", "_").replace(".", "_"),
# Default timeout is moderate (5min). This causes the test to be terminated while
# Saucelabs browsers keep running. Ultimately resulting in failing tests and browsers
# unnecessarily being acquired. Our specified Saucelabs idle timeout is 10min, so we use
# Bazel's long timeout (15min). This ensures that Karma can shut down properly.
timeout = "long",
karma = "//tools/saucelabs:karma-saucelabs",
tags = [
"exclusive",
"manual",
"no-remote-exec",
"saucelabs",
],
deps = ["//%s" % test],
)
for test in SAUCE_TEST_SUITE_TARGETS
]
# To run manually:
# Setup your SAUCE_USERNAME, SAUCE_ACCESS_KEY & SAUCE_TUNNEL_IDENTIFIER.
# If on OSX, also set SAUCE_CONNECT to the path of your `sc` binary.
# environment variables and run:
# ```
# yarn bazel run //tools/saucelabs:sauce_service_setup
# yarn bazel test //:saucelabs_unit_tests_poc_suite --config=saucelabs
# ```
# See /tools/saucelabs/README.md for more info on karma Saucelabs tests under Bazel.
test_suite(
name = "saucelabs_unit_tests_poc_suite",
tags = ["manual"],
tests = ["//:saucelabs_unit_tests_poc_%s" % test.replace("/", "_").replace(":", "_").replace(".", "_") for test in SAUCE_TEST_SUITE_TARGETS],
)

View File

@ -214,54 +214,69 @@ def pkg_npm(name, substitutions = {}, **kwargs):
visibility = visibility,
)
def karma_web_test(bootstrap = [], deps = [], data = [], runtime_deps = [], **kwargs):
"""Default values for karma_web_test"""
if not bootstrap:
bootstrap = ["//:web_test_bootstrap_scripts"]
local_deps = [
def karma_web_test_suite(name, **kwargs):
"""Default values for karma_web_test_suite"""
# Default value for bootstrap
bootstrap = kwargs.pop("bootstrap", [
"//:web_test_bootstrap_scripts",
])
# Add common deps
deps = kwargs.pop("deps", []) + [
"@npm//karma-browserstack-launcher",
"@npm//karma-sauce-launcher",
"@npm//:node_modules/tslib/tslib.js",
"//tools/rxjs:rxjs_umd_modules",
"//packages/zone.js:npm_package",
] + deps
local_runtime_deps = [
"//tools/testing:browser",
] + runtime_deps
]
# Add common runtime deps
runtime_deps = kwargs.pop("runtime_deps", []) + [
"//tools/testing:browser",
]
data = kwargs.pop("data", [])
tags = kwargs.pop("tags", [])
_karma_web_test_suite(
name = name,
runtime_deps = runtime_deps,
bootstrap = bootstrap,
deps = deps,
browsers = ["//tools/browsers:chromium"],
data = data,
tags = tags,
**kwargs
)
# Add a saucelabs target for these karma tests
_karma_web_test(
runtime_deps = local_runtime_deps,
name = "saucelabs_%s" % name,
# Default timeout is moderate (5min). This causes the test to be terminated while
# Saucelabs browsers keep running. Ultimately resulting in failing tests and browsers
# unnecessarily being acquired. Our specified Saucelabs idle timeout is 10min, so we use
# Bazel's long timeout (15min). This ensures that Karma can shut down properly.
timeout = "long",
runtime_deps = runtime_deps,
bootstrap = bootstrap,
config_file = "//:karma-js.conf.js",
deps = local_deps,
deps = deps,
data = data + [
"//:browser-providers.conf.js",
"//tools:jasmine-seed-generator.js",
],
karma = "//tools/saucelabs:karma-saucelabs",
tags = tags + [
"exclusive",
"manual",
"no-remote-exec",
"saucelabs",
],
configuration_env_vars = ["KARMA_WEB_TEST_MODE"],
**kwargs
)
def karma_web_test_suite(bootstrap = [], deps = [], runtime_deps = [], **kwargs):
"""Default values for karma_web_test_suite"""
if not bootstrap:
bootstrap = ["//:web_test_bootstrap_scripts"]
local_deps = [
"@npm//:node_modules/tslib/tslib.js",
"//tools/rxjs:rxjs_umd_modules",
] + deps
local_runtime_deps = [
"//tools/testing:browser",
] + runtime_deps
_karma_web_test_suite(
runtime_deps = local_runtime_deps,
bootstrap = bootstrap,
deps = local_deps,
browsers = ["//tools/browsers:chromium"],
**kwargs
)
def protractor_web_test_suite(**kwargs):
"""Default values for protractor_web_test_suite"""