From 75357ecb32275a98b6adba3cad5e0f6af87d7810 Mon Sep 17 00:00:00 2001 From: Greg Magolan Date: Mon, 17 Dec 2018 22:09:39 -0800 Subject: [PATCH] build(bazel): run a number of web tests with karma_web_test in saucelabs in CircleCI (#27721) PR Close #27721 --- .bazelrc | 16 ++++ .circleci/config.yml | 36 +++++++++ BUILD.bazel | 47 +++++++++++- karma-js.conf.js | 53 ++++++++++--- package.json | 2 +- packages/common/http/test/BUILD.bazel | 2 + packages/common/http/testing/test/BUILD.bazel | 2 + packages/common/test/BUILD.bazel | 2 + packages/core/test/BUILD.bazel | 2 + packages/forms/test/BUILD.bazel | 2 + packages/http/test/BUILD.bazel | 3 +- packages/router/test/BUILD.bazel | 2 + scripts/saucelabs/start-tunnel.sh | 7 +- tools/BUILD.bazel | 5 +- tools/defaults.bzl | 76 ++++++++++++++++++- 15 files changed, 237 insertions(+), 20 deletions(-) diff --git a/.bazelrc b/.bazelrc index 59fd7b8332..5883d9e898 100644 --- a/.bazelrc +++ b/.bazelrc @@ -52,6 +52,22 @@ build --incompatible_strict_action_env run --incompatible_strict_action_env test --incompatible_strict_action_env +############################### +# Saucelabs support # +# Turn on these settings with # +# --config=saucelabs # +############################### + +# Expose SauceLabs environment to actions +# These environment variables are needed by +# web_test_karma to run on Saucelabs +test:saucelabs --action_env=SAUCE_USERNAME +test:saucelabs --action_env=SAUCE_ACCESS_KEY +test:saucelabs --action_env=SAUCE_READY_FILE +test:saucelabs --action_env=SAUCE_PID_FILE +test:saucelabs --action_env=SAUCE_TUNNEL_IDENTIFIER +test:saucelabs --define=KARMA_WEB_TEST_MODE=SL_REQUIRED + ############################### # Release support # # Turn on these settings with # diff --git a/.circleci/config.yml b/.circleci/config.yml index 2a0d7d2bbd..1a38d28c4c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -182,6 +182,37 @@ jobs: path: dist/bin/packages/core/test/bundling/todo/bundle.min.js.br destination: core/todo/bundle.br + test_saucelabs: + <<: *job_defaults + # In order to avoid the bottleneck of having a slow host machine, we acquire a better + # container for this job. This is necessary because we launch a lot of browsers concurrently + # and therefore the tunnel and Karma need to process a lot of file requests and tests. + resource_class: xlarge + steps: + - checkout: + <<: *post_checkout + - *restore_cache + - *init_environment + - *yarn_install + - run: + name: Preparing environment for running tests on Saucelabs. + command: setSecretVar SAUCE_ACCESS_KEY $(echo $SAUCE_ACCESS_KEY | rev) + - run: + name: Starting Saucelabs tunnel + command: ./scripts/saucelabs/start-tunnel.sh + background: true + # Waits for the Saucelabs tunnel to be ready. This ensures that we don't run tests + # too early without Saucelabs not being ready. + - run: ./scripts/saucelabs/wait-for-tunnel.sh + # All web tests are contained within a single //:test_web_all target for Saucelabs + # as running each set of tests as a separate target will attempt to acquire too + # many browsers on Saucelabs (7 per target currently) and some tests will always + # fail to acquire browsers. For example: + # 14 02 2019 19:52:33.170:WARN [launcher]: chrome beta on SauceLabs have not captured in 180000 ms, killing. + # //packages/forms/test:web_test_sauce TIMEOUT in 315.0s + - run: yarn bazel test --config=saucelabs //:test_web_all + - run: ./scripts/saucelabs/stop-tunnel.sh + test_aio: <<: *job_defaults docker: @@ -612,7 +643,12 @@ workflows: saucelabs_tests: jobs: + - test_saucelabs - legacy-unit-tests-saucelabs + # Don't open up multiple saucelabs tunnels at the same + # time to minimize flakes + requires: + - test_saucelabs triggers: - schedule: # Runs the Saucelabs legacy tests every hour. We still want to run Saucelabs diff --git a/BUILD.bazel b/BUILD.bazel index 049d72fe26..49f61c4b18 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -1,22 +1,29 @@ package(default_visibility = ["//visibility:public"]) load("@build_bazel_rules_nodejs//:defs.bzl", "node_modules_filegroup") +load("//tools:defaults.bzl", "karma_web_test") exports_files([ "tsconfig.json", "LICENSE", "protractor-perf.conf.js", + "karma-js.conf.js", + "browser-providers.conf.js", ]) filegroup( name = "web_test_bootstrap_scripts", # do not sort srcs = [ - "@ngdeps//node_modules/reflect-metadata:Reflect.js", + "@ngdeps//node_modules/core-js:client/core.js", "@ngdeps//node_modules/zone.js:dist/zone.js", "@ngdeps//node_modules/zone.js:dist/zone-testing.js", "@ngdeps//node_modules/zone.js:dist/task-tracking.js", "//:test-events.js", + "//:shims_for_IE.js", + # Including systemjs because it defines `__eval`, which produces correct stack traces. + "@ngdeps//node_modules/systemjs:dist/system.src.js", + "@ngdeps//node_modules/reflect-metadata:Reflect.js", ], ) @@ -54,3 +61,41 @@ nodejs_binary( install_source_map_support = False, templated_args = ["--node_options=--expose-gc"], ) + +# To run a karma_web_test target locally on SauceLabs: +# 1) have SAUCE_USERNAME, SAUCE_ACCESS_KEY (and optionally a SAUCE_TUNNEL_IDENTIFIER) set in your environment +# 2) open a sauce connection with `./scripts/saucelabs/start-tunnel.sh` +# NOTE: start-tunnel.sh uses `node_modules/sauce-connect` which is current linux specific: +# "sauce-connect": "https://saucelabs.com/downloads/sc-4.5.3-linux.tar.gz". +# On OSX or Windows you'll need to use the appropriate sauce-connect binary. +# 3) run target with `yarn bazel test --config=saucelabs ` +karma_web_test( + name = "test_web_all", + tags = [ + "local", + "manual", + "saucelabs", + ], + deps = [ + "//packages/common/http/test:test_lib", + "//packages/common/http/testing/test:test_lib", + "//packages/common/test:test_lib", + "//packages/core/test:test_lib", + "//packages/forms/test:test_lib", + "//packages/http/test:test_lib", + # "//packages/router/test:test_lib", + # //packages/router/test:test_lib fails with: + # IE 11.0.0 (Windows 8.1.0.0) bootstrap should restore the scrolling position FAILED + # Expected undefined to equal 5000. + # at stack (eval code:2338:11) + # at buildExpectationResult (eval code:2305:5) + # at expectationResultFactory (eval code:858:11) + # at Spec.prototype.addExpectationResult (eval code:487:5) + # at addExpectationResult (eval code:802:9) + # at Anonymous function (eval code:2252:7) + # at Anonymous function (eval code:339:25) + # at step (eval code:133:17) + # at Anonymous function (eval code:114:50) + # at fulfilled (eval code:104:47) + ], +) diff --git a/karma-js.conf.js b/karma-js.conf.js index e1575ed0ad..f73adf9df0 100644 --- a/karma-js.conf.js +++ b/karma-js.conf.js @@ -12,8 +12,7 @@ const {generateSeed} = require('./tools/jasmine-seed-generator'); // Karma configuration // Generated on Thu Sep 25 2014 11:52:02 GMT-0700 (PDT) module.exports = function(config) { - config.set({ - + const conf = { frameworks: ['jasmine'], client: { @@ -116,6 +115,7 @@ module.exports = function(config) { }, reporters: ['dots'], + sauceLabs: { testName: 'Angular2', retryLimit: 3, @@ -135,28 +135,57 @@ module.exports = function(config) { pollingTimeout: 10000, }, - browsers: ['Chrome'], + // Try "websocket" for a faster transmission first. Fallback to "polling" if necessary. + transports: ['websocket', 'polling'], port: 9876, captureTimeout: 180000, browserDisconnectTimeout: 180000, browserDisconnectTolerance: 3, browserNoActivityTimeout: 300000, - }); + } + + if (process.env['SAUCE_TUNNEL_IDENTIFIER']) { + console.log(`SAUCE_TUNNEL_IDENTIFIER: ${process.env.SAUCE_TUNNEL_IDENTIFIER}`); - if (process.env.CIRCLECI) { const tunnelIdentifier = process.env['SAUCE_TUNNEL_IDENTIFIER']; // Setup the Saucelabs plugin so that it can launch browsers using the proper tunnel. - config.sauceLabs.build = tunnelIdentifier; - config.sauceLabs.tunnelIdentifier = tunnelIdentifier; + conf.sauceLabs.build = tunnelIdentifier; + conf.sauceLabs.tunnelIdentifier = tunnelIdentifier; // Setup the Browserstack plugin so that it can launch browsers using the proper tunnel. // TODO: This is currently not used because BS doesn't run on the CI. Consider removing. - config.browserStack.build = tunnelIdentifier; - config.browserStack.tunnelIdentifier = tunnelIdentifier; - - // Try "websocket" for a faster transmission first. Fallback to "polling" if necessary. - config.transports = ['websocket', 'polling']; + conf.browserStack.build = tunnelIdentifier; + conf.browserStack.tunnelIdentifier = tunnelIdentifier; } + + if (process.env.KARMA_WEB_TEST_MODE) { + // KARMA_WEB_TEST_MODE is used to setup karma to run in + // SauceLabs or Browserstack + console.log(`KARMA_WEB_TEST_MODE: ${process.env.KARMA_WEB_TEST_MODE}`); + + switch (process.env.KARMA_WEB_TEST_MODE) { + case 'SL_REQUIRED': + conf.browsers = browserProvidersConf.sauceAliases.CI_REQUIRED; + break; + case 'SL_OPTIONAL': + conf.browsers = browserProvidersConf.sauceAliases.CI_OPTIONAL; + break; + case 'BS_REQUIRED': + conf.browsers = browserProvidersConf.browserstackAliases.CI_REQUIRED; + break; + case 'BS_OPTIONAL': + conf.browsers = browserProvidersConf.browserstackAliases.CI_OPTIONAL; + break; + default: + throw new Error( + `Unrecognized process.env.KARMA_WEB_TEST_MODE: ${process.env.KARMA_WEB_TEST_MODE}`); + } + } else { + // Run the test locally + conf.browsers = [process.env['DISPLAY'] ? 'Chrome' : 'ChromeHeadless']; + } + + config.set(conf); }; diff --git a/package.json b/package.json index 9ab06b409e..5f3daba35e 100644 --- a/package.json +++ b/package.json @@ -75,6 +75,7 @@ "jasmine-core": "^3.1.0", "jquery": "3.0.0", "karma": "^3.1.4", + "karma-browserstack-launcher": "^1.3.0", "magic-string": "^0.25.0", "materialize-css": "1.0.0", "minimist": "1.2.0", @@ -131,7 +132,6 @@ "gulp-tslint": "8.1.2", "husky": "^0.14.3", "jpm": "1.3.1", - "karma-browserstack-launcher": "^1.3.0", "karma-chrome-launcher": "^2.2.0", "karma-jasmine": "^1.1.2", "karma-sauce-launcher": "^2.0.2", diff --git a/packages/common/http/test/BUILD.bazel b/packages/common/http/test/BUILD.bazel index e3c6f91998..122b3b78d5 100644 --- a/packages/common/http/test/BUILD.bazel +++ b/packages/common/http/test/BUILD.bazel @@ -6,6 +6,8 @@ ts_library( srcs = glob( ["**/*.ts"], ), + # Visible to //:test_web_all target + visibility = ["//:__pkg__"], deps = [ "//packages/common/http", "//packages/common/http/testing", diff --git a/packages/common/http/testing/test/BUILD.bazel b/packages/common/http/testing/test/BUILD.bazel index 95d1c3eeb9..c8c150c274 100644 --- a/packages/common/http/testing/test/BUILD.bazel +++ b/packages/common/http/testing/test/BUILD.bazel @@ -6,6 +6,8 @@ ts_library( srcs = glob( ["**/*.ts"], ), + # Visible to //:test_web_all target + visibility = ["//:__pkg__"], deps = [ "//packages/common/http", "//packages/common/http/testing", diff --git a/packages/common/test/BUILD.bazel b/packages/common/test/BUILD.bazel index 3a7c7cf38f..a4c83de627 100644 --- a/packages/common/test/BUILD.bazel +++ b/packages/common/test/BUILD.bazel @@ -6,6 +6,8 @@ ts_library( srcs = glob( ["**/*.ts"], ), + # Visible to //:test_web_all target + visibility = ["//:__pkg__"], deps = [ "//packages/common", "//packages/common/locales", diff --git a/packages/core/test/BUILD.bazel b/packages/core/test/BUILD.bazel index e5d2f4e067..face8ab22b 100644 --- a/packages/core/test/BUILD.bazel +++ b/packages/core/test/BUILD.bazel @@ -11,6 +11,8 @@ ts_library( "**/*_node_only_spec.ts", ], ), + # Visible to //:test_web_all target + visibility = ["//:__pkg__"], deps = [ "//packages/animations", "//packages/animations/browser", diff --git a/packages/forms/test/BUILD.bazel b/packages/forms/test/BUILD.bazel index 7a0b8b5d57..3a5d16fdba 100644 --- a/packages/forms/test/BUILD.bazel +++ b/packages/forms/test/BUILD.bazel @@ -4,6 +4,8 @@ ts_library( name = "test_lib", testonly = True, srcs = glob(["**/*.ts"]), + # Visible to //:test_web_all target + visibility = ["//:__pkg__"], deps = [ "//packages/core", "//packages/core/testing", diff --git a/packages/http/test/BUILD.bazel b/packages/http/test/BUILD.bazel index 5581b37e00..df2af84980 100644 --- a/packages/http/test/BUILD.bazel +++ b/packages/http/test/BUILD.bazel @@ -4,6 +4,8 @@ ts_library( name = "test_lib", testonly = True, srcs = glob(["**/*.ts"]), + # Visible to //:test_web_all target + visibility = ["//:__pkg__"], deps = [ "//packages/core", "//packages/core/testing", @@ -28,6 +30,5 @@ ts_web_test_suite( name = "test_web", deps = [ ":test_lib", - "@ngdeps//karma", ], ) diff --git a/packages/router/test/BUILD.bazel b/packages/router/test/BUILD.bazel index 55496311d1..49fcf3253d 100644 --- a/packages/router/test/BUILD.bazel +++ b/packages/router/test/BUILD.bazel @@ -4,6 +4,8 @@ ts_library( name = "test_lib", testonly = True, srcs = glob(["**/*.ts"]), + # Visible to //:test_web_all target + visibility = ["//:__pkg__"], deps = [ "//packages/common", "//packages/common/testing", diff --git a/scripts/saucelabs/start-tunnel.sh b/scripts/saucelabs/start-tunnel.sh index ea9b2a77dc..6ccf68a32e 100755 --- a/scripts/saucelabs/start-tunnel.sh +++ b/scripts/saucelabs/start-tunnel.sh @@ -7,16 +7,17 @@ readonly currentDir=$(cd $(dirname $0); pwd) # Command arguments that will be passed to sauce-connect. sauceArgs="" -if [[ ! -z "${SAUCE_READY_FILE}" ]]; then +if [[ ! -z "${SAUCE_READY_FILE:-}" ]]; then + mkdir -p $(dirname ${SAUCE_READY_FILE}) sauceArgs="${sauceArgs} --readyfile ${SAUCE_READY_FILE}" fi -if [[ ! -z "${SAUCE_PID_FILE}" ]]; then +if [[ ! -z "${SAUCE_PID_FILE:-}" ]]; then mkdir -p $(dirname ${SAUCE_PID_FILE}) sauceArgs="${sauceArgs} --pidfile ${SAUCE_PID_FILE}" fi -if [[ ! -z "${SAUCE_TUNNEL_IDENTIFIER}" ]]; then +if [[ ! -z "${SAUCE_TUNNEL_IDENTIFIER:-}" ]]; then sauceArgs="${sauceArgs} --tunnel-identifier ${SAUCE_TUNNEL_IDENTIFIER}" fi diff --git a/tools/BUILD.bazel b/tools/BUILD.bazel index 905df3c90b..83e4953fa9 100644 --- a/tools/BUILD.bazel +++ b/tools/BUILD.bazel @@ -2,7 +2,10 @@ package(default_visibility = ["//visibility:public"]) load("@npm_bazel_typescript//:defs.bzl", "ts_config") -exports_files(["tsconfig.json"]) +exports_files([ + "tsconfig.json", + "jasmine-seed-generator.js", +]) ts_config( name = "tsconfig-test", diff --git a/tools/defaults.bzl b/tools/defaults.bzl index 6dfbef7eef..2993efd87f 100644 --- a/tools/defaults.bzl +++ b/tools/defaults.bzl @@ -1,7 +1,7 @@ """Re-export of some bazel rules with repository-wide defaults.""" -load("@npm_bazel_karma//:defs.bzl", _ts_web_test_suite = "ts_web_test_suite") load("@build_bazel_rules_nodejs//:defs.bzl", _jasmine_node_test = "jasmine_node_test", _nodejs_binary = "nodejs_binary", _npm_package = "npm_package") +load("@npm_bazel_karma//:defs.bzl", _karma_web_test = "karma_web_test", _karma_web_test_suite = "karma_web_test_suite", _ts_web_test = "ts_web_test", _ts_web_test_suite = "ts_web_test_suite") load("@npm_bazel_typescript//:defs.bzl", _ts_library = "ts_library") load("//packages/bazel:index.bzl", _ng_module = "ng_module", _ng_package = "ng_package") load("//packages/bazel/src:ng_rollup_bundle.bzl", _ng_rollup_bundle = "ng_rollup_bundle") @@ -159,6 +159,26 @@ def npm_package(name, replacements = {}, **kwargs): **kwargs ) +def ts_web_test(bootstrap = [], deps = [], runtime_deps = [], **kwargs): + """Default values for ts_web_test""" + if not bootstrap: + bootstrap = ["//:web_test_bootstrap_scripts"] + local_deps = [ + "@ngdeps//node_modules/tslib:tslib.js", + "//tools/rxjs:rxjs_umd_modules", + ] + deps + local_runtime_deps = [ + "//tools/testing:browser", + ] + runtime_deps + + _ts_web_test( + runtime_deps = local_runtime_deps, + bootstrap = bootstrap, + deps = local_deps, + karma = _DEFAULT_KARMA_BIN, + **kwargs + ) + def ts_web_test_suite(bootstrap = [], deps = [], runtime_deps = [], **kwargs): """Default values for ts_web_test_suite""" if not bootstrap: @@ -190,6 +210,60 @@ def ts_web_test_suite(bootstrap = [], deps = [], runtime_deps = [], **kwargs): **kwargs ) +def karma_web_test(bootstrap = [], deps = [], data = [], runtime_deps = [], **kwargs): + """Default values for karma_web_test""" + if not bootstrap: + bootstrap = ["//:web_test_bootstrap_scripts"] + local_deps = [ + "@ngdeps//karma-browserstack-launcher", + "@ngdeps//node_modules/tslib:tslib.js", + "//tools/rxjs:rxjs_umd_modules", + ] + deps + local_runtime_deps = [ + "//tools/testing:browser", + ] + runtime_deps + + _karma_web_test( + runtime_deps = local_runtime_deps, + bootstrap = bootstrap, + config_file = "//:karma-js.conf.js", + deps = local_deps, + karma = _DEFAULT_KARMA_BIN, + data = data + [ + "//:browser-providers.conf.js", + "//tools:jasmine-seed-generator.js", + ], + configuration_env_vars = ["KARMA_WEB_TEST_MODE"], + **kwargs + ) + +def karma_web_test_suite(bootstrap = [], deps = [], **kwargs): + """Default values for karma_web_test_suite""" + if not bootstrap: + bootstrap = ["//:web_test_bootstrap_scripts"] + local_deps = [ + "@ngdeps//node_modules/tslib:tslib.js", + "//tools/rxjs:rxjs_umd_modules", + ] + deps + + _karma_web_test_suite( + bootstrap = bootstrap, + deps = local_deps, + karma = _DEFAULT_KARMA_BIN, + # Run unit tests on local Chromium by default. + # You can exclude tests based on tags, e.g. to skip Firefox testing, + # `yarn bazel test --test_tag_filters=-browser:firefox-local [targets]` + browsers = [ + "@io_bazel_rules_webtesting//browsers:chromium-local", + # Don't test on local Firefox by default, for faster builds. + # We think that bugs in Angular tend to be caught the same in any + # evergreen browser. + # "@io_bazel_rules_webtesting//browsers:firefox-local", + # TODO(alexeagle): add remote browsers on SauceLabs + ], + **kwargs + ) + def nodejs_binary(data = [], **kwargs): """Default values for nodejs_binary""" _nodejs_binary(