build(bazel): run a number of web tests with karma_web_test in saucelabs in CircleCI (#27721)

PR Close #27721

.bazelrc

@@ -52,6 +52,22 @@ build --incompatible_strict_action_env
 run --incompatible_strict_action_env
 test --incompatible_strict_action_env
 
+###############################
+# Saucelabs support           #
+# Turn on these settings with #
+#  --config=saucelabs         #
+###############################
+
+# Expose SauceLabs environment to actions
+# These environment variables are needed by
+# web_test_karma to run on Saucelabs
+test:saucelabs --action_env=SAUCE_USERNAME
+test:saucelabs --action_env=SAUCE_ACCESS_KEY
+test:saucelabs --action_env=SAUCE_READY_FILE
+test:saucelabs --action_env=SAUCE_PID_FILE
+test:saucelabs --action_env=SAUCE_TUNNEL_IDENTIFIER
+test:saucelabs --define=KARMA_WEB_TEST_MODE=SL_REQUIRED
+
 ###############################
 # Release support             #
 # Turn on these settings with #

.circleci/config.yml

@@ -182,6 +182,37 @@ jobs:
           path: dist/bin/packages/core/test/bundling/todo/bundle.min.js.br
           destination: core/todo/bundle.br
 
+  test_saucelabs:
+    <<: *job_defaults
+    # In order to avoid the bottleneck of having a slow host machine, we acquire a better
+    # container for this job. This is necessary because we launch a lot of browsers concurrently
+    # and therefore the tunnel and Karma need to process a lot of file requests and tests.
+    resource_class: xlarge
+    steps:
+      - checkout:
+          <<: *post_checkout
+      - *restore_cache
+      - *init_environment
+      - *yarn_install
+      - run:
+          name: Preparing environment for running tests on Saucelabs.
+          command: setSecretVar SAUCE_ACCESS_KEY $(echo $SAUCE_ACCESS_KEY | rev)
+      - run:
+          name: Starting Saucelabs tunnel
+          command: ./scripts/saucelabs/start-tunnel.sh
+          background: true
+        # Waits for the Saucelabs tunnel to be ready. This ensures that we don't run tests
+        # too early without Saucelabs not being ready.
+      - run: ./scripts/saucelabs/wait-for-tunnel.sh
+        # All web tests are contained within a single //:test_web_all target for Saucelabs
+        # as running each set of tests as a separate target will attempt to acquire too
+        # many browsers on Saucelabs (7 per target currently) and some tests will always
+        # fail to acquire browsers. For example:
+        # 14 02 2019 19:52:33.170:WARN [launcher]: chrome beta on SauceLabs have not captured in 180000 ms, killing.
+        # //packages/forms/test:web_test_sauce TIMEOUT in 315.0s
+      - run: yarn bazel test --config=saucelabs //:test_web_all
+      - run: ./scripts/saucelabs/stop-tunnel.sh
+
   test_aio:
     <<: *job_defaults
     docker:
@@ -612,7 +643,12 @@ workflows:
 
   saucelabs_tests:
     jobs:
+      - test_saucelabs
       - legacy-unit-tests-saucelabs
+          # Don't open up multiple saucelabs tunnels at the same
+          # time to minimize flakes
+          requires:
+            - test_saucelabs
     triggers:
       - schedule:
           # Runs the Saucelabs legacy tests every hour. We still want to run Saucelabs

BUILD.bazel

@@ -1,22 +1,29 @@
 package(default_visibility = ["//visibility:public"])
 
 load("@build_bazel_rules_nodejs//:defs.bzl", "node_modules_filegroup")
+load("//tools:defaults.bzl", "karma_web_test")
 
 exports_files([
     "tsconfig.json",
     "LICENSE",
     "protractor-perf.conf.js",
+    "karma-js.conf.js",
+    "browser-providers.conf.js",
 ])
 
 filegroup(
     name = "web_test_bootstrap_scripts",
     # do not sort
     srcs = [
-        "@ngdeps//node_modules/reflect-metadata:Reflect.js",
+        "@ngdeps//node_modules/core-js:client/core.js",
         "@ngdeps//node_modules/zone.js:dist/zone.js",
         "@ngdeps//node_modules/zone.js:dist/zone-testing.js",
         "@ngdeps//node_modules/zone.js:dist/task-tracking.js",
         "//:test-events.js",
+        "//:shims_for_IE.js",
+        # Including systemjs because it defines `__eval`, which produces correct stack traces.
+        "@ngdeps//node_modules/systemjs:dist/system.src.js",
+        "@ngdeps//node_modules/reflect-metadata:Reflect.js",
     ],
 )
 
@@ -54,3 +61,41 @@ nodejs_binary(
     install_source_map_support = False,
     templated_args = ["--node_options=--expose-gc"],
 )
+
+# To run a karma_web_test target locally on SauceLabs:
+# 1) have SAUCE_USERNAME, SAUCE_ACCESS_KEY (and optionally a SAUCE_TUNNEL_IDENTIFIER) set in your environment
+# 2) open a sauce connection with `./scripts/saucelabs/start-tunnel.sh`
+#    NOTE: start-tunnel.sh uses `node_modules/sauce-connect` which is current linux specific:
+#          "sauce-connect": "https://saucelabs.com/downloads/sc-4.5.3-linux.tar.gz".
+#          On OSX or Windows you'll need to use the appropriate sauce-connect binary.
+# 3) run target with `yarn bazel test --config=saucelabs <target>`
+karma_web_test(
+    name = "test_web_all",
+    tags = [
+        "local",
+        "manual",
+        "saucelabs",
+    ],
+    deps = [
+        "//packages/common/http/test:test_lib",
+        "//packages/common/http/testing/test:test_lib",
+        "//packages/common/test:test_lib",
+        "//packages/core/test:test_lib",
+        "//packages/forms/test:test_lib",
+        "//packages/http/test:test_lib",
+        # "//packages/router/test:test_lib",
+        # //packages/router/test:test_lib fails with:
+        # IE 11.0.0 (Windows 8.1.0.0) bootstrap should restore the scrolling position FAILED
+        # 	Expected undefined to equal 5000.
+        # 	   at stack (eval code:2338:11)
+        # 	   at buildExpectationResult (eval code:2305:5)
+        # 	   at expectationResultFactory (eval code:858:11)
+        # 	   at Spec.prototype.addExpectationResult (eval code:487:5)
+        # 	   at addExpectationResult (eval code:802:9)
+        # 	   at Anonymous function (eval code:2252:7)
+        # 	   at Anonymous function (eval code:339:25)
+        # 	   at step (eval code:133:17)
+        # 	   at Anonymous function (eval code:114:50)
+        # 	   at fulfilled (eval code:104:47)
+    ],
+)

karma-js.conf.js

@@ -12,8 +12,7 @@ const {generateSeed} = require('./tools/jasmine-seed-generator');
 // Karma configuration
 // Generated on Thu Sep 25 2014 11:52:02 GMT-0700 (PDT)
 module.exports = function(config) {
-  config.set({
+  const conf = {
-
     frameworks: ['jasmine'],
 
     client: {
@@ -116,6 +115,7 @@ module.exports = function(config) {
     },
 
     reporters: ['dots'],
+
     sauceLabs: {
       testName: 'Angular2',
       retryLimit: 3,
@@ -135,28 +135,57 @@ module.exports = function(config) {
       pollingTimeout: 10000,
     },
 
-    browsers: ['Chrome'],
+    // Try "websocket" for a faster transmission first. Fallback to "polling" if necessary.
+    transports: ['websocket', 'polling'],
 
     port: 9876,
     captureTimeout: 180000,
     browserDisconnectTimeout: 180000,
     browserDisconnectTolerance: 3,
     browserNoActivityTimeout: 300000,
-  });
+  }
+
+  if (process.env['SAUCE_TUNNEL_IDENTIFIER']) {
+    console.log(`SAUCE_TUNNEL_IDENTIFIER: ${process.env.SAUCE_TUNNEL_IDENTIFIER}`);
 
-  if (process.env.CIRCLECI) {
     const tunnelIdentifier = process.env['SAUCE_TUNNEL_IDENTIFIER'];
 
     // Setup the Saucelabs plugin so that it can launch browsers using the proper tunnel.
-    config.sauceLabs.build = tunnelIdentifier;
+    conf.sauceLabs.build = tunnelIdentifier;
-    config.sauceLabs.tunnelIdentifier = tunnelIdentifier;
+    conf.sauceLabs.tunnelIdentifier = tunnelIdentifier;
 
     // Setup the Browserstack plugin so that it can launch browsers using the proper tunnel.
     // TODO: This is currently not used because BS doesn't run on the CI. Consider removing.
-    config.browserStack.build = tunnelIdentifier;
+    conf.browserStack.build = tunnelIdentifier;
-    config.browserStack.tunnelIdentifier = tunnelIdentifier;
+    conf.browserStack.tunnelIdentifier = tunnelIdentifier;
-
-    // Try "websocket" for a faster transmission first. Fallback to "polling" if necessary.
-    config.transports = ['websocket', 'polling'];
   }
+
+  if (process.env.KARMA_WEB_TEST_MODE) {
+    // KARMA_WEB_TEST_MODE is used to setup karma to run in
+    // SauceLabs or Browserstack
+    console.log(`KARMA_WEB_TEST_MODE: ${process.env.KARMA_WEB_TEST_MODE}`);
+
+    switch (process.env.KARMA_WEB_TEST_MODE) {
+      case 'SL_REQUIRED':
+        conf.browsers = browserProvidersConf.sauceAliases.CI_REQUIRED;
+        break;
+      case 'SL_OPTIONAL':
+        conf.browsers = browserProvidersConf.sauceAliases.CI_OPTIONAL;
+        break;
+      case 'BS_REQUIRED':
+        conf.browsers = browserProvidersConf.browserstackAliases.CI_REQUIRED;
+        break;
+      case 'BS_OPTIONAL':
+        conf.browsers = browserProvidersConf.browserstackAliases.CI_OPTIONAL;
+        break;
+      default:
+        throw new Error(
+            `Unrecognized process.env.KARMA_WEB_TEST_MODE: ${process.env.KARMA_WEB_TEST_MODE}`);
+    }
+  } else {
+    // Run the test locally
+    conf.browsers = [process.env['DISPLAY'] ? 'Chrome' : 'ChromeHeadless'];
+  }
+
+  config.set(conf);
 };

package.json

@@ -75,6 +75,7 @@
     "jasmine-core": "^3.1.0",
     "jquery": "3.0.0",
     "karma": "^3.1.4",
+    "karma-browserstack-launcher": "^1.3.0",
     "magic-string": "^0.25.0",
     "materialize-css": "1.0.0",
     "minimist": "1.2.0",
@@ -131,7 +132,6 @@
     "gulp-tslint": "8.1.2",
     "husky": "^0.14.3",
     "jpm": "1.3.1",
-    "karma-browserstack-launcher": "^1.3.0",
     "karma-chrome-launcher": "^2.2.0",
     "karma-jasmine": "^1.1.2",
     "karma-sauce-launcher": "^2.0.2",

packages/common/http/test/BUILD.bazel

@@ -6,6 +6,8 @@ ts_library(
     srcs = glob(
         ["**/*.ts"],
     ),
+    # Visible to //:test_web_all target
+    visibility = ["//:__pkg__"],
     deps = [
         "//packages/common/http",
         "//packages/common/http/testing",

packages/common/http/testing/test/BUILD.bazel

@@ -6,6 +6,8 @@ ts_library(
     srcs = glob(
         ["**/*.ts"],
     ),
+    # Visible to //:test_web_all target
+    visibility = ["//:__pkg__"],
     deps = [
         "//packages/common/http",
         "//packages/common/http/testing",

packages/common/test/BUILD.bazel

@@ -6,6 +6,8 @@ ts_library(
     srcs = glob(
         ["**/*.ts"],
     ),
+    # Visible to //:test_web_all target
+    visibility = ["//:__pkg__"],
     deps = [
         "//packages/common",
         "//packages/common/locales",

packages/core/test/BUILD.bazel

@@ -11,6 +11,8 @@ ts_library(
             "**/*_node_only_spec.ts",
         ],
     ),
+    # Visible to //:test_web_all target
+    visibility = ["//:__pkg__"],
     deps = [
         "//packages/animations",
         "//packages/animations/browser",

packages/forms/test/BUILD.bazel

@@ -4,6 +4,8 @@ ts_library(
     name = "test_lib",
     testonly = True,
     srcs = glob(["**/*.ts"]),
+    # Visible to //:test_web_all target
+    visibility = ["//:__pkg__"],
     deps = [
         "//packages/core",
         "//packages/core/testing",

packages/http/test/BUILD.bazel

@@ -4,6 +4,8 @@ ts_library(
     name = "test_lib",
     testonly = True,
     srcs = glob(["**/*.ts"]),
+    # Visible to //:test_web_all target
+    visibility = ["//:__pkg__"],
     deps = [
         "//packages/core",
         "//packages/core/testing",
@@ -28,6 +30,5 @@ ts_web_test_suite(
     name = "test_web",
     deps = [
         ":test_lib",
-        "@ngdeps//karma",
     ],
 )

packages/router/test/BUILD.bazel

@@ -4,6 +4,8 @@ ts_library(
     name = "test_lib",
     testonly = True,
     srcs = glob(["**/*.ts"]),
+    # Visible to //:test_web_all target
+    visibility = ["//:__pkg__"],
     deps = [
         "//packages/common",
         "//packages/common/testing",

scripts/saucelabs/start-tunnel.sh

@@ -7,16 +7,17 @@ readonly currentDir=$(cd $(dirname $0); pwd)
 # Command arguments that will be passed to sauce-connect.
 sauceArgs=""
 
-if [[ ! -z "${SAUCE_READY_FILE}" ]]; then
+if [[ ! -z "${SAUCE_READY_FILE:-}" ]]; then
+  mkdir -p $(dirname ${SAUCE_READY_FILE})
   sauceArgs="${sauceArgs} --readyfile ${SAUCE_READY_FILE}"
 fi
 
-if [[ ! -z "${SAUCE_PID_FILE}" ]]; then
+if [[ ! -z "${SAUCE_PID_FILE:-}" ]]; then
   mkdir -p $(dirname ${SAUCE_PID_FILE})
   sauceArgs="${sauceArgs} --pidfile ${SAUCE_PID_FILE}"
 fi
 
-if [[ ! -z "${SAUCE_TUNNEL_IDENTIFIER}" ]]; then
+if [[ ! -z "${SAUCE_TUNNEL_IDENTIFIER:-}" ]]; then
   sauceArgs="${sauceArgs} --tunnel-identifier ${SAUCE_TUNNEL_IDENTIFIER}"
 fi
 

tools/BUILD.bazel

@@ -2,7 +2,10 @@ package(default_visibility = ["//visibility:public"])
 
 load("@npm_bazel_typescript//:defs.bzl", "ts_config")
 
-exports_files(["tsconfig.json"])
+exports_files([
+    "tsconfig.json",
+    "jasmine-seed-generator.js",
+])
 
 ts_config(
     name = "tsconfig-test",

tools/defaults.bzl

@@ -1,7 +1,7 @@
 """Re-export of some bazel rules with repository-wide defaults."""
 
-load("@npm_bazel_karma//:defs.bzl", _ts_web_test_suite = "ts_web_test_suite")
 load("@build_bazel_rules_nodejs//:defs.bzl", _jasmine_node_test = "jasmine_node_test", _nodejs_binary = "nodejs_binary", _npm_package = "npm_package")
+load("@npm_bazel_karma//:defs.bzl", _karma_web_test = "karma_web_test", _karma_web_test_suite = "karma_web_test_suite", _ts_web_test = "ts_web_test", _ts_web_test_suite = "ts_web_test_suite")
 load("@npm_bazel_typescript//:defs.bzl", _ts_library = "ts_library")
 load("//packages/bazel:index.bzl", _ng_module = "ng_module", _ng_package = "ng_package")
 load("//packages/bazel/src:ng_rollup_bundle.bzl", _ng_rollup_bundle = "ng_rollup_bundle")
@@ -159,6 +159,26 @@ def npm_package(name, replacements = {}, **kwargs):
         **kwargs
     )
 
+def ts_web_test(bootstrap = [], deps = [], runtime_deps = [], **kwargs):
+    """Default values for ts_web_test"""
+    if not bootstrap:
+        bootstrap = ["//:web_test_bootstrap_scripts"]
+    local_deps = [
+        "@ngdeps//node_modules/tslib:tslib.js",
+        "//tools/rxjs:rxjs_umd_modules",
+    ] + deps
+    local_runtime_deps = [
+        "//tools/testing:browser",
+    ] + runtime_deps
+
+    _ts_web_test(
+        runtime_deps = local_runtime_deps,
+        bootstrap = bootstrap,
+        deps = local_deps,
+        karma = _DEFAULT_KARMA_BIN,
+        **kwargs
+    )
+
 def ts_web_test_suite(bootstrap = [], deps = [], runtime_deps = [], **kwargs):
     """Default values for ts_web_test_suite"""
     if not bootstrap:
@@ -190,6 +210,60 @@ def ts_web_test_suite(bootstrap = [], deps = [], runtime_deps = [], **kwargs):
         **kwargs
     )
 
+def karma_web_test(bootstrap = [], deps = [], data = [], runtime_deps = [], **kwargs):
+    """Default values for karma_web_test"""
+    if not bootstrap:
+        bootstrap = ["//:web_test_bootstrap_scripts"]
+    local_deps = [
+        "@ngdeps//karma-browserstack-launcher",
+        "@ngdeps//node_modules/tslib:tslib.js",
+        "//tools/rxjs:rxjs_umd_modules",
+    ] + deps
+    local_runtime_deps = [
+        "//tools/testing:browser",
+    ] + runtime_deps
+
+    _karma_web_test(
+        runtime_deps = local_runtime_deps,
+        bootstrap = bootstrap,
+        config_file = "//:karma-js.conf.js",
+        deps = local_deps,
+        karma = _DEFAULT_KARMA_BIN,
+        data = data + [
+            "//:browser-providers.conf.js",
+            "//tools:jasmine-seed-generator.js",
+        ],
+        configuration_env_vars = ["KARMA_WEB_TEST_MODE"],
+        **kwargs
+    )
+
+def karma_web_test_suite(bootstrap = [], deps = [], **kwargs):
+    """Default values for karma_web_test_suite"""
+    if not bootstrap:
+        bootstrap = ["//:web_test_bootstrap_scripts"]
+    local_deps = [
+        "@ngdeps//node_modules/tslib:tslib.js",
+        "//tools/rxjs:rxjs_umd_modules",
+    ] + deps
+
+    _karma_web_test_suite(
+        bootstrap = bootstrap,
+        deps = local_deps,
+        karma = _DEFAULT_KARMA_BIN,
+        # Run unit tests on local Chromium by default.
+        # You can exclude tests based on tags, e.g. to skip Firefox testing,
+        #   `yarn bazel test --test_tag_filters=-browser:firefox-local [targets]`
+        browsers = [
+            "@io_bazel_rules_webtesting//browsers:chromium-local",
+            # Don't test on local Firefox by default, for faster builds.
+            # We think that bugs in Angular tend to be caught the same in any
+            # evergreen browser.
+            # "@io_bazel_rules_webtesting//browsers:firefox-local",
+            # TODO(alexeagle): add remote browsers on SauceLabs
+        ],
+        **kwargs
+    )
+
 def nodejs_binary(data = [], **kwargs):
     """Default values for nodejs_binary"""
     _nodejs_binary(