ci: run acceptance tests on saucelabs with ivy (#34277)

Currently we only run Saucelabs on PRs using the legacy View Engine build. Switching that build to Ivy is not trivial and there are various options: 1. Updating the R3 switches to use POST_R3 by default. At first glance, this doesn't look easy because the current ngtsc switch logic seems to be unidirectional (only PRE_R3 to POST_R3). 2. Updating the legacy setup to run with Ivy. This sounds like the easiest solution at first.. but it turns out to be way more complicated. Packages would need to be built with ngtsc using legacy tools (i.e. first building the compiler-cli; and then building packages) and View Engine only tests would need to be determined and filtered out. Basically it will result in re-auditing all test targets. This is contradictory to the fact that we have this information in Bazel already. 3. Creating a new job that runs tests on Saucelabs with Bazel. We specify fine-grained test targets that should run. This would be a good start (e.g. acceptance tests) and also would mean that we do not continue maintaining the legacy setup.. This commit implements the third option as it allows us to move forward with the general Bazel migration. We don't want to spend too much time on our legacy setup since it will be removed anyway in the future. PR Close #34277
2019-12-06 16:17:09 +01:00 · 2019-12-06 16:17:09 +01:00 · 6d3a25d897
parent 3bbd12d560
commit 6d3a25d897
13 changed files with 73 additions and 29 deletions
--- a/.circleci/bazel.common.rc
+++ b/.circleci/bazel.common.rc
@ -13,3 +13,7 @@ test --flaky_test_attempts=2

 # More details on failures
 build --verbose_failures=true
+
+# For saucelabs tests we don't want to enable flaky test attempts. Karma has its own integrated
+# retry mechanism and we do not want to retry unnecessarily if Karma already tried multiple times.
+test:saucelabs --flaky_test_attempts=1
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@ -330,7 +330,7 @@ jobs:
  # [BUILD.bazel](https://github.com/angular/angular/blob/ef44f51d5/BUILD.bazel#L66-L92)).
  #
  # NOTE: This is currently limited to master builds only. See the `default_workflow` configuration.
-  test_saucelabs_bazel:
+  saucelabs_view_engine:
    executor:
      name: default-executor
      # In order to avoid the bottleneck of having a slow host machine, we acquire a better
@ -344,9 +344,9 @@ jobs:
      - setup_bazel_rbe
      - run:
          name: Run Bazel tests in saucelabs
-          # All web tests are contained within a single //:test_web_all target for Saucelabs
-          # as running each set of tests as a separate target will attempt to acquire too
-          # many browsers on Saucelabs (7 per target currently) and some tests will always
+          # All web tests are contained within a single //:saucelabs_unit_tests_poc target
+          # for Saucelabs as running each set of tests as a separate target will attempt to acquire
+          # too many browsers on Saucelabs (7 per target currently) and some tests will always
          # fail to acquire browsers. For example:
          # 14 02 2019 19:52:33.170:WARN [launcher]: chrome beta on SauceLabs have not captured in 180000 ms, killing.
          # //packages/forms/test:web_test_sauce TIMEOUT in 315.0s
@ -355,11 +355,34 @@ jobs:
              --tunnel-id angular-${CIRCLE_BUILD_NUM}-${CIRCLE_NODE_INDEX} \
              --username $SAUCE_USERNAME \
              --key $(echo $SAUCE_ACCESS_KEY | rev) \
-              yarn bazel test //:test_web_all
+              yarn bazel test //:saucelabs_unit_tests_poc --config=saucelabs
          no_output_timeout: 20m
      - notify_webhook_on_fail:
          webhook_url_env_var: SLACK_DEV_INFRA_CI_FAILURES_WEBHOOK_URL

+  saucelabs_ivy:
+    executor:
+      name: default-executor
+      # In order to avoid the bottleneck of having a slow host machine, we acquire a better
+      # container for this job. This is necessary because we launch a lot of browsers concurrently
+      # and therefore the tunnel and Karma need to process a lot of file requests and tests.
+      resource_class: xlarge
+    steps:
+      - custom_attach_workspace
+      - init_environment
+      - setup_circleci_bazel_config
+      - setup_bazel_rbe
+      - run:
+          name: Run Bazel tests on Saucelabs
+          # Runs the //:saucelabs_tests target with Saucelabs and Ivy.
+          command:  |
+            ./scripts/saucelabs/run-bazel-via-tunnel.sh \
+              --tunnel-id angular-${CIRCLE_BUILD_NUM}-${CIRCLE_NODE_INDEX} \
+              --username $SAUCE_USERNAME \
+              --key $(echo $SAUCE_ACCESS_KEY | rev) \
+              -- yarn bazel test //:saucelabs_unit_tests --config=ivy --config=saucelabs
+          no_output_timeout: 20m
+
  test_aio:
    # Needed because the AIO tests and the PWA score test depend on Chrome being available.
    executor: browsers-executor
@ -819,11 +842,14 @@ workflows:
      - legacy-unit-tests-saucelabs:
          requires:
            - setup
-      - test_saucelabs_bazel:
+      - saucelabs_ivy:
+            requires:
+              - test_ivy_aot
+      - saucelabs_view_engine:
          # This job is currently a PoC and a subset of `legacy-unit-tests-saucelabs`. Running on
          # master only to avoid wasting resources.
-          #
-          # TODO: Run this job on all branches (including PRs) as soon as it is not a PoC.
+          # TODO: Run this job on all branches (including PRs) as soon as it is not a PoC and
+          # we can remove the legacy saucelabs job.
          <<: *only_on_master
          requires:
            - setup
@ -876,6 +902,7 @@ workflows:
            - test
            - test_ivy_aot
            - integration_test
+            - saucelabs_ivy
            # Only publish if `aio`/`docs` tests using the locally built Angular packages pass
            - test_aio_local
            - test_aio_local_viewengine
--- a/BUILD.bazel
+++ b/BUILD.bazel
@ -47,17 +47,28 @@ filegroup(
    ],
 )

-# To run a karma_web_test target locally on SauceLabs:
-# 1) have SAUCE_USERNAME, SAUCE_ACCESS_KEY (and optionally a SAUCE_TUNNEL_IDENTIFIER) set in your environment
-# 2) open a sauce connection with `./scripts/saucelabs/start-tunnel.sh`
-#    NOTE: start-tunnel.sh uses `node_modules/sauce-connect` which is current linux specific:
-#          "sauce-connect": "https://saucelabs.com/downloads/sc-4.5.3-linux.tar.gz".
-#          On OSX or Windows you'll need to use the appropriate sauce-connect binary.
-# 3) run target with `yarn bazel test --config=saucelabs <target>`
-#    NOTE: --config=saucelabs is required as it makes the SAUCE_XXX environment variables available to
-#          the action. See /.bazelrc.
+# To run a karma_web_test target manually, run the "./scripts/saucelabs/run-bazel-via-tunnel.sh"
+# script. Note: If you are on MacOS or Windows, you need to manually start the Saucelabs tunnel
+# because the script only supports the linux Saucelabs tunnel binary. We combine all tests into
+# a single "karma_web_test" target because running them as separate targets in parallel can result
+# in to too many browsers being acquired at the same time. This will then result in tests being
+# flaky. This target runs in CI with Saucelabs and Ivy.
 karma_web_test(
-    name = "test_web_all",
+    name = "saucelabs_unit_tests",
+    tags = [
+        "local",
+        "manual",
+        "saucelabs",
+    ],
+    deps = [
+        "//packages/core/test/acceptance:acceptance_lib",
+    ],
+)
+
+karma_web_test(
+    # This target runs in CI with View Engine as a Saucelabs and Bazel proof-of-concept. It's a
+    # subset of the legacy saucelabs tests.
+    name = "saucelabs_unit_tests_poc",
    tags = [
        "local",
        "manual",
@ -65,7 +76,7 @@ karma_web_test(
    ],
    deps = [
        # We combine all tests into a single karma_web_test target
-        # as running them as seperate targets in parallel leads to too many
+        # as running them as separate targets in parallel leads to too many
        # browsers being acquired at once in SauceLabs and the tests flake out
        # TODO: this is an example subset of tests below, add all remaining angular tests
        "//packages/common/http/test:test_lib",
--- a/packages/common/http/test/BUILD.bazel
+++ b/packages/common/http/test/BUILD.bazel
@ -6,7 +6,7 @@ ts_library(
    srcs = glob(
        ["**/*.ts"],
    ),
-    # Visible to //:test_web_all target
+    # Visible to //:saucelabs_unit_tests_poc target
    visibility = ["//:__pkg__"],
    deps = [
        "//packages/common/http",
--- a/packages/common/http/testing/test/BUILD.bazel
+++ b/packages/common/http/testing/test/BUILD.bazel
@ -6,7 +6,7 @@ ts_library(
    srcs = glob(
        ["**/*.ts"],
    ),
-    # Visible to //:test_web_all target
+    # Visible to //:saucelabs_unit_tests_poc target
    visibility = ["//:__pkg__"],
    deps = [
        "//packages/common/http",
--- a/packages/common/test/BUILD.bazel
+++ b/packages/common/test/BUILD.bazel
@ -6,7 +6,7 @@ ts_library(
    srcs = glob(
        ["**/*.ts"],
    ),
-    # Visible to //:test_web_all target
+    # Visible to //:saucelabs_unit_tests_poc target
    visibility = ["//:__pkg__"],
    deps = [
        "//packages/common",
--- a/packages/core/test/BUILD.bazel
+++ b/packages/core/test/BUILD.bazel
@ -11,7 +11,7 @@ ts_library(
            "**/*_node_only_spec.ts",
        ],
    ),
-    # Visible to //:test_web_all target
+    # Visible to //:saucelabs_unit_tests_poc target
    visibility = ["//:__pkg__"],
    deps = [
        "//packages/animations",
--- a/packages/core/test/acceptance/BUILD.bazel
+++ b/packages/core/test/acceptance/BUILD.bazel
@ -8,6 +8,8 @@ ts_library(
    srcs = glob(
        ["**/*.ts"],
    ),
+    # Visible to //:saucelabs_unit_tests
+    visibility = ["//:__pkg__"],
    deps = [
        "//packages/animations",
        "//packages/animations/browser",
--- a/packages/forms/test/BUILD.bazel
+++ b/packages/forms/test/BUILD.bazel
@ -4,7 +4,7 @@ ts_library(
    name = "test_lib",
    testonly = True,
    srcs = glob(["**/*.ts"]),
-    # Visible to //:test_web_all target
+    # Visible to //:saucelabs_unit_tests_poc target
    visibility = ["//:__pkg__"],
    deps = [
        "//packages/common",
--- a/packages/http/test/BUILD.bazel
+++ b/packages/http/test/BUILD.bazel
@ -4,7 +4,7 @@ ts_library(
    name = "test_lib",
    testonly = True,
    srcs = glob(["**/*.ts"]),
-    # Visible to //:test_web_all target
+    # Visible to //:saucelabs_unit_tests_poc target
    visibility = ["//:__pkg__"],
    deps = [
        "//packages/common",
--- a/packages/router/test/BUILD.bazel
+++ b/packages/router/test/BUILD.bazel
@ -4,7 +4,7 @@ ts_library(
    name = "test_lib",
    testonly = True,
    srcs = glob(["**/*.ts"]),
-    # Visible to //:test_web_all target
+    # Visible to //:saucelabs_unit_tests_poc target
    visibility = ["//:__pkg__"],
    deps = [
        "//packages/common",
--- a/packages/zone.js/test/karma_test.bzl
+++ b/packages/zone.js/test/karma_test.bzl
@ -127,7 +127,7 @@ def karma_test(name, env_srcs, env_deps, env_entry_point, test_srcs, test_deps,
                    ":assets/import.html",
                ],
                tags = ["zone_karma_test"],
-                # Visible to //:test_web_all target
+                # Visible to //:saucelabs_unit_tests_poc target
                visibility = ["//:__pkg__"],
                runtime_deps = [
                    "@npm//karma-browserstack-launcher",
--- a/scripts/saucelabs/run-bazel-via-tunnel.sh
+++ b/scripts/saucelabs/run-bazel-via-tunnel.sh
@ -158,9 +158,9 @@ set +e
  cd $GIT_ROOT_DIR && \
  # Run bazel command with saucelabs specific environment variables passed to the action
  # The KARMA_WEB_TEST_MODE and SAUCE_TUNNEL_IDENTIFIER environment variables provide
-  # envirnment variables to be read in the karma configuration file to set correct
+  # environment variables to be read in the karma configuration file to set correct
  # configurations for karma saucelabs and browser configs.
-  # Usage of these envirnment variables can be seen in this repo in
+  # Usage of these environment variables can be seen in this repo in
  # /karma-js.conf.js and /browser-providers.conf.js
  eval "$USER_COMMAND --define=KARMA_WEB_TEST_MODE=SL_REQUIRED \
                      --action_env=SAUCE_USERNAME=$SAUCE_USERNAME \