build: run angular/material2 unit tests on ci (#28197)

PR Close #28197

.circleci/config.yml

@@ -291,14 +291,16 @@ jobs:
           name: Wait for preview and run tests
           command: node aio/scripts/test-preview.js $CI_PULL_REQUEST $CI_COMMIT $CI_AIO_MIN_PWA_SCORE
 
-  # This job exists only for backwards-compatibility with old scripts and tests
+
-  # that rely on the pre-Bazel dist/packages-dist layout.
+  # The `build-npm-packages` tasks exist for backwards-compatibility with old scripts and
-  # It duplicates some work with the job above: we build the bazel packages
+  # tests that rely on the pre-Bazel `dist/packages-dist` output structure (build.sh).
-  # twice. Even though we have a remote cache, these jobs will typically run in
+  # Having multiple jobs that independently build in this manner duplicates some work; we build
-  # parallel so up-to-date outputs will not be available at the time the build
+  # the bazel packages more than once. Even though we have a remote cache, these jobs will
+  # typically run in parallel so up-to-date outputs will not be available at the time the build
   # starts.
-  # No new jobs should depend on this one.
+
-  build-packages-dist:
+  # Build the view engine npm packages. No new jobs should depend on this.
+  build-npm-packages:
     <<: *job_defaults
     resource_class: xlarge
     steps:
@@ -319,6 +321,29 @@ jobs:
           root: dist
           paths:
             - packages-dist
+
+
+  # Build the ivy npm packages.
+  build-ivy-npm-packages:
+    <<: *job_defaults
+    resource_class: xlarge
+    steps:
+      - checkout:
+          <<: *post_checkout
+      - restore_cache:
+          key: *cache_key
+      - *define_env_vars
+      - *setup_circleci_bazel_config
+      - *yarn_install
+      - *setup_bazel_remote_execution
+
+      - run: scripts/build-ivy-npm-packages.sh
+
+      # Save the npm packages from //packages/... for other workflow jobs to read
+      # https://circleci.com/docs/2.0/workflows/#using-workspaces-to-share-data-among-jobs
+      - persist_to_workspace:
+          root: dist
+          paths:
             - packages-dist-ivy-aot
 
   # We run the integration tests outside of Bazel for now.
@@ -484,6 +509,21 @@ jobs:
       # the ESM5/ES2015 output. See: https://github.com/angular/angular/issues/27966
       # - run: yarn gulp source-map-test
 
+  # Job to run unit tests from angular/material2. Needs a browser since all
+  # component unit tests assume they're running in the browser environment.
+  material-unit-tests:
+    <<: *job_defaults
+    resource_class: xlarge
+    docker:
+      - image: *browsers_docker_image
+    steps:
+      - checkout:
+          <<: *post_checkout
+      - *define_env_vars
+      - attach_workspace:
+          at: dist
+      - run: ./scripts/ci/run_angular_material_unit_tests.sh
+
 workflows:
   version: 2
   default_workflow:
@@ -491,7 +531,8 @@ workflows:
       - lint
       - test
       - test_ivy_aot
-      - build-packages-dist
+      - build-npm-packages
+      - build-ivy-npm-packages
       - test_aio
       - legacy-unit-tests-local
       - legacy-unit-tests-saucelabs
@@ -500,22 +541,22 @@ workflows:
             - test_aio
       - legacy-e2e-tests:
           requires:
-            - build-packages-dist
+            - build-npm-packages
       - legacy-misc-tests:
           requires:
-            - build-packages-dist
+            - build-npm-packages
       - test_aio_local:
           requires:
-            - build-packages-dist
+            - build-npm-packages
       - test_aio_local_ivy:
           requires:
-            - build-packages-dist
+            - build-npm-packages
       - test_aio_tools:
           requires:
-            - build-packages-dist
+            - build-npm-packages
       - test_docs_examples:
           requires:
-            - build-packages-dist
+            - build-npm-packages
       - aio_preview:
           # Only run on PR builds. (There can be no previews for non-PR builds.)
           filters:
@@ -526,7 +567,7 @@ workflows:
             - aio_preview
       - integration_test:
           requires:
-            - build-packages-dist
+            - build-npm-packages
       - publish_snapshot:
           # Note: no filters on this job because we want it to run for all upstream branches
           # We'd really like to filter out pull requests here, but not yet available:
@@ -541,13 +582,17 @@ workflows:
             - test_aio_local
             - test_aio_local_ivy
             - test_docs_examples
-            # Get the artifacts to publish from the build-packages-dist job
+            # Get the artifacts to publish from the build-npm-packages job
             # since the publishing script expects the legacy outputs layout.
-            - build-packages-dist
+            - build-npm-packages
+            - build-ivy-npm-packages
             - legacy-e2e-tests
             - legacy-misc-tests
             - legacy-unit-tests-local
             - legacy-unit-tests-saucelabs
+      - material-unit-tests:
+          requires:
+            - build-ivy-npm-packages
 
 
   aio_monitoring:

scripts/build-ivy-npm-packages.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+source ./scripts/package-builder.sh
+
+# Build the ivy packages
+buildTargetPackages "dist/packages-dist-ivy-aot" "aot" "Ivy AOT"

scripts/build-packages-dist.sh

@@ -1,69 +1,6 @@
 #!/usr/bin/env bash
-# Build the dist/packages-dist directory in the same fashion as the legacy
-# /build.sh script, by building the npm packages with Bazel and copying files.
-# This is needed for scripts and tests which are not updated to the Bazel output
-# layout (which always matches the input layout).
-# Do not add new dependencies on this script, instead adapt scripts to use the
-# new layout, and write new tests as Bazel targets.
 
-set -u -e -o pipefail
+source ./scripts/package-builder.sh
 
-cd "$(dirname "$0")"
+# Build the legacy (view engine) npm packages into dist/packages-dist
-
+buildTargetPackages "dist/packages-dist" "legacy" "Production"
-# basedir is the workspace root
-readonly basedir=$(pwd)/..
-# We need to resolve the Bazel binary in the node modules because running Bazel
-# through `yarn bazel` causes additional output that throws off command stdout.
-readonly bazelBin=$(yarn bin)/bazel
-readonly bin=$(${bazelBin} info bazel-bin)
-
-function buildTargetPackages() {
-  targets="$1"
-  destPath="$2"
-  compileMode="$3"
-  desc="$4"
-
-  echo "##################################"
-  echo "scripts/build-packages-dist.sh:"
-  echo "  building @angular/* npm packages"
-  echo "  mode: ${desc}"
-  echo "##################################"
-
-  # Use --config=release so that snapshot builds get published with embedded version info
-  echo "$targets" | xargs ${bazelBin} build --config=release --define=compile=$compileMode
-
-  [ -d "${basedir}/${destPath}" ] || mkdir -p $basedir/${destPath}
-
-  dirs=`echo "$targets" | sed -e 's/\/\/packages\/\(.*\):npm_package/\1/'`
-
-  for pkg in $dirs; do
-    # Skip any that don't have an "npm_package" target
-    srcDir="${bin}/packages/${pkg}/npm_package"
-    destDir="${basedir}/${destPath}/${pkg}"
-    if [ -d $srcDir ]; then
-      echo "# Copy artifacts to ${destDir}"
-      rm -rf $destDir
-      cp -R $srcDir $destDir
-      chmod -R u+w $destDir
-    fi
-  done
-}
-
-# Ideally these integration tests should run under bazel, and just list the npm
-# packages in their deps[].
-# Until then, we have to manually run bazel first to create the npm packages we
-# want to test.
-BAZEL_TARGETS=`${bazelBin} query --output=label 'attr("tags", "\[.*release-with-framework.*\]", //packages/...) intersect kind(".*_package", //packages/...)'`
-buildTargetPackages "$BAZEL_TARGETS" "dist/packages-dist" "legacy" "Production"
-
-# We don't use the ivy build in the integration tests, only when publishing
-# snapshots.
-# This logic matches what we use in the .circleci/config.yml file to short-
-# circuit execution of the publish-packages job.
-[[  "${CI_PULL_REQUEST-}" != "false"
-    || "${CI_REPO_OWNER-}" != "angular"
-    || "${CI_REPO_NAME-}" != "angular"
-    || "${CI_BRANCH}" != "master"
-]] && exit 0
-
-buildTargetPackages "$BAZEL_TARGETS" "dist/packages-dist-ivy-aot" "aot" "Ivy AOT"

scripts/ci/run_angular_material_unit_tests.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+set -u -e -o pipefail
+
+
+# This script runs unit tests from angular/material2.
+
+# Save the dir for the root of the Angular repo.
+angular_dir=$(pwd)
+
+# Clone the angular/material2 repo into tmp so we can run the tests from there.
+# We specifically use /tmp here because we want the cloned repo to be completely
+# isolated from angular/angular in order to avoid any bad interactions between their
+# separate build setups. Also note that this is using the ivy-2019 branch, which has
+# previously been set up to work with ivy.
+cd /tmp
+rm -rf /tmp/material2
+git clone --depth 1 --branch ivy-2019 https://github.com/angular/material2.git
+
+# Install dependencies for the freshly cloned repo.
+cd /tmp/material2
+yarn install --frozen-lockfile --non-interactive # TODO: cache
+
+# Install this version of Angular into the freshly cloned repo.
+rm -rf /tmp/material2/node_modules/@angular/*
+cp -r ${angular_dir}/dist/packages-dist-ivy-aot/* /tmp/material2/node_modules/@angular/
+
+# The angular/material2 CI sets TEST_PLATFORM to either local, saucelabs, or browserstack.
+# For angular/angular, we only want to run the local tests.
+export TEST_PLATFORM=local
+
+# Append the test blacklist into angular/material2's karma-test-shim.js.
+# This filters out known-failing tests because the goal is to prevent regressions.
+cat ${angular_dir}/tools/material-ci/angular_material_test_blacklist.js >> /tmp/material2/test/karma-test-shim.js
+
+# Now actually run the tests.
+yarn gulp test:single-run

scripts/package-builder.sh

@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+
+# Build the dist/packages-dist directory in the same fashion as the legacy
+# /build.sh script, by building the npm packages with Bazel and copying files.
+# This is needed for scripts and tests which are not updated to the Bazel output
+# layout (which always matches the input layout).
+# Do not add new dependencies on this script, instead adapt scripts to use the
+# new layout, and write new tests as Bazel targets.
+#
+# Ideally integration tests should run under bazel, and just consume the npm
+# packages via `deps`. Until that works, we manually build the npm packages and then
+# copy the results to the appropriate `dist` location.
+
+set -u -e -o pipefail
+
+cd "$(dirname "$0")"
+
+# basedir is the workspace root
+readonly base_dir=$(pwd)/..
+# We need to resolve the Bazel binary in the node modules because running Bazel
+# through `yarn bazel` causes additional output that throws off command stdout.
+readonly bazel_bin=$(yarn bin)/bazel
+readonly bin=$(${bazel_bin} info bazel-bin)
+
+function buildTargetPackages() {
+  # List of targets to build, e.g. core, common, compiler, etc.
+  targets=$(${bazel_bin} query --output=label 'attr("tags", "\[.*release-with-framework.*\]", //packages/...) intersect kind(".*_package", //packages/...)')
+
+  # Path to the output directory into which we copy the npm packages.
+  dest_path="$1"
+
+  # Either "legacy" (view engine) or "aot" (ivy)
+  compile_mode="$2"
+
+  # Human-readable description of the build.
+  desc="$3"
+
+  echo "##################################"
+  echo "scripts/build-packages-dist.sh:"
+  echo "  building @angular/* npm packages"
+  echo "  mode: ${desc}"
+  echo "##################################"
+
+  # Use --config=release so that snapshot builds get published with embedded version info
+  echo "$targets" | xargs ${bazel_bin} build --config=release --define=compile=${compile_mode}
+
+  [[ -d "${base_dir}/${dest_path}" ]] || mkdir -p ${base_dir}/${dest_path}
+
+  dirs=`echo "$targets" | sed -e 's/\/\/packages\/\(.*\):npm_package/\1/'`
+
+  for pkg in ${dirs}; do
+    # Skip any that don't have an "npm_package" target
+    src_dir="${bin}/packages/${pkg}/npm_package"
+    dest_dir="${base_dir}/${dest_path}/${pkg}"
+    if [[ -d ${src_dir} ]]; then
+      echo "# Copy artifacts to ${dest_dir}"
+      rm -rf ${dest_dir}
+      cp -R ${src_dir} ${dest_dir}
+      chmod -R u+w ${dest_dir}
+    fi
+  done
+}
+

tools/material-ci/instructions.md

@@ -0,0 +1,24 @@
+### Unit tests for Angular CDK/Material
+The unit tests from angular/material2 run on CircleCI under the `material-unit-tests` job.
+Known failing tests are skipped based on the blacklist in
+`tools/material-ci/angular_material_test_blacklist.js`. Whenever the root cause of a known failure
+is identified, the `notes` field for the corresponding tests should be updated. Whenever a failure
+is resolved, the corresponding tests should be removed from the blacklist.
+
+### Debugging
+To debug a failure, you need to work against the angular/material2 repo:
+1. Clone `angular/material2`
+2. Checkout the `ivy-2019` branch
+3. Run `yarn`
+4. Run `scripts/ivy/install-angular.sh path/to/local/angular/repo`
+5. Run `gulp test`
+
+### Regenerating the blacklist
+If a problem has been fixed, you can regenerate the blacklist by:
+1. Clone `angular/material2`
+2. Checkout the `ivy-2019` branch
+3. Run `yarn`
+4. Run `scripts/ivy/install-angular.sh path/to/local/angular/repo`
+5. Run `gulp test`. Let it finish. It will take a few minutes.
+6. Run `scripts/ivy/generate-blacklist.js`
+7. Copy the new blacklist from `dist/angular_material_test_blacklist.js`