From 7e46a6d99dd773610d88f89144074b5adc67e879 Mon Sep 17 00:00:00 2001 From: JoostK Date: Thu, 13 Jun 2019 21:06:14 +0200 Subject: [PATCH] test(ivy): update Material to recent commit from master branch (#31569) Previously, the ivy-2019 branch of the Material (aka components) repo was used, which contains some changes that were necessary to work with Ivy. These changes are not longer necessary, as Material's master branch is fully working with Ivy today. To be up-to-date with recent Material development and its support for more recent dependencies, e.g. TypeScript, it is desirable for us to be on a newer version of Material. This commit moves the Material tests away from the ivy-2019 branch, to a recent commit on master. We are not targeting the master branch itself, as that would introduce a moving target into Angular's CI checks, which is undesirable. Lastly, the usage of gulp to run Material's tests is changed into using Bazel, as Material itself is now also built with Bazel. PR Close #31569 --- .circleci/config.yml | 29 +++------ .circleci/env.sh | 2 +- .circleci/setup-rbe.sh | 20 ++++++ scripts/ci/clone_angular_material_repo.sh | 2 +- scripts/ci/run_angular_material_unit_tests.sh | 26 +++++--- scripts/ci/update-deps-to-dist-packages.js | 63 +++++++++++++++++++ 6 files changed, 110 insertions(+), 32 deletions(-) create mode 100755 .circleci/setup-rbe.sh create mode 100644 scripts/ci/update-deps-to-dist-packages.js diff --git a/.circleci/config.yml b/.circleci/config.yml index 7e4f6cc65e..7623635a3f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -58,17 +58,7 @@ var_5: &setup_bazel_remote_execution # cause decryption failures based on the openssl version. https://stackoverflow.com/a/39641378/4317734 openssl aes-256-cbc -d -in .circleci/gcp_token -md md5 -k "$CI_REPO_NAME" -out /home/circleci/.gcp_credentials echo "export GOOGLE_APPLICATION_CREDENTIALS=/home/circleci/.gcp_credentials" >> $BASH_ENV - touch .bazelrc.user - sudo bash -c "echo -e 'build --config=remote\n' >> .bazelrc.user" - sudo bash -c "echo -e 'build:remote --remote_accept_cached=true\n' >> .bazelrc.user" - echo "Reading from remote cache for bazel remote jobs." - if [[ "$CI_PULL_REQUEST" == "false" ]]; then - sudo bash -c "echo -e 'build:remote --remote_upload_local_results=true\n' >> .bazelrc.user" - echo "Uploading local build results to remote cache." - else - sudo bash -c "echo -e 'build:remote --remote_upload_local_results=false\n' >> .bazelrc.user" - echo "Not uploading local build results to remote cache." - fi + ./.circleci/setup-rbe.sh .bazelrc.user # Settings common to each job var_6: &job_defaults @@ -625,19 +615,12 @@ jobs: resource_class: xlarge docker: - image: *browsers_docker_image - # The Material unit tests support splitting the browsers across multiple CircleCI - # instances. Since by default this job launches two browsers, we run each browser - # in its own container instance. - # https://github.com/angular/material2/blob/7baeaa797b19da2d2998f0d26f6fede3c8a13714/test/karma.conf.js#L107-L110 - parallelism: 2 - environment: - # The Material unit tests also support launching the same browser multiple times by - # sharding individual specs across the defined multiple instances. - # See: https://github.com/angular/material2/blob/7baeaa797b19da2d2998f0d26f6fede3c8a13714/test/karma.conf.js#L113-L116 - KARMA_PARALLEL_BROWSERS: 3 steps: - *attach_workspace - *init_environment + # Although RBE is configured below for the Material repo, also setup RBE in the Angular repo + # to provision Angular's GCP token into the environment variables. + - *setup_bazel_remote_execution - run: name: "Cloning Material repository" command: ./scripts/ci/clone_angular_material_repo.sh @@ -659,6 +642,10 @@ jobs: key: v2-angular-material-{{ checksum "/tmp/material2/yarn.lock" }} paths: - "/tmp/material2/node_modules" + - run: + name: "Setup Bazel RBE remote execution in Material repo" + command: | + ./.circleci/setup-rbe.sh "${MATERIAL_REPO_TMP_DIR}/.bazelrc.user" - run: name: "Running Material unit tests" command: ./scripts/ci/run_angular_material_unit_tests.sh diff --git a/.circleci/env.sh b/.circleci/env.sh index 878cdfa091..6b41bcd13f 100755 --- a/.circleci/env.sh +++ b/.circleci/env.sh @@ -77,7 +77,7 @@ setPublicVar SAUCE_READY_FILE_TIMEOUT 120 # their separate build setups. setPublicVar MATERIAL_REPO_TMP_DIR "/tmp/material2" setPublicVar MATERIAL_REPO_URL "https://github.com/angular/material2.git" -setPublicVar MATERIAL_REPO_BRANCH "ivy-2019" +setPublicVar MATERIAL_REPO_TAG "8.1.0" # Source `$BASH_ENV` to make the variables available immediately. source $BASH_ENV; diff --git a/.circleci/setup-rbe.sh b/.circleci/setup-rbe.sh new file mode 100755 index 0000000000..07bbc6bd5d --- /dev/null +++ b/.circleci/setup-rbe.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -u -e -o pipefail + +# The path of the .bazelrc.user file to update should be passed as first parameter to this script. +# This allows to setup RBE for both the Angular repo and the Material repo. +bazelrc_user="$1" + +echo "Writing RBE configuration to ${bazelrc_user}" + +touch ${bazelrc_user} +echo -e 'build --config=remote\n' >> ${bazelrc_user} +echo -e 'build:remote --remote_accept_cached=true\n' >> ${bazelrc_user} +echo "Reading from remote cache for bazel remote jobs." +if [[ "$CI_PULL_REQUEST" == "false" ]]; then + echo -e 'build:remote --remote_upload_local_results=true\n' >> ${bazelrc_user} + echo "Uploading local build results to remote cache." +else + echo -e 'build:remote --remote_upload_local_results=false\n' >> ${bazelrc_user} + echo "Not uploading local build results to remote cache." +fi diff --git a/scripts/ci/clone_angular_material_repo.sh b/scripts/ci/clone_angular_material_repo.sh index c85492566a..9eb9b0030b 100755 --- a/scripts/ci/clone_angular_material_repo.sh +++ b/scripts/ci/clone_angular_material_repo.sh @@ -6,5 +6,5 @@ set -u -e -o pipefail rm -rf ${MATERIAL_REPO_TMP_DIR} # Clone the Material repository into the given temporary directory. -git clone --depth 1 --branch ${MATERIAL_REPO_BRANCH} ${MATERIAL_REPO_URL} \ +git clone --depth 1 --branch ${MATERIAL_REPO_TAG} ${MATERIAL_REPO_URL} \ ${MATERIAL_REPO_TMP_DIR} diff --git a/scripts/ci/run_angular_material_unit_tests.sh b/scripts/ci/run_angular_material_unit_tests.sh index 01c6ad9204..b044daf953 100755 --- a/scripts/ci/run_angular_material_unit_tests.sh +++ b/scripts/ci/run_angular_material_unit_tests.sh @@ -6,20 +6,28 @@ set -u -e -o pipefail # Save the dir for the root of the Angular repo. angular_dir=$(pwd) +# Disable full template type check, as Material doesn't build cleanly with it enabled. +# See https://github.com/angular/components/pull/16373 for details. +# The "ivyTemplateTypeCheck" flag is set to True so that a minimum amount of type checking still +# occurs, at a level compatible with that of VE's type checking. This ensures Ivy's type checker +# is still tested against the Material repo, albeit in its non-strict mode. +sed -i'.bak' "s/\(_ENABLE_NG_TYPE_CHECKING = \)True/\1False/g" ${MATERIAL_REPO_TMP_DIR}/tools/defaults.bzl +sed -i'.bak' "s/\(\"ivyTemplateTypeCheck\": \)False/\1True/g" dist/packages-dist-ivy-aot/bazel/src/ng_module.bzl + # Switch into Material directory. cd ${MATERIAL_REPO_TMP_DIR} -# Install this version of Angular into the freshly cloned repo. -rm -rf ./node_modules/@angular/* -cp -r ${angular_dir}/dist/packages-dist-ivy-aot/* ./node_modules/@angular/ - -# The angular/material2 CI sets TEST_PLATFORM to either "local", "saucelabs", or "browserstack". -# For angular/angular, we only want to run the "local" tests. -export TEST_PLATFORM=local +# Updates Material's package.json to refer to the packages-dist-ivy-aot directory. +# Note that it's not necessary to perform a yarn install, as Bazel performs its own yarn install. +node ${angular_dir}/scripts/ci/update-deps-to-dist-packages.js ${MATERIAL_REPO_TMP_DIR}/package.json ${angular_dir}/dist/packages-dist-ivy-aot/ # Append the test blocklist into angular/material2's karma-test-shim.js. # This filters out known-failing tests because the goal is to prevent regressions. cat ${angular_dir}/tools/material-ci/angular_material_test_blocklist.js >> ./test/karma-test-shim.js -# Now actually run the tests. -yarn gulp test:single-run +# Create a symlink for the Bazel binary installed through NPM, as running through Yarn introduces OOM errors. +./scripts/circleci/setup_bazel_binary.sh + +# Now actually run the tests. The dev-app target is excluded as it fails to compile due to +# limitations in Ivy's type checker (see FW-1352 and FW-1433) +bazel test src/... --deleted_packages=//src/dev-app --build_tag_filters=-docs-package,-e2e,-browser:firefox-local --test_tag_filters=-e2e,-browser:firefox-local --define=compile=aot diff --git a/scripts/ci/update-deps-to-dist-packages.js b/scripts/ci/update-deps-to-dist-packages.js new file mode 100644 index 0000000000..a64a30a61c --- /dev/null +++ b/scripts/ci/update-deps-to-dist-packages.js @@ -0,0 +1,63 @@ +/** + * This script updates a package.json file by replacing all dependencies and devDependencies + * such that all packages from the @angular scope point to the packages-dist directory. + * + * Please be aware that updating of versions might introduce compatibility issues. For instance, + * if a peer dependency of Angular, e.g. "typescript" changes, the package.json that is updated + * by this script will not have updated the "typescript" dependency to satisfy the peer dependency + * requirement. As a result, incompatibility errors might occur. + */ +'use strict'; + +const {yellow, green} = require('chalk'); +const {existsSync, writeFileSync} = require('fs'); +const {resolve} = require('path'); + +const [, , packageJsonPath, packagesDistRoot] = process.argv; + +const packageJson = require(packageJsonPath); + +const updated = []; +const skipped = []; +function updateDeps(dependencies) { + for (const packageName of Object.keys(dependencies)) { + // We're only interested to update packages in the @angular scope + if (!packageName.startsWith('@angular/')) { + continue; + } + + // Within the packages-dist directory there's no scope name + const packageNameWithoutScope = packageName.replace('@angular/', ''); + const packagePath = resolve(packagesDistRoot, packageNameWithoutScope); + + // Check whether the package exists in packages-dist. Not all packages + // in the @angular scope are published from the main Angular repo. + if (existsSync(packagePath)) { + // Update the dependency to point to the packages-dist location. + dependencies[packageName] = `file:${packagePath}`; + updated.push(packageName); + } else { + skipped.push(packageName); + } + } +} + + +// Update dependencies from @angular scope to those in the packages-dist folder +updateDeps(packageJson.dependencies); +updateDeps(packageJson.devDependencies); + +// Write the updated package.json contents +writeFileSync(packageJsonPath, JSON.stringify(packageJson, null, 2)); + +// Log all packages that were updated +if (updated.length > 0) { + console.log(green(`Updated ${packageJsonPath} to packages in ${packagesDistRoot}:`)); + console.log(` ${updated.join('\n ')}\n`); +} + +// Log the packages that were skipped, as they were not present in the packages-dist directory +if (skipped.length > 0) { + console.log(yellow(`Did not update packages that were not present in ${packagesDistRoot}:`)); + console.log(` ${skipped.join('\n ')}\n`); +}