ci: publish build snapshots from Bazel/CircleCI (#23512)

This uses a new script and CircleCI job called "build-packages-dist"
which shims the new Bazel build to produce outputs matching the legacy
build. We'll use this to get AIO testing onto CircleCI as well.

We move the integration tests to a new circleCI job that depends on this
one, as well as the build publishing job.

Note that every PR will have a trivial green publishing status, because
we always create this job even for PRs. We'd rather not - see
https://discuss.circleci.com/t/workflows-pull-request-filter/14396/4

PR Close #23512

.circleci/config.yml

@@ -63,7 +63,7 @@ jobs:
       - run: yarn install --frozen-lockfile --non-interactive
       - run: ./node_modules/.bin/gulp lint
 
-  build:
+  test:
     <<: *job_defaults
     resource_class: xlarge
     steps:
@@ -87,10 +87,6 @@ jobs:
       # NOTE: Angular developers should typically just bazel build //packages/... or bazel test //packages/...
       - run: bazel query --output=label //... | xargs bazel test
 
-      # We run the integration tests outside of Bazel for now.
-      # See comments inside this script.
-      - run: xvfb-run --auto-servernum ./integration/run_tests.sh
-
       # CircleCI will allow us to go back and view/download these artifacts from past builds.
       # Also we can use a service like https://buildsize.org/ to automatically track binary size of these artifacts.
       - store_artifacts:
@@ -112,6 +108,66 @@ jobs:
             - "node_modules"
             - "~/bazel_repository_cache"
 
+  # This job exists only for backwards-compatibility with old scripts and tests
+  # that rely on the pre-Bazel dist/packages-dist layout.
+  # It duplicates some work with the job above: we build the bazel packages
+  # twice. Even though we have a remote cache, these jobs will typically run in
+  # parallel so up-to-date outputs will not be available at the time the build
+  # starts.
+  # No new jobs should depend on this one.
+  build-packages-dist:
+    <<: *job_defaults
+    resource_class: xlarge
+    steps:
+      - *define_env_vars
+      - checkout:
+          <<: *post_checkout
+      # See remote cache documentation in /docs/BAZEL.md
+      - run: .circleci/setup_cache.sh
+      - run: sudo cp .circleci/bazel.rc /etc/bazel.bazelrc
+      - *setup-bazel-remote-cache
+
+      - run: bazel run @yarn//:yarn
+      - run: scripts/build-packages-dist.sh
+
+      # Save the npm packages from //packages/... for other workflow jobs to read
+      # https://circleci.com/docs/2.0/workflows/#using-workspaces-to-share-data-among-jobs
+      - persist_to_workspace:
+          root: dist
+          paths:
+            - packages-dist
+
+  # We run the integration tests outside of Bazel for now.
+  # They are a separate workflow job so that they can be easily re-run.
+  # When the tests are ported to bazel test targets, they should move to the "test"
+  # job above, as part of the bazel test command. That has flaky_test_attempts so the
+  # need to re-run manually should be alleviated.
+  # See comments inside the integration/run_tests.sh script.
+  integration_test:
+    <<: *job_defaults
+    steps:
+      - *define_env_vars
+      - checkout:
+          <<: *post_checkout
+      - attach_workspace:
+          at: dist
+      - run: xvfb-run --auto-servernum ./integration/run_tests.sh
+
+  # This job updates the content of repos like github.com/angular/core-builds
+  # for every green build on angular/angular.
+  publish_snapshot:
+    <<: *job_defaults
+    steps:
+      - checkout:
+          <<: *post_checkout
+      - attach_workspace:
+          at: dist
+      # CircleCI has a config setting to force SSH for all github connections
+      # This is not compatible with our mechanism of using a Personal Access Token
+      # Clear the global setting
+      - run: git config --global --unset "url.ssh://git@github.com.insteadof"
+      - run: ./scripts/ci/publish-build-artifacts.sh
+
   aio_monitoring:
     <<: *job_defaults
     steps:
@@ -126,7 +182,20 @@ workflows:
   default_workflow:
     jobs:
       - lint
-      - build
+      - test
+      - build-packages-dist
+      - integration_test:
+          requires:
+            - build-packages-dist
+      - publish_snapshot:
+          # Note: no filters on this job because we want it to run for all upstream branches
+          # We'd really like to filter out pull requests here, but not yet available:
+          # https://discuss.circleci.com/t/workflows-pull-request-filter/14396/4
+          # Instead, the publish-build-artifacts.sh script just terminates when
+          # CIRCLE_PULL_REQUEST is set.
+          requires:
+            - build-packages-dist
+
   aio_monitoring:
     jobs:
       - aio_monitoring

.circleci/github_token

@@ -0,0 +1 @@
+Salted__ê÷û<EFBFBD>Ë“]ê×þO<>ʤu'÷–UzhŽ®ìbEÕ]+ÉxCèY-ÿ?ýc"qÒ;ƲK@l#ÒxÞ€<C39E>IÊ1&w0ç+á\p/Ož;Â

docs/DEVELOPER.md

@@ -133,11 +133,10 @@ $ gulp lint
 
 ## Publishing snapshot builds
 
-When the `master` branch successfully builds on Travis, it automatically publishes build artifacts
+When a build of any branch on the upstream fork angular/angular is green on CircleCI,
+it automatically publishes build artifacts
 to repositories in the Angular org, eg. the `@angular/core` package is published to
 http://github.com/angular/core-builds.
-The ES2015 version of Angular is published to a different branch in these repos, for example
-http://github.com/angular/core-builds#master-es2015
 
 You may find that your un-merged change needs some validation from external participants.
 Rather than requiring them to pull your Pull Request and build Angular locally, you can

integration/cli-hello-world/package.json

@@ -5,7 +5,7 @@
   "scripts": {
     "ng": "ng",
     "start": "ng serve",
-    "build": "ng build --prod",
+    "build": "ng build --prod --progress false",
     "test": "ng test",
     "lint": "ng lint",
     "e2e": "ng e2e"

integration/run_tests.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-set -e -o pipefail
+set -u -e -o pipefail
 
 # see https://circleci.com/docs/2.0/env-vars/#circleci-built-in-environment-variables
 CI=${CI:-false}
@@ -9,41 +9,21 @@ cd "$(dirname "$0")"
 
 # basedir is the workspace root
 readonly basedir=$(pwd)/..
-readonly bin=$(bazel info bazel-bin)
-
-echo "#################################"
-echo "Building @angular/* npm packages "
-echo "#################################"
-
-# Ideally these integration tests should run under bazel, and just list the npm
-# packages in their deps[].
-# Until then, we have to manually run bazel first to create the npm packages we
-# want to test.
-bazel query --output=label 'kind(.*_package, //packages/...)' \
-  | xargs bazel build
-
-# Allow this test to run even if dist/ doesn't exist yet.
-# Under Bazel we don't need to create the dist folder to run the integration tests
-[ -d "${basedir}/dist/packages-dist" ] || mkdir -p $basedir/dist/packages-dist
-# Each package is a subdirectory of bazel-bin/packages/
-for pkg in $(ls ${bin}/packages); do
-  # Skip any that don't have an "npm_package" target
-  if [ -d "${bin}/packages/${pkg}/npm_package" ]; then
-    echo "# Copy artifacts to dist/packages-dist/${pkg}"
-    rm -rf ${basedir}/dist/packages-dist/${pkg}
-    cp -R ${bin}/packages/${pkg}/npm_package ${basedir}/dist/packages-dist/${pkg}
-  fi
-done
-chmod -R u+w ${basedir}/dist/packages-dist/
 
 # Track payload size functions
-# TODO(alexeagle): finish migrating these to buildsize.org
 if $CI; then
   # We don't install this by default because it contains some broken Bazel setup
   # and also it's a very big dependency that we never use except when publishing
   # payload sizes on CI.
-  yarn add -D firebase-tools@3.12.0
+  yarn add --silent -D firebase-tools@3.12.0
   source ${basedir}/scripts/ci/payload-size.sh
+
+  # NB: we don't run build-packages-dist.sh because we expect that it was done
+  # by an earlier job in the CircleCI workflow.
+else
+  # Not on CircleCI so let's build the packages-dist directory.
+  # This should be fast on incremental re-build.
+  ${basedir}/scripts/build-packages-dist.sh
 fi
 
 # Workaround https://github.com/yarnpkg/yarn/issues/2165
@@ -64,7 +44,7 @@ for testDir in $(ls | grep -v node_modules) ; do
   (
     cd $testDir
     rm -rf dist
-    pwd
+
     yarn install --cache-folder ../$cache
     yarn test || exit 1
     # Track payload size for cli-hello-world and hello_world__closure and the render3 tests

scripts/build-packages-dist.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+# Build the dist/packages-dist directory in the same fashion as the legacy
+# /build.sh script, by building the npm packages with Bazel and copying files.
+# This is needed for scripts and tests which are not updated to the Bazel output
+# layout (which always matches the input layout).
+# Do not add new dependencies on this script, instead adapt scripts to use the
+# new layout, and write new tests as Bazel targets.
+
+set -u -e -o pipefail
+
+cd "$(dirname "$0")"
+
+# basedir is the workspace root
+readonly basedir=$(pwd)/..
+
+echo "##################################"
+echo "scripts/build-packages-dist.sh:"
+echo "  building @angular/* npm packages"
+echo "##################################"
+# Ideally these integration tests should run under bazel, and just list the npm
+# packages in their deps[].
+# Until then, we have to manually run bazel first to create the npm packages we
+# want to test.
+bazel query --output=label 'kind(.*_package, //packages/...)' \
+  | xargs bazel build
+readonly bin=$(bazel info bazel-bin)
+
+# Create the legacy dist/packages-dist folder
+[ -d "${basedir}/dist/packages-dist" ] || mkdir -p $basedir/dist/packages-dist
+# Each package is a subdirectory of bazel-bin/packages/
+for pkg in $(ls ${bin}/packages); do
+  # Skip any that don't have an "npm_package" target
+  srcDir="${bin}/packages/${pkg}/npm_package"
+  destDir="${basedir}/dist/packages-dist/${pkg}"
+  if [ -d $srcDir ]; then
+    echo "# Copy artifacts to ${destDir}"
+    rm -rf $destDir
+    cp -R $srcDir $destDir
+    chmod -R u+w $destDir
+  fi
+done

scripts/ci/deploy.sh

@@ -15,12 +15,7 @@ if [[ ${TRAVIS_TEST_RESULT=0} == 1 ]]; then
 fi
 
 
-# Don't deploy if not running against angular/angular
+# Don't deploy Angular.io if we are running in a fork
-# TODO(i): because we don't let deploy to run outside of angular/angular folks can't use their
-#   private travis build to deploy anywhere. This is likely ok, but this means that @alexeagle's
-#   fancy setup to publish ES2015 packages to github -build repos no longer works. This is ok
-#   since with flat modules we'll have this feature built-in. We should still go and remove
-#   stuff that Alex put in for this from publish-build-artifacts.sh
 if [[ ${TRAVIS_REPO_SLUG} != "angular/angular" ]]; then
   echo "Skipping deploy because this is not angular/angular."
   exit 0
@@ -28,19 +23,6 @@ fi
 
 
 case ${CI_MODE} in
-
-  e2e)
-    # Don't deploy if this is a PR build
-    if [[ ${TRAVIS_PULL_REQUEST} != "false" ]]; then
-      echo "Skipping deploy because this is a PR build."
-      exit 0
-    fi
-
-    travisFoldStart "deploy.packages"
-      ${thisDir}/publish-build-artifacts.sh
-    travisFoldEnd "deploy.packages"
-    ;;
-
   aio)
     travisFoldStart "deploy.aio"
     (

scripts/ci/publish-build-artifacts.sh

@@ -4,8 +4,6 @@ set -x -u -e -o pipefail
 
 # Setup environment
 readonly thisDir=$(cd $(dirname $0); pwd)
-source ${thisDir}/_travis-fold.sh
-
 
 # Find the most recent tag that is reachable from the current commit.
 # This is shallow clone of the repo, so we might need to fetch more commits to
@@ -63,22 +61,12 @@ function publishRepo {
   rm -rf $REPO_DIR/*
   cp -R $ARTIFACTS_DIR/* $REPO_DIR/
 
-  # Replace $$ANGULAR_VERSION$$ with the build version.
   BUILD_VER="${LATEST_TAG}+${SHORT_SHA}"
-  if [[ ${TRAVIS} ]]; then
+  if [[ ${CI} ]]; then
-    find $REPO_DIR/ -type f -name package.json -print0 | xargs -0 sed -i "s/\\\$\\\$ANGULAR_VERSION\\\$\\\$/${BUILD_VER}/g"
-
-    # Find umd.js and umd.min.js
-    UMD_FILES=$(find $REPO_DIR/ -type f -name "*.umd*.js" -print)
-    for UMD_FILE in ${UMD_FILES}; do
-      sed -i "s/\\\$\\\$ANGULAR_VERSION\\\$\\\$/${BUILD_VER}/g" ${UMD_FILE}
-    done
-
     (
+      # The file ~/.git_credentials is created below
       cd $REPO_DIR && \
-      git config credential.helper "store --file=.git/credentials" && \
+      git config credential.helper "store --file=$HOME/.git_credentials"
-      # SECURITY CRITICAL: DO NOT use shell to expand vars since it could be logged and leaked.
-      node -e "console.log('https://'+process.env.GITHUB_TOKEN_ANGULAR+':@github.com')" > .git/credentials
     )
   fi
   echo `date` > $REPO_DIR/BUILD_INFO
@@ -130,23 +118,22 @@ function publishPackages {
 }
 
 # See docs/DEVELOPER.md for help
-CUR_BRANCH=${TRAVIS_BRANCH:-$(git symbolic-ref --short HEAD)}
+CUR_BRANCH=${CIRCLE_BRANCH:-$(git symbolic-ref --short HEAD)}
 if [ $# -gt 0 ]; then
   ORG=$1
   publishPackages "ssh" dist/packages-dist $CUR_BRANCH
-  if [[ -e dist/packages-dist-es2015 ]]; then
-    publishPackages "ssh" dist/packages-dist-es2015 ${CUR_BRANCH}-es2015
-  fi
 
 elif [[ \
-    "$TRAVIS_REPO_SLUG" == "angular/angular" && \
+    "$CIRCLE_PROJECT_USERNAME" == "angular" && \
-    "$TRAVIS_PULL_REQUEST" == "false" && \
+    "$CIRCLE_PROJECT_REPONAME" == "angular" && \
-    "$CI_MODE" == "e2e" ]]; then
+    ! -v CIRCLE_PULL_REQUEST ]]; then
   ORG="angular"
+  # $KEY is set on CI only for non-PR builds. See /.circleci/README.md
+  openssl aes-256-cbc -d -in .circleci/github_token -k "${KEY}" -out "${HOME}/.git_credentials"
+
   publishPackages "http" dist/packages-dist $CUR_BRANCH
-  if [[ -e dist/packages-dist-es2015 ]]; then
+  # Clean up the credentials file out of caution
-    publishPackages "http" dist/packages-dist-es2015 ${CUR_BRANCH}-es2015
+  rm "${HOME}/.git_credentials"
-  fi
 
 else
   echo "Not building the upstream/${CUR_BRANCH} branch, build artifacts won't be published."