diff --git a/.circleci/config.yml b/.circleci/config.yml index 0084b57dbf..8e7cb232a5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -63,7 +63,7 @@ jobs: - run: yarn install --frozen-lockfile --non-interactive - run: ./node_modules/.bin/gulp lint - build: + test: <<: *job_defaults resource_class: xlarge steps: @@ -87,10 +87,6 @@ jobs: # NOTE: Angular developers should typically just bazel build //packages/... or bazel test //packages/... - run: bazel query --output=label //... | xargs bazel test - # We run the integration tests outside of Bazel for now. - # See comments inside this script. - - run: xvfb-run --auto-servernum ./integration/run_tests.sh - # CircleCI will allow us to go back and view/download these artifacts from past builds. # Also we can use a service like https://buildsize.org/ to automatically track binary size of these artifacts. - store_artifacts: @@ -112,6 +108,66 @@ jobs: - "node_modules" - "~/bazel_repository_cache" + # This job exists only for backwards-compatibility with old scripts and tests + # that rely on the pre-Bazel dist/packages-dist layout. + # It duplicates some work with the job above: we build the bazel packages + # twice. Even though we have a remote cache, these jobs will typically run in + # parallel so up-to-date outputs will not be available at the time the build + # starts. + # No new jobs should depend on this one. + build-packages-dist: + <<: *job_defaults + resource_class: xlarge + steps: + - *define_env_vars + - checkout: + <<: *post_checkout + # See remote cache documentation in /docs/BAZEL.md + - run: .circleci/setup_cache.sh + - run: sudo cp .circleci/bazel.rc /etc/bazel.bazelrc + - *setup-bazel-remote-cache + + - run: bazel run @yarn//:yarn + - run: scripts/build-packages-dist.sh + + # Save the npm packages from //packages/... for other workflow jobs to read + # https://circleci.com/docs/2.0/workflows/#using-workspaces-to-share-data-among-jobs + - persist_to_workspace: + root: dist + paths: + - packages-dist + + # We run the integration tests outside of Bazel for now. + # They are a separate workflow job so that they can be easily re-run. + # When the tests are ported to bazel test targets, they should move to the "test" + # job above, as part of the bazel test command. That has flaky_test_attempts so the + # need to re-run manually should be alleviated. + # See comments inside the integration/run_tests.sh script. + integration_test: + <<: *job_defaults + steps: + - *define_env_vars + - checkout: + <<: *post_checkout + - attach_workspace: + at: dist + - run: xvfb-run --auto-servernum ./integration/run_tests.sh + + # This job updates the content of repos like github.com/angular/core-builds + # for every green build on angular/angular. + publish_snapshot: + <<: *job_defaults + steps: + - checkout: + <<: *post_checkout + - attach_workspace: + at: dist + # CircleCI has a config setting to force SSH for all github connections + # This is not compatible with our mechanism of using a Personal Access Token + # Clear the global setting + - run: git config --global --unset "url.ssh://git@github.com.insteadof" + - run: ./scripts/ci/publish-build-artifacts.sh + aio_monitoring: <<: *job_defaults steps: @@ -126,7 +182,20 @@ workflows: default_workflow: jobs: - lint - - build + - test + - build-packages-dist + - integration_test: + requires: + - build-packages-dist + - publish_snapshot: + # Note: no filters on this job because we want it to run for all upstream branches + # We'd really like to filter out pull requests here, but not yet available: + # https://discuss.circleci.com/t/workflows-pull-request-filter/14396/4 + # Instead, the publish-build-artifacts.sh script just terminates when + # CIRCLE_PULL_REQUEST is set. + requires: + - build-packages-dist + aio_monitoring: jobs: - aio_monitoring diff --git a/.circleci/github_token b/.circleci/github_token new file mode 100644 index 0000000000..1c80ccfe59 --- /dev/null +++ b/.circleci/github_token @@ -0,0 +1 @@ +Salted__ê÷ûË“]ê×þOʤu'÷–UzhŽ®ìbEÕ]+ÉxCèY-ÿ?ýc"qÒ;ƲK@l#ÒxÞ€IÊ1&w0ç+á\p/Ož; \ No newline at end of file diff --git a/docs/DEVELOPER.md b/docs/DEVELOPER.md index 9c3c69b637..e0120e27c4 100644 --- a/docs/DEVELOPER.md +++ b/docs/DEVELOPER.md @@ -133,11 +133,10 @@ $ gulp lint ## Publishing snapshot builds -When the `master` branch successfully builds on Travis, it automatically publishes build artifacts +When a build of any branch on the upstream fork angular/angular is green on CircleCI, +it automatically publishes build artifacts to repositories in the Angular org, eg. the `@angular/core` package is published to http://github.com/angular/core-builds. -The ES2015 version of Angular is published to a different branch in these repos, for example -http://github.com/angular/core-builds#master-es2015 You may find that your un-merged change needs some validation from external participants. Rather than requiring them to pull your Pull Request and build Angular locally, you can diff --git a/integration/cli-hello-world/package.json b/integration/cli-hello-world/package.json index 27e5180488..c929434fc9 100644 --- a/integration/cli-hello-world/package.json +++ b/integration/cli-hello-world/package.json @@ -5,7 +5,7 @@ "scripts": { "ng": "ng", "start": "ng serve", - "build": "ng build --prod", + "build": "ng build --prod --progress false", "test": "ng test", "lint": "ng lint", "e2e": "ng e2e" diff --git a/integration/run_tests.sh b/integration/run_tests.sh index 8cde03608f..ca35c2d444 100755 --- a/integration/run_tests.sh +++ b/integration/run_tests.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -e -o pipefail +set -u -e -o pipefail # see https://circleci.com/docs/2.0/env-vars/#circleci-built-in-environment-variables CI=${CI:-false} @@ -9,41 +9,21 @@ cd "$(dirname "$0")" # basedir is the workspace root readonly basedir=$(pwd)/.. -readonly bin=$(bazel info bazel-bin) - -echo "#################################" -echo "Building @angular/* npm packages " -echo "#################################" - -# Ideally these integration tests should run under bazel, and just list the npm -# packages in their deps[]. -# Until then, we have to manually run bazel first to create the npm packages we -# want to test. -bazel query --output=label 'kind(.*_package, //packages/...)' \ - | xargs bazel build - -# Allow this test to run even if dist/ doesn't exist yet. -# Under Bazel we don't need to create the dist folder to run the integration tests -[ -d "${basedir}/dist/packages-dist" ] || mkdir -p $basedir/dist/packages-dist -# Each package is a subdirectory of bazel-bin/packages/ -for pkg in $(ls ${bin}/packages); do - # Skip any that don't have an "npm_package" target - if [ -d "${bin}/packages/${pkg}/npm_package" ]; then - echo "# Copy artifacts to dist/packages-dist/${pkg}" - rm -rf ${basedir}/dist/packages-dist/${pkg} - cp -R ${bin}/packages/${pkg}/npm_package ${basedir}/dist/packages-dist/${pkg} - fi -done -chmod -R u+w ${basedir}/dist/packages-dist/ # Track payload size functions -# TODO(alexeagle): finish migrating these to buildsize.org if $CI; then # We don't install this by default because it contains some broken Bazel setup # and also it's a very big dependency that we never use except when publishing # payload sizes on CI. - yarn add -D firebase-tools@3.12.0 + yarn add --silent -D firebase-tools@3.12.0 source ${basedir}/scripts/ci/payload-size.sh + + # NB: we don't run build-packages-dist.sh because we expect that it was done + # by an earlier job in the CircleCI workflow. +else + # Not on CircleCI so let's build the packages-dist directory. + # This should be fast on incremental re-build. + ${basedir}/scripts/build-packages-dist.sh fi # Workaround https://github.com/yarnpkg/yarn/issues/2165 @@ -64,7 +44,7 @@ for testDir in $(ls | grep -v node_modules) ; do ( cd $testDir rm -rf dist - pwd + yarn install --cache-folder ../$cache yarn test || exit 1 # Track payload size for cli-hello-world and hello_world__closure and the render3 tests diff --git a/scripts/build-packages-dist.sh b/scripts/build-packages-dist.sh new file mode 100755 index 0000000000..6ebc5e7b7c --- /dev/null +++ b/scripts/build-packages-dist.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Build the dist/packages-dist directory in the same fashion as the legacy +# /build.sh script, by building the npm packages with Bazel and copying files. +# This is needed for scripts and tests which are not updated to the Bazel output +# layout (which always matches the input layout). +# Do not add new dependencies on this script, instead adapt scripts to use the +# new layout, and write new tests as Bazel targets. + +set -u -e -o pipefail + +cd "$(dirname "$0")" + +# basedir is the workspace root +readonly basedir=$(pwd)/.. + +echo "##################################" +echo "scripts/build-packages-dist.sh:" +echo " building @angular/* npm packages" +echo "##################################" +# Ideally these integration tests should run under bazel, and just list the npm +# packages in their deps[]. +# Until then, we have to manually run bazel first to create the npm packages we +# want to test. +bazel query --output=label 'kind(.*_package, //packages/...)' \ + | xargs bazel build +readonly bin=$(bazel info bazel-bin) + +# Create the legacy dist/packages-dist folder +[ -d "${basedir}/dist/packages-dist" ] || mkdir -p $basedir/dist/packages-dist +# Each package is a subdirectory of bazel-bin/packages/ +for pkg in $(ls ${bin}/packages); do + # Skip any that don't have an "npm_package" target + srcDir="${bin}/packages/${pkg}/npm_package" + destDir="${basedir}/dist/packages-dist/${pkg}" + if [ -d $srcDir ]; then + echo "# Copy artifacts to ${destDir}" + rm -rf $destDir + cp -R $srcDir $destDir + chmod -R u+w $destDir + fi +done diff --git a/scripts/ci/deploy.sh b/scripts/ci/deploy.sh index e88bac203a..ba674a265d 100755 --- a/scripts/ci/deploy.sh +++ b/scripts/ci/deploy.sh @@ -15,12 +15,7 @@ if [[ ${TRAVIS_TEST_RESULT=0} == 1 ]]; then fi -# Don't deploy if not running against angular/angular -# TODO(i): because we don't let deploy to run outside of angular/angular folks can't use their -# private travis build to deploy anywhere. This is likely ok, but this means that @alexeagle's -# fancy setup to publish ES2015 packages to github -build repos no longer works. This is ok -# since with flat modules we'll have this feature built-in. We should still go and remove -# stuff that Alex put in for this from publish-build-artifacts.sh +# Don't deploy Angular.io if we are running in a fork if [[ ${TRAVIS_REPO_SLUG} != "angular/angular" ]]; then echo "Skipping deploy because this is not angular/angular." exit 0 @@ -28,19 +23,6 @@ fi case ${CI_MODE} in - - e2e) - # Don't deploy if this is a PR build - if [[ ${TRAVIS_PULL_REQUEST} != "false" ]]; then - echo "Skipping deploy because this is a PR build." - exit 0 - fi - - travisFoldStart "deploy.packages" - ${thisDir}/publish-build-artifacts.sh - travisFoldEnd "deploy.packages" - ;; - aio) travisFoldStart "deploy.aio" ( diff --git a/scripts/ci/publish-build-artifacts.sh b/scripts/ci/publish-build-artifacts.sh index 9e7c006ee7..5e50ed2090 100755 --- a/scripts/ci/publish-build-artifacts.sh +++ b/scripts/ci/publish-build-artifacts.sh @@ -4,8 +4,6 @@ set -x -u -e -o pipefail # Setup environment readonly thisDir=$(cd $(dirname $0); pwd) -source ${thisDir}/_travis-fold.sh - # Find the most recent tag that is reachable from the current commit. # This is shallow clone of the repo, so we might need to fetch more commits to @@ -63,22 +61,12 @@ function publishRepo { rm -rf $REPO_DIR/* cp -R $ARTIFACTS_DIR/* $REPO_DIR/ - # Replace $$ANGULAR_VERSION$$ with the build version. BUILD_VER="${LATEST_TAG}+${SHORT_SHA}" - if [[ ${TRAVIS} ]]; then - find $REPO_DIR/ -type f -name package.json -print0 | xargs -0 sed -i "s/\\\$\\\$ANGULAR_VERSION\\\$\\\$/${BUILD_VER}/g" - - # Find umd.js and umd.min.js - UMD_FILES=$(find $REPO_DIR/ -type f -name "*.umd*.js" -print) - for UMD_FILE in ${UMD_FILES}; do - sed -i "s/\\\$\\\$ANGULAR_VERSION\\\$\\\$/${BUILD_VER}/g" ${UMD_FILE} - done - + if [[ ${CI} ]]; then ( + # The file ~/.git_credentials is created below cd $REPO_DIR && \ - git config credential.helper "store --file=.git/credentials" && \ - # SECURITY CRITICAL: DO NOT use shell to expand vars since it could be logged and leaked. - node -e "console.log('https://'+process.env.GITHUB_TOKEN_ANGULAR+':@github.com')" > .git/credentials + git config credential.helper "store --file=$HOME/.git_credentials" ) fi echo `date` > $REPO_DIR/BUILD_INFO @@ -130,23 +118,22 @@ function publishPackages { } # See docs/DEVELOPER.md for help -CUR_BRANCH=${TRAVIS_BRANCH:-$(git symbolic-ref --short HEAD)} +CUR_BRANCH=${CIRCLE_BRANCH:-$(git symbolic-ref --short HEAD)} if [ $# -gt 0 ]; then ORG=$1 publishPackages "ssh" dist/packages-dist $CUR_BRANCH - if [[ -e dist/packages-dist-es2015 ]]; then - publishPackages "ssh" dist/packages-dist-es2015 ${CUR_BRANCH}-es2015 - fi elif [[ \ - "$TRAVIS_REPO_SLUG" == "angular/angular" && \ - "$TRAVIS_PULL_REQUEST" == "false" && \ - "$CI_MODE" == "e2e" ]]; then + "$CIRCLE_PROJECT_USERNAME" == "angular" && \ + "$CIRCLE_PROJECT_REPONAME" == "angular" && \ + ! -v CIRCLE_PULL_REQUEST ]]; then ORG="angular" + # $KEY is set on CI only for non-PR builds. See /.circleci/README.md + openssl aes-256-cbc -d -in .circleci/github_token -k "${KEY}" -out "${HOME}/.git_credentials" + publishPackages "http" dist/packages-dist $CUR_BRANCH - if [[ -e dist/packages-dist-es2015 ]]; then - publishPackages "http" dist/packages-dist-es2015 ${CUR_BRANCH}-es2015 - fi + # Clean up the credentials file out of caution + rm "${HOME}/.git_credentials" else echo "Not building the upstream/${CUR_BRANCH} branch, build artifacts won't be published."