ci: publish build snapshots from Bazel/CircleCI (#23512)
This uses a new script and CircleCI job called "build-packages-dist" which shims the new Bazel build to produce outputs matching the legacy build. We'll use this to get AIO testing onto CircleCI as well. We move the integration tests to a new circleCI job that depends on this one, as well as the build publishing job. Note that every PR will have a trivial green publishing status, because we always create this job even for PRs. We'd rather not - see https://discuss.circleci.com/t/workflows-pull-request-filter/14396/4 PR Close #23512
This commit is contained in:
Alex Eagle
Victor Berchet
parent
60e5507076
commit
b26ac1c22f
|
|
@ -63,7 +63,7 @@ jobs:
|
|||
- run: yarn install --frozen-lockfile --non-interactive
|
||||
- run: ./node_modules/.bin/gulp lint
|
||||
|
||||
build:
|
||||
test:
|
||||
<<: *job_defaults
|
||||
resource_class: xlarge
|
||||
steps:
|
||||
|
|
@ -87,10 +87,6 @@ jobs:
|
|||
# NOTE: Angular developers should typically just bazel build //packages/... or bazel test //packages/...
|
||||
- run: bazel query --output=label //... | xargs bazel test
|
||||
|
||||
# We run the integration tests outside of Bazel for now.
|
||||
# See comments inside this script.
|
||||
- run: xvfb-run --auto-servernum ./integration/run_tests.sh
|
||||
|
||||
# CircleCI will allow us to go back and view/download these artifacts from past builds.
|
||||
# Also we can use a service like https://buildsize.org/ to automatically track binary size of these artifacts.
|
||||
- store_artifacts:
|
||||
|
|
@ -112,6 +108,66 @@ jobs:
|
|||
- "node_modules"
|
||||
- "~/bazel_repository_cache"
|
||||
|
||||
# This job exists only for backwards-compatibility with old scripts and tests
|
||||
# that rely on the pre-Bazel dist/packages-dist layout.
|
||||
# It duplicates some work with the job above: we build the bazel packages
|
||||
# twice. Even though we have a remote cache, these jobs will typically run in
|
||||
# parallel so up-to-date outputs will not be available at the time the build
|
||||
# starts.
|
||||
# No new jobs should depend on this one.
|
||||
build-packages-dist:
|
||||
<<: *job_defaults
|
||||
resource_class: xlarge
|
||||
steps:
|
||||
- *define_env_vars
|
||||
- checkout:
|
||||
<<: *post_checkout
|
||||
# See remote cache documentation in /docs/BAZEL.md
|
||||
- run: .circleci/setup_cache.sh
|
||||
- run: sudo cp .circleci/bazel.rc /etc/bazel.bazelrc
|
||||
- *setup-bazel-remote-cache
|
||||
|
||||
- run: bazel run @yarn//:yarn
|
||||
- run: scripts/build-packages-dist.sh
|
||||
|
||||
# Save the npm packages from //packages/... for other workflow jobs to read
|
||||
# https://circleci.com/docs/2.0/workflows/#using-workspaces-to-share-data-among-jobs
|
||||
- persist_to_workspace:
|
||||
root: dist
|
||||
paths:
|
||||
- packages-dist
|
||||
|
||||
# We run the integration tests outside of Bazel for now.
|
||||
# They are a separate workflow job so that they can be easily re-run.
|
||||
# When the tests are ported to bazel test targets, they should move to the "test"
|
||||
# job above, as part of the bazel test command. That has flaky_test_attempts so the
|
||||
# need to re-run manually should be alleviated.
|
||||
# See comments inside the integration/run_tests.sh script.
|
||||
integration_test:
|
||||
<<: *job_defaults
|
||||
steps:
|
||||
- *define_env_vars
|
||||
- checkout:
|
||||
<<: *post_checkout
|
||||
- attach_workspace:
|
||||
at: dist
|
||||
- run: xvfb-run --auto-servernum ./integration/run_tests.sh
|
||||
|
||||
# This job updates the content of repos like github.com/angular/core-builds
|
||||
# for every green build on angular/angular.
|
||||
publish_snapshot:
|
||||
<<: *job_defaults
|
||||
steps:
|
||||
- checkout:
|
||||
<<: *post_checkout
|
||||
- attach_workspace:
|
||||
at: dist
|
||||
# CircleCI has a config setting to force SSH for all github connections
|
||||
# This is not compatible with our mechanism of using a Personal Access Token
|
||||
# Clear the global setting
|
||||
- run: git config --global --unset "url.ssh://git@github.com.insteadof"
|
||||
- run: ./scripts/ci/publish-build-artifacts.sh
|
||||
|
||||
aio_monitoring:
|
||||
<<: *job_defaults
|
||||
steps:
|
||||
|
|
@ -126,7 +182,20 @@ workflows:
|
|||
default_workflow:
|
||||
jobs:
|
||||
- lint
|
||||
- build
|
||||
- test
|
||||
- build-packages-dist
|
||||
- integration_test:
|
||||
requires:
|
||||
- build-packages-dist
|
||||
- publish_snapshot:
|
||||
# Note: no filters on this job because we want it to run for all upstream branches
|
||||
# We'd really like to filter out pull requests here, but not yet available:
|
||||
# https://discuss.circleci.com/t/workflows-pull-request-filter/14396/4
|
||||
# Instead, the publish-build-artifacts.sh script just terminates when
|
||||
# CIRCLE_PULL_REQUEST is set.
|
||||
requires:
|
||||
- build-packages-dist
|
||||
|
||||
aio_monitoring:
|
||||
jobs:
|
||||
- aio_monitoring
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
Salted__ê÷û<EFBFBD>Ë“]ê×þO<>ʤu'÷–UzhŽ®ìbEÕ]+ÉxCèY-ÿ?ýc"qÒ;ƲK@l#ÒxÞ€<C39E>IÊ1&w0ç+á\p/Ož;Â
|
||||
|
|
@ -133,11 +133,10 @@ $ gulp lint
|
|||
|
||||
## Publishing snapshot builds
|
||||
|
||||
When the `master` branch successfully builds on Travis, it automatically publishes build artifacts
|
||||
When a build of any branch on the upstream fork angular/angular is green on CircleCI,
|
||||
it automatically publishes build artifacts
|
||||
to repositories in the Angular org, eg. the `@angular/core` package is published to
|
||||
http://github.com/angular/core-builds.
|
||||
The ES2015 version of Angular is published to a different branch in these repos, for example
|
||||
http://github.com/angular/core-builds#master-es2015
|
||||
|
||||
You may find that your un-merged change needs some validation from external participants.
|
||||
Rather than requiring them to pull your Pull Request and build Angular locally, you can
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
"scripts": {
|
||||
"ng": "ng",
|
||||
"start": "ng serve",
|
||||
"build": "ng build --prod",
|
||||
"build": "ng build --prod --progress false",
|
||||
"test": "ng test",
|
||||
"lint": "ng lint",
|
||||
"e2e": "ng e2e"
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
set -e -o pipefail
|
||||
set -u -e -o pipefail
|
||||
|
||||
# see https://circleci.com/docs/2.0/env-vars/#circleci-built-in-environment-variables
|
||||
CI=${CI:-false}
|
||||
|
|
@ -9,41 +9,21 @@ cd "$(dirname "$0")"
|
|||
|
||||
# basedir is the workspace root
|
||||
readonly basedir=$(pwd)/..
|
||||
readonly bin=$(bazel info bazel-bin)
|
||||
|
||||
echo "#################################"
|
||||
echo "Building @angular/* npm packages "
|
||||
echo "#################################"
|
||||
|
||||
# Ideally these integration tests should run under bazel, and just list the npm
|
||||
# packages in their deps[].
|
||||
# Until then, we have to manually run bazel first to create the npm packages we
|
||||
# want to test.
|
||||
bazel query --output=label 'kind(.*_package, //packages/...)' \
|
||||
| xargs bazel build
|
||||
|
||||
# Allow this test to run even if dist/ doesn't exist yet.
|
||||
# Under Bazel we don't need to create the dist folder to run the integration tests
|
||||
[ -d "${basedir}/dist/packages-dist" ] || mkdir -p $basedir/dist/packages-dist
|
||||
# Each package is a subdirectory of bazel-bin/packages/
|
||||
for pkg in $(ls ${bin}/packages); do
|
||||
# Skip any that don't have an "npm_package" target
|
||||
if [ -d "${bin}/packages/${pkg}/npm_package" ]; then
|
||||
echo "# Copy artifacts to dist/packages-dist/${pkg}"
|
||||
rm -rf ${basedir}/dist/packages-dist/${pkg}
|
||||
cp -R ${bin}/packages/${pkg}/npm_package ${basedir}/dist/packages-dist/${pkg}
|
||||
fi
|
||||
done
|
||||
chmod -R u+w ${basedir}/dist/packages-dist/
|
||||
|
||||
# Track payload size functions
|
||||
# TODO(alexeagle): finish migrating these to buildsize.org
|
||||
if $CI; then
|
||||
# We don't install this by default because it contains some broken Bazel setup
|
||||
# and also it's a very big dependency that we never use except when publishing
|
||||
# payload sizes on CI.
|
||||
yarn add -D firebase-tools@3.12.0
|
||||
yarn add --silent -D firebase-tools@3.12.0
|
||||
source ${basedir}/scripts/ci/payload-size.sh
|
||||
|
||||
# NB: we don't run build-packages-dist.sh because we expect that it was done
|
||||
# by an earlier job in the CircleCI workflow.
|
||||
else
|
||||
# Not on CircleCI so let's build the packages-dist directory.
|
||||
# This should be fast on incremental re-build.
|
||||
${basedir}/scripts/build-packages-dist.sh
|
||||
fi
|
||||
|
||||
# Workaround https://github.com/yarnpkg/yarn/issues/2165
|
||||
|
|
@ -64,7 +44,7 @@ for testDir in $(ls | grep -v node_modules) ; do
|
|||
(
|
||||
cd $testDir
|
||||
rm -rf dist
|
||||
pwd
|
||||
|
||||
yarn install --cache-folder ../$cache
|
||||
yarn test || exit 1
|
||||
# Track payload size for cli-hello-world and hello_world__closure and the render3 tests
|
||||
|
|
|
|||
|
|
@ -0,0 +1,41 @@
|
|||
#!/usr/bin/env bash
|
||||
# Build the dist/packages-dist directory in the same fashion as the legacy
|
||||
# /build.sh script, by building the npm packages with Bazel and copying files.
|
||||
# This is needed for scripts and tests which are not updated to the Bazel output
|
||||
# layout (which always matches the input layout).
|
||||
# Do not add new dependencies on this script, instead adapt scripts to use the
|
||||
# new layout, and write new tests as Bazel targets.
|
||||
|
||||
set -u -e -o pipefail
|
||||
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
# basedir is the workspace root
|
||||
readonly basedir=$(pwd)/..
|
||||
|
||||
echo "##################################"
|
||||
echo "scripts/build-packages-dist.sh:"
|
||||
echo " building @angular/* npm packages"
|
||||
echo "##################################"
|
||||
# Ideally these integration tests should run under bazel, and just list the npm
|
||||
# packages in their deps[].
|
||||
# Until then, we have to manually run bazel first to create the npm packages we
|
||||
# want to test.
|
||||
bazel query --output=label 'kind(.*_package, //packages/...)' \
|
||||
| xargs bazel build
|
||||
readonly bin=$(bazel info bazel-bin)
|
||||
|
||||
# Create the legacy dist/packages-dist folder
|
||||
[ -d "${basedir}/dist/packages-dist" ] || mkdir -p $basedir/dist/packages-dist
|
||||
# Each package is a subdirectory of bazel-bin/packages/
|
||||
for pkg in $(ls ${bin}/packages); do
|
||||
# Skip any that don't have an "npm_package" target
|
||||
srcDir="${bin}/packages/${pkg}/npm_package"
|
||||
destDir="${basedir}/dist/packages-dist/${pkg}"
|
||||
if [ -d $srcDir ]; then
|
||||
echo "# Copy artifacts to ${destDir}"
|
||||
rm -rf $destDir
|
||||
cp -R $srcDir $destDir
|
||||
chmod -R u+w $destDir
|
||||
fi
|
||||
done
|
||||
|
|
@ -15,12 +15,7 @@ if [[ ${TRAVIS_TEST_RESULT=0} == 1 ]]; then
|
|||
fi
|
||||
|
||||
|
||||
# Don't deploy if not running against angular/angular
|
||||
# TODO(i): because we don't let deploy to run outside of angular/angular folks can't use their
|
||||
# private travis build to deploy anywhere. This is likely ok, but this means that @alexeagle's
|
||||
# fancy setup to publish ES2015 packages to github -build repos no longer works. This is ok
|
||||
# since with flat modules we'll have this feature built-in. We should still go and remove
|
||||
# stuff that Alex put in for this from publish-build-artifacts.sh
|
||||
# Don't deploy Angular.io if we are running in a fork
|
||||
if [[ ${TRAVIS_REPO_SLUG} != "angular/angular" ]]; then
|
||||
echo "Skipping deploy because this is not angular/angular."
|
||||
exit 0
|
||||
|
|
@ -28,19 +23,6 @@ fi
|
|||
|
||||
|
||||
case ${CI_MODE} in
|
||||
|
||||
e2e)
|
||||
# Don't deploy if this is a PR build
|
||||
if [[ ${TRAVIS_PULL_REQUEST} != "false" ]]; then
|
||||
echo "Skipping deploy because this is a PR build."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
travisFoldStart "deploy.packages"
|
||||
${thisDir}/publish-build-artifacts.sh
|
||||
travisFoldEnd "deploy.packages"
|
||||
;;
|
||||
|
||||
aio)
|
||||
travisFoldStart "deploy.aio"
|
||||
(
|
||||
|
|
|
|||
|
|
@ -4,8 +4,6 @@ set -x -u -e -o pipefail
|
|||
|
||||
# Setup environment
|
||||
readonly thisDir=$(cd $(dirname $0); pwd)
|
||||
source ${thisDir}/_travis-fold.sh
|
||||
|
||||
|
||||
# Find the most recent tag that is reachable from the current commit.
|
||||
# This is shallow clone of the repo, so we might need to fetch more commits to
|
||||
|
|
@ -63,22 +61,12 @@ function publishRepo {
|
|||
rm -rf $REPO_DIR/*
|
||||
cp -R $ARTIFACTS_DIR/* $REPO_DIR/
|
||||
|
||||
# Replace $$ANGULAR_VERSION$$ with the build version.
|
||||
BUILD_VER="${LATEST_TAG}+${SHORT_SHA}"
|
||||
if [[ ${TRAVIS} ]]; then
|
||||
find $REPO_DIR/ -type f -name package.json -print0 | xargs -0 sed -i "s/\\\$\\\$ANGULAR_VERSION\\\$\\\$/${BUILD_VER}/g"
|
||||
|
||||
# Find umd.js and umd.min.js
|
||||
UMD_FILES=$(find $REPO_DIR/ -type f -name "*.umd*.js" -print)
|
||||
for UMD_FILE in ${UMD_FILES}; do
|
||||
sed -i "s/\\\$\\\$ANGULAR_VERSION\\\$\\\$/${BUILD_VER}/g" ${UMD_FILE}
|
||||
done
|
||||
|
||||
if [[ ${CI} ]]; then
|
||||
(
|
||||
# The file ~/.git_credentials is created below
|
||||
cd $REPO_DIR && \
|
||||
git config credential.helper "store --file=.git/credentials" && \
|
||||
# SECURITY CRITICAL: DO NOT use shell to expand vars since it could be logged and leaked.
|
||||
node -e "console.log('https://'+process.env.GITHUB_TOKEN_ANGULAR+':@github.com')" > .git/credentials
|
||||
git config credential.helper "store --file=$HOME/.git_credentials"
|
||||
)
|
||||
fi
|
||||
echo `date` > $REPO_DIR/BUILD_INFO
|
||||
|
|
@ -130,23 +118,22 @@ function publishPackages {
|
|||
}
|
||||
|
||||
# See docs/DEVELOPER.md for help
|
||||
CUR_BRANCH=${TRAVIS_BRANCH:-$(git symbolic-ref --short HEAD)}
|
||||
CUR_BRANCH=${CIRCLE_BRANCH:-$(git symbolic-ref --short HEAD)}
|
||||
if [ $# -gt 0 ]; then
|
||||
ORG=$1
|
||||
publishPackages "ssh" dist/packages-dist $CUR_BRANCH
|
||||
if [[ -e dist/packages-dist-es2015 ]]; then
|
||||
publishPackages "ssh" dist/packages-dist-es2015 ${CUR_BRANCH}-es2015
|
||||
fi
|
||||
|
||||
elif [[ \
|
||||
"$TRAVIS_REPO_SLUG" == "angular/angular" && \
|
||||
"$TRAVIS_PULL_REQUEST" == "false" && \
|
||||
"$CI_MODE" == "e2e" ]]; then
|
||||
"$CIRCLE_PROJECT_USERNAME" == "angular" && \
|
||||
"$CIRCLE_PROJECT_REPONAME" == "angular" && \
|
||||
! -v CIRCLE_PULL_REQUEST ]]; then
|
||||
ORG="angular"
|
||||
# $KEY is set on CI only for non-PR builds. See /.circleci/README.md
|
||||
openssl aes-256-cbc -d -in .circleci/github_token -k "${KEY}" -out "${HOME}/.git_credentials"
|
||||
|
||||
publishPackages "http" dist/packages-dist $CUR_BRANCH
|
||||
if [[ -e dist/packages-dist-es2015 ]]; then
|
||||
publishPackages "http" dist/packages-dist-es2015 ${CUR_BRANCH}-es2015
|
||||
fi
|
||||
# Clean up the credentials file out of caution
|
||||
rm "${HOME}/.git_credentials"
|
||||
|
||||
else
|
||||
echo "Not building the upstream/${CUR_BRANCH} branch, build artifacts won't be published."
|
||||
|
|
|
|||
Loading…
Reference in New Issue