From cc956744d7c637620e8c46155cc2ea0dc52cb383 Mon Sep 17 00:00:00 2001 From: Joey Perrott Date: Fri, 17 Jan 2020 09:06:48 -0800 Subject: [PATCH] ci: move setup of bazel configurations into the env init scripts (#34834) Since we always set up bazel on both our windows and linux CI runs, we should consider this configuration part of the environment setup. PR Close #34834 --- .circleci/bazel.linux.rc | 4 ++++ .circleci/config.yml | 34 ---------------------------------- .circleci/env.sh | 19 ++++++++++++++++++- .circleci/setup-rbe.sh | 20 -------------------- .circleci/windows-env.ps1 | 12 ++++++++++++ 5 files changed, 34 insertions(+), 55 deletions(-) delete mode 100755 .circleci/setup-rbe.sh diff --git a/.circleci/bazel.linux.rc b/.circleci/bazel.linux.rc index 2088971ef9..4889f934b2 100644 --- a/.circleci/bazel.linux.rc +++ b/.circleci/bazel.linux.rc @@ -15,3 +15,7 @@ build --repository_cache=/home/circleci/bazel_repository_cache # Limit Bazel to consuming resources that fit in CircleCI "xlarge" class # https://circleci.com/docs/2.0/configuration-reference/#resource_class build --local_resources=14336,8.0,1.0 + +# All build executed remotely should be done using our RBE configuration. +build:remote --google_default_credentials +build --config=remote diff --git a/.circleci/config.yml b/.circleci/config.yml index 0770145109..a8dc173d5b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -172,33 +172,11 @@ commands: - *cache_key_win_fallback # Reinstall to get windows binaries. - run: yarn install --frozen-lockfile --non-interactive - - setup_circleci_bazel_config_win # Install @bazel/bazel globally and use that for the first run. # Workaround for https://github.com/bazelbuild/rules_nodejs/issues/894 - run: yarn global add @bazel/bazel@$env:BAZEL_VERSION - run: bazel info - setup_circleci_bazel_config: - description: Set up CircleCI bazel configuration on Linux - steps: - - run: sudo cp .circleci/bazel.linux.rc $HOME/.bazelrc - # We need ensure that the same default digest is used for encoding and decoding - # with openssl. Openssl versions might have different default digests which can - # cause decryption failures based on the openssl version. https://stackoverflow.com/a/39641378/4317734 - - run: openssl aes-256-cbc -d -in .circleci/gcp_token -md md5 -k "$CI_REPO_NAME" -out /home/circleci/.gcp_credentials - - run: echo "export GOOGLE_APPLICATION_CREDENTIALS=/home/circleci/.gcp_credentials" >> $BASH_ENV - - run: ./.circleci/setup-rbe.sh .bazelrc.user - - setup_circleci_bazel_config_win: - description: Set up CircleCI bazel configuration on Windows - steps: - - run: copy .circleci\bazel.windows.rc $env:USERPROFILE\.bazelrc - - run: mkdir $env:APPDATA\gcloud - # We need ensure that the same default digest is used for encoding and decoding - # with openssl. Openssl versions might have different default digests which can - # cause decryption failures based on the openssl version. https://stackoverflow.com/a/39641378/4317734 - - run: openssl aes-256-cbc -d -in .circleci\gcp_token -md md5 -out "$env:APPDATA\gcloud\application_default_credentials.json" -k "$env:CIRCLE_PROJECT_REPONAME" - notify_webhook_on_fail: description: Notify a webhook about failure parameters: @@ -283,7 +261,6 @@ jobs: steps: - custom_attach_workspace - init_environment - - setup_circleci_bazel_config - run: command: yarn bazel test //... --build_tag_filters=-ivy-only --test_tag_filters=-ivy-only no_output_timeout: 20m @@ -296,7 +273,6 @@ jobs: steps: - custom_attach_workspace - init_environment - - setup_circleci_bazel_config # We need to explicitly specify the --symlink_prefix option because otherwise we would # not be able to easily find the output bin directory when uploading artifacts for size # measurements. @@ -338,7 +314,6 @@ jobs: steps: - custom_attach_workspace - init_environment - - setup_circleci_bazel_config - run: name: Run Bazel tests in saucelabs # All web tests are contained within a single //:saucelabs_unit_tests_poc target @@ -367,7 +342,6 @@ jobs: steps: - custom_attach_workspace - init_environment - - setup_circleci_bazel_config - run: name: Run Bazel tests on Saucelabs # Runs the //:saucelabs_tests target with Saucelabs and Ivy. @@ -522,7 +496,6 @@ jobs: steps: - custom_attach_workspace - init_environment - - setup_circleci_bazel_config - run: node scripts/build-packages-dist.js # Save the npm packages from //packages/... for other workflow jobs to read @@ -548,7 +521,6 @@ jobs: steps: - custom_attach_workspace - init_environment - - setup_circleci_bazel_config - run: node scripts/build-ivy-npm-packages.js # Save the npm packages from //packages/... for other workflow jobs to read @@ -729,7 +701,6 @@ jobs: steps: - custom_attach_workspace - init_environment - - setup_circleci_bazel_config # Restore the cache before cloning the repository because the clone script re-uses # the restored repository if present. This reduces the amount of times the components # repository needs to be cloned (this is slow and increases based on commits in the repo). @@ -754,10 +725,6 @@ jobs: # Material directory must be kept in sync with the `$MATERIAL_REPO_TMP_DIR` env variable. # It needs to be hardcoded here, because env variables interpolation is not supported. - "/tmp/material2" - - run: - name: "Setup Bazel RBE remote execution in Material repo" - command: | - ./.circleci/setup-rbe.sh "${MATERIAL_REPO_TMP_DIR}/.bazelrc.user" - run: # Updates Material's `package.json` to refer to the `packages-dist` directory. Note # that it's not necessary to perform a yarn install, as Bazel runs it automatically. @@ -774,7 +741,6 @@ jobs: steps: - custom_attach_workspace - init_environment - - setup_circleci_bazel_config # Install - run: yarn --cwd packages/zone.js install --frozen-lockfile --non-interactive # Run zone.js tools tests diff --git a/.circleci/env.sh b/.circleci/env.sh index 4c16850cd0..f8d12f719c 100755 --- a/.circleci/env.sh +++ b/.circleci/env.sh @@ -77,5 +77,22 @@ setPublicVar MATERIAL_REPO_BRANCH "master" # **NOTE**: When updating the commit SHA, also update the cache key in the CircleCI "config.yml". setPublicVar MATERIAL_REPO_COMMIT "97a7e2babbccd3dc58e7b3364004e45ed5bd9968" -# Source `$BASH_ENV` to make the variables available immediately. + +#################################################################################################### +# Decrypt GCP Credentials and store them as the Google default credentials. +#################################################################################################### +mkdir -p "$HOME/.config/gcloud"; +openssl aes-256-cbc -d -in "${projectDir}/.circleci/gcp_token" \ + -md md5 -k "$CIRCLE_PROJECT_REPONAME" -out "$HOME/.config/gcloud/application_default_credentials.json" +#################################################################################################### +# Set bazel configuration for CircleCI runs. +#################################################################################################### +cp "${projectDir}/.circleci/bazel.linux.rc" "$HOME/.bazelrc"; + +#################################################################################################### +#################################################################################################### +## Source `$BASH_ENV` to make the variables available immediately. ## +## ***NOTE: This must remain the the last action in this script*** ## +#################################################################################################### +#################################################################################################### source $BASH_ENV; diff --git a/.circleci/setup-rbe.sh b/.circleci/setup-rbe.sh deleted file mode 100755 index 07bbc6bd5d..0000000000 --- a/.circleci/setup-rbe.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash -set -u -e -o pipefail - -# The path of the .bazelrc.user file to update should be passed as first parameter to this script. -# This allows to setup RBE for both the Angular repo and the Material repo. -bazelrc_user="$1" - -echo "Writing RBE configuration to ${bazelrc_user}" - -touch ${bazelrc_user} -echo -e 'build --config=remote\n' >> ${bazelrc_user} -echo -e 'build:remote --remote_accept_cached=true\n' >> ${bazelrc_user} -echo "Reading from remote cache for bazel remote jobs." -if [[ "$CI_PULL_REQUEST" == "false" ]]; then - echo -e 'build:remote --remote_upload_local_results=true\n' >> ${bazelrc_user} - echo "Uploading local build results to remote cache." -else - echo -e 'build:remote --remote_upload_local_results=false\n' >> ${bazelrc_user} - echo "Not uploading local build results to remote cache." -fi diff --git a/.circleci/windows-env.ps1 b/.circleci/windows-env.ps1 index 39008b37b4..49ad2cf866 100644 --- a/.circleci/windows-env.ps1 +++ b/.circleci/windows-env.ps1 @@ -27,6 +27,18 @@ Add-Content $profile $bazelVersionGlobalVar git config --global --unset url.ssh://git@github.com.insteadOf +#################################################################################################### +# Decrypt GCP Credentials and store them as the Google default credentials. +#################################################################################################### +mkdir ${env:APPDATA}\gcloud +openssl aes-256-cbc -d -in .circleci\gcp_token -md md5 -out "$env:APPDATA\gcloud\application_default_credentials.json" -k "$env:CIRCLE_PROJECT_REPONAME" + +#################################################################################################### +# Set bazel configuration for CircleCI runs. +#################################################################################################### +copy .circleci\bazel.windows.rc ${Env:USERPROFILE}\.bazelrc + + # These Bazel prereqs aren't needed because the CircleCI image already includes them. # choco install nodejs --version 10.16.0 --no-progress # choco install yarn --version 1.16.0 --no-progress