From 418e9cf3c43140db9e53e9e6677365a23b8240f2 Mon Sep 17 00:00:00 2001 From: Joey Perrott Date: Thu, 17 Oct 2019 12:04:06 -0700 Subject: [PATCH] build: address comments, restructure setup-rbe.sh script (#33109) PR Close #33109 --- .bazelrc | 4 +- docs/DEVELOPER.md | 6 ++- scripts/local-dev/get-email | 24 ++++++++++ scripts/local-dev/setup-rbe.sh | 82 +++++++++++++++++++++++----------- 4 files changed, 86 insertions(+), 30 deletions(-) create mode 100644 scripts/local-dev/get-email diff --git a/.bazelrc b/.bazelrc index 3c5de7486f..e19e3fb688 100644 --- a/.bazelrc +++ b/.bazelrc @@ -71,8 +71,6 @@ test --test_output=errors # any bazel target. This is a temporary flag until codebase is permanently switched to Ivy. build --define=compile=legacy -build --google_default_credentials - ####################### # Remote HTTP Caching # ####################### @@ -86,6 +84,7 @@ build --remote_upload_local_results=false # --config=-http-caching # ###################################### build:remote-http-caching --remote_upload_local_results=true +build:remote-http-caching --google_default_credentials ################################## # Remote Build Execution support # @@ -99,6 +98,7 @@ import %workspace%/third_party/github.com/bazelbuild/bazel-toolchains/bazelrc/.b # Increase the default number of jobs by 50% because our build has lots of # parallelism build:remote --jobs=150 +build:remote --google_default_credentials # Toolchain and platform related flags build:remote --host_javabase=@rbe_ubuntu1604_angular//java:jdk diff --git a/docs/DEVELOPER.md b/docs/DEVELOPER.md index 2d3be7cb07..6803a6e07e 100644 --- a/docs/DEVELOPER.md +++ b/docs/DEVELOPER.md @@ -169,12 +169,14 @@ It will automatically recognize `*.bazel` and `*.bzl` files. ### Remote Build Execution and Remote Caching Bazel builds in the Angular repository use a shared http cache. When a build occurs a hash of the inputs is computed and checked against available outputs in the shared http cache. If an output is found, it is used as the output for the -build action rather than performing the build locally. +build action rather than performing the build locally. + +> Remote Build Execution and uploading to the Remote Cache requires authentication as a google.com or angular.io account. ### --config=remote-http-caching flag The `--config=remote-http-caching` flag can be added to enable uploading of build results to the shared http cache. This flag can be added to the `.bazelrc.user` file using the script at `scripts/local-dev/setup-rbe.sh`. ### --config=remote flag -The `--config=remote-http-caching` flag can be added to enable remote execution of builds. This flag can be added to +The `--config=remote` flag can be added to enable remote execution of builds. This flag can be added to the `.bazelrc.user` file using the script at `scripts/local-dev/setup-rbe.sh`. \ No newline at end of file diff --git a/scripts/local-dev/get-email b/scripts/local-dev/get-email new file mode 100644 index 0000000000..ba678cb006 --- /dev/null +++ b/scripts/local-dev/get-email @@ -0,0 +1,24 @@ +#!/usr/bin/env node + +/** + * Retrieves the email value from a json object. This assumes + * the email attributes is at the base of the object, as returned + * by Google Cloud's tokeninfo api. + */ + +// Read information being piped in. +var stdin = process.openStdin(); +// Stored data stream. +var data = ""; + +// Store each chunk of the stream in data. +stdin.on('data', chunk => data += chunk); + +// After stream ends, parse data and get value requested. +stdin.on('end', () => { + // The JSON object, to be accessed. + let output = JSON.parse(data); + + // Print the output to STDOUT. + console.log(output['email']); +}); diff --git a/scripts/local-dev/setup-rbe.sh b/scripts/local-dev/setup-rbe.sh index 5210460d4b..22a1f16daa 100755 --- a/scripts/local-dev/setup-rbe.sh +++ b/scripts/local-dev/setup-rbe.sh @@ -1,11 +1,15 @@ #!/bin/bash # A script for automatically configuring a user's local dev # environment to use Remote Build Execution. - # Short cuts to set output as bold and normal bold=$(tput bold) normal=$(tput sgr0) +########################################################### +# Setup/Confirm Environment # +########################################################### +# The full path location of the script +full_script_path="$(pwd)/$(dirname ${BASH_SOURCE[0]})" # Determine the root directory of the Angular github repo. project_directory=$(git rev-parse --show-toplevel 2> /dev/null) if [[ $? -ne 0 ]]; then @@ -20,32 +24,49 @@ if [ ! -x "$(command -v gcloud)" ]; then exit 1 fi -# Confirm the user is already logged into gcloud, if they aren't -# attempt to login -echo "Checking gcloud login state" -gcloud auth application-default print-access-token &> /dev/null -if [[ $? -ne 0 ]]; then - echo "Not currently logged into gcloud. Starting gcloud login now" +# The full path to the .bazelrc.user file +bazelrc_user_filepath="$project_directory/.bazelrc.user" + +########################################################### +# Action Functions # +########################################################### +# Log into gcloud +function gcloud_login() { gcloud auth application-default login if [[ $? -ne 0 ]]; then echo "gcloud login failed. Aborting" exit 2 fi -fi -access_token=$(gcloud auth application-default print-access-token) -current_account=$(curl -s https://www.googleapis.com/oauth2/v1/tokeninfo?access_token=$access_token | jq -r '.email') -if [[ ! $current_account =~ (angular\.io$)|(google\.com$) ]]; then - echo "Currently an angular.io or google.com account must be used for remote Bazel usage" - echo "Please login instead using the correct account with the following command, then rerun" - echo " gcloud auth application-default login" - exit 3 -fi -echo "Logged in as $current_account"; +} -# The full path to the .bazelrc.user file -bazelrc_user_filepath="$project_directory/.bazelrc.user" -# Create the bazelrc.user file, echo the config flags into the file. -touch $bazelrc_user_filepath +# Confirm the user is already logged into gcloud, if they aren't +# attempt to login. After login, confirm the logged in account +# is from the correct domain. +function confirm_gcloud_login() { + echo "Checking gcloud login state" + gcloud auth application-default print-access-token &> /dev/null + if [[ $? -ne 0 ]]; then + echo "Not currently logged into gcloud. Starting gcloud login now" + gcloud_login + fi + access_token=$(gcloud auth application-default print-access-token) + current_account=$(curl -s https://www.googleapis.com/oauth2/v1/tokeninfo?access_token=$access_token | node $full_script_path/get-email) + if [[ ! $current_account =~ (angular\.io$)|(google\.com$) ]]; then + echo "Logged in as $current_account"; + echo "An angular.io or google.com account must be used for remote Bazel usage." + echo "Please login instead using an account from one of these domains." + read -p "Rerun login command now? [Y/y]" + if [[ $REPLY =~ ^[Yy]$ ]]; then + gcloud_login + confirm_gcloud_login + return + else + echo "Exiting..." + exit 3 + fi + fi + echo "Logged in as $current_account"; +} # Prompts to add a flag to the .bazelrc.user file if its not already in place function add_flag() { @@ -62,14 +83,23 @@ function add_flag() { echo } +########################################################### +# RBE Setup Script # +########################################################### +# Create the bazelrc.user file, echo the config flags into the file. +touch $bazelrc_user_filepath + +# Ensure default credentials are valid. +confirm_gcloud_login + # Add extra line space before config setup. echo +# Remote builds +echo "The ${bold}remote${normal} flag enables RBE, builds run remotely when possible and caching" +echo "occurs in the RBE context" +add_flag "build --config=remote" + # Remote HTTP Caching echo "The ${bold}remote-http-caching${normal} flag enables uploading build results to the http cache," echo "but not does enable remote builds" add_flag "build --config=remote-http-caching" - -# Remote builds -echo "The ${bold}remote${normal} flag enables RBE, builds occurs remotely when possible and caching" -echo "occurs in the RBE context" -add_flag "build --config=remote"