build: ensure that refs and shas for PRs only need to be requested once (#36207)
This is done by requesting the refs and shas for the PR when the env.sh script is run. Additionally, the env.sh script is now setup to write all of the environment variables created to a cache file and subsequent loads of the environment load the values from there. The get-refs-and-shas-for-target.js script now also first attempts to load the refs and shas from an environment variable before falling back to requesting from github via the API. PR Close #36207
This commit is contained in:
Joey Perrott
Misko Hevery
parent
22710fc353
commit
d37dad82f1
|
|
@ -226,6 +226,7 @@ jobs:
|
|||
executor: default-executor
|
||||
steps:
|
||||
- checkout
|
||||
- init_environment
|
||||
- run:
|
||||
name: Rebase PR on target branch
|
||||
# After checkout, rebase on top of target branch.
|
||||
|
|
@ -244,7 +245,6 @@ jobs:
|
|||
keys:
|
||||
- *cache_key
|
||||
- *cache_key_fallback
|
||||
- init_environment
|
||||
- run:
|
||||
name: Running Yarn install
|
||||
command: yarn install --frozen-lockfile --non-interactive
|
||||
|
|
|
|||
125
.circleci/env.sh
125
.circleci/env.sh
|
|
@ -3,72 +3,87 @@
|
|||
# Variables
|
||||
readonly projectDir=$(realpath "$(dirname ${BASH_SOURCE[0]})/..")
|
||||
readonly envHelpersPath="$projectDir/.circleci/env-helpers.inc.sh";
|
||||
readonly bashEnvCachePath="$projectDir/.circleci/bash_env_cache";
|
||||
|
||||
# Load helpers and make them available everywhere (through `$BASH_ENV`).
|
||||
source $envHelpersPath;
|
||||
echo "source $envHelpersPath;" >> $BASH_ENV;
|
||||
if [ -f $bashEnvCachePath ]; then
|
||||
# Since a bash env cache is present, load this into the $BASH_ENV
|
||||
cat "$bashEnvCachePath" >> $BASH_ENV;
|
||||
echo "BASH environment loaded from cached value at $bashEnvCachePath";
|
||||
else
|
||||
# Since no bash env cache is present, build out $BASH_ENV values.
|
||||
|
||||
# Load helpers and make them available everywhere (through `$BASH_ENV`).
|
||||
source $envHelpersPath;
|
||||
echo "source $envHelpersPath;" >> $BASH_ENV;
|
||||
|
||||
|
||||
####################################################################################################
|
||||
# Define PUBLIC environment variables for CircleCI.
|
||||
####################################################################################################
|
||||
# See https://circleci.com/docs/2.0/env-vars/#built-in-environment-variables for more info.
|
||||
####################################################################################################
|
||||
setPublicVar PROJECT_ROOT "$projectDir";
|
||||
setPublicVar CI_AIO_MIN_PWA_SCORE "95";
|
||||
# This is the branch being built; e.g. `pull/12345` for PR builds.
|
||||
setPublicVar CI_BRANCH "$CIRCLE_BRANCH";
|
||||
setPublicVar CI_BUILD_URL "$CIRCLE_BUILD_URL";
|
||||
setPublicVar CI_COMMIT "$CIRCLE_SHA1";
|
||||
# `CI_COMMIT_RANGE` is only used on push builds (a.k.a. non-PR, non-scheduled builds and rerun
|
||||
# workflows of such builds).
|
||||
setPublicVar CI_COMMIT_RANGE "$CIRCLE_GIT_BASE_REVISION..$CIRCLE_GIT_REVISION";
|
||||
setPublicVar CI_PULL_REQUEST "${CIRCLE_PR_NUMBER:-false}";
|
||||
setPublicVar CI_REPO_NAME "$CIRCLE_PROJECT_REPONAME";
|
||||
setPublicVar CI_REPO_OWNER "$CIRCLE_PROJECT_USERNAME";
|
||||
####################################################################################################
|
||||
# Define PUBLIC environment variables for CircleCI.
|
||||
####################################################################################################
|
||||
# See https://circleci.com/docs/2.0/env-vars/#built-in-environment-variables for more info.
|
||||
####################################################################################################
|
||||
setPublicVar PROJECT_ROOT "$projectDir";
|
||||
setPublicVar CI_AIO_MIN_PWA_SCORE "95";
|
||||
# This is the branch being built; e.g. `pull/12345` for PR builds.
|
||||
setPublicVar CI_BRANCH "$CIRCLE_BRANCH";
|
||||
setPublicVar CI_BUILD_URL "$CIRCLE_BUILD_URL";
|
||||
setPublicVar CI_COMMIT "$CIRCLE_SHA1";
|
||||
# `CI_COMMIT_RANGE` is only used on push builds (a.k.a. non-PR, non-scheduled builds and rerun
|
||||
# workflows of such builds).
|
||||
setPublicVar CI_COMMIT_RANGE "$CIRCLE_GIT_BASE_REVISION..$CIRCLE_GIT_REVISION";
|
||||
setPublicVar CI_PULL_REQUEST "${CIRCLE_PR_NUMBER:-false}";
|
||||
setPublicVar CI_REPO_NAME "$CIRCLE_PROJECT_REPONAME";
|
||||
setPublicVar CI_REPO_OWNER "$CIRCLE_PROJECT_USERNAME";
|
||||
|
||||
# Store a PR's refs and shas so they don't need to be requested multiple times.
|
||||
setPublicVar GITHUB_REFS_AND_SHAS $(node tools/utils/get-refs-and-shas-for-target.js ${CIRCLE_PR_NUMBER:-false} | awk '{ gsub(/"/,"\\\"") } 1');
|
||||
|
||||
|
||||
####################################################################################################
|
||||
# Define "lazy" PUBLIC environment variables for CircleCI.
|
||||
# (I.e. functions to set an environment variable when called.)
|
||||
####################################################################################################
|
||||
createPublicVarSetter CI_STABLE_BRANCH "\$(npm info @angular/core dist-tags.latest | sed -r 's/^\\s*([0-9]+\\.[0-9]+)\\.[0-9]+.*$/\\1.x/')";
|
||||
####################################################################################################
|
||||
# Define "lazy" PUBLIC environment variables for CircleCI.
|
||||
# (I.e. functions to set an environment variable when called.)
|
||||
####################################################################################################
|
||||
createPublicVarSetter CI_STABLE_BRANCH "\$(npm info @angular/core dist-tags.latest | sed -r 's/^\\s*([0-9]+\\.[0-9]+)\\.[0-9]+.*$/\\1.x/')";
|
||||
|
||||
|
||||
####################################################################################################
|
||||
# Define SECRET environment variables for CircleCI.
|
||||
####################################################################################################
|
||||
setSecretVar CI_SECRET_AIO_DEPLOY_FIREBASE_TOKEN "$AIO_DEPLOY_TOKEN";
|
||||
setSecretVar CI_SECRET_PAYLOAD_FIREBASE_TOKEN "$ANGULAR_PAYLOAD_TOKEN";
|
||||
####################################################################################################
|
||||
# Define SECRET environment variables for CircleCI.
|
||||
####################################################################################################
|
||||
setSecretVar CI_SECRET_AIO_DEPLOY_FIREBASE_TOKEN "$AIO_DEPLOY_TOKEN";
|
||||
setSecretVar CI_SECRET_PAYLOAD_FIREBASE_TOKEN "$ANGULAR_PAYLOAD_TOKEN";
|
||||
|
||||
|
||||
####################################################################################################
|
||||
# Define SauceLabs environment variables for CircleCI.
|
||||
####################################################################################################
|
||||
setPublicVar SAUCE_USERNAME "angular-framework";
|
||||
setSecretVar SAUCE_ACCESS_KEY "0c731274ed5f-cbc9-16f4-021a-9835e39f";
|
||||
# TODO(josephperrott): Remove environment variables once all saucelabs tests are via bazel method.
|
||||
setPublicVar SAUCE_LOG_FILE /tmp/angular/sauce-connect.log
|
||||
setPublicVar SAUCE_READY_FILE /tmp/angular/sauce-connect-ready-file.lock
|
||||
setPublicVar SAUCE_PID_FILE /tmp/angular/sauce-connect-pid-file.lock
|
||||
setPublicVar SAUCE_TUNNEL_IDENTIFIER "angular-framework-${CIRCLE_BUILD_NUM}-${CIRCLE_NODE_INDEX}"
|
||||
# Amount of seconds we wait for sauceconnect to establish a tunnel instance. In order to not
|
||||
# acquire CircleCI instances for too long if sauceconnect failed, we need a connect timeout.
|
||||
setPublicVar SAUCE_READY_FILE_TIMEOUT 120
|
||||
####################################################################################################
|
||||
# Define SauceLabs environment variables for CircleCI.
|
||||
####################################################################################################
|
||||
setPublicVar SAUCE_USERNAME "angular-framework";
|
||||
setSecretVar SAUCE_ACCESS_KEY "0c731274ed5f-cbc9-16f4-021a-9835e39f";
|
||||
# TODO(josephperrott): Remove environment variables once all saucelabs tests are via bazel method.
|
||||
setPublicVar SAUCE_LOG_FILE /tmp/angular/sauce-connect.log
|
||||
setPublicVar SAUCE_READY_FILE /tmp/angular/sauce-connect-ready-file.lock
|
||||
setPublicVar SAUCE_PID_FILE /tmp/angular/sauce-connect-pid-file.lock
|
||||
setPublicVar SAUCE_TUNNEL_IDENTIFIER "angular-framework-${CIRCLE_BUILD_NUM}-${CIRCLE_NODE_INDEX}"
|
||||
# Amount of seconds we wait for sauceconnect to establish a tunnel instance. In order to not
|
||||
# acquire CircleCI instances for too long if sauceconnect failed, we need a connect timeout.
|
||||
setPublicVar SAUCE_READY_FILE_TIMEOUT 120
|
||||
|
||||
|
||||
####################################################################################################
|
||||
# Define environment variables for the `angular/components` repo unit tests job.
|
||||
####################################################################################################
|
||||
# We specifically use a directory within "/tmp" here because we want the cloned repo to be
|
||||
# completely isolated from angular/angular in order to avoid any bad interactions between
|
||||
# their separate build setups. **NOTE**: When updating the temporary directory, also update
|
||||
# the `save_cache` path configuration in `config.yml`
|
||||
setPublicVar COMPONENTS_REPO_TMP_DIR "/tmp/angular-components-repo"
|
||||
setPublicVar COMPONENTS_REPO_URL "https://github.com/angular/components.git"
|
||||
setPublicVar COMPONENTS_REPO_BRANCH "master"
|
||||
# **NOTE**: When updating the commit SHA, also update the cache key in the CircleCI `config.yml`.
|
||||
setPublicVar COMPONENTS_REPO_COMMIT "598db096e668aa7e9debd56eedfd127b7a55e371"
|
||||
####################################################################################################
|
||||
# Define environment variables for the `angular/components` repo unit tests job.
|
||||
####################################################################################################
|
||||
# We specifically use a directory within "/tmp" here because we want the cloned repo to be
|
||||
# completely isolated from angular/angular in order to avoid any bad interactions between
|
||||
# their separate build setups. **NOTE**: When updating the temporary directory, also update
|
||||
# the `save_cache` path configuration in `config.yml`
|
||||
setPublicVar COMPONENTS_REPO_TMP_DIR "/tmp/angular-components-repo"
|
||||
setPublicVar COMPONENTS_REPO_URL "https://github.com/angular/components.git"
|
||||
setPublicVar COMPONENTS_REPO_BRANCH "master"
|
||||
# **NOTE**: When updating the commit SHA, also update the cache key in the CircleCI `config.yml`.
|
||||
setPublicVar COMPONENTS_REPO_COMMIT "598db096e668aa7e9debd56eedfd127b7a55e371"
|
||||
|
||||
# Save the created BASH_ENV into the bash env cache file.
|
||||
cat "$BASH_ENV" >> $bashEnvCachePath;
|
||||
fi
|
||||
|
||||
|
||||
####################################################################################################
|
||||
|
|
|
|||
|
|
@ -6,6 +6,11 @@
|
|||
* found in the LICENSE file at https://angular.io/license
|
||||
*/
|
||||
|
||||
// NOTE: When invoked directly via node, this script will take the first positional
|
||||
// arguement as to be the PR number, and log out the ref and sha information in its
|
||||
// JSON format. For other usages, the function to get the ref and sha information
|
||||
// may be imported by another script to be invoked.
|
||||
|
||||
// This script uses `console` to print messages to the user.
|
||||
// tslint:disable:no-console
|
||||
|
||||
|
|
@ -16,7 +21,18 @@ const exec = util.promisify(child_process.exec);
|
|||
|
||||
async function requestDataFromGithub(url) {
|
||||
// GitHub requires a user agent: https://developer.github.com/v3/#user-agent-required
|
||||
const options = {headers: {'User-Agent': 'angular'}};
|
||||
let options = {headers: {'User-Agent': 'angular'}};
|
||||
|
||||
// If a github token is present, use it for authorization.
|
||||
const githubToken = process.env.TOKEN || process.env.GITHUB_TOKEN || '';
|
||||
if (githubToken) {
|
||||
options = {
|
||||
headers: {
|
||||
Authorization: `token ${githubToken}`,
|
||||
...options.headers,
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
https
|
||||
|
|
@ -54,9 +70,19 @@ async function requestDataFromGithub(url) {
|
|||
.on('error', (e) => { reject(e); });
|
||||
});
|
||||
}
|
||||
// clang-format off
|
||||
// clang keeps trying to put the function name on the next line.
|
||||
async function getRefsAndShasForTarget(prNumber, suppressLog) {
|
||||
// clang-format on
|
||||
// If the environment variable already contains the refs and shas, reuse them.
|
||||
if (process.env['GITHUB_REFS_AND_SHAS']) {
|
||||
suppressLog ||
|
||||
console.info(`Retrieved refs and SHAs for PR ${prNumber} from environment variables.`);
|
||||
return JSON.parse(process.env['GITHUB_REFS_AND_SHAS']);
|
||||
}
|
||||
|
||||
module.exports = async function getRefsAndShasForTarget(prNumber) {
|
||||
console.log(`Getting refs and SHAs for PR ${prNumber} on angular/angular.`);
|
||||
suppressLog ||
|
||||
console.info(`Getting refs and SHAs for PR ${prNumber} on angular/angular from Github.`);
|
||||
const pullsUrl = `https://api.github.com/repos/angular/angular/pulls/${prNumber}`;
|
||||
const result = await requestDataFromGithub(pullsUrl);
|
||||
|
||||
|
|
@ -84,6 +110,19 @@ module.exports = async function getRefsAndShasForTarget(prNumber) {
|
|||
latestShaOfTargetBranch: latestShaOfTargetBranch.trim(),
|
||||
latestShaOfPrBranch: latestShaOfPrBranch.trim(),
|
||||
};
|
||||
|
||||
return output;
|
||||
};
|
||||
}
|
||||
|
||||
// If the script is called directly, log the output of the refs and sha for the
|
||||
// requested PR.
|
||||
if (require.main === module) {
|
||||
const run = async() => {
|
||||
const prNumber = Number.parseInt(process.argv[2], 10);
|
||||
if (!!prNumber) {
|
||||
console.info(JSON.stringify(await getRefsAndShasForTarget(prNumber, true)));
|
||||
}
|
||||
};
|
||||
run();
|
||||
}
|
||||
|
||||
module.exports = getRefsAndShasForTarget;
|
||||
|
|
|
|||
Loading…
Reference in New Issue