ci: remove reliance on Github API for CI setup (#36500)
Previously our CI during the setup process has made requests to the Github API to determine the target branch and shas. With this change, this information is now determined via git commands using pipeline parameters from CircleCI. PR Close #36500
This commit is contained in:
Joey Perrott
Matias Niemelä
parent
5e79799b89
commit
4480ba3e29
|
|
@ -236,7 +236,7 @@ jobs:
|
|||
git config user.name "angular-ci"
|
||||
git config user.email "angular-ci"
|
||||
# Rebase PR on top of target branch.
|
||||
node tools/rebase-pr.js angular/angular ${CIRCLE_PR_NUMBER}
|
||||
node tools/rebase-pr.js
|
||||
else
|
||||
echo "This build is not over a PR, nothing to do."
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ else
|
|||
####################################################################################################
|
||||
# See https://circleci.com/docs/2.0/env-vars/#built-in-environment-variables for more info.
|
||||
####################################################################################################
|
||||
setPublicVar CI "$CI"
|
||||
setPublicVar PROJECT_ROOT "$projectDir";
|
||||
setPublicVar CI_AIO_MIN_PWA_SCORE "95";
|
||||
# This is the branch being built; e.g. `pull/12345` for PR builds.
|
||||
|
|
@ -36,9 +37,8 @@ else
|
|||
setPublicVar CI_PULL_REQUEST "${CIRCLE_PR_NUMBER:-false}";
|
||||
setPublicVar CI_REPO_NAME "$CIRCLE_PROJECT_REPONAME";
|
||||
setPublicVar CI_REPO_OWNER "$CIRCLE_PROJECT_USERNAME";
|
||||
|
||||
# Store a PR's refs and shas so they don't need to be requested multiple times.
|
||||
setPublicVar GITHUB_REFS_AND_SHAS $(node tools/utils/get-refs-and-shas-for-target.js ${CIRCLE_PR_NUMBER:-false} | awk '{ gsub(/"/,"\\\"") } 1');
|
||||
setPublicVar CI_PR_REPONAME "$CIRCLE_PR_REPONAME";
|
||||
setPublicVar CI_PR_USERNAME "$CIRCLE_PR_USERNAME";
|
||||
|
||||
|
||||
####################################################################################################
|
||||
|
|
@ -82,7 +82,7 @@ else
|
|||
setPublicVar COMPONENTS_REPO_BRANCH "master"
|
||||
# **NOTE**: When updating the commit SHA, also update the cache key in the CircleCI `config.yml`.
|
||||
setPublicVar COMPONENTS_REPO_COMMIT "598db096e668aa7e9debd56eedfd127b7a55e371"
|
||||
|
||||
|
||||
# Save the created BASH_ENV into the bash env cache file.
|
||||
cat "$BASH_ENV" >> $bashEnvCachePath;
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -40,33 +40,26 @@
|
|||
const util = require('util');
|
||||
const child_process = require('child_process');
|
||||
const exec = util.promisify(child_process.exec);
|
||||
const getRefsAndShasForTarget = require('./utils/get-refs-and-shas-for-target');
|
||||
const getRefsAndShasForChange = require('./utils/git-get-changeset-refs');
|
||||
|
||||
// CLI validation
|
||||
if (process.argv.length != 4) {
|
||||
console.error(`This script requires the GitHub repository and PR number as arguments.`);
|
||||
console.error(`Example: node tools/rebase-pr.js angular/angular 123`);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
// Run
|
||||
_main(...process.argv.slice(2)).catch(err => {
|
||||
_main().catch(err => {
|
||||
console.log('Failed to rebase on top of target branch.\n');
|
||||
console.error(err);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
|
||||
// Helpers
|
||||
async function _main(repository, prNumber) {
|
||||
const target = await getRefsAndShasForTarget(prNumber);
|
||||
async function _main() {
|
||||
const refs = await getRefsAndShasForChange();
|
||||
|
||||
// Log known refs and shas
|
||||
console.log(`--------------------------------`);
|
||||
console.log(` Target Branch: ${target.base.ref}`);
|
||||
console.log(` Latest Commit for Target Branch: ${target.latestShaOfTargetBranch}`);
|
||||
console.log(` Latest Commit for PR: ${target.latestShaOfPrBranch}`);
|
||||
console.log(` First Common Ancestor SHA: ${target.commonAncestorSha}`);
|
||||
console.log(` Target Branch: ${refs.base.ref}`);
|
||||
console.log(` Latest Commit for Target Branch: ${refs.target.latestSha}`);
|
||||
console.log(` Latest Commit for PR: ${refs.base.latestSha}`);
|
||||
console.log(` First Common Ancestor SHA: ${refs.commonAncestorSha}`);
|
||||
console.log(`--------------------------------`);
|
||||
console.log();
|
||||
|
||||
|
|
@ -74,27 +67,27 @@ async function _main(repository, prNumber) {
|
|||
|
||||
// Get the count of commits between the latest commit from origin and the common ancestor SHA.
|
||||
const {stdout: commitCount} =
|
||||
await exec(`git rev-list --count origin/${target.base.ref}...${target.commonAncestorSha}`);
|
||||
await exec(`git rev-list --count origin/${refs.base.ref}...${refs.commonAncestorSha}`);
|
||||
console.log(`Checking ${commitCount.trim()} commits for changes in the CircleCI config file.`);
|
||||
|
||||
// Check if the files changed between the latest commit from origin and the common ancestor SHA
|
||||
// includes the CircleCI config.
|
||||
const {stdout: circleCIConfigChanged} = await exec(`git diff --name-only origin/${
|
||||
target.base.ref} ${target.commonAncestorSha} -- .circleci/config.yml`);
|
||||
refs.base.ref} ${refs.commonAncestorSha} -- .circleci/config.yml`);
|
||||
|
||||
if (!!circleCIConfigChanged) {
|
||||
throw Error(`
|
||||
CircleCI config on ${target.base.ref} has been modified since commit ${
|
||||
target.commonAncestorSha.slice(0, 7)},
|
||||
CircleCI config on ${refs.base.ref} has been modified since commit ${
|
||||
refs.commonAncestorSha.slice(0, 7)},
|
||||
which this PR is based on.
|
||||
|
||||
Please rebase the PR on ${target.base.ref} after fetching from upstream.
|
||||
Please rebase the PR on ${refs.base.ref} after fetching from upstream.
|
||||
|
||||
Rebase instructions for PR Author, please run the following commands:
|
||||
|
||||
git fetch upstream ${target.base.ref};
|
||||
git checkout ${target.head.ref};
|
||||
git rebase upstream/${target.base.ref};
|
||||
git fetch upstream ${refs.base.ref};
|
||||
git checkout ${refs.head.ref};
|
||||
git rebase upstream/${refs.base.ref};
|
||||
git push --force-with-lease;
|
||||
`);
|
||||
} else {
|
||||
|
|
@ -103,7 +96,7 @@ async function _main(repository, prNumber) {
|
|||
console.log();
|
||||
|
||||
// Rebase the PR.
|
||||
console.log(`Rebasing current branch on ${target.base.ref}.`);
|
||||
await exec(`git rebase origin/${target.base.ref}`);
|
||||
console.log(`Rebasing current branch on ${refs.base.ref}.`);
|
||||
await exec(`git rebase origin/${refs.base.ref}`);
|
||||
console.log('Rebase successful.');
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,132 +0,0 @@
|
|||
/**
|
||||
* @license
|
||||
* Copyright Google Inc. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by an MIT-style license that can be
|
||||
* found in the LICENSE file at https://angular.io/license
|
||||
*/
|
||||
|
||||
// NOTE: When invoked directly via node, this script will take the first positional
|
||||
// arguement as to be the PR number, and log out the ref and sha information in its
|
||||
// JSON format. For other usages, the function to get the ref and sha information
|
||||
// may be imported by another script to be invoked.
|
||||
|
||||
// This script uses `console` to print messages to the user.
|
||||
// tslint:disable:no-console
|
||||
|
||||
const https = require('https');
|
||||
const util = require('util');
|
||||
const child_process = require('child_process');
|
||||
const exec = util.promisify(child_process.exec);
|
||||
|
||||
async function requestDataFromGithub(url) {
|
||||
// GitHub requires a user agent: https://developer.github.com/v3/#user-agent-required
|
||||
let options = {headers: {'User-Agent': 'angular'}};
|
||||
|
||||
// If a github token is present, use it for authorization.
|
||||
const githubToken = process.env.TOKEN || process.env.GITHUB_TOKEN || '';
|
||||
if (githubToken) {
|
||||
options = {
|
||||
headers: {
|
||||
Authorization: `token ${githubToken}`,
|
||||
...options.headers,
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
https
|
||||
.get(
|
||||
url, options,
|
||||
(res) => {
|
||||
const {statusCode} = res;
|
||||
const contentType = res.headers['content-type'];
|
||||
let rawData = '';
|
||||
|
||||
res.on('data', (chunk) => {
|
||||
rawData += chunk;
|
||||
});
|
||||
res.on('end', () => {
|
||||
let error;
|
||||
if (statusCode !== 200) {
|
||||
error = new Error(
|
||||
`Request Failed.\nStatus Code: ${statusCode}.\nResponse: ${rawData}`);
|
||||
} else if (!/^application\/json/.test(contentType)) {
|
||||
error = new Error(
|
||||
'Invalid content-type.\n' +
|
||||
`Expected application/json but received ${contentType}`);
|
||||
}
|
||||
|
||||
if (error) {
|
||||
reject(error);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
resolve(JSON.parse(rawData));
|
||||
} catch (e) {
|
||||
reject(e);
|
||||
}
|
||||
});
|
||||
})
|
||||
.on('error', (e) => {
|
||||
reject(e);
|
||||
});
|
||||
});
|
||||
}
|
||||
// clang-format off
|
||||
// clang keeps trying to put the function name on the next line.
|
||||
async function getRefsAndShasForTarget(prNumber, suppressLog) {
|
||||
// clang-format on
|
||||
// If the environment variable already contains the refs and shas, reuse them.
|
||||
if (process.env['GITHUB_REFS_AND_SHAS']) {
|
||||
suppressLog ||
|
||||
console.info(`Retrieved refs and SHAs for PR ${prNumber} from environment variables.`);
|
||||
return JSON.parse(process.env['GITHUB_REFS_AND_SHAS']);
|
||||
}
|
||||
|
||||
suppressLog ||
|
||||
console.info(`Getting refs and SHAs for PR ${prNumber} on angular/angular from Github.`);
|
||||
const pullsUrl = `https://api.github.com/repos/angular/angular/pulls/${prNumber}`;
|
||||
const result = await requestDataFromGithub(pullsUrl);
|
||||
|
||||
// Ensure the base ref is up to date
|
||||
await exec(`git fetch origin ${result.base.ref}`);
|
||||
|
||||
// The sha of the latest commit on the target branch.
|
||||
const {stdout: latestShaOfTargetBranch} = await exec(`git rev-parse origin/${result.base.ref}`);
|
||||
// The sha of the latest commit on the PR.
|
||||
const {stdout: latestShaOfPrBranch} = await exec(`git rev-parse HEAD`);
|
||||
// The first common SHA in the history of the target branch and the latest commit in the PR.
|
||||
const {stdout: commonAncestorSha} =
|
||||
await exec(`git merge-base origin/${result.base.ref} ${latestShaOfPrBranch}`);
|
||||
|
||||
const output = {
|
||||
base: {
|
||||
ref: result.base.ref,
|
||||
sha: result.base.sha,
|
||||
},
|
||||
head: {
|
||||
ref: result.head.ref,
|
||||
sha: result.head.sha,
|
||||
},
|
||||
commonAncestorSha: commonAncestorSha.trim(),
|
||||
latestShaOfTargetBranch: latestShaOfTargetBranch.trim(),
|
||||
latestShaOfPrBranch: latestShaOfPrBranch.trim(),
|
||||
};
|
||||
return output;
|
||||
}
|
||||
|
||||
// If the script is called directly, log the output of the refs and sha for the
|
||||
// requested PR.
|
||||
if (require.main === module) {
|
||||
const run = async () => {
|
||||
const prNumber = Number.parseInt(process.argv[2], 10);
|
||||
if (!!prNumber) {
|
||||
console.info(JSON.stringify(await getRefsAndShasForTarget(prNumber, true)));
|
||||
}
|
||||
};
|
||||
run();
|
||||
}
|
||||
|
||||
module.exports = getRefsAndShasForTarget;
|
||||
|
|
@ -0,0 +1,180 @@
|
|||
/**
|
||||
* @license
|
||||
* Copyright Google Inc. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by an MIT-style license that can be
|
||||
* found in the LICENSE file at https://angular.io/license
|
||||
*/
|
||||
const {execSync} = require('child_process');
|
||||
|
||||
/** A regex to select a ref that matches our semver refs. */
|
||||
const semverRegex = /^(\d+)\.(\d+)\.x$/;
|
||||
|
||||
/**
|
||||
* Synchronously executes the command.
|
||||
*
|
||||
* Return the trimmed stdout as a string, with an added attribute of the exit code.
|
||||
*/
|
||||
function exec(command, allowStderr = true) {
|
||||
let output = new String();
|
||||
output.code = 0;
|
||||
try {
|
||||
output += execSync(command, {stdio: ['pipe', 'pipe', 'pipe']}).toString().trim();
|
||||
} catch (err) {
|
||||
allowStderr && console.error(err.stderr.toString());
|
||||
output.code = err.status;
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sort a list of fullpath refs into a list and then provide the first entry.
|
||||
*
|
||||
* The sort order will first find master ref, and then any semver ref, followed
|
||||
* by the rest of the refs in the order provided.
|
||||
*
|
||||
* Branches are sorted in this order as work is primarily done on master, and
|
||||
* otherwise on a semver branch. If neither of those were to match, the most
|
||||
* likely correct branch will be the first one encountered in the list.
|
||||
*/
|
||||
function getRefFromBranchList(gitOutput, remote) {
|
||||
const branches = gitOutput.split('\n').map(b => b.split('/').slice(1).join('').trim());
|
||||
return branches.sort((a, b) => {
|
||||
if (a === 'master') {
|
||||
return -1;
|
||||
}
|
||||
if (b === 'master') {
|
||||
return 1;
|
||||
}
|
||||
const aIsSemver = semverRegex.test(a);
|
||||
const bIsSemver = semverRegex.test(b);
|
||||
if (aIsSemver && bIsSemver) {
|
||||
const [, aMajor, aMinor] = a.match(semverRegex);
|
||||
const [, bMajor, bMinor] = b.match(semverRegex);
|
||||
return parseInt(bMajor, 10) - parseInt(aMajor, 10) ||
|
||||
parseInt(aMinor, 10) - parseInt(bMinor, 10) || 0;
|
||||
}
|
||||
if (aIsSemver) {
|
||||
return -1;
|
||||
}
|
||||
if (bIsSemver) {
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
})[0];
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the full sha of the ref provided.
|
||||
*
|
||||
* example: 1bc0c1a6c01ede7168f22fa9b3508ba51f1f464e
|
||||
*/
|
||||
function getShaFromRef(ref) {
|
||||
return exec(`git rev-parse ${ref}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the list of branches which contain the provided sha, sorted in descending order
|
||||
* by committerdate.
|
||||
*
|
||||
* example:
|
||||
* upstream/master
|
||||
* upstream/9.0.x
|
||||
* upstream/test
|
||||
* upstream/1.1.x
|
||||
*/
|
||||
function getBranchListForSha(sha, remote) {
|
||||
return exec(`git branch -r '${remote}/*' --sort=-committerdate --contains ${sha}`);
|
||||
}
|
||||
|
||||
/** Get the common ancestor sha of the two provided shas. */
|
||||
function getCommonAncestorSha(sha1, sha2) {
|
||||
return exec(`git merge-base ${sha1} ${sha2}`);
|
||||
}
|
||||
|
||||
/** Removes the remote from git. */
|
||||
function removeRemote(remote) {
|
||||
exec(`git remote remove ${remote}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds the remote to git, if it doesn't already exist. Returns a boolean indicating
|
||||
* whether the remote was added by the command.
|
||||
*/
|
||||
function addRemote(remote) {
|
||||
return !exec(`git remote add ${remote} https://github.com/${remote}/angular.git`, false).code;
|
||||
}
|
||||
|
||||
/** Fetch latest from the remote. */
|
||||
function fetchRemote(remote) {
|
||||
exec(`git fetch ${remote}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the nearest ref which the HEAD has a parent commit.
|
||||
*
|
||||
* Checks up to a limit of 100 previous shas.
|
||||
*/
|
||||
function getParentBranchForHead(remote) {
|
||||
// Get the latest for the remote.
|
||||
fetchRemote(remote);
|
||||
|
||||
let headCount = 0;
|
||||
while (headCount < 100) {
|
||||
// Attempt to get the ref on the remote for the sha.
|
||||
const branches = getBranchListForSha(`HEAD~${headCount}`, remote);
|
||||
const ref = getRefFromBranchList(branches, remote);
|
||||
// If the ref exists, get the sha and latest sha for the remote ref.
|
||||
if (ref) {
|
||||
const sha = getShaFromRef(`HEAD~${headCount}`);
|
||||
const latestSha = getShaFromRef(`${remote}/${ref}`);
|
||||
return {ref, sha, latestSha, remote};
|
||||
}
|
||||
headCount++;
|
||||
}
|
||||
return {ref: '', latestSha: '', sha, remote};
|
||||
}
|
||||
|
||||
/** Get the ref and latest shas for the provided sha on a specific remote. */
|
||||
function getRefAndShas(sha, remote) {
|
||||
// Ensure the remote is defined in git.
|
||||
let markRemoteForClean = addRemote(remote);
|
||||
// Get the latest from the remote.
|
||||
fetchRemote(remote);
|
||||
|
||||
// Get the ref on the remote for the sha provided.
|
||||
const branches = getBranchListForSha(sha, remote);
|
||||
const ref = getRefFromBranchList(branches, remote);
|
||||
|
||||
// Get the latest sha on the discovered remote ref.
|
||||
const latestSha = getShaFromRef(`${remote}/${ref}`);
|
||||
|
||||
// Clean up the remote if it didn't exist before execution.
|
||||
if (markRemoteForClean) {
|
||||
removeRemote(remote);
|
||||
}
|
||||
|
||||
return {remote, ref, latestSha, sha};
|
||||
}
|
||||
|
||||
|
||||
/** Gets the refs and shas for the base and target of the current environment. */
|
||||
function getRefsAndShasForChange() {
|
||||
let base, target;
|
||||
if (process.env['CI']) {
|
||||
base = getRefAndShas(process.env['CI_GIT_BASE_REVISION'], process.env['CI_REPO_OWNER']);
|
||||
target = getRefAndShas(process.env['CI_GIT_REVISION'], process.env['CI_PR_USERNAME']);
|
||||
} else {
|
||||
const originSha = getShaFromRef(`HEAD`);
|
||||
target = getRefAndShas(originSha, 'origin');
|
||||
base = getParentBranchForHead('upstream');
|
||||
}
|
||||
const commonAncestorSha = getCommonAncestorSha(base.sha, target.sha);
|
||||
return {
|
||||
base,
|
||||
target,
|
||||
commonAncestorSha,
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = getRefsAndShasForChange;
|
||||
Loading…
Reference in New Issue