From d72ae7b71e082d8178b019aa1808cbbf0918e94d Mon Sep 17 00:00:00 2001 From: George Kalpakas Date: Thu, 20 Dec 2018 15:49:22 +0200 Subject: [PATCH] ci: compute commit range for rerun workflows (#27775) On push builds, CircleCI provides `CIRCLE_COMPARE_URL`, which we use to extract the commit range for a given build. When a workflow is rerun (e.g. to recover from a flaked job), `CIRCLE_COMPARE_URL` is not defined, causing some jobs to fail. This commit fixes it by retrieving the compare URL from the original workflow. It uses a slow process involving a (potentially large) number of requests to CircleCI API. It depends on the (undocumented) fact, that the `workspace_id` is the same on all rerun workflows and the same as the original `workflow_id`. PR Close #27775 --- .circleci/env.sh | 12 ++- .circleci/get-commit-range.js | 149 ++++++++++++++++++++++++++++++++++ 2 files changed, 157 insertions(+), 4 deletions(-) create mode 100644 .circleci/get-commit-range.js diff --git a/.circleci/env.sh b/.circleci/env.sh index afb59b991d..5b24056b6c 100755 --- a/.circleci/env.sh +++ b/.circleci/env.sh @@ -1,7 +1,10 @@ #!/usr/bin/env bash -# Load helpers and make them available everywhere (through `$BASH_ENV`). +# Variables readonly envHelpersPath="`dirname $0`/env-helpers.inc.sh"; +readonly getCommitRangePath="`dirname $0`/get-commit-range.js"; + +# Load helpers and make them available everywhere (through `$BASH_ENV`). source $envHelpersPath; echo "source $envHelpersPath;" >> $BASH_ENV; @@ -16,9 +19,10 @@ setPublicVar CI_AIO_MIN_PWA_SCORE "95"; # This is the branch being built; e.g. `pull/12345` for PR builds. setPublicVar CI_BRANCH "$CIRCLE_BRANCH"; setPublicVar CI_COMMIT "$CIRCLE_SHA1"; -# `CI_COMMIT_RANGE` will only be available when `CIRCLE_COMPARE_URL` is also available, -# i.e. on push builds (a.k.a. non-PR builds). That is fine, since we only need it in push builds. -setPublicVar CI_COMMIT_RANGE "$(sed -r 's|^.*/([0-9a-f]+\.\.\.[0-9a-f]+)$|\1|i' <<< ${CIRCLE_COMPARE_URL:-})"; +# `CI_COMMIT_RANGE` will only be available when `CIRCLE_COMPARE_URL` is also available (or can be +# retrieved via `get-compare-url.js`), i.e. on push builds (a.k.a. non-PR, non-scheduled builds and +# rerun workflows of such builds). That is fine, since we only need it in push builds. +setPublicVar CI_COMMIT_RANGE "`[[ ${CIRCLE_PR_NUMBER:-false} != false ]] && echo "" || node $getCommitRangePath "$CIRCLE_BUILD_NUM" "$CIRCLE_COMPARE_URL"`"; setPublicVar CI_PULL_REQUEST "${CIRCLE_PR_NUMBER:-false}"; setPublicVar CI_REPO_NAME "$CIRCLE_PROJECT_REPONAME"; setPublicVar CI_REPO_OWNER "$CIRCLE_PROJECT_USERNAME"; diff --git a/.circleci/get-commit-range.js b/.circleci/get-commit-range.js new file mode 100644 index 0000000000..d54930f1ef --- /dev/null +++ b/.circleci/get-commit-range.js @@ -0,0 +1,149 @@ +#!/usr/bin/env node + +/** + * **Usage:** + * ``` + * node get-commit-range [ []] + * ``` + * + * Returns the value of the `CIRCLE_COMPARE_URL` environment variable (if defined) or, if this is + * not a PR build (i.e. `CIRCLE_PR_NUMBER` is not defined), retrieves the equivalent of + * `CIRCLE_COMPARE_URL` for jobs that are part of a rerun workflow. + * + * **Context:** + * CircleCI sets the `CIRCLE_COMPARE_URL` environment variable (from which we can extract the commit + * range) on push builds (a.k.a. non-PR, non-scheduled builds). Yet, when a workflow is rerun + * (either from the beginning or from failed jobs) - e.g. when a job flakes - CircleCI does not set + * the `CIRCLE_COMPARE_URL`. + * + * **Implementation details:** + * This script relies on the fact that all rerun workflows share the same CircleCI workspace and the + * (undocumented) fact that the workspace ID happens to be the same as the workflow ID that first + * created it. + * + * For example, for a job on push build workflow, the CircleCI API will return data that look like: + * ```js + * { + * compare: 'THE_COMPARE_URL_WE_ARE_LOOKING_FOR', + * //... + * previous: { + * // ... + * build_num: 12345, + * }, + * //... + * workflows: { + * //... + * workflow_id: 'SOME_ID_A', + * workspace_id: 'SOME_ID_A', // Same as `workflow_id`. + * } + * } + * ``` + * + * If the workflow is rerun, the data for jobs on the new workflow will look like: + * ```js + * { + * compare: null, // ¯\_(ツ)_/¯ + * //... + * previous: { + * // ... + * build_num: 23456, + * }, + * //... + * workflows: { + * //... + * workflow_id: 'SOME_ID_B', + * workspace_id: 'SOME_ID_A', // Different from current `workflow_id`. + * // Same as original `workflow_id`. \o/ + * } + * } + * ``` + * + * This script uses the `previous.build_num` (which points to the previous build number on the same + * branch) to traverse the jobs backwards, until it finds a job from the original workflow. Such a + * job (if found) should also contain the compare URL. + * + * **NOTE 1:** + * This is only useful on workflows which are created by rerunning a workflow for which + * `CIRCLE_COMPARE_URL` was defined. + * + * **NOTE 2:** + * The `circleToken` will be used for CircleCI API requests if provided, but it is not needed for + * accessing the read-only endpoints that we need (as long as the current project is FOSS and the + * corresponding setting is turned on in "Advanced Settings" in the project dashboard). + * + * --- + * Inspired by https://circleci.com/orbs/registry/orb/iynere/compare-url + * (source code: https://github.com/iynere/compare-url-orb). + */ + +// Imports +const {get: httpsGet} = require('https'); + +// Constants +const API_URL_BASE = 'https://circleci.com/api/v1.1/project/github/angular/angular'; +const COMPARE_URL_RE = /^.*\/([0-9a-f]+\.\.\.[0-9a-f]+)$/i; + +// Run +_main(process.argv.slice(2)); + +// Helpers +async function _main([buildNumber, compareUrl = '', circleToken = '']) { + try { + if (!buildNumber || isNaN(buildNumber)) { + throw new Error( + 'Missing or invalid arguments.\n' + + 'Expected: buildNumber (number), compareUrl? (string), circleToken? (string)'); + } + + if (!compareUrl) { + compareUrl = await getCompareUrl(buildNumber, circleToken); + } + + const commitRangeMatch = COMPARE_URL_RE.exec(compareUrl) + const commitRange = commitRangeMatch ? commitRangeMatch[1] : ''; + + console.log(commitRange); + } catch (err) { + console.error(err); + process.exit(1); + } +} + +function getBuildInfo(buildNumber, circleToken) { + console.error(`BUILD ${buildNumber}`); + const url = `${API_URL_BASE}/${buildNumber}?circle-token=${circleToken}`; + return getJson(url); +} + +async function getCompareUrl(buildNumber, circleToken) { + let info = await getBuildInfo(buildNumber, circleToken); + const targetWorkflowId = info.workflows.workspace_id; + + while (info.workflows.workflow_id !== targetWorkflowId) { + info = await getBuildInfo(info.previous.build_num, circleToken); + } + + return info.compare || ''; +} + +function getJson(url) { + return new Promise((resolve, reject) => { + const opts = {headers: {Accept: 'application/json'}}; + const onResponse = res => { + const statusCode = res.statusCode || -1; + const isSuccess = (200 <= statusCode) && (statusCode <= 400); + let responseText = ''; + + res. + on('error', reject). + on('data', d => responseText += d). + on('end', () => isSuccess ? + resolve(JSON.parse(responseText)) : + reject(`Error getting '${url}' (status ${statusCode}):\n${responseText}`)); + }; + + httpsGet(url, opts, onResponse). + on('error', reject). + end(); + }); +}