diff --git a/dev-infra/BUILD.bazel b/dev-infra/BUILD.bazel index 1906e83c04..065f6efad7 100644 --- a/dev-infra/BUILD.bazel +++ b/dev-infra/BUILD.bazel @@ -8,6 +8,7 @@ ts_library( ], module_name = "@angular/dev-infra-private", deps = [ + "//dev-infra/caretaker", "//dev-infra/commit-message", "//dev-infra/format", "//dev-infra/pr", diff --git a/dev-infra/caretaker/BUILD.bazel b/dev-infra/caretaker/BUILD.bazel new file mode 100644 index 0000000000..268ab0f616 --- /dev/null +++ b/dev-infra/caretaker/BUILD.bazel @@ -0,0 +1,26 @@ +load("@npm_bazel_typescript//:index.bzl", "ts_library") + +ts_library( + name = "caretaker", + srcs = [ + "cli.ts", + ], + module_name = "@angular/dev-infra-private/caretaker", + visibility = ["//dev-infra:__subpackages__"], + deps = [ + "//dev-infra/caretaker/check", + "@npm//@types/yargs", + "@npm//yargs", + ], +) + +ts_library( + name = "config", + srcs = [ + "config.ts", + ], + visibility = ["//dev-infra:__subpackages__"], + deps = [ + "//dev-infra/utils", + ], +) diff --git a/dev-infra/caretaker/check/BUILD.bazel b/dev-infra/caretaker/check/BUILD.bazel new file mode 100644 index 0000000000..e4a8644799 --- /dev/null +++ b/dev-infra/caretaker/check/BUILD.bazel @@ -0,0 +1,21 @@ +load("@npm_bazel_typescript//:index.bzl", "ts_library") + +ts_library( + name = "check", + srcs = glob(["*.ts"]), + module_name = "@angular/dev-infra-private/caretaker/service-statuses", + visibility = ["//dev-infra:__subpackages__"], + deps = [ + "//dev-infra/caretaker:config", + "//dev-infra/utils", + "@npm//@types/fs-extra", + "@npm//@types/node", + "@npm//@types/node-fetch", + "@npm//@types/yargs", + "@npm//multimatch", + "@npm//node-fetch", + "@npm//typed-graphqlify", + "@npm//yaml", + "@npm//yargs", + ], +) diff --git a/dev-infra/caretaker/check/check.ts b/dev-infra/caretaker/check/check.ts new file mode 100644 index 0000000000..9fdfb8b984 --- /dev/null +++ b/dev-infra/caretaker/check/check.ts @@ -0,0 +1,27 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +import {GitClient} from '../../utils/git'; +import {getCaretakerConfig} from '../config'; + +import {printG3Comparison} from './g3'; +import {printGithubTasks} from './github'; +import {printServiceStatuses} from './services'; + + +/** Check the status of services which Angular caretakers need to monitor. */ +export async function checkServiceStatuses(githubToken: string) { + /** The configuration for the caretaker commands. */ + const config = getCaretakerConfig(); + /** The GitClient for interacting with git and Github. */ + const git = new GitClient(githubToken, config); + + await printServiceStatuses(); + await printGithubTasks(git, config.caretaker); + await printG3Comparison(git); +} diff --git a/dev-infra/caretaker/check/cli.ts b/dev-infra/caretaker/check/cli.ts new file mode 100644 index 0000000000..8e50c7f261 --- /dev/null +++ b/dev-infra/caretaker/check/cli.ts @@ -0,0 +1,39 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +import {Arguments, Argv, CommandModule} from 'yargs'; + +import {addGithubTokenFlag} from '../../utils/yargs'; + +import {checkServiceStatuses} from './check'; + + +export interface CaretakerCheckOptions { + githubToken: string; +} + +/** URL to the Github page where personal access tokens can be generated. */ +export const GITHUB_TOKEN_GENERATE_URL = `https://github.com/settings/tokens`; + +/** Builds the command. */ +function builder(yargs: Argv) { + return addGithubTokenFlag(yargs); +} + +/** Handles the command. */ +async function handler({githubToken}: Arguments) { + await checkServiceStatuses(githubToken); +} + +/** yargs command module for checking status information for the repository */ +export const CheckModule: CommandModule<{}, CaretakerCheckOptions> = { + handler, + builder, + command: 'check', + describe: 'Check the status of information the caretaker manages for the repository', +}; diff --git a/dev-infra/caretaker/check/g3.ts b/dev-infra/caretaker/check/g3.ts new file mode 100644 index 0000000000..bba0f0b4c1 --- /dev/null +++ b/dev-infra/caretaker/check/g3.ts @@ -0,0 +1,123 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +import {existsSync, readFileSync} from 'fs-extra'; +import * as multimatch from 'multimatch'; +import {join} from 'path'; +import {parse as parseYaml} from 'yaml'; + +import {getRepoBaseDir} from '../../utils/config'; +import {bold, debug, info} from '../../utils/console'; +import {GitClient} from '../../utils/git'; + +/** Compare the upstream master to the upstream g3 branch, if it exists. */ +export async function printG3Comparison(git: GitClient) { + const angularRobotFilePath = join(getRepoBaseDir(), '.github/angular-robot.yml'); + if (!existsSync(angularRobotFilePath)) { + return debug('No angular robot configuration file exists, skipping.'); + } + + /** The configuration defined for the angular robot. */ + const robotConfig = parseYaml(readFileSync(angularRobotFilePath).toString()); + /** The files to be included in the g3 sync. */ + const includeFiles = robotConfig?.merge?.g3Status?.include || []; + /** The files to be expected in the g3 sync. */ + const excludeFiles = robotConfig?.merge?.g3Status?.exclude || []; + + if (includeFiles.length === 0 && excludeFiles.length === 0) { + debug('No g3Status include or exclude lists are defined in the angular robot configuration,'); + debug('skipping.'); + return; + } + + /** Random prefix to create unique branch names. */ + const randomPrefix = `prefix${Math.floor(Math.random() * 1000000)}`; + /** Ref name of the temporary master branch. */ + const masterRef = `${randomPrefix}-master`; + /** Ref name of the temporary g3 branch. */ + const g3Ref = `${randomPrefix}-g3`; + /** Url of the ref for fetching master and g3 branches. */ + const refUrl = `https://github.com/${git.remoteConfig.owner}/${git.remoteConfig.name}.git`; + /** The result fo the fetch command. */ + const fetchResult = git.runGraceful(['fetch', refUrl, `master:${masterRef}`, `g3:${g3Ref}`]); + + // If the upstream repository does not have a g3 branch to compare to, skip the comparison. + if (fetchResult.status !== 0) { + if (fetchResult.stderr.includes(`couldn't find remote ref g3`)) { + return debug('No g3 branch exists on upstream, skipping.'); + } + throw Error('Fetch of master and g3 branches for comparison failed.'); + } + + /** The statistical information about the git diff between master and g3. */ + const stats = getDiffStats(git); + + // Delete the temporarily created mater and g3 branches. + git.runGraceful(['branch', '-D', masterRef, g3Ref]); + + info.group(bold('g3 branch check')); + info(`${stats.commits} commits between g3 and master`); + if (stats.files === 0) { + info('✅ No sync is needed at this time'); + } else { + info(`${stats.files} files changed, ${stats.insertions} insertions(+), ${ + stats.deletions} deletions(-) will be included in the next sync`); + } + info.groupEnd(); + info(); + + + /** + * Get git diff stats between master and g3, for all files and filtered to only g3 affecting + * files. + */ + function getDiffStats(git: GitClient) { + /** The diff stats to be returned. */ + const stats = { + insertions: 0, + deletions: 0, + files: 0, + commits: 0, + }; + + + // Determine the number of commits between master and g3 refs. */ + stats.commits = parseInt(git.run(['rev-list', '--count', `${g3Ref}..${masterRef}`]).stdout, 10); + + // Get the numstat information between master and g3 + git.run(['diff', `${g3Ref}...${masterRef}`, '--numstat']) + .stdout + // Remove the extra space after git's output. + .trim() + // Split each line of git output into array + .split('\n') + // Split each line from the git output into components parts: insertions, + // deletions and file name respectively + .map(line => line.split('\t')) + // Parse number value from the insertions and deletions values + // Example raw line input: + // 10\t5\tsrc/file/name.ts + .map(line => [Number(line[0]), Number(line[1]), line[2]] as [number, number, string]) + // Add each line's value to the diff stats, and conditionally to the g3 + // stats as well if the file name is included in the files synced to g3. + .forEach(([insertions, deletions, fileName]) => { + if (checkMatchAgainstIncludeAndExclude(fileName, includeFiles, excludeFiles)) { + stats.insertions += insertions; + stats.deletions += deletions; + stats.files += 1; + } + }); + return stats; + } + + /** Determine whether the file name passes both include and exclude checks. */ + function checkMatchAgainstIncludeAndExclude( + file: string, includes: string[], excludes: string[]) { + return multimatch(multimatch(file, includes), excludes, {flipNegate: true}).length !== 0; + } +} diff --git a/dev-infra/caretaker/check/github.ts b/dev-infra/caretaker/check/github.ts new file mode 100644 index 0000000000..5020c11531 --- /dev/null +++ b/dev-infra/caretaker/check/github.ts @@ -0,0 +1,56 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +import {alias, params, types} from 'typed-graphqlify'; + +import {bold, debug, info} from '../../utils/console'; +import {GitClient} from '../../utils/git'; +import {CaretakerConfig} from '../config'; + + +interface GithubInfoQuery { + [key: string]: { + issueCount: number, + }; +} + +/** Retrieve the number of matching issues for each github query. */ +export async function printGithubTasks(git: GitClient, config: CaretakerConfig) { + if (!config.githubQueries?.length) { + debug('No github queries defined in the configuration, skipping.'); + return; + } + info.group(bold('Github Tasks')); + await getGithubInfo(git, config); + info.groupEnd(); + info(); +} + +/** Retrieve query match counts and log discovered counts to the console. */ +async function getGithubInfo(git: GitClient, {githubQueries: queries = []}: CaretakerConfig) { + /** The query object for graphql. */ + const graphQlQuery: {[key: string]: {issueCount: number}} = {}; + /** The Github search filter for the configured repository. */ + const repoFilter = `repo:${git.remoteConfig.owner}/${git.remoteConfig.name}`; + queries.forEach(({name, query}) => { + /** The name of the query, with spaces removed to match GraphQL requirements. */ + const queryKey = alias(name.replace(/ /g, ''), 'search'); + graphQlQuery[queryKey] = params( + { + type: 'ISSUE', + query: `"${repoFilter} ${query.replace(/"/g, '\\"')}"`, + }, + {issueCount: types.number}, + ); + }); + /** The results of the generated github query. */ + const results = await git.github.graphql.query(graphQlQuery); + Object.values(results).forEach((result, i) => { + info(`${queries[i]?.name.padEnd(25)} ${result.issueCount}`); + }); +} diff --git a/dev-infra/caretaker/check/services.ts b/dev-infra/caretaker/check/services.ts new file mode 100644 index 0000000000..8fd3a2dbc3 --- /dev/null +++ b/dev-infra/caretaker/check/services.ts @@ -0,0 +1,79 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +import fetch from 'node-fetch'; + +import {bold, green, info, red} from '../../utils/console'; + +/** The status levels for services. */ +enum ServiceStatus { + GREEN, + RED +} + +/** The results of checking the status of a service */ +interface StatusCheckResult { + status: ServiceStatus; + description: string; + lastUpdated: Date; +} + +/** Retrieve and log stasuses for all of the services of concern. */ +export async function printServiceStatuses() { + info.group(bold(`Service Statuses (checked: ${new Date().toLocaleString()})`)); + logStatus('CircleCI', await getCircleCiStatus()); + logStatus('Github', await getGithubStatus()); + logStatus('NPM', await getNpmStatus()); + logStatus('Saucelabs', await getSaucelabsStatus()); + info.groupEnd(); + info(); +} + + +/** Log the status of the service to the console. */ +function logStatus(serviceName: string, status: StatusCheckResult) { + serviceName = serviceName.padEnd(15); + if (status.status === ServiceStatus.GREEN) { + info(`${serviceName} ${green('✅')}`); + } else if (status.status === ServiceStatus.RED) { + info.group(`${serviceName} ${red('❌')} (Updated: ${status.lastUpdated.toLocaleString()})`); + info(` Details: ${status.description}`); + info.groupEnd(); + } +} + +/** Gets the service status information for Saucelabs. */ +async function getSaucelabsStatus(): Promise { + return getStatusFromStandardApi('https://status.us-west-1.saucelabs.com/api/v2/status.json'); +} + +/** Gets the service status information for NPM. */ +async function getNpmStatus(): Promise { + return getStatusFromStandardApi('https://status.npmjs.org/api/v2/status.json'); +} + +/** Gets the service status information for CircleCI. */ +async function getCircleCiStatus(): Promise { + return getStatusFromStandardApi('https://status.circleci.com/api/v2/status.json'); +} + +/** Gets the service status information for Github. */ +async function getGithubStatus(): Promise { + return getStatusFromStandardApi('https://www.githubstatus.com/api/v2/status.json'); +} + +/** Retrieve the status information for a service which uses a standard API response. */ +async function getStatusFromStandardApi(url: string) { + const result = await fetch(url).then(result => result.json()); + const status = result.status.indicator === 'none' ? ServiceStatus.GREEN : ServiceStatus.RED; + return { + status, + description: result.status.description, + lastUpdated: new Date(result.page.updated_at) + }; +} diff --git a/dev-infra/caretaker/cli.ts b/dev-infra/caretaker/cli.ts new file mode 100644 index 0000000000..dd464699e8 --- /dev/null +++ b/dev-infra/caretaker/cli.ts @@ -0,0 +1,16 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +import {Argv} from 'yargs'; +import {CheckModule} from './check/cli'; + + +/** Build the parser for the caretaker commands. */ +export function buildCaretakerParser(yargs: Argv) { + return yargs.command(CheckModule); +} diff --git a/dev-infra/caretaker/config.ts b/dev-infra/caretaker/config.ts new file mode 100644 index 0000000000..8ac7ea6303 --- /dev/null +++ b/dev-infra/caretaker/config.ts @@ -0,0 +1,24 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +import {assertNoErrors, getConfig, NgDevConfig} from '../utils/config'; + +export interface CaretakerConfig { + githubQueries?: {name: string; query: string;}[]; +} + +/** Retrieve and validate the config as `CaretakerConfig`. */ +export function getCaretakerConfig() { + // List of errors encountered validating the config. + const errors: string[] = []; + // The non-validated config object. + const config: Partial> = getConfig(); + + assertNoErrors(errors); + return config as Required; +} diff --git a/dev-infra/cli.ts b/dev-infra/cli.ts index 5901dd9fe1..5ec25a6245 100644 --- a/dev-infra/cli.ts +++ b/dev-infra/cli.ts @@ -14,6 +14,7 @@ import {buildFormatParser} from './format/cli'; import {buildReleaseParser} from './release/cli'; import {buildPrParser} from './pr/cli'; import {captureLogOutputForCommand} from './utils/console'; +import {buildCaretakerParser} from './caretaker/cli'; yargs.scriptName('ng-dev') .middleware(captureLogOutputForCommand) @@ -25,6 +26,7 @@ yargs.scriptName('ng-dev') .command('pullapprove ', '', buildPullapproveParser) .command('release ', '', buildReleaseParser) .command('ts-circular-deps ', '', tsCircularDependenciesBuilder) + .command('caretaker ', '', buildCaretakerParser) .wrap(120) .strict() .parse(); diff --git a/dev-infra/utils/console.ts b/dev-infra/utils/console.ts index d8676c48c4..9687d090f3 100644 --- a/dev-infra/utils/console.ts +++ b/dev-infra/utils/console.ts @@ -19,6 +19,7 @@ import {getRepoBaseDir} from './config'; export const red: typeof chalk = chalk.red; export const green: typeof chalk = chalk.green; export const yellow: typeof chalk = chalk.yellow; +export const bold: typeof chalk = chalk.bold; /** Prompts the user with a confirmation question and a specified message. */ export async function promptConfirm(message: string, defaultValue = false): Promise { diff --git a/dev-infra/utils/git/index.ts b/dev-infra/utils/git/index.ts index 93364731ad..15f2592f1a 100644 --- a/dev-infra/utils/git/index.ts +++ b/dev-infra/utils/git/index.ts @@ -86,10 +86,10 @@ export class GitClient { /** * Spawns a given Git command process. Does not throw if the command fails. Additionally, * if there is any stderr output, the output will be printed. This makes it easier to - * debug failed commands. + * info failed commands. */ runGraceful(args: string[], options: SpawnSyncOptions = {}): SpawnSyncReturns { - // To improve the debugging experience in case something fails, we print all executed + // To improve the infoging experience in case something fails, we print all executed // Git commands. Note that we do not want to print the token if is contained in the // command. It's common to share errors with others if the tool failed. info('Executing: git', this.omitGithubTokenFromMessage(args.join(' '))); diff --git a/package.json b/package.json index 0f3965c849..88cc917984 100644 --- a/package.json +++ b/package.json @@ -81,13 +81,14 @@ "@types/jasmine-ajax": "^3.3.1", "@types/jasminewd2": "^2.0.8", "@types/minimist": "^1.2.0", + "@types/multimatch": "^4.0.0", "@types/node": "^12.11.1", "@types/node-fetch": "^2.5.7", "@types/selenium-webdriver": "3.0.7", "@types/semver": "^6.0.2", "@types/shelljs": "^0.8.6", "@types/systemjs": "0.19.32", - "@types/yaml": "^1.2.0", + "@types/yaml": "^1.9.7", "@types/yargs": "^15.0.5", "@webcomponents/custom-elements": "^1.1.0", "angular": "npm:angular@1.7", @@ -152,7 +153,7 @@ "tslint": "6.1.3", "typescript": "~4.0.2", "xhr2": "0.2.0", - "yaml": "^1.7.2", + "yaml": "^1.10.0", "yargs": "^15.4.1" }, "// 2": "devDependencies are not used under Bazel. Many can be removed after test.sh is deleted.", diff --git a/yarn.lock b/yarn.lock index 0188a14eff..c8918b79fc 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2243,6 +2243,13 @@ resolved "https://registry.yarnpkg.com/@types/minimist/-/minimist-1.2.0.tgz#69a23a3ad29caf0097f06eda59b361ee2f0639f6" integrity sha1-aaI6OtKcrwCX8G7aWbNh7i8GOfY= +"@types/multimatch@^4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@types/multimatch/-/multimatch-4.0.0.tgz#e14237ade6cba7b79fe3a1a5d4e9579613cee6b6" + integrity sha512-xS26gtqY5QASmfU/6jb5vj7F0D0SctgRGtwXsKSNng1knk/OewjISlkMwGonkMCbZCqSoW3s6nL0sAtTlzbL/g== + dependencies: + multimatch "*" + "@types/node-fetch@^2.5.7": version "2.5.7" resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.5.7.tgz#20a2afffa882ab04d44ca786449a276f9f6bbf3c" @@ -2347,10 +2354,12 @@ "@types/source-list-map" "*" source-map "^0.6.1" -"@types/yaml@^1.2.0": - version "1.2.0" - resolved "https://registry.yarnpkg.com/@types/yaml/-/yaml-1.2.0.tgz#4ed577fc4ebbd6b829b28734e56d10c9e6984e09" - integrity sha512-GW8b9qM+ebgW3/zjzPm0I1NxMvLaz/YKT9Ph6tTb+Fkeyzd9yLTvQ6ciQ2MorTRmb/qXmfjMerRpG4LviixaqQ== +"@types/yaml@^1.9.7": + version "1.9.7" + resolved "https://registry.yarnpkg.com/@types/yaml/-/yaml-1.9.7.tgz#2331f36e0aac91311a63d33eb026c21687729679" + integrity sha512-8WMXRDD1D+wCohjfslHDgICd2JtMATZU8CkhH8LVJqcJs6dyYj5TGptzP8wApbmEullGBSsCEzzap73DQ1HJaA== + dependencies: + yaml "*" "@types/yargs-parser@*": version "15.0.0" @@ -10714,7 +10723,7 @@ multicast-dns@^6.0.1: dns-packet "^1.3.1" thunky "^1.0.2" -multimatch@^4.0.0: +multimatch@*, multimatch@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/multimatch/-/multimatch-4.0.0.tgz#8c3c0f6e3e8449ada0af3dd29efb491a375191b3" integrity sha512-lDmx79y1z6i7RNx0ZGCPq1bzJ6ZoDDKbvh7jxr9SJcWLkShMzXrHbYVpTdnhNM5MXpDUxCQ4DgqVttVXlBgiBQ== @@ -16342,6 +16351,11 @@ yallist@^4.0.0: resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72" integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A== +yaml@*, yaml@^1.10.0: + version "1.10.0" + resolved "https://registry.yarnpkg.com/yaml/-/yaml-1.10.0.tgz#3b593add944876077d4d683fee01081bd9fff31e" + integrity sha512-yr2icI4glYaNG+KWONODapy2/jDdMSDnrONSjblABjD9B4Z5LgiircSt8m8sRZFNi08kG9Sm0uSHtEmP3zaEGg== + yaml@^1.7.2: version "1.8.3" resolved "https://registry.yarnpkg.com/yaml/-/yaml-1.8.3.tgz#2f420fca58b68ce3a332d0ca64be1d191dd3f87a"