iSharkFly-Docs
/
angular-docs-cn
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(dev-infra): run caretaker checks asyncronously (#39086) 

Run each check in the caretaker check process asyncronously.

PR Close #39086
This commit is contained in:
Joey Perrott 2020-09-14 11:24:15 -07:00
parent 1e91f84697
commit 2d79780384
15 changed files with 1194 additions and 483 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
dev-infra/caretaker/BUILD.bazel
View File

@ -1,10 +1,12 @@
load("@npm//@bazel/typescript:index.bzl", "ts_library")
load("//tools:defaults.bzl", "jasmine_node_test")
ts_library(
name = "caretaker",
srcs = glob([
"**/*.ts",
]),
srcs = glob(
["**/*.ts"],
exclude = ["**/*.spec.ts"],
),
module_name = "@angular/dev-infra-private/caretaker",
visibility = ["//dev-infra:__subpackages__"],
deps = [
@ -20,3 +22,27 @@ ts_library(
"@npm//yargs",
],
)
ts_library(
name = "test_lib",
testonly = True,
srcs = glob(["**/*.spec.ts"]),
deps = [
":caretaker",
"//dev-infra/release/versioning",
"//dev-infra/utils",
"//dev-infra/utils/testing",
"@npm//@types/jasmine",
"@npm//@types/node",
"@npm//@types/semver",
"@npm//semver",
],
)
jasmine_node_test(
name = "test",
bootstrap = ["//tools/testing:node_no_angular_es5"],
deps = [
":test_lib",
],
)

40
dev-infra/caretaker/check/base.spec.ts Normal file
View File

@ -0,0 +1,40 @@
/**
* @license
* Copyright Google LLC All Rights Reserved.
*
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/
import {BaseModule} from './base';
/** Data mocking as the "retrieved data". */
const exampleData = 'this is example data' as const;
/** A simple usage of the BaseModule to illustrate the workings built into the abstract class. */
class ConcreteBaseModule extends BaseModule<typeof exampleData> {
async retrieveData() {
return exampleData;
}
async printToTerminal() {}
}
describe('BaseModule', () => {
let retrieveDataSpy: jasmine.Spy;
beforeEach(() => {
retrieveDataSpy = spyOn(ConcreteBaseModule.prototype, 'retrieveData');
});
it('begins retrieving data during construction', () => {
new ConcreteBaseModule({} as any, {} as any);
expect(retrieveDataSpy).toHaveBeenCalled();
});
it('makes the data available via the data attribute', async () => {
retrieveDataSpy.and.callThrough();
const module = new ConcreteBaseModule({} as any, {} as any);
expect(await module.data).toBe(exampleData);
});
});

25
dev-infra/caretaker/check/base.ts Normal file
View File

@ -0,0 +1,25 @@
/**
* @license
* Copyright Google LLC All Rights Reserved.
*
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/
import {NgDevConfig} from '../../utils/config';
import {GitClient} from '../../utils/git/index';
import {CaretakerConfig} from '../config';
/** The BaseModule to extend modules for caretaker checks from. */
export abstract class BaseModule<Data> {
/** The data for the module. */
readonly data = this.retrieveData();
constructor(
protected git: GitClient, protected config: NgDevConfig<{caretaker: CaretakerConfig}>) {}
/** Asyncronously retrieve data for the module. */
protected abstract async retrieveData(): Promise<Data>;
/** Print the information discovered for the module to the terminal. */
abstract async printToTerminal(): Promise<void>;
}

30
dev-infra/caretaker/check/check.ts
View File

@ -9,11 +9,18 @@
import {GitClient} from '../../utils/git/index';
import {getCaretakerConfig} from '../config';
import {printCiStatus} from './ci';
import {printG3Comparison} from './g3';
import {printGithubTasks} from './github';
import {printServiceStatuses} from './services';
import {CiModule} from './ci';
import {G3Module} from './g3';
import {GithubQueriesModule} from './github';
import {ServicesModule} from './services';
/** List of modules checked for the caretaker check command. */
const moduleList = [
GithubQueriesModule,
ServicesModule,
CiModule,
G3Module,
];
/** Check the status of services which Angular caretakers need to monitor. */
export async function checkServiceStatuses(githubToken: string) {
@ -23,10 +30,15 @@ export async function checkServiceStatuses(githubToken: string) {
const git = new GitClient(githubToken, config);
// Prevent logging of the git commands being executed during the check.
GitClient.LOG_COMMANDS = false;
/** List of instances of Caretaker Check modules */
const caretakerCheckModules = moduleList.map(module => new module(git, config));
// TODO(josephperrott): Allow these checks to be loaded in parallel.
await printServiceStatuses();
await printGithubTasks(git, config.caretaker);
await printG3Comparison(git);
await printCiStatus(git);
// Module's `data` is casted as Promise<unknown> because the data types of the `module`'s `data`
// promises do not match typings, however our usage here is only to determine when the promise
// resolves.
await Promise.all(caretakerCheckModules.map(module => module.data as Promise<unknown>));
for (const module of caretakerCheckModules) {
await module.printToTerminal();
}
}

118
dev-infra/caretaker/check/ci.spec.ts Normal file
View File

@ -0,0 +1,118 @@
/**
* @license
* Copyright Google LLC All Rights Reserved.
*
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/
import {SemVer} from 'semver';
import {ReleaseTrain} from '../../release/versioning';
import * as versioning from '../../release/versioning/active-release-trains';
import * as console from '../../utils/console';
import {buildVirtualGitClient, mockNgDevConfig, VirtualGitClient} from '../../utils/testing';
import {CiModule} from './ci';
describe('CiModule', () => {
let fetchActiveReleaseTrainsSpy: jasmine.Spy;
let getBranchStatusFromCiSpy: jasmine.Spy;
let infoSpy: jasmine.Spy;
let debugSpy: jasmine.Spy;
let virtualGitClient: VirtualGitClient;
beforeEach(() => {
virtualGitClient = buildVirtualGitClient();
fetchActiveReleaseTrainsSpy = spyOn(versioning, 'fetchActiveReleaseTrains');
getBranchStatusFromCiSpy = spyOn(CiModule.prototype, 'getBranchStatusFromCi' as any);
infoSpy = spyOn(console, 'info');
debugSpy = spyOn(console, 'debug');
});
describe('getting data for active trains', () => {
it('handles active rc train', async () => {
const trains = buildMockActiveReleaseTrains(true);
fetchActiveReleaseTrainsSpy.and.resolveTo(trains);
const module = new CiModule(virtualGitClient, {caretaker: {}, ...mockNgDevConfig});
await module.data;
expect(getBranchStatusFromCiSpy).toHaveBeenCalledWith(trains.releaseCandidate.branchName);
expect(getBranchStatusFromCiSpy).toHaveBeenCalledWith(trains.latest.branchName);
expect(getBranchStatusFromCiSpy).toHaveBeenCalledWith(trains.next.branchName);
expect(getBranchStatusFromCiSpy).toHaveBeenCalledTimes(3);
});
it('handles an inactive rc train', async () => {
const trains = buildMockActiveReleaseTrains(false);
fetchActiveReleaseTrainsSpy.and.resolveTo(trains);
const module = new CiModule(virtualGitClient, {caretaker: {}, ...mockNgDevConfig});
await module.data;
expect(getBranchStatusFromCiSpy).toHaveBeenCalledWith(trains.latest.branchName);
expect(getBranchStatusFromCiSpy).toHaveBeenCalledWith(trains.next.branchName);
expect(getBranchStatusFromCiSpy).toHaveBeenCalledTimes(2);
});
it('aggregates information into a useful structure', async () => {
const trains = buildMockActiveReleaseTrains(false);
fetchActiveReleaseTrainsSpy.and.resolveTo(trains);
getBranchStatusFromCiSpy.and.returnValue('success');
const module = new CiModule(virtualGitClient, {caretaker: {}, ...mockNgDevConfig});
const data = await module.data;
expect(data[0]).toEqual(
{active: false, name: 'releaseCandidate', label: '', status: 'not found'});
expect(data[1]).toEqual({
active: true,
name: 'latest-branch',
label: 'latest (latest-branch)',
status: 'success',
});
});
});
it('prints the data retrieved', async () => {
const fakeData = Promise.resolve([
{
active: true,
name: 'name0',
label: 'label0',
status: 'success',
},
{
active: false,
name: 'name1',
label: 'label1',
status: 'failed',
},
]);
fetchActiveReleaseTrainsSpy.and.resolveTo([]);
const module = new CiModule(virtualGitClient, {caretaker: {}, ...mockNgDevConfig});
Object.defineProperty(module, 'data', {value: fakeData});
await module.printToTerminal();
expect(debugSpy).toHaveBeenCalledWith('No active release train for name1');
expect(infoSpy).toHaveBeenCalledWith('label0 ✅');
});
});
/** Build a mock set of ActiveReleaseTrains. */
function buildMockActiveReleaseTrains(withRc: false): versioning.ActiveReleaseTrains&
{releaseCandidate: null};
function buildMockActiveReleaseTrains(withRc: true): versioning.ActiveReleaseTrains&
{releaseCandidate: ReleaseTrain};
function buildMockActiveReleaseTrains(withRc: boolean): versioning.ActiveReleaseTrains {
const baseResult = {
isMajor: false,
version: new SemVer('0.0.0'),
};
return {
releaseCandidate: withRc ? {branchName: 'rc-branch', ...baseResult} : null,
latest: {branchName: 'latest-branch', ...baseResult},
next: {branchName: 'next-branch', ...baseResult}
};
}

114
dev-infra/caretaker/check/ci.ts
View File

@ -7,56 +7,82 @@
*/
import fetch from 'node-fetch';
import {fetchActiveReleaseTrains} from '../../release/versioning/index';
import {fetchActiveReleaseTrains, ReleaseTrain} from '../../release/versioning/index';
import {bold, debug, info} from '../../utils/console';
import {GitClient} from '../../utils/git/index';
import {BaseModule} from './base';
/** The results of checking the status of CI. */
interface StatusCheckResult {
status: 'success'|'failed';
}
/** The result of checking a branch on CI. */
type CiBranchStatus = 'success'|'failed'|'not found';
/** Retrieve and log status of CI for the project. */
export async function printCiStatus(git: GitClient) {
const releaseTrains = await fetchActiveReleaseTrains({api: git.github, ...git.remoteConfig});
/** A list of results for checking CI branches. */
type CiData = {
active: boolean,
name: string,
label: string,
status: CiBranchStatus,
}[];
info.group(bold(`CI`));
for (const [trainName, train] of Object.entries(releaseTrains)) {
if (train === null) {
debug(`No active release train for ${trainName}`);
continue;
export class CiModule extends BaseModule<CiData> {
async retrieveData() {
const gitRepoWithApi = {api: this.git.github, ...this.git.remoteConfig};
const releaseTrains = await fetchActiveReleaseTrains(gitRepoWithApi);
const ciResultPromises = Object.entries(releaseTrains).map(async ([trainName, train]: [
string, ReleaseTrain|null
]) => {
if (train === null) {
return {
active: false,
name: trainName,
label: '',
status: 'not found' as const,
};
}
return {
active: true,
name: train.branchName,
label: `${trainName} (${train.branchName})`,
status: await this.getBranchStatusFromCi(train.branchName),
};
});
return await Promise.all(ciResultPromises);
}
async printToTerminal() {
const data = await this.data;
const minLabelLength = Math.max(...data.map(result => result.label.length));
info.group(bold(`CI`));
data.forEach(result => {
if (result.active === false) {
debug(`No active release train for ${result.name}`);
return;
}
const label = result.label.padEnd(minLabelLength);
if (result.status === 'not found') {
info(`${result.name} was not found on CircleCI`);
} else if (result.status === 'success') {
info(`${label} ✅`);
} else {
info(`${label} ❌`);
}
});
info.groupEnd();
info();
}
/** Get the CI status of a given branch from CircleCI. */
private async getBranchStatusFromCi(branch: string): Promise<CiBranchStatus> {
const {owner, name} = this.git.remoteConfig;
const url = `https://circleci.com/gh/${owner}/${name}/tree/${branch}.svg?style=shield`;
const result = await fetch(url).then(result => result.text());
if (result && !result.includes('no builds')) {
return result.includes('passing') ? 'success' : 'failed';
}
const status = await getStatusOfBranch(git, train.branchName);
await printStatus(`${trainName.padEnd(6)} (${train.branchName})`, status);
}
info.groupEnd();
info();
}
/** Log the status of CI for a given branch to the console. */
async function printStatus(label: string, status: StatusCheckResult|null) {
const branchName = label.padEnd(16);
if (status === null) {
info(`${branchName} was not found on CircleCI`);
} else if (status.status === 'success') {
info(`${branchName} ✅`);
} else {
info(`${branchName} ❌`);
return 'not found';
}
}
/** Get the CI status of a given branch from CircleCI. */
async function getStatusOfBranch(git: GitClient, branch: string): Promise<StatusCheckResult|null> {
const {owner, name} = git.remoteConfig;
const url = `https://circleci.com/gh/${owner}/${name}/tree/${branch}.svg?style=shield`;
const result = await fetch(url).then(result => result.text());
if (result && !result.includes('no builds')) {
return {
status: result.includes('passing') ? 'success' : 'failed',
};
}
return null;
}

109
dev-infra/caretaker/check/g3.spec.ts Normal file
View File

@ -0,0 +1,109 @@
/**
* @license
* Copyright Google LLC All Rights Reserved.
*
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/
import {SpawnSyncReturns} from 'child_process';
import * as console from '../../utils/console';
import {buildVirtualGitClient, mockNgDevConfig, VirtualGitClient} from '../../utils/testing';
import {G3Module, G3StatsData} from './g3';
describe('G3Module', () => {
let getG3FileIncludeAndExcludeLists: jasmine.Spy;
let getLatestShas: jasmine.Spy;
let getDiffStats: jasmine.Spy;
let infoSpy: jasmine.Spy;
let virtualGitClient: VirtualGitClient;
beforeEach(() => {
virtualGitClient = buildVirtualGitClient();
getG3FileIncludeAndExcludeLists =
spyOn(G3Module.prototype, 'getG3FileIncludeAndExcludeLists' as any).and.returnValue(null);
getLatestShas = spyOn(G3Module.prototype, 'getLatestShas' as any).and.returnValue(null);
getDiffStats = spyOn(G3Module.prototype, 'getDiffStats' as any).and.returnValue(null);
infoSpy = spyOn(console, 'info');
});
describe('gathering stats', () => {
it('unless the g3 merge config is not defined in the angular robot file', async () => {
getG3FileIncludeAndExcludeLists.and.returnValue(null);
getLatestShas.and.returnValue({g3: 'abc123', master: 'zxy987'});
const module = new G3Module(virtualGitClient, {caretaker: {}, ...mockNgDevConfig});
expect(getDiffStats).not.toHaveBeenCalled();
expect(await module.data).toBe(undefined);
});
it('unless the branch shas are not able to be retrieved', async () => {
getLatestShas.and.returnValue(null);
getG3FileIncludeAndExcludeLists.and.returnValue({include: ['file1'], exclude: []});
const module = new G3Module(virtualGitClient, {caretaker: {}, ...mockNgDevConfig});
expect(getDiffStats).not.toHaveBeenCalled();
expect(await module.data).toBe(undefined);
});
it('for the files which are being synced to g3', async () => {
getLatestShas.and.returnValue({g3: 'abc123', master: 'zxy987'});
getG3FileIncludeAndExcludeLists.and.returnValue({include: ['project1/*'], exclude: []});
getDiffStats.and.callThrough();
spyOn(virtualGitClient, 'run').and.callFake((args: string[]): any => {
const output: Partial<SpawnSyncReturns<string>> = {};
if (args[0] === 'rev-list') {
output.stdout = '3';
}
if (args[0] === 'diff') {
output.stdout = '5\t6\tproject1/file1\n2\t3\tproject2/file2\n7\t1\tproject1/file3\n';
}
return output;
});
const module = new G3Module(virtualGitClient, {caretaker: {}, ...mockNgDevConfig});
const {insertions, deletions, files, commits} = (await module.data) as G3StatsData;
expect(insertions).toBe(12);
expect(deletions).toBe(7);
expect(files).toBe(2);
expect(commits).toBe(3);
});
});
describe('printing the data retrieved', () => {
it('if files are discovered needing to sync', async () => {
const fakeData = Promise.resolve({
insertions: 25,
deletions: 10,
files: 2,
commits: 2,
});
const module = new G3Module(virtualGitClient, {caretaker: {}, ...mockNgDevConfig});
Object.defineProperty(module, 'data', {value: fakeData});
await module.printToTerminal();
expect(infoSpy).toHaveBeenCalledWith(
'2 files changed, 25 insertions(+), 10 deletions(-) from 2 commits will be included in the next sync');
});
it('if no files need to sync', async () => {
const fakeData = Promise.resolve({
insertions: 0,
deletions: 0,
files: 0,
commits: 25,
});
const module = new G3Module(virtualGitClient, {caretaker: {}, ...mockNgDevConfig});
Object.defineProperty(module, 'data', {value: fakeData});
await module.printToTerminal();
expect(infoSpy).toHaveBeenCalledWith('25 commits between g3 and master');
expect(infoSpy).toHaveBeenCalledWith('✅ No sync is needed at this time');
});
});
});

146
dev-infra/caretaker/check/g3.ts
View File

@ -10,76 +10,71 @@ import {existsSync, readFileSync} from 'fs';
import * as multimatch from 'multimatch';
import {join} from 'path';
import {parse as parseYaml} from 'yaml';
import {getRepoBaseDir} from '../../utils/config';
import {bold, debug, info} from '../../utils/console';
import {GitClient} from '../../utils/git/index';
import {bold, debug, error, info} from '../../utils/console';
/** Compare the upstream master to the upstream g3 branch, if it exists. */
export async function printG3Comparison(git: GitClient) {
const angularRobotFilePath = join(getRepoBaseDir(), '.github/angular-robot.yml');
if (!existsSync(angularRobotFilePath)) {
return debug('No angular robot configuration file exists, skipping.');
import {BaseModule} from './base';
/** Information expressing the difference between the master and g3 branches */
export interface G3StatsData {
insertions: number;
deletions: number;
files: number;
commits: number;
}
export class G3Module extends BaseModule<G3StatsData|void> {
async retrieveData() {
const toCopyToG3 = this.getG3FileIncludeAndExcludeLists();
const latestSha = this.getLatestShas();
if (toCopyToG3 === null || latestSha === null) {
return;
}
return this.getDiffStats(
latestSha.g3, latestSha.master, toCopyToG3.include, toCopyToG3.exclude);
}
/** The configuration defined for the angular robot. */
const robotConfig = parseYaml(readFileSync(angularRobotFilePath).toString());
/** The files to be included in the g3 sync. */
const includeFiles = robotConfig?.merge?.g3Status?.include || [];
/** The files to be expected in the g3 sync. */
const excludeFiles = robotConfig?.merge?.g3Status?.exclude || [];
if (includeFiles.length === 0 && excludeFiles.length === 0) {
debug('No g3Status include or exclude lists are defined in the angular robot configuration,');
debug('skipping.');
return;
async printToTerminal() {
const stats = await this.data;
if (!stats) {
return;
}
info.group(bold('g3 branch check'));
if (stats.files === 0) {
info(`${stats.commits} commits between g3 and master`);
info('✅ No sync is needed at this time');
} else {
info(
`${stats.files} files changed, ${stats.insertions} insertions(+), ${stats.deletions} ` +
`deletions(-) from ${stats.commits} commits will be included in the next sync`);
}
info.groupEnd();
info();
}
/** The latest sha for the g3 branch. */
const g3Ref = getShaForBranchLatest('g3');
/** The latest sha for the master branch. */
const masterRef = getShaForBranchLatest('master');
if (!g3Ref && !masterRef) {
return debug('Exiting early as either the g3 or master was unable to be retrieved');
}
/** The statistical information about the git diff between master and g3. */
const stats = getDiffStats();
info.group(bold('g3 branch check'));
info(`${stats.commits} commits between g3 and master`);
if (stats.files === 0) {
info('✅ No sync is needed at this time');
} else {
info(`${stats.files} files changed, ${stats.insertions} insertions(+), ${
stats.deletions} deletions(-) will be included in the next sync`);
}
info.groupEnd();
info();
/** Fetch and retrieve the latest sha for a specific branch. */
function getShaForBranchLatest(branch: string) {
private getShaForBranchLatest(branch: string) {
const {owner, name} = this.git.remoteConfig;
/** The result fo the fetch command. */
const fetchResult = git.runGraceful([
'fetch', '-q', `https://github.com/${git.remoteConfig.owner}/${git.remoteConfig.name}.git`,
branch
]);
const fetchResult =
this.git.runGraceful(['fetch', '-q', `https://github.com/${owner}/${name}.git`, branch]);
if (fetchResult.status !== 0 &&
fetchResult.stderr.includes(`couldn't find remote ref ${branch}`)) {
debug(`No '${branch}' branch exists on upstream, skipping.`);
return false;
return null;
}
return git.runGraceful(['rev-parse', 'FETCH_HEAD']).stdout.trim();
return this.git.runGraceful(['rev-parse', 'FETCH_HEAD']).stdout.trim();
}
/**
* Get git diff stats between master and g3, for all files and filtered to only g3 affecting
* files.
*/
function getDiffStats() {
private getDiffStats(
g3Ref: string, masterRef: string, includeFiles: string[], excludeFiles: string[]) {
/** The diff stats to be returned. */
const stats = {
insertions: 0,
@ -89,10 +84,11 @@ export async function printG3Comparison(git: GitClient) {
};
// Determine the number of commits between master and g3 refs. */
stats.commits = parseInt(git.run(['rev-list', '--count', `${g3Ref}..${masterRef}`]).stdout, 10);
stats.commits =
parseInt(this.git.run(['rev-list', '--count', `${g3Ref}..${masterRef}`]).stdout, 10);
// Get the numstat information between master and g3
git.run(['diff', `${g3Ref}...${masterRef}`, '--numstat'])
this.git.run(['diff', `${g3Ref}...${masterRef}`, '--numstat'])
.stdout
// Remove the extra space after git's output.
.trim()
@ -100,7 +96,7 @@ export async function printG3Comparison(git: GitClient) {
.split('\n')
// Split each line from the git output into components parts: insertions,
// deletions and file name respectively
.map(line => line.split('\t'))
.map(line => line.trim().split('\t'))
// Parse number value from the insertions and deletions values
// Example raw line input:
// 10\t5\tsrc/file/name.ts
@ -108,7 +104,7 @@ export async function printG3Comparison(git: GitClient) {
// Add each line's value to the diff stats, and conditionally to the g3
// stats as well if the file name is included in the files synced to g3.
.forEach(([insertions, deletions, fileName]) => {
if (checkMatchAgainstIncludeAndExclude(fileName, includeFiles, excludeFiles)) {
if (this.checkMatchAgainstIncludeAndExclude(fileName, includeFiles, excludeFiles)) {
stats.insertions += insertions;
stats.deletions += deletions;
stats.files += 1;
@ -116,12 +112,46 @@ export async function printG3Comparison(git: GitClient) {
});
return stats;
}
/** Determine whether the file name passes both include and exclude checks. */
function checkMatchAgainstIncludeAndExclude(
file: string, includes: string[], excludes: string[]) {
private checkMatchAgainstIncludeAndExclude(file: string, includes: string[], excludes: string[]) {
return (
multimatch.call(undefined, file, includes).length >= 1 &&
multimatch.call(undefined, file, excludes).length === 0);
}
private getG3FileIncludeAndExcludeLists() {
const angularRobotFilePath = join(getRepoBaseDir(), '.github/angular-robot.yml');
if (!existsSync(angularRobotFilePath)) {
debug('No angular robot configuration file exists, skipping.');
return null;
}
/** The configuration defined for the angular robot. */
const robotConfig = parseYaml(readFileSync(angularRobotFilePath).toString());
/** The files to be included in the g3 sync. */
const include: string[] = robotConfig?.merge?.g3Status?.include || [];
/** The files to be expected in the g3 sync. */
const exclude: string[] = robotConfig?.merge?.g3Status?.exclude || [];
if (include.length === 0 && exclude.length === 0) {
debug('No g3Status include or exclude lists are defined in the angular robot configuration');
return null;
}
return {include, exclude};
}
private getLatestShas() {
/** The latest sha for the g3 branch. */
const g3 = this.getShaForBranchLatest('g3');
/** The latest sha for the master branch. */
const master = this.getShaForBranchLatest('master');
if (g3 === null || master === null) {
debug('Either the g3 or master was unable to be retrieved');
return null;
}
return {g3, master};
}
}

121
dev-infra/caretaker/check/github.spec.ts Normal file
View File

@ -0,0 +1,121 @@
/**
* @license
* Copyright Google LLC All Rights Reserved.
*
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/
import * as console from '../../utils/console';
import {GithubGraphqlClient} from '../../utils/git/github';
import {buildVirtualGitClient, mockNgDevConfig, VirtualGitClient} from '../../utils/testing';
import {GithubQueriesModule} from './github';
describe('GithubQueriesModule', () => {
let githubApiSpy: jasmine.Spy;
let infoSpy: jasmine.Spy;
let infoGroupSpy: jasmine.Spy;
let virtualGitClient: VirtualGitClient;
beforeEach(() => {
githubApiSpy = spyOn(GithubGraphqlClient.prototype, 'query')
.and.throwError(
'The graphql query response must always be manually defined in a test.');
virtualGitClient = buildVirtualGitClient();
infoGroupSpy = spyOn(console.info, 'group');
infoSpy = spyOn(console, 'info');
});
describe('gathering stats', () => {
it('unless githubQueries are `undefined`', async () => {
const module = new GithubQueriesModule(
virtualGitClient, {...mockNgDevConfig, caretaker: {githubQueries: undefined}});
expect(await module.data).toBe(undefined);
});
it('unless githubQueries are an empty array', async () => {
const module = new GithubQueriesModule(
virtualGitClient, {...mockNgDevConfig, caretaker: {githubQueries: []}});
expect(await module.data).toBe(undefined);
});
it('for the requestd Github queries', async () => {
githubApiSpy.and.returnValue({
'keynamewithspaces': {issueCount: 1, nodes: [{url: 'http://gituhb.com/owner/name/issue/1'}]}
});
const module = new GithubQueriesModule(virtualGitClient, {
...mockNgDevConfig,
caretaker: {githubQueries: [{name: 'key name with spaces', query: 'issue: yes'}]}
});
expect(await module.data).toEqual([{
queryName: 'key name with spaces',
count: 1,
queryUrl: 'https://github.com/owner/name/issues?q=issue:%20yes',
matchedUrls: ['http://gituhb.com/owner/name/issue/1'],
}]);
});
});
describe('printing the data retrieved', () => {
it('if there are no matches of the query', async () => {
const fakeData = Promise.resolve([
{
queryName: 'query1',
count: 0,
queryUrl: 'https://github.com/owner/name/issues?q=issue:%20no',
matchedUrls: [],
},
{
queryName: 'query2',
count: 0,
queryUrl: 'https://github.com/owner/name/issues?q=something',
matchedUrls: [],
},
]);
const module = new GithubQueriesModule(virtualGitClient, {caretaker: {}, ...mockNgDevConfig});
Object.defineProperty(module, 'data', {value: fakeData});
await module.printToTerminal();
expect(infoGroupSpy).toHaveBeenCalledWith('Github Tasks');
expect(infoSpy).toHaveBeenCalledWith('query1 0');
expect(infoSpy).toHaveBeenCalledWith('query2 0');
});
it('if there are maches of the query', async () => {
const fakeData = Promise.resolve([
{
queryName: 'query1',
count: 1,
queryUrl: 'https://github.com/owner/name/issues?q=issue:%20yes',
matchedUrls: ['http://gituhb.com/owner/name/issue/1'],
},
{
queryName: 'query2',
count: 0,
queryUrl: 'https://github.com/owner/name/issues?q=something',
matchedUrls: [],
},
]);
const module = new GithubQueriesModule(virtualGitClient, {caretaker: {}, ...mockNgDevConfig});
Object.defineProperty(module, 'data', {value: fakeData});
await module.printToTerminal();
expect(infoGroupSpy).toHaveBeenCalledWith('Github Tasks');
expect(infoSpy).toHaveBeenCalledWith('query1 1');
expect(infoGroupSpy)
.toHaveBeenCalledWith('https://github.com/owner/name/issues?q=issue:%20yes');
expect(infoSpy).toHaveBeenCalledWith('- http://gituhb.com/owner/name/issue/1');
expect(infoSpy).toHaveBeenCalledWith('query2 0');
});
});
});

157
dev-infra/caretaker/check/github.ts
View File

@ -9,75 +9,112 @@
import {alias, onUnion, params, types} from 'typed-graphqlify';
import {bold, debug, info} from '../../utils/console';
import {GitClient} from '../../utils/git/index';
import {CaretakerConfig} from '../config';
import {BaseModule} from './base';
/** A list of generated results for a github query. */
type GithubQueryResults = {
queryName: string,
count: number,
queryUrl: string,
matchedUrls: string[],
}[];
/** The fragment for a result from Github's api for a Github query. */
const GithubQueryResultFragment = {
issueCount: types.number,
nodes: [{...onUnion({
PullRequest: {
url: types.string,
},
Issue: {
url: types.string,
},
})}],
};
/** An object containing results of multiple queries. */
type GithubQueryResult = {
[key: string]: typeof GithubQueryResultFragment;
};
/**
* Cap the returned issues in the queries to an arbitrary 100. At that point, caretaker has a lot
* Cap the returned issues in the queries to an arbitrary 20. At that point, caretaker has a lot
* of work to do and showing more than that isn't really useful.
*/
const MAX_RETURNED_ISSUES = 20;
/** Retrieve the number of matching issues for each github query. */
export async function printGithubTasks(git: GitClient, config?: CaretakerConfig) {
if (!config?.githubQueries?.length) {
debug('No github queries defined in the configuration, skipping.');
return;
}
info.group(bold(`Github Tasks`));
await getGithubInfo(git, config);
info.groupEnd();
info();
}
export class GithubQueriesModule extends BaseModule<GithubQueryResults|void> {
async retrieveData() {
// Non-null assertion is used here as the check for undefined immediately follows to confirm the
// assertion. Typescript's type filtering does not seem to work as needed to understand
// whether githubQueries is undefined or not.
let queries = this.config.caretaker?.githubQueries!;
if (queries === undefined || queries.length === 0) {
debug('No github queries defined in the configuration, skipping');
return;
}
/** Retrieve query match counts and log discovered counts to the console. */
async function getGithubInfo(git: GitClient, {githubQueries: queries = []}: CaretakerConfig) {
/** The query object for graphql. */
const graphQlQuery: {
[key: string]: {
issueCount: number,
nodes: Array<{url: string}>,
/** The results of the generated github query. */
const queryResult = await this.git.github.graphql.query(this.buildGraphqlQuery(queries));
const results = Object.values(queryResult);
const {owner, name: repo} = this.git.remoteConfig;
return results.map((result, i) => {
return {
queryName: queries[i].name,
count: result.issueCount,
queryUrl: encodeURI(`https://github.com/${owner}/${repo}/issues?q=${queries[i].query}`),
matchedUrls: result.nodes.map(node => node.url)
};
});
}
/** Build a Graphql query statement for the provided queries. */
private buildGraphqlQuery(queries: NonNullable<CaretakerConfig['githubQueries']>) {
/** The query object for graphql. */
const graphQlQuery: GithubQueryResult = {};
const {owner, name: repo} = this.git.remoteConfig;
/** The Github search filter for the configured repository. */
const repoFilter = `repo:${owner}/${repo}`;
queries.forEach(({name, query}) => {
/** The name of the query, with spaces removed to match GraphQL requirements. */
const queryKey = alias(name.replace(/ /g, ''), 'search');
graphQlQuery[queryKey] = params(
{
type: 'ISSUE',
first: MAX_RETURNED_ISSUES,
query: `"${repoFilter} ${query.replace(/"/g, '\\"')}"`,
},
{...GithubQueryResultFragment});
});
return graphQlQuery;
}
async printToTerminal() {
const queryResults = await this.data;
if (!queryResults) {
return;
}
} = {};
/** The Github search filter for the configured repository. */
const repoFilter = `repo:${git.remoteConfig.owner}/${git.remoteConfig.name}`;
queries.forEach(({name, query}) => {
/** The name of the query, with spaces removed to match GraphQL requirements. */
const queryKey = alias(name.replace(/ /g, ''), 'search');
graphQlQuery[queryKey] = params(
{
type: 'ISSUE',
first: MAX_RETURNED_ISSUES,
query: `"${repoFilter} ${query.replace(/"/g, '\\"')}"`,
},
{
issueCount: types.number,
nodes: [{...onUnion({
PullRequest: {
url: types.string,
},
Issue: {
url: types.string,
},
})}],
},
);
});
/** The results of the generated github query. */
const results = await git.github.graphql.query(graphQlQuery);
Object.values(results).forEach((result, i) => {
info(`${queries[i]?.name.padEnd(25)} ${result.issueCount}`);
if (result.issueCount > 0) {
const {owner, name: repo} = git.remoteConfig;
const url = encodeURI(`https://github.com/${owner}/${repo}/issues?q=${queries[i]?.query}`);
info.group(`${url}`);
if (result.nodes.length === MAX_RETURNED_ISSUES && result.nodes.length < result.issueCount) {
info(`(first ${MAX_RETURNED_ISSUES})`);
info.group(bold('Github Tasks'));
const minQueryNameLength = Math.max(...queryResults.map(result => result.queryName.length));
for (const queryResult of queryResults) {
info(`${queryResult.queryName.padEnd(minQueryNameLength)} ${queryResult.count}`);
if (queryResult.count > 0) {
info.group(queryResult.queryUrl);
queryResult.matchedUrls.forEach(url => info(`- ${url}`));
if (queryResult.count > MAX_RETURNED_ISSUES) {
info(`... ${queryResult.count - MAX_RETURNED_ISSUES} additional matches`);
}
info.groupEnd();
}
for (const node of result.nodes) {
info(`- ${node.url}`);
}
info.groupEnd();
}
});
info.groupEnd();
info();
}
}

69
dev-infra/caretaker/check/services.spec.ts Normal file
View File

@ -0,0 +1,69 @@
/**
* @license
* Copyright Google LLC All Rights Reserved.
*
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/
import * as console from '../../utils/console';
import {buildVirtualGitClient, mockNgDevConfig, VirtualGitClient} from '../../utils/testing';
import {services, ServicesModule} from './services';
describe('ServicesModule', () => {
let getStatusFromStandardApiSpy: jasmine.Spy;
let infoSpy: jasmine.Spy;
let infoGroupSpy: jasmine.Spy;
let virtualGitClient: VirtualGitClient;
services.splice(0, Infinity, {url: 'fakeStatus.com/api.json', name: 'Service Name'});
beforeEach(() => {
getStatusFromStandardApiSpy = spyOn(ServicesModule.prototype, 'getStatusFromStandardApi');
virtualGitClient = buildVirtualGitClient();
infoGroupSpy = spyOn(console.info, 'group');
infoSpy = spyOn(console, 'info');
});
describe('gathering status', () => {
it('for each of the services', async () => {
new ServicesModule(virtualGitClient, {caretaker: {}, ...mockNgDevConfig});
expect(getStatusFromStandardApiSpy)
.toHaveBeenCalledWith({url: 'fakeStatus.com/api.json', name: 'Service Name'});
});
});
describe('printing the data retrieved', () => {
it('for each service ', async () => {
const fakeData = Promise.resolve([
{
name: 'Service 1',
status: 'passing',
description: 'Everything is working great',
lastUpdated: new Date(0),
},
{
name: 'Service 2',
status: 'failing',
description: 'Literally everything is broken',
lastUpdated: new Date(0),
},
]);
const module = new ServicesModule(virtualGitClient, {caretaker: {}, ...mockNgDevConfig});
Object.defineProperty(module, 'data', {value: fakeData});
await module.printToTerminal();
expect(infoGroupSpy).toHaveBeenCalledWith('Service Statuses');
expect(infoSpy).toHaveBeenCalledWith('Service 1 ✅');
expect(infoGroupSpy)
.toHaveBeenCalledWith(`Service 2 ❌ (Updated: ${new Date(0).toLocaleString()})`);
expect(infoSpy).toHaveBeenCalledWith(' Details: Literally everything is broken');
});
});
});

112
dev-infra/caretaker/check/services.ts
View File

@ -8,72 +8,74 @@
import fetch from 'node-fetch';
import {bold, green, info, red} from '../../utils/console';
import {bold, info} from '../../utils/console';
import {BaseModule} from './base';
/** The status levels for services. */
enum ServiceStatus {
GREEN,
RED
interface ServiceConfig {
name: string;
url: string;
}
/** The results of checking the status of a service */
interface StatusCheckResult {
status: ServiceStatus;
name: string;
status: 'passing'|'failing';
description: string;
lastUpdated: Date;
}
/** Retrieve and log stasuses for all of the services of concern. */
export async function printServiceStatuses() {
info.group(bold(`Service Statuses (checked: ${new Date().toLocaleString()})`));
logStatus('CircleCI', await getCircleCiStatus());
logStatus('Github', await getGithubStatus());
logStatus('NPM', await getNpmStatus());
logStatus('Saucelabs', await getSaucelabsStatus());
info.groupEnd();
info();
}
/** List of services Angular relies on. */
export const services: ServiceConfig[] = [
{
url: 'https://status.us-west-1.saucelabs.com/api/v2/status.json',
name: 'Saucelabs',
},
{
url: 'https://status.npmjs.org/api/v2/status.json',
name: 'Npm',
},
{
url: 'https://status.circleci.com/api/v2/status.json',
name: 'CircleCi',
},
{
url: 'https://www.githubstatus.com/api/v2/status.json',
name: 'Github',
},
];
export class ServicesModule extends BaseModule<StatusCheckResult[]> {
async retrieveData() {
return Promise.all(services.map(service => this.getStatusFromStandardApi(service)));
}
/** Log the status of the service to the console. */
function logStatus(serviceName: string, status: StatusCheckResult) {
serviceName = serviceName.padEnd(15);
if (status.status === ServiceStatus.GREEN) {
info(`${serviceName} ${green('✅')}`);
} else if (status.status === ServiceStatus.RED) {
info.group(`${serviceName} ${red('❌')} (Updated: ${status.lastUpdated.toLocaleString()})`);
info(` Details: ${status.description}`);
async printToTerminal() {
const statuses = await this.data;
const serviceNameMinLength = Math.max(...statuses.map(service => service.name.length));
info.group(bold('Service Statuses'));
for (const status of statuses) {
const name = status.name.padEnd(serviceNameMinLength);
if (status.status === 'passing') {
info(`${name} ✅`);
} else {
info.group(`${name} ❌ (Updated: ${status.lastUpdated.toLocaleString()})`);
info(` Details: ${status.description}`);
info.groupEnd();
}
}
info.groupEnd();
info();
}
/** Retrieve the status information for a service which uses a standard API response. */
async getStatusFromStandardApi(service: ServiceConfig): Promise<StatusCheckResult> {
const result = await fetch(service.url).then(result => result.json());
const status = result.status.indicator === 'none' ? 'passing' : 'failing';
return {
name: service.name,
status,
description: result.status.description,
lastUpdated: new Date(result.page.updated_at)
};
}
}
/** Gets the service status information for Saucelabs. */
async function getSaucelabsStatus(): Promise<StatusCheckResult> {
return getStatusFromStandardApi('https://status.us-west-1.saucelabs.com/api/v2/status.json');
}
/** Gets the service status information for NPM. */
async function getNpmStatus(): Promise<StatusCheckResult> {
return getStatusFromStandardApi('https://status.npmjs.org/api/v2/status.json');
}
/** Gets the service status information for CircleCI. */
async function getCircleCiStatus(): Promise<StatusCheckResult> {
return getStatusFromStandardApi('https://status.circleci.com/api/v2/status.json');
}
/** Gets the service status information for Github. */
async function getGithubStatus(): Promise<StatusCheckResult> {
return getStatusFromStandardApi('https://www.githubstatus.com/api/v2/status.json');
}
/** Retrieve the status information for a service which uses a standard API response. */
async function getStatusFromStandardApi(url: string) {
const result = await fetch(url).then(result => result.json());
const status = result.status.indicator === 'none' ? ServiceStatus.GREEN : ServiceStatus.RED;
return {
status,
description: result.status.description,
lastUpdated: new Date(result.page.updated_at)
};
}

2
dev-infra/commit-message/BUILD.bazel
View File

@ -39,6 +39,6 @@ jasmine_node_test(
name = "test",
bootstrap = ["//tools/testing:node_no_angular_es5"],
deps = [
"test_lib",
":test_lib",
],
)

576
dev-infra/ng-dev.js
View File

@ -1089,58 +1089,14 @@ function getLtsNpmDistTagOfMajor(major) {
return `v${major}-lts`;
}
/**
* @license
* Copyright Google LLC All Rights Reserved.
*
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/
/** Retrieve and log status of CI for the project. */
function printCiStatus(git) {
return tslib.__awaiter(this, void 0, void 0, function* () {
const releaseTrains = yield fetchActiveReleaseTrains(Object.assign({ api: git.github }, git.remoteConfig));
info.group(bold(`CI`));
for (const [trainName, train] of Object.entries(releaseTrains)) {
if (train === null) {
debug(`No active release train for ${trainName}`);
continue;
}
const status = yield getStatusOfBranch(git, train.branchName);
yield printStatus(`${trainName.padEnd(6)} (${train.branchName})`, status);
}
info.groupEnd();
info();
});
}
/** Log the status of CI for a given branch to the console. */
function printStatus(label, status) {
return tslib.__awaiter(this, void 0, void 0, function* () {
const branchName = label.padEnd(16);
if (status === null) {
info(`${branchName} was not found on CircleCI`);
}
else if (status.status === 'success') {
info(`${branchName} ✅`);
}
else {
info(`${branchName} ❌`);
}
});
}
/** Get the CI status of a given branch from CircleCI. */
function getStatusOfBranch(git, branch) {
return tslib.__awaiter(this, void 0, void 0, function* () {
const { owner, name } = git.remoteConfig;
const url = `https://circleci.com/gh/${owner}/${name}/tree/${branch}.svg?style=shield`;
const result = yield fetch(url).then(result => result.text());
if (result && !result.includes('no builds')) {
return {
status: result.includes('passing') ? 'success' : 'failed',
};
}
return null;
});
/** The BaseModule to extend modules for caretaker checks from. */
class BaseModule {
constructor(git, config) {
this.git = git;
this.config = config;
/** The data for the module. */
this.data = this.retrieveData();
}
}
/**
@ -1150,103 +1106,193 @@ function getStatusOfBranch(git, branch) {
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/
/** Compare the upstream master to the upstream g3 branch, if it exists. */
function printG3Comparison(git) {
var _a, _b, _c, _d;
return tslib.__awaiter(this, void 0, void 0, function* () {
class CiModule extends BaseModule {
retrieveData() {
return tslib.__awaiter(this, void 0, void 0, function* () {
const gitRepoWithApi = Object.assign({ api: this.git.github }, this.git.remoteConfig);
const releaseTrains = yield fetchActiveReleaseTrains(gitRepoWithApi);
const ciResultPromises = Object.entries(releaseTrains).map(([trainName, train]) => tslib.__awaiter(this, void 0, void 0, function* () {
if (train === null) {
return {
active: false,
name: trainName,
label: '',
status: 'not found',
};
}
return {
active: true,
name: train.branchName,
label: `${trainName} (${train.branchName})`,
status: yield this.getBranchStatusFromCi(train.branchName),
};
}));
return yield Promise.all(ciResultPromises);
});
}
printToTerminal() {
return tslib.__awaiter(this, void 0, void 0, function* () {
const data = yield this.data;
const minLabelLength = Math.max(...data.map(result => result.label.length));
info.group(bold(`CI`));
data.forEach(result => {
if (result.active === false) {
debug(`No active release train for ${result.name}`);
return;
}
const label = result.label.padEnd(minLabelLength);
if (result.status === 'not found') {
info(`${result.name} was not found on CircleCI`);
}
else if (result.status === 'success') {
info(`${label} ✅`);
}
else {
info(`${label} ❌`);
}
});
info.groupEnd();
info();
});
}
/** Get the CI status of a given branch from CircleCI. */
getBranchStatusFromCi(branch) {
return tslib.__awaiter(this, void 0, void 0, function* () {
const { owner, name } = this.git.remoteConfig;
const url = `https://circleci.com/gh/${owner}/${name}/tree/${branch}.svg?style=shield`;
const result = yield fetch(url).then(result => result.text());
if (result && !result.includes('no builds')) {
return result.includes('passing') ? 'success' : 'failed';
}
return 'not found';
});
}
}
/**
* @license
* Copyright Google LLC All Rights Reserved.
*
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/
class G3Module extends BaseModule {
retrieveData() {
return tslib.__awaiter(this, void 0, void 0, function* () {
const toCopyToG3 = this.getG3FileIncludeAndExcludeLists();
const latestSha = this.getLatestShas();
if (toCopyToG3 === null || latestSha === null) {
return;
}
return this.getDiffStats(latestSha.g3, latestSha.master, toCopyToG3.include, toCopyToG3.exclude);
});
}
printToTerminal() {
return tslib.__awaiter(this, void 0, void 0, function* () {
const stats = yield this.data;
if (!stats) {
return;
}
info.group(bold('g3 branch check'));
if (stats.files === 0) {
info(`${stats.commits} commits between g3 and master`);
info('✅ No sync is needed at this time');
}
else {
info(`${stats.files} files changed, ${stats.insertions} insertions(+), ${stats.deletions} ` +
`deletions(-) from ${stats.commits} commits will be included in the next sync`);
}
info.groupEnd();
info();
});
}
/** Fetch and retrieve the latest sha for a specific branch. */
getShaForBranchLatest(branch) {
const { owner, name } = this.git.remoteConfig;
/** The result fo the fetch command. */
const fetchResult = this.git.runGraceful(['fetch', '-q', `https://github.com/${owner}/${name}.git`, branch]);
if (fetchResult.status !== 0 &&
fetchResult.stderr.includes(`couldn't find remote ref ${branch}`)) {
debug(`No '${branch}' branch exists on upstream, skipping.`);
return null;
}
return this.git.runGraceful(['rev-parse', 'FETCH_HEAD']).stdout.trim();
}
/**
* Get git diff stats between master and g3, for all files and filtered to only g3 affecting
* files.
*/
getDiffStats(g3Ref, masterRef, includeFiles, excludeFiles) {
/** The diff stats to be returned. */
const stats = {
insertions: 0,
deletions: 0,
files: 0,
commits: 0,
};
// Determine the number of commits between master and g3 refs. */
stats.commits =
parseInt(this.git.run(['rev-list', '--count', `${g3Ref}..${masterRef}`]).stdout, 10);
// Get the numstat information between master and g3
this.git.run(['diff', `${g3Ref}...${masterRef}`, '--numstat'])
.stdout
// Remove the extra space after git's output.
.trim()
// Split each line of git output into array
.split('\n')
// Split each line from the git output into components parts: insertions,
// deletions and file name respectively
.map(line => line.trim().split('\t'))
// Parse number value from the insertions and deletions values
// Example raw line input:
// 10\t5\tsrc/file/name.ts
.map(line => [Number(line[0]), Number(line[1]), line[2]])
// Add each line's value to the diff stats, and conditionally to the g3
// stats as well if the file name is included in the files synced to g3.
.forEach(([insertions, deletions, fileName]) => {
if (this.checkMatchAgainstIncludeAndExclude(fileName, includeFiles, excludeFiles)) {
stats.insertions += insertions;
stats.deletions += deletions;
stats.files += 1;
}
});
return stats;
}
/** Determine whether the file name passes both include and exclude checks. */
checkMatchAgainstIncludeAndExclude(file, includes, excludes) {
return (multimatch.call(undefined, file, includes).length >= 1 &&
multimatch.call(undefined, file, excludes).length === 0);
}
getG3FileIncludeAndExcludeLists() {
var _a, _b, _c, _d;
const angularRobotFilePath = path.join(getRepoBaseDir(), '.github/angular-robot.yml');
if (!fs.existsSync(angularRobotFilePath)) {
return debug('No angular robot configuration file exists, skipping.');
debug('No angular robot configuration file exists, skipping.');
return null;
}
/** The configuration defined for the angular robot. */
const robotConfig = yaml.parse(fs.readFileSync(angularRobotFilePath).toString());
/** The files to be included in the g3 sync. */
const includeFiles = ((_b = (_a = robotConfig === null || robotConfig === void 0 ? void 0 : robotConfig.merge) === null || _a === void 0 ? void 0 : _a.g3Status) === null || _b === void 0 ? void 0 : _b.include) || [];
const include = ((_b = (_a = robotConfig === null || robotConfig === void 0 ? void 0 : robotConfig.merge) === null || _a === void 0 ? void 0 : _a.g3Status) === null || _b === void 0 ? void 0 : _b.include) || [];
/** The files to be expected in the g3 sync. */
const excludeFiles = ((_d = (_c = robotConfig === null || robotConfig === void 0 ? void 0 : robotConfig.merge) === null || _c === void 0 ? void 0 : _c.g3Status) === null || _d === void 0 ? void 0 : _d.exclude) || [];
if (includeFiles.length === 0 && excludeFiles.length === 0) {
debug('No g3Status include or exclude lists are defined in the angular robot configuration,');
debug('skipping.');
return;
const exclude = ((_d = (_c = robotConfig === null || robotConfig === void 0 ? void 0 : robotConfig.merge) === null || _c === void 0 ? void 0 : _c.g3Status) === null || _d === void 0 ? void 0 : _d.exclude) || [];
if (include.length === 0 && exclude.length === 0) {
debug('No g3Status include or exclude lists are defined in the angular robot configuration');
return null;
}
return { include, exclude };
}
getLatestShas() {
/** The latest sha for the g3 branch. */
const g3Ref = getShaForBranchLatest('g3');
const g3 = this.getShaForBranchLatest('g3');
/** The latest sha for the master branch. */
const masterRef = getShaForBranchLatest('master');
if (!g3Ref && !masterRef) {
return debug('Exiting early as either the g3 or master was unable to be retrieved');
const master = this.getShaForBranchLatest('master');
if (g3 === null || master === null) {
debug('Either the g3 or master was unable to be retrieved');
return null;
}
/** The statistical information about the git diff between master and g3. */
const stats = getDiffStats();
info.group(bold('g3 branch check'));
info(`${stats.commits} commits between g3 and master`);
if (stats.files === 0) {
info('✅ No sync is needed at this time');
}
else {
info(`${stats.files} files changed, ${stats.insertions} insertions(+), ${stats.deletions} deletions(-) will be included in the next sync`);
}
info.groupEnd();
info();
/** Fetch and retrieve the latest sha for a specific branch. */
function getShaForBranchLatest(branch) {
/** The result fo the fetch command. */
const fetchResult = git.runGraceful([
'fetch', '-q', `https://github.com/${git.remoteConfig.owner}/${git.remoteConfig.name}.git`,
branch
]);
if (fetchResult.status !== 0 &&
fetchResult.stderr.includes(`couldn't find remote ref ${branch}`)) {
debug(`No '${branch}' branch exists on upstream, skipping.`);
return false;
}
return git.runGraceful(['rev-parse', 'FETCH_HEAD']).stdout.trim();
}
/**
* Get git diff stats between master and g3, for all files and filtered to only g3 affecting
* files.
*/
function getDiffStats() {
/** The diff stats to be returned. */
const stats = {
insertions: 0,
deletions: 0,
files: 0,
commits: 0,
};
// Determine the number of commits between master and g3 refs. */
stats.commits = parseInt(git.run(['rev-list', '--count', `${g3Ref}..${masterRef}`]).stdout, 10);
// Get the numstat information between master and g3
git.run(['diff', `${g3Ref}...${masterRef}`, '--numstat'])
.stdout
// Remove the extra space after git's output.
.trim()
// Split each line of git output into array
.split('\n')
// Split each line from the git output into components parts: insertions,
// deletions and file name respectively
.map(line => line.split('\t'))
// Parse number value from the insertions and deletions values
// Example raw line input:
// 10\t5\tsrc/file/name.ts
.map(line => [Number(line[0]), Number(line[1]), line[2]])
// Add each line's value to the diff stats, and conditionally to the g3
// stats as well if the file name is included in the files synced to g3.
.forEach(([insertions, deletions, fileName]) => {
if (checkMatchAgainstIncludeAndExclude(fileName, includeFiles, excludeFiles)) {
stats.insertions += insertions;
stats.deletions += deletions;
stats.files += 1;
}
});
return stats;
}
/** Determine whether the file name passes both include and exclude checks. */
function checkMatchAgainstIncludeAndExclude(file, includes, excludes) {
return (multimatch.call(undefined, file, includes).length >= 1 &&
multimatch.call(undefined, file, excludes).length === 0);
}
});
return { g3, master };
}
}
/**
@ -1256,32 +1302,56 @@ function printG3Comparison(git) {
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/
/** The fragment for a result from Github's api for a Github query. */
const GithubQueryResultFragment = {
issueCount: typedGraphqlify.types.number,
nodes: [Object.assign({}, typedGraphqlify.onUnion({
PullRequest: {
url: typedGraphqlify.types.string,
},
Issue: {
url: typedGraphqlify.types.string,
},
}))],
};
/**
* Cap the returned issues in the queries to an arbitrary 100. At that point, caretaker has a lot
* Cap the returned issues in the queries to an arbitrary 20. At that point, caretaker has a lot
* of work to do and showing more than that isn't really useful.
*/
const MAX_RETURNED_ISSUES = 20;
/** Retrieve the number of matching issues for each github query. */
function printGithubTasks(git, config) {
var _a;
return tslib.__awaiter(this, void 0, void 0, function* () {
if (!((_a = config === null || config === void 0 ? void 0 : config.githubQueries) === null || _a === void 0 ? void 0 : _a.length)) {
debug('No github queries defined in the configuration, skipping.');
return;
}
info.group(bold(`Github Tasks`));
yield getGithubInfo(git, config);
info.groupEnd();
info();
});
}
/** Retrieve query match counts and log discovered counts to the console. */
function getGithubInfo(git, { githubQueries: queries = [] }) {
return tslib.__awaiter(this, void 0, void 0, function* () {
class GithubQueriesModule extends BaseModule {
retrieveData() {
var _a;
return tslib.__awaiter(this, void 0, void 0, function* () {
// Non-null assertion is used here as the check for undefined immediately follows to confirm the
// assertion. Typescript's type filtering does not seem to work as needed to understand
// whether githubQueries is undefined or not.
let queries = (_a = this.config.caretaker) === null || _a === void 0 ? void 0 : _a.githubQueries;
if (queries === undefined || queries.length === 0) {
debug('No github queries defined in the configuration, skipping');
return;
}
/** The results of the generated github query. */
const queryResult = yield this.git.github.graphql.query(this.buildGraphqlQuery(queries));
const results = Object.values(queryResult);
const { owner, name: repo } = this.git.remoteConfig;
return results.map((result, i) => {
return {
queryName: queries[i].name,
count: result.issueCount,
queryUrl: encodeURI(`https://github.com/${owner}/${repo}/issues?q=${queries[i].query}`),
matchedUrls: result.nodes.map(node => node.url)
};
});
});
}
/** Build a Graphql query statement for the provided queries. */
buildGraphqlQuery(queries) {
/** The query object for graphql. */
const graphQlQuery = {};
const { owner, name: repo } = this.git.remoteConfig;
/** The Github search filter for the configured repository. */
const repoFilter = `repo:${git.remoteConfig.owner}/${git.remoteConfig.name}`;
const repoFilter = `repo:${owner}/${repo}`;
queries.forEach(({ name, query }) => {
/** The name of the query, with spaces removed to match GraphQL requirements. */
const queryKey = typedGraphqlify.alias(name.replace(/ /g, ''), 'search');
@ -1289,37 +1359,33 @@ function getGithubInfo(git, { githubQueries: queries = [] }) {
type: 'ISSUE',
first: MAX_RETURNED_ISSUES,
query: `"${repoFilter} ${query.replace(/"/g, '\\"')}"`,
}, {
issueCount: typedGraphqlify.types.number,
nodes: [Object.assign({}, typedGraphqlify.onUnion({
PullRequest: {
url: typedGraphqlify.types.string,
},
Issue: {
url: typedGraphqlify.types.string,
},
}))],
});
}, Object.assign({}, GithubQueryResultFragment));
});
/** The results of the generated github query. */
const results = yield git.github.graphql.query(graphQlQuery);
Object.values(results).forEach((result, i) => {
var _a, _b;
info(`${(_a = queries[i]) === null || _a === void 0 ? void 0 : _a.name.padEnd(25)} ${result.issueCount}`);
if (result.issueCount > 0) {
const { owner, name: repo } = git.remoteConfig;
const url = encodeURI(`https://github.com/${owner}/${repo}/issues?q=${(_b = queries[i]) === null || _b === void 0 ? void 0 : _b.query}`);
info.group(`${url}`);
if (result.nodes.length === MAX_RETURNED_ISSUES && result.nodes.length < result.issueCount) {
info(`(first ${MAX_RETURNED_ISSUES})`);
}
for (const node of result.nodes) {
info(`- ${node.url}`);
}
info.groupEnd();
return graphQlQuery;
}
printToTerminal() {
return tslib.__awaiter(this, void 0, void 0, function* () {
const queryResults = yield this.data;
if (!queryResults) {
return;
}
info.group(bold('Github Tasks'));
const minQueryNameLength = Math.max(...queryResults.map(result => result.queryName.length));
for (const queryResult of queryResults) {
info(`${queryResult.queryName.padEnd(minQueryNameLength)} ${queryResult.count}`);
if (queryResult.count > 0) {
info.group(queryResult.queryUrl);
queryResult.matchedUrls.forEach(url => info(`- ${url}`));
if (queryResult.count > MAX_RETURNED_ISSUES) {
info(`... ${queryResult.count - MAX_RETURNED_ISSUES} additional matches`);
}
info.groupEnd();
}
}
info.groupEnd();
info();
});
});
}
}
/**
@ -1329,71 +1395,64 @@ function getGithubInfo(git, { githubQueries: queries = [] }) {
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/
/** The status levels for services. */
var ServiceStatus;
(function (ServiceStatus) {
ServiceStatus[ServiceStatus["GREEN"] = 0] = "GREEN";
ServiceStatus[ServiceStatus["RED"] = 1] = "RED";
})(ServiceStatus || (ServiceStatus = {}));
/** Retrieve and log stasuses for all of the services of concern. */
function printServiceStatuses() {
return tslib.__awaiter(this, void 0, void 0, function* () {
info.group(bold(`Service Statuses (checked: ${new Date().toLocaleString()})`));
logStatus('CircleCI', yield getCircleCiStatus());
logStatus('Github', yield getGithubStatus());
logStatus('NPM', yield getNpmStatus());
logStatus('Saucelabs', yield getSaucelabsStatus());
info.groupEnd();
info();
});
}
/** Log the status of the service to the console. */
function logStatus(serviceName, status) {
serviceName = serviceName.padEnd(15);
if (status.status === ServiceStatus.GREEN) {
info(`${serviceName} ${green('✅')}`);
/** List of services Angular relies on. */
const services = [
{
url: 'https://status.us-west-1.saucelabs.com/api/v2/status.json',
name: 'Saucelabs',
},
{
url: 'https://status.npmjs.org/api/v2/status.json',
name: 'Npm',
},
{
url: 'https://status.circleci.com/api/v2/status.json',
name: 'CircleCi',
},
{
url: 'https://www.githubstatus.com/api/v2/status.json',
name: 'Github',
},
];
class ServicesModule extends BaseModule {
retrieveData() {
return tslib.__awaiter(this, void 0, void 0, function* () {
return Promise.all(services.map(service => this.getStatusFromStandardApi(service)));
});
}
else if (status.status === ServiceStatus.RED) {
info.group(`${serviceName} ${red('❌')} (Updated: ${status.lastUpdated.toLocaleString()})`);
info(` Details: ${status.description}`);
info.groupEnd();
printToTerminal() {
return tslib.__awaiter(this, void 0, void 0, function* () {
const statuses = yield this.data;
const serviceNameMinLength = Math.max(...statuses.map(service => service.name.length));
info.group(bold('Service Statuses'));
for (const status of statuses) {
const name = status.name.padEnd(serviceNameMinLength);
if (status.status === 'passing') {
info(`${name} ✅`);
}
else {
info.group(`${name} ❌ (Updated: ${status.lastUpdated.toLocaleString()})`);
info(` Details: ${status.description}`);
info.groupEnd();
}
}
info.groupEnd();
info();
});
}
/** Retrieve the status information for a service which uses a standard API response. */
getStatusFromStandardApi(service) {
return tslib.__awaiter(this, void 0, void 0, function* () {
const result = yield fetch(service.url).then(result => result.json());
const status = result.status.indicator === 'none' ? 'passing' : 'failing';
return {
name: service.name,
status,
description: result.status.description,
lastUpdated: new Date(result.page.updated_at)
};
});
}
}
/** Gets the service status information for Saucelabs. */
function getSaucelabsStatus() {
return tslib.__awaiter(this, void 0, void 0, function* () {
return getStatusFromStandardApi('https://status.us-west-1.saucelabs.com/api/v2/status.json');
});
}
/** Gets the service status information for NPM. */
function getNpmStatus() {
return tslib.__awaiter(this, void 0, void 0, function* () {
return getStatusFromStandardApi('https://status.npmjs.org/api/v2/status.json');
});
}
/** Gets the service status information for CircleCI. */
function getCircleCiStatus() {
return tslib.__awaiter(this, void 0, void 0, function* () {
return getStatusFromStandardApi('https://status.circleci.com/api/v2/status.json');
});
}
/** Gets the service status information for Github. */
function getGithubStatus() {
return tslib.__awaiter(this, void 0, void 0, function* () {
return getStatusFromStandardApi('https://www.githubstatus.com/api/v2/status.json');
});
}
/** Retrieve the status information for a service which uses a standard API response. */
function getStatusFromStandardApi(url) {
return tslib.__awaiter(this, void 0, void 0, function* () {
const result = yield fetch(url).then(result => result.json());
const status = result.status.indicator === 'none' ? ServiceStatus.GREEN : ServiceStatus.RED;
return {
status,
description: result.status.description,
lastUpdated: new Date(result.page.updated_at)
};
});
}
/**
@ -1403,6 +1462,13 @@ function getStatusFromStandardApi(url) {
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/
/** List of modules checked for the caretaker check command. */
const moduleList = [
GithubQueriesModule,
ServicesModule,
CiModule,
G3Module,
];
/** Check the status of services which Angular caretakers need to monitor. */
function checkServiceStatuses(githubToken) {
return tslib.__awaiter(this, void 0, void 0, function* () {
@ -1412,11 +1478,15 @@ function checkServiceStatuses(githubToken) {
const git = new GitClient(githubToken, config);
// Prevent logging of the git commands being executed during the check.
GitClient.LOG_COMMANDS = false;
// TODO(josephperrott): Allow these checks to be loaded in parallel.
yield printServiceStatuses();
yield printGithubTasks(git, config.caretaker);
yield printG3Comparison(git);
yield printCiStatus(git);
/** List of instances of Caretaker Check modules */
const caretakerCheckModules = moduleList.map(module => new module(git, config));
// Module's `data` is casted as Promise<unknown> because the data types of the `module`'s `data`
// promises do not match typings, however our usage here is only to determine when the promise
// resolves.
yield Promise.all(caretakerCheckModules.map(module => module.data));
for (const module of caretakerCheckModules) {
yield module.printToTerminal();
}
});
}

26
dev-infra/utils/testing/virtual-git-client.ts
View File

@ -9,8 +9,25 @@
import {SpawnSyncOptions, SpawnSyncReturns} from 'child_process';
import * as parseArgs from 'minimist';
import {NgDevConfig} from '../config';
import {GitClient} from '../git/index';
/**
* Temporary directory which will be used as project directory in tests. Note that
* this environment variable is automatically set by Bazel for tests.
*/
export const testTmpDir: string = process.env['TEST_TMPDIR']!;
/** A mock instance of a configuration for the ng-dev toolset for default testing. */
export const mockNgDevConfig: NgDevConfig = {
github: {
name: 'name',
owner: 'owner',
}
};
/** Type describing a Git head. */
interface GitHead {
/** Name of the head. Not defined in a detached state. */
@ -171,3 +188,12 @@ export class VirtualGitClient extends GitClient {
};
}
}
/**
* Builds a Virtual Git Client instance with the provided config and set the temporary test
* directory.
*/
export function buildVirtualGitClient(config = mockNgDevConfig, tmpDir = testTmpDir) {
return (new VirtualGitClient(undefined, config, tmpDir));
}