honeymoose
/
angular-cn
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor(dev-infra): set up new method for checking range of commits (#41341) 

Check a range of commits by retrieving the log files to be parsed with the expected
format for the parser.

This change is in part of a larger set of changes making the process for obtaining
and parsing commits for release note creation and message validation consistent.
This consistency will make it easier to debug as well as ease the design of tooling
which is built on top of these processes.

PR Close #41341
This commit is contained in:
Joey Perrott 2021-03-24 16:17:15 -07:00 committed by Alex Rickabaugh
parent 18bc9ffb51
commit 381ea9d7d4
14 changed files with 208 additions and 131 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
dev-infra/commit-message/BUILD.bazel
View File

@ -12,11 +12,13 @@ ts_library(
deps = [
"//dev-infra/utils",
"@npm//@types/conventional-commits-parser",
"@npm//@types/git-raw-commits",
"@npm//@types/inquirer",
"@npm//@types/node",
"@npm//@types/shelljs",
"@npm//@types/yargs",
"@npm//conventional-commits-parser",
"@npm//git-raw-commits",
"@npm//inquirer",
"@npm//shelljs",
"@npm//yargs",

2
dev-infra/commit-message/config.ts
View File

@ -31,7 +31,7 @@ export function getCommitMessageConfig() {
return config as Required<typeof config>;
}
/** Scope requirement level to be set for each commit type. */
/** Scope requirement level to be set for each commit type. */
export enum ScopeRequirement {
Required,
Optional,

57
dev-infra/commit-message/parse.ts
View File

@ -8,8 +8,6 @@
import {Commit as ParsedCommit, Options, sync as parse} from 'conventional-commits-parser';
import {exec} from '../utils/shelljs';
/** A parsed commit, containing the information needed to validate the commit. */
export interface Commit {
@ -43,6 +41,30 @@ export interface Commit {
isRevert: boolean;
}
/**
* A list of tuples expressing the fields to extract from each commit log entry. The tuple contains
* two values, the first is the key for the property and the second is the template shortcut for the
* git log command.
*/
const commitFields = {
hash: '%H',
shortHash: '%h',
author: '%aN',
};
/** The additional fields to be included in commit log entries for parsing. */
export type CommitFields = typeof commitFields;
/** The commit fields described as git log format entries for parsing. */
export const commitFieldsAsFormat = (fields: CommitFields) => {
return Object.entries(fields).map(([key, value]) => `%n-${key}-%n${value}`).join('');
};
/**
* The git log format template to create git log entries for parsing.
*
* The conventional commits parser expects to parse the standard git log raw body (%B) into its
* component parts. Additionally it will parse additional fields with keys defined by
* `-{key name}-` separated by new lines.
* */
export const gitLogFormatForParsing = `%B${commitFieldsAsFormat(commitFields)}`;
/** Markers used to denote the start of a note section in a commit. */
enum NoteSections {
BREAKING_CHANGE = 'BREAKING CHANGE',
@ -87,7 +109,9 @@ const parseOptions: Options&{notesPattern: (keywords: string) => RegExp} = {
/** Parse a full commit message into its composite parts. */
export function parseCommitMessage(fullText: string): Commit {
export function parseCommitMessage(fullText: string|Buffer): Commit {
// Ensure the fullText symbol is a `string`, even if a Buffer was provided.
fullText = fullText.toString();
/** The commit message text with the fixup and squash markers stripped out. */
const strippedCommitMsg = fullText.replace(FIXUP_PREFIX_RE, '')
.replace(SQUASH_PREFIX_RE, '')
@ -126,30 +150,3 @@ export function parseCommitMessage(fullText: string): Commit {
isRevert: REVERT_PREFIX_RE.test(fullText),
};
}
/** Retrieve and parse each commit message in a provide range. */
export function parseCommitMessagesForRange(range: string): Commit[] {
/** A random number used as a split point in the git log result. */
const randomValueSeparator = `${Math.random()}`;
/**
* Custom git log format that provides the commit header and body, separated as expected with the
* custom separator as the trailing value.
*/
const gitLogFormat = `%s%n%n%b${randomValueSeparator}`;
// Retrieve the commits in the provided range.
const result = exec(`git log --reverse --format=${gitLogFormat} ${range}`);
if (result.code) {
throw new Error(`Failed to get all commits in the range:\n ${result.stderr}`);
}
return result
// Separate the commits from a single string into individual commits.
.split(randomValueSeparator)
// Remove extra space before and after each commit message.
.map(l => l.trim())
// Remove any superfluous lines which remain from the split.
.filter(line => !!line)
// Parse each commit message.
.map(commit => parseCommitMessage(commit));
}

4
dev-infra/commit-message/restore-commit-message/cli.ts
View File

@ -50,11 +50,11 @@ async function handler({fileEnvVariable, file, source}: Arguments<RestoreCommitM
}
throw new Error(
'No file path and commit message source provide. Provide values via positional command ' +
'No file path and commit message source provide. Provide values via positional command ' +
'arguments, or via the --file-env-variable flag');
}
/** yargs command module describing the command. */
/** yargs command module describing the command. */
export const RestoreCommitMessageModule: CommandModule<{}, RestoreCommitMessageOptions> = {
handler,
builder,

32
dev-infra/commit-message/utils.ts Normal file
View File

@ -0,0 +1,32 @@
/**
* @license
* Copyright Google LLC All Rights Reserved.
*
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/
import * as gitCommits_ from 'git-raw-commits';
import {Commit, gitLogFormatForParsing, parseCommitMessage} from './parse';
// Set `gitCommits` as this imported value to address "Cannot call a namespace" error.
const gitCommits = gitCommits_;
/**
* Find all commits within the given range and return an object describing those.
*/
export function getCommitsInRange(from: string, to: string = 'HEAD'): Promise<Commit[]> {
return new Promise((resolve, reject) => {
/** List of parsed commit objects. */
const commits: Commit[] = [];
/** Stream of raw git commit strings in the range provided. */
const commitStream = gitCommits({from, to, format: gitLogFormatForParsing});
// Accumulate the parsed commits for each commit from the Readable stream into an array, then
// resolve the promise with the array when the Readable stream ends.
commitStream.on('data', (commit: Buffer) => commits.push(parseCommitMessage(commit)));
commitStream.on('error', (err: Error) => reject(err));
commitStream.on('end', () => resolve(commits));
});
}

2
dev-infra/commit-message/validate-file/cli.ts
View File

@ -53,7 +53,7 @@ async function handler({error, file, fileEnvVariable}: Arguments<ValidateFileOpt
validateFile(filePath, error);
}
/** yargs command module describing the command. */
/** yargs command module describing the command. */
export const ValidateFileModule: CommandModule<{}, ValidateFileOptions> = {
handler,
builder,

28
dev-infra/commit-message/validate-range/cli.ts
View File

@ -14,21 +14,27 @@ import {validateCommitRange} from './validate-range';
export interface ValidateRangeOptions {
range: string;
startingRef: string;
endingRef: string;
}
/** Builds the command. */
function builder(yargs: Argv) {
return yargs.option('range', {
description: 'The range of commits to check, e.g. --range abc123..xyz456',
demandOption: ' A range must be provided, e.g. --range abc123..xyz456',
type: 'string',
requiresArg: true,
});
return yargs
.positional('startingRef', {
description: 'The first ref in the range to select',
type: 'string',
demandOption: true,
})
.positional('endingRef', {
description: 'The last ref in the range to select',
type: 'string',
default: 'HEAD',
});
}
/** Handles the command. */
async function handler({range}: Arguments<ValidateRangeOptions>) {
async function handler({startingRef, endingRef}: Arguments<ValidateRangeOptions>) {
// If on CI, and no pull request number is provided, assume the branch
// being run on is an upstream branch.
if (process.env['CI'] && process.env['CI_PULL_REQUEST'] === 'false') {
@ -38,13 +44,13 @@ async function handler({range}: Arguments<ValidateRangeOptions>) {
info(`Skipping check of provided commit range`);
return;
}
validateCommitRange(range);
await validateCommitRange(startingRef, endingRef);
}
/** yargs command module describing the command. */
/** yargs command module describing the command. */
export const ValidateRangeModule: CommandModule<{}, ValidateRangeOptions> = {
handler,
builder,
command: 'validate-range',
command: 'validate-range <starting-ref> [ending-ref]',
describe: 'Validate a range of commit messages',
};

18
dev-infra/commit-message/validate-range/validate-range.ts
View File

@ -5,8 +5,9 @@
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/
import {error, info} from '../../utils/console';
import {Commit, parseCommitMessagesForRange} from '../parse';
import {error, green, info, red} from '../../utils/console';
import {Commit} from '../parse';
import {getCommitsInRange} from '../utils';
import {printValidationErrors, validateCommitMessage, ValidateCommitMessageOptions} from '../validate';
// Whether the provided commit is a fixup commit.
@ -16,12 +17,13 @@ const isNonFixup = (commit: Commit) => !commit.isFixup;
const extractCommitHeader = (commit: Commit) => commit.header;
/** Validate all commits in a provided git commit range. */
export function validateCommitRange(range: string) {
export async function validateCommitRange(from: string, to: string) {
/** A list of tuples of the commit header string and a list of error messages for the commit. */
const errors: [commitHeader: string, errors: string[]][] = [];
/** A list of parsed commit messages from the range. */
const commits = parseCommitMessagesForRange(range);
info(`Examining ${commits.length} commit(s) in the provided range: ${range}`);
const commits = await getCommitsInRange(from, to);
info(`Examining ${commits.length} commit(s) in the provided range: ${from}..${to}`);
/**
* Whether all commits in the range are valid, commits are allowed to be fixup commits for other
@ -32,7 +34,7 @@ export function validateCommitRange(range: string) {
disallowSquash: true,
nonFixupCommitHeaders: isNonFixup(commit) ?
undefined :
commits.slice(0, i).filter(isNonFixup).map(extractCommitHeader)
commits.slice(i + 1).filter(isNonFixup).map(extractCommitHeader)
};
const {valid, errors: localErrors} = validateCommitMessage(commit, options);
if (localErrors.length) {
@ -42,9 +44,9 @@ export function validateCommitRange(range: string) {
});
if (allCommitsInRangeValid) {
info('√ All commit messages in range valid.');
info(green('√ All commit messages in range valid.'));
} else {
error('✘ Invalid commit message');
error(red('✘ Invalid commit message'));
errors.forEach(([header, validationErrors]) => {
error.group(header);
printValidationErrors(validationErrors);

175
dev-infra/ng-dev.js
View File

@ -20,6 +20,7 @@ var semver = require('semver');
var multimatch = require('multimatch');
var yaml = require('yaml');
var conventionalCommitsParser = require('conventional-commits-parser');
var gitCommits_ = require('git-raw-commits');
var cliProgress = require('cli-progress');
var os = require('os');
var minimatch = require('minimatch');
@ -1588,11 +1589,11 @@ function handler$1({ fileEnvVariable, file, source }) {
restoreCommitMessage(fileFromEnv, sourceFromEnv);
return;
}
throw new Error('No file path and commit message source provide. Provide values via positional command ' +
throw new Error('No file path and commit message source provide. Provide values via positional command ' +
'arguments, or via the --file-env-variable flag');
});
}
/** yargs command module describing the command. */
/** yargs command module describing the command. */
const RestoreCommitMessageModule = {
handler: handler$1,
builder: builder$1,
@ -1621,7 +1622,7 @@ function getCommitMessageConfig() {
assertNoErrors(errors);
return config;
}
/** Scope requirement level to be set for each commit type. */
/** Scope requirement level to be set for each commit type. */
var ScopeRequirement;
(function (ScopeRequirement) {
ScopeRequirement[ScopeRequirement["Required"] = 0] = "Required";
@ -1684,6 +1685,28 @@ const COMMIT_TYPES = {
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/
/**
* A list of tuples expressing the fields to extract from each commit log entry. The tuple contains
* two values, the first is the key for the property and the second is the template shortcut for the
* git log command.
*/
const commitFields = {
hash: '%H',
shortHash: '%h',
author: '%aN',
};
/** The commit fields described as git log format entries for parsing. */
const commitFieldsAsFormat = (fields) => {
return Object.entries(fields).map(([key, value]) => `%n-${key}-%n${value}`).join('');
};
/**
* The git log format template to create git log entries for parsing.
*
* The conventional commits parser expects to parse the standard git log raw body (%B) into its
* component parts. Additionally it will parse additional fields with keys defined by
* `-{key name}-` separated by new lines.
* */
const gitLogFormatForParsing = `%B${commitFieldsAsFormat(commitFields)}`;
/** Markers used to denote the start of a note section in a commit. */
var NoteSections;
(function (NoteSections) {
@ -1728,6 +1751,8 @@ const parseOptions = {
};
/** Parse a full commit message into its composite parts. */
function parseCommitMessage(fullText) {
// Ensure the fullText symbol is a `string`, even if a Buffer was provided.
fullText = fullText.toString();
/** The commit message text with the fixup and squash markers stripped out. */
const strippedCommitMsg = fullText.replace(FIXUP_PREFIX_RE, '')
.replace(SQUASH_PREFIX_RE, '')
@ -1764,30 +1789,6 @@ function parseCommitMessage(fullText) {
isRevert: REVERT_PREFIX_RE.test(fullText),
};
}
/** Retrieve and parse each commit message in a provide range. */
function parseCommitMessagesForRange(range) {
/** A random number used as a split point in the git log result. */
const randomValueSeparator = `${Math.random()}`;
/**
* Custom git log format that provides the commit header and body, separated as expected with the
* custom separator as the trailing value.
*/
const gitLogFormat = `%s%n%n%b${randomValueSeparator}`;
// Retrieve the commits in the provided range.
const result = exec(`git log --reverse --format=${gitLogFormat} ${range}`);
if (result.code) {
throw new Error(`Failed to get all commits in the range:\n ${result.stderr}`);
}
return result
// Separate the commits from a single string into individual commits.
.split(randomValueSeparator)
// Remove extra space before and after each commit message.
.map(l => l.trim())
// Remove any superfluous lines which remain from the split.
.filter(line => !!line)
// Parse each commit message.
.map(commit => parseCommitMessage(commit));
}
/**
* @license
@ -2012,7 +2013,7 @@ function handler$2({ error, file, fileEnvVariable }) {
validateFile(filePath, error);
});
}
/** yargs command module describing the command. */
/** yargs command module describing the command. */
const ValidateFileModule = {
handler: handler$2,
builder: builder$2,
@ -2027,48 +2028,69 @@ const ValidateFileModule = {
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/
// Set `gitCommits` as this imported value to address "Cannot call a namespace" error.
const gitCommits = gitCommits_;
/**
* Find all commits within the given range and return an object describing those.
*/
function getCommitsInRange(from, to = 'HEAD') {
return new Promise((resolve, reject) => {
/** List of parsed commit objects. */
const commits = [];
/** Stream of raw git commit strings in the range provided. */
const commitStream = gitCommits({ from, to, format: gitLogFormatForParsing });
// Accumulate the parsed commits for each commit from the Readable stream into an array, then
// resolve the promise with the array when the Readable stream ends.
commitStream.on('data', (commit) => commits.push(parseCommitMessage(commit)));
commitStream.on('error', (err) => reject(err));
commitStream.on('end', () => resolve(commits));
});
}
// Whether the provided commit is a fixup commit.
const isNonFixup = (commit) => !commit.isFixup;
// Extracts commit header (first line of commit message).
const extractCommitHeader = (commit) => commit.header;
/** Validate all commits in a provided git commit range. */
function validateCommitRange(range) {
/** A list of tuples of the commit header string and a list of error messages for the commit. */
const errors = [];
/** A list of parsed commit messages from the range. */
const commits = parseCommitMessagesForRange(range);
info(`Examining ${commits.length} commit(s) in the provided range: ${range}`);
/**
* Whether all commits in the range are valid, commits are allowed to be fixup commits for other
* commits in the provided commit range.
*/
const allCommitsInRangeValid = commits.every((commit, i) => {
const options = {
disallowSquash: true,
nonFixupCommitHeaders: isNonFixup(commit) ?
undefined :
commits.slice(0, i).filter(isNonFixup).map(extractCommitHeader)
};
const { valid, errors: localErrors } = validateCommitMessage(commit, options);
if (localErrors.length) {
errors.push([commit.header, localErrors]);
}
return valid;
});
if (allCommitsInRangeValid) {
info('√ All commit messages in range valid.');
}
else {
error('✘ Invalid commit message');
errors.forEach(([header, validationErrors]) => {
error.group(header);
printValidationErrors(validationErrors);
error.groupEnd();
function validateCommitRange(from, to) {
return tslib.__awaiter(this, void 0, void 0, function* () {
/** A list of tuples of the commit header string and a list of error messages for the commit. */
const errors = [];
/** A list of parsed commit messages from the range. */
const commits = yield getCommitsInRange(from, to);
info(`Examining ${commits.length} commit(s) in the provided range: ${from}..${to}`);
/**
* Whether all commits in the range are valid, commits are allowed to be fixup commits for other
* commits in the provided commit range.
*/
const allCommitsInRangeValid = commits.every((commit, i) => {
const options = {
disallowSquash: true,
nonFixupCommitHeaders: isNonFixup(commit) ?
undefined :
commits.slice(i + 1).filter(isNonFixup).map(extractCommitHeader)
};
const { valid, errors: localErrors } = validateCommitMessage(commit, options);
if (localErrors.length) {
errors.push([commit.header, localErrors]);
}
return valid;
});
// Exit with a non-zero exit code if invalid commit messages have
// been discovered.
process.exit(1);
}
if (allCommitsInRangeValid) {
info(green('√ All commit messages in range valid.'));
}
else {
error(red('✘ Invalid commit message'));
errors.forEach(([header, validationErrors]) => {
error.group(header);
printValidationErrors(validationErrors);
error.groupEnd();
});
// Exit with a non-zero exit code if invalid commit messages have
// been discovered.
process.exit(1);
}
});
}
/**
@ -2080,15 +2102,20 @@ function validateCommitRange(range) {
*/
/** Builds the command. */
function builder$3(yargs) {
return yargs.option('range', {
description: 'The range of commits to check, e.g. --range abc123..xyz456',
demandOption: ' A range must be provided, e.g. --range abc123..xyz456',
return yargs
.positional('startingRef', {
description: 'The first ref in the range to select',
type: 'string',
requiresArg: true,
demandOption: true,
})
.positional('endingRef', {
description: 'The last ref in the range to select',
type: 'string',
default: 'HEAD',
});
}
/** Handles the command. */
function handler$3({ range }) {
function handler$3({ startingRef, endingRef }) {
return tslib.__awaiter(this, void 0, void 0, function* () {
// If on CI, and no pull request number is provided, assume the branch
// being run on is an upstream branch.
@ -2099,14 +2126,14 @@ function handler$3({ range }) {
info(`Skipping check of provided commit range`);
return;
}
validateCommitRange(range);
yield validateCommitRange(startingRef, endingRef);
});
}
/** yargs command module describing the command. */
/** yargs command module describing the command. */
const ValidateRangeModule = {
handler: handler$3,
builder: builder$3,
command: 'validate-range',
command: 'validate-range <starting-ref> [ending-ref]',
describe: 'Validate a range of commit messages',
};
@ -4210,7 +4237,7 @@ function handler$6(_a) {
});
});
}
/** yargs command module describing the command. */
/** yargs command module describing the command. */
var MergeCommandModule = {
handler: handler$6,
builder: builder$6,
@ -4294,7 +4321,7 @@ function rebasePr(prNumber, githubToken, config = getConfig()) {
info(`Fetching ${fullBaseRef} to rebase #${prNumber} on`);
git.run(['fetch', '-q', baseRefUrl, baseRefName]);
const commonAncestorSha = git.run(['merge-base', 'HEAD', 'FETCH_HEAD']).stdout.trim();
const commits = parseCommitMessagesForRange(`${commonAncestorSha}..HEAD`);
const commits = yield getCommitsInRange(commonAncestorSha, 'HEAD');
let squashFixups = commits.filter((commit) => commit.isFixup).length === 0 ?
false :
yield promptConfirm(`PR #${prNumber} contains fixup commits, would you like to squash them during rebase?`, true);

2
dev-infra/pr/merge/cli.ts
View File

@ -41,7 +41,7 @@ async function handler({pr, githubToken, branchPrompt}: Arguments<MergeCommandOp
await mergePullRequest(pr, githubToken, {branchPrompt});
}
/** yargs command module describing the command. */
/** yargs command module describing the command. */
export const MergeCommandModule: CommandModule<{}, MergeCommandOptions> = {
handler,
builder,

5
dev-infra/pr/rebase/index.ts
View File

@ -8,7 +8,8 @@
import {types as graphQLTypes} from 'typed-graphqlify';
import {Commit, parseCommitMessagesForRange} from '../../commit-message/parse';
import {Commit} from '../../commit-message/parse';
import {getCommitsInRange} from '../../commit-message/utils';
import {getConfig, NgDevConfig} from '../../utils/config';
import {error, info, promptConfirm} from '../../utils/console';
import {addTokenToGitHttpsUrl} from '../../utils/git/github-urls';
@ -93,7 +94,7 @@ export async function rebasePr(
const commonAncestorSha = git.run(['merge-base', 'HEAD', 'FETCH_HEAD']).stdout.trim();
const commits = parseCommitMessagesForRange(`${commonAncestorSha}..HEAD`);
const commits = await getCommitsInRange(commonAncestorSha, 'HEAD');
let squashFixups = commits.filter((commit: Commit) => commit.isFixup).length === 0 ?
false :

1
dev-infra/tmpl-package.json
View File

@ -17,6 +17,7 @@
"chalk": "<from-root>",
"cli-progress": "<from-root>",
"conventional-commits-parser": "<from-root>",
"git-raw-commits": "<from-root>",
"glob": "<from-root>",
"inquirer": "<from-root>",
"minimatch": "<from-root>",

2
package.json
View File

@ -168,6 +168,7 @@
"@octokit/graphql": "^4.3.1",
"@types/cli-progress": "^3.4.2",
"@types/conventional-commits-parser": "^3.0.1",
"@types/git-raw-commits": "^2.0.0",
"@types/minimist": "^1.2.0",
"@yarnpkg/lockfile": "^1.1.0",
"browserstacktunnel-wrapper": "^2.0.4",
@ -182,6 +183,7 @@
"entities": "1.1.1",
"firebase-tools": "^7.11.0",
"firefox-profile": "1.0.3",
"git-raw-commits": "^2.0.10",
"glob": "7.1.2",
"gulp": "^4.0.2",
"gulp-conventional-changelog": "^2.0.35",

9
yarn.lock
View File

@ -2495,6 +2495,13 @@
dependencies:
"@types/node" "*"
"@types/git-raw-commits@^2.0.0":
version "2.0.0"
resolved "https://registry.yarnpkg.com/@types/git-raw-commits/-/git-raw-commits-2.0.0.tgz#157e9e4709db0748fb1aa623f8927ddd4864bac6"
integrity sha512-sHXOKjKqu1kQxbxkZiz2s0yx2kc7g20g6tE98LYGq5jQyT9r+GRyTn19NBfPotOlXhGO6oPvYT3tdnPl8MYYyA==
dependencies:
"@types/node" "*"
"@types/glob@*", "@types/glob@^7.1.1":
version "7.1.1"
resolved "https://registry.yarnpkg.com/@types/glob/-/glob-7.1.1.tgz#aa59a1c6e3fbc421e07ccd31a944c30eba521575"
@ -7647,7 +7654,7 @@ git-raw-commits@2.0.0:
split2 "^2.0.0"
through2 "^2.0.0"
git-raw-commits@^2.0.8:
git-raw-commits@^2.0.10, git-raw-commits@^2.0.8:
version "2.0.10"
resolved "https://registry.yarnpkg.com/git-raw-commits/-/git-raw-commits-2.0.10.tgz#e2255ed9563b1c9c3ea6bd05806410290297bbc1"
integrity sha512-sHhX5lsbG9SOO6yXdlwgEMQ/ljIn7qMpAbJZCGfXX2fq5T8M5SrDnpYk9/4HswTildcIqatsWa91vty6VhWSaQ==