refactor(dev-infra): set up new method for checking range of commits (#41341)

Check a range of commits by retrieving the log files to be parsed with the expected
format for the parser.

This change is in part of a larger set of changes making the process for obtaining
and parsing commits for release note creation and message validation consistent.
This consistency will make it easier to debug as well as ease the design of tooling
which is built on top of these processes.

PR Close #41341

dev-infra/commit-message/BUILD.bazel

@@ -12,11 +12,13 @@ ts_library(
     deps = [
         "//dev-infra/utils",
         "@npm//@types/conventional-commits-parser",
+        "@npm//@types/git-raw-commits",
         "@npm//@types/inquirer",
         "@npm//@types/node",
         "@npm//@types/shelljs",
         "@npm//@types/yargs",
         "@npm//conventional-commits-parser",
+        "@npm//git-raw-commits",
         "@npm//inquirer",
         "@npm//shelljs",
         "@npm//yargs",

dev-infra/commit-message/parse.ts

@@ -8,8 +8,6 @@
 
 import {Commit as ParsedCommit, Options, sync as parse} from 'conventional-commits-parser';
 
-import {exec} from '../utils/shelljs';
-
 
 /** A parsed commit, containing the information needed to validate the commit. */
 export interface Commit {
@@ -43,6 +41,30 @@ export interface Commit {
   isRevert: boolean;
 }
 
+/**
+ * A list of tuples expressing the fields to extract from each commit log entry. The tuple contains
+ * two values, the first is the key for the property and the second is the template shortcut for the
+ * git log command.
+ */
+const commitFields = {
+  hash: '%H',
+  shortHash: '%h',
+  author: '%aN',
+};
+/** The additional fields to be included in commit log entries for parsing. */
+export type CommitFields = typeof commitFields;
+/** The commit fields described as git log format entries for parsing. */
+export const commitFieldsAsFormat = (fields: CommitFields) => {
+  return Object.entries(fields).map(([key, value]) => `%n-${key}-%n${value}`).join('');
+};
+/**
+ * The git log format template to create git log entries for parsing.
+ *
+ * The conventional commits parser expects to parse the standard git log raw body (%B) into its
+ * component parts. Additionally it will parse additional fields with keys defined by
+ * `-{key name}-` separated by new lines.
+ * */
+export const gitLogFormatForParsing = `%B${commitFieldsAsFormat(commitFields)}`;
 /** Markers used to denote the start of a note section in a commit. */
 enum NoteSections {
   BREAKING_CHANGE = 'BREAKING CHANGE',
@@ -87,7 +109,9 @@ const parseOptions: Options&{notesPattern: (keywords: string) => RegExp} = {
 
 
 /** Parse a full commit message into its composite parts. */
-export function parseCommitMessage(fullText: string): Commit {
+export function parseCommitMessage(fullText: string|Buffer): Commit {
+  // Ensure the fullText symbol is a `string`, even if a Buffer was provided.
+  fullText = fullText.toString();
   /** The commit message text with the fixup and squash markers stripped out. */
   const strippedCommitMsg = fullText.replace(FIXUP_PREFIX_RE, '')
                                 .replace(SQUASH_PREFIX_RE, '')
@@ -126,30 +150,3 @@ export function parseCommitMessage(fullText: string): Commit {
     isRevert: REVERT_PREFIX_RE.test(fullText),
   };
 }
-
-/** Retrieve and parse each commit message in a provide range. */
-export function parseCommitMessagesForRange(range: string): Commit[] {
-  /** A random number used as a split point in the git log result. */
-  const randomValueSeparator = `${Math.random()}`;
-  /**
-   * Custom git log format that provides the commit header and body, separated as expected with the
-   * custom separator as the trailing value.
-   */
-  const gitLogFormat = `%s%n%n%b${randomValueSeparator}`;
-
-  // Retrieve the commits in the provided range.
-  const result = exec(`git log --reverse --format=${gitLogFormat} ${range}`);
-  if (result.code) {
-    throw new Error(`Failed to get all commits in the range:\n  ${result.stderr}`);
-  }
-
-  return result
-      // Separate the commits from a single string into individual commits.
-      .split(randomValueSeparator)
-      // Remove extra space before and after each commit message.
-      .map(l => l.trim())
-      // Remove any superfluous lines which remain from the split.
-      .filter(line => !!line)
-      // Parse each commit message.
-      .map(commit => parseCommitMessage(commit));
-}

dev-infra/commit-message/utils.ts

@@ -0,0 +1,32 @@
+/**
+ * @license
+ * Copyright Google LLC All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+import * as gitCommits_ from 'git-raw-commits';
+
+import {Commit, gitLogFormatForParsing, parseCommitMessage} from './parse';
+
+// Set `gitCommits` as this imported value to address "Cannot call a namespace" error.
+const gitCommits = gitCommits_;
+
+
+/**
+ * Find all commits within the given range and return an object describing those.
+ */
+export function getCommitsInRange(from: string, to: string = 'HEAD'): Promise<Commit[]> {
+  return new Promise((resolve, reject) => {
+    /** List of parsed commit objects. */
+    const commits: Commit[] = [];
+    /** Stream of raw git commit strings in the range provided. */
+    const commitStream = gitCommits({from, to, format: gitLogFormatForParsing});
+
+    // Accumulate the parsed commits for each commit from the Readable stream into an array, then
+    // resolve the promise with the array when the Readable stream ends.
+    commitStream.on('data', (commit: Buffer) => commits.push(parseCommitMessage(commit)));
+    commitStream.on('error', (err: Error) => reject(err));
+    commitStream.on('end', () => resolve(commits));
+  });
+}

dev-infra/commit-message/validate-range/cli.ts

@@ -14,21 +14,27 @@ import {validateCommitRange} from './validate-range';
 
 
 export interface ValidateRangeOptions {
-  range: string;
+  startingRef: string;
+  endingRef: string;
 }
 
 /** Builds the command. */
 function builder(yargs: Argv) {
-  return yargs.option('range', {
+  return yargs
-    description: 'The range of commits to check, e.g. --range abc123..xyz456',
+      .positional('startingRef', {
-    demandOption: '  A range must be provided, e.g. --range abc123..xyz456',
+        description: 'The first ref in the range to select',
         type: 'string',
-    requiresArg: true,
+        demandOption: true,
+      })
+      .positional('endingRef', {
+        description: 'The last ref in the range to select',
+        type: 'string',
+        default: 'HEAD',
       });
 }
 
 /** Handles the command. */
-async function handler({range}: Arguments<ValidateRangeOptions>) {
+async function handler({startingRef, endingRef}: Arguments<ValidateRangeOptions>) {
   // If on CI, and no pull request number is provided, assume the branch
   // being run on is an upstream branch.
   if (process.env['CI'] && process.env['CI_PULL_REQUEST'] === 'false') {
@@ -38,13 +44,13 @@ async function handler({range}: Arguments<ValidateRangeOptions>) {
     info(`Skipping check of provided commit range`);
     return;
   }
-  validateCommitRange(range);
+  await validateCommitRange(startingRef, endingRef);
 }
 
 /** yargs command module describing the command. */
 export const ValidateRangeModule: CommandModule<{}, ValidateRangeOptions> = {
   handler,
   builder,
-  command: 'validate-range',
+  command: 'validate-range <starting-ref> [ending-ref]',
   describe: 'Validate a range of commit messages',
 };

dev-infra/commit-message/validate-range/validate-range.ts

@@ -5,8 +5,9 @@
  * Use of this source code is governed by an MIT-style license that can be
  * found in the LICENSE file at https://angular.io/license
  */
-import {error, info} from '../../utils/console';
+import {error, green, info, red} from '../../utils/console';
-import {Commit, parseCommitMessagesForRange} from '../parse';
+import {Commit} from '../parse';
+import {getCommitsInRange} from '../utils';
 import {printValidationErrors, validateCommitMessage, ValidateCommitMessageOptions} from '../validate';
 
 // Whether the provided commit is a fixup commit.
@@ -16,12 +17,13 @@ const isNonFixup = (commit: Commit) => !commit.isFixup;
 const extractCommitHeader = (commit: Commit) => commit.header;
 
 /** Validate all commits in a provided git commit range. */
-export function validateCommitRange(range: string) {
+export async function validateCommitRange(from: string, to: string) {
   /** A list of tuples of the commit header string and a list of error messages for the commit. */
   const errors: [commitHeader: string, errors: string[]][] = [];
+
   /** A list of parsed commit messages from the range. */
-  const commits = parseCommitMessagesForRange(range);
+  const commits = await getCommitsInRange(from, to);
-  info(`Examining ${commits.length} commit(s) in the provided range: ${range}`);
+  info(`Examining ${commits.length} commit(s) in the provided range: ${from}..${to}`);
 
   /**
    * Whether all commits in the range are valid, commits are allowed to be fixup commits for other
@@ -32,7 +34,7 @@ export function validateCommitRange(range: string) {
       disallowSquash: true,
       nonFixupCommitHeaders: isNonFixup(commit) ?
           undefined :
-          commits.slice(0, i).filter(isNonFixup).map(extractCommitHeader)
+          commits.slice(i + 1).filter(isNonFixup).map(extractCommitHeader)
     };
     const {valid, errors: localErrors} = validateCommitMessage(commit, options);
     if (localErrors.length) {
@@ -42,9 +44,9 @@ export function validateCommitRange(range: string) {
   });
 
   if (allCommitsInRangeValid) {
-    info('√  All commit messages in range valid.');
+    info(green('√  All commit messages in range valid.'));
   } else {
-    error('✘  Invalid commit message');
+    error(red('✘  Invalid commit message'));
     errors.forEach(([header, validationErrors]) => {
       error.group(header);
       printValidationErrors(validationErrors);

dev-infra/ng-dev.js

@@ -20,6 +20,7 @@ var semver = require('semver');
 var multimatch = require('multimatch');
 var yaml = require('yaml');
 var conventionalCommitsParser = require('conventional-commits-parser');
+var gitCommits_ = require('git-raw-commits');
 var cliProgress = require('cli-progress');
 var os = require('os');
 var minimatch = require('minimatch');
@@ -1684,6 +1685,28 @@ const COMMIT_TYPES = {
  * Use of this source code is governed by an MIT-style license that can be
  * found in the LICENSE file at https://angular.io/license
  */
+/**
+ * A list of tuples expressing the fields to extract from each commit log entry. The tuple contains
+ * two values, the first is the key for the property and the second is the template shortcut for the
+ * git log command.
+ */
+const commitFields = {
+    hash: '%H',
+    shortHash: '%h',
+    author: '%aN',
+};
+/** The commit fields described as git log format entries for parsing. */
+const commitFieldsAsFormat = (fields) => {
+    return Object.entries(fields).map(([key, value]) => `%n-${key}-%n${value}`).join('');
+};
+/**
+ * The git log format template to create git log entries for parsing.
+ *
+ * The conventional commits parser expects to parse the standard git log raw body (%B) into its
+ * component parts. Additionally it will parse additional fields with keys defined by
+ * `-{key name}-` separated by new lines.
+ * */
+const gitLogFormatForParsing = `%B${commitFieldsAsFormat(commitFields)}`;
 /** Markers used to denote the start of a note section in a commit. */
 var NoteSections;
 (function (NoteSections) {
@@ -1728,6 +1751,8 @@ const parseOptions = {
 };
 /** Parse a full commit message into its composite parts. */
 function parseCommitMessage(fullText) {
+    // Ensure the fullText symbol is a `string`, even if a Buffer was provided.
+    fullText = fullText.toString();
     /** The commit message text with the fixup and squash markers stripped out. */
     const strippedCommitMsg = fullText.replace(FIXUP_PREFIX_RE, '')
         .replace(SQUASH_PREFIX_RE, '')
@@ -1764,30 +1789,6 @@ function parseCommitMessage(fullText) {
         isRevert: REVERT_PREFIX_RE.test(fullText),
     };
 }
-/** Retrieve and parse each commit message in a provide range. */
-function parseCommitMessagesForRange(range) {
-    /** A random number used as a split point in the git log result. */
-    const randomValueSeparator = `${Math.random()}`;
-    /**
-     * Custom git log format that provides the commit header and body, separated as expected with the
-     * custom separator as the trailing value.
-     */
-    const gitLogFormat = `%s%n%n%b${randomValueSeparator}`;
-    // Retrieve the commits in the provided range.
-    const result = exec(`git log --reverse --format=${gitLogFormat} ${range}`);
-    if (result.code) {
-        throw new Error(`Failed to get all commits in the range:\n  ${result.stderr}`);
-    }
-    return result
-        // Separate the commits from a single string into individual commits.
-        .split(randomValueSeparator)
-        // Remove extra space before and after each commit message.
-        .map(l => l.trim())
-        // Remove any superfluous lines which remain from the split.
-        .filter(line => !!line)
-        // Parse each commit message.
-        .map(commit => parseCommitMessage(commit));
-}
 
 /**
  * @license
@@ -2027,17 +2028,37 @@ const ValidateFileModule = {
  * Use of this source code is governed by an MIT-style license that can be
  * found in the LICENSE file at https://angular.io/license
  */
+// Set `gitCommits` as this imported value to address "Cannot call a namespace" error.
+const gitCommits = gitCommits_;
+/**
+ * Find all commits within the given range and return an object describing those.
+ */
+function getCommitsInRange(from, to = 'HEAD') {
+    return new Promise((resolve, reject) => {
+        /** List of parsed commit objects. */
+        const commits = [];
+        /** Stream of raw git commit strings in the range provided. */
+        const commitStream = gitCommits({ from, to, format: gitLogFormatForParsing });
+        // Accumulate the parsed commits for each commit from the Readable stream into an array, then
+        // resolve the promise with the array when the Readable stream ends.
+        commitStream.on('data', (commit) => commits.push(parseCommitMessage(commit)));
+        commitStream.on('error', (err) => reject(err));
+        commitStream.on('end', () => resolve(commits));
+    });
+}
+
 // Whether the provided commit is a fixup commit.
 const isNonFixup = (commit) => !commit.isFixup;
 // Extracts commit header (first line of commit message).
 const extractCommitHeader = (commit) => commit.header;
 /** Validate all commits in a provided git commit range. */
-function validateCommitRange(range) {
+function validateCommitRange(from, to) {
+    return tslib.__awaiter(this, void 0, void 0, function* () {
         /** A list of tuples of the commit header string and a list of error messages for the commit. */
         const errors = [];
         /** A list of parsed commit messages from the range. */
-    const commits = parseCommitMessagesForRange(range);
+        const commits = yield getCommitsInRange(from, to);
-    info(`Examining ${commits.length} commit(s) in the provided range: ${range}`);
+        info(`Examining ${commits.length} commit(s) in the provided range: ${from}..${to}`);
         /**
          * Whether all commits in the range are valid, commits are allowed to be fixup commits for other
          * commits in the provided commit range.
@@ -2047,7 +2068,7 @@ function validateCommitRange(range) {
                 disallowSquash: true,
                 nonFixupCommitHeaders: isNonFixup(commit) ?
                     undefined :
-                commits.slice(0, i).filter(isNonFixup).map(extractCommitHeader)
+                    commits.slice(i + 1).filter(isNonFixup).map(extractCommitHeader)
             };
             const { valid, errors: localErrors } = validateCommitMessage(commit, options);
             if (localErrors.length) {
@@ -2056,10 +2077,10 @@ function validateCommitRange(range) {
             return valid;
         });
         if (allCommitsInRangeValid) {
-        info('√  All commit messages in range valid.');
+            info(green('√  All commit messages in range valid.'));
         }
         else {
-        error('✘  Invalid commit message');
+            error(red('✘  Invalid commit message'));
             errors.forEach(([header, validationErrors]) => {
                 error.group(header);
                 printValidationErrors(validationErrors);
@@ -2069,6 +2090,7 @@ function validateCommitRange(range) {
             // been discovered.
             process.exit(1);
         }
+    });
 }
 
 /**
@@ -2080,15 +2102,20 @@ function validateCommitRange(range) {
  */
 /** Builds the command. */
 function builder$3(yargs) {
-    return yargs.option('range', {
+    return yargs
-        description: 'The range of commits to check, e.g. --range abc123..xyz456',
+        .positional('startingRef', {
-        demandOption: '  A range must be provided, e.g. --range abc123..xyz456',
+        description: 'The first ref in the range to select',
         type: 'string',
-        requiresArg: true,
+        demandOption: true,
+    })
+        .positional('endingRef', {
+        description: 'The last ref in the range to select',
+        type: 'string',
+        default: 'HEAD',
     });
 }
 /** Handles the command. */
-function handler$3({ range }) {
+function handler$3({ startingRef, endingRef }) {
     return tslib.__awaiter(this, void 0, void 0, function* () {
         // If on CI, and no pull request number is provided, assume the branch
         // being run on is an upstream branch.
@@ -2099,14 +2126,14 @@ function handler$3({ range }) {
             info(`Skipping check of provided commit range`);
             return;
         }
-        validateCommitRange(range);
+        yield validateCommitRange(startingRef, endingRef);
     });
 }
 /** yargs command module describing the command. */
 const ValidateRangeModule = {
     handler: handler$3,
     builder: builder$3,
-    command: 'validate-range',
+    command: 'validate-range <starting-ref> [ending-ref]',
     describe: 'Validate a range of commit messages',
 };
 
@@ -4294,7 +4321,7 @@ function rebasePr(prNumber, githubToken, config = getConfig()) {
             info(`Fetching ${fullBaseRef} to rebase #${prNumber} on`);
             git.run(['fetch', '-q', baseRefUrl, baseRefName]);
             const commonAncestorSha = git.run(['merge-base', 'HEAD', 'FETCH_HEAD']).stdout.trim();
-            const commits = parseCommitMessagesForRange(`${commonAncestorSha}..HEAD`);
+            const commits = yield getCommitsInRange(commonAncestorSha, 'HEAD');
             let squashFixups = commits.filter((commit) => commit.isFixup).length === 0 ?
                 false :
                 yield promptConfirm(`PR #${prNumber} contains fixup commits, would you like to squash them during rebase?`, true);

dev-infra/pr/rebase/index.ts

@@ -8,7 +8,8 @@
 
 import {types as graphQLTypes} from 'typed-graphqlify';
 
-import {Commit, parseCommitMessagesForRange} from '../../commit-message/parse';
+import {Commit} from '../../commit-message/parse';
+import {getCommitsInRange} from '../../commit-message/utils';
 import {getConfig, NgDevConfig} from '../../utils/config';
 import {error, info, promptConfirm} from '../../utils/console';
 import {addTokenToGitHttpsUrl} from '../../utils/git/github-urls';
@@ -93,7 +94,7 @@ export async function rebasePr(
 
     const commonAncestorSha = git.run(['merge-base', 'HEAD', 'FETCH_HEAD']).stdout.trim();
 
-    const commits = parseCommitMessagesForRange(`${commonAncestorSha}..HEAD`);
+    const commits = await getCommitsInRange(commonAncestorSha, 'HEAD');
 
     let squashFixups = commits.filter((commit: Commit) => commit.isFixup).length === 0 ?
         false :

dev-infra/tmpl-package.json

@@ -17,6 +17,7 @@
     "chalk": "<from-root>",
     "cli-progress": "<from-root>",
     "conventional-commits-parser": "<from-root>",
+    "git-raw-commits": "<from-root>",
     "glob": "<from-root>",
     "inquirer": "<from-root>",
     "minimatch": "<from-root>",

package.json

@@ -168,6 +168,7 @@
     "@octokit/graphql": "^4.3.1",
     "@types/cli-progress": "^3.4.2",
     "@types/conventional-commits-parser": "^3.0.1",
+    "@types/git-raw-commits": "^2.0.0",
     "@types/minimist": "^1.2.0",
     "@yarnpkg/lockfile": "^1.1.0",
     "browserstacktunnel-wrapper": "^2.0.4",
@@ -182,6 +183,7 @@
     "entities": "1.1.1",
     "firebase-tools": "^7.11.0",
     "firefox-profile": "1.0.3",
+    "git-raw-commits": "^2.0.10",
     "glob": "7.1.2",
     "gulp": "^4.0.2",
     "gulp-conventional-changelog": "^2.0.35",

yarn.lock

@@ -2495,6 +2495,13 @@
   dependencies:
     "@types/node" "*"
 
+"@types/git-raw-commits@^2.0.0":
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/@types/git-raw-commits/-/git-raw-commits-2.0.0.tgz#157e9e4709db0748fb1aa623f8927ddd4864bac6"
+  integrity sha512-sHXOKjKqu1kQxbxkZiz2s0yx2kc7g20g6tE98LYGq5jQyT9r+GRyTn19NBfPotOlXhGO6oPvYT3tdnPl8MYYyA==
+  dependencies:
+    "@types/node" "*"
+
 "@types/glob@*", "@types/glob@^7.1.1":
   version "7.1.1"
   resolved "https://registry.yarnpkg.com/@types/glob/-/glob-7.1.1.tgz#aa59a1c6e3fbc421e07ccd31a944c30eba521575"
@@ -7647,7 +7654,7 @@ git-raw-commits@2.0.0:
     split2 "^2.0.0"
     through2 "^2.0.0"
 
-git-raw-commits@^2.0.8:
+git-raw-commits@^2.0.10, git-raw-commits@^2.0.8:
   version "2.0.10"
   resolved "https://registry.yarnpkg.com/git-raw-commits/-/git-raw-commits-2.0.10.tgz#e2255ed9563b1c9c3ea6bd05806410290297bbc1"
   integrity sha512-sHhX5lsbG9SOO6yXdlwgEMQ/ljIn7qMpAbJZCGfXX2fq5T8M5SrDnpYk9/4HswTildcIqatsWa91vty6VhWSaQ==