ci: add verification of the pullapprove config (#35060)

Verify that all files in the repo are covered by the pullapprove config
and that all rules in the pullapprove config match at least one file
in the repo.

PR Close #35060

.circleci/config.yml

@@ -271,6 +271,7 @@ jobs:
               (echo -e "\n.bzl files have lint errors. Please run ''yarn bazel:lint-fix''"; exit 1)'
 
       - run: yarn gulp lint
+      - run: node tools/pullapprove/verify.js
 
   test:
     executor:

package.json

@@ -69,6 +69,7 @@
     "@types/semver": "^6.0.2",
     "@types/shelljs": "^0.8.6",
     "@types/systemjs": "0.19.32",
+    "@types/yaml": "^1.2.0",
     "@types/yargs": "^11.1.1",
     "@webcomponents/custom-elements": "^1.0.4",
     "angular": "npm:angular@1.7",
@@ -104,6 +105,7 @@
     "karma-sourcemap-loader": "^0.3.7",
     "magic-string": "^0.25.0",
     "materialize-css": "1.0.0",
+    "minimatch": "^3.0.4",
     "minimist": "1.2.0",
     "node-uuid": "1.4.8",
     "nodejs-websocket": "^1.7.2",
@@ -127,6 +129,7 @@
     "tslint": "5.7.0",
     "typescript": "~3.7.4",
     "xhr2": "0.1.4",
+    "yaml": "^1.7.2",
     "yargs": "13.1.0"
   },
   "// 2": "devDependencies are not used under Bazel. Many can be removed after test.sh is deleted.",

tools/pullapprove/verify.js

@@ -0,0 +1,206 @@
+#!/usr/bin/env node
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+/* tslint:disable:no-console  */
+const parseYaml = require('yaml').parse;
+const readFileSync = require('fs').readFileSync;
+const Minimatch = require('minimatch').Minimatch;
+const {exec, set, cd} = require('shelljs');
+const path = require('path');
+
+// Exit early on shelljs errors
+set('-e');
+
+// Regex Matcher for contains_any_globs conditions
+const CONTAINS_ANY_GLOBS_REGEX = /^'([^']+)',?$/;
+
+// Full path of the angular project directory
+const ANGULAR_PROJECT_DIR = path.resolve(__dirname, '../..');
+// Change to the Angular project directory
+cd(ANGULAR_PROJECT_DIR);
+
+// Whether to log verbosely
+const VERBOSE_MODE = process.argv.includes('-v');
+// Full path to PullApprove config file
+const PULL_APPROVE_YAML_PATH = path.resolve(ANGULAR_PROJECT_DIR, '.pullapprove.yml');
+// All relative path file names in the git repo, this is retrieved using git rather
+// that a glob so that we only get files that are checked in, ignoring things like
+// node_modules, .bazelrc.user, etc
+const ALL_FILES = exec('git ls-tree --full-tree -r --name-only HEAD', {silent: true})
+                      .trim()
+                      .split('\n')
+                      .filter(_ => _);
+if (!ALL_FILES.length) {
+  console.error(
+      `No files were found to be in the git tree, did you run this command from \n` +
+      `inside the angular repository?`);
+  process.exit(1);
+}
+
+/** Gets the glob matching information from each group's condition. */
+function getGlobMatchersFromCondition(groupName, condition) {
+  const trimmedCondition = condition.trim();
+  const globMatchers = [];
+  const badConditionLines = [];
+
+  // If the condition starts with contains_any_globs, evaluate all of the globs
+  if (trimmedCondition.startsWith('contains_any_globs')) {
+    trimmedCondition.split('\n')
+        .slice(1, -1)
+        .map(glob => {
+          const trimmedGlob = glob.trim();
+          const match = trimmedGlob.match(CONTAINS_ANY_GLOBS_REGEX);
+          if (!match) {
+            badConditionLines.push(trimmedGlob);
+            return;
+          }
+          return match[1];
+        })
+        .filter(globString => !!globString)
+        .forEach(globString => globMatchers.push({
+          group: groupName,
+          glob: globString,
+          matcher: new Minimatch(globString, {dot: true}),
+          matchCount: 0,
+        }));
+  }
+  return [globMatchers, badConditionLines];
+}
+
+/** Create logs for each review group. */
+function logGroups(groups) {
+  Array.from(groups.entries()).sort().forEach(([groupName, globs]) => {
+    console.groupCollapsed(groupName);
+    Array.from(globs.values())
+        .sort((a, b) => b.matchCount - a.matchCount)
+        .forEach(glob => console.log(`${glob.glob} - ${glob.matchCount}`));
+    console.groupEnd();
+  });
+}
+
+/** Logs a header within a text drawn box. */
+function logHeader(...params) {
+  const totalWidth = 80;
+  const fillWidth = totalWidth - 2;
+  const headerText = params.join(' ').substr(0, fillWidth);
+  const leftSpace = Math.ceil((fillWidth - headerText.length) / 2);
+  const rightSpace = fillWidth - leftSpace - headerText.length;
+  const fill = (count, content) => content.repeat(count);
+
+  console.log(`┌${fill(fillWidth, '─')}┐`);
+  console.log(`│${fill(leftSpace, ' ')}${headerText}${fill(rightSpace, ' ')}│`);
+  console.log(`└${fill(fillWidth, '─')}┘`);
+}
+
+/** Runs the pull approve verification check on provided files. */
+function runVerification(files) {
+  // All of the globs created for each group's conditions.
+  const allGlobs = [];
+  // The pull approve config file.
+  const pullApprove = readFileSync(PULL_APPROVE_YAML_PATH, {encoding: 'utf8'});
+  // All of the PullApprove groups, parsed from the PullApprove yaml file.
+  const parsedPullApproveGroups = parseYaml(pullApprove).groups;
+  // All files which were found to match a condition in PullApprove.
+  const matchedFiles = new Set();
+  // All files which were not found to match a condition in PullApprove.
+  const unmatchedFiles = new Set();
+  // All PullApprove groups which matched at least one file.
+  const matchedGroups = new Map();
+  // All PullApprove groups which did not match at least one file.
+  const unmatchedGroups = new Map();
+  // All condition lines which were not able to be correctly parsed, by group.
+  const badConditionLinesByGroup = new Map();
+  // Total number of condition lines which were not able to be correctly parsed.
+  let badConditionLineCount = 0;
+
+  // Get all of the globs from the PullApprove group conditions.
+  Object.entries(parsedPullApproveGroups).map(([groupName, group]) => {
+    for (const condition of group.conditions) {
+      const [matchers, badConditions] = getGlobMatchersFromCondition(groupName, condition);
+      if (badConditions.length) {
+        badConditionLinesByGroup.set(groupName, badConditions);
+        badConditionLineCount += badConditions.length;
+      }
+      allGlobs.push(...matchers);
+    }
+  });
+
+  if (badConditionLineCount) {
+    console.log(`Discovered ${badConditionLineCount} parsing errors in PullApprove conditions`);
+    console.log(`Attempted parsing using: ${CONTAINS_ANY_GLOBS_REGEX}`);
+    console.log();
+    console.log(`Unable to properly parse the following line(s) by group:`);
+    for (const [groupName, badConditionLines] of badConditionLinesByGroup.entries()) {
+      console.log(`- ${groupName}:`);
+      badConditionLines.forEach(line => console.log(`    ${line}`));
+    }
+    console.log();
+    console.log(
+        `Please correct the invalid conditions, before PullApprove verification can be completed`);
+    process.exit(1);
+  }
+
+  // Check each file for if it is matched by a PullApprove condition.
+  for (let file of files) {
+    const matched = allGlobs.filter(glob => glob.matcher.match(file));
+    matched.length ? matchedFiles.add(file) : unmatchedFiles.add(file);
+    matched.forEach(glob => glob.matchCount++);
+  }
+
+  // Add each glob for each group to a map either matched or unmatched.
+  allGlobs.forEach(glob => {
+    const groups = glob.matchCount ? matchedGroups : unmatchedGroups;
+    const globs = groups.get(glob.group) || new Map();
+    // Set the globs map in the groups map
+    groups.set(glob.group, globs);
+    // Set the glob in the globs map
+    globs.set(glob.glob, glob);
+  });
+
+  // PullApprove is considered verified if no files or groups are found to be unsed.
+  const verificationSucceeded = !(unmatchedFiles.size || unmatchedGroups.size);
+
+  /**
+   * Overall result
+   */
+  logHeader('Result');
+  if (verificationSucceeded) {
+    console.log('PullApprove verification succeeded!');
+  } else {
+    console.log(`PullApprove verification failed.\n`);
+    console.log(`Please update '.pullapprove.yml' to ensure that all necessary`);
+    console.log(`files/directories have owners and all patterns that appear in`);
+    console.log(`the file correspond to actual files/directories in the repo.`);
+  }
+  /**
+   * File by file Summary
+   */
+  logHeader('PullApprove file match results');
+  console.groupCollapsed(`Matched Files (${matchedFiles.size} files)`);
+  VERBOSE_MODE && matchedFiles.forEach(file => console.log(file));
+  console.groupEnd();
+  console.groupCollapsed(`Unmatched Files (${unmatchedFiles.size} files)`);
+  unmatchedFiles.forEach(file => console.log(file));
+  console.groupEnd();
+
+  /**
+   * Group by group Summary
+   */
+  logHeader('PullApprove group matches');
+  console.groupCollapsed(`Matched Groups (${matchedGroups.size} groups)`);
+  VERBOSE_MODE && logGroups(matchedGroups);
+  console.groupEnd();
+  console.groupCollapsed(`Unmatched Groups (${unmatchedGroups.size} groups)`);
+  logGroups(unmatchedGroups);
+  console.groupEnd();
+
+  // Provide correct exit code based on verification success.
+  process.exit(verificationSucceeded ? 0 : 1);
+}
+
+runVerification(ALL_FILES);

yarn.lock

@@ -936,6 +936,13 @@
     js-levenshtein "^1.1.3"
     semver "^5.5.0"
 
+"@babel/runtime@^7.6.3":
+  version "7.8.3"
+  resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.8.3.tgz#0811944f73a6c926bb2ad35e918dcc1bfab279f1"
+  integrity sha512-fVHx1rzEmwB130VTkLnxR+HmxcTjGzH12LYQcFFoBwakMd3aOMD4OsRN7tGG/UOYE2ektgFrS8uACAoRk1CY0w==
+  dependencies:
+    regenerator-runtime "^0.13.2"
+
 "@babel/template@^7.7.4":
   version "7.7.4"
   resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.7.4.tgz#428a7d9eecffe27deac0a98e23bf8e3675d2a77b"
@@ -1720,6 +1727,11 @@
     "@types/source-list-map" "*"
     source-map "^0.6.1"
 
+"@types/yaml@^1.2.0":
+  version "1.2.0"
+  resolved "https://registry.yarnpkg.com/@types/yaml/-/yaml-1.2.0.tgz#4ed577fc4ebbd6b829b28734e56d10c9e6984e09"
+  integrity sha512-GW8b9qM+ebgW3/zjzPm0I1NxMvLaz/YKT9Ph6tTb+Fkeyzd9yLTvQ6ciQ2MorTRmb/qXmfjMerRpG4LviixaqQ==
+
 "@types/yargs@^11.1.1":
   version "11.1.1"
   resolved "https://registry.yarnpkg.com/@types/yargs/-/yargs-11.1.1.tgz#2e724257167fd6b615dbe4e54301e65fe597433f"
@@ -12375,7 +12387,7 @@ regenerate@^1.4.0:
   resolved "https://registry.yarnpkg.com/regenerate/-/regenerate-1.4.0.tgz#4a856ec4b56e4077c557589cae85e7a4c8869a11"
   integrity sha512-1G6jJVDWrt0rK99kBjvEtziZNCICAuvIPkSiUFIQxVP06RCVpq3dmDo2oi6ABpYaDYaTRr67BEhL8r1wgEZZKg==
 
-regenerator-runtime@0.13.3:
+regenerator-runtime@0.13.3, regenerator-runtime@^0.13.2:
   version "0.13.3"
   resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.13.3.tgz#7cf6a77d8f5c6f60eb73c5fc1955b2ceb01e6bf5"
   integrity sha512-naKIZz2GQ8JWh///G7L3X6LaQUAMp2lvb1rvwwsURe/VXwD6VMfr+/1NuNw3ag8v2kY1aQ/go5SNn79O9JU7yw==
@@ -15636,6 +15648,13 @@ yallist@^4.0.0:
   resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72"
   integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==
 
+yaml@^1.7.2:
+  version "1.7.2"
+  resolved "https://registry.yarnpkg.com/yaml/-/yaml-1.7.2.tgz#f26aabf738590ab61efaca502358e48dc9f348b2"
+  integrity sha512-qXROVp90sb83XtAoqE8bP9RwAkTTZbugRUTm5YeFCBfNRPEp2YzTeqWiz7m5OORHzEvrA/qcGS8hp/E+MMROYw==
+  dependencies:
+    "@babel/runtime" "^7.6.3"
+
 yargs-parser@^11.1.1:
   version "11.1.1"
   resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-11.1.1.tgz#879a0865973bca9f6bab5cbdf3b1c67ec7d3bcf4"