build: create tool for validating typescript circular dependencies (#35647)

Creates a tool for validating TypeScript circular dependencies. The tool
has been designed in a way that allows us to slowly burn down the amount
of circular dependencies while ensuring that we don't regress.

The tool doesn't rely on Madge since it doesn't provide a programmatic
way for doing path mapping. We need path mapping since we also want to
check for cycles across different entry-points or packages. The tool
uses the TypeScript AST to manually collect cycles. This code is not
a lot of bloat and also gives us more flexibility (if we ever need it).

Closes #35041.

PR Close #35647

package.json

@@ -34,7 +34,8 @@
     "lint": "yarn -s tslint && yarn gulp lint",
     "tslint": "tsc -p tools/tsconfig.json && tslint -c tslint.json \"+(packages|modules|scripts|tools)/**/*.+(js|ts)\"",
     "public-api:check": "node goldens/public-api/manage.js test",
-    "public-api:update": "node goldens/public-api/manage.js accept"
+    "public-api:update": "node goldens/public-api/manage.js accept",
+    "ts-circular-deps": "ts-node --project tools/ts-circular-dependencies/tsconfig.json tools/ts-circular-dependencies/index.ts"
   },
   "// 1": "dependencies are used locally and by bazel",
   "dependencies": {

tools/ts-circular-dependencies/analyzer.ts

@@ -0,0 +1,133 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import {readFileSync} from 'fs';
+import {dirname, join, resolve} from 'path';
+import * as ts from 'typescript';
+
+import {getFileStatus} from './file_system';
+import {getModuleReferences} from './parser';
+
+export type ModuleResolver = (specifier: string) => string | null;
+
+/**
+ * Reference chains describe a sequence of source files which are connected through imports.
+ * e.g. `file_a.ts` imports `file_b.ts`, whereas `file_b.ts` imports `file_c.ts`. The reference
+ * chain data structure could be used to represent this import sequence.
+ */
+export type ReferenceChain<T = ts.SourceFile> = T[];
+
+/** Default extensions that the analyzer uses for resolving imports. */
+const DEFAULT_EXTENSIONS = ['ts', 'js', 'd.ts'];
+
+/**
+ * Analyzer that can be used to detect import cycles within source files. It supports
+ * custom module resolution, source file caching and collects unresolved specifiers.
+ */
+export class Analyzer {
+  private _sourceFileCache = new Map<string, ts.SourceFile>();
+
+  unresolvedModules = new Set<string>();
+  unresolvedFiles = new Map<string, string[]>();
+
+  constructor(
+      public resolveModuleFn?: ModuleResolver, public extensions: string[] = DEFAULT_EXTENSIONS) {}
+
+  /** Finds all cycles in the specified source file. */
+  findCycles(sf: ts.SourceFile, visited = new WeakSet<ts.SourceFile>(), path: ReferenceChain = []):
+      ReferenceChain[] {
+    const previousIndex = path.indexOf(sf);
+    // If the given node is already part of the current path, then a cycle has
+    // been found. Add the reference chain which represents the cycle to the results.
+    if (previousIndex !== -1) {
+      return [path.slice(previousIndex)];
+    }
+    // If the node has already been visited, then it's not necessary to go check its edges
+    // again. Cycles would have been already detected and collected in the first check.
+    if (visited.has(sf)) {
+      return [];
+    }
+    path.push(sf);
+    visited.add(sf);
+    // Go through all edges, which are determined through import/exports, and collect cycles.
+    const result: ReferenceChain[] = [];
+    for (const ref of getModuleReferences(sf)) {
+      const targetFile = this._resolveImport(ref, sf.fileName);
+      if (targetFile !== null) {
+        result.push(...this.findCycles(this.getSourceFile(targetFile), visited, path.slice()));
+      }
+    }
+    return result;
+  }
+
+  /** Gets the TypeScript source file of the specified path. */
+  getSourceFile(filePath: string): ts.SourceFile {
+    const resolvedPath = resolve(filePath);
+    if (this._sourceFileCache.has(resolvedPath)) {
+      return this._sourceFileCache.get(resolvedPath) !;
+    }
+    const fileContent = readFileSync(resolvedPath, 'utf8');
+    const sourceFile =
+        ts.createSourceFile(resolvedPath, fileContent, ts.ScriptTarget.Latest, false);
+    this._sourceFileCache.set(resolvedPath, sourceFile);
+    return sourceFile;
+  }
+
+  /** Resolves the given import specifier with respect to the specified containing file path. */
+  private _resolveImport(specifier: string, containingFilePath: string): string|null {
+    if (specifier.charAt(0) === '.') {
+      const resolvedPath = this._resolveFileSpecifier(specifier, containingFilePath);
+      if (resolvedPath === null) {
+        this._trackUnresolvedFileImport(specifier, containingFilePath);
+      }
+      return resolvedPath;
+    }
+    if (this.resolveModuleFn) {
+      const targetFile = this.resolveModuleFn(specifier);
+      if (targetFile !== null) {
+        const resolvedPath = this._resolveFileSpecifier(targetFile);
+        if (resolvedPath !== null) {
+          return resolvedPath;
+        }
+      }
+    }
+    this.unresolvedModules.add(specifier);
+    return null;
+  }
+
+  /** Tracks the given file import as unresolved. */
+  private _trackUnresolvedFileImport(specifier: string, originFilePath: string) {
+    if (!this.unresolvedFiles.has(originFilePath)) {
+      this.unresolvedFiles.set(originFilePath, [specifier]);
+    }
+    this.unresolvedFiles.get(originFilePath) !.push(specifier);
+  }
+
+  /** Resolves the given import specifier to the corresponding source file. */
+  private _resolveFileSpecifier(specifier: string, containingFilePath?: string): string|null {
+    const importFullPath =
+        containingFilePath !== undefined ? join(dirname(containingFilePath), specifier) : specifier;
+    const stat = getFileStatus(importFullPath);
+    if (stat && stat.isFile()) {
+      return importFullPath;
+    }
+    for (const extension of this.extensions) {
+      const pathWithExtension = `${importFullPath}.${extension}`;
+      const stat = getFileStatus(pathWithExtension);
+      if (stat && stat.isFile()) {
+        return pathWithExtension;
+      }
+    }
+    // Directories should be considered last. TypeScript first looks for source files, then
+    // falls back to directories if no file with appropriate extension could be found.
+    if (stat && stat.isDirectory()) {
+      return this._resolveFileSpecifier(join(importFullPath, 'index'));
+    }
+    return null;
+  }
+}

tools/ts-circular-dependencies/file_system.ts

@@ -0,0 +1,23 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import {Stats, statSync} from 'fs';
+
+/** Gets the status of the specified file. Returns null if the file does not exist. */
+export function getFileStatus(filePath: string): Stats|null {
+  try {
+    return statSync(filePath);
+  } catch {
+    return null;
+  }
+}
+
+/** Ensures that the specified path uses forward slashes as delimiter. */
+export function convertPathToForwardSlash(path: string) {
+  return path.replace(/\\/g, '/');
+}

tools/ts-circular-dependencies/golden.ts

@@ -0,0 +1,58 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import {relative} from 'path';
+
+import {ReferenceChain} from './analyzer';
+import {convertPathToForwardSlash} from './file_system';
+
+export type CircularDependency = ReferenceChain<string>;
+export type Golden = CircularDependency[];
+
+/**
+ * Converts a list of reference chains to a JSON-compatible golden object. Reference chains
+ * by default use TypeScript source file objects. In order to make those chains printable,
+ * the source file objects are mapped to their relative file names.
+ */
+export function convertReferenceChainToGolden(refs: ReferenceChain[], baseDir: string): Golden {
+  return refs.map(
+      chain => chain.map(({fileName}) => convertPathToForwardSlash(relative(baseDir, fileName))));
+}
+
+/**
+ * Compares the specified goldens and returns two lists that describe newly
+ * added circular dependencies, or fixed circular dependencies.
+ */
+export function compareGoldens(actual: Golden, expected: Golden) {
+  const newCircularDeps: CircularDependency[] = [];
+  const fixedCircularDeps: CircularDependency[] = [];
+  actual.forEach(a => {
+    if (!expected.find(e => isSameCircularDependency(a, e))) {
+      newCircularDeps.push(a);
+    }
+  });
+  expected.forEach(e => {
+    if (!actual.find(a => isSameCircularDependency(e, a))) {
+      fixedCircularDeps.push(e);
+    }
+  });
+  return {newCircularDeps, fixedCircularDeps};
+}
+
+/** Checks whether the specified circular dependencies are equal. */
+function isSameCircularDependency(actual: CircularDependency, expected: CircularDependency) {
+  if (actual.length !== expected.length) {
+    return false;
+  }
+  for (let i = 0; i < actual.length; i++) {
+    if (actual[i] !== expected[i]) {
+      return false;
+    }
+  }
+  return true;
+}

tools/ts-circular-dependencies/index.ts

@@ -0,0 +1,149 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import {existsSync, readFileSync, writeFileSync} from 'fs';
+import {sync as globSync} from 'glob';
+import {join, relative, resolve} from 'path';
+import * as ts from 'typescript';
+import * as yargs from 'yargs';
+import chalk from 'chalk';
+
+import {Analyzer, ReferenceChain} from './analyzer';
+import {compareGoldens, convertReferenceChainToGolden, Golden} from './golden';
+import {convertPathToForwardSlash} from './file_system';
+
+const projectDir = join(__dirname, '../../');
+const packagesDir = join(projectDir, 'packages/');
+// The default glob does not capture deprecated packages such as http, or the webworker platform.
+const defaultGlob =
+    join(packagesDir, '!(http|platform-webworker|platform-webworker-dynamic)/**/*.ts');
+
+if (require.main === module) {
+  const {_: command, goldenFile, glob, baseDir, warnings} =
+      yargs.help()
+          .version(false)
+          .strict()
+          .command('check <golden-file>', 'Checks if the circular dependencies have changed.')
+          .command('approve <golden-file>', 'Approves the current circular dependencies.')
+          .demandCommand()
+          .option(
+              'approve',
+              {type: 'boolean', description: 'Approves the current circular dependencies.'})
+          .option('warnings', {type: 'boolean', description: 'Prints all warnings.'})
+          .option('base-dir', {
+            type: 'string',
+            description: 'Base directory used for shortening paths in the golden file.',
+            default: projectDir,
+            defaultDescription: 'Project directory'
+          })
+          .option('glob', {
+            type: 'string',
+            description: 'Glob that matches source files which should be checked.',
+            default: defaultGlob,
+            defaultDescription: 'All release packages'
+          })
+          .argv;
+  const isApprove = command.includes('approve');
+  process.exit(main(baseDir, isApprove, goldenFile, glob, warnings));
+}
+
+/**
+ * Runs the ts-circular-dependencies tool.
+ * @param baseDir Base directory which is used to build up relative file paths in goldens.
+ * @param approve Whether the detected circular dependencies should be approved.
+ * @param goldenFile Path to the golden file.
+ * @param glob Glob that is used to collect all source files which should be checked/approved.
+ * @param printWarnings Whether warnings should be printed. Warnings for unresolved modules/files
+ *     are not printed by default.
+ * @returns Status code.
+ */
+export function main(
+    baseDir: string, approve: boolean, goldenFile: string, glob: string,
+    printWarnings: boolean): number {
+  const analyzer = new Analyzer(resolveModule);
+  const cycles: ReferenceChain[] = [];
+  const checkedNodes = new WeakSet<ts.SourceFile>();
+
+  globSync(glob, {absolute: true}).forEach(filePath => {
+    const sourceFile = analyzer.getSourceFile(filePath);
+    cycles.push(...analyzer.findCycles(sourceFile, checkedNodes));
+  });
+
+  const actual = convertReferenceChainToGolden(cycles, baseDir);
+
+  console.info(
+      chalk.green(`   Current number of cycles: ${chalk.yellow(cycles.length.toString())}`));
+
+  if (approve) {
+    writeFileSync(goldenFile, JSON.stringify(actual, null, 2));
+    console.info(chalk.green('✅  Updated golden file.'));
+    return 0;
+  } else if (!existsSync(goldenFile)) {
+    console.error(chalk.red(`❌  Could not find golden file: ${goldenFile}`));
+    return 1;
+  }
+
+  // By default, warnings for unresolved files or modules are not printed. This is because
+  // it's common that third-party modules are not resolved/visited. Also generated files
+  // from the View Engine compiler (i.e. factories, summaries) cannot be resolved.
+  if (printWarnings &&
+      (analyzer.unresolvedFiles.size !== 0 || analyzer.unresolvedModules.size !== 0)) {
+    console.info(chalk.yellow('The following imports could not be resolved:'));
+    analyzer.unresolvedModules.forEach(specifier => console.info(`  • ${specifier}`));
+    analyzer.unresolvedFiles.forEach((value, key) => {
+      console.info(`  • ${getRelativePath(baseDir, key)}`);
+      value.forEach(specifier => console.info(`      ${specifier}`));
+    });
+  }
+
+  const expected: Golden = JSON.parse(readFileSync(goldenFile, 'utf8'));
+  const {fixedCircularDeps, newCircularDeps} = compareGoldens(actual, expected);
+  const isMatching = fixedCircularDeps.length === 0 && newCircularDeps.length === 0;
+
+  if (isMatching) {
+    console.info(chalk.green('✅  Golden matches current circular dependencies.'));
+    return 0;
+  }
+
+  console.error(chalk.red('❌  Golden does not match current circular dependencies.'));
+  if (newCircularDeps.length !== 0) {
+    console.error(chalk.yellow(`   New circular dependencies which are not allowed:`));
+    newCircularDeps.forEach(c => console.error(`     • ${convertReferenceChainToString(c)}`));
+  }
+  if (fixedCircularDeps.length !== 0) {
+    console.error(
+        chalk.yellow(`   Fixed circular dependencies that need to be removed from the golden:`));
+    fixedCircularDeps.forEach(c => console.error(`     • ${convertReferenceChainToString(c)}`));
+    console.info();
+    console.info(chalk.yellow(
+        `   Please update the golden. The following command can be ` +
+        `run: yarn ts-circular-deps approve ${getRelativePath(baseDir, goldenFile)}.`));
+  }
+  return 1;
+}
+
+/** Gets the specified path relative to the base directory. */
+function getRelativePath(baseDir: string, path: string) {
+  return convertPathToForwardSlash(relative(baseDir, path));
+}
+
+/** Converts the given reference chain to its string representation. */
+function convertReferenceChainToString(chain: ReferenceChain<string>) {
+  return chain.join(' → ');
+}
+
+/**
+ * Custom module resolver that maps specifiers starting with `@angular/` to the
+ * local packages folder.
+ */
+function resolveModule(specifier: string) {
+  if (specifier.startsWith('@angular/')) {
+    return packagesDir + specifier.substr('@angular/'.length);
+  }
+  return null;
+}

tools/ts-circular-dependencies/parser.ts

@@ -0,0 +1,27 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import * as ts from 'typescript';
+
+/**
+ * Finds all module references in the specified source file.
+ * @param node Source file which should be parsed.
+ * @returns List of import specifiers in the source file.
+ */
+export function getModuleReferences(node: ts.SourceFile): string[] {
+  const references: string[] = [];
+  const visitNode = (node: ts.Node) => {
+    if ((ts.isImportDeclaration(node) || ts.isExportDeclaration(node)) &&
+        node.moduleSpecifier !== undefined && ts.isStringLiteral(node.moduleSpecifier)) {
+      references.push(node.moduleSpecifier.text);
+    }
+    ts.forEachChild(node, visitNode);
+  };
+  ts.forEachChild(node, visitNode);
+  return references;
+}

tools/ts-circular-dependencies/tsconfig.json

@@ -0,0 +1,7 @@
+{
+  "compilerOptions": {
+    "module": "commonjs",
+    "target": "es2015",
+    "strict": true
+  }
+}