fix(ngcc): capture dynamic import expressions as well as declarations (#37075)

Previously we only checked for static import declaration statements.
This commit also finds import paths from dynamic import expressions.

Also this commit should speed up processing: Previously we were parsing
the source code contents into a `ts.SourceFile` and then walking the parsed
AST to find import paths.
Generating an AST is unnecessary work and it is faster and creates less
memory pressure to just scan the source code contents with the TypeScript
scanner, identifying import paths from the tokens.

PR Close #37075

packages/compiler-cli/ngcc/src/dependencies/esm_dependency_host.ts

@@ -13,20 +13,204 @@ import {DependencyHostBase} from './dependency_host';
  * Helper functions for computing dependencies.
  */
 export class EsmDependencyHost extends DependencyHostBase {
+  // By skipping trivia here we don't have to account for it in the processing below
+  // It has no relevance to capturing imports.
+  private scanner = ts.createScanner(ts.ScriptTarget.Latest, /* skipTrivia */ true);
+
   protected canSkipFile(fileContents: string): boolean {
     return !hasImportOrReexportStatements(fileContents);
   }
 
   protected extractImports(file: AbsoluteFsPath, fileContents: string): Set<string> {
-    const imports: string[] = [];
+    const imports = new Set<string>();
-    // Parse the source into a TypeScript AST and then walk it looking for imports and re-exports.
+    const templateStack: ts.SyntaxKind[] = [];
-    const sf =
+    let lastToken: ts.SyntaxKind = ts.SyntaxKind.Unknown;
-        ts.createSourceFile(file, fileContents, ts.ScriptTarget.ES2015, false, ts.ScriptKind.JS);
+    let currentToken: ts.SyntaxKind = ts.SyntaxKind.Unknown;
-    return new Set(sf.statements
+
-                       // filter out statements that are not imports or reexports
+    this.scanner.setText(fileContents);
-                       .filter(isStringImportOrReexport)
+
-                       // Grab the id of the module that is being imported
+    while ((currentToken = this.scanner.scan()) !== ts.SyntaxKind.EndOfFileToken) {
-                       .map(stmt => stmt.moduleSpecifier.text));
+      switch (currentToken) {
+        case ts.SyntaxKind.TemplateHead:
+          templateStack.push(currentToken);
+          break;
+        case ts.SyntaxKind.OpenBraceToken:
+          if (templateStack.length > 0) {
+            templateStack.push(currentToken);
+          }
+          break;
+        case ts.SyntaxKind.CloseBraceToken:
+          if (templateStack.length > 0) {
+            const templateToken = templateStack[templateStack.length - 1];
+            if (templateToken === ts.SyntaxKind.TemplateHead) {
+              currentToken = this.scanner.reScanTemplateToken(/* isTaggedTemplate */ false);
+              if (currentToken === ts.SyntaxKind.TemplateTail) {
+                templateStack.pop();
+              }
+            } else {
+              templateStack.pop();
+            }
+          }
+          break;
+        case ts.SyntaxKind.SlashToken:
+        case ts.SyntaxKind.SlashEqualsToken:
+          if (canPrecedeARegex(lastToken)) {
+            currentToken = this.scanner.reScanSlashToken();
+          }
+          break;
+        case ts.SyntaxKind.ImportKeyword:
+          const importPath = this.extractImportPath();
+          if (importPath !== null) {
+            imports.add(importPath);
+          }
+          break;
+        case ts.SyntaxKind.ExportKeyword:
+          const reexportPath = this.extractReexportPath();
+          if (reexportPath !== null) {
+            imports.add(reexportPath);
+          }
+          break;
+      }
+      lastToken = currentToken;
+    }
+
+    // Clear the text from the scanner.
+    this.scanner.setText('');
+
+    return imports;
+  }
+
+
+  /**
+   * We have found an `import` token so now try to identify the import path.
+   *
+   * This method will use the current state of `this.scanner` to extract a string literal module
+   * specifier. It expects that the current state of the scanner is that an `import` token has just
+   * been scanned.
+   *
+   * The following forms of import are matched:
+   *
+   * * `import "module-specifier";`
+   * * `import("module-specifier")`
+   * * `import defaultBinding from "module-specifier";`
+   * * `import defaultBinding, * as identifier from "module-specifier";`
+   * * `import defaultBinding, {...} from "module-specifier";`
+   * * `import * as identifier from "module-specifier";`
+   * * `import {...} from "module-specifier";`
+   *
+   * @returns the import path or null if there is no import or it is not a string literal.
+   */
+  protected extractImportPath(): string|null {
+    // Check for side-effect import
+    let sideEffectImportPath = this.tryStringLiteral();
+    if (sideEffectImportPath !== null) {
+      return sideEffectImportPath;
+    }
+
+    let kind: ts.SyntaxKind|null = this.scanner.getToken();
+
+    // Check for dynamic import expression
+    if (kind === ts.SyntaxKind.OpenParenToken) {
+      return this.tryStringLiteral();
+    }
+
+    // Check for defaultBinding
+    if (kind === ts.SyntaxKind.Identifier) {
+      // Skip default binding
+      kind = this.scanner.scan();
+      if (kind === ts.SyntaxKind.CommaToken) {
+        // Skip comma that indicates additional import bindings
+        kind = this.scanner.scan();
+      }
+    }
+
+    // Check for namespace import clause
+    if (kind === ts.SyntaxKind.AsteriskToken) {
+      kind = this.skipNamespacedClause();
+      if (kind === null) {
+        return null;
+      }
+    }
+    // Check for named imports clause
+    else if (kind === ts.SyntaxKind.OpenBraceToken) {
+      kind = this.skipNamedClause();
+    }
+
+    // Expect a `from` clause, if not bail out
+    if (kind !== ts.SyntaxKind.FromKeyword) {
+      return null;
+    }
+
+    return this.tryStringLiteral();
+  }
+
+  /**
+   * We have found an `export` token so now try to identify a re-export path.
+   *
+   * This method will use the current state of `this.scanner` to extract a string literal module
+   * specifier. It expects that the current state of the scanner is that an `export` token has
+   * just been scanned.
+   *
+   * There are three forms of re-export that are matched:
+   *
+   * * `export * from '...';
+   * * `export * as alias from '...';
+   * * `export {...} from '...';
+   */
+  protected extractReexportPath(): string|null {
+    // Skip the `export` keyword
+    let token: ts.SyntaxKind|null = this.scanner.scan();
+    if (token === ts.SyntaxKind.AsteriskToken) {
+      token = this.skipNamespacedClause();
+      if (token === null) {
+        return null;
+      }
+    } else if (token === ts.SyntaxKind.OpenBraceToken) {
+      token = this.skipNamedClause();
+    }
+    // Expect a `from` clause, if not bail out
+    if (token !== ts.SyntaxKind.FromKeyword) {
+      return null;
+    }
+    return this.tryStringLiteral();
+  }
+
+  protected skipNamespacedClause(): ts.SyntaxKind|null {
+    // Skip past the `*`
+    let token = this.scanner.scan();
+    // Check for a `* as identifier` alias clause
+    if (token === ts.SyntaxKind.AsKeyword) {
+      // Skip past the `as` keyword
+      token = this.scanner.scan();
+      // Expect an identifier, if not bail out
+      if (token !== ts.SyntaxKind.Identifier) {
+        return null;
+      }
+      // Skip past the identifier
+      token = this.scanner.scan();
+    }
+    return token;
+  }
+
+  protected skipNamedClause(): ts.SyntaxKind {
+    let braceCount = 1;
+    // Skip past the initial opening brace `{`
+    let token = this.scanner.scan();
+    // Search for the matching closing brace `}`
+    while (braceCount > 0 && token !== ts.SyntaxKind.EndOfFileToken) {
+      if (token === ts.SyntaxKind.OpenBraceToken) {
+        braceCount++;
+      } else if (token === ts.SyntaxKind.CloseBraceToken) {
+        braceCount--;
+      }
+      token = this.scanner.scan();
+    }
+    return token;
+  }
+
+  protected tryStringLiteral(): string|null {
+    return this.scanner.scan() === ts.SyntaxKind.StringLiteral ? this.scanner.getTokenValue() :
+                                                                 null;
   }
 }
 
@@ -56,3 +240,25 @@ export function isStringImportOrReexport(stmt: ts.Statement): stmt is ts.ImportD
       ts.isExportDeclaration(stmt) && !!stmt.moduleSpecifier &&
       ts.isStringLiteral(stmt.moduleSpecifier);
 }
+
+
+function canPrecedeARegex(kind: ts.SyntaxKind): boolean {
+  switch (kind) {
+    case ts.SyntaxKind.Identifier:
+    case ts.SyntaxKind.StringLiteral:
+    case ts.SyntaxKind.NumericLiteral:
+    case ts.SyntaxKind.BigIntLiteral:
+    case ts.SyntaxKind.RegularExpressionLiteral:
+    case ts.SyntaxKind.ThisKeyword:
+    case ts.SyntaxKind.PlusPlusToken:
+    case ts.SyntaxKind.MinusMinusToken:
+    case ts.SyntaxKind.CloseParenToken:
+    case ts.SyntaxKind.CloseBracketToken:
+    case ts.SyntaxKind.CloseBraceToken:
+    case ts.SyntaxKind.TrueKeyword:
+    case ts.SyntaxKind.FalseKeyword:
+      return false;
+    default:
+      return true;
+  }
+}

packages/compiler-cli/ngcc/test/dependencies/esm_dependency_host_spec.ts

@@ -57,6 +57,17 @@ runInEachFileSystem(() => {
         expect(dependencies.has(_('/node_modules/lib-1/sub-1'))).toBe(true);
       });
 
+      it('should resolve all the external dynamic imports of the source file', () => {
+        const {dependencies, missing, deepImports} = createDependencyInfo();
+        host.collectDependencies(
+            _('/external/dynamic/index.js'), {dependencies, missing, deepImports});
+        expect(dependencies.size).toBe(2);
+        expect(missing.size).toBe(0);
+        expect(deepImports.size).toBe(0);
+        expect(dependencies.has(_('/node_modules/lib-1'))).toBe(true);
+        expect(dependencies.has(_('/node_modules/lib-1/sub-1'))).toBe(true);
+      });
+
       it('should capture missing external imports', () => {
         const {dependencies, missing, deepImports} = createDependencyInfo();
         host.collectDependencies(
@@ -184,6 +195,13 @@ runInEachFileSystem(() => {
         },
         {name: _('/external/imports/package.json'), contents: '{"esm2015": "./index.js"}'},
         {name: _('/external/imports/index.metadata.json'), contents: 'MOCK METADATA'},
+        {
+          name: _('/external/dynamic/index.js'),
+          contents:
+              `async function foo() { await const x = import('lib-1');\n const promise = import('lib-1/sub-1'); }`
+        },
+        {name: _('/external/dynamic/package.json'), contents: '{"esm2015": "./index.js"}'},
+        {name: _('/external/dynamic/index.metadata.json'), contents: 'MOCK METADATA'},
         {
           name: _('/external/re-exports/index.js'),
           contents: `export {X} from 'lib-1';\nexport {\n   Y,\n   Z\n} from 'lib-1/sub-1';`