From 07a80161182298e8fdacf0ece0413f8c0200f030 Mon Sep 17 00:00:00 2001 From: Pete Bacon Darwin Date: Thu, 4 Jun 2020 08:43:04 +0100 Subject: [PATCH] fix(ngcc): capture dynamic import expressions as well as declarations (#37075) Previously we only checked for static import declaration statements. This commit also finds import paths from dynamic import expressions. Also this commit should speed up processing: Previously we were parsing the source code contents into a `ts.SourceFile` and then walking the parsed AST to find import paths. Generating an AST is unnecessary work and it is faster and creates less memory pressure to just scan the source code contents with the TypeScript scanner, identifying import paths from the tokens. PR Close #37075 --- .../src/dependencies/esm_dependency_host.ts | 224 +++++++++++++++++- .../dependencies/esm_dependency_host_spec.ts | 18 ++ 2 files changed, 233 insertions(+), 9 deletions(-) diff --git a/packages/compiler-cli/ngcc/src/dependencies/esm_dependency_host.ts b/packages/compiler-cli/ngcc/src/dependencies/esm_dependency_host.ts index 71e8aa2923..c8ad1718bf 100644 --- a/packages/compiler-cli/ngcc/src/dependencies/esm_dependency_host.ts +++ b/packages/compiler-cli/ngcc/src/dependencies/esm_dependency_host.ts @@ -13,20 +13,204 @@ import {DependencyHostBase} from './dependency_host'; * Helper functions for computing dependencies. */ export class EsmDependencyHost extends DependencyHostBase { + // By skipping trivia here we don't have to account for it in the processing below + // It has no relevance to capturing imports. + private scanner = ts.createScanner(ts.ScriptTarget.Latest, /* skipTrivia */ true); + protected canSkipFile(fileContents: string): boolean { return !hasImportOrReexportStatements(fileContents); } protected extractImports(file: AbsoluteFsPath, fileContents: string): Set { - const imports: string[] = []; - // Parse the source into a TypeScript AST and then walk it looking for imports and re-exports. - const sf = - ts.createSourceFile(file, fileContents, ts.ScriptTarget.ES2015, false, ts.ScriptKind.JS); - return new Set(sf.statements - // filter out statements that are not imports or reexports - .filter(isStringImportOrReexport) - // Grab the id of the module that is being imported - .map(stmt => stmt.moduleSpecifier.text)); + const imports = new Set(); + const templateStack: ts.SyntaxKind[] = []; + let lastToken: ts.SyntaxKind = ts.SyntaxKind.Unknown; + let currentToken: ts.SyntaxKind = ts.SyntaxKind.Unknown; + + this.scanner.setText(fileContents); + + while ((currentToken = this.scanner.scan()) !== ts.SyntaxKind.EndOfFileToken) { + switch (currentToken) { + case ts.SyntaxKind.TemplateHead: + templateStack.push(currentToken); + break; + case ts.SyntaxKind.OpenBraceToken: + if (templateStack.length > 0) { + templateStack.push(currentToken); + } + break; + case ts.SyntaxKind.CloseBraceToken: + if (templateStack.length > 0) { + const templateToken = templateStack[templateStack.length - 1]; + if (templateToken === ts.SyntaxKind.TemplateHead) { + currentToken = this.scanner.reScanTemplateToken(/* isTaggedTemplate */ false); + if (currentToken === ts.SyntaxKind.TemplateTail) { + templateStack.pop(); + } + } else { + templateStack.pop(); + } + } + break; + case ts.SyntaxKind.SlashToken: + case ts.SyntaxKind.SlashEqualsToken: + if (canPrecedeARegex(lastToken)) { + currentToken = this.scanner.reScanSlashToken(); + } + break; + case ts.SyntaxKind.ImportKeyword: + const importPath = this.extractImportPath(); + if (importPath !== null) { + imports.add(importPath); + } + break; + case ts.SyntaxKind.ExportKeyword: + const reexportPath = this.extractReexportPath(); + if (reexportPath !== null) { + imports.add(reexportPath); + } + break; + } + lastToken = currentToken; + } + + // Clear the text from the scanner. + this.scanner.setText(''); + + return imports; + } + + + /** + * We have found an `import` token so now try to identify the import path. + * + * This method will use the current state of `this.scanner` to extract a string literal module + * specifier. It expects that the current state of the scanner is that an `import` token has just + * been scanned. + * + * The following forms of import are matched: + * + * * `import "module-specifier";` + * * `import("module-specifier")` + * * `import defaultBinding from "module-specifier";` + * * `import defaultBinding, * as identifier from "module-specifier";` + * * `import defaultBinding, {...} from "module-specifier";` + * * `import * as identifier from "module-specifier";` + * * `import {...} from "module-specifier";` + * + * @returns the import path or null if there is no import or it is not a string literal. + */ + protected extractImportPath(): string|null { + // Check for side-effect import + let sideEffectImportPath = this.tryStringLiteral(); + if (sideEffectImportPath !== null) { + return sideEffectImportPath; + } + + let kind: ts.SyntaxKind|null = this.scanner.getToken(); + + // Check for dynamic import expression + if (kind === ts.SyntaxKind.OpenParenToken) { + return this.tryStringLiteral(); + } + + // Check for defaultBinding + if (kind === ts.SyntaxKind.Identifier) { + // Skip default binding + kind = this.scanner.scan(); + if (kind === ts.SyntaxKind.CommaToken) { + // Skip comma that indicates additional import bindings + kind = this.scanner.scan(); + } + } + + // Check for namespace import clause + if (kind === ts.SyntaxKind.AsteriskToken) { + kind = this.skipNamespacedClause(); + if (kind === null) { + return null; + } + } + // Check for named imports clause + else if (kind === ts.SyntaxKind.OpenBraceToken) { + kind = this.skipNamedClause(); + } + + // Expect a `from` clause, if not bail out + if (kind !== ts.SyntaxKind.FromKeyword) { + return null; + } + + return this.tryStringLiteral(); + } + + /** + * We have found an `export` token so now try to identify a re-export path. + * + * This method will use the current state of `this.scanner` to extract a string literal module + * specifier. It expects that the current state of the scanner is that an `export` token has + * just been scanned. + * + * There are three forms of re-export that are matched: + * + * * `export * from '...'; + * * `export * as alias from '...'; + * * `export {...} from '...'; + */ + protected extractReexportPath(): string|null { + // Skip the `export` keyword + let token: ts.SyntaxKind|null = this.scanner.scan(); + if (token === ts.SyntaxKind.AsteriskToken) { + token = this.skipNamespacedClause(); + if (token === null) { + return null; + } + } else if (token === ts.SyntaxKind.OpenBraceToken) { + token = this.skipNamedClause(); + } + // Expect a `from` clause, if not bail out + if (token !== ts.SyntaxKind.FromKeyword) { + return null; + } + return this.tryStringLiteral(); + } + + protected skipNamespacedClause(): ts.SyntaxKind|null { + // Skip past the `*` + let token = this.scanner.scan(); + // Check for a `* as identifier` alias clause + if (token === ts.SyntaxKind.AsKeyword) { + // Skip past the `as` keyword + token = this.scanner.scan(); + // Expect an identifier, if not bail out + if (token !== ts.SyntaxKind.Identifier) { + return null; + } + // Skip past the identifier + token = this.scanner.scan(); + } + return token; + } + + protected skipNamedClause(): ts.SyntaxKind { + let braceCount = 1; + // Skip past the initial opening brace `{` + let token = this.scanner.scan(); + // Search for the matching closing brace `}` + while (braceCount > 0 && token !== ts.SyntaxKind.EndOfFileToken) { + if (token === ts.SyntaxKind.OpenBraceToken) { + braceCount++; + } else if (token === ts.SyntaxKind.CloseBraceToken) { + braceCount--; + } + token = this.scanner.scan(); + } + return token; + } + + protected tryStringLiteral(): string|null { + return this.scanner.scan() === ts.SyntaxKind.StringLiteral ? this.scanner.getTokenValue() : + null; } } @@ -56,3 +240,25 @@ export function isStringImportOrReexport(stmt: ts.Statement): stmt is ts.ImportD ts.isExportDeclaration(stmt) && !!stmt.moduleSpecifier && ts.isStringLiteral(stmt.moduleSpecifier); } + + +function canPrecedeARegex(kind: ts.SyntaxKind): boolean { + switch (kind) { + case ts.SyntaxKind.Identifier: + case ts.SyntaxKind.StringLiteral: + case ts.SyntaxKind.NumericLiteral: + case ts.SyntaxKind.BigIntLiteral: + case ts.SyntaxKind.RegularExpressionLiteral: + case ts.SyntaxKind.ThisKeyword: + case ts.SyntaxKind.PlusPlusToken: + case ts.SyntaxKind.MinusMinusToken: + case ts.SyntaxKind.CloseParenToken: + case ts.SyntaxKind.CloseBracketToken: + case ts.SyntaxKind.CloseBraceToken: + case ts.SyntaxKind.TrueKeyword: + case ts.SyntaxKind.FalseKeyword: + return false; + default: + return true; + } +} \ No newline at end of file diff --git a/packages/compiler-cli/ngcc/test/dependencies/esm_dependency_host_spec.ts b/packages/compiler-cli/ngcc/test/dependencies/esm_dependency_host_spec.ts index f20b5036d1..f9e80c2b4d 100644 --- a/packages/compiler-cli/ngcc/test/dependencies/esm_dependency_host_spec.ts +++ b/packages/compiler-cli/ngcc/test/dependencies/esm_dependency_host_spec.ts @@ -57,6 +57,17 @@ runInEachFileSystem(() => { expect(dependencies.has(_('/node_modules/lib-1/sub-1'))).toBe(true); }); + it('should resolve all the external dynamic imports of the source file', () => { + const {dependencies, missing, deepImports} = createDependencyInfo(); + host.collectDependencies( + _('/external/dynamic/index.js'), {dependencies, missing, deepImports}); + expect(dependencies.size).toBe(2); + expect(missing.size).toBe(0); + expect(deepImports.size).toBe(0); + expect(dependencies.has(_('/node_modules/lib-1'))).toBe(true); + expect(dependencies.has(_('/node_modules/lib-1/sub-1'))).toBe(true); + }); + it('should capture missing external imports', () => { const {dependencies, missing, deepImports} = createDependencyInfo(); host.collectDependencies( @@ -184,6 +195,13 @@ runInEachFileSystem(() => { }, {name: _('/external/imports/package.json'), contents: '{"esm2015": "./index.js"}'}, {name: _('/external/imports/index.metadata.json'), contents: 'MOCK METADATA'}, + { + name: _('/external/dynamic/index.js'), + contents: + `async function foo() { await const x = import('lib-1');\n const promise = import('lib-1/sub-1'); }` + }, + {name: _('/external/dynamic/package.json'), contents: '{"esm2015": "./index.js"}'}, + {name: _('/external/dynamic/index.metadata.json'), contents: 'MOCK METADATA'}, { name: _('/external/re-exports/index.js'), contents: `export {X} from 'lib-1';\nexport {\n Y,\n Z\n} from 'lib-1/sub-1';`