fix(compiler): switch to 'referencedFiles' for shim generation (#36211)

Shim generation was built on a lie.

Shims are files added to the program which aren't original files authored by
the user, but files authored effectively by the compiler. These fall into
two categories: files which will be generated (like the .ngfactory shims we
generate for View Engine compatibility) as well as files used internally in
compilation (like the __ng_typecheck__.ts file).

Previously, shim generation was driven by the `rootFiles` passed to the
compiler as input. These are effectively the `files` listed in the
`tsconfig.json`. Each shim generator (e.g. the `FactoryGenerator`) would
examine the `rootFiles` and produce a list of shim file names which it would
be responsible for generating. These names would then be added to the
`rootFiles` when the program was created.

The fatal flaw here is that `rootFiles` does not always account for all of
the files in the program. In fact, it's quite rare that it does. Users don't
typically specify every file directly in `files`. Instead, they rely on
TypeScript, during program creation, starting with a few root files and
transitively discovering all of the files in the program.

This happens, however, during `ts.createProgram`, which is too late to add
new files to the `rootFiles` list.

As a result, shim generation was only including shims for files actually
listed in the `tsconfig.json` file, and not for the transitive set of files
in the user's program as it should.

This commit completely rewrites shim generation to use a different technique
for adding files to the program, inspired by View Engine's shim generator.
In this new technique, as the program is being created and `ts.SourceFile`s
are being requested from the `NgCompilerHost`, shims for those files are
generated and a reference to them is patched onto the original file's
`ts.SourceFile.referencedFiles`. This causes TS to think that the original
file references the shim, and causes the shim to be included in the program.
The original `referencedFiles` array is saved and restored after program
creation, hiding this little hack from the rest of the system.

The new shim generation engine differentiates between two kinds of shims:
top-level shims (such as the flat module entrypoint file and
__ng_typecheck__.ts) and per-file shims such as ngfactory or ngsummary
files. The former are included via `rootFiles` as before, the latter are
included via the `referencedFiles` of their corresponding original files.

As a result of this change, shims are now correctly generated for all files
in the program, not just the ones named in `tsconfig.json`.

A few mitigating factors prevented this bug from being realized until now:

* in g3, `files` does include the transitive closure of files in the program
* in CLI apps, shims are not really used

This change also makes use of a novel technique for associating information
with source files: the use of an `NgExtension` `Symbol` to patch the
information directly onto the AST object. This is used in several
circumstances:

* For shims, metadata about a `ts.SourceFile`'s status as a shim and its
  origins are held in the extension data.
* For original files, the original `referencedFiles` are stashed in the
  extension data for later restoration.

The main benefit of this technique is a lot less bookkeeping around `Map`s
of `ts.SourceFile`s to various kinds of data, which need to be tracked/
invalidated as part of incremental builds.

This technique is based on designs used internally in the TypeScript
compiler and is serving as a prototype of this design in ngtsc. If it works
well, it could have benefits across the rest of the compiler.

PR Close #36211

packages/compiler-cli/src/ngtsc/core/BUILD.bazel

@@ -29,6 +29,7 @@ ts_library(
         "//packages/compiler-cli/src/ngtsc/routing",
         "//packages/compiler-cli/src/ngtsc/scope",
         "//packages/compiler-cli/src/ngtsc/shims",
+        "//packages/compiler-cli/src/ngtsc/shims:api",
         "//packages/compiler-cli/src/ngtsc/switch",
         "//packages/compiler-cli/src/ngtsc/transform",
         "//packages/compiler-cli/src/ngtsc/typecheck",

packages/compiler-cli/src/ngtsc/core/src/compiler.ts

@@ -145,13 +145,10 @@ export class NgCompiler {
     }
     setIncrementalDriver(tsProgram, this.incrementalDriver);
 
-    this.ignoreForDiagnostics = new Set([
+    this.ignoreForDiagnostics =
-      this.typeCheckFile,
+        new Set(tsProgram.getSourceFiles().filter(sf => this.host.isShim(sf)));
-      ...host.factoryFiles.map(fileName => getSourceFileOrError(tsProgram, fileName)),
-      ...host.summaryFiles.map(fileName => getSourceFileOrError(tsProgram, fileName)),
-    ]);
 
-    this.ignoreForEmit = new Set([this.typeCheckFile]);
+    this.ignoreForEmit = this.host.ignoreForEmit;
   }
 
   /**

packages/compiler-cli/src/ngtsc/core/src/host.ts

@@ -11,10 +11,11 @@ import * as ts from 'typescript';
 import {ErrorCode, ngErrorCode} from '../../diagnostics';
 import {findFlatIndexEntryPoint, FlatIndexGenerator} from '../../entry_point';
 import {AbsoluteFsPath, resolve} from '../../file_system';
-import {FactoryGenerator, FactoryTracker, ShimGenerator, SummaryGenerator, TypeCheckShimGenerator} from '../../shims';
+import {FactoryGenerator, FactoryTracker, isShim, ShimAdapter, ShimReferenceTagger, SummaryGenerator} from '../../shims';
-import {typeCheckFilePath} from '../../typecheck';
+import {PerFileShimGenerator, TopLevelShimGenerator} from '../../shims/api';
+import {typeCheckFilePath, TypeCheckShimGenerator} from '../../typecheck';
 import {normalizeSeparators} from '../../util/src/path';
-import {getRootDirs} from '../../util/src/typescript';
+import {getRootDirs, isDtsPath, isNonDeclarationTsPath} from '../../util/src/typescript';
 import {ExtendedTsCompilerHost, NgCompilerOptions, UnifiedModulesHost} from '../api';
 
 // A persistent source of bugs in CompilerHost delegation has been the addition by TS of new,
@@ -96,34 +97,48 @@ export class NgCompilerHost extends DelegatingCompilerHost implements
   readonly inputFiles: ReadonlyArray<string>;
   readonly rootDirs: ReadonlyArray<AbsoluteFsPath>;
   readonly typeCheckFile: AbsoluteFsPath;
-  readonly factoryFiles: AbsoluteFsPath[];
+
-  readonly summaryFiles: AbsoluteFsPath[];
 
   constructor(
       delegate: ExtendedTsCompilerHost, inputFiles: ReadonlyArray<string>,
-      rootDirs: ReadonlyArray<AbsoluteFsPath>, private shims: ShimGenerator[],
+      rootDirs: ReadonlyArray<AbsoluteFsPath>, private shimAdapter: ShimAdapter,
-      entryPoint: AbsoluteFsPath|null, typeCheckFile: AbsoluteFsPath,
+      private shimTagger: ShimReferenceTagger, entryPoint: AbsoluteFsPath|null,
-      factoryFiles: AbsoluteFsPath[], summaryFiles: AbsoluteFsPath[],
+      typeCheckFile: AbsoluteFsPath, factoryTracker: FactoryTracker|null,
-      factoryTracker: FactoryTracker|null, diagnostics: ts.Diagnostic[]) {
+      diagnostics: ts.Diagnostic[]) {
     super(delegate);
 
     this.factoryTracker = factoryTracker;
     this.entryPoint = entryPoint;
     this.typeCheckFile = typeCheckFile;
-    this.factoryFiles = factoryFiles;
-    this.summaryFiles = summaryFiles;
     this.diagnostics = diagnostics;
-    this.inputFiles = inputFiles;
+    this.inputFiles = [...inputFiles, ...shimAdapter.extraInputFiles];
     this.rootDirs = rootDirs;
   }
 
+  /**
+   * Retrieves a set of `ts.SourceFile`s which should not be emitted as JS files.
+   *
+   * Available after this host is used to create a `ts.Program` (which causes all the files in the
+   * program to be enumerated).
+   */
+  get ignoreForEmit(): Set<ts.SourceFile> {
+    return this.shimAdapter.ignoreForEmit;
+  }
+
+  /**
+   * Performs cleanup that needs to happen after a `ts.Program` has been created using this host.
+   */
+  postProgramCreationCleanup(): void {
+    this.shimTagger.finalize();
+  }
+
   /**
    * Create an `NgCompilerHost` from a delegate host, an array of input filenames, and the full set
    * of TypeScript and Angular compiler options.
    */
   static wrap(
-      delegate: ts.CompilerHost, inputFiles: ReadonlyArray<string>,
+      delegate: ts.CompilerHost, inputFiles: ReadonlyArray<string>, options: NgCompilerOptions,
-      options: NgCompilerOptions): NgCompilerHost {
+      oldProgram: ts.Program|null): NgCompilerHost {
     // TODO(alxhub): remove the fallback to allowEmptyCodegenFiles after verifying that the rest of
     // our build tooling is no longer relying on it.
     const allowEmptyCodegenFiles = options.allowEmptyCodegenFiles || false;
@@ -135,54 +150,41 @@ export class NgCompilerHost extends DelegatingCompilerHost implements
         options.generateNgSummaryShims :
         allowEmptyCodegenFiles;
 
-    let rootFiles = [...inputFiles];
-    let normalizedInputFiles = inputFiles.map(n => resolve(n));
 
-    const generators: ShimGenerator[] = [];
+    const topLevelShimGenerators: TopLevelShimGenerator[] = [];
-    let summaryGenerator: SummaryGenerator|null = null;
+    const perFileShimGenerators: PerFileShimGenerator[] = [];
-    let summaryFiles: AbsoluteFsPath[];
 
     if (shouldGenerateSummaryShims) {
       // Summary generation.
-      summaryGenerator = SummaryGenerator.forRootFiles(normalizedInputFiles);
+      perFileShimGenerators.push(new SummaryGenerator());
-      generators.push(summaryGenerator);
-      summaryFiles = summaryGenerator.getSummaryFileNames();
-    } else {
-      summaryFiles = [];
     }
 
     let factoryTracker: FactoryTracker|null = null;
-    let factoryFiles: AbsoluteFsPath[];
     if (shouldGenerateFactoryShims) {
-      // Factory generation.
+      const factoryGenerator = new FactoryGenerator();
-      const factoryGenerator = FactoryGenerator.forRootFiles(normalizedInputFiles);
+      perFileShimGenerators.push(factoryGenerator);
-      const factoryFileMap = factoryGenerator.factoryFileMap;
 
-      factoryFiles = Array.from(factoryFileMap.keys());
+      factoryTracker = factoryGenerator;
-      rootFiles.push(...factoryFiles);
-      generators.push(factoryGenerator);
-
-      factoryTracker = new FactoryTracker(factoryGenerator);
-    } else {
-      factoryFiles = [];
     }
 
-    // Done separately to preserve the order of factory files before summary files in rootFiles.
-    // TODO(alxhub): validate that this is necessary.
-    rootFiles.push(...summaryFiles);
-
-
     const rootDirs = getRootDirs(delegate, options as ts.CompilerOptions);
 
     const typeCheckFile = typeCheckFilePath(rootDirs);
-    generators.push(new TypeCheckShimGenerator(typeCheckFile));
+    topLevelShimGenerators.push(new TypeCheckShimGenerator(typeCheckFile));
-    rootFiles.push(typeCheckFile);
 
     let diagnostics: ts.Diagnostic[] = [];
 
+    const normalizedTsInputFiles: AbsoluteFsPath[] = [];
+    for (const inputFile of inputFiles) {
+      if (!isNonDeclarationTsPath(inputFile)) {
+        continue;
+      }
+      normalizedTsInputFiles.push(resolve(inputFile));
+    }
+
     let entryPoint: AbsoluteFsPath|null = null;
     if (options.flatModuleOutFile != null && options.flatModuleOutFile !== '') {
-      entryPoint = findFlatIndexEntryPoint(normalizedInputFiles);
+      entryPoint = findFlatIndexEntryPoint(normalizedTsInputFiles);
       if (entryPoint === null) {
         // This error message talks specifically about having a single .ts file in "files". However
         // the actual logic is a bit more permissive. If a single file exists, that will be taken,
@@ -206,37 +208,49 @@ export class NgCompilerHost extends DelegatingCompilerHost implements
         const flatModuleOutFile = normalizeSeparators(options.flatModuleOutFile);
         const flatIndexGenerator =
             new FlatIndexGenerator(entryPoint, flatModuleOutFile, flatModuleId);
-        generators.push(flatIndexGenerator);
+        topLevelShimGenerators.push(flatIndexGenerator);
-        rootFiles.push(flatIndexGenerator.flatIndexPath);
       }
     }
 
+    const shimAdapter = new ShimAdapter(
+        delegate, normalizedTsInputFiles, topLevelShimGenerators, perFileShimGenerators,
+        oldProgram);
+    const shimTagger =
+        new ShimReferenceTagger(perFileShimGenerators.map(gen => gen.extensionPrefix));
     return new NgCompilerHost(
-        delegate, rootFiles, rootDirs, generators, entryPoint, typeCheckFile, factoryFiles,
+        delegate, inputFiles, rootDirs, shimAdapter, shimTagger, entryPoint, typeCheckFile,
-        summaryFiles, factoryTracker, diagnostics);
+        factoryTracker, diagnostics);
+  }
+
+  /**
+   * Check whether the given `ts.SourceFile` is a shim file.
+   *
+   * If this returns false, the file is user-provided.
+   */
+  isShim(sf: ts.SourceFile): boolean {
+    return isShim(sf);
   }
 
   getSourceFile(
       fileName: string, languageVersion: ts.ScriptTarget,
       onError?: ((message: string) => void)|undefined,
       shouldCreateNewSourceFile?: boolean|undefined): ts.SourceFile|undefined {
-    for (let i = 0; i < this.shims.length; i++) {
+    // Is this a previously known shim?
-      const generator = this.shims[i];
+    const shimSf = this.shimAdapter.maybeGenerate(resolve(fileName));
-      // TypeScript internal paths are guaranteed to be POSIX-like absolute file paths.
+    if (shimSf !== null) {
-      const absoluteFsPath = resolve(fileName);
+      // Yes, so return it.
-      if (generator.recognize(absoluteFsPath)) {
+      return shimSf;
-        const readFile = (originalFile: string) => {
-          return this.delegate.getSourceFile(
-                     originalFile, languageVersion, onError, shouldCreateNewSourceFile) ||
-              null;
-        };
-
-        return generator.generate(absoluteFsPath, readFile) || undefined;
-      }
     }
 
-    return this.delegate.getSourceFile(
+    // No, so it's a file which might need shims (or a file which doesn't exist).
-        fileName, languageVersion, onError, shouldCreateNewSourceFile);
+    const sf =
+        this.delegate.getSourceFile(fileName, languageVersion, onError, shouldCreateNewSourceFile);
+    if (sf === undefined) {
+      return undefined;
+    }
+
+    this.shimTagger.tag(sf);
+    return sf;
   }
 
   fileExists(fileName: string): boolean {
@@ -245,8 +259,10 @@ export class NgCompilerHost extends DelegatingCompilerHost implements
     //  2) at least one of the shim generators recognizes it
     // Note that we can pass the file name as branded absolute fs path because TypeScript
     // internally only passes POSIX-like paths.
+    //
+    // Also note that the `maybeGenerate` check below checks for both `null` and `undefined`.
     return this.delegate.fileExists(fileName) ||
-        this.shims.some(shim => shim.recognize(resolve(fileName)));
+        this.shimAdapter.maybeGenerate(resolve(fileName)) != null;
   }
 
   get unifiedModulesHost(): UnifiedModulesHost|null {

packages/compiler-cli/src/ngtsc/core/test/compiler_test.ts

@@ -43,7 +43,7 @@ runInEachFileSystem(() => {
         strictTemplates: true,
       };
       const baseHost = new NgtscCompilerHost(getFileSystem(), options);
-      const host = NgCompilerHost.wrap(baseHost, [COMPONENT], options);
+      const host = NgCompilerHost.wrap(baseHost, [COMPONENT], options, /* oldProgram */ null);
       const program = ts.createProgram({host, options, rootNames: host.inputFiles});
       const compiler = new NgCompiler(host, options, program);
 

packages/compiler-cli/src/ngtsc/core/test/host_spec.ts

@@ -0,0 +1,52 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import {absoluteFrom as _, FileSystem, getFileSystem, NgtscCompilerHost} from '../../file_system';
+import {runInEachFileSystem} from '../../file_system/testing';
+import {NgCompilerOptions} from '../api';
+import {NgCompilerHost} from '../src/host';
+
+runInEachFileSystem(() => {
+  let fs!: FileSystem;
+  beforeEach(() => {
+    fs = getFileSystem();
+    fs.ensureDir(_('/'));
+  });
+
+  describe('NgCompilerHost', () => {
+    it('should add factory shims for all root files', () => {
+      // This test verifies that per-file shims are guaranteed to be generated for each file in the
+      // "root" input files, regardless of whether `referencedFiles`-based shimming is enabled. As
+      // it turns out, setting `noResolve` in TypeScript's options disables this kind of shimming,
+      // so `NgCompilerHost` needs to ensure at least the root files still get shims directly.
+
+      // File is named index.ts to trigger flat module entrypoint logic
+      // (which adds a top-level shim).
+      const fileName = _('/index.ts');
+      fs.writeFile(fileName, `export class Test {}`);
+
+      const options: NgCompilerOptions = {
+        // Using noResolve means that TS will not follow `referencedFiles`, which is how shims are
+        // normally included in the program.
+        noResolve: true,
+        rootDir: _('/'),
+
+        // Both a top-level and a per-file shim are enabled.
+        flatModuleOutFile: './entry',
+        generateNgFactoryShims: true,
+      };
+      const baseHost = new NgtscCompilerHost(fs, options);
+      const host = NgCompilerHost.wrap(baseHost, [fileName], options, null);
+
+      // A shim file should be included for the index.ts ngfactory, but not entry.ts since that
+      // itself is a shim.
+      expect(host.inputFiles).toContain(_('/index.ngfactory.ts'));
+      expect(host.inputFiles).not.toContain(_('/entry.ngfactory.ts'));
+    });
+  });
+});

packages/compiler-cli/src/ngtsc/entry_point/src/generator.ts

@@ -11,11 +11,12 @@
 import * as ts from 'typescript';
 
 import {AbsoluteFsPath, dirname, join} from '../../file_system';
-import {ShimGenerator} from '../../shims';
+import {TopLevelShimGenerator} from '../../shims';
 import {relativePathBetween} from '../../util/src/path';
 
-export class FlatIndexGenerator implements ShimGenerator {
+export class FlatIndexGenerator implements TopLevelShimGenerator {
   readonly flatIndexPath: string;
+  readonly shouldEmit = true;
 
   constructor(
       readonly entryPoint: AbsoluteFsPath, relativeFlatIndexPath: string,
@@ -24,11 +25,7 @@ export class FlatIndexGenerator implements ShimGenerator {
         join(dirname(entryPoint), relativeFlatIndexPath).replace(/\.js$/, '') + '.ts';
   }
 
-  recognize(fileName: string): boolean {
+  makeTopLevelShim(): ts.SourceFile {
-    return fileName === this.flatIndexPath;
-  }
-
-  generate(): ts.SourceFile {
     const relativeEntryPoint = relativePathBetween(this.flatIndexPath, this.entryPoint);
     const contents = `/**
  * Generated bundle index. Do not edit.

packages/compiler-cli/src/ngtsc/program.ts

@@ -66,12 +66,14 @@ export class NgtscProgram implements api.Program {
     }
     this.closureCompilerEnabled = !!options.annotateForClosureCompiler;
 
-    this.host = NgCompilerHost.wrap(delegateHost, rootNames, options);
-
     const reuseProgram = oldProgram && oldProgram.reuseTsProgram;
+    this.host = NgCompilerHost.wrap(delegateHost, rootNames, options, reuseProgram ?? null);
+
     this.tsProgram = ts.createProgram(this.host.inputFiles, options, this.host, reuseProgram);
     this.reuseTsProgram = this.tsProgram;
 
+    this.host.postProgramCreationCleanup();
+
     // Create the NgCompiler which will drive the rest of the compilation.
     this.compiler =
         new NgCompiler(this.host, options, this.tsProgram, reuseProgram, this.perfRecorder);

packages/compiler-cli/src/ngtsc/shims/BUILD.bazel

@@ -2,12 +2,22 @@ load("//tools:defaults.bzl", "ts_library")
 
 package(default_visibility = ["//visibility:public"])
 
+ts_library(
+    name = "api",
+    srcs = ["api.ts"],
+    deps = [
+        "//packages/compiler-cli/src/ngtsc/file_system",
+        "@npm//typescript",
+    ],
+)
+
 ts_library(
     name = "shims",
     srcs = ["index.ts"] + glob([
         "src/**/*.ts",
     ]),
     deps = [
+        ":api",
         "//packages/compiler",
         "//packages/compiler-cli/src/ngtsc/file_system",
         "//packages/compiler-cli/src/ngtsc/imports",

packages/compiler-cli/src/ngtsc/shims/README.md

@@ -1,11 +1,84 @@
-Deals with the creation of generated factory files.
+# Shims
+
+The shims package deals with the specification and generation of "shim files". These are files which are not part of the user's original program, but are added by the compiler by user request or in support of certain features. For example, users can request that the compiler produce `.ngfactory` files alongside user files to support migration from View Engine (which used `.ngfactory` files) to Ivy which does not.
+
+## API
+
+Shim generation is exposed through two interfaces: `TopLevelShimGenerator` and `PerFileShimGenerator`. Each implementation of one of these interfaces produces one or more shims of a particular type.
+
+A top-level shim is a shim which is a "singleton" with respect to the program - it's one file that's generated and added in addition to all the user files.
+
+A per-file shim is a shim generated from the contents of a particular file (like how `.ngfactory` shims are generated for each user input file, if requested).
+
+Shims from either kind of generator can be emittable, in which case their `ts.SourceFile`s will be transpiled to JS and emitted alongside the user's code, or non-emittable, which means the user is unlikely to be aware of their existence.
+
+This API is used both by the shim generators in this package as well as for other types of shims generated by other compiler subsystems.
+
+## Implementation
+
+The shim package exposes two specific pieces of functionality related to the integration of shims into the creation of a `ts.Program`:
+
+* A `ShimReferenceTagger` which "tags" `ts.SourceFile`s prior to program creation, and creates links from each original file to all of the per-file shims which need to be created for those file.
+* A `ShimAdapter` which is used by an implementation of `ts.CompilerHost` to include shims in any program created via the host.
+
+### `ShimAdapter`
+
+The shim adapter is responsible for recognizing when a path being loaded corresponds to a shim, and producing a `ts.SourceFile` for the shim if so.
+
+Recognizing a shim filename involves two steps. First, the path itself must match a pattern for a particular `PerFileShimGenerator`'s shims (for example, NgFactory shims end in `.ngfactory.ts`). From this filename, the "source" filename can be inferred (actually several source filenames, since the source file might be `.ts` or `.tsx`). Even if a path matches the pattern, it's only a valid shim if the source file actually exists.
+
+Once a filename has been recognized, the `ShimAdapter` caches the generated shim source file and can quickly produce it on request.
+
+#### Shim loading in practice
+
+As TS starts from the root files and walks imports and references, it discovers new files which are part of the program. It will discover shim files in two different ways:
+
+* As references on their source files (those added by `ShimReferenceTagger`).
+* As imports written by users.
+
+This means that it's not guaranteed for a source file to be loaded before its shim.
+
+### `ShimReferenceTagger`
+
+During program creation, TypeScript enumerates the `.ts` files on disk (the original files) and includes them into the program. However, each original file may have many associated shim files, which are not referenced and do not exist on disk, but still need to be included as well.
+
+The mechanism used to do this is "reference tagging", which is performed by the `ShimReferenceTagger`.
+
+`ts.SourceFile`s have a `referencedFiles` property, which contains paths extracted from any `/// <reference>` comments within the file. If a `ts.SourceFile` with references is included in a program, so are its referenced files.
+
+This mechanism is (ab)used by the `ShimReferenceTagger` to create references from each original file to its shims, causing them to be loaded as well.
+
+Once the program has been created, the `referencedFiles` properties can be restored to their original values via the `cleanup()` operation. This is necessary as `ts.SourceFile`s may live on in various caches for much longer than the duration of a single compilation.
+
+### Expando
+
+The shim system needs to keep track of various pieces of metadata for `ts.SourceFile`s:
+
+* Whether or not they're shims, and if so which generator created them.
+* If the file is not a shim, then the original `referenceFiles` for that file (so it can be restored later).
+
+Instead of `Map`s keyed with `ts.SourceFile`s which could lead to memory leaks, this information is instead patched directly onto the `ts.SourceFile` instances using an expando symbol property `NgExtension`.
+
+
+## Usage
+
+### Factory shim generation
 
 Generated factory files create a catch-22 in ngtsc. Their contents depends on static analysis of the current program, yet they're also importable from the current program. This importability gives rise to the requirement that the contents of the generated file must be known before program creation, so that imports of it are valid. However, until the program is created, the analysis to determine the contents of the generated file cannot take place.
 
 ngc used to get away with this because the analysis phase did not depend on program creation but on the metadata collection / global analysis process.
 
-ngtsc is forced to take a different approach. A lightweight analysis pipeline which does not rely on the ts.TypeChecker (and thus can run before the program is created) is used to estimate the contents of a generated file, in a way that allows the program to be created. A transformer then operates on this estimated file during emit and replaces the estimated contents with accurate information.
+ngtsc is forced to take a different approach. A lightweight analysis pipeline which does not rely on the `ts.TypeChecker` (and thus can run before the program is created) is used to estimate the contents of a generated file, in a way that allows the program to be created. A transformer then operates on this estimated file during emit and replaces the estimated contents with accurate information.
 
 It is important that this estimate be an overestimate, as type-checking will always be run against the estimated file, and must succeed in every case where it would have succeeded with accurate info.
 
-This directory contains the utilities for generating, updating, and incorporating these factory files into a ts.Program.
+### Summary shim generation
+
+Summary shim generation is simpler than factory generation, and can be generated from a `ts.SourceFile` without needing to be cleaned up later.
+
+### Other uses of shims
+
+A few other systems in the compiler make use of shim generation as well.
+
+* `entry_point` generates a flat module index (in the way View Engine used to) using a shim.
+* `typecheck` includes template type-checking code in the program using a shim generator.

packages/compiler-cli/src/ngtsc/shims/api.ts

@@ -0,0 +1,51 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+import * as ts from 'typescript';
+import {AbsoluteFsPath} from '../file_system';
+
+/**
+ * Generates a single shim file for the entire program.
+ */
+export interface TopLevelShimGenerator {
+  /**
+   * Whether this shim should be emitted during TypeScript emit.
+   */
+  readonly shouldEmit: boolean;
+
+  /**
+   * Create a `ts.SourceFile` representing the shim, with the correct filename.
+   */
+  makeTopLevelShim(): ts.SourceFile;
+}
+
+/**
+ * Generates a shim file for each original `ts.SourceFile` in the user's program, with a file
+ * extension prefix.
+ */
+export interface PerFileShimGenerator {
+  /**
+   * The extension prefix which will be used for the shim.
+   *
+   * Knowing this allows the `ts.CompilerHost` implementation which is consuming this shim generator
+   * to predict the shim filename, which is useful when a previous `ts.Program` already includes a
+   * generated version of the shim.
+   */
+  readonly extensionPrefix: string;
+
+  /**
+   * Whether shims produced by this generator should be emitted during TypeScript emit.
+   */
+  readonly shouldEmit: boolean;
+
+  /**
+   * Generate the shim for a given original `ts.SourceFile`, with the given filename.
+   */
+  generateShimForFile(
+      sf: ts.SourceFile, genFilePath: AbsoluteFsPath,
+      priorShimSf: ts.SourceFile|null): ts.SourceFile;
+}

packages/compiler-cli/src/ngtsc/shims/index.ts

@@ -8,8 +8,9 @@
 
 /// <reference types="node" />
 
-export {ShimGenerator} from './src/api';
+export {PerFileShimGenerator, TopLevelShimGenerator} from './api';
-export {FactoryGenerator, FactoryInfo, generatedFactoryTransform} from './src/factory_generator';
+export {ShimAdapter} from './src/adapter';
-export {FactoryTracker} from './src/factory_tracker';
+export {isShim} from './src/expando';
+export {FactoryGenerator, FactoryInfo, FactoryTracker, generatedFactoryTransform} from './src/factory_generator';
+export {ShimReferenceTagger} from './src/reference_tagger';
 export {SummaryGenerator} from './src/summary_generator';
-export {TypeCheckShimGenerator} from './src/typecheck_shim';

packages/compiler-cli/src/ngtsc/shims/src/adapter.ts

@@ -0,0 +1,229 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import * as ts from 'typescript';
+
+import {absoluteFrom, absoluteFromSourceFile, AbsoluteFsPath} from '../../file_system';
+import {isDtsPath} from '../../util/src/typescript';
+import {PerFileShimGenerator, TopLevelShimGenerator} from '../api';
+
+import {isFileShimSourceFile, isShim, NgExtension, sfExtensionData} from './expando';
+import {makeShimFileName} from './util';
+
+interface ShimGeneratorData {
+  generator: PerFileShimGenerator;
+  test: RegExp;
+  suffix: string;
+}
+
+/**
+ * Generates and tracks shim files for each original `ts.SourceFile`.
+ *
+ * The `ShimAdapter` provides an API that's designed to be used by a `ts.CompilerHost`
+ * implementation and allows it to include synthetic "shim" files in the program that's being
+ * created. It works for both freshly created programs as well as with reuse of an older program
+ * (which already may contain shim files and thus have a different creation flow).
+ */
+export class ShimAdapter {
+  /**
+   * A map of shim file names to the `ts.SourceFile` generated for those shims.
+   */
+  private shims = new Map<AbsoluteFsPath, ts.SourceFile>();
+
+  /**
+   * A map of shim file names to existing shims which were part of a previous iteration of this
+   * program.
+   *
+   * Not all of these shims will be inherited into this program.
+   */
+  private priorShims = new Map<AbsoluteFsPath, ts.SourceFile>();
+
+  /**
+   * File names which are already known to not be shims.
+   *
+   * This allows for short-circuit returns without the expense of running regular expressions
+   * against the filename repeatedly.
+   */
+  private notShims = new Set<AbsoluteFsPath>();
+
+  /**
+   * The shim generators supported by this adapter as well as extra precalculated data facilitating
+   * their use.
+   */
+  private generators: ShimGeneratorData[] = [];
+
+  /**
+   * A `Set` of shim `ts.SourceFile`s which should not be emitted.
+   */
+  readonly ignoreForEmit = new Set<ts.SourceFile>();
+
+  /**
+   * A list of extra filenames which should be considered inputs to program creation.
+   *
+   * This includes any top-level shims generated for the program, as well as per-file shim names for
+   * those files which are included in the root files of the program.
+   */
+  readonly extraInputFiles: ReadonlyArray<AbsoluteFsPath>;
+
+  constructor(
+      private delegate: Pick<ts.CompilerHost, 'getSourceFile'|'fileExists'>,
+      tsRootFiles: AbsoluteFsPath[], topLevelGenerators: TopLevelShimGenerator[],
+      perFileGenerators: PerFileShimGenerator[], oldProgram: ts.Program|null) {
+    // Initialize `this.generators` with a regex that matches each generator's paths.
+    for (const gen of perFileGenerators) {
+      // This regex matches paths for shims from this generator. The first (and only) capture group
+      // extracts the filename prefix, which can be used to find the original file that was used to
+      // generate this shim.
+      const pattern = `^(.*)\\.${gen.extensionPrefix}\\.ts$`;
+      const regexp = new RegExp(pattern, 'i');
+      this.generators.push({
+        generator: gen,
+        test: regexp,
+        suffix: `.${gen.extensionPrefix}.ts`,
+      });
+    }
+    // Process top-level generators and pre-generate their shims. Accumulate the list of filenames
+    // as extra input files.
+    const extraInputFiles: AbsoluteFsPath[] = [];
+
+    for (const gen of topLevelGenerators) {
+      const sf = gen.makeTopLevelShim();
+      sfExtensionData(sf).isTopLevelShim = true;
+
+      if (!gen.shouldEmit) {
+        this.ignoreForEmit.add(sf);
+      }
+
+      const fileName = absoluteFromSourceFile(sf);
+      this.shims.set(fileName, sf);
+      extraInputFiles.push(fileName);
+    }
+
+    // Add to that list the per-file shims associated with each root file. This is needed because
+    // reference tagging alone may not work in TS compilations that have `noResolve` set. Such
+    // compilations rely on the list of input files completely describing the program.
+    for (const rootFile of tsRootFiles) {
+      for (const gen of this.generators) {
+        extraInputFiles.push(makeShimFileName(rootFile, gen.suffix));
+      }
+    }
+
+    this.extraInputFiles = extraInputFiles;
+
+    // If an old program is present, extract all per-file shims into a map, which will be used to
+    // generate new versions of those shims.
+    if (oldProgram !== null) {
+      for (const oldSf of oldProgram.getSourceFiles()) {
+        if (oldSf.isDeclarationFile || !isFileShimSourceFile(oldSf)) {
+          continue;
+        }
+
+        this.priorShims.set(absoluteFromSourceFile(oldSf), oldSf);
+      }
+    }
+  }
+
+  /**
+   * Produce a shim `ts.SourceFile` if `fileName` refers to a shim file which should exist in the
+   * program.
+   *
+   * If `fileName` does not refer to a potential shim file, `null` is returned. If a corresponding
+   * base file could not be determined, `undefined` is returned instead.
+   */
+  maybeGenerate(fileName: AbsoluteFsPath): ts.SourceFile|null|undefined {
+    // Fast path: either this filename has been proven not to be a shim before, or it is a known
+    // shim and no generation is required.
+    if (this.notShims.has(fileName)) {
+      return null;
+    } else if (this.shims.has(fileName)) {
+      return this.shims.get(fileName)!;
+    }
+
+    // .d.ts files can't be shims.
+    if (isDtsPath(fileName)) {
+      this.notShims.add(fileName);
+      return null;
+    }
+
+    // This is the first time seeing this path. Try to match it against a shim generator.
+    for (const record of this.generators) {
+      const match = record.test.exec(fileName);
+      if (match === null) {
+        continue;
+      }
+
+      // The path matched. Extract the filename prefix without the extension.
+      const prefix = match[1];
+      // This _might_ be a shim, if an underlying base file exists. The base file might be .ts or
+      // .tsx.
+      let baseFileName = absoluteFrom(prefix + '.ts');
+      if (!this.delegate.fileExists(baseFileName)) {
+        // No .ts file by that name - try .tsx.
+        baseFileName = absoluteFrom(prefix + '.tsx');
+        if (!this.delegate.fileExists(baseFileName)) {
+          // This isn't a shim after all since there is no original file which would have triggered
+          // its generation, even though the path is right. There are a few reasons why this could
+          // occur:
+          //
+          // * when resolving an import to an .ngfactory.d.ts file, the module resolution algorithm
+          //   will first look for an .ngfactory.ts file in its place, which will be requested here.
+          // * when the user writes a bad import.
+          // * when a file is present in one compilation and removed in the next incremental step.
+          //
+          // Note that this does not add the filename to `notShims`, so this path is not cached.
+          // That's okay as these cases above are edge cases and do not occur regularly in normal
+          // operations.
+          return undefined;
+        }
+      }
+
+      // Retrieve the original file for which the shim will be generated.
+      const inputFile = this.delegate.getSourceFile(baseFileName, ts.ScriptTarget.Latest);
+      if (inputFile === undefined || isShim(inputFile)) {
+        // Something strange happened here. This case is also not cached in `notShims`, but this
+        // path is not expected to occur in reality so this shouldn't be a problem.
+        return null;
+      }
+
+      // Actually generate and cache the shim.
+      return this.generateSpecific(fileName, record.generator, inputFile);
+    }
+
+    // No generator matched.
+    this.notShims.add(fileName);
+    return null;
+  }
+
+  private generateSpecific(
+      fileName: AbsoluteFsPath, generator: PerFileShimGenerator,
+      inputFile: ts.SourceFile): ts.SourceFile {
+    let priorShimSf: ts.SourceFile|null = null;
+    if (this.priorShims.has(fileName)) {
+      // In the previous program a shim with this name already existed. It's passed to the shim
+      // generator which may reuse it instead of generating a fresh shim.
+
+      priorShimSf = this.priorShims.get(fileName)!;
+      this.priorShims.delete(fileName);
+    }
+
+    const shimSf = generator.generateShimForFile(inputFile, fileName, priorShimSf);
+
+    // Mark the new generated source file as a shim that originated from this generator.
+    sfExtensionData(shimSf).fileShim = {
+      extension: generator.extensionPrefix,
+      generatedFrom: absoluteFromSourceFile(inputFile),
+    };
+
+    if (!generator.shouldEmit) {
+      this.ignoreForEmit.add(shimSf);
+    }
+
+    this.shims.set(fileName, shimSf);
+    return shimSf;
+  }
+}

packages/compiler-cli/src/ngtsc/shims/src/api.ts

@@ -1,27 +0,0 @@
-/**
- * @license
- * Copyright Google Inc. All Rights Reserved.
- *
- * Use of this source code is governed by an MIT-style license that can be
- * found in the LICENSE file at https://angular.io/license
- */
-import * as ts from 'typescript';
-import {AbsoluteFsPath} from '../../file_system';
-
-export interface ShimGenerator {
-  /**
-   * Returns `true` if this generator is intended to handle the given file.
-   */
-  recognize(fileName: AbsoluteFsPath): boolean;
-
-  /**
-   * Generate a shim's `ts.SourceFile` for the given original file.
-   *
-   * `readFile` is a function which allows the generator to look up the contents of existing source
-   * files. It returns null if the requested file doesn't exist.
-   *
-   * If `generate` returns null, then the shim generator declines to generate the file after all.
-   */
-  generate(genFileName: AbsoluteFsPath, readFile: (fileName: string) => ts.SourceFile | null):
-      ts.SourceFile|null;
-}

packages/compiler-cli/src/ngtsc/shims/src/expando.ts

@@ -0,0 +1,102 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import * as ts from 'typescript';
+
+import {AbsoluteFsPath} from '../../file_system';
+
+/**
+ * A `Symbol` which is used to patch extension data onto `ts.SourceFile`s.
+ */
+export const NgExtension = Symbol('NgExtension');
+
+/**
+ * Contents of the `NgExtension` property of a `ts.SourceFile`.
+ */
+export interface NgExtensionData {
+  isTopLevelShim: boolean;
+  fileShim: NgFileShimData|null;
+  originalReferencedFiles: ReadonlyArray<ts.FileReference>|null;
+}
+
+/**
+ * A `ts.SourceFile` which may or may not have `NgExtension` data.
+ */
+interface MaybeNgExtendedSourceFile extends ts.SourceFile {
+  [NgExtension]?: NgExtensionData;
+}
+
+/**
+ * A `ts.SourceFile` which has `NgExtension` data.
+ */
+export interface NgExtendedSourceFile extends ts.SourceFile {
+  /**
+   * Overrides the type of `referencedFiles` to be writeable.
+   */
+  referencedFiles: ts.FileReference[];
+
+  [NgExtension]: NgExtensionData;
+}
+
+/**
+ * Narrows a `ts.SourceFile` if it has an `NgExtension` property.
+ */
+export function isExtended(sf: ts.SourceFile): sf is NgExtendedSourceFile {
+  return (sf as MaybeNgExtendedSourceFile)[NgExtension] !== undefined;
+}
+
+/**
+ * Returns the `NgExtensionData` for a given `ts.SourceFile`, adding it if none exists.
+ */
+export function sfExtensionData(sf: ts.SourceFile): NgExtensionData {
+  const extSf = sf as MaybeNgExtendedSourceFile;
+  if (extSf[NgExtension] !== undefined) {
+    // The file already has extension data, so return it directly.
+    return extSf[NgExtension]!;
+  }
+
+  // The file has no existing extension data, so add it and return it.
+  const extension: NgExtensionData = {
+    isTopLevelShim: false,
+    fileShim: null,
+    originalReferencedFiles: null,
+  };
+  extSf[NgExtension] = extension;
+  return extension;
+}
+
+/**
+ * Data associated with a per-shim instance `ts.SourceFile`.
+ */
+export interface NgFileShimData {
+  generatedFrom: AbsoluteFsPath;
+  extension: string;
+}
+
+/**
+ * An `NgExtendedSourceFile` that is a per-file shim and has `NgFileShimData`.
+ */
+export interface NgFileShimSourceFile extends NgExtendedSourceFile {
+  [NgExtension]: NgExtensionData&{
+    fileShim: NgFileShimData,
+  };
+}
+
+/**
+ * Check whether `sf` is a per-file shim `ts.SourceFile`.
+ */
+export function isFileShimSourceFile(sf: ts.SourceFile): sf is NgFileShimSourceFile {
+  return isExtended(sf) && sf[NgExtension].fileShim !== null;
+}
+
+/**
+ * Check whether `sf` is a shim `ts.SourceFile` (either a per-file shim or a top-level shim).
+ */
+export function isShim(sf: ts.SourceFile): boolean {
+  return isExtended(sf) && (sf[NgExtension].fileShim !== null || sf[NgExtension].isTopLevelShim);
+}

packages/compiler-cli/src/ngtsc/shims/src/factory_generator.ts

@@ -7,51 +7,50 @@
  */
 import * as ts from 'typescript';
 
-import {absoluteFrom, AbsoluteFsPath, basename} from '../../file_system';
+import {absoluteFromSourceFile, AbsoluteFsPath, basename} from '../../file_system';
 import {ImportRewriter} from '../../imports';
-import {isNonDeclarationTsPath} from '../../util/src/typescript';
+import {PerFileShimGenerator} from '../api';
 
-import {ShimGenerator} from './api';
 import {generatedModuleName} from './util';
 
 const TS_DTS_SUFFIX = /(\.d)?\.ts$/;
 const STRIP_NG_FACTORY = /(.*)NgFactory$/;
 
+/**
+ * Maintains a mapping of which symbols in a .ngfactory file have been used.
+ *
+ * .ngfactory files are generated with one symbol per defined class in the source file, regardless
+ * of whether the classes in the source files are NgModules (because that isn't known at the time
+ * the factory files are generated). A `FactoryTracker` supports removing factory symbols which
+ * didn't end up being NgModules, by tracking the ones which are.
+ */
+export interface FactoryTracker {
+  readonly sourceInfo: Map<string, FactoryInfo>;
+
+  track(sf: ts.SourceFile, factorySymbolName: string): void;
+}
+
 /**
  * Generates ts.SourceFiles which contain variable declarations for NgFactories for every exported
  * class of an input ts.SourceFile.
  */
-export class FactoryGenerator implements ShimGenerator {
+export class FactoryGenerator implements PerFileShimGenerator, FactoryTracker {
-  private constructor(private map: Map<AbsoluteFsPath, AbsoluteFsPath>) {}
+  readonly sourceInfo = new Map<string, FactoryInfo>();
+  private sourceToFactorySymbols = new Map<string, Set<string>>();
 
-  get factoryFileMap(): Map<AbsoluteFsPath, AbsoluteFsPath> {
+  readonly shouldEmit = true;
-    return this.map;
+  readonly extensionPrefix = 'ngfactory';
-  }
 
-  get factoryFileNames(): AbsoluteFsPath[] {
+  generateShimForFile(sf: ts.SourceFile, genFilePath: AbsoluteFsPath): ts.SourceFile {
-    return Array.from(this.map.keys());
+    const absoluteSfPath = absoluteFromSourceFile(sf);
-  }
 
-  recognize(fileName: AbsoluteFsPath): boolean {
+    const relativePathToSource = './' + basename(sf.fileName).replace(TS_DTS_SUFFIX, '');
-    return this.map.has(fileName);
-  }
-
-  generate(genFilePath: AbsoluteFsPath, readFile: (fileName: string) => ts.SourceFile | null):
-      ts.SourceFile|null {
-    const originalPath = this.map.get(genFilePath)!;
-    const original = readFile(originalPath);
-    if (original === null) {
-      return null;
-    }
-
-    const relativePathToSource = './' + basename(original.fileName).replace(TS_DTS_SUFFIX, '');
     // Collect a list of classes that need to have factory types emitted for them. This list is
     // overly broad as at this point the ts.TypeChecker hasn't been created, and can't be used to
     // semantically understand which decorated types are actually decorated with Angular decorators.
     //
     // The exports generated here are pruned in the factory transform during emit.
-    const symbolNames = original
+    const symbolNames = sf.statements
-                            .statements
                             // Pick out top level class declarations...
                             .filter(ts.isClassDeclaration)
                             // which are named, exported, and have decorators.
@@ -66,7 +65,7 @@ export class FactoryGenerator implements ShimGenerator {
 
     // If there is a top-level comment in the original file, copy it over at the top of the
     // generated factory file. This is important for preserving any load-bearing jsdoc comments.
-    const leadingComment = getFileoverviewComment(original);
+    const leadingComment = getFileoverviewComment(sf);
     if (leadingComment !== null) {
       // Leading comments must be separated from the rest of the contents by a blank line.
       sourceText = leadingComment + '\n\n';
@@ -94,22 +93,23 @@ export class FactoryGenerator implements ShimGenerator {
     // factory transformer if it ends up not being needed.
     sourceText += '\nexport const ɵNonEmptyModule = true;';
 
-    const genFile = ts.createSourceFile(
+    const genFile =
-        genFilePath, sourceText, original.languageVersion, true, ts.ScriptKind.TS);
+        ts.createSourceFile(genFilePath, sourceText, sf.languageVersion, true, ts.ScriptKind.TS);
-    if (original.moduleName !== undefined) {
+    if (sf.moduleName !== undefined) {
-      genFile.moduleName =
+      genFile.moduleName = generatedModuleName(sf.moduleName, sf.fileName, '.ngfactory');
-          generatedModuleName(original.moduleName, original.fileName, '.ngfactory');
     }
+
+    const moduleSymbolNames = new Set<string>();
+    this.sourceToFactorySymbols.set(absoluteSfPath, moduleSymbolNames);
+    this.sourceInfo.set(genFilePath, {sourceFilePath: absoluteSfPath, moduleSymbolNames});
+
     return genFile;
   }
 
-  static forRootFiles(files: ReadonlyArray<AbsoluteFsPath>): FactoryGenerator {
+  track(sf: ts.SourceFile, factorySymbolName: string): void {
-    const map = new Map<AbsoluteFsPath, AbsoluteFsPath>();
+    if (this.sourceToFactorySymbols.has(sf.fileName)) {
-    files.filter(sourceFile => isNonDeclarationTsPath(sourceFile))
+      this.sourceToFactorySymbols.get(sf.fileName)!.add(factorySymbolName);
-        .forEach(
+    }
-            sourceFile =>
-                map.set(absoluteFrom(sourceFile.replace(/\.ts$/, '.ngfactory.ts')), sourceFile));
-    return new FactoryGenerator(map);
   }
 }
 

packages/compiler-cli/src/ngtsc/shims/src/factory_tracker.ts

@@ -1,38 +0,0 @@
-/**
- * @license
- * Copyright Google Inc. All Rights Reserved.
- *
- * Use of this source code is governed by an MIT-style license that can be
- * found in the LICENSE file at https://angular.io/license
- */
-
-import * as ts from 'typescript';
-
-import {FactoryGenerator, FactoryInfo} from './factory_generator';
-
-/**
- * Maintains a mapping of which symbols in a .ngfactory file have been used.
- *
- * .ngfactory files are generated with one symbol per defined class in the source file, regardless
- * of whether the classes in the source files are NgModules (because that isn't known at the time
- * the factory files are generated). The `FactoryTracker` exists to support removing factory symbols
- * which didn't end up being NgModules, by tracking the ones which are.
- */
-export class FactoryTracker {
-  readonly sourceInfo = new Map<string, FactoryInfo>();
-  private sourceToFactorySymbols = new Map<string, Set<string>>();
-
-  constructor(generator: FactoryGenerator) {
-    generator.factoryFileMap.forEach((sourceFilePath, factoryPath) => {
-      const moduleSymbolNames = new Set<string>();
-      this.sourceToFactorySymbols.set(sourceFilePath, moduleSymbolNames);
-      this.sourceInfo.set(factoryPath, {sourceFilePath, moduleSymbolNames});
-    });
-  }
-
-  track(sf: ts.SourceFile, factorySymbolName: string): void {
-    if (this.sourceToFactorySymbols.has(sf.fileName)) {
-      this.sourceToFactorySymbols.get(sf.fileName)!.add(factorySymbolName);
-    }
-  }
-}

packages/compiler-cli/src/ngtsc/shims/src/reference_tagger.ts

@@ -0,0 +1,83 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import * as ts from 'typescript';
+
+import {absoluteFrom, absoluteFromSourceFile} from '../../file_system';
+
+import {isExtended as isExtendedSf, isShim, NgExtension, sfExtensionData} from './expando';
+import {makeShimFileName} from './util';
+
+/**
+ * Manipulates the `referencedFiles` property of `ts.SourceFile`s to add references to shim files
+ * for each original source file, causing the shims to be loaded into the program as well.
+ *
+ * `ShimReferenceTagger`s are intended to operate during program creation only.
+ */
+export class ShimReferenceTagger {
+  private suffixes: string[];
+
+  /**
+   * Tracks which original files have been processed and had shims generated if necessary.
+   *
+   * This is used to avoid generating shims twice for the same file.
+   */
+  private tagged = new Set<ts.SourceFile>();
+
+  /**
+   * Whether shim tagging is currently being performed.
+   */
+  private enabled: boolean = true;
+
+  constructor(shimExtensions: string[]) {
+    this.suffixes = shimExtensions.map(extension => `.${extension}.ts`);
+  }
+
+  /**
+   * Tag `sf` with any needed references if it's not a shim itself.
+   */
+  tag(sf: ts.SourceFile): void {
+    if (!this.enabled || sf.isDeclarationFile || isShim(sf) || this.tagged.has(sf)) {
+      return;
+    }
+
+    sfExtensionData(sf).originalReferencedFiles = sf.referencedFiles;
+    const referencedFiles = [...sf.referencedFiles];
+
+    const sfPath = absoluteFromSourceFile(sf);
+    for (const suffix of this.suffixes) {
+      referencedFiles.push({
+        fileName: makeShimFileName(sfPath, suffix),
+        pos: 0,
+        end: 0,
+      });
+    }
+
+    sf.referencedFiles = referencedFiles;
+    this.tagged.add(sf);
+  }
+
+  /**
+   * Restore the original `referencedFiles` values of all tagged `ts.SourceFile`s and disable the
+   * `ShimReferenceTagger`.
+   */
+  finalize(): void {
+    this.enabled = false;
+    for (const sf of this.tagged) {
+      if (!isExtendedSf(sf)) {
+        continue;
+      }
+
+      const extensionData = sfExtensionData(sf);
+      if (extensionData.originalReferencedFiles !== null) {
+        sf.referencedFiles = extensionData.originalReferencedFiles! as ts.FileReference[];
+      }
+    }
+    this.tagged.clear();
+  }
+}

packages/compiler-cli/src/ngtsc/shims/src/summary_generator.ts

@@ -8,38 +8,23 @@
 
 import * as ts from 'typescript';
 
-import {absoluteFrom, AbsoluteFsPath} from '../../file_system';
+import {AbsoluteFsPath} from '../../file_system';
-import {isNonDeclarationTsPath} from '../../util/src/typescript';
+import {PerFileShimGenerator} from '../api';
 
-import {ShimGenerator} from './api';
 import {generatedModuleName} from './util';
 
-export class SummaryGenerator implements ShimGenerator {
+export class SummaryGenerator implements PerFileShimGenerator {
-  private constructor(private map: Map<AbsoluteFsPath, AbsoluteFsPath>) {}
+  readonly shouldEmit = true;
-
+  readonly extensionPrefix = 'ngsummary';
-  getSummaryFileNames(): AbsoluteFsPath[] {
-    return Array.from(this.map.keys());
-  }
-
-  recognize(fileName: AbsoluteFsPath): boolean {
-    return this.map.has(fileName);
-  }
-
-  generate(genFilePath: AbsoluteFsPath, readFile: (fileName: string) => ts.SourceFile | null):
-      ts.SourceFile|null {
-    const originalPath = this.map.get(genFilePath)!;
-    const original = readFile(originalPath);
-    if (original === null) {
-      return null;
-    }
 
+  generateShimForFile(sf: ts.SourceFile, genFilePath: AbsoluteFsPath): ts.SourceFile {
     // Collect a list of classes that need to have factory types emitted for them. This list is
     // overly broad as at this point the ts.TypeChecker has not been created and so it can't be used
     // to semantically understand which decorators are Angular decorators. It's okay to output an
     // overly broad set of summary exports as the exports are no-ops anyway, and summaries are a
     // compatibility layer which will be removed after Ivy is enabled.
     const symbolNames: string[] = [];
-    for (const stmt of original.statements) {
+    for (const stmt of sf.statements) {
       if (ts.isClassDeclaration(stmt)) {
         // If the class isn't exported, or if it's not decorated, then skip it.
         if (!isExported(stmt) || stmt.decorators === undefined || stmt.name === undefined) {
@@ -73,23 +58,13 @@ export class SummaryGenerator implements ShimGenerator {
       varLines.push(`export const ɵempty = null;`);
     }
     const sourceText = varLines.join('\n');
-    const genFile = ts.createSourceFile(
+    const genFile =
-        genFilePath, sourceText, original.languageVersion, true, ts.ScriptKind.TS);
+        ts.createSourceFile(genFilePath, sourceText, sf.languageVersion, true, ts.ScriptKind.TS);
-    if (original.moduleName !== undefined) {
+    if (sf.moduleName !== undefined) {
-      genFile.moduleName =
+      genFile.moduleName = generatedModuleName(sf.moduleName, sf.fileName, '.ngsummary');
-          generatedModuleName(original.moduleName, original.fileName, '.ngsummary');
     }
     return genFile;
   }
-
-  static forRootFiles(files: ReadonlyArray<AbsoluteFsPath>): SummaryGenerator {
-    const map = new Map<AbsoluteFsPath, AbsoluteFsPath>();
-    files.filter(sourceFile => isNonDeclarationTsPath(sourceFile))
-        .forEach(
-            sourceFile =>
-                map.set(absoluteFrom(sourceFile.replace(/\.ts$/, '.ngsummary.ts')), sourceFile));
-    return new SummaryGenerator(map);
-  }
 }
 
 function isExported(decl: ts.Declaration): boolean {

packages/compiler-cli/src/ngtsc/shims/src/util.ts

@@ -6,6 +6,17 @@
  * found in the LICENSE file at https://angular.io/license
  */
 
+import {absoluteFrom, AbsoluteFsPath} from '../../file_system';
+
+const TS_EXTENSIONS = /\.tsx?$/i;
+
+/**
+ * Replace the .ts or .tsx extension of a file with the shim filename suffix.
+ */
+export function makeShimFileName(fileName: AbsoluteFsPath, suffix: string): AbsoluteFsPath {
+  return absoluteFrom(fileName.replace(TS_EXTENSIONS, suffix));
+}
+
 export function generatedModuleName(
     originalModuleName: string, originalFileName: string, genSuffix: string): string {
   let moduleName: string;

packages/compiler-cli/src/ngtsc/shims/test/BUILD.bazel

@@ -10,7 +10,11 @@ ts_library(
     ]),
     deps = [
         "//packages:types",
+        "//packages/compiler-cli/src/ngtsc/file_system",
+        "//packages/compiler-cli/src/ngtsc/file_system/testing",
         "//packages/compiler-cli/src/ngtsc/shims",
+        "//packages/compiler-cli/src/ngtsc/shims:api",
+        "//packages/compiler-cli/src/ngtsc/testing",
         "@npm//typescript",
     ],
 )

packages/compiler-cli/src/ngtsc/shims/test/adapter_spec.ts

@@ -0,0 +1,83 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import * as ts from 'typescript';
+
+import {absoluteFrom as _} from '../../file_system';
+import {runInEachFileSystem} from '../../file_system/testing';
+import {makeProgram} from '../../testing';
+import {ShimAdapter} from '../src/adapter';
+import {TestShimGenerator} from './util';
+
+runInEachFileSystem(() => {
+  describe('ShimAdapter', () => {
+    it('should recognize a basic shim name', () => {
+      const {host} = makeProgram([{
+        name: _('/test.ts'),
+        contents: `export class A {}`,
+      }]);
+
+      const adapter =
+          new ShimAdapter(host, [], [], [new TestShimGenerator()], /* oldProgram */ null);
+      const shimSf = adapter.maybeGenerate(_('/test.testshim.ts'));
+      expect(shimSf).not.toBeNull();
+      expect(shimSf!.fileName).toBe(_('/test.testshim.ts'));
+      expect(shimSf!.text).toContain('SHIM_FOR_FILE');
+    });
+
+    it('should not recognize a normal file in the program', () => {
+      const {host} = makeProgram([{
+        name: _('/test.ts'),
+        contents: `export class A {}`,
+      }]);
+
+      const adapter =
+          new ShimAdapter(host, [], [], [new TestShimGenerator()], /* oldProgram */ null);
+      const shimSf = adapter.maybeGenerate(_('/test.ts'));
+      expect(shimSf).toBeNull();
+    });
+
+    it('should not recognize a shim-named file without a source file', () => {
+      const {host} = makeProgram([{
+        name: _('/test.ts'),
+        contents: `export class A {}`,
+      }]);
+
+      const adapter =
+          new ShimAdapter(host, [], [], [new TestShimGenerator()], /* oldProgram */ null);
+      const shimSf = adapter.maybeGenerate(_('/other.testshim.ts'));
+
+      // Expect undefined, not null, since that indicates a valid shim path but an invalid source
+      // file.
+      expect(shimSf).toBeUndefined();
+    });
+
+    it('should detect a prior shim if one is available', () => {
+      // Create a shim via the ShimAdapter, then create a second ShimAdapter simulating an
+      // incremental compilation, with a stub passed for the oldProgram that includes the original
+      // shim file. Verify that the new ShimAdapter incorporates the original shim in generation of
+      // the new one.
+      const {host, program} = makeProgram([
+        {
+          name: _('/test.ts'),
+          contents: `export class A {}`,
+        },
+      ]);
+      const adapter =
+          new ShimAdapter(host, [], [], [new TestShimGenerator()], /* oldProgram */ null);
+      const originalShim = adapter.maybeGenerate(_('/test.testshim.ts'))!;
+      const oldProgramStub = {
+        getSourceFiles: () => [...program.getSourceFiles(), originalShim],
+      } as unknown as ts.Program;
+
+      const adapter2 = new ShimAdapter(host, [], [], [new TestShimGenerator()], oldProgramStub);
+      const newShim = adapter.maybeGenerate(_('/test.testshim.ts'));
+      expect(newShim).toBe(originalShim);
+    });
+  });
+});

packages/compiler-cli/src/ngtsc/shims/test/reference_tagger_spec.ts

@@ -0,0 +1,119 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import * as ts from 'typescript';
+
+import {absoluteFrom as _, AbsoluteFsPath, getSourceFileOrError} from '../../file_system';
+import {runInEachFileSystem} from '../../file_system/testing';
+import {makeProgram} from '../../testing';
+import {ShimAdapter} from '../src/adapter';
+import {ShimReferenceTagger} from '../src/reference_tagger';
+
+import {TestShimGenerator} from './util';
+
+runInEachFileSystem(() => {
+  describe('ShimReferenceTagger', () => {
+    it('should tag a source file with its appropriate shims', () => {
+      const tagger = new ShimReferenceTagger(['test1', 'test2']);
+
+      const fileName = _('/file.ts');
+      const sf = makeArbitrarySf(fileName);
+      expect(sf.referencedFiles).toEqual([]);
+
+      tagger.tag(sf);
+      expectReferencedFiles(sf, ['/file.test1.ts', '/file.test2.ts']);
+    });
+
+    it('should not tag .d.ts files', () => {
+      const tagger = new ShimReferenceTagger(['test1', 'test2']);
+
+      const fileName = _('/file.d.ts');
+      const sf = makeArbitrarySf(fileName);
+
+      expectReferencedFiles(sf, []);
+      tagger.tag(sf);
+      expectReferencedFiles(sf, []);
+    });
+
+    it('should not tag shim files', () => {
+      const tagger = new ShimReferenceTagger(['test1', 'test2']);
+      const fileName = _('/file.ts');
+      const {host} = makeProgram([
+        {name: fileName, contents: 'export declare const UNIMPORTANT = true;'},
+      ]);
+      const shimAdapter =
+          new ShimAdapter(host, [], [], [new TestShimGenerator()], /* oldProgram */ null);
+
+      const shimSf = shimAdapter.maybeGenerate(_('/file.testshim.ts'))!;
+      expect(shimSf.referencedFiles).toEqual([]);
+
+      tagger.tag(shimSf);
+      expect(shimSf.referencedFiles).toEqual([]);
+    });
+
+    it('should remove tags during finalization', () => {
+      const tagger = new ShimReferenceTagger(['test1', 'test2']);
+
+      const fileName = _('/file.ts');
+      const sf = makeArbitrarySf(fileName);
+
+      expectReferencedFiles(sf, []);
+
+      tagger.tag(sf);
+      expectReferencedFiles(sf, ['/file.test1.ts', '/file.test2.ts']);
+
+      tagger.finalize();
+      expectReferencedFiles(sf, []);
+    });
+
+    it('should not remove references it did not add during finalization', () => {
+      const tagger = new ShimReferenceTagger(['test1', 'test2']);
+      const fileName = _('/file.ts');
+      const libFileName = _('/lib.d.ts');
+
+      const sf = makeSf(fileName, `
+        /// <reference path="/lib.d.ts" />
+        export const UNIMPORTANT = true;
+      `);
+
+      expectReferencedFiles(sf, [libFileName]);
+
+      tagger.tag(sf);
+      expectReferencedFiles(sf, ['/file.test1.ts', '/file.test2.ts', libFileName]);
+
+      tagger.finalize();
+      expectReferencedFiles(sf, [libFileName]);
+    });
+
+    it('should not tag shims after finalization', () => {
+      const tagger = new ShimReferenceTagger(['test1', 'test2']);
+      tagger.finalize();
+
+      const fileName = _('/file.ts');
+      const sf = makeArbitrarySf(fileName);
+
+      tagger.tag(sf);
+      expectReferencedFiles(sf, []);
+    });
+  });
+});
+
+function makeSf(fileName: AbsoluteFsPath, contents: string): ts.SourceFile {
+  return ts.createSourceFile(fileName, contents, ts.ScriptTarget.Latest, true, ts.ScriptKind.TS);
+}
+
+function makeArbitrarySf(fileName: AbsoluteFsPath): ts.SourceFile {
+  const declare = fileName.endsWith('.d.ts') ? 'declare ' : '';
+  return makeSf(fileName, `export ${declare}const UNIMPORTANT = true;`);
+}
+
+function expectReferencedFiles(sf: ts.SourceFile, files: string[]): void {
+  const actual = sf.referencedFiles.map(f => _(f.fileName)).sort();
+  const expected = files.map(fileName => _(fileName)).sort();
+  expect(actual).toEqual(expected);
+}

packages/compiler-cli/src/ngtsc/shims/test/util.ts

@@ -0,0 +1,27 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import * as ts from 'typescript';
+
+import {absoluteFromSourceFile, AbsoluteFsPath} from '../../file_system';
+import {PerFileShimGenerator} from '../api';
+
+export class TestShimGenerator implements PerFileShimGenerator {
+  readonly shouldEmit = false;
+  readonly extensionPrefix = 'testshim';
+
+  generateShimForFile(sf: ts.SourceFile, genFilePath: AbsoluteFsPath, priorSf: ts.SourceFile|null):
+      ts.SourceFile {
+    if (priorSf !== null) {
+      return priorSf;
+    }
+    const path = absoluteFromSourceFile(sf);
+    return ts.createSourceFile(
+        genFilePath, `export const SHIM_FOR_FILE = '${path}';\n`, ts.ScriptTarget.Latest, true);
+  }
+}

packages/compiler-cli/src/ngtsc/tsc_plugin.ts

@@ -79,7 +79,7 @@ export class NgTscPlugin implements TscPlugin {
       host: ts.CompilerHost&UnifiedModulesHost, inputFiles: readonly string[],
       options: ts.CompilerOptions): PluginCompilerHost {
     this.options = {...this.ngOptions, ...options} as NgCompilerOptions;
-    this.host = NgCompilerHost.wrap(host, inputFiles, this.options);
+    this.host = NgCompilerHost.wrap(host, inputFiles, this.options, /* oldProgram */ null);
     return this.host;
   }
 

packages/compiler-cli/src/ngtsc/typecheck/BUILD.bazel

@@ -13,6 +13,7 @@ ts_library(
         "//packages/compiler-cli/src/ngtsc/imports",
         "//packages/compiler-cli/src/ngtsc/metadata",
         "//packages/compiler-cli/src/ngtsc/reflection",
+        "//packages/compiler-cli/src/ngtsc/shims:api",
         "//packages/compiler-cli/src/ngtsc/translator",
         "//packages/compiler-cli/src/ngtsc/util",
         "@npm//@types/node",

packages/compiler-cli/src/ngtsc/typecheck/index.ts

@@ -9,5 +9,6 @@
 export * from './src/api';
 export {TypeCheckContext} from './src/context';
 export {TemplateDiagnostic, isTemplateDiagnostic} from './src/diagnostics';
+export {TypeCheckShimGenerator} from './src/shim';
 export {TypeCheckProgramHost} from './src/host';
 export {typeCheckFilePath} from './src/type_check_file';

packages/compiler-cli/src/ngtsc/typecheck/src/shim.ts

@@ -9,8 +9,7 @@
 import * as ts from 'typescript';
 
 import {AbsoluteFsPath} from '../../file_system';
-
+import {TopLevelShimGenerator} from '../../shims/api';
-import {ShimGenerator} from './api';
 
 /**
  * A `ShimGenerator` which adds a type-checking file to the `ts.Program`.
@@ -19,17 +18,14 @@ import {ShimGenerator} from './api';
  * information in the main program when creating the type-checking program if the set of files in
  * each are exactly the same. Thus, the main program also needs the synthetic type-checking file.
  */
-export class TypeCheckShimGenerator implements ShimGenerator {
+export class TypeCheckShimGenerator implements TopLevelShimGenerator {
   constructor(private typeCheckFile: AbsoluteFsPath) {}
 
-  recognize(fileName: AbsoluteFsPath): boolean {
+  readonly shouldEmit = false;
-    return fileName === this.typeCheckFile;
-  }
 
-  generate(genFileName: AbsoluteFsPath, readFile: (fileName: string) => ts.SourceFile | null):
+  makeTopLevelShim(): ts.SourceFile {
-      ts.SourceFile|null {
     return ts.createSourceFile(
-        genFileName, 'export const USED_FOR_NG_TYPE_CHECKING = true;', ts.ScriptTarget.Latest, true,
+        this.typeCheckFile, 'export const USED_FOR_NG_TYPE_CHECKING = true;',
-        ts.ScriptKind.TS);
+        ts.ScriptTarget.Latest, true, ts.ScriptKind.TS);
   }
 }

packages/compiler-cli/test/ngtsc/env.ts

@@ -158,11 +158,16 @@ export class NgtscTestEnvironment {
     this.multiCompileHostExt.invalidate(absFilePath);
   }
 
-  tsconfig(extraOpts: {[key: string]: string|boolean|null} = {}, extraRootDirs?: string[]): void {
+  tsconfig(
+      extraOpts: {[key: string]: string|boolean|null} = {}, extraRootDirs?: string[],
+      files?: string[]): void {
     const tsconfig: {[key: string]: any} = {
       extends: './tsconfig-base.json',
       angularCompilerOptions: {...extraOpts, enableIvy: true},
     };
+    if (files !== undefined) {
+      tsconfig['files'] = files;
+    }
     if (extraRootDirs !== undefined) {
       tsconfig.compilerOptions = {
         rootDirs: ['.', ...extraRootDirs],

packages/compiler-cli/test/ngtsc/ngtsc_spec.ts

@@ -3468,6 +3468,56 @@ runInEachFileSystem(os => {
         env.tsconfig({'generateNgFactoryShims': true});
       });
 
+      it('should be able to depend on an existing factory shim', () => {
+        // This test verifies that ngfactory files from the compilations of dependencies are
+        // available to import in a fresh compilation. It is derived from a bug observed in g3 where
+        // the shim system accidentally caused TypeScript to think that *.ngfactory.ts files always
+        // exist.
+        env.write('other.ngfactory.d.ts', `
+          export class OtherNgFactory {}
+        `);
+        env.write('test.ts', `
+          import {OtherNgFactory} from './other.ngfactory';
+
+          class DoSomethingWith extends OtherNgFactory {}
+        `);
+        expect(env.driveDiagnostics()).toEqual([]);
+      });
+
+      it('should generate factory shims for files not listed in root files', () => {
+        // This test verifies that shims are generated for all files in the user's program, even if
+        // only a subset of those files are listed in the tsconfig as root files.
+
+        env.tsconfig({'generateNgFactoryShims': true}, /* extraRootDirs */ undefined, [
+          absoluteFrom('/test.ts'),
+        ]);
+        env.write('test.ts', `
+          import {Component} from '@angular/core';
+
+          import {OtherCmp} from './other';
+
+          @Component({
+            selector: 'test',
+            template: '...',
+          })
+          export class TestCmp {
+            constructor(other: OtherCmp) {}
+          }
+        `);
+        env.write('other.ts', `
+          import {Component} from '@angular/core';
+
+          @Component({
+            selector: 'other',
+            template: '...',
+          })
+          export class OtherCmp {}
+        `);
+        env.driveMain();
+
+        expect(env.getContents('other.ngfactory.js')).toContain('OtherCmp');
+      });
+
       it('should generate correct type annotation for NgModuleFactory calls in ngfactories', () => {
         env.write('test.ts', `
         import {Component} from '@angular/core';

packages/compiler-cli/test/ngtsc/template_typecheck_spec.ts

@@ -1838,6 +1838,29 @@ export declare class AnimationEvent {
            expect(diags.length).toBe(0);
          });
     });
+
+    describe('stability', () => {
+      // This section tests various scenarios which have more complex ts.Program setups and thus
+      // exercise edge cases of the template type-checker.
+      it('should accept a program with a flat index', () => {
+        // This test asserts that flat indices don't have any negative interactions with the
+        // generation of template type-checking code in the program.
+        env.tsconfig({fullTemplateTypeCheck: true, flatModuleOutFile: 'flat.js'});
+        env.write('test.ts', `
+          import {Component} from '@angular/core';
+
+          @Component({
+            selector: 'test-cmp',
+            template: '{{expr}}'
+          })
+          export class TestCmp {
+            expr = 'string';
+          }
+        `);
+
+        expect(env.driveDiagnostics()).toEqual([]);
+      });
+    });
   });
 });