perf(ivy): reuse prior analysis work during incremental builds (#34288)

Previously, the compiler performed an incremental build by analyzing and
resolving all classes in the program (even unchanged ones) and then using
the dependency graph information to determine which .js files were stale and
needed to be re-emitted. This algorithm produced "correct" rebuilds, but the
cost of re-analyzing the entire program turned out to be higher than
anticipated, especially for component-heavy compilations.

To achieve performant rebuilds, it is necessary to reuse previous analysis
results if possible. Doing this safely requires knowing when prior work is
viable and when it is stale and needs to be re-done.

The new algorithm implemented by this commit is such:

1) Each incremental build starts with knowledge of the last known good
   dependency graph and analysis results from the last successful build,
   plus of course information about the set of files changed.

2) The previous dependency graph's information is used to determine the
   set of source files which have "logically" changed. A source file is
   considered logically changed if it or any of its dependencies have
   physically changed (on disk) since the last successful compilation. Any
   logically unchanged dependencies have their dependency information copied
   over to the new dependency graph.

3) During the `TraitCompiler`'s loop to consider all source files in the
   program, if a source file is logically unchanged then its previous
   analyses are "adopted" (and their 'register' steps are run). If the file
   is logically changed, then it is re-analyzed as usual.

4) Then, incremental build proceeds as before, with the new dependency graph
   being used to determine the set of files which require re-emitting.

This analysis reuse avoids template parsing operations in many circumstances
and significantly reduces the time it takes ngtsc to rebuild a large
application.

Future work will increase performance even more, by tackling a variety of
other opportunities to reuse or avoid work.

PR Close #34288

packages/compiler-cli/ngcc/BUILD.bazel

@@ -17,6 +17,7 @@ ts_library(
         "//packages/compiler-cli/src/ngtsc/diagnostics",
         "//packages/compiler-cli/src/ngtsc/file_system",
         "//packages/compiler-cli/src/ngtsc/imports",
+        "//packages/compiler-cli/src/ngtsc/incremental:api",
         "//packages/compiler-cli/src/ngtsc/metadata",
         "//packages/compiler-cli/src/ngtsc/partial_evaluator",
         "//packages/compiler-cli/src/ngtsc/perf",

packages/compiler-cli/ngcc/src/analysis/decoration_analyzer.ts

@@ -27,7 +27,7 @@ import {isDefined} from '../utils';
 
 import {DefaultMigrationHost} from './migration_host';
 import {AnalyzedClass, AnalyzedFile, CompiledClass, CompiledFile, DecorationAnalyses} from './types';
-import {analyzeDecorators, isWithinPackage} from './util';
+import {NOOP_DEPENDENCY_TRACKER, analyzeDecorators, isWithinPackage} from './util';
 
 
 /**
@@ -79,7 +79,8 @@ export class DecorationAnalyzer {
   scopeRegistry = new LocalModuleScopeRegistry(
       this.metaRegistry, this.dtsModuleScopeResolver, this.refEmitter, this.aliasingHost);
   fullRegistry = new CompoundMetadataRegistry([this.metaRegistry, this.scopeRegistry]);
-  evaluator = new PartialEvaluator(this.reflectionHost, this.typeChecker);
+  evaluator =
+      new PartialEvaluator(this.reflectionHost, this.typeChecker, /* dependencyTracker */ null);
   moduleResolver = new ModuleResolver(this.program, this.options, this.host);
   importGraph = new ImportGraph(this.moduleResolver);
   cycleAnalyzer = new CycleAnalyzer(this.importGraph);
@@ -90,9 +91,9 @@ export class DecorationAnalyzer {
         /* defaultPreserveWhitespaces */ false,
         /* i18nUseExternalIds */ true, this.bundle.enableI18nLegacyMessageIdFormat,
         this.moduleResolver, this.cycleAnalyzer, this.refEmitter, NOOP_DEFAULT_IMPORT_RECORDER,
-        /* annotateForClosureCompiler */ false),
+        NOOP_DEPENDENCY_TRACKER, /* annotateForClosureCompiler */ false),
     // clang-format off
-    // See the note in ngtsc about why this cast is needed.
+        // See the note in ngtsc about why this cast is needed.
     new DirectiveDecoratorHandler(
         this.reflectionHost, this.evaluator, this.fullRegistry, NOOP_DEFAULT_IMPORT_RECORDER,
         this.isCore, /* annotateForClosureCompiler */ false) as DecoratorHandler<unknown, unknown, unknown>,

packages/compiler-cli/ngcc/src/analysis/util.ts

@@ -9,8 +9,9 @@ import * as ts from 'typescript';
 
 import {isFatalDiagnosticError} from '../../../src/ngtsc/diagnostics';
 import {AbsoluteFsPath, absoluteFromSourceFile, relative} from '../../../src/ngtsc/file_system';
+import {DependencyTracker} from '../../../src/ngtsc/incremental/api';
 import {Decorator} from '../../../src/ngtsc/reflection';
-import {DecoratorHandler, DetectResult, HandlerFlags, HandlerPrecedence} from '../../../src/ngtsc/transform';
+import {DecoratorHandler, HandlerFlags, HandlerPrecedence} from '../../../src/ngtsc/transform';
 import {NgccClassSymbol} from '../host/ngcc_host';
 
 import {AnalyzedClass, MatchingHandler} from './types';
@@ -103,3 +104,12 @@ export function analyzeDecorators(
     diagnostics: allDiagnostics.length > 0 ? allDiagnostics : undefined,
   };
 }
+
+class NoopDependencyTracker implements DependencyTracker {
+  addDependency(): void {}
+  addResourceDependency(): void {}
+  addTransitiveDependency(): void {}
+  addTransitiveResources(): void {}
+}
+
+export const NOOP_DEPENDENCY_TRACKER: DependencyTracker = new NoopDependencyTracker();

packages/compiler-cli/ngcc/test/analysis/migration_host_spec.ts

@@ -298,7 +298,7 @@ runInEachFileSystem(() => {
 });
 
 class TestHandler implements DecoratorHandler<unknown, unknown, unknown> {
-  constructor(protected name: string, protected log: string[]) {}
+  constructor(readonly name: string, protected log: string[]) {}
 
   precedence = HandlerPrecedence.PRIMARY;
   detect(node: ClassDeclaration, decorators: Decorator[]|null): DetectResult<unknown>|undefined {

packages/compiler-cli/ngcc/test/analysis/references_registry_spec.ts

@@ -47,7 +47,7 @@ runInEachFileSystem(() => {
       const testArrayExpression = testArrayDeclaration.initializer !;
 
       const reflectionHost = new TypeScriptReflectionHost(checker);
-      const evaluator = new PartialEvaluator(reflectionHost, checker);
+      const evaluator = new PartialEvaluator(reflectionHost, checker, /* dependencyTracker */ null);
       const registry = new NgccReferencesRegistry(reflectionHost);
 
       const references = (evaluator.evaluate(testArrayExpression) as any[]).filter(isReference);

packages/compiler-cli/src/ngtsc/annotations/BUILD.bazel

@@ -13,6 +13,7 @@ ts_library(
         "//packages/compiler-cli/src/ngtsc/diagnostics",
         "//packages/compiler-cli/src/ngtsc/file_system",
         "//packages/compiler-cli/src/ngtsc/imports",
+        "//packages/compiler-cli/src/ngtsc/incremental:api",
         "//packages/compiler-cli/src/ngtsc/indexer",
         "//packages/compiler-cli/src/ngtsc/metadata",
         "//packages/compiler-cli/src/ngtsc/partial_evaluator",

packages/compiler-cli/src/ngtsc/annotations/src/component.ts

@@ -13,6 +13,7 @@ import {CycleAnalyzer} from '../../cycles';
 import {ErrorCode, FatalDiagnosticError} from '../../diagnostics';
 import {absoluteFrom, relative} from '../../file_system';
 import {DefaultImportRecorder, ModuleResolver, Reference, ReferenceEmitter} from '../../imports';
+import {DependencyTracker} from '../../incremental/api';
 import {IndexingContext} from '../../indexer';
 import {DirectiveMeta, MetadataReader, MetadataRegistry, extractDirectiveGuards} from '../../metadata';
 import {flattenInheritedDirectiveMetadata} from '../../metadata/src/inheritance';
@@ -21,7 +22,6 @@ import {ClassDeclaration, Decorator, ReflectionHost, reflectObjectLiteral} from
 import {ComponentScopeReader, LocalModuleScopeRegistry} from '../../scope';
 import {AnalysisOutput, CompileResult, DecoratorHandler, DetectResult, HandlerFlags, HandlerPrecedence, ResolveResult} from '../../transform';
 import {TemplateSourceMapping, TypeCheckContext} from '../../typecheck';
-import {NoopResourceDependencyRecorder, ResourceDependencyRecorder} from '../../util/src/resource_recorder';
 import {tsSourceMapBug29300Fixed} from '../../util/src/ts_source_map_bug_29300';
 import {SubsetOfKeys} from '../../util/src/typescript';
 
@@ -73,9 +73,7 @@ export class ComponentDecoratorHandler implements
       private enableI18nLegacyMessageIdFormat: boolean, private moduleResolver: ModuleResolver,
       private cycleAnalyzer: CycleAnalyzer, private refEmitter: ReferenceEmitter,
       private defaultImportRecorder: DefaultImportRecorder,
-      private annotateForClosureCompiler: boolean,
+      private depTracker: DependencyTracker|null, private annotateForClosureCompiler: boolean) {}
-      private resourceDependencies:
-          ResourceDependencyRecorder = new NoopResourceDependencyRecorder()) {}
 
   private literalCache = new Map<Decorator, ts.ObjectLiteralExpression>();
   private elementSchemaRegistry = new DomElementSchemaRegistry();
@@ -88,6 +86,7 @@ export class ComponentDecoratorHandler implements
   private preanalyzeTemplateCache = new Map<ts.Declaration, PreanalyzedTemplate>();
 
   readonly precedence = HandlerPrecedence.PRIMARY;
+  readonly name = ComponentDecoratorHandler.name;
 
   detect(node: ClassDeclaration, decorators: Decorator[]|null): DetectResult<Decorator>|undefined {
     if (!decorators) {
@@ -255,7 +254,9 @@ export class ComponentDecoratorHandler implements
         const resourceUrl = this.resourceLoader.resolve(styleUrl, containingFile);
         const resourceStr = this.resourceLoader.load(resourceUrl);
         styles.push(resourceStr);
-        this.resourceDependencies.recordResourceDependency(node.getSourceFile(), resourceUrl);
+        if (this.depTracker !== null) {
+          this.depTracker.addResourceDependency(node.getSourceFile(), resourceUrl);
+        }
       }
     }
     if (component.has('styles')) {
@@ -638,7 +639,9 @@ export class ComponentDecoratorHandler implements
     templateSourceMapping: TemplateSourceMapping
   } {
     const templateStr = this.resourceLoader.load(resourceUrl);
-    this.resourceDependencies.recordResourceDependency(node.getSourceFile(), resourceUrl);
+    if (this.depTracker !== null) {
+      this.depTracker.addResourceDependency(node.getSourceFile(), resourceUrl);
+    }
     const parseTemplate = (options?: ParseTemplateOptions) => this._parseTemplate(
         component, templateStr, sourceMapUrl(resourceUrl),
         /* templateRange */ undefined,

packages/compiler-cli/src/ngtsc/annotations/src/directive.ts

@@ -45,6 +45,7 @@ export class DirectiveDecoratorHandler implements
       private isCore: boolean, private annotateForClosureCompiler: boolean) {}
 
   readonly precedence = HandlerPrecedence.PRIMARY;
+  readonly name = DirectiveDecoratorHandler.name;
 
   detect(node: ClassDeclaration, decorators: Decorator[]|null):
       DetectResult<Decorator|null>|undefined {

packages/compiler-cli/src/ngtsc/annotations/src/injectable.ts

@@ -42,6 +42,7 @@ export class InjectableDecoratorHandler implements
       private errorOnDuplicateProv = true) {}
 
   readonly precedence = HandlerPrecedence.SHARED;
+  readonly name = InjectableDecoratorHandler.name;
 
   detect(node: ClassDeclaration, decorators: Decorator[]|null): DetectResult<Decorator>|undefined {
     if (!decorators) {

packages/compiler-cli/src/ngtsc/annotations/src/ng_module.ts

@@ -36,13 +36,15 @@ export interface NgModuleAnalysis {
   factorySymbolName: string;
 }
 
+export interface NgModuleResolution { injectorImports: Expression[]; }
+
 /**
  * Compiles @NgModule annotations to ngModuleDef fields.
  *
  * TODO(alxhub): handle injector side of things as well.
  */
 export class NgModuleDecoratorHandler implements
-    DecoratorHandler<Decorator, NgModuleAnalysis, unknown> {
+    DecoratorHandler<Decorator, NgModuleAnalysis, NgModuleResolution> {
   constructor(
       private reflector: ReflectionHost, private evaluator: PartialEvaluator,
       private metaReader: MetadataReader, private metaRegistry: MetadataRegistry,
@@ -54,6 +56,7 @@ export class NgModuleDecoratorHandler implements
       private annotateForClosureCompiler: boolean, private localeId?: string) {}
 
   readonly precedence = HandlerPrecedence.PRIMARY;
+  readonly name = NgModuleDecoratorHandler.name;
 
   detect(node: ClassDeclaration, decorators: Decorator[]|null): DetectResult<Decorator>|undefined {
     if (!decorators) {
@@ -270,9 +273,13 @@ export class NgModuleDecoratorHandler implements
     }
   }
 
-  resolve(node: ClassDeclaration, analysis: Readonly<NgModuleAnalysis>): ResolveResult<unknown> {
+  resolve(node: ClassDeclaration, analysis: Readonly<NgModuleAnalysis>):
+      ResolveResult<NgModuleResolution> {
     const scope = this.scopeRegistry.getScopeOfModule(node);
     const diagnostics = this.scopeRegistry.getDiagnosticsOfModule(node) || undefined;
+    const data: NgModuleResolution = {
+      injectorImports: [],
+    };
 
     if (scope !== null) {
       // Using the scope information, extend the injector's imports using the modules that are
@@ -280,7 +287,7 @@ export class NgModuleDecoratorHandler implements
       const context = getSourceFile(node);
       for (const exportRef of analysis.exports) {
         if (isNgModule(exportRef.node, scope.compilation)) {
-          analysis.inj.imports.push(this.refEmitter.emit(exportRef, context));
+          data.injectorImports.push(this.refEmitter.emit(exportRef, context));
         }
       }
 
@@ -296,17 +303,25 @@ export class NgModuleDecoratorHandler implements
     }
 
     if (scope === null || scope.reexports === null) {
-      return {diagnostics};
+      return {data, diagnostics};
     } else {
       return {
+        data,
         diagnostics,
         reexports: scope.reexports,
       };
     }
   }
 
-  compile(node: ClassDeclaration, analysis: Readonly<NgModuleAnalysis>): CompileResult[] {
+  compile(
-    const ngInjectorDef = compileInjector(analysis.inj);
+      node: ClassDeclaration, analysis: Readonly<NgModuleAnalysis>,
+      resolution: Readonly<NgModuleResolution>): CompileResult[] {
+    //  Merge the injector imports (which are 'exports' that were later found to be NgModules)
+    //  computed during resolution with the ones from analysis.
+    const ngInjectorDef = compileInjector({
+      ...analysis.inj,
+      imports: [...analysis.inj.imports, ...resolution.injectorImports],
+    });
     const ngModuleDef = compileNgModule(analysis.mod);
     const ngModuleStatements = ngModuleDef.additionalStatements;
     if (analysis.metadataStmt !== null) {

packages/compiler-cli/src/ngtsc/annotations/src/pipe.ts

@@ -32,6 +32,7 @@ export class PipeDecoratorHandler implements DecoratorHandler<Decorator, PipeHan
       private isCore: boolean) {}
 
   readonly precedence = HandlerPrecedence.PRIMARY;
+  readonly name = PipeDecoratorHandler.name;
 
   detect(node: ClassDeclaration, decorators: Decorator[]|null): DetectResult<Decorator>|undefined {
     if (!decorators) {

packages/compiler-cli/src/ngtsc/annotations/test/component_spec.ts

@@ -47,7 +47,7 @@ runInEachFileSystem(() => {
       ]);
       const checker = program.getTypeChecker();
       const reflectionHost = new TypeScriptReflectionHost(checker);
-      const evaluator = new PartialEvaluator(reflectionHost, checker);
+      const evaluator = new PartialEvaluator(reflectionHost, checker, /* dependencyTracker */ null);
       const moduleResolver = new ModuleResolver(program, options, host);
       const importGraph = new ImportGraph(moduleResolver);
       const cycleAnalyzer = new CycleAnalyzer(importGraph);
@@ -64,7 +64,8 @@ runInEachFileSystem(() => {
           /* isCore */ false, new NoopResourceLoader(), /* rootDirs */[''],
           /* defaultPreserveWhitespaces */ false, /* i18nUseExternalIds */ true,
           /* enableI18nLegacyMessageIdFormat */ false, moduleResolver, cycleAnalyzer, refEmitter,
-          NOOP_DEFAULT_IMPORT_RECORDER, /* annotateForClosureCompiler */ false);
+          NOOP_DEFAULT_IMPORT_RECORDER, /* depTracker */ null,
+          /* annotateForClosureCompiler */ false);
       const TestCmp = getDeclaration(program, _('/entry.ts'), 'TestCmp', isNamedClassDeclaration);
       const detected = handler.detect(TestCmp, reflectionHost.getDecoratorsOfDeclaration(TestCmp));
       if (detected === undefined) {

packages/compiler-cli/src/ngtsc/annotations/test/directive_spec.ts

@@ -42,7 +42,7 @@ runInEachFileSystem(() => {
 
       const checker = program.getTypeChecker();
       const reflectionHost = new TestReflectionHost(checker);
-      const evaluator = new PartialEvaluator(reflectionHost, checker);
+      const evaluator = new PartialEvaluator(reflectionHost, checker, /* dependencyTracker */ null);
       const metaReader = new LocalMetadataRegistry();
       const dtsReader = new DtsMetadataReader(checker, reflectionHost);
       const scopeRegistry = new LocalModuleScopeRegistry(

packages/compiler-cli/src/ngtsc/annotations/test/ng_module_spec.ts

@@ -57,7 +57,7 @@ runInEachFileSystem(() => {
       ]);
       const checker = program.getTypeChecker();
       const reflectionHost = new TypeScriptReflectionHost(checker);
-      const evaluator = new PartialEvaluator(reflectionHost, checker);
+      const evaluator = new PartialEvaluator(reflectionHost, checker, /* dependencyTracker */ null);
       const referencesRegistry = new NoopReferencesRegistry();
       const metaRegistry = new LocalMetadataRegistry();
       const metaReader = new CompoundMetadataReader([metaRegistry]);

packages/compiler-cli/src/ngtsc/incremental/BUILD.bazel

@@ -8,12 +8,22 @@ ts_library(
         "src/**/*.ts",
     ]),
     deps = [
+        ":api",
         "//packages/compiler-cli/src/ngtsc/imports",
         "//packages/compiler-cli/src/ngtsc/metadata",
         "//packages/compiler-cli/src/ngtsc/partial_evaluator",
         "//packages/compiler-cli/src/ngtsc/reflection",
         "//packages/compiler-cli/src/ngtsc/scope",
+        "//packages/compiler-cli/src/ngtsc/transform",
         "//packages/compiler-cli/src/ngtsc/util",
         "@npm//typescript",
     ],
 )
+
+ts_library(
+    name = "api",
+    srcs = ["api.ts"],
+    deps = [
+        "@npm//typescript",
+    ],
+)

packages/compiler-cli/src/ngtsc/incremental/api.ts

@@ -0,0 +1,53 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import * as ts from 'typescript';
+
+/**
+ * Interface of the incremental build engine.
+ *
+ * `W` is a generic type representing a unit of work. This is generic to avoid a cyclic dependency
+ * between the incremental engine API definition and its consumer(s).
+ */
+export interface IncrementalBuild<W> {
+  /**
+   * Retrieve the prior analysis work, if any, done for the given source file.
+   */
+  priorWorkFor(sf: ts.SourceFile): W[]|null;
+}
+
+/**
+ * Tracks dependencies between source files or resources in the application.
+ */
+export interface DependencyTracker<T extends{fileName: string} = ts.SourceFile> {
+  /**
+   * Record that the file `from` depends on the file `on`.
+   */
+  addDependency(from: T, on: T): void;
+
+  /**
+   * Record that the file `from` depends on the resource file `on`.
+   */
+  addResourceDependency(from: T, on: string): void;
+
+  /**
+   * Record that the file `from` depends on the file `on` as well as `on`'s direct dependencies.
+   *
+   * This operation is reified immediately, so if future dependencies are added to `on` they will
+   * not automatically be added to `from`.
+   */
+  addTransitiveDependency(from: T, on: T): void;
+
+  /**
+   * Record that the file `from` depends on the resource dependencies of `resourcesOf`.
+   *
+   * This operation is reified immediately, so if future resource dependencies are added to
+   * `resourcesOf` they will not automatically be added to `from`.
+   */
+  addTransitiveResources(from: T, resourcesOf: T): void;
+}

packages/compiler-cli/src/ngtsc/incremental/src/README.md

@@ -2,11 +2,40 @@
 
 This package contains logic related to incremental compilation in ngtsc.
 
-In particular, it tracks dependencies between `ts.SourceFile`s, so the compiler can make intelligent decisions about when it's safe to skip certain operations. The main class performing this task is the `IncrementalDriver`.
+In particular, it tracks dependencies between `ts.SourceFile`s, so the compiler can make intelligent decisions about when it's safe to skip certain operations.
 
 # What optimizations are made?
 
-ngtsc makes a decision to skip the emit of a file if it can prove that the contents of the file will not have changed. To prove this, two conditions must be true.
+ngtsc currently makes two optimizations: reuse of prior analysis work, and the skipping of file emits.
+
+## Reuse of analyses
+
+If a build has succeeded previously, ngtsc has available the analyses of all Angular classes in the prior program, as well as the dependency graph which outlines inter-file dependencies. This is known as the "last good compilation".
+
+When the next build begins, ngtsc follows a simple algorithm which reuses prior work where possible:
+
+1) For each input file, ngtsc makes a determination as to whether the file is "logically changed".
+
+"Logically changed" means that either:
+
+* The file itself has physically changed on disk, or
+* One of the file's dependencies has physically changed on disk.
+
+Either of these conditions invalidates the previous analysis of the file.
+
+2) ngtsc begins constructing a new dependency graph.
+
+For each logically unchanged file, its dependencies are copied wholesale into the new graph.
+
+3) ngtsc begins analyzing each file in the program.
+
+If the file is logically unchanged, ngtsc will reuse the previous analysis and only call the 'register' phase of compilation, to apply any necessary side effects.
+
+If the file is logically changed, ngtsc will re-analyze it.
+
+## Skipping emit
+
+ngtsc makes a decision to skip the emit of a file if it can prove that the contents of the file will not have changed since the last good compilation. To prove this, two conditions must be true.
 
 * The input file itself must not have changed since the previous compilation.
 
@@ -16,6 +45,14 @@ The second condition is challenging to prove, as Angular allows statically evalu
 
 The emit of a file is the most expensive part of TypeScript/Angular compilation, so skipping emits when they are not necessary is one of the most valuable things the compiler can do to improve incremental build performance.
 
+## The two dependency graphs
+
+For both of the above optimizations, ngtsc makes use of dependency information extracted from the program. But these usages are subtly different.
+
+To reuse previous analyses, ngtsc uses the _prior_ compilation's dependency graph, plus the information about which files have changed, to determine whether it's safe to reuse the prior compilation's work.
+
+To skip emit, ngtsc uses the _current_ compilation's dependency graph, coupled with the information about which files have changed since the last successful build, to determine the set of outputs that need to be re-emitted.
+
 # How does incremental compilation work?
 
 The initial compilation is no different from a standalone compilation; the compiler is unaware that incremental compilation will be utilized.
@@ -28,7 +65,7 @@ This information is leveraged in two major ways:
 
 2) An `IncrementalDriver` instance is constructed from the old and new `ts.Program`s, and the previous program's `IncrementalDriver`.
 
-The compiler then proceeds normally, analyzing all of the Angular code within the program. As a part of this process, the compiler maps out all of the dependencies between files in the `IncrementalDriver`.
+The compiler then proceeds normally, using the `IncrementalDriver` to manage the reuse of any pertinent information while processing the new program. As a part of this process, the compiler (again) maps out all of the dependencies between files.
 
 ## Determination of files to emit
 
@@ -51,9 +88,10 @@ On every invocation, the compiler receives (or can easily determine) several pie
 
 With this information, the compiler can perform rebuild optimizations:
 
-1. The compiler analyzes the full program and generates a dependency graph, which describes the relationships between files in the program.
+1. The compiler uses the last good compilation's dependency graph to determine which parts of its analysis work can be reused.
-2. Based on this graph, the compiler can make a determination for each TS file whether it needs to be re-emitted or can safely be skipped. This produces a set called `pendingEmit` of every file which requires a re-emit.
+2. The compiler analyzes the rest of the program and generates an updated dependency graph, which describes the relationships between files in the program as they are currently.
-3. The compiler cycles through the files and emits those which are necessary, removing them from `pendingEmit`.
+3. Based on this graph, the compiler can make a determination for each TS file whether it needs to be re-emitted or can safely be skipped. This produces a set called `pendingEmit` of every file which requires a re-emit.
+4. The compiler cycles through the files and emits those which are necessary, removing them from `pendingEmit`.
 
 Theoretically, after this process `pendingEmit` should be empty. As a precaution against errors which might happen in the future, `pendingEmit` is also passed into future compilations, so any files which previously were determined to need an emit (but have not been successfully produced yet) will be retried on subsequent compilations. This is mostly relevant if a client of `ngtsc` attempts to implement emit-on-error functionality.
 
@@ -79,10 +117,6 @@ If a new build is started after a successful build, only `pendingEmit` from the
 
 There is plenty of room for improvement here, with diminishing returns for the work involved.
 
-## Optimization of re-analysis
-
-Currently, the compiler re-analyzes the entire `ts.Program` on each compilation. Under certain circumstances it may be possible for the compiler to reuse parts of the previous compilation's analysis rather than repeat the work, if it can be proven to be safe.
-
 ## Semantic dependency tracking
 
 Currently the compiler tracks dependencies only at the file level, and will re-emit dependent files if they _may_ have been affected by a change. Often times a change though does _not_ require updates to dependent files.
@@ -92,3 +126,16 @@ For example, today a component's `NgModule` and all of the other components whic
 In contrast, if the component's _selector_ changes, then all those dependent files do need to be updated since their `directiveDefs` might have changed.
 
 Currently the compiler does not distinguish these two cases, and conservatively always re-emits the entire NgModule chain. It would be possible to break the dependency graph down into finer-grained nodes and distinguish between updates that affect the component, vs updates that affect its dependents. This would be a huge win, but is exceedingly complex.
+
+## Skipping template type-checking
+
+For certain kinds of changes, it may be possible to avoid the cost of generating and checking the template type-checking file. Several levels of this can be imagined.
+
+For resource-only changes, only the component(s) which have changed resources need to be re-checked. No other components could be affected, so previously produced diagnostics are still valid.
+
+For arbitrary source changes, things get a bit more complicated. A change to any .ts file could affect types anywhere in the program (think `declare global ...`). If a set of affected components can be determined (perhaps via the import graph that the cycle analyzer extracts?) and it can be proven that the change does not impact any global types (exactly how to do this is left as an exercise for  the reader), then type-checking could be skipped for other components in the mix.
+
+If the above is too complex, then certain kinds of type changes might allow for the reuse of the text of the template type-checking file, if it can be proven that none of the inputs to its generation have changed. This is useful for two very important reasons.
+
+1) Generating (and subsequently parsing) the template type-checking file itself is expensive.
+2) Under ideal conditions, after an initial template type-checking program is created, it may be possible to reuse it for emit _and_ type-checking in subsequent builds. This would be a pretty advanced optimization but would save creation of a second `ts.Program` on each valid rebuild.

packages/compiler-cli/src/ngtsc/incremental/src/dependency_tracking.ts

@@ -0,0 +1,142 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import * as ts from 'typescript';
+
+import {DependencyTracker} from '../api';
+
+/**
+ * An implementation of the `DependencyTracker` dependency graph API.
+ *
+ * The `FileDependencyGraph`'s primary job is to determine whether a given file has "logically"
+ * changed, given the set of physical changes (direct changes to files on disk).
+ *
+ * A file is logically changed if at least one of three conditions is met:
+ *
+ * 1. The file itself has physically changed.
+ * 2. One of its dependencies has physically changed.
+ * 3. One of its resource dependencies has physically changed.
+ */
+export class FileDependencyGraph<T extends{fileName: string} = ts.SourceFile> implements
+    DependencyTracker<T> {
+  private nodes = new Map<T, FileNode>();
+
+  addDependency(from: T, on: T): void { this.nodeFor(from).dependsOn.add(on.fileName); }
+
+  addResourceDependency(from: T, resource: string): void {
+    this.nodeFor(from).usesResources.add(resource);
+  }
+
+  addTransitiveDependency(from: T, on: T): void {
+    const nodeFrom = this.nodeFor(from);
+    nodeFrom.dependsOn.add(on.fileName);
+
+    const nodeOn = this.nodeFor(on);
+    for (const dep of nodeOn.dependsOn) {
+      nodeFrom.dependsOn.add(dep);
+    }
+  }
+
+  addTransitiveResources(from: T, resourcesOf: T): void {
+    const nodeFrom = this.nodeFor(from);
+    const nodeOn = this.nodeFor(resourcesOf);
+    for (const dep of nodeOn.usesResources) {
+      nodeFrom.usesResources.add(dep);
+    }
+  }
+
+  isStale(sf: T, changedTsPaths: Set<string>, changedResources: Set<string>): boolean {
+    return isLogicallyChanged(sf, this.nodeFor(sf), changedTsPaths, EMPTY_SET, changedResources);
+  }
+
+  /**
+   * Update the current dependency graph from a previous one, incorporating a set of physical
+   * changes.
+   *
+   * This method performs two tasks:
+   *
+   * 1. For files which have not logically changed, their dependencies from `previous` are added to
+   *    `this` graph.
+   * 2. For files which have logically changed, they're added to a set of logically changed files
+   *    which is eventually returned.
+   *
+   * In essence, for build `n`, this method performs:
+   *
+   * G(n) + L(n) = G(n - 1) + P(n)
+   *
+   * where:
+   *
+   * G(n) = the dependency graph of build `n`
+   * L(n) = the logically changed files from build n - 1 to build n.
+   * P(n) = the physically changed files from build n - 1 to build n.
+   */
+  updateWithPhysicalChanges(
+      previous: FileDependencyGraph<T>, changedTsPaths: Set<string>, deletedTsPaths: Set<string>,
+      changedResources: Set<string>): Set<string> {
+    const logicallyChanged = new Set<string>();
+
+    for (const sf of previous.nodes.keys()) {
+      const node = previous.nodeFor(sf);
+      if (isLogicallyChanged(sf, node, changedTsPaths, deletedTsPaths, changedResources)) {
+        logicallyChanged.add(sf.fileName);
+      } else if (!deletedTsPaths.has(sf.fileName)) {
+        this.nodes.set(sf, {
+          dependsOn: new Set(node.dependsOn),
+          usesResources: new Set(node.usesResources),
+        });
+      }
+    }
+
+    return logicallyChanged;
+  }
+
+  private nodeFor(sf: T): FileNode {
+    if (!this.nodes.has(sf)) {
+      this.nodes.set(sf, {
+        dependsOn: new Set<string>(),
+        usesResources: new Set<string>(),
+      });
+    }
+    return this.nodes.get(sf) !;
+  }
+}
+
+/**
+ * Determine whether `sf` has logically changed, given its dependencies and the set of physically
+ * changed files and resources.
+ */
+function isLogicallyChanged<T extends{fileName: string}>(
+    sf: T, node: FileNode, changedTsPaths: ReadonlySet<string>, deletedTsPaths: ReadonlySet<string>,
+    changedResources: ReadonlySet<string>): boolean {
+  // A file is logically changed if it has physically changed itself (including being deleted).
+  if (changedTsPaths.has(sf.fileName) || deletedTsPaths.has(sf.fileName)) {
+    return true;
+  }
+
+  // A file is logically changed if one of its dependencies has physically cxhanged.
+  for (const dep of node.dependsOn) {
+    if (changedTsPaths.has(dep) || deletedTsPaths.has(dep)) {
+      return true;
+    }
+  }
+
+  // A file is logically changed if one of its resources has physically changed.
+  for (const dep of node.usesResources) {
+    if (changedResources.has(dep)) {
+      return true;
+    }
+  }
+  return false;
+}
+
+interface FileNode {
+  dependsOn: Set<string>;
+  usesResources: Set<string>;
+}
+
+const EMPTY_SET: ReadonlySet<any> = new Set<any>();

packages/compiler-cli/src/ngtsc/incremental/src/state.ts

@@ -8,13 +8,15 @@
 
 import * as ts from 'typescript';
 
-import {DependencyTracker} from '../../partial_evaluator';
+import {ClassRecord, TraitCompiler} from '../../transform';
-import {ResourceDependencyRecorder} from '../../util/src/resource_recorder';
+import {IncrementalBuild} from '../api';
+
+import {FileDependencyGraph} from './dependency_tracking';
 
 /**
  * Drives an incremental build, by tracking changes and determining which files need to be emitted.
  */
-export class IncrementalDriver implements DependencyTracker, ResourceDependencyRecorder {
+export class IncrementalDriver implements IncrementalBuild<ClassRecord> {
   /**
    * State of the current build.
    *
@@ -22,12 +24,9 @@ export class IncrementalDriver implements DependencyTracker, ResourceDependencyR
    */
   private state: BuildState;
 
-  /**
+  private constructor(
-   * Tracks metadata related to each `ts.SourceFile` in the program.
+      state: PendingBuildState, private allTsFiles: Set<ts.SourceFile>,
-   */
+      readonly depGraph: FileDependencyGraph, private logicalChanges: Set<string>|null) {
-  private metadata = new Map<ts.SourceFile, FileMetadata>();
-
-  private constructor(state: PendingBuildState, private allTsFiles: Set<ts.SourceFile>) {
     this.state = state;
   }
 
@@ -55,6 +54,7 @@ export class IncrementalDriver implements DependencyTracker, ResourceDependencyR
         pendingEmit: oldDriver.state.pendingEmit,
         changedResourcePaths: new Set<string>(),
         changedTsPaths: new Set<string>(),
+        lastGood: oldDriver.state.lastGood,
       };
     }
 
@@ -101,7 +101,7 @@ export class IncrementalDriver implements DependencyTracker, ResourceDependencyR
       }
     }
 
-    // The last step is to remove any deleted files from the state.
+    // The next step is to remove any deleted files from the state.
     for (const filePath of deletedTsPaths) {
       state.pendingEmit.delete(filePath);
 
@@ -110,8 +110,29 @@ export class IncrementalDriver implements DependencyTracker, ResourceDependencyR
       state.changedTsPaths.delete(filePath);
     }
 
-    // `state` now reflects the initial compilation state of the current
+    // Now, changedTsPaths contains physically changed TS paths. Use the previous program's logical
-    return new IncrementalDriver(state, new Set<ts.SourceFile>(tsOnlyFiles(newProgram)));
+    // dependency graph to determine logically changed files.
+    const depGraph = new FileDependencyGraph();
+
+    // If a previous compilation exists, use its dependency graph to determine the set of logically
+    // changed files.
+    let logicalChanges: Set<string>|null = null;
+    if (state.lastGood !== null) {
+      // Extract the set of logically changed files. At the same time, this operation populates the
+      // current (fresh) dependency graph with information about those files which have not
+      // logically changed.
+      logicalChanges = depGraph.updateWithPhysicalChanges(
+          state.lastGood.depGraph, state.changedTsPaths, deletedTsPaths,
+          state.changedResourcePaths);
+      for (const fileName of state.changedTsPaths) {
+        logicalChanges.add(fileName);
+      }
+    }
+
+    // `state` now reflects the initial pending state of the current compilation.
+
+    return new IncrementalDriver(
+        state, new Set<ts.SourceFile>(tsOnlyFiles(newProgram)), depGraph, logicalChanges);
   }
 
   static fresh(program: ts.Program): IncrementalDriver {
@@ -124,12 +145,14 @@ export class IncrementalDriver implements DependencyTracker, ResourceDependencyR
       pendingEmit: new Set<string>(tsFiles.map(sf => sf.fileName)),
       changedResourcePaths: new Set<string>(),
       changedTsPaths: new Set<string>(),
+      lastGood: null,
     };
 
-    return new IncrementalDriver(state, new Set(tsFiles));
+    return new IncrementalDriver(
+        state, new Set(tsFiles), new FileDependencyGraph(), /* logicalChanges */ null);
   }
 
-  recordSuccessfulAnalysis(): void {
+  recordSuccessfulAnalysis(traitCompiler: TraitCompiler): void {
     if (this.state.kind !== BuildStateKind.Pending) {
       // Changes have already been incorporated.
       return;
@@ -140,12 +163,7 @@ export class IncrementalDriver implements DependencyTracker, ResourceDependencyR
     const state: PendingBuildState = this.state;
 
     for (const sf of this.allTsFiles) {
-      // It's safe to skip emitting a file if:
+      if (this.depGraph.isStale(sf, state.changedTsPaths, state.changedResourcePaths)) {
-      // 1) it hasn't changed
-      // 2) none if its resource dependencies have changed
-      // 3) none of its source dependencies have changed
-      if (state.changedTsPaths.has(sf.fileName) || this.hasChangedResourceDependencies(sf) ||
-          this.getFileDependencies(sf).some(dep => state.changedTsPaths.has(dep.fileName))) {
         // Something has changed which requires this file be re-emitted.
         pendingEmit.add(sf.fileName);
       }
@@ -155,6 +173,13 @@ export class IncrementalDriver implements DependencyTracker, ResourceDependencyR
     this.state = {
       kind: BuildStateKind.Analyzed,
       pendingEmit,
+
+      // Since this compilation was successfully analyzed, update the "last good" artifacts to the
+      // ones from the current compilation.
+      lastGood: {
+        depGraph: this.depGraph,
+        traitCompiler: traitCompiler,
+      }
     };
   }
 
@@ -162,59 +187,20 @@ export class IncrementalDriver implements DependencyTracker, ResourceDependencyR
 
   safeToSkipEmit(sf: ts.SourceFile): boolean { return !this.state.pendingEmit.has(sf.fileName); }
 
-  trackFileDependency(dep: ts.SourceFile, src: ts.SourceFile) {
+  priorWorkFor(sf: ts.SourceFile): ClassRecord[]|null {
-    const metadata = this.ensureMetadata(src);
+    if (this.state.lastGood === null || this.logicalChanges === null) {
-    metadata.fileDependencies.add(dep);
+      // There is no previous good build, so no prior work exists.
-  }
+      return null;
-
+    } else if (this.logicalChanges.has(sf.fileName)) {
-  trackFileDependencies(deps: ts.SourceFile[], src: ts.SourceFile) {
+      // Prior work might exist, but would be stale as the file in question has logically changed.
-    const metadata = this.ensureMetadata(src);
+      return null;
-    for (const dep of deps) {
+    } else {
-      metadata.fileDependencies.add(dep);
+      // Prior work might exist, and if it does then it's usable!
+      return this.state.lastGood.traitCompiler.recordsFor(sf);
     }
   }
-
-  getFileDependencies(file: ts.SourceFile): ts.SourceFile[] {
-    if (!this.metadata.has(file)) {
-      return [];
-    }
-    const meta = this.metadata.get(file) !;
-    return Array.from(meta.fileDependencies);
-  }
-
-  recordResourceDependency(file: ts.SourceFile, resourcePath: string): void {
-    const metadata = this.ensureMetadata(file);
-    metadata.resourcePaths.add(resourcePath);
-  }
-
-  private ensureMetadata(sf: ts.SourceFile): FileMetadata {
-    const metadata = this.metadata.get(sf) || new FileMetadata();
-    this.metadata.set(sf, metadata);
-    return metadata;
-  }
-
-  private hasChangedResourceDependencies(sf: ts.SourceFile): boolean {
-    if (!this.metadata.has(sf)) {
-      return false;
-    }
-    const resourceDeps = this.metadata.get(sf) !.resourcePaths;
-    return Array.from(resourceDeps.keys())
-        .some(
-            resourcePath => this.state.kind === BuildStateKind.Pending &&
-                this.state.changedResourcePaths.has(resourcePath));
-  }
 }
 
-/**
- * Information about the whether a source file can have analysis or emission can be skipped.
- */
-class FileMetadata {
-  /** A set of source files that this file depends upon. */
-  fileDependencies = new Set<ts.SourceFile>();
-  resourcePaths = new Set<string>();
-}
-
-
 type BuildState = PendingBuildState | AnalyzedBuildState;
 
 enum BuildStateKind {
@@ -247,6 +233,26 @@ interface BaseBuildState {
    * See the README.md for more information on this algorithm.
    */
   pendingEmit: Set<string>;
+
+
+  /**
+   * Specific aspects of the last compilation which successfully completed analysis, if any.
+   */
+  lastGood: {
+    /**
+     * The dependency graph from the last successfully analyzed build.
+     *
+     * This is used to determine the logical impact of physical file changes.
+     */
+    depGraph: FileDependencyGraph;
+
+    /**
+     * The `TraitCompiler` from the last successfully analyzed build.
+     *
+     * This is used to extract "prior work" which might be reusable in this compilation.
+     */
+    traitCompiler: TraitCompiler;
+  }|null;
 }
 
 /**

packages/compiler-cli/src/ngtsc/partial_evaluator/BUILD.bazel

@@ -12,6 +12,7 @@ ts_library(
         "//packages:types",
         "//packages/compiler",
         "//packages/compiler-cli/src/ngtsc/imports",
+        "//packages/compiler-cli/src/ngtsc/incremental:api",
         "//packages/compiler-cli/src/ngtsc/reflection",
         "//packages/compiler-cli/src/ngtsc/util",
         "@npm//@types/node",

packages/compiler-cli/src/ngtsc/partial_evaluator/index.ts

@@ -7,5 +7,5 @@
  */
 
 export {DynamicValue} from './src/dynamic';
-export {DependencyTracker, ForeignFunctionResolver, PartialEvaluator} from './src/interface';
+export {ForeignFunctionResolver, PartialEvaluator} from './src/interface';
 export {BuiltinFn, EnumValue, ResolvedValue, ResolvedValueArray, ResolvedValueMap} from './src/result';

packages/compiler-cli/src/ngtsc/partial_evaluator/src/interface.ts

@@ -9,19 +9,12 @@
 import * as ts from 'typescript';
 
 import {Reference} from '../../imports';
+import {DependencyTracker} from '../../incremental/api';
 import {ReflectionHost} from '../../reflection';
 
 import {StaticInterpreter} from './interpreter';
 import {ResolvedValue} from './result';
 
-/**
- * Implement this interface to record dependency relations between
- * source files.
- */
-export interface DependencyTracker {
-  trackFileDependency(dep: ts.SourceFile, src: ts.SourceFile): void;
-}
-
 export type ForeignFunctionResolver =
     (node: Reference<ts.FunctionDeclaration|ts.MethodDeclaration|ts.FunctionExpression>,
      args: ReadonlyArray<ts.Expression>) => ts.Expression | null;
@@ -29,7 +22,7 @@ export type ForeignFunctionResolver =
 export class PartialEvaluator {
   constructor(
       private host: ReflectionHost, private checker: ts.TypeChecker,
-      private dependencyTracker?: DependencyTracker) {}
+      private dependencyTracker: DependencyTracker|null) {}
 
   evaluate(expr: ts.Expression, foreignFunctionResolver?: ForeignFunctionResolver): ResolvedValue {
     const interpreter = new StaticInterpreter(this.host, this.checker, this.dependencyTracker);

packages/compiler-cli/src/ngtsc/partial_evaluator/src/interpreter.ts

@@ -10,12 +10,13 @@ import * as ts from 'typescript';
 
 import {Reference} from '../../imports';
 import {OwningModule} from '../../imports/src/references';
+import {DependencyTracker} from '../../incremental/api';
 import {Declaration, InlineDeclaration, ReflectionHost} from '../../reflection';
 import {isDeclaration} from '../../util/src/typescript';
 
 import {ArrayConcatBuiltinFn, ArraySliceBuiltinFn} from './builtin';
 import {DynamicValue} from './dynamic';
-import {DependencyTracker, ForeignFunctionResolver} from './interface';
+import {ForeignFunctionResolver} from './interface';
 import {BuiltinFn, EnumValue, ResolvedModule, ResolvedValue, ResolvedValueArray, ResolvedValueMap} from './result';
 import {evaluateTsHelperInline} from './ts_helpers';
 
@@ -89,7 +90,7 @@ interface Context {
 export class StaticInterpreter {
   constructor(
       private host: ReflectionHost, private checker: ts.TypeChecker,
-      private dependencyTracker?: DependencyTracker) {}
+      private dependencyTracker: DependencyTracker|null) {}
 
   visit(node: ts.Expression, context: Context): ResolvedValue {
     return this.visitExpression(node, context);
@@ -249,8 +250,8 @@ export class StaticInterpreter {
   }
 
   private visitDeclaration(node: ts.Declaration, context: Context): ResolvedValue {
-    if (this.dependencyTracker) {
+    if (this.dependencyTracker !== null) {
-      this.dependencyTracker.trackFileDependency(node.getSourceFile(), context.originatingFile);
+      this.dependencyTracker.addDependency(context.originatingFile, node.getSourceFile());
     }
     if (this.host.isClass(node)) {
       return this.getReference(node, context);

packages/compiler-cli/src/ngtsc/partial_evaluator/test/BUILD.bazel

@@ -14,6 +14,7 @@ ts_library(
         "//packages/compiler-cli/src/ngtsc/file_system",
         "//packages/compiler-cli/src/ngtsc/file_system/testing",
         "//packages/compiler-cli/src/ngtsc/imports",
+        "//packages/compiler-cli/src/ngtsc/incremental:api",
         "//packages/compiler-cli/src/ngtsc/partial_evaluator",
         "//packages/compiler-cli/src/ngtsc/reflection",
         "//packages/compiler-cli/src/ngtsc/testing",

packages/compiler-cli/src/ngtsc/partial_evaluator/test/evaluator_spec.ts

@@ -6,14 +6,17 @@
  * found in the LICENSE file at https://angular.io/license
  */
 import * as ts from 'typescript';
+
 import {absoluteFrom, getSourceFileOrError} from '../../file_system';
 import {runInEachFileSystem} from '../../file_system/testing';
 import {Reference} from '../../imports';
+import {DependencyTracker} from '../../incremental/api';
 import {FunctionDefinition, TsHelperFn, TypeScriptReflectionHost} from '../../reflection';
 import {getDeclaration, makeProgram} from '../../testing';
 import {DynamicValue} from '../src/dynamic';
 import {PartialEvaluator} from '../src/interface';
 import {EnumValue} from '../src/result';
+
 import {evaluate, firstArgFfr, makeEvaluator, makeExpression, owningModuleOf} from './utils';
 
 runInEachFileSystem(() => {
@@ -551,7 +554,7 @@ runInEachFileSystem(() => {
             },
           ]);
       const reflectionHost = new TsLibAwareReflectionHost(checker);
-      const evaluator = new PartialEvaluator(reflectionHost, checker);
+      const evaluator = new PartialEvaluator(reflectionHost, checker, null);
       const value = evaluator.evaluate(expression);
       expect(value).toEqual([1, 2, 3]);
     });
@@ -572,44 +575,42 @@ runInEachFileSystem(() => {
             },
           ]);
       const reflectionHost = new TsLibAwareReflectionHost(checker);
-      const evaluator = new PartialEvaluator(reflectionHost, checker);
+      const evaluator = new PartialEvaluator(reflectionHost, checker, null);
       const value = evaluator.evaluate(expression);
       expect(value).toEqual([1, 2, 3]);
     });
 
     describe('(visited file tracking)', () => {
       it('should track each time a source file is visited', () => {
-        const trackFileDependency = jasmine.createSpy('DependencyTracker');
+        const addDependency = jasmine.createSpy('DependencyTracker');
         const {expression, checker} = makeExpression(
             `class A { static foo = 42; } function bar() { return A.foo; }`, 'bar()');
-        const evaluator = makeEvaluator(checker, {trackFileDependency});
+        const evaluator = makeEvaluator(checker, {...fakeDepTracker, addDependency});
         evaluator.evaluate(expression);
-        expect(trackFileDependency).toHaveBeenCalledTimes(2);  // two declaration visited
+        expect(addDependency).toHaveBeenCalledTimes(2);  // two declaration visited
-        expect(
+        expect(addDependency.calls.allArgs().map(args => [args[0].fileName, args[1].fileName]))
-            trackFileDependency.calls.allArgs().map(args => [args[0].fileName, args[1].fileName]))
             .toEqual([[_('/entry.ts'), _('/entry.ts')], [_('/entry.ts'), _('/entry.ts')]]);
       });
 
       it('should track imported source files', () => {
-        const trackFileDependency = jasmine.createSpy('DependencyTracker');
+        const addDependency = jasmine.createSpy('DependencyTracker');
         const {expression, checker} =
             makeExpression(`import {Y} from './other'; const A = Y;`, 'A', [
               {name: _('/other.ts'), contents: `export const Y = 'test';`},
               {name: _('/not-visited.ts'), contents: `export const Z = 'nope';`}
             ]);
-        const evaluator = makeEvaluator(checker, {trackFileDependency});
+        const evaluator = makeEvaluator(checker, {...fakeDepTracker, addDependency});
         evaluator.evaluate(expression);
-        expect(trackFileDependency).toHaveBeenCalledTimes(2);
+        expect(addDependency).toHaveBeenCalledTimes(2);
-        expect(
+        expect(addDependency.calls.allArgs().map(args => [args[0].fileName, args[1].fileName]))
-            trackFileDependency.calls.allArgs().map(args => [args[0].fileName, args[1].fileName]))
             .toEqual([
               [_('/entry.ts'), _('/entry.ts')],
-              [_('/other.ts'), _('/entry.ts')],
+              [_('/entry.ts'), _('/other.ts')],
             ]);
       });
 
       it('should track files passed through during re-exports', () => {
-        const trackFileDependency = jasmine.createSpy('DependencyTracker');
+        const addDependency = jasmine.createSpy('DependencyTracker');
         const {expression, checker} =
             makeExpression(`import * as mod from './direct-reexport';`, 'mod.value.property', [
               {name: _('/const.ts'), contents: 'export const value = {property: "test"};'},
@@ -626,16 +627,15 @@ runInEachFileSystem(() => {
                 contents: `export {value} from './indirect-reexport';`
               },
             ]);
-        const evaluator = makeEvaluator(checker, {trackFileDependency});
+        const evaluator = makeEvaluator(checker, {...fakeDepTracker, addDependency});
         evaluator.evaluate(expression);
-        expect(trackFileDependency).toHaveBeenCalledTimes(2);
+        expect(addDependency).toHaveBeenCalledTimes(2);
-        expect(
+        expect(addDependency.calls.allArgs().map(args => [args[0].fileName, args[1].fileName]))
-            trackFileDependency.calls.allArgs().map(args => [args[0].fileName, args[1].fileName]))
             .toEqual([
-              [_('/direct-reexport.ts'), _('/entry.ts')],
+              [_('/entry.ts'), _('/direct-reexport.ts')],
               // Not '/indirect-reexport.ts' or '/def.ts'.
               // TS skips through them when finding the original symbol for `value`
-              [_('/const.ts'), _('/entry.ts')],
+              [_('/entry.ts'), _('/const.ts')],
             ]);
       });
     });
@@ -675,3 +675,10 @@ runInEachFileSystem(() => {
     }
   }
 });
+
+const fakeDepTracker: DependencyTracker = {
+  addDependency: () => undefined,
+  addResourceDependency: () => undefined,
+  addTransitiveDependency: () => undefined,
+  addTransitiveResources: () => undefined,
+};

packages/compiler-cli/src/ngtsc/partial_evaluator/test/utils.ts

@@ -6,12 +6,14 @@
  * found in the LICENSE file at https://angular.io/license
  */
 import * as ts from 'typescript';
+
 import {absoluteFrom} from '../../file_system';
 import {TestFile} from '../../file_system/testing';
 import {Reference} from '../../imports';
+import {DependencyTracker} from '../../incremental/api';
 import {TypeScriptReflectionHost} from '../../reflection';
 import {getDeclaration, makeProgram} from '../../testing';
-import {DependencyTracker, ForeignFunctionResolver, PartialEvaluator} from '../src/interface';
+import {ForeignFunctionResolver, PartialEvaluator} from '../src/interface';
 import {ResolvedValue} from '../src/result';
 
 export function makeExpression(code: string, expr: string, supportingFiles: TestFile[] = []): {
@@ -40,7 +42,7 @@ export function makeExpression(code: string, expr: string, supportingFiles: Test
 export function makeEvaluator(
     checker: ts.TypeChecker, tracker?: DependencyTracker): PartialEvaluator {
   const reflectionHost = new TypeScriptReflectionHost(checker);
-  return new PartialEvaluator(reflectionHost, checker, tracker);
+  return new PartialEvaluator(reflectionHost, checker, tracker !== undefined ? tracker : null);
 }
 
 export function evaluate<T extends ResolvedValue>(

packages/compiler-cli/src/ngtsc/program.ts

@@ -257,13 +257,8 @@ export class NgtscProgram implements api.Program {
     await Promise.all(promises);
 
     this.perfRecorder.stop(analyzeSpan);
-    this.compilation.resolve();
 
-    this.recordNgModuleScopeDependencies();
+    this.resolveCompilation(this.compilation);
-
-    // At this point, analysis is complete and the compiler can now calculate which files need to be
-    // emitted, so do that.
-    this.incrementalDriver.recordSuccessfulAnalysis();
   }
 
   listLazyRoutes(entryRoute?: string|undefined): api.LazyRoute[] {
@@ -339,17 +334,22 @@ export class NgtscProgram implements api.Program {
         this.perfRecorder.stop(analyzeFileSpan);
       }
       this.perfRecorder.stop(analyzeSpan);
-      this.compilation.resolve();
 
-      this.recordNgModuleScopeDependencies();
+      this.resolveCompilation(this.compilation);
-
-      // At this point, analysis is complete and the compiler can now calculate which files need to
-      // be emitted, so do that.
-      this.incrementalDriver.recordSuccessfulAnalysis();
     }
     return this.compilation;
   }
 
+  private resolveCompilation(compilation: TraitCompiler): void {
+    compilation.resolve();
+
+    this.recordNgModuleScopeDependencies();
+
+    // At this point, analysis is complete and the compiler can now calculate which files need to
+    // be emitted, so do that.
+    this.incrementalDriver.recordSuccessfulAnalysis(compilation);
+  }
+
   emit(opts?: {
     emitFlags?: api.EmitFlags,
     cancellationToken?: ts.CancellationToken,
@@ -616,7 +616,8 @@ export class NgtscProgram implements api.Program {
       this.aliasingHost = new FileToModuleAliasingHost(this.fileToModuleHost);
     }
 
-    const evaluator = new PartialEvaluator(this.reflector, checker, this.incrementalDriver);
+    const evaluator =
+        new PartialEvaluator(this.reflector, checker, this.incrementalDriver.depGraph);
     const dtsReader = new DtsMetadataReader(checker, this.reflector);
     const localMetaRegistry = new LocalMetadataRegistry();
     const localMetaReader: MetadataReader = localMetaRegistry;
@@ -654,7 +655,7 @@ export class NgtscProgram implements api.Program {
           this.options.preserveWhitespaces || false, this.options.i18nUseExternalIds !== false,
           this.options.enableI18nLegacyMessageIdFormat !== false, this.moduleResolver,
           this.cycleAnalyzer, this.refEmitter, this.defaultImportTracker,
-          this.closureCompilerEnabled, this.incrementalDriver),
+          this.incrementalDriver.depGraph, this.closureCompilerEnabled),
       // TODO(alxhub): understand why the cast here is necessary (something to do with `null` not
       // being assignable to `unknown` when wrapped in `Readonly`).
       new DirectiveDecoratorHandler(
@@ -674,7 +675,7 @@ export class NgtscProgram implements api.Program {
     ];
 
     return new TraitCompiler(
-        handlers, this.reflector, this.perfRecorder,
+        handlers, this.reflector, this.perfRecorder, this.incrementalDriver,
         this.options.compileNonExportedClasses !== false, this.dtsTransforms);
   }
 
@@ -684,38 +685,39 @@ export class NgtscProgram implements api.Program {
    */
   private recordNgModuleScopeDependencies() {
     const recordSpan = this.perfRecorder.start('recordDependencies');
+    const depGraph = this.incrementalDriver.depGraph;
+
     for (const scope of this.scopeRegistry !.getCompilationScopes()) {
       const file = scope.declaration.getSourceFile();
       const ngModuleFile = scope.ngModule.getSourceFile();
 
       // A change to any dependency of the declaration causes the declaration to be invalidated,
       // which requires the NgModule to be invalidated as well.
-      const deps = this.incrementalDriver.getFileDependencies(file);
+      depGraph.addTransitiveDependency(ngModuleFile, file);
-      this.incrementalDriver.trackFileDependencies(deps, ngModuleFile);
 
       // A change to the NgModule file should cause the declaration itself to be invalidated.
-      this.incrementalDriver.trackFileDependency(ngModuleFile, file);
+      depGraph.addDependency(file, ngModuleFile);
 
-      // A change to any directive/pipe in the compilation scope should cause the declaration to be
+      const meta = this.metaReader !.getDirectiveMetadata(new Reference(scope.declaration));
-      // invalidated.
+      if (meta !== null && meta.isComponent) {
-      for (const directive of scope.directives) {
+        // If a component's template changes, it might have affected the import graph, and thus the
-        const dirSf = directive.ref.node.getSourceFile();
+        // remote scoping feature which is activated in the event of potential import cycles. Thus,
+        // the module depends not only on the transitive dependencies of the component, but on its
+        // resources as well.
+        depGraph.addTransitiveResources(ngModuleFile, file);
 
-        // When a directive in scope is updated, the declaration needs to be recompiled as e.g.
+        // A change to any directive/pipe in the compilation scope should cause the component to be
-        // a selector may have changed.
+        // invalidated.
-        this.incrementalDriver.trackFileDependency(dirSf, file);
+        for (const directive of scope.directives) {
-
+          // When a directive in scope is updated, the component needs to be recompiled as e.g. a
-        // When any of the dependencies of the declaration changes, the NgModule scope may be
+          // selector may have changed.
-        // affected so a component within scope must be recompiled. Only components need to be
+          depGraph.addTransitiveDependency(file, directive.ref.node.getSourceFile());
-        // recompiled, as directives are not dependent upon the compilation scope.
+        }
-        if (directive.isComponent) {
+        for (const pipe of scope.pipes) {
-          this.incrementalDriver.trackFileDependencies(deps, dirSf);
+          // When a pipe in scope is updated, the component needs to be recompiled as e.g. the
+          // pipe's name may have changed.
+          depGraph.addTransitiveDependency(file, pipe.ref.node.getSourceFile());
         }
-      }
-      for (const pipe of scope.pipes) {
-        // When a pipe in scope is updated, the declaration needs to be recompiled as e.g.
-        // the pipe's name may have changed.
-        this.incrementalDriver.trackFileDependency(pipe.ref.node.getSourceFile(), file);
       }
     }
     this.perfRecorder.stop(recordSpan);

packages/compiler-cli/src/ngtsc/transform/BUILD.bazel

@@ -11,6 +11,7 @@ ts_library(
         "//packages/compiler",
         "//packages/compiler-cli/src/ngtsc/diagnostics",
         "//packages/compiler-cli/src/ngtsc/imports",
+        "//packages/compiler-cli/src/ngtsc/incremental:api",
         "//packages/compiler-cli/src/ngtsc/indexer",
         "//packages/compiler-cli/src/ngtsc/modulewithproviders",
         "//packages/compiler-cli/src/ngtsc/perf",

packages/compiler-cli/src/ngtsc/transform/index.ts

@@ -7,6 +7,6 @@
  */
 
 export * from './src/api';
-export {TraitCompiler} from './src/compilation';
+export {ClassRecord, TraitCompiler} from './src/compilation';
 export {declarationTransformFactory, DtsTransformRegistry, IvyDeclarationDtsTransform, ReturnTypeTransform} from './src/declaration';
 export {ivyTransformFactory} from './src/transform';

packages/compiler-cli/src/ngtsc/transform/src/api.ts

@@ -73,6 +73,8 @@ export enum HandlerFlags {
  * @param `R` The type of resolution metadata produced by `resolve`.
  */
 export interface DecoratorHandler<D, A, R> {
+  readonly name: string;
+
   /**
    * The precedence of a handler controls how it interacts with other handlers that match the same
    * class.

packages/compiler-cli/src/ngtsc/transform/src/compilation.ts

@@ -11,6 +11,7 @@ import * as ts from 'typescript';
 
 import {ErrorCode, FatalDiagnosticError} from '../../diagnostics';
 import {ImportRewriter} from '../../imports';
+import {IncrementalBuild} from '../../incremental/api';
 import {IndexingContext} from '../../indexer';
 import {ModuleWithProvidersScanner} from '../../modulewithproviders';
 import {PerfRecorder} from '../../perf';
@@ -22,10 +23,11 @@ import {AnalysisOutput, CompileResult, DecoratorHandler, DetectResult, HandlerPr
 import {DtsTransformRegistry} from './declaration';
 import {Trait, TraitState} from './trait';
 
+
 /**
  * Records information about a specific class that has matched traits.
  */
-interface ClassRecord {
+export interface ClassRecord {
   /**
    * The `ClassDeclaration` of the class which has Angular traits applied.
    */
@@ -59,7 +61,13 @@ interface ClassRecord {
 /**
  * The heart of Angular compilation.
  *
- * The `TraitCompiler` is responsible for processing all classes in the program and
+ * The `TraitCompiler` is responsible for processing all classes in the program. Any time a
+ * `DecoratorHandler` matches a class, a "trait" is created to represent that Angular aspect of the
+ * class (such as the class having a component definition).
+ *
+ * The `TraitCompiler` transitions each trait through the various phases of compilation, culminating
+ * in the production of `CompileResult`s instructing the compiler to apply various mutations to the
+ * class (like adding fields or type declarations).
  */
 export class TraitCompiler {
   /**
@@ -76,19 +84,17 @@ export class TraitCompiler {
 
   private reexportMap = new Map<string, Map<string, [string, string]>>();
 
-  /**
+  private handlersByName = new Map<string, DecoratorHandler<unknown, unknown, unknown>>();
-   * @param handlers array of `DecoratorHandler`s which will be executed against each class in the
+
-   * program
-   * @param checker TypeScript `TypeChecker` instance for the program
-   * @param reflector `ReflectionHost` through which all reflection operations will be performed
-   * @param coreImportsFrom a TypeScript `SourceFile` which exports symbols needed for Ivy imports
-   * when compiling @angular/core, or `null` if the current program is not @angular/core. This is
-   * `null` in most cases.
-   */
   constructor(
       private handlers: DecoratorHandler<unknown, unknown, unknown>[],
       private reflector: ReflectionHost, private perf: PerfRecorder,
-      private compileNonExportedClasses: boolean, private dtsTransforms: DtsTransformRegistry) {}
+      private incrementalBuild: IncrementalBuild<ClassRecord>,
+      private compileNonExportedClasses: boolean, private dtsTransforms: DtsTransformRegistry) {
+    for (const handler of handlers) {
+      this.handlersByName.set(handler.name, handler);
+    }
+  }
 
   analyzeSync(sf: ts.SourceFile): void { this.analyze(sf, false); }
 
@@ -101,6 +107,16 @@ export class TraitCompiler {
     // type of 'void', so `undefined` is used instead.
     const promises: Promise<void>[] = [];
 
+    const priorWork = this.incrementalBuild.priorWorkFor(sf);
+    if (priorWork !== null) {
+      for (const priorRecord of priorWork) {
+        this.adopt(priorRecord);
+      }
+
+      // Skip the rest of analysis, as this file's prior traits are being reused.
+      return;
+    }
+
     const visit = (node: ts.Node): void => {
       if (isNamedClassDeclaration(node)) {
         this.analyzeClass(node, preanalyze ? promises : null);
@@ -117,6 +133,60 @@ export class TraitCompiler {
     }
   }
 
+  recordsFor(sf: ts.SourceFile): ClassRecord[]|null {
+    if (!this.fileToClasses.has(sf)) {
+      return null;
+    }
+    const records: ClassRecord[] = [];
+    for (const clazz of this.fileToClasses.get(sf) !) {
+      records.push(this.classes.get(clazz) !);
+    }
+    return records;
+  }
+
+  /**
+   * Import a `ClassRecord` from a previous compilation.
+   *
+   * Traits from the `ClassRecord` have accurate metadata, but the `handler` is from the old program
+   * and needs to be updated (matching is done by name). A new pending trait is created and then
+   * transitioned to analyzed using the previous analysis. If the trait is in the errored state,
+   * instead the errors are copied over.
+   */
+  private adopt(priorRecord: ClassRecord): void {
+    const record: ClassRecord = {
+      hasPrimaryHandler: priorRecord.hasPrimaryHandler,
+      hasWeakHandlers: priorRecord.hasWeakHandlers,
+      metaDiagnostics: priorRecord.metaDiagnostics,
+      node: priorRecord.node,
+      traits: [],
+    };
+
+    for (const priorTrait of priorRecord.traits) {
+      const handler = this.handlersByName.get(priorTrait.handler.name) !;
+      let trait: Trait<unknown, unknown, unknown> = Trait.pending(handler, priorTrait.detected);
+
+      if (priorTrait.state === TraitState.ANALYZED || priorTrait.state === TraitState.RESOLVED) {
+        trait = trait.toAnalyzed(priorTrait.analysis);
+        if (trait.handler.register !== undefined) {
+          trait.handler.register(record.node, trait.analysis);
+        }
+      } else if (priorTrait.state === TraitState.SKIPPED) {
+        trait = trait.toSkipped();
+      } else if (priorTrait.state === TraitState.ERRORED) {
+        trait = trait.toErrored(priorTrait.diagnostics);
+      }
+
+      record.traits.push(trait);
+    }
+
+    this.classes.set(record.node, record);
+    const sf = record.node.getSourceFile();
+    if (!this.fileToClasses.has(sf)) {
+      this.fileToClasses.set(sf, new Set<ClassDeclaration>());
+    }
+    this.fileToClasses.get(sf) !.add(record.node);
+  }
+
   private scanClassForTraits(clazz: ClassDeclaration): ClassRecord|null {
     if (!this.compileNonExportedClasses && !isExported(clazz)) {
       return null;

packages/compiler-cli/src/ngtsc/util/BUILD.bazel

@@ -10,6 +10,7 @@ ts_library(
     deps = [
         "//packages:types",
         "//packages/compiler-cli/src/ngtsc/file_system",
+        "//packages/compiler-cli/src/ngtsc/incremental:api",
         "@npm//@types/node",
         "@npm//typescript",
     ],

packages/compiler-cli/src/ngtsc/util/src/resource_recorder.ts

@@ -1,20 +0,0 @@
-
-/**
- * @license
- * Copyright Google Inc. All Rights Reserved.
- *
- * Use of this source code is governed by an MIT-style license that can be
- * found in the LICENSE file at https://angular.io/license
- */
-import * as ts from 'typescript';
-
-/**
- * Implement this interface to record what resources a source file depends upon.
- */
-export interface ResourceDependencyRecorder {
-  recordResourceDependency(file: ts.SourceFile, resourcePath: string): void;
-}
-
-export class NoopResourceDependencyRecorder implements ResourceDependencyRecorder {
-  recordResourceDependency(): void {}
-}

packages/compiler-cli/test/metadata/evaluator_spec.ts

@@ -159,7 +159,7 @@ describe('Evaluator', () => {
     });
   });
 
-  it('should support referene to a declared module type', () => {
+  it('should support reference to a declared module type', () => {
     const declared = program.getSourceFile('declared.ts') !;
     const aDecl = findVar(declared, 'a') !;
     expect(evaluator.evaluateNode(aDecl.type !)).toEqual({

packages/compiler-cli/test/ngtsc/incremental_error_spec.ts

@@ -32,13 +32,14 @@ runInEachFileSystem(() => {
 
     function expectToHaveWritten(files: string[]): void {
       const set = env.getFilesWrittenSinceLastFlush();
+
+      const expectedSet = new Set<string>();
       for (const file of files) {
-        expect(set).toContain(file);
+        expectedSet.add(file);
-        expect(set).toContain(file.replace(/\.js$/, '.d.ts'));
+        expectedSet.add(file.replace(/\.js$/, '.d.ts'));
       }
 
-      // Validate that 2x the size of `files` have been written (one .d.ts, one .js) and no more.
+      expect(set).toEqual(expectedSet);
-      expect(set.size).toBe(2 * files.length);
 
       // Reset for the next compilation.
       env.flushWrittenFileTracking();
@@ -479,7 +480,7 @@ runInEachFileSystem(() => {
           '/other.js',
 
           // Because a.html changed
-          '/a.js',
+          '/a.js', '/module.js',
 
           // b.js and module.js should not be re-emitted, because specifically when tracking
           // resource dependencies, the compiler knows that a change to a resource file only affects

packages/compiler-cli/test/ngtsc/incremental_spec.ts

@@ -160,7 +160,7 @@ runInEachFileSystem(() => {
       expect(written).toContain('/bar_directive.js');
       expect(written).toContain('/bar_component.js');
       expect(written).toContain('/bar_module.js');
-      expect(written).not.toContain('/foo_component.js');
+      expect(written).toContain('/foo_component.js');
       expect(written).not.toContain('/foo_pipe.js');
       expect(written).not.toContain('/foo_module.js');
     });
@@ -251,7 +251,7 @@ runInEachFileSystem(() => {
       expect(written).toContain('/foo_module.js');
     });
 
-    it('should rebuild only a Component (but with the correct CompilationScope) if its template has changed',
+    it('should rebuild only a Component (but with the correct CompilationScope) and its module if its template has changed',
        () => {
          setupFooBarProgram(env);
 
@@ -262,7 +262,9 @@ runInEachFileSystem(() => {
          const written = env.getFilesWrittenSinceLastFlush();
          expect(written).not.toContain('/bar_directive.js');
          expect(written).toContain('/bar_component.js');
-         expect(written).not.toContain('/bar_module.js');
+         // /bar_module.js should also be re-emitted, because remote scoping of BarComponent might
+         // have been affected.
+         expect(written).toContain('/bar_module.js');
          expect(written).not.toContain('/foo_component.js');
          expect(written).not.toContain('/foo_pipe.js');
          expect(written).not.toContain('/foo_module.js');

packages/core/schematics/migrations/missing-injectable/transform.ts

@@ -41,8 +41,8 @@ export class MissingInjectableTransform {
   constructor(
       private typeChecker: ts.TypeChecker,
       private getUpdateRecorder: (sf: ts.SourceFile) => UpdateRecorder) {
-    this.providersEvaluator =
+    this.providersEvaluator = new ProvidersEvaluator(
-        new ProvidersEvaluator(new TypeScriptReflectionHost(typeChecker), typeChecker);
+        new TypeScriptReflectionHost(typeChecker), typeChecker, /* dependencyTracker */ null);
   }
 
   recordChanges() { this.importManager.recordChanges(); }

packages/core/schematics/migrations/module-with-providers/transform.ts

@@ -24,8 +24,9 @@ const TODO_COMMENT = 'TODO: The following node requires a generic type for `Modu
 
 export class ModuleWithProvidersTransform {
   private printer = ts.createPrinter();
-  private partialEvaluator: PartialEvaluator =
+  private partialEvaluator: PartialEvaluator = new PartialEvaluator(
-      new PartialEvaluator(new TypeScriptReflectionHost(this.typeChecker), this.typeChecker);
+      new TypeScriptReflectionHost(this.typeChecker), this.typeChecker,
+      /* dependencyTracker */ null);
 
   constructor(
       private typeChecker: ts.TypeChecker,

packages/core/schematics/migrations/undecorated-classes-with-di/index.ts

@@ -82,8 +82,8 @@ function runUndecoratedClassesMigration(
 
   const {program, compiler} = programData;
   const typeChecker = program.getTypeChecker();
-  const partialEvaluator =
+  const partialEvaluator = new PartialEvaluator(
-      new PartialEvaluator(new TypeScriptReflectionHost(typeChecker), typeChecker);
+      new TypeScriptReflectionHost(typeChecker), typeChecker, /* dependencyTracker */ null);
   const declarationCollector = new NgDeclarationCollector(typeChecker, partialEvaluator);
   const sourceFiles = program.getSourceFiles().filter(
       s => !s.isDeclarationFile && !program.isSourceFileFromExternalLibrary(s));