perf(ivy): basic incremental compilation for ngtsc (#29380)

This commit introduces a mechanism for incremental compilation to the ngtsc
compiler.

Previously, incremental information was used in the construction of the
ts.Program for subsequent compilations, but was not used in ngtsc itself.

This commit adds an IncrementalState class, which tracks state between ngtsc
compilations. Currently, this supports skipping the TypeScript emit step
when the compiler can prove the contents of emit have not changed.

This is implemented for @Injectables as well as for files which don't
contain any Angular decorated types. These are the only files which can be
proven to be safe today.

See ngtsc/incremental/README.md for more details.

PR Close #29380

packages/compiler-cli/BUILD.bazel

@@ -28,6 +28,7 @@ ts_library(
         "//packages/compiler-cli/src/ngtsc/diagnostics",
         "//packages/compiler-cli/src/ngtsc/entry_point",
         "//packages/compiler-cli/src/ngtsc/imports",
+        "//packages/compiler-cli/src/ngtsc/incremental",
         "//packages/compiler-cli/src/ngtsc/partial_evaluator",
         "//packages/compiler-cli/src/ngtsc/path",
         "//packages/compiler-cli/src/ngtsc/perf",

packages/compiler-cli/src/ngtsc/annotations/src/injectable.ts

@@ -50,6 +50,8 @@ export class InjectableDecoratorHandler implements
 
   analyze(node: ClassDeclaration, decorator: Decorator): AnalysisOutput<InjectableHandlerData> {
     return {
+      // @Injectable()s cannot depend on other files for their compilation output.
+      allowSkipAnalysisAndEmit: true,
       analysis: {
         meta: extractInjectableMetadata(
             node, decorator, this.reflector, this.defaultImportRecorder, this.isCore,

packages/compiler-cli/src/ngtsc/incremental/BUILD.bazel

@@ -0,0 +1,13 @@
+load("//tools:defaults.bzl", "ts_library")
+
+package(default_visibility = ["//visibility:public"])
+
+ts_library(
+    name = "incremental",
+    srcs = ["index.ts"] + glob([
+        "src/**/*.ts",
+    ]),
+    deps = [
+        "@npm//typescript",
+    ],
+)

packages/compiler-cli/src/ngtsc/incremental/index.ts

@@ -0,0 +1,9 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+export {IncrementalState} from './src/state';

packages/compiler-cli/src/ngtsc/incremental/src/README.md

@@ -0,0 +1,43 @@
+# What is the `incremental` package?
+
+This package contains logic related to incremental compilation in ngtsc.
+
+In particular, it tracks metadata for `ts.SourceFile`s in between compilations, so the compiler can make intelligent decisions about when to skip certain operations and rely on previously analyzed data.
+
+# How does incremental compilation work?
+
+The initial compilation is no different from a standalone compilation; the compiler is unaware that incremental compilation will be utilized.
+
+When an `NgtscProgram` is created for a _subsequent_ compilation, it is initialized with the `NgtscProgram` from the previous compilation. It is therefore able to take advantage of any information present in the previous compilation to optimize the next one.
+
+This information is leveraged in two major ways:
+
+1) The previous `ts.Program` itself is used to create the next `ts.Program`, allowing TypeScript internally to leverage information from the previous compile in much the same way.
+
+2) An `IncrementalState` instance is constructed from the previous compilation's `IncrementalState` and its `ts.Program`.
+
+After this initialization, the `IncrementalState` contains the knowledge from the previous compilation which will be used to optimize the next one.
+
+# What optimizations can be made?
+
+Currently, ngtsc makes a decision to skip the emit of a file if it can prove that the contents of the file will not have changed. To prove this, two conditions must be true.
+
+* The input file itself must not have changed since the previous compilation.
+
+* As a result of analyzing the file, no dependencies must exist where the output of compilation could vary depending on the contents of any other file.
+
+The second condition is challenging, as Angular allows statically evaluated expressions in lots of contexts that could result in changes from file to file. For example, the `name` of an `@Pipe` could be a reference to a constant in a different file.
+
+Therefore, only two types of files meet these conditions and can be optimized today:
+
+* Files with no Angular decorated classes at all.
+
+* Files with only `@Injectable`s.
+
+# What optimizations are possible in the future?
+
+There is plenty of room for improvement here, with diminishing returns for the work involved.
+
+* The compiler could track the dependencies of each file being compiled, and know whether an `@Pipe` gets its name from a second file, for example. This is sufficient to skip the analysis and emit of more files when none of the dependencies have changed.
+
+* The compiler could also perform analysis on files which _have_ changed dependencies, and skip emit if the analysis indicates nothing has changed which would affect the file being emitted.

packages/compiler-cli/src/ngtsc/incremental/src/state.ts

@@ -0,0 +1,86 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import * as ts from 'typescript';
+
+/**
+ * Accumulates state between compilations.
+ */
+export class IncrementalState {
+  private constructor(
+      private unchangedFiles: Set<ts.SourceFile>,
+      private metadata: Map<ts.SourceFile, FileMetadata>) {}
+
+  static reconcile(previousState: IncrementalState, oldProgram: ts.Program, newProgram: ts.Program):
+      IncrementalState {
+    const unchangedFiles = new Set<ts.SourceFile>();
+    const metadata = new Map<ts.SourceFile, FileMetadata>();
+
+    // Compute the set of files that's unchanged.
+    const oldFiles = new Set<ts.SourceFile>();
+    for (const oldFile of oldProgram.getSourceFiles()) {
+      if (!oldFile.isDeclarationFile) {
+        oldFiles.add(oldFile);
+      }
+    }
+
+    // Look for files in the new program which haven't changed.
+    for (const newFile of newProgram.getSourceFiles()) {
+      if (oldFiles.has(newFile)) {
+        unchangedFiles.add(newFile);
+
+        // Copy over metadata for the unchanged file if available.
+        if (previousState.metadata.has(newFile)) {
+          metadata.set(newFile, previousState.metadata.get(newFile) !);
+        }
+      }
+    }
+
+    return new IncrementalState(unchangedFiles, metadata);
+  }
+
+  static fresh(): IncrementalState {
+    return new IncrementalState(new Set<ts.SourceFile>(), new Map<ts.SourceFile, FileMetadata>());
+  }
+
+  safeToSkipEmit(sf: ts.SourceFile): boolean {
+    if (!this.unchangedFiles.has(sf)) {
+      // The file has changed since the last run, and must be re-emitted.
+      return false;
+    }
+
+    // The file hasn't changed since the last emit. Whether or not it's safe to emit depends on
+    // what metadata was gathered about the file.
+
+    if (!this.metadata.has(sf)) {
+      // The file has no metadata from the previous or current compilations, so it must be emitted.
+      return false;
+    }
+
+    const meta = this.metadata.get(sf) !;
+
+    // Check if this file was explicitly marked as safe. This would only be done if every
+    // `DecoratorHandler` agreed that the file didn't depend on any other file's contents.
+    if (meta.safeToSkipEmitIfUnchanged) {
+      return true;
+    }
+
+    // The file wasn't explicitly marked as safe to skip emitting, so require an emit.
+    return false;
+  }
+
+  markFileAsSafeToSkipEmitIfUnchanged(sf: ts.SourceFile): void {
+    this.metadata.set(sf, {
+      safeToSkipEmitIfUnchanged: true,
+    });
+  }
+}
+
+interface FileMetadata {
+  safeToSkipEmitIfUnchanged: boolean;
+}

packages/compiler-cli/src/ngtsc/program.ts

@@ -18,6 +18,7 @@ import {CycleAnalyzer, ImportGraph} from './cycles';
 import {ErrorCode, ngErrorCode} from './diagnostics';
 import {FlatIndexGenerator, ReferenceGraph, checkForPrivateExports, findFlatIndexEntryPoint} from './entry_point';
 import {AbsoluteModuleStrategy, AliasGenerator, AliasStrategy, DefaultImportTracker, FileToModuleHost, FileToModuleStrategy, ImportRewriter, LocalIdentifierStrategy, LogicalProjectStrategy, ModuleResolver, NoopImportRewriter, R3SymbolsImportRewriter, Reference, ReferenceEmitter} from './imports';
+import {IncrementalState} from './incremental';
 import {PartialEvaluator} from './partial_evaluator';
 import {AbsoluteFsPath, LogicalFileSystem} from './path';
 import {NOOP_PERF_RECORDER, PerfRecorder, PerfTracker} from './perf';
@@ -60,6 +61,7 @@ export class NgtscProgram implements api.Program {
   private defaultImportTracker: DefaultImportTracker;
   private perfRecorder: PerfRecorder = NOOP_PERF_RECORDER;
   private perfTracker: PerfTracker|null = null;
+  private incrementalState: IncrementalState;
 
   constructor(
       rootNames: ReadonlyArray<string>, private options: api.CompilerOptions,
@@ -143,6 +145,13 @@ export class NgtscProgram implements api.Program {
     this.moduleResolver = new ModuleResolver(this.tsProgram, options, this.host);
     this.cycleAnalyzer = new CycleAnalyzer(new ImportGraph(this.moduleResolver));
     this.defaultImportTracker = new DefaultImportTracker();
+    if (oldProgram === undefined) {
+      this.incrementalState = IncrementalState.fresh();
+    } else {
+      const oldNgtscProgram = oldProgram as NgtscProgram;
+      this.incrementalState = IncrementalState.reconcile(
+          oldNgtscProgram.incrementalState, oldNgtscProgram.tsProgram, this.tsProgram);
+    }
   }
 
   getTsProgram(): ts.Program { return this.tsProgram; }
@@ -332,6 +341,10 @@ export class NgtscProgram implements api.Program {
         continue;
       }
 
+      if (this.incrementalState.safeToSkipEmit(targetSourceFile)) {
+        continue;
+      }
+
       const fileEmitSpan = this.perfRecorder.start('emitFile', targetSourceFile);
       emitResults.push(emitCallback({
         targetSourceFile,
@@ -440,8 +453,8 @@ export class NgtscProgram implements api.Program {
     ];
 
     return new IvyCompilation(
-        handlers, checker, this.reflector, this.importRewriter, this.perfRecorder,
-        this.sourceToFactorySymbols);
+        handlers, checker, this.reflector, this.importRewriter, this.incrementalState,
+        this.perfRecorder, this.sourceToFactorySymbols);
   }
 
   private get reflector(): TypeScriptReflectionHost {

packages/compiler-cli/src/ngtsc/transform/BUILD.bazel

@@ -11,6 +11,7 @@ ts_library(
         "//packages/compiler",
         "//packages/compiler-cli/src/ngtsc/diagnostics",
         "//packages/compiler-cli/src/ngtsc/imports",
+        "//packages/compiler-cli/src/ngtsc/incremental",
         "//packages/compiler-cli/src/ngtsc/perf",
         "//packages/compiler-cli/src/ngtsc/reflection",
         "//packages/compiler-cli/src/ngtsc/translator",

packages/compiler-cli/src/ngtsc/transform/src/api.ts

@@ -110,6 +110,7 @@ export interface AnalysisOutput<A> {
   diagnostics?: ts.Diagnostic[];
   factorySymbolName?: string;
   typeCheck?: boolean;
+  allowSkipAnalysisAndEmit?: boolean;
 }
 
 /**

packages/compiler-cli/src/ngtsc/transform/src/compilation.ts

@@ -11,6 +11,7 @@ import * as ts from 'typescript';
 
 import {ErrorCode, FatalDiagnosticError} from '../../diagnostics';
 import {ImportRewriter} from '../../imports';
+import {IncrementalState} from '../../incremental';
 import {PerfRecorder} from '../../perf';
 import {ClassDeclaration, ReflectionHost, isNamedClassDeclaration, reflectNameOfDeclaration} from '../../reflection';
 import {TypeCheckContext} from '../../typecheck';
@@ -76,7 +77,8 @@ export class IvyCompilation {
   constructor(
       private handlers: DecoratorHandler<any, any>[], private checker: ts.TypeChecker,
       private reflector: ReflectionHost, private importRewriter: ImportRewriter,
-      private perf: PerfRecorder, private sourceToFactorySymbols: Map<string, Set<string>>|null) {}
+      private incrementalState: IncrementalState, private perf: PerfRecorder,
+      private sourceToFactorySymbols: Map<string, Set<string>>|null) {}
 
 
   get exportStatements(): Map<string, Map<string, [string, string]>> { return this.reexportMap; }
@@ -170,6 +172,10 @@ export class IvyCompilation {
   private analyze(sf: ts.SourceFile, preanalyze: boolean): Promise<void>|undefined {
     const promises: Promise<void>[] = [];
 
+    // This flag begins as true for the file. If even one handler is matched and does not explicitly
+    // state that analysis/emit can be skipped, then the flag will be set to false.
+    let allowSkipAnalysisAndEmit = true;
+
     const analyzeClass = (node: ClassDeclaration): void => {
       const ivyClass = this.detectHandlersForClass(node);
 
@@ -197,6 +203,11 @@ export class IvyCompilation {
               this.sourceToFactorySymbols.get(sf.fileName) !.add(match.analyzed.factorySymbolName);
             }
 
+            // Update the allowSkipAnalysisAndEmit flag - it will only remain true if match.analyzed
+            // also explicitly specifies a value of true for the flag.
+            allowSkipAnalysisAndEmit =
+                allowSkipAnalysisAndEmit && (!!match.analyzed.allowSkipAnalysisAndEmit);
+
           } catch (err) {
             if (err instanceof FatalDiagnosticError) {
               this._diagnostics.push(err.toDiagnostic());
@@ -239,9 +250,19 @@ export class IvyCompilation {
 
     visit(sf);
 
+    const updateIncrementalState = () => {
+      if (allowSkipAnalysisAndEmit) {
+        this.incrementalState.markFileAsSafeToSkipEmitIfUnchanged(sf);
+      }
+    };
+
     if (preanalyze && promises.length > 0) {
-      return Promise.all(promises).then(() => undefined);
+      return Promise.all(promises).then(() => {
+        updateIncrementalState();
+        return undefined;
+      });
     } else {
+      updateIncrementalState();
       return undefined;
     }
   }

packages/compiler-cli/test/ngtsc/incremental_spec.ts

@@ -0,0 +1,47 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import {NgtscTestEnvironment} from './env';
+
+describe('ngtsc incremental compilation', () => {
+  let env !: NgtscTestEnvironment;
+  beforeEach(() => {
+    env = NgtscTestEnvironment.setup();
+    env.enableMultipleCompilations();
+    env.tsconfig();
+  });
+
+  it('should compile incrementally', () => {
+    env.write('service.ts', `
+      import {Injectable} from '@angular/core';
+
+      @Injectable()
+      export class Service {}
+    `);
+    env.write('test.ts', `
+      import {Component} from '@angular/core';
+      import {Service} from './service';
+
+      @Component({selector: 'cmp', template: 'cmp'})
+      export class Cmp {
+        constructor(service: Service) {}
+      }
+    `);
+    env.driveMain();
+    env.flushWrittenFileTracking();
+
+    // Pretend a change was made to test.ts.
+    env.invalidateCachedFile('test.ts');
+    env.driveMain();
+    const written = env.getFilesWrittenSinceLastFlush();
+
+    // The component should be recompiled, but not the service.
+    expect(written).toContain('/test.js');
+    expect(written).not.toContain('/service.js');
+  });
+});