feat(ivy): track file dependencies due to partial evaluation (#30238)

As part of incremental compilation performance improvements, we need
to track the dependencies of files due to expressions being evaluated by
the `PartialEvaluator`.

The `PartialEvaluator` now accepts a `DependencyTracker` object, which is
used to track which files are visited when evaluating an expression.
The interpreter computes this `originatingFile` and stores it in the evaluation
`Context` so it can pass this to the `DependencyTracker.

The `IncrementalState` object implements this interface, which allows it to be
passed to the `PartialEvaluator` and so capture the file dependencies.

PR Close #30238

packages/compiler-cli/src/ngtsc/incremental/BUILD.bazel

@@ -8,6 +8,7 @@ ts_library(
         "src/**/*.ts",
     ]),
     deps = [
+        "//packages/compiler-cli/src/ngtsc/partial_evaluator",
         "@npm//typescript",
     ],
 )

packages/compiler-cli/src/ngtsc/incremental/src/state.ts

@@ -7,11 +7,12 @@
  */
 
 import * as ts from 'typescript';
+import {DependencyTracker} from '../../partial_evaluator';
 
 /**
  * Accumulates state between compilations.
  */
-export class IncrementalState {
+export class IncrementalState implements DependencyTracker {
   private constructor(
       private unchangedFiles: Set<ts.SourceFile>,
       private metadata: Map<ts.SourceFile, FileMetadata>) {}
@@ -75,12 +76,28 @@ export class IncrementalState {
   }
 
   markFileAsSafeToSkipEmitIfUnchanged(sf: ts.SourceFile): void {
-    this.metadata.set(sf, {
-      safeToSkipEmitIfUnchanged: true,
-    });
+    const metadata = this.ensureMetadata(sf);
+    metadata.safeToSkipEmitIfUnchanged = true;
+  }
+
+  trackFileDependency(dep: ts.SourceFile, src: ts.SourceFile) {
+    const metadata = this.ensureMetadata(src);
+    metadata.fileDependencies.add(dep);
+  }
+
+  private ensureMetadata(sf: ts.SourceFile): FileMetadata {
+    const metadata = this.metadata.get(sf) || new FileMetadata();
+    this.metadata.set(sf, metadata);
+    return metadata;
   }
 }
 
-interface FileMetadata {
-  safeToSkipEmitIfUnchanged: boolean;
+/**
+ * Information about the whether a source file can have analysis or emission can be skipped.
+ */
+class FileMetadata {
+  /** True if this file has no dependency changes that require it to be re-emitted. */
+  safeToSkipEmitIfUnchanged = false;
+  /** A set of source files that this file depends upon. */
+  fileDependencies = new Set<ts.SourceFile>();
 }

packages/compiler-cli/src/ngtsc/partial_evaluator/index.ts

@@ -7,5 +7,5 @@
  */
 
 export {DynamicValue} from './src/dynamic';
-export {ForeignFunctionResolver, PartialEvaluator} from './src/interface';
+export {DependencyTracker, ForeignFunctionResolver, PartialEvaluator} from './src/interface';
 export {BuiltinFn, EnumValue, ResolvedValue, ResolvedValueArray, ResolvedValueMap} from './src/result';

packages/compiler-cli/src/ngtsc/partial_evaluator/src/interface.ts

@@ -14,23 +14,27 @@ import {ReflectionHost} from '../../reflection';
 import {StaticInterpreter} from './interpreter';
 import {ResolvedValue} from './result';
 
+/**
+ * Implement this interface to record dependency relations between
+ * source files.
+ */
+export interface DependencyTracker {
+  trackFileDependency(dep: ts.SourceFile, src: ts.SourceFile): void;
+}
+
 export type ForeignFunctionResolver =
     (node: Reference<ts.FunctionDeclaration|ts.MethodDeclaration|ts.FunctionExpression>,
      args: ReadonlyArray<ts.Expression>) => ts.Expression | null;
 
-export type VisitedFilesCallback = (sf: ts.SourceFile) => void;
-
 export class PartialEvaluator {
-  constructor(private host: ReflectionHost, private checker: ts.TypeChecker) {}
+  constructor(
+      private host: ReflectionHost, private checker: ts.TypeChecker,
+      private dependencyTracker?: DependencyTracker) {}
 
-  evaluate(
-      expr: ts.Expression, foreignFunctionResolver?: ForeignFunctionResolver,
-      visitedFilesCb?: VisitedFilesCallback): ResolvedValue {
-    const interpreter = new StaticInterpreter(this.host, this.checker, visitedFilesCb);
-    if (visitedFilesCb) {
-      visitedFilesCb(expr.getSourceFile());
-    }
+  evaluate(expr: ts.Expression, foreignFunctionResolver?: ForeignFunctionResolver): ResolvedValue {
+    const interpreter = new StaticInterpreter(this.host, this.checker, this.dependencyTracker);
     return interpreter.visit(expr, {
+      originatingFile: expr.getSourceFile(),
       absoluteModuleName: null,
       resolutionContext: expr.getSourceFile().fileName,
       scope: new Map<ts.ParameterDeclaration, ResolvedValue>(), foreignFunctionResolver,

packages/compiler-cli/src/ngtsc/partial_evaluator/src/interpreter.ts

@@ -14,7 +14,7 @@ import {Declaration, ReflectionHost} from '../../reflection';
 
 import {ArrayConcatBuiltinFn, ArraySliceBuiltinFn} from './builtin';
 import {DynamicValue} from './dynamic';
-import {ForeignFunctionResolver, VisitedFilesCallback} from './interface';
+import {DependencyTracker, ForeignFunctionResolver} from './interface';
 import {BuiltinFn, EnumValue, ResolvedValue, ResolvedValueArray, ResolvedValueMap} from './result';
 
 
@@ -64,6 +64,7 @@ const UNARY_OPERATORS = new Map<ts.SyntaxKind, (a: any) => any>([
 ]);
 
 interface Context {
+  originatingFile: ts.SourceFile;
   /**
    * The module name (if any) which was used to reach the currently resolving symbols.
    */
@@ -81,7 +82,7 @@ interface Context {
 export class StaticInterpreter {
   constructor(
       private host: ReflectionHost, private checker: ts.TypeChecker,
-      private visitedFilesCb?: VisitedFilesCallback) {}
+      private dependencyTracker?: DependencyTracker) {}
 
   visit(node: ts.Expression, context: Context): ResolvedValue {
     return this.visitExpression(node, context);
@@ -226,8 +227,8 @@ export class StaticInterpreter {
   }
 
   private visitDeclaration(node: ts.Declaration, context: Context): ResolvedValue {
-    if (this.visitedFilesCb) {
-      this.visitedFilesCb(node.getSourceFile());
+    if (this.dependencyTracker) {
+      this.dependencyTracker.trackFileDependency(node.getSourceFile(), context.originatingFile);
     }
     if (this.host.isClass(node)) {
       return this.getReference(node, context);

packages/compiler-cli/src/ngtsc/partial_evaluator/test/BUILD.bazel

@@ -15,6 +15,7 @@ ts_library(
         "//packages/compiler-cli/src/ngtsc/partial_evaluator",
         "//packages/compiler-cli/src/ngtsc/reflection",
         "//packages/compiler-cli/src/ngtsc/testing",
+        "//packages/compiler-cli/src/ngtsc/util",
         "@npm//typescript",
     ],
 )

packages/compiler-cli/src/ngtsc/partial_evaluator/test/evaluator_spec.ts

@@ -9,45 +9,11 @@
 import * as ts from 'typescript';
 
 import {Reference} from '../../imports';
-import {TypeScriptReflectionHost} from '../../reflection';
 import {getDeclaration, makeProgram} from '../../testing/in_memory_typescript';
 import {DynamicValue} from '../src/dynamic';
-import {ForeignFunctionResolver, PartialEvaluator} from '../src/interface';
-import {EnumValue, ResolvedValue} from '../src/result';
+import {EnumValue} from '../src/result';
 
-function makeExpression(
-    code: string, expr: string, supportingFiles: {name: string, contents: string}[] = []): {
-  expression: ts.Expression,
-  host: ts.CompilerHost,
-  checker: ts.TypeChecker,
-  program: ts.Program,
-  options: ts.CompilerOptions
-} {
-  const {program, options, host} = makeProgram(
-      [{name: 'entry.ts', contents: `${code}; const target$ = ${expr};`}, ...supportingFiles]);
-  const checker = program.getTypeChecker();
-  const decl = getDeclaration(program, 'entry.ts', 'target$', ts.isVariableDeclaration);
-  return {
-    expression: decl.initializer !,
-    host,
-    options,
-    checker,
-    program,
-  };
-}
-
-function makeEvaluator(checker: ts.TypeChecker): PartialEvaluator {
-  const reflectionHost = new TypeScriptReflectionHost(checker);
-  return new PartialEvaluator(reflectionHost, checker);
-}
-
-function evaluate<T extends ResolvedValue>(
-    code: string, expr: string, supportingFiles: {name: string, contents: string}[] = [],
-    foreignFunctionResolver?: ForeignFunctionResolver): T {
-  const {expression, checker} = makeExpression(code, expr, supportingFiles);
-  const evaluator = makeEvaluator(checker);
-  return evaluator.evaluate(expression, foreignFunctionResolver) as T;
-}
+import {evaluate, firstArgFfr, makeEvaluator, makeExpression, owningModuleOf} from './utils';
 
 describe('ngtsc metadata', () => {
   it('reads a file correctly', () => {
@@ -157,7 +123,7 @@ describe('ngtsc metadata', () => {
   });
 
   it('imports work', () => {
-    const {program, options, host} = makeProgram([
+    const {program} = makeProgram([
       {name: 'second.ts', contents: 'export function foo(bar) { return bar; }'},
       {
         name: 'entry.ts',
@@ -168,10 +134,9 @@ describe('ngtsc metadata', () => {
       },
     ]);
     const checker = program.getTypeChecker();
-    const reflectionHost = new TypeScriptReflectionHost(checker);
     const result = getDeclaration(program, 'entry.ts', 'target$', ts.isVariableDeclaration);
     const expr = result.initializer !;
-    const evaluator = new PartialEvaluator(reflectionHost, checker);
+    const evaluator = makeEvaluator(checker);
     const resolved = evaluator.evaluate(expr);
     if (!(resolved instanceof Reference)) {
       return fail('Expected expression to resolve to a reference');
@@ -185,7 +150,7 @@ describe('ngtsc metadata', () => {
   });
 
   it('absolute imports work', () => {
-    const {program, options, host} = makeProgram([
+    const {program} = makeProgram([
       {name: 'node_modules/some_library/index.d.ts', contents: 'export declare function foo(bar);'},
       {
         name: 'entry.ts',
@@ -196,10 +161,9 @@ describe('ngtsc metadata', () => {
       },
     ]);
     const checker = program.getTypeChecker();
-    const reflectionHost = new TypeScriptReflectionHost(checker);
     const result = getDeclaration(program, 'entry.ts', 'target$', ts.isVariableDeclaration);
     const expr = result.initializer !;
-    const evaluator = new PartialEvaluator(reflectionHost, checker);
+    const evaluator = makeEvaluator(checker);
     const resolved = evaluator.evaluate(expr);
     if (!(resolved instanceof Reference)) {
       return fail('Expected expression to resolve to an absolute reference');
@@ -296,11 +260,10 @@ describe('ngtsc metadata', () => {
       {name: 'entry.ts', contents: `const prop = 42; const target$ = {prop};`},
     ]);
     const checker = program.getTypeChecker();
-    const reflectionHost = new TypeScriptReflectionHost(checker);
     const result = getDeclaration(program, 'entry.ts', 'target$', ts.isVariableDeclaration);
     const expr = result.initializer !as ts.ObjectLiteralExpression;
     const prop = expr.properties[0] as ts.ShorthandPropertyAssignment;
-    const evaluator = new PartialEvaluator(reflectionHost, checker);
+    const evaluator = makeEvaluator(checker);
     const resolved = evaluator.evaluate(prop.name);
     expect(resolved).toBe(42);
   });
@@ -314,10 +277,9 @@ describe('ngtsc metadata', () => {
       },
     ]);
     const checker = program.getTypeChecker();
-    const reflectionHost = new TypeScriptReflectionHost(checker);
     const result = getDeclaration(program, 'entry.ts', 'target$', ts.isVariableDeclaration);
     const expr = result.initializer !as ts.ObjectLiteralExpression;
-    const evaluator = new PartialEvaluator(reflectionHost, checker);
+    const evaluator = makeEvaluator(checker);
     const resolved = evaluator.evaluate(expr);
     if (!(resolved instanceof Map)) {
       return fail('Should have resolved to a Map');
@@ -369,34 +331,34 @@ describe('ngtsc metadata', () => {
 
   describe('(visited file tracking)', () => {
     it('should track each time a source file is visited', () => {
-      const visitedFilesSpy = jasmine.createSpy('visitedFilesCb');
+      const trackFileDependency = jasmine.createSpy('DependencyTracker');
       const {expression, checker} =
           makeExpression(`class A { static foo = 42; } function bar() { return A.foo; }`, 'bar()');
-      const evaluator = makeEvaluator(checker);
-      evaluator.evaluate(expression, undefined, visitedFilesSpy);
-      expect(visitedFilesSpy)
-          .toHaveBeenCalledTimes(3);  // The initial expression, followed by two declaration visited
-      expect(visitedFilesSpy.calls.allArgs().map(args => args[0].fileName)).toEqual([
-        '/entry.ts', '/entry.ts', '/entry.ts'
-      ]);
+      const evaluator = makeEvaluator(checker, {trackFileDependency});
+      evaluator.evaluate(expression);
+      expect(trackFileDependency).toHaveBeenCalledTimes(2);  // two declaration visited
+      expect(trackFileDependency.calls.allArgs().map(args => [args[0].fileName, args[1].fileName]))
+          .toEqual([['/entry.ts', '/entry.ts'], ['/entry.ts', '/entry.ts']]);
     });
 
     it('should track imported source files', () => {
-      const visitedFilesSpy = jasmine.createSpy('visitedFilesCb');
+      const trackFileDependency = jasmine.createSpy('DependencyTracker');
       const {expression, checker} = makeExpression(`import {Y} from './other'; const A = Y;`, 'A', [
         {name: 'other.ts', contents: `export const Y = 'test';`},
         {name: 'not-visited.ts', contents: `export const Z = 'nope';`}
       ]);
-      const evaluator = makeEvaluator(checker);
-      evaluator.evaluate(expression, undefined, visitedFilesSpy);
-      expect(visitedFilesSpy).toHaveBeenCalledTimes(3);
-      expect(visitedFilesSpy.calls.allArgs().map(args => args[0].fileName)).toEqual([
-        '/entry.ts', '/entry.ts', '/other.ts'
-      ]);
+      const evaluator = makeEvaluator(checker, {trackFileDependency});
+      evaluator.evaluate(expression);
+      expect(trackFileDependency).toHaveBeenCalledTimes(2);
+      expect(trackFileDependency.calls.allArgs().map(args => [args[0].fileName, args[1].fileName]))
+          .toEqual([
+            ['/entry.ts', '/entry.ts'],
+            ['/other.ts', '/entry.ts'],
+          ]);
     });
 
     it('should track files passed through during re-exports', () => {
-      const visitedFilesSpy = jasmine.createSpy('visitedFilesCb');
+      const trackFileDependency = jasmine.createSpy('DependencyTracker');
       const {expression, checker} =
           makeExpression(`import * as mod from './direct-reexport';`, 'mod.value.property', [
             {name: 'const.ts', contents: 'export const value = {property: "test"};'},
@@ -404,26 +366,16 @@ describe('ngtsc metadata', () => {
             {name: 'indirect-reexport.ts', contents: `import value from './def'; export {value};`},
             {name: 'direct-reexport.ts', contents: `export {value} from './indirect-reexport';`},
           ]);
-      const evaluator = makeEvaluator(checker);
-      evaluator.evaluate(expression, undefined, visitedFilesSpy);
-      expect(visitedFilesSpy).toHaveBeenCalledTimes(3);
-      expect(visitedFilesSpy.calls.allArgs().map(args => args[0].fileName)).toEqual([
-        '/entry.ts',
-        '/direct-reexport.ts',
-        // Not '/indirect-reexport.ts' or '/def.ts'.
-        // TS skips through them when finding the original symbol for `value`
-        '/const.ts',
-      ]);
+      const evaluator = makeEvaluator(checker, {trackFileDependency});
+      evaluator.evaluate(expression);
+      expect(trackFileDependency).toHaveBeenCalledTimes(2);
+      expect(trackFileDependency.calls.allArgs().map(args => [args[0].fileName, args[1].fileName]))
+          .toEqual([
+            ['/direct-reexport.ts', '/entry.ts'],
+            // Not '/indirect-reexport.ts' or '/def.ts'.
+            // TS skips through them when finding the original symbol for `value`
+            ['/const.ts', '/entry.ts'],
+          ]);
     });
   });
 });
-
-function owningModuleOf(ref: Reference): string|null {
-  return ref.bestGuessOwningModule !== null ? ref.bestGuessOwningModule.specifier : null;
-}
-
-function firstArgFfr(
-    node: Reference<ts.FunctionDeclaration|ts.MethodDeclaration|ts.FunctionExpression>,
-    args: ReadonlyArray<ts.Expression>): ts.Expression {
-  return args[0];
-}

packages/compiler-cli/src/ngtsc/partial_evaluator/test/utils.ts

@@ -0,0 +1,60 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import * as ts from 'typescript';
+
+import {Reference} from '../../imports';
+import {TypeScriptReflectionHost} from '../../reflection';
+import {getDeclaration, makeProgram} from '../../testing/in_memory_typescript';
+import {DependencyTracker, ForeignFunctionResolver, PartialEvaluator} from '../src/interface';
+import {ResolvedValue} from '../src/result';
+
+export function makeExpression(
+    code: string, expr: string, supportingFiles: {name: string, contents: string}[] = []): {
+  expression: ts.Expression,
+  host: ts.CompilerHost,
+  checker: ts.TypeChecker,
+  program: ts.Program,
+  options: ts.CompilerOptions
+} {
+  const {program, options, host} = makeProgram(
+      [{name: 'entry.ts', contents: `${code}; const target$ = ${expr};`}, ...supportingFiles]);
+  const checker = program.getTypeChecker();
+  const decl = getDeclaration(program, 'entry.ts', 'target$', ts.isVariableDeclaration);
+  return {
+    expression: decl.initializer !,
+    host,
+    options,
+    checker,
+    program,
+  };
+}
+
+export function makeEvaluator(
+    checker: ts.TypeChecker, tracker?: DependencyTracker): PartialEvaluator {
+  const reflectionHost = new TypeScriptReflectionHost(checker);
+  return new PartialEvaluator(reflectionHost, checker, tracker);
+}
+
+export function evaluate<T extends ResolvedValue>(
+    code: string, expr: string, supportingFiles: {name: string, contents: string}[] = [],
+    foreignFunctionResolver?: ForeignFunctionResolver): T {
+  const {expression, checker} = makeExpression(code, expr, supportingFiles);
+  const evaluator = makeEvaluator(checker);
+  return evaluator.evaluate(expression, foreignFunctionResolver) as T;
+}
+
+export function owningModuleOf(ref: Reference): string|null {
+  return ref.bestGuessOwningModule !== null ? ref.bestGuessOwningModule.specifier : null;
+}
+
+export function firstArgFfr(
+    node: Reference<ts.FunctionDeclaration|ts.MethodDeclaration|ts.FunctionExpression>,
+    args: ReadonlyArray<ts.Expression>): ts.Expression {
+  return args[0];
+}

packages/compiler-cli/src/ngtsc/program.ts

@@ -457,7 +457,7 @@ export class NgtscProgram implements api.Program {
       aliasGenerator = new AliasGenerator(this.fileToModuleHost);
     }
 
-    const evaluator = new PartialEvaluator(this.reflector, checker);
+    const evaluator = new PartialEvaluator(this.reflector, checker, this.incrementalState);
     const dtsReader = new DtsMetadataReader(checker, this.reflector);
     const localMetaRegistry = new LocalMetadataRegistry();
     const depScopeReader = new MetadataDtsModuleScopeResolver(dtsReader, aliasGenerator);