feat(ivy): support templateUrl for ngtsc (#24704)

This commit adds support for templateUrl in component templates within
ngtsc. The compilation pipeline is split into sync and async versions,
where asynchronous compilation invokes a special preanalyze() phase of
analysis. The preanalyze() phase can optionally return a Promise which
will delay compilation until it resolves.

A ResourceLoader interface is used to resolve templateUrls to template
strings and can return results either synchronously or asynchronously.
During sync compilation it is an error if the ResourceLoader returns a
Promise.

Two ResourceLoader implementations are provided. One uses 'fs' to read
resources directly from disk and is chosen if the CompilerHost doesn't
provide a readResource method. The other wraps the readResource method
from CompilerHost if it's provided.

PR Close #24704

packages/compiler-cli/src/ngtsc/annotations/index.ts

@@ -6,6 +6,7 @@
  * found in the LICENSE file at https://angular.io/license
  */
 
+export {ResourceLoader} from './src/api';
 export {ComponentDecoratorHandler} from './src/component';
 export {DirectiveDecoratorHandler} from './src/directive';
 export {InjectableDecoratorHandler} from './src/injectable';

packages/compiler-cli/src/ngtsc/annotations/src/api.ts

@@ -0,0 +1,12 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+export interface ResourceLoader {
+  preload?(url: string): Promise<void>|undefined;
+  load(url: string): string;
+}

packages/compiler-cli/src/ngtsc/annotations/src/component.ts

@@ -7,12 +7,14 @@
  */
 
 import {ConstantPool, Expression, R3ComponentMetadata, R3DirectiveMetadata, WrappedNodeExpr, compileComponentFromMetadata, makeBindingParser, parseTemplate} from '@angular/compiler';
+import * as path from 'path';
 import * as ts from 'typescript';
 
 import {Decorator, ReflectionHost} from '../../host';
 import {reflectObjectLiteral, staticallyResolve} from '../../metadata';
 import {AnalysisOutput, CompileResult, DecoratorHandler} from '../../transform';
 
+import {ResourceLoader} from './api';
 import {extractDirectiveMetadata} from './directive';
 import {SelectorScopeRegistry} from './selector_scope';
 import {isAngularCore} from './util';
@@ -25,21 +27,35 @@ const EMPTY_MAP = new Map<string, Expression>();
 export class ComponentDecoratorHandler implements DecoratorHandler<R3ComponentMetadata> {
   constructor(
       private checker: ts.TypeChecker, private reflector: ReflectionHost,
-      private scopeRegistry: SelectorScopeRegistry, private isCore: boolean) {}
+      private scopeRegistry: SelectorScopeRegistry, private isCore: boolean,
+      private resourceLoader: ResourceLoader) {}
+
+  private literalCache = new Map<Decorator, ts.ObjectLiteralExpression>();
+
 
   detect(decorators: Decorator[]): Decorator|undefined {
     return decorators.find(
         decorator => decorator.name === 'Component' && (this.isCore || isAngularCore(decorator)));
   }
 
+  preanalyze(node: ts.ClassDeclaration, decorator: Decorator): Promise<void>|undefined {
+    const meta = this._resolveLiteral(decorator);
+    const component = reflectObjectLiteral(meta);
+
+    if (this.resourceLoader.preload !== undefined && component.has('templateUrl')) {
+      const templateUrl = staticallyResolve(component.get('templateUrl') !, this.checker);
+      if (typeof templateUrl !== 'string') {
+        throw new Error(`templateUrl should be a string`);
+      }
+      const url = path.posix.resolve(path.dirname(node.getSourceFile().fileName), templateUrl);
+      return this.resourceLoader.preload(url);
+    }
+    return undefined;
+  }
+
   analyze(node: ts.ClassDeclaration, decorator: Decorator): AnalysisOutput<R3ComponentMetadata> {
-    if (decorator.args === null || decorator.args.length !== 1) {
-      throw new Error(`Incorrect number of arguments to @Component decorator`);
-    }
-    const meta = decorator.args[0];
-    if (!ts.isObjectLiteralExpression(meta)) {
-      throw new Error(`Decorator argument must be literal.`);
-    }
+    const meta = this._resolveLiteral(decorator);
+    this.literalCache.delete(decorator);
 
     // @Component inherits @Directive, so begin by extracting the @Directive metadata and building
     // on it.
@@ -55,15 +71,24 @@ export class ComponentDecoratorHandler implements DecoratorHandler<R3ComponentMe
     // Next, read the `@Component`-specific fields.
     const component = reflectObjectLiteral(meta);
 
-    // Resolve and parse the template.
-    if (!component.has('template')) {
-      throw new Error(`For now, components must directly have a template.`);
+    let templateStr: string|null = null;
+    if (component.has('templateUrl')) {
+      const templateUrl = staticallyResolve(component.get('templateUrl') !, this.checker);
+      if (typeof templateUrl !== 'string') {
+        throw new Error(`templateUrl should be a string`);
       }
+      const url = path.posix.resolve(path.dirname(node.getSourceFile().fileName), templateUrl);
+      templateStr = this.resourceLoader.load(url);
+    } else if (component.has('template')) {
       const templateExpr = component.get('template') !;
-    const templateStr = staticallyResolve(templateExpr, this.checker);
-    if (typeof templateStr !== 'string') {
+      const resolvedTemplate = staticallyResolve(templateExpr, this.checker);
+      if (typeof resolvedTemplate !== 'string') {
         throw new Error(`Template must statically resolve to a string: ${node.name!.text}`);
       }
+      templateStr = resolvedTemplate;
+    } else {
+      throw new Error(`Component has no template or templateUrl`);
+    }
 
     let preserveWhitespaces: boolean = false;
     if (component.has('preserveWhitespaces')) {
@@ -123,4 +148,20 @@ export class ComponentDecoratorHandler implements DecoratorHandler<R3ComponentMe
       type: res.type,
     };
   }
+
+  private _resolveLiteral(decorator: Decorator): ts.ObjectLiteralExpression {
+    if (this.literalCache.has(decorator)) {
+      return this.literalCache.get(decorator) !;
+    }
+    if (decorator.args === null || decorator.args.length !== 1) {
+      throw new Error(`Incorrect number of arguments to @Component decorator`);
+    }
+    const meta = decorator.args[0];
+    if (!ts.isObjectLiteralExpression(meta)) {
+      throw new Error(`Decorator argument must be literal.`);
+    }
+
+    this.literalCache.set(decorator, meta);
+    return meta;
+  }
 }

packages/compiler-cli/src/ngtsc/program.ts

@@ -12,17 +12,28 @@ import * as ts from 'typescript';
 
 import * as api from '../transformers/api';
 
-import {ComponentDecoratorHandler, DirectiveDecoratorHandler, InjectableDecoratorHandler, NgModuleDecoratorHandler, PipeDecoratorHandler, SelectorScopeRegistry} from './annotations';
+import {ComponentDecoratorHandler, DirectiveDecoratorHandler, InjectableDecoratorHandler, NgModuleDecoratorHandler, PipeDecoratorHandler, ResourceLoader, SelectorScopeRegistry} from './annotations';
 import {CompilerHost} from './compiler_host';
 import {TypeScriptReflectionHost} from './metadata';
+import {FileResourceLoader, HostResourceLoader} from './resource_loader';
 import {IvyCompilation, ivyTransformFactory} from './transform';
 
 export class NgtscProgram implements api.Program {
   private tsProgram: ts.Program;
+  private resourceLoader: ResourceLoader;
+  private compilation: IvyCompilation|undefined = undefined;
+
+  private _coreImportsFrom: ts.SourceFile|null|undefined = undefined;
+  private _reflector: TypeScriptReflectionHost|undefined = undefined;
+  private _isCore: boolean|undefined = undefined;
 
   constructor(
       rootNames: ReadonlyArray<string>, private options: api.CompilerOptions,
       private host: api.CompilerHost, oldProgram?: api.Program) {
+    this.resourceLoader = host.readResource !== undefined ?
+        new HostResourceLoader(host.readResource.bind(host)) :
+        new FileResourceLoader();
+
     this.tsProgram =
         ts.createProgram(rootNames, options, host, oldProgram && oldProgram.getTsProgram());
   }
@@ -62,7 +73,16 @@ export class NgtscProgram implements api.Program {
     return [];
   }
 
-  loadNgStructureAsync(): Promise<void> { return Promise.resolve(); }
+  async loadNgStructureAsync(): Promise<void> {
+    if (this.compilation === undefined) {
+      this.compilation = this.makeCompilation();
+
+      await this.tsProgram.getSourceFiles()
+          .filter(file => !file.fileName.endsWith('.d.ts'))
+          .map(file => this.compilation !.analyzeAsync(file))
+          .filter((result): result is Promise<void> => result !== undefined);
+    }
+  }
 
   listLazyRoutes(entryRoute?: string|undefined): api.LazyRoute[] {
     throw new Error('Method not implemented.');
@@ -88,30 +108,13 @@ export class NgtscProgram implements api.Program {
     mergeEmitResultsCallback?: api.TsMergeEmitResultsCallback
   }): ts.EmitResult {
     const emitCallback = opts && opts.emitCallback || defaultEmitCallback;
-    const mergeEmitResultsCallback = opts && opts.mergeEmitResultsCallback || mergeEmitResults;
 
-    const checker = this.tsProgram.getTypeChecker();
-    const isCore = isAngularCorePackage(this.tsProgram);
-    const reflector = new TypeScriptReflectionHost(checker);
-    const scopeRegistry = new SelectorScopeRegistry(checker, reflector);
-
-    // Set up the IvyCompilation, which manages state for the Ivy transformer.
-    const handlers = [
-      new ComponentDecoratorHandler(checker, reflector, scopeRegistry, isCore),
-      new DirectiveDecoratorHandler(checker, reflector, scopeRegistry, isCore),
-      new InjectableDecoratorHandler(reflector, isCore),
-      new NgModuleDecoratorHandler(checker, reflector, scopeRegistry, isCore),
-      new PipeDecoratorHandler(reflector, isCore),
-    ];
-
-    const coreImportsFrom = isCore && getR3SymbolsFile(this.tsProgram) || null;
-
-    const compilation = new IvyCompilation(handlers, checker, reflector, coreImportsFrom);
-
-    // Analyze every source file in the program.
+    if (this.compilation === undefined) {
+      this.compilation = this.makeCompilation();
       this.tsProgram.getSourceFiles()
           .filter(file => !file.fileName.endsWith('.d.ts'))
-        .forEach(file => compilation.analyze(file));
+          .forEach(file => this.compilation !.analyzeSync(file));
+    }
 
     // Since there is no .d.ts transformation API, .d.ts files are transformed during write.
     const writeFile: ts.WriteFileCallback =
@@ -120,7 +123,8 @@ export class NgtscProgram implements api.Program {
          sourceFiles: ReadonlyArray<ts.SourceFile>) => {
           if (fileName.endsWith('.d.ts')) {
             data = sourceFiles.reduce(
-                (data, sf) => compilation.transformedDtsFor(sf.fileName, data, fileName), data);
+                (data, sf) => this.compilation !.transformedDtsFor(sf.fileName, data, fileName),
+                data);
           }
           this.host.writeFile(fileName, data, writeByteOrderMark, onError, sourceFiles);
         };
@@ -133,11 +137,49 @@ export class NgtscProgram implements api.Program {
       options: this.options,
       emitOnlyDtsFiles: false, writeFile,
       customTransformers: {
-        before: [ivyTransformFactory(compilation, reflector, coreImportsFrom)],
+        before: [ivyTransformFactory(this.compilation !, this.reflector, this.coreImportsFrom)],
       },
     });
     return emitResult;
   }
+
+  private makeCompilation(): IvyCompilation {
+    const checker = this.tsProgram.getTypeChecker();
+    const scopeRegistry = new SelectorScopeRegistry(checker, this.reflector);
+
+    // Set up the IvyCompilation, which manages state for the Ivy transformer.
+    const handlers = [
+      new ComponentDecoratorHandler(
+          checker, this.reflector, scopeRegistry, this.isCore, this.resourceLoader),
+      new DirectiveDecoratorHandler(checker, this.reflector, scopeRegistry, this.isCore),
+      new InjectableDecoratorHandler(this.reflector, this.isCore),
+      new NgModuleDecoratorHandler(checker, this.reflector, scopeRegistry, this.isCore),
+      new PipeDecoratorHandler(this.reflector, this.isCore),
+    ];
+
+    return new IvyCompilation(handlers, checker, this.reflector, this.coreImportsFrom);
+  }
+
+  private get reflector(): TypeScriptReflectionHost {
+    if (this._reflector === undefined) {
+      this._reflector = new TypeScriptReflectionHost(this.tsProgram.getTypeChecker());
+    }
+    return this._reflector;
+  }
+
+  private get coreImportsFrom(): ts.SourceFile|null {
+    if (this._coreImportsFrom === undefined) {
+      this._coreImportsFrom = this.isCore && getR3SymbolsFile(this.tsProgram) || null;
+    }
+    return this._coreImportsFrom;
+  }
+
+  private get isCore(): boolean {
+    if (this._isCore === undefined) {
+      this._isCore = isAngularCorePackage(this.tsProgram);
+    }
+    return this._isCore;
+  }
 }
 
 const defaultEmitCallback: api.TsEmitCallback =

packages/compiler-cli/src/ngtsc/resource_loader.ts

@@ -0,0 +1,59 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import * as fs from 'fs';
+
+import {ResourceLoader} from './annotations';
+
+/**
+ * `ResourceLoader` which delegates to a `CompilerHost` resource loading method.
+ */
+export class HostResourceLoader implements ResourceLoader {
+  private cache = new Map<string, string>();
+  private fetching = new Set<string>();
+
+  constructor(private host: (url: string) => string | Promise<string>) {}
+
+  preload(url: string): Promise<void>|undefined {
+    if (this.cache.has(url) || this.fetching.has(url)) {
+      return undefined;
+    }
+
+    const result = this.host(url);
+    if (typeof result === 'string') {
+      this.cache.set(url, result);
+      return undefined;
+    } else {
+      this.fetching.add(url);
+      return result.then(str => {
+        this.fetching.delete(url);
+        this.cache.set(url, str);
+      });
+    }
+  }
+
+  load(url: string): string {
+    if (this.cache.has(url)) {
+      return this.cache.get(url) !;
+    }
+
+    const result = this.host(url);
+    if (typeof result !== 'string') {
+      throw new Error(`HostResourceLoader: host(${url}) returned a Promise`);
+    }
+    this.cache.set(url, result);
+    return result;
+  }
+}
+
+/**
+ * `ResourceLoader` which directly uses the filesystem to resolve resources synchronously.
+ */
+export class FileResourceLoader implements ResourceLoader {
+  load(url: string): string { return fs.readFileSync(url, 'utf8'); }
+}

packages/compiler-cli/src/ngtsc/transform/src/api.ts

@@ -26,6 +26,15 @@ export interface DecoratorHandler<A> {
    */
   detect(decorator: Decorator[]): Decorator|undefined;
 
+
+  /**
+   * Asynchronously perform pre-analysis on the decorator/class combination.
+   *
+   * `preAnalyze` is optional and is not guaranteed to be called through all compilation flows. It
+   * will only be called if asynchronicity is supported in the CompilerHost.
+   */
+  preanalyze?(node: ts.Declaration, decorator: Decorator): Promise<void>|undefined;
+
   /**
    * Perform analysis on the decorator/class combination, producing instructions for compilation
    * if successful, or an array of diagnostic messages if the analysis fails or the decorator

packages/compiler-cli/src/ngtsc/transform/src/compilation.ts

@@ -6,7 +6,6 @@
  * found in the LICENSE file at https://angular.io/license
  */
 
-import {Expression, Type} from '@angular/compiler';
 import * as ts from 'typescript';
 
 import {Decorator, ReflectionHost} from '../../host';
@@ -14,9 +13,6 @@ import {reflectNameOfDeclaration} from '../../metadata/src/reflector';
 
 import {AnalysisOutput, CompileResult, DecoratorHandler} from './api';
 import {DtsFileTransformer} from './declaration';
-import {ImportManager, translateType} from './translator';
-
-
 
 /**
  * Record of an adapter which decided to emit a static field, and the analysis it performed to
@@ -45,6 +41,8 @@ export class IvyCompilation {
    * Tracks the `DtsFileTransformer`s for each TS file that needs .d.ts transformations.
    */
   private dtsMap = new Map<string, DtsFileTransformer>();
+  private _diagnostics: ts.Diagnostic[] = [];
+
 
   /**
    * @param handlers array of `DecoratorHandler`s which will be executed against each class in the
@@ -59,11 +57,18 @@ export class IvyCompilation {
       private handlers: DecoratorHandler<any>[], private checker: ts.TypeChecker,
       private reflector: ReflectionHost, private coreImportsFrom: ts.SourceFile|null) {}
 
+
+  analyzeSync(sf: ts.SourceFile): void { return this.analyze(sf, false); }
+
+  analyzeAsync(sf: ts.SourceFile): Promise<void>|undefined { return this.analyze(sf, true); }
+
   /**
    * Analyze a source file and produce diagnostics for it (if any).
    */
-  analyze(sf: ts.SourceFile): ts.Diagnostic[] {
-    const diagnostics: ts.Diagnostic[] = [];
+  private analyze(sf: ts.SourceFile, preanalyze: false): undefined;
+  private analyze(sf: ts.SourceFile, preanalyze: true): Promise<void>|undefined;
+  private analyze(sf: ts.SourceFile, preanalyze: boolean): Promise<void>|undefined {
+    const promises: Promise<void>[] = [];
 
     const analyzeClass = (node: ts.Declaration): void => {
       // The first step is to reflect the decorators.
@@ -79,6 +84,7 @@ export class IvyCompilation {
           return;
         }
 
+        const completeAnalysis = () => {
           // Check for multiple decorators on the same node. Technically speaking this
           // could be supported, but right now it's an error.
           if (this.analysis.has(node)) {
@@ -88,15 +94,29 @@ export class IvyCompilation {
           // Run analysis on the decorator. This will produce either diagnostics, an
           // analysis result, or both.
           const analysis = adapter.analyze(node, decorator);
-        if (analysis.diagnostics !== undefined) {
-          diagnostics.push(...analysis.diagnostics);
-        }
+
           if (analysis.analysis !== undefined) {
             this.analysis.set(node, {
               adapter,
               analysis: analysis.analysis, decorator,
             });
           }
+
+          if (analysis.diagnostics !== undefined) {
+            this._diagnostics.push(...analysis.diagnostics);
+          }
+        };
+
+        if (preanalyze && adapter.preanalyze !== undefined) {
+          const preanalysis = adapter.preanalyze(node, decorator);
+          if (preanalysis !== undefined) {
+            promises.push(preanalysis.then(() => completeAnalysis()));
+          } else {
+            completeAnalysis();
+          }
+        } else {
+          completeAnalysis();
+        }
       });
     };
 
@@ -109,7 +129,12 @@ export class IvyCompilation {
     };
 
     visit(sf);
-    return diagnostics;
+
+    if (preanalyze && promises.length > 0) {
+      return Promise.all(promises).then(() => undefined);
+    } else {
+      return undefined;
+    }
   }
 
   /**
@@ -166,6 +191,8 @@ export class IvyCompilation {
     return this.dtsMap.get(tsFileName) !.transform(dtsOriginalSource, tsFileName);
   }
 
+  get diagnostics(): ReadonlyArray<ts.Diagnostic> { return this._diagnostics; }
+
   private getDtsTransformer(tsFileName: string): DtsFileTransformer {
     if (!this.dtsMap.has(tsFileName)) {
       this.dtsMap.set(tsFileName, new DtsFileTransformer(this.coreImportsFrom));

packages/compiler-cli/test/ngtsc/ngtsc_spec.ts

@@ -147,6 +147,27 @@ describe('ngtsc behavioral tests', () => {
     expect(dtsContents).toContain('static ngComponentDef: i0.ComponentDef<TestCmp, \'test-cmp\'>');
   });
 
+  it('should compile Components without errors', () => {
+    writeConfig();
+    write('test.ts', `
+        import {Component} from '@angular/core';
+
+        @Component({
+          selector: 'test-cmp',
+          templateUrl: './dir/test.html',
+        })
+        export class TestCmp {}
+    `);
+    write('dir/test.html', '<p>Hello World</p>');
+
+    const exitCode = main(['-p', basePath], errorSpy);
+    expect(errorSpy).not.toHaveBeenCalled();
+    expect(exitCode).toBe(0);
+
+    const jsContents = getContents('test.js');
+    expect(jsContents).toContain('Hello World');
+  });
+
   it('should compile NgModules without errors', () => {
     writeConfig();
     write('test.ts', `