feat(ivy): add source mappings to compiled Angular templates (#28055)

During analysis, the `ComponentDecoratorHandler` passes the component
template to the `parseTemplate()` function. Previously, there was little or
no information about the original source file, where the template is found,
passed when calling this function.

Now, we correctly compute the URL of the source of the template, both
for external `templateUrl` and in-line `template` cases. Further in the
in-line template case we compute the character range of the template
in its containing source file; *but only in the case that the template is
a simple string literal*. If the template is actually a dynamic value like
an interpolated string or a function call, then we do not try to add the
originating source file information.

The translator that converts Ivy AST nodes to TypeScript now adds these
template specific source mappings, which account for the file where
the template was found, to the templates to support stepping through the
template creation and update code when debugging an Angular application.

Note that some versions of TypeScript have a bug which means they cannot
support external template source-maps. We check for this via the
`canSourceMapExternalTemplates()` helper function and avoid trying to
add template mappings to external templates if not supported.

PR Close #28055

packages/compiler-cli/src/ngtsc/annotations/BUILD.bazel

@@ -18,6 +18,7 @@ ts_library(
         "//packages/compiler-cli/src/ngtsc/routing",
         "//packages/compiler-cli/src/ngtsc/transform",
         "//packages/compiler-cli/src/ngtsc/typecheck",
+        "//packages/compiler-cli/src/ngtsc/util",
         "@ngdeps//@types/node",
         "@ngdeps//typescript",
     ],

packages/compiler-cli/src/ngtsc/annotations/src/component.ts

@@ -6,7 +6,7 @@
  * found in the LICENSE file at https://angular.io/license
  */
 
-import {ConstantPool, CssSelector, DEFAULT_INTERPOLATION_CONFIG, DomElementSchemaRegistry, ElementSchemaRegistry, Expression, ExternalExpr, InterpolationConfig, R3ComponentMetadata, R3DirectiveMetadata, SelectorMatcher, Statement, TmplAstNode, WrappedNodeExpr, compileComponentFromMetadata, makeBindingParser, parseTemplate} from '@angular/compiler';
+import {ConstantPool, CssSelector, DEFAULT_INTERPOLATION_CONFIG, DomElementSchemaRegistry, Expression, ExternalExpr, InterpolationConfig, LexerRange, R3ComponentMetadata, SelectorMatcher, Statement, TmplAstNode, WrappedNodeExpr, compileComponentFromMetadata, makeBindingParser, parseTemplate} from '@angular/compiler';
 import * as path from 'path';
 import * as ts from 'typescript';
 
@@ -16,7 +16,8 @@ import {ModuleResolver, Reference, ResolvedReference} from '../../imports';
 import {EnumValue, PartialEvaluator} from '../../partial_evaluator';
 import {Decorator, ReflectionHost, filterToMembersWithDecorator, reflectObjectLiteral} from '../../reflection';
 import {AnalysisOutput, CompileResult, DecoratorHandler} from '../../transform';
-import {TypeCheckContext, TypeCheckableDirectiveMeta} from '../../typecheck';
+import {TypeCheckContext} from '../../typecheck';
+import {tsSourceMapBug29300Fixed} from '../../util/src/ts_source_map_bug_29300';
 
 import {ResourceLoader} from './api';
 import {extractDirectiveMetadata, extractQueriesFromDecorator, parseFieldArrayValue, queriesFromFields} from './directive';
@@ -119,24 +120,56 @@ export class ComponentDecoratorHandler implements
     // Next, read the `@Component`-specific fields.
     const {decoratedElements, decorator: component, metadata} = directiveResult;
 
+    // Go through the root directories for this project, and select the one with the smallest
+    // relative path representation.
+    const filePath = node.getSourceFile().fileName;
+    const relativeContextFilePath = this.rootDirs.reduce<string|undefined>((previous, rootDir) => {
+      const candidate = path.posix.relative(rootDir, filePath);
+      if (previous === undefined || candidate.length < previous.length) {
+        return candidate;
+      } else {
+        return previous;
+      }
+    }, undefined) !;
+
     let templateStr: string|null = null;
+    let templateUrl: string = '';
+    let templateRange: LexerRange|undefined;
+    let escapedString: boolean = false;
+
     if (component.has('templateUrl')) {
       const templateUrlExpr = component.get('templateUrl') !;
-      const templateUrl = this.evaluator.evaluate(templateUrlExpr);
+      const evalTemplateUrl = this.evaluator.evaluate(templateUrlExpr);
-      if (typeof templateUrl !== 'string') {
+      if (typeof evalTemplateUrl !== 'string') {
         throw new FatalDiagnosticError(
             ErrorCode.VALUE_HAS_WRONG_TYPE, templateUrlExpr, 'templateUrl must be a string');
       }
-      const resolvedTemplateUrl = this.resourceLoader.resolve(templateUrl, containingFile);
+      templateUrl = this.resourceLoader.resolve(evalTemplateUrl, containingFile);
-      templateStr = this.resourceLoader.load(resolvedTemplateUrl);
+      templateStr = this.resourceLoader.load(templateUrl);
+      if (!tsSourceMapBug29300Fixed()) {
+        // By removing the template URL we are telling the translator not to try to
+        // map the external source file to the generated code, since the version
+        // of TS that is running does not support it.
+        templateUrl = '';
+      }
     } else if (component.has('template')) {
       const templateExpr = component.get('template') !;
-      const resolvedTemplate = this.evaluator.evaluate(templateExpr);
+      // We only support SourceMaps for inline templates that are simple string literals.
-      if (typeof resolvedTemplate !== 'string') {
+      if (ts.isStringLiteral(templateExpr) || ts.isNoSubstitutionTemplateLiteral(templateExpr)) {
-        throw new FatalDiagnosticError(
+        // the start and end of the `templateExpr` node includes the quotation marks, which we must
-            ErrorCode.VALUE_HAS_WRONG_TYPE, templateExpr, 'template must be a string');
+        // strip
+        templateRange = getTemplateRange(templateExpr);
+        templateStr = templateExpr.getSourceFile().text;
+        templateUrl = relativeContextFilePath;
+        escapedString = true;
+      } else {
+        const resolvedTemplate = this.evaluator.evaluate(templateExpr);
+        if (typeof resolvedTemplate !== 'string') {
+          throw new FatalDiagnosticError(
+              ErrorCode.VALUE_HAS_WRONG_TYPE, templateExpr, 'template must be a string');
+        }
+        templateStr = resolvedTemplate;
       }
-      templateStr = resolvedTemplate;
     } else {
       throw new FatalDiagnosticError(
           ErrorCode.COMPONENT_MISSING_TEMPLATE, decorator.node, 'component is missing a template');
@@ -157,18 +190,6 @@ export class ComponentDecoratorHandler implements
         new WrappedNodeExpr(component.get('viewProviders') !) :
         null;
 
-    // Go through the root directories for this project, and select the one with the smallest
-    // relative path representation.
-    const filePath = node.getSourceFile().fileName;
-    const relativeContextFilePath = this.rootDirs.reduce<string|undefined>((previous, rootDir) => {
-      const candidate = path.posix.relative(rootDir, filePath);
-      if (previous === undefined || candidate.length < previous.length) {
-        return candidate;
-      } else {
-        return previous;
-      }
-    }, undefined) !;
-
     let interpolation: InterpolationConfig = DEFAULT_INTERPOLATION_CONFIG;
     if (component.has('interpolation')) {
       const expr = component.get('interpolation') !;
@@ -182,9 +203,11 @@ export class ComponentDecoratorHandler implements
       interpolation = InterpolationConfig.fromArray(value as[string, string]);
     }
 
-    const template = parseTemplate(
+    const template = parseTemplate(templateStr, templateUrl, {
-        templateStr, `${node.getSourceFile().fileName}#${node.name!.text}/template.html`,
+      preserveWhitespaces,
-        {preserveWhitespaces, interpolationConfig: interpolation});
+      interpolationConfig: interpolation,
+      range: templateRange, escapedString
+    });
     if (template.errors !== undefined) {
       throw new Error(
           `Errors parsing template: ${template.errors.map(e => e.toString()).join(', ')}`);
@@ -402,3 +425,15 @@ export class ComponentDecoratorHandler implements
     return this.cycleAnalyzer.wouldCreateCycle(origin, imported);
   }
 }
+
+function getTemplateRange(templateExpr: ts.Expression) {
+  const startPos = templateExpr.getStart() + 1;
+  const {line, character} =
+      ts.getLineAndCharacterOfPosition(templateExpr.getSourceFile(), startPos);
+  return {
+    startPos,
+    startLine: line,
+    startCol: character,
+    endPos: templateExpr.getEnd() - 1,
+  };
+}

packages/compiler-cli/src/ngtsc/translator/src/translator.ts

@@ -91,6 +91,7 @@ export function translateType(type: Type, imports: ImportManager): ts.TypeNode {
 }
 
 class ExpressionTranslatorVisitor implements ExpressionVisitor, StatementVisitor {
+  private externalSourceFiles = new Map<string, ts.SourceMapSource>();
   constructor(private imports: ImportManager) {}
 
   visitDeclareVarStmt(stmt: DeclareVarStmt, context: Context): ts.VariableStatement {
@@ -153,7 +154,9 @@ class ExpressionTranslatorVisitor implements ExpressionVisitor, StatementVisitor
   }
 
   visitReadVarExpr(ast: ReadVarExpr, context: Context): ts.Identifier {
-    return ts.createIdentifier(ast.name !);
+    const identifier = ts.createIdentifier(ast.name !);
+    this.setSourceMapRange(identifier, ast);
+    return identifier;
   }
 
   visitWriteVarExpr(expr: WriteVarExpr, context: Context): ts.Expression {
@@ -175,9 +178,11 @@ class ExpressionTranslatorVisitor implements ExpressionVisitor, StatementVisitor
 
   visitInvokeMethodExpr(ast: InvokeMethodExpr, context: Context): ts.CallExpression {
     const target = ast.receiver.visitExpression(this, context);
-    return ts.createCall(
+    const call = ts.createCall(
         ast.name !== null ? ts.createPropertyAccess(target, ast.name) : target, undefined,
         ast.args.map(arg => arg.visitExpression(this, context)));
+    this.setSourceMapRange(call, ast);
+    return call;
   }
 
   visitInvokeFunctionExpr(ast: InvokeFunctionExpr, context: Context): ts.CallExpression {
@@ -187,6 +192,7 @@ class ExpressionTranslatorVisitor implements ExpressionVisitor, StatementVisitor
     if (ast.pure) {
       ts.addSyntheticLeadingComment(expr, ts.SyntaxKind.MultiLineCommentTrivia, '@__PURE__', false);
     }
+    this.setSourceMapRange(expr, ast);
     return expr;
   }
 
@@ -197,13 +203,16 @@ class ExpressionTranslatorVisitor implements ExpressionVisitor, StatementVisitor
   }
 
   visitLiteralExpr(ast: LiteralExpr, context: Context): ts.Expression {
+    let expr: ts.Expression;
     if (ast.value === undefined) {
-      return ts.createIdentifier('undefined');
+      expr = ts.createIdentifier('undefined');
     } else if (ast.value === null) {
-      return ts.createNull();
+      expr = ts.createNull();
     } else {
-      return ts.createLiteral(ast.value);
+      expr = ts.createLiteral(ast.value);
     }
+    this.setSourceMapRange(expr, ast);
+    return expr;
   }
 
   visitExternalExpr(ast: ExternalExpr, context: Context): ts.PropertyAccessExpression
@@ -269,7 +278,10 @@ class ExpressionTranslatorVisitor implements ExpressionVisitor, StatementVisitor
   }
 
   visitLiteralArrayExpr(ast: LiteralArrayExpr, context: Context): ts.ArrayLiteralExpression {
-    return ts.createArrayLiteral(ast.entries.map(expr => expr.visitExpression(this, context)));
+    const expr =
+        ts.createArrayLiteral(ast.entries.map(expr => expr.visitExpression(this, context)));
+    this.setSourceMapRange(expr, ast);
+    return expr;
   }
 
   visitLiteralMapExpr(ast: LiteralMapExpr, context: Context): ts.ObjectLiteralExpression {
@@ -277,7 +289,9 @@ class ExpressionTranslatorVisitor implements ExpressionVisitor, StatementVisitor
         entry => ts.createPropertyAssignment(
             entry.quoted ? ts.createLiteral(entry.key) : ts.createIdentifier(entry.key),
             entry.value.visitExpression(this, context)));
-    return ts.createObjectLiteral(entries);
+    const expr = ts.createObjectLiteral(entries);
+    this.setSourceMapRange(expr, ast);
+    return expr;
   }
 
   visitCommaExpr(ast: CommaExpr, context: Context): never {
@@ -289,6 +303,20 @@ class ExpressionTranslatorVisitor implements ExpressionVisitor, StatementVisitor
   visitTypeofExpr(ast: TypeofExpr, context: Context): ts.TypeOfExpression {
     return ts.createTypeOf(ast.expr.visitExpression(this, context));
   }
+
+  private setSourceMapRange(expr: ts.Expression, ast: Expression) {
+    if (ast.sourceSpan) {
+      const {start, end} = ast.sourceSpan;
+      const {url, content} = start.file;
+      if (url) {
+        if (!this.externalSourceFiles.has(url)) {
+          this.externalSourceFiles.set(url, ts.createSourceMapSource(url, content, pos => pos));
+        }
+        const source = this.externalSourceFiles.get(url);
+        ts.setSourceMapRange(expr, {pos: start.offset, end: end.offset, source});
+      }
+    }
+  }
 }
 
 export class TypeTranslatorVisitor implements ExpressionVisitor, TypeVisitor {

packages/compiler-cli/src/ngtsc/util/src/ts_source_map_bug_29300.ts

@@ -0,0 +1,68 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+import * as ts from 'typescript';
+
+let _tsSourceMapBug29300Fixed: boolean|undefined;
+
+/**
+ * Test the current version of TypeScript to see if it has fixed the external SourceMap
+ * file bug: https://github.com/Microsoft/TypeScript/issues/29300.
+ *
+ * The bug is fixed in TS 3.3+ but this check avoid us having to rely upon the version number,
+ * and allows us to gracefully fail if the TS version still has the bug.
+ *
+ * We check for the bug by compiling a very small program `a;` and transforming it to `b;`,
+ * where we map the new `b` identifier to an external source file, which has different lines to
+ * the original source file.  If the bug is fixed then the output SourceMap should contain
+ * mappings that correspond ot the correct line/col pairs for this transformed node.
+ *
+ * @returns true if the bug is fixed.
+ */
+export function tsSourceMapBug29300Fixed() {
+  if (_tsSourceMapBug29300Fixed === undefined) {
+    let writtenFiles: {[filename: string]: string} = {};
+    const sourceFile =
+        ts.createSourceFile('test.ts', 'a;', ts.ScriptTarget.ES2015, true, ts.ScriptKind.TS);
+    const host = {
+      getSourceFile(): ts.SourceFile | undefined{return sourceFile;},
+      fileExists(): boolean{return true;},
+      readFile(): string | undefined{return '';},
+      writeFile(fileName: string, data: string) { writtenFiles[fileName] = data; },
+      getDefaultLibFileName(): string{return '';},
+      getCurrentDirectory(): string{return '';},
+      getDirectories(): string[]{return [];},
+      getCanonicalFileName(): string{return '';},
+      useCaseSensitiveFileNames(): boolean{return true;},
+      getNewLine(): string{return '\n';},
+    };
+
+    const transform = (context: ts.TransformationContext) => {
+      return (node: ts.SourceFile) => ts.visitNode(node, visitor);
+      function visitor(node: ts.Node): ts.Node {
+        if (ts.isIdentifier(node) && node.text === 'a') {
+          const newNode = ts.createIdentifier('b');
+          ts.setSourceMapRange(newNode, {
+            pos: 16,
+            end: 16,
+            source: ts.createSourceMapSource('test.html', 'abc\ndef\nghi\njkl\nmno\npqr')
+          });
+          return newNode;
+        }
+        return ts.visitEachChild(node, visitor, context);
+      }
+    };
+
+    const program = ts.createProgram(['test.ts'], {sourceMap: true}, host);
+    program.emit(sourceFile, undefined, undefined, undefined, {after: [transform]});
+    // The first two mappings in the source map should look like:
+    // [0,1,4,0] col 0 => source file 1, row 4, column 0)
+    // [1,0,0,0] col 1 => source file 1, row 4, column 0)
+    _tsSourceMapBug29300Fixed = /ACIA,CAAA/.test(writtenFiles['test.js.map']);
+  }
+  return _tsSourceMapBug29300Fixed;
+}

packages/compiler-cli/test/ngtsc/BUILD.bazel

@@ -8,7 +8,10 @@ ts_library(
         "//packages/compiler",
         "//packages/compiler-cli",
         "//packages/compiler-cli/src/ngtsc/routing",
+        "//packages/compiler-cli/src/ngtsc/util",
         "//packages/compiler-cli/test:test_utils",
+        "@ngdeps//@types/source-map",
+        "@ngdeps//source-map",
         "@ngdeps//typescript",
     ],
 )

packages/compiler-cli/test/ngtsc/sourcemap_utils.ts

@@ -0,0 +1,103 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import {MappingItem, SourceMapConsumer} from 'source-map';
+import {NgtscTestEnvironment} from './env';
+
+class TestSourceFile {
+  private lineStarts: number[];
+
+  constructor(public url: string, public contents: string) {
+    this.lineStarts = this.getLineStarts();
+  }
+
+  getSegment(key: 'generated'|'original', start: MappingItem|any, end: MappingItem|any): string {
+    const startLine = start[key + 'Line'];
+    const startCol = start[key + 'Column'];
+    const endLine = end[key + 'Line'];
+    const endCol = end[key + 'Column'];
+    return this.contents.substring(
+        this.lineStarts[startLine - 1] + startCol, this.lineStarts[endLine - 1] + endCol);
+  }
+
+  getSourceMapFileName(generatedContents: string): string {
+    const match = /\/\/# sourceMappingURL=(.+)/.exec(generatedContents);
+    if (!match) {
+      throw new Error('Generated contents does not contain a sourceMappingURL');
+    }
+    return match[1];
+  }
+
+  private getLineStarts(): number[] {
+    const lineStarts = [0];
+    let currentPos = 0;
+    const lines = this.contents.split('\n');
+    lines.forEach(line => {
+      currentPos += line.length + 1;
+      lineStarts.push(currentPos);
+    });
+    return lineStarts;
+  }
+}
+
+/**
+ * A mapping of a segment of generated text to a segment of source text.
+ */
+export interface SegmentMapping {
+  /** The generated text in this segment. */
+  generated: string;
+  /** The source text in this segment. */
+  source: string;
+  /** The URL of the source file for this segment. */
+  sourceUrl: string;
+}
+
+/**
+ * Process a generated file to extract human understandable segment mappings.
+ * These mappings are easier to compare in unit tests that the raw SourceMap mappings.
+ * @param env the environment that holds the source and generated files.
+ * @param generatedFileName The name of the generated file to process.
+ * @returns An array of segment mappings for each mapped segment in the given generated file.
+ */
+export function getMappedSegments(
+    env: NgtscTestEnvironment, generatedFileName: string): SegmentMapping[] {
+  const generated = new TestSourceFile(generatedFileName, env.getContents(generatedFileName));
+  const sourceMapFileName = generated.getSourceMapFileName(generated.contents);
+
+  const sources = new Map<string, TestSourceFile>();
+  const mappings: MappingItem[] = [];
+
+  const mapContents = env.getContents(sourceMapFileName);
+  const sourceMapConsumer = new SourceMapConsumer(JSON.parse(mapContents));
+  sourceMapConsumer.eachMapping(item => {
+    if (!sources.has(item.source)) {
+      sources.set(item.source, new TestSourceFile(item.source, env.getContents(item.source)));
+    }
+    mappings.push(item);
+  });
+
+  const segments: SegmentMapping[] = [];
+  let currentMapping = mappings.shift();
+  while (currentMapping) {
+    const nextMapping = mappings.shift();
+    if (nextMapping) {
+      const source = sources.get(currentMapping.source) !;
+      const segment = {
+        generated: generated.getSegment('generated', currentMapping, nextMapping),
+        source: source.getSegment('original', currentMapping, nextMapping),
+        sourceUrl: source.url
+      };
+      if (segment.generated !== segment.source) {
+        segments.push(segment);
+      }
+    }
+    currentMapping = nextMapping;
+  }
+
+  return segments;
+}

packages/compiler/src/compiler.ts

@@ -74,6 +74,7 @@ export * from './ml_parser/html_parser';
 export * from './ml_parser/html_tags';
 export * from './ml_parser/interpolation_config';
 export * from './ml_parser/tags';
+export {LexerRange} from './ml_parser/lexer';
 export {NgModuleCompiler} from './ng_module_compiler';
 export {ArrayType, AssertNotNull, BinaryOperator, BinaryOperatorExpr, BuiltinMethod, BuiltinType, BuiltinTypeName, BuiltinVar, CastExpr, ClassField, ClassMethod, ClassStmt, CommaExpr, CommentStmt, ConditionalExpr, DeclareFunctionStmt, DeclareVarStmt, Expression, ExpressionStatement, ExpressionType, ExpressionVisitor, ExternalExpr, ExternalReference, FunctionExpr, IfStmt, InstantiateExpr, InvokeFunctionExpr, InvokeMethodExpr, JSDocCommentStmt, LiteralArrayExpr, LiteralExpr, LiteralMapExpr, MapType, NotExpr, ReadKeyExpr, ReadPropExpr, ReadVarExpr, ReturnStatement, StatementVisitor, ThrowStmt, TryCatchStmt, Type, TypeVisitor, WrappedNodeExpr, WriteKeyExpr, WritePropExpr, WriteVarExpr, StmtModifier, Statement, TypeofExpr, collectExternalReferences} from './output/output_ast';
 export {EmitterVisitorContext} from './output/abstract_emitter';