fix(compiler-cli): compute source-mappings for localized strings (#38645)

Previously, localized strings had very limited or incorrect source-mapping
information available.

Now the i18n AST nodes and related output AST nodes include source-span
information about message-parts and placeholders - including closing tag
placeholders.

This information is then used when generating the final localized string
ASTs to ensure that the correct source-mapping is rendered.

See #38588 (comment)

PR Close #38645

packages/compiler-cli/src/ngtsc/translator/src/translator.ts

@@ -6,7 +6,7 @@
  * found in the LICENSE file at https://angular.io/license
  */
 
-import {ArrayType, AssertNotNull, BinaryOperator, BinaryOperatorExpr, BuiltinType, BuiltinTypeName, CastExpr, ClassStmt, CommaExpr, CommentStmt, ConditionalExpr, DeclareFunctionStmt, DeclareVarStmt, Expression, ExpressionStatement, ExpressionType, ExpressionVisitor, ExternalExpr, FunctionExpr, IfStmt, InstantiateExpr, InvokeFunctionExpr, InvokeMethodExpr, JSDocCommentStmt, LiteralArrayExpr, LiteralExpr, LiteralMapExpr, MapType, NotExpr, ReadKeyExpr, ReadPropExpr, ReadVarExpr, ReturnStatement, Statement, StatementVisitor, StmtModifier, ThrowStmt, TryCatchStmt, Type, TypeofExpr, TypeVisitor, WrappedNodeExpr, WriteKeyExpr, WritePropExpr, WriteVarExpr} from '@angular/compiler';
+import {ArrayType, AssertNotNull, BinaryOperator, BinaryOperatorExpr, BuiltinType, BuiltinTypeName, CastExpr, ClassStmt, CommaExpr, CommentStmt, ConditionalExpr, DeclareFunctionStmt, DeclareVarStmt, Expression, ExpressionStatement, ExpressionType, ExpressionVisitor, ExternalExpr, FunctionExpr, IfStmt, InstantiateExpr, InvokeFunctionExpr, InvokeMethodExpr, JSDocCommentStmt, LiteralArrayExpr, LiteralExpr, LiteralMapExpr, MapType, NotExpr, ParseSourceSpan, ReadKeyExpr, ReadPropExpr, ReadVarExpr, ReturnStatement, Statement, StatementVisitor, StmtModifier, ThrowStmt, TryCatchStmt, Type, TypeofExpr, TypeVisitor, WrappedNodeExpr, WriteKeyExpr, WritePropExpr, WriteVarExpr} from '@angular/compiler';
 import {LocalizedString, UnaryOperator, UnaryOperatorExpr} from '@angular/compiler/src/output/output_ast';
 import * as ts from 'typescript';
 
@@ -212,7 +212,7 @@ class ExpressionTranslatorVisitor implements ExpressionVisitor, StatementVisitor
 
   visitReadVarExpr(ast: ReadVarExpr, context: Context): ts.Identifier {
     const identifier = ts.createIdentifier(ast.name!);
-    this.setSourceMapRange(identifier, ast);
+    this.setSourceMapRange(identifier, ast.sourceSpan);
     return identifier;
   }
 
@@ -244,7 +244,7 @@ class ExpressionTranslatorVisitor implements ExpressionVisitor, StatementVisitor
     const call = ts.createCall(
         ast.name !== null ? ts.createPropertyAccess(target, ast.name) : target, undefined,
         ast.args.map(arg => arg.visitExpression(this, context)));
-    this.setSourceMapRange(call, ast);
+    this.setSourceMapRange(call, ast.sourceSpan);
     return call;
   }
 
@@ -255,7 +255,7 @@ class ExpressionTranslatorVisitor implements ExpressionVisitor, StatementVisitor
     if (ast.pure) {
       ts.addSyntheticLeadingComment(expr, ts.SyntaxKind.MultiLineCommentTrivia, '@__PURE__', false);
     }
-    this.setSourceMapRange(expr, ast);
+    this.setSourceMapRange(expr, ast.sourceSpan);
     return expr;
   }
 
@@ -274,15 +274,15 @@ class ExpressionTranslatorVisitor implements ExpressionVisitor, StatementVisitor
     } else {
       expr = ts.createLiteral(ast.value);
     }
-    this.setSourceMapRange(expr, ast);
+    this.setSourceMapRange(expr, ast.sourceSpan);
     return expr;
   }
 
   visitLocalizedString(ast: LocalizedString, context: Context): ts.Expression {
     const localizedString = this.scriptTarget >= ts.ScriptTarget.ES2015 ?
-        createLocalizedStringTaggedTemplate(ast, context, this) :
+        this.createLocalizedStringTaggedTemplate(ast, context) :
-        createLocalizedStringFunctionCall(ast, context, this, this.imports);
+        this.createLocalizedStringFunctionCall(ast, context);
-    this.setSourceMapRange(localizedString, ast);
+    this.setSourceMapRange(localizedString, ast.sourceSpan);
     return localizedString;
   }
 
@@ -395,7 +395,7 @@ class ExpressionTranslatorVisitor implements ExpressionVisitor, StatementVisitor
   visitLiteralArrayExpr(ast: LiteralArrayExpr, context: Context): ts.ArrayLiteralExpression {
     const expr =
         ts.createArrayLiteral(ast.entries.map(expr => expr.visitExpression(this, context)));
-    this.setSourceMapRange(expr, ast);
+    this.setSourceMapRange(expr, ast.sourceSpan);
     return expr;
   }
 
@@ -405,7 +405,7 @@ class ExpressionTranslatorVisitor implements ExpressionVisitor, StatementVisitor
             entry.quoted ? ts.createLiteral(entry.key) : ts.createIdentifier(entry.key),
             entry.value.visitExpression(this, context)));
     const expr = ts.createObjectLiteral(entries);
-    this.setSourceMapRange(expr, ast);
+    this.setSourceMapRange(expr, ast.sourceSpan);
     return expr;
   }
 
@@ -424,9 +424,111 @@ class ExpressionTranslatorVisitor implements ExpressionVisitor, StatementVisitor
     return ts.createTypeOf(ast.expr.visitExpression(this, context));
   }
 
-  private setSourceMapRange(expr: ts.Expression, ast: Expression) {
+  /**
-    if (ast.sourceSpan) {
+   * Translate the `LocalizedString` node into a `TaggedTemplateExpression` for ES2015 formatted
-      const {start, end} = ast.sourceSpan;
+   * output.
+   */
+  private createLocalizedStringTaggedTemplate(ast: LocalizedString, context: Context):
+      ts.TaggedTemplateExpression {
+    let template: ts.TemplateLiteral;
+    const length = ast.messageParts.length;
+    const metaBlock = ast.serializeI18nHead();
+    if (length === 1) {
+      template = ts.createNoSubstitutionTemplateLiteral(metaBlock.cooked, metaBlock.raw);
+      this.setSourceMapRange(template, ast.getMessagePartSourceSpan(0));
+    } else {
+      // Create the head part
+      const head = ts.createTemplateHead(metaBlock.cooked, metaBlock.raw);
+      this.setSourceMapRange(head, ast.getMessagePartSourceSpan(0));
+      const spans: ts.TemplateSpan[] = [];
+      // Create the middle parts
+      for (let i = 1; i < length - 1; i++) {
+        const resolvedExpression = ast.expressions[i - 1].visitExpression(this, context);
+        this.setSourceMapRange(resolvedExpression, ast.getPlaceholderSourceSpan(i - 1));
+        const templatePart = ast.serializeI18nTemplatePart(i);
+        const templateMiddle = createTemplateMiddle(templatePart.cooked, templatePart.raw);
+        this.setSourceMapRange(templateMiddle, ast.getMessagePartSourceSpan(i));
+        const templateSpan = ts.createTemplateSpan(resolvedExpression, templateMiddle);
+        spans.push(templateSpan);
+      }
+      // Create the tail part
+      const resolvedExpression = ast.expressions[length - 2].visitExpression(this, context);
+      this.setSourceMapRange(resolvedExpression, ast.getPlaceholderSourceSpan(length - 2));
+      const templatePart = ast.serializeI18nTemplatePart(length - 1);
+      const templateTail = createTemplateTail(templatePart.cooked, templatePart.raw);
+      this.setSourceMapRange(templateTail, ast.getMessagePartSourceSpan(length - 1));
+      spans.push(ts.createTemplateSpan(resolvedExpression, templateTail));
+      // Put it all together
+      template = ts.createTemplateExpression(head, spans);
+    }
+    const expression = ts.createTaggedTemplate(ts.createIdentifier('$localize'), template);
+    this.setSourceMapRange(expression, ast.sourceSpan);
+    return expression;
+  }
+
+  /**
+   * Translate the `LocalizedString` node into a `$localize` call using the imported
+   * `__makeTemplateObject` helper for ES5 formatted output.
+   */
+  private createLocalizedStringFunctionCall(ast: LocalizedString, context: Context) {
+    // A `$localize` message consists `messageParts` and `expressions`, which get interleaved
+    // together. The interleaved pieces look like:
+    // `[messagePart0, expression0, messagePart1, expression1, messagePart2]`
+    //
+    // Note that there is always a message part at the start and end, and so therefore
+    // `messageParts.length === expressions.length + 1`.
+    //
+    // Each message part may be prefixed with "metadata", which is wrapped in colons (:) delimiters.
+    // The metadata is attached to the first and subsequent message parts by calls to
+    // `serializeI18nHead()` and `serializeI18nTemplatePart()` respectively.
+
+    // The first message part (i.e. `ast.messageParts[0]`) is used to initialize `messageParts`
+    // array.
+    const messageParts = [ast.serializeI18nHead()];
+    const expressions: any[] = [];
+
+    // The rest of the `ast.messageParts` and each of the expressions are `ast.expressions` pushed
+    // into the arrays. Note that `ast.messagePart[i]` corresponds to `expressions[i-1]`
+    for (let i = 1; i < ast.messageParts.length; i++) {
+      expressions.push(ast.expressions[i - 1].visitExpression(this, context));
+      messageParts.push(ast.serializeI18nTemplatePart(i));
+    }
+
+    // The resulting downlevelled tagged template string uses a call to the `__makeTemplateObject()`
+    // helper, so we must ensure it has been imported.
+    const {moduleImport, symbol} =
+        this.imports.generateNamedImport('tslib', '__makeTemplateObject');
+    const __makeTemplateObjectHelper = (moduleImport === null) ?
+        ts.createIdentifier(symbol) :
+        ts.createPropertyAccess(ts.createIdentifier(moduleImport), ts.createIdentifier(symbol));
+
+    // Generate the call in the form:
+    // `$localize(__makeTemplateObject(cookedMessageParts, rawMessageParts), ...expressions);`
+    const cookedLiterals = messageParts.map(
+        (messagePart, i) =>
+            this.createLiteral(messagePart.cooked, ast.getMessagePartSourceSpan(i)));
+    const rawLiterals = messageParts.map(
+        (messagePart, i) => this.createLiteral(messagePart.raw, ast.getMessagePartSourceSpan(i)));
+    return ts.createCall(
+        /* expression */ ts.createIdentifier('$localize'),
+        /* typeArguments */ undefined,
+        /* argumentsArray */[
+          ts.createCall(
+              /* expression */ __makeTemplateObjectHelper,
+              /* typeArguments */ undefined,
+              /* argumentsArray */
+              [
+                ts.createArrayLiteral(cookedLiterals),
+                ts.createArrayLiteral(rawLiterals),
+              ]),
+          ...expressions,
+        ]);
+  }
+
+
+  private setSourceMapRange(expr: ts.Node, sourceSpan: ParseSourceSpan|null) {
+    if (sourceSpan) {
+      const {start, end} = sourceSpan;
       const {url, content} = start.file;
       if (url) {
         if (!this.externalSourceFiles.has(url)) {
@@ -437,6 +539,12 @@ class ExpressionTranslatorVisitor implements ExpressionVisitor, StatementVisitor
       }
     }
   }
+
+  private createLiteral(text: string, span: ParseSourceSpan|null) {
+    const literal = ts.createStringLiteral(text);
+    this.setSourceMapRange(literal, span);
+    return literal;
+  }
 }
 
 export class TypeTranslatorVisitor implements ExpressionVisitor, TypeVisitor {
@@ -662,40 +770,6 @@ export class TypeTranslatorVisitor implements ExpressionVisitor, TypeVisitor {
   }
 }
 
-/**
- * Translate the `LocalizedString` node into a `TaggedTemplateExpression` for ES2015 formatted
- * output.
- */
-function createLocalizedStringTaggedTemplate(
-    ast: LocalizedString, context: Context, visitor: ExpressionVisitor) {
-  let template: ts.TemplateLiteral;
-  const length = ast.messageParts.length;
-  const metaBlock = ast.serializeI18nHead();
-  if (length === 1) {
-    template = ts.createNoSubstitutionTemplateLiteral(metaBlock.cooked, metaBlock.raw);
-  } else {
-    // Create the head part
-    const head = ts.createTemplateHead(metaBlock.cooked, metaBlock.raw);
-    const spans: ts.TemplateSpan[] = [];
-    // Create the middle parts
-    for (let i = 1; i < length - 1; i++) {
-      const resolvedExpression = ast.expressions[i - 1].visitExpression(visitor, context);
-      const templatePart = ast.serializeI18nTemplatePart(i);
-      const templateMiddle = createTemplateMiddle(templatePart.cooked, templatePart.raw);
-      spans.push(ts.createTemplateSpan(resolvedExpression, templateMiddle));
-    }
-    // Create the tail part
-    const resolvedExpression = ast.expressions[length - 2].visitExpression(visitor, context);
-    const templatePart = ast.serializeI18nTemplatePart(length - 1);
-    const templateTail = createTemplateTail(templatePart.cooked, templatePart.raw);
-    spans.push(ts.createTemplateSpan(resolvedExpression, templateTail));
-    // Put it all together
-    template = ts.createTemplateExpression(head, spans);
-  }
-  return ts.createTaggedTemplate(ts.createIdentifier('$localize'), template);
-}
-
-
 // HACK: Use this in place of `ts.createTemplateMiddle()`.
 // Revert once https://github.com/microsoft/TypeScript/issues/35374 is fixed
 function createTemplateMiddle(cooked: string, raw: string): ts.TemplateMiddle {
@@ -710,59 +784,4 @@ function createTemplateTail(cooked: string, raw: string): ts.TemplateTail {
   const node: ts.TemplateLiteralLikeNode = ts.createTemplateHead(cooked, raw);
   (node.kind as ts.SyntaxKind) = ts.SyntaxKind.TemplateTail;
   return node as ts.TemplateTail;
-}
+}
-
-/**
- * Translate the `LocalizedString` node into a `$localize` call using the imported
- * `__makeTemplateObject` helper for ES5 formatted output.
- */
-function createLocalizedStringFunctionCall(
-    ast: LocalizedString, context: Context, visitor: ExpressionVisitor, imports: ImportManager) {
-  // A `$localize` message consists `messageParts` and `expressions`, which get interleaved
-  // together. The interleaved pieces look like:
-  // `[messagePart0, expression0, messagePart1, expression1, messagePart2]`
-  //
-  // Note that there is always a message part at the start and end, and so therefore
-  // `messageParts.length === expressions.length + 1`.
-  //
-  // Each message part may be prefixed with "metadata", which is wrapped in colons (:) delimiters.
-  // The metadata is attached to the first and subsequent message parts by calls to
-  // `serializeI18nHead()` and `serializeI18nTemplatePart()` respectively.
-
-  // The first message part (i.e. `ast.messageParts[0]`) is used to initialize `messageParts` array.
-  const messageParts = [ast.serializeI18nHead()];
-  const expressions: any[] = [];
-
-  // The rest of the `ast.messageParts` and each of the expressions are `ast.expressions` pushed
-  // into the arrays. Note that `ast.messagePart[i]` corresponds to `expressions[i-1]`
-  for (let i = 1; i < ast.messageParts.length; i++) {
-    expressions.push(ast.expressions[i - 1].visitExpression(visitor, context));
-    messageParts.push(ast.serializeI18nTemplatePart(i));
-  }
-
-  // The resulting downlevelled tagged template string uses a call to the `__makeTemplateObject()`
-  // helper, so we must ensure it has been imported.
-  const {moduleImport, symbol} = imports.generateNamedImport('tslib', '__makeTemplateObject');
-  const __makeTemplateObjectHelper = (moduleImport === null) ?
-      ts.createIdentifier(symbol) :
-      ts.createPropertyAccess(ts.createIdentifier(moduleImport), ts.createIdentifier(symbol));
-
-  // Generate the call in the form:
-  // `$localize(__makeTemplateObject(cookedMessageParts, rawMessageParts), ...expressions);`
-  return ts.createCall(
-      /* expression */ ts.createIdentifier('$localize'),
-      /* typeArguments */ undefined,
-      /* argumentsArray */[
-        ts.createCall(
-            /* expression */ __makeTemplateObjectHelper,
-            /* typeArguments */ undefined,
-            /* argumentsArray */
-            [
-              ts.createArrayLiteral(
-                  messageParts.map(messagePart => ts.createStringLiteral(messagePart.cooked))),
-              ts.createArrayLiteral(
-                  messageParts.map(messagePart => ts.createStringLiteral(messagePart.raw))),
-            ]),
-        ...expressions,
-      ]);
-}

packages/compiler-cli/test/ngtsc/template_mapping_spec.ts

@@ -24,7 +24,7 @@ runInEachFileSystem((os) => {
 
     beforeEach(() => {
       env = NgtscTestEnvironment.setup(testFiles);
-      env.tsconfig();
+      env.tsconfig({sourceMap: true, target: 'es2015', enableI18nLegacyMessageIdFormat: false});
     });
 
     describe('Inline templates', () => {
@@ -360,6 +360,90 @@ runInEachFileSystem((os) => {
         });
       });
 
+      describe('$localize', () => {
+        it('should create simple i18n message source-mapping', () => {
+          const mappings = compileAndMap(`<div i18n>Hello, World!</div>`);
+          expect(mappings).toContain({
+            source: '<div i18n>',
+            generated: 'i0.ɵɵelementStart(0, "div")',
+            sourceUrl: '../test.ts',
+          });
+          expect(mappings).toContain({
+            source: 'Hello, World!',
+            generated: '`Hello, World!`',
+            sourceUrl: '../test.ts',
+          });
+        });
+
+        it('should create placeholder source-mappings', () => {
+          const mappings = compileAndMap(`<div i18n>Hello, {{name}}!</div>`);
+          expect(mappings).toContain({
+            source: '<div i18n>',
+            generated: 'i0.ɵɵelementStart(0, "div")',
+            sourceUrl: '../test.ts',
+          });
+          expect(mappings).toContain({
+            source: '</div>',
+            generated: 'i0.ɵɵelementEnd()',
+            sourceUrl: '../test.ts',
+          });
+          expect(mappings).toContain({
+            source: 'Hello, ',
+            generated: '`Hello, ${',
+            sourceUrl: '../test.ts',
+          });
+          expect(mappings).toContain({
+            source: '{{name}}',
+            generated: '"\\uFFFD0\\uFFFD"',
+            sourceUrl: '../test.ts',
+          });
+          expect(mappings).toContain({
+            source: '!',
+            generated: '}:INTERPOLATION:!`',
+            sourceUrl: '../test.ts',
+          });
+        });
+
+        it('should create tag (container) placeholder source-mappings', () => {
+          const mappings = compileAndMap(`<div i18n>Hello, <b>World</b>!</div>`);
+          expect(mappings).toContain({
+            source: '<div i18n>',
+            generated: 'i0.ɵɵelementStart(0, "div")',
+            sourceUrl: '../test.ts',
+          });
+          expect(mappings).toContain({
+            source: '</div>',
+            generated: 'i0.ɵɵelementEnd()',
+            sourceUrl: '../test.ts',
+          });
+          expect(mappings).toContain({
+            source: 'Hello, ',
+            generated: '`Hello, ${',
+            sourceUrl: '../test.ts',
+          });
+          expect(mappings).toContain({
+            source: '<b>',
+            generated: '"\\uFFFD#2\\uFFFD"',
+            sourceUrl: '../test.ts',
+          });
+          expect(mappings).toContain({
+            source: 'World',
+            generated: '}:START_BOLD_TEXT:World${',
+            sourceUrl: '../test.ts',
+          });
+          expect(mappings).toContain({
+            source: '</b>',
+            generated: '"\\uFFFD/#2\\uFFFD"',
+            sourceUrl: '../test.ts',
+          });
+          expect(mappings).toContain({
+            source: '!',
+            generated: '}:CLOSE_BOLD_TEXT:!`',
+            sourceUrl: '../test.ts',
+          });
+        });
+      });
+
       it('should create (simple string) inline template source-mapping', () => {
         const mappings = compileAndMap('<div>this is a test</div><div>{{ 1 + 2 }}</div>');
 
@@ -520,7 +604,6 @@ runInEachFileSystem((os) => {
     function compileAndMap(template: string, templateUrl: string|null = null) {
       const templateConfig = templateUrl ? `templateUrl: '${templateUrl}'` :
                                            ('template: `' + template.replace(/`/g, '\\`') + '`');
-      env.tsconfig({sourceMap: true});
       env.write('test.ts', `
         import {Component} from '@angular/core';
 

packages/compiler/src/i18n/i18n_ast.ts

@@ -87,7 +87,9 @@ export class TagPlaceholder implements Node {
   constructor(
       public tag: string, public attrs: {[k: string]: string}, public startName: string,
       public closeName: string, public children: Node[], public isVoid: boolean,
-      public sourceSpan: ParseSourceSpan, public closeSourceSpan: ParseSourceSpan|null) {}
+      // TODO sourceSpan should cover all (we need a startSourceSpan and endSourceSpan)
+      public sourceSpan: ParseSourceSpan, public startSourceSpan: ParseSourceSpan|null,
+      public endSourceSpan: ParseSourceSpan|null) {}
 
   visit(visitor: Visitor, context?: any): any {
     return visitor.visitTagPlaceholder(this, context);
@@ -152,7 +154,7 @@ export class CloneVisitor implements Visitor {
     const children = ph.children.map(n => n.visit(this, context));
     return new TagPlaceholder(
         ph.tag, ph.attrs, ph.startName, ph.closeName, children, ph.isVoid, ph.sourceSpan,
-        ph.closeSourceSpan);
+        ph.startSourceSpan, ph.endSourceSpan);
   }
 
   visitPlaceholder(ph: Placeholder, context?: any): Placeholder {

packages/compiler/src/i18n/i18n_parser.ts

@@ -93,8 +93,8 @@ class _I18nVisitor implements html.Visitor {
     }
 
     const node = new i18n.TagPlaceholder(
-        el.name, attrs, startPhName, closePhName, children, isVoid, el.startSourceSpan,
+        el.name, attrs, startPhName, closePhName, children, isVoid, el.sourceSpan,
-        el.endSourceSpan);
+        el.startSourceSpan, el.endSourceSpan);
     return context.visitNodeFn(el, node);
   }
 

packages/compiler/src/i18n/message_bundle.ts

@@ -95,7 +95,7 @@ class MapPlaceholderNames extends i18n.CloneVisitor {
     const children = ph.children.map(n => n.visit(this, mapper));
     return new i18n.TagPlaceholder(
         ph.tag, ph.attrs, startName, closeName, children, ph.isVoid, ph.sourceSpan,
-        ph.closeSourceSpan);
+        ph.startSourceSpan, ph.endSourceSpan);
   }
 
   visitPlaceholder(ph: i18n.Placeholder, mapper: PlaceholderMapper): i18n.Placeholder {

packages/compiler/src/render3/view/i18n/localize_utils.ts

@@ -48,10 +48,10 @@ class LocalizeSerializerVisitor implements i18n.Visitor {
   }
 
   visitTagPlaceholder(ph: i18n.TagPlaceholder, context: o.MessagePiece[]): any {
-    context.push(this.createPlaceholderPiece(ph.startName, ph.sourceSpan));
+    context.push(this.createPlaceholderPiece(ph.startName, ph.startSourceSpan ?? ph.sourceSpan));
     if (!ph.isVoid) {
       ph.children.forEach(child => child.visit(this, context));
-      context.push(this.createPlaceholderPiece(ph.closeName, ph.closeSourceSpan ?? ph.sourceSpan));
+      context.push(this.createPlaceholderPiece(ph.closeName, ph.endSourceSpan ?? ph.sourceSpan));
     }
   }
 

packages/compiler/test/i18n/serializers/i18n_ast_spec.ts

@@ -41,7 +41,7 @@ import {_extractMessages} from '../i18n_parser_spec';
               new i18n.IcuPlaceholder(null!, '', null!),
             ],
             null!);
-        const tag = new i18n.TagPlaceholder('', {}, '', '', [container], false, null!, null);
+        const tag = new i18n.TagPlaceholder('', {}, '', '', [container], false, null!, null, null);
         const icu = new i18n.Icu('', '', {tag}, null!);
 
         icu.visit(visitor);