fix(localize): render text of extracted placeholders (#38536)

Formats like XLIFF allow the text of the original source to
be included as metadata. This commit fixes the message
extractor to also render this text when available.

PR Close #38536

packages/localize/src/tools/src/extract/extraction.ts

@@ -77,6 +77,20 @@ export class MessageExtractor {
     for (const message of messages) {
       if (message.location !== undefined) {
         message.location = this.getOriginalLocation(sourceFile, message.location);
+
+        if (message.messagePartLocations) {
+          message.messagePartLocations = message.messagePartLocations.map(
+              location => location && this.getOriginalLocation(sourceFile, location));
+        }
+
+        if (message.substitutionLocations) {
+          const placeholderNames = Object.keys(message.substitutionLocations);
+          for (const placeholderName of placeholderNames) {
+            const location = message.substitutionLocations[placeholderName];
+            message.substitutionLocations[placeholderName] =
+                location && this.getOriginalLocation(sourceFile, location);
+          }
+        }
       }
     }
   }

packages/localize/src/tools/src/extract/translation_files/xliff1_translation_serializer.ts

@@ -67,7 +67,8 @@ export class Xliff1TranslationSerializer implements TranslationSerializer {
     const length = message.messageParts.length - 1;
     for (let i = 0; i < length; i++) {
       this.serializeTextPart(xml, message.messageParts[i]);
-      xml.startTag('x', {id: message.placeholderNames[i]}, {selfClosing: true});
+      const location = message.substitutionLocations?.[message.placeholderNames[i]];
+      this.serializePlaceholder(xml, message.placeholderNames[i], location?.text);
     }
     this.serializeTextPart(xml, message.messageParts[length]);
   }
@@ -77,11 +78,19 @@ export class Xliff1TranslationSerializer implements TranslationSerializer {
     const length = pieces.length - 1;
     for (let i = 0; i < length; i += 2) {
       xml.text(pieces[i]);
-      xml.startTag('x', {id: pieces[i + 1]}, {selfClosing: true});
+      this.serializePlaceholder(xml, pieces[i + 1], undefined);
     }
     xml.text(pieces[length]);
   }
 
+  private serializePlaceholder(xml: XmlFile, id: string, text: string|undefined): void {
+    const attrs: Record<string, string> = {id};
+    if (text !== undefined) {
+      attrs['equiv-text'] = text;
+    }
+    xml.startTag('x', attrs, {selfClosing: true});
+  }
+
   private serializeNote(xml: XmlFile, name: string, value: string): void {
     xml.startTag('note', {priority: '1', from: name}, {preserveWhitespace: true});
     xml.text(value);

packages/localize/src/tools/src/extract/translation_files/xliff2_translation_serializer.ts

@@ -6,7 +6,7 @@
  * found in the LICENSE file at https://angular.io/license
  */
 import {AbsoluteFsPath, relative} from '@angular/compiler-cli/src/ngtsc/file_system';
-import {ɵParsedMessage} from '@angular/localize';
+import {ɵParsedMessage, ɵSourceLocation} from '@angular/localize';
 
 import {extractIcuPlaceholders} from './icu_parsing';
 import {TranslationSerializer} from './translation_serializer';
@@ -80,7 +80,7 @@ export class Xliff2TranslationSerializer implements TranslationSerializer {
     const length = message.messageParts.length - 1;
     for (let i = 0; i < length; i++) {
       this.serializeTextPart(xml, message.messageParts[i]);
-      this.serializePlaceholder(xml, message.placeholderNames[i]);
+      this.serializePlaceholder(xml, message.placeholderNames[i], message.substitutionLocations);
     }
     this.serializeTextPart(xml, message.messageParts[length]);
   }
@@ -90,24 +90,40 @@ export class Xliff2TranslationSerializer implements TranslationSerializer {
     const length = pieces.length - 1;
     for (let i = 0; i < length; i += 2) {
       xml.text(pieces[i]);
-      this.serializePlaceholder(xml, pieces[i + 1]);
+      this.serializePlaceholder(xml, pieces[i + 1], undefined);
     }
     xml.text(pieces[length]);
   }
 
-  private serializePlaceholder(xml: XmlFile, placeholderName: string): void {
+  private serializePlaceholder(
+      xml: XmlFile, placeholderName: string,
+      substitutionLocations: Record<string, ɵSourceLocation|undefined>|undefined): void {
+    const text = substitutionLocations?.[placeholderName]?.text;
+
     if (placeholderName.startsWith('START_')) {
-      xml.startTag('pc', {
+      const closingPlaceholderName = placeholderName.replace(/^START/, 'CLOSE');
+      const closingText = substitutionLocations?.[closingPlaceholderName]?.text;
+      const attrs: Record<string, string> = {
         id: `${this.currentPlaceholderId++}`,
         equivStart: placeholderName,
-        equivEnd: placeholderName.replace(/^START/, 'CLOSE')
+        equivEnd: closingPlaceholderName,
-      });
+      };
+      if (text !== undefined) {
+        attrs.dispStart = text;
+      }
+      if (closingText !== undefined) {
+        attrs.dispEnd = closingText;
+      }
+      xml.startTag('pc', attrs);
     } else if (placeholderName.startsWith('CLOSE_')) {
       xml.endTag('pc');
     } else {
-      xml.startTag(
+      const attrs:
-          'ph', {id: `${this.currentPlaceholderId++}`, equiv: placeholderName},
+          Record<string, string> = {id: `${this.currentPlaceholderId++}`, equiv: placeholderName};
-          {selfClosing: true});
+      if (text !== undefined) {
+        attrs.disp = text;
+      }
+      xml.startTag('ph', attrs, {selfClosing: true});
     }
   }
 

packages/localize/src/tools/test/extract/integration/main_spec.ts

@@ -77,7 +77,8 @@ runInEachFileSystem(() => {
             `    "8669027859022295761": "try{$PH}me",`,
             `    "custom-id": "Custom id message",`,
             `    "273296103957933077": "Legacy id message",`,
-            `    "custom-id-2": "Custom and legacy message"`,
+            `    "custom-id-2": "Custom and legacy message",`,
+            `    "2932901491976224757": "pre{$START_TAG_SPAN}inner-pre{$START_BOLD_TEXT}bold{$CLOSE_BOLD_TEXT}inner-post{$CLOSE_TAG_SPAN}post"`,
             `  }`,
             `}`,
           ].join('\n'));
@@ -127,6 +128,8 @@ runInEachFileSystem(() => {
                     '12345678901234567890' :
                     '273296103957933077'}"><source>test_files/test.js:5</source>Legacy id message</msg>`,
             `  <msg id="custom-id-2"><source>test_files/test.js:7</source>Custom and legacy message</msg>`,
+            `  <msg id="2932901491976224757"><source>test_files/test.js:8,10</source>pre<ph name="START_TAG_SPAN"/>` +
+                `inner-pre<ph name="START_BOLD_TEXT"/>bold<ph name="CLOSE_BOLD_TEXT"/>inner-post<ph name="CLOSE_TAG_SPAN"/>post</msg>`,
             `</messagebundle>\n`,
           ].join('\n'));
         });
@@ -149,14 +152,14 @@ runInEachFileSystem(() => {
             `  <file source-language="en-CA" datatype="plaintext">`,
             `    <body>`,
             `      <trans-unit id="3291030485717846467" datatype="html">`,
-            `        <source>Hello, <x id="PH"/>!</source>`,
+            `        <source>Hello, <x id="PH" equiv-text="name"/>!</source>`,
             `        <context-group purpose="location">`,
             `          <context context-type="sourcefile">test_files/test.js</context>`,
             `          <context context-type="linenumber">2</context>`,
             `        </context-group>`,
             `      </trans-unit>`,
             `      <trans-unit id="8669027859022295761" datatype="html">`,
-            `        <source>try<x id="PH"/>me</source>`,
+            `        <source>try<x id="PH" equiv-text="40 + 2"/>me</source>`,
             `        <context-group purpose="location">`,
             `          <context context-type="sourcefile">test_files/test.js</context>`,
             `          <context context-type="linenumber">3</context>`,
@@ -185,6 +188,15 @@ runInEachFileSystem(() => {
             `          <context context-type="linenumber">8</context>`,
             `        </context-group>`,
             `      </trans-unit>`,
+            `      <trans-unit id="2932901491976224757" datatype="html">`,
+            `        <source>pre<x id="START_TAG_SPAN" equiv-text="&apos;&lt;span&gt;&apos;"/>` +
+                `inner-pre<x id="START_BOLD_TEXT" equiv-text="&apos;&lt;b&gt;&apos;"/>bold<x id="CLOSE_BOLD_TEXT" equiv-text="&apos;&lt;/b&gt;&apos;"/>` +
+                `inner-post<x id="CLOSE_TAG_SPAN" equiv-text="&apos;&lt;/span&gt;&apos;"/>post</source>`,
+            `        <context-group purpose="location">`,
+            `          <context context-type="sourcefile">test_files/test.js</context>`,
+            `          <context context-type="linenumber">9,10</context>`,
+            `        </context-group>`,
+            `      </trans-unit>`,
             `    </body>`,
             `  </file>`,
             `</xliff>\n`,
@@ -209,12 +221,12 @@ runInEachFileSystem(() => {
             `  <file>`,
             `    <unit id="3291030485717846467">`,
             `      <segment>`,
-            `        <source>Hello, <ph id="0" equiv="PH"/>!</source>`,
+            `        <source>Hello, <ph id="0" equiv="PH" disp="name"/>!</source>`,
             `      </segment>`,
             `    </unit>`,
             `    <unit id="8669027859022295761">`,
             `      <segment>`,
-            `        <source>try<ph id="0" equiv="PH"/>me</source>`,
+            `        <source>try<ph id="0" equiv="PH" disp="40 + 2"/>me</source>`,
             `      </segment>`,
             `    </unit>`,
             `    <unit id="custom-id">`,
@@ -232,6 +244,13 @@ runInEachFileSystem(() => {
             `        <source>Custom and legacy message</source>`,
             `      </segment>`,
             `    </unit>`,
+            `    <unit id="2932901491976224757">`,
+            `      <segment>`,
+            `        <source>pre<pc id="0" equivStart="START_TAG_SPAN" equivEnd="CLOSE_TAG_SPAN" dispStart="&apos;&lt;span&gt;&apos;" dispEnd="&apos;&lt;/span&gt;&apos;">` +
+                `inner-pre<pc id="1" equivStart="START_BOLD_TEXT" equivEnd="CLOSE_BOLD_TEXT" dispStart="&apos;&lt;b&gt;&apos;" dispEnd="&apos;&lt;/b&gt;&apos;">bold</pc>` +
+                `inner-post</pc>post</source>`,
+            `      </segment>`,
+            `    </unit>`,
             `  </file>`,
             `</xliff>\n`,
           ].join('\n'));
@@ -260,7 +279,7 @@ runInEachFileSystem(() => {
              `  <file source-language="en-CA" datatype="plaintext">`,
              `    <body>`,
              `      <trans-unit id="157258427077572998" datatype="html">`,
-             `        <source>Message in <x id="a-file"/>!</source>`,
+             `        <source>Message in <x id="a-file" equiv-text="file"/>!</source>`,
              `        <context-group purpose="location">`,
              // These source file paths are due to how Bazel TypeScript compilation source-maps work
              `          <context context-type="sourcefile">../packages/localize/src/tools/test/extract/integration/test_files/src/a.ts</context>`,
@@ -268,7 +287,7 @@ runInEachFileSystem(() => {
              `        </context-group>`,
              `      </trans-unit>`,
              `      <trans-unit id="7829869508202074508" datatype="html">`,
-             `        <source>Message in <x id="b-file"/>!</source>`,
+             `        <source>Message in <x id="b-file" equiv-text="file"/>!</source>`,
              `        <context-group purpose="location">`,
              `          <context context-type="sourcefile">../packages/localize/src/tools/test/extract/integration/test_files/src/b.ts</context>`,
              `          <context context-type="linenumber">3</context>`,

packages/localize/src/tools/test/extract/integration/test_files/test.js

@@ -6,3 +6,5 @@ var legacyMessage =
     $localize`:␟1234567890123456789012345678901234567890␟12345678901234567890:Legacy id message`;
 var customAndLegacyMessage =
     $localize`:@@custom-id-2␟1234567890123456789012345678901234567890␟12345678901234567890:Custom and legacy message`;
+var containers = $localize`pre${'<span>'}:START_TAG_SPAN:inner-pre${'<b>'}:START_BOLD_TEXT:bold${
+                          '</b>'}:CLOSE_BOLD_TEXT:inner-post${'</span>'}:CLOSE_TAG_SPAN:post`;

packages/localize/src/tools/test/extract/translation_files/mock_message.ts

@@ -6,7 +6,7 @@
  * found in the LICENSE file at https://angular.io/license
  */
 import {ɵParsedMessage} from '@angular/localize';
-import {SourceLocation} from '@angular/localize/src/utils';
+import {MessageId, SourceLocation} from '@angular/localize/src/utils';
 
 export interface MockMessageOptions {
   customId?: string;
@@ -14,6 +14,8 @@ export interface MockMessageOptions {
   description?: string;
   location?: SourceLocation;
   legacyIds?: string[];
+  messagePartLocations?: (SourceLocation|undefined)[];
+  substitutionLocations?: Record<string, SourceLocation|undefined>;
 }
 
 /**
@@ -21,23 +23,18 @@ export interface MockMessageOptions {
  * `TranslationSerializer` tests.
  */
 export function mockMessage(
-    id: string, messageParts: string[], placeholderNames: string[],
+    id: MessageId, messageParts: string[], placeholderNames: string[],
-    {customId, meaning = '', description = '', location, legacyIds = []}: MockMessageOptions):
+    options: MockMessageOptions): ɵParsedMessage {
-    ɵParsedMessage {
   let text = messageParts[0];
   for (let i = 1; i < messageParts.length; i++) {
     text += `{$${placeholderNames[i - 1]}}${messageParts[i]}`;
   }
   return {
-    id: customId || id,  // customId trumps id
+    substitutions: [],
+    ...options,
+    id: options.customId || id,  // customId trumps id
     text,
     messageParts,
     placeholderNames,
-    customId,
-    description,
-    meaning,
-    substitutions: [],
-    legacyIds,
-    location,
   };
 }

packages/localize/src/tools/test/extract/translation_files/xliff1_translation_serializer_spec.ts

@@ -7,7 +7,7 @@
  */
 import {absoluteFrom} from '@angular/compiler-cli/src/ngtsc/file_system';
 import {runInEachFileSystem} from '@angular/compiler-cli/src/ngtsc/file_system/testing';
-import {ɵParsedMessage} from '@angular/localize';
+import {ɵParsedMessage, ɵSourceLocation} from '@angular/localize';
 
 import {Xliff1TranslationSerializer} from '../../../src/extract/translation_files/xliff1_translation_serializer';
 
@@ -18,6 +18,18 @@ runInEachFileSystem(() => {
     [false, true].forEach(useLegacyIds => {
       describe(`renderFile() [using ${useLegacyIds ? 'legacy' : 'canonical'} ids]`, () => {
         it('should convert a set of parsed messages into an XML string', () => {
+          const phLocation: ɵSourceLocation = {
+            start: {line: 0, column: 10},
+            end: {line: 1, column: 15},
+            file: absoluteFrom('/project/file.ts'),
+            text: 'placeholder + 1'
+          };
+          const messagePartLocation: ɵSourceLocation = {
+            start: {line: 0, column: 5},
+            end: {line: 0, column: 10},
+            file: absoluteFrom('/project/file.ts'),
+            text: 'message part'
+          };
           const messages: ɵParsedMessage[] = [
             mockMessage('12345', ['a', 'b', 'c'], ['PH', 'PH_1'], {
               meaning: 'some meaning',
@@ -31,6 +43,8 @@ runInEachFileSystem(() => {
             mockMessage('54321', ['a', 'b', 'c'], ['PH', 'PH_1'], {
               customId: 'someId',
               legacyIds: ['87654321FEDCBA0987654321FEDCBA0987654321', '563965274788097516'],
+              messagePartLocations: [undefined, messagePartLocation, undefined],
+              substitutionLocations: {'PH': phLocation, 'PH_1': undefined},
             }),
             mockMessage(
                 '67890', ['a', '', 'c'], ['START_TAG_SPAN', 'CLOSE_TAG_SPAN'],
@@ -71,7 +85,7 @@ runInEachFileSystem(() => {
             `        <note priority="1" from="meaning">some meaning</note>`,
             `      </trans-unit>`,
             `      <trans-unit id="someId" datatype="html">`,
-            `        <source>a<x id="PH"/>b<x id="PH_1"/>c</source>`,
+            `        <source>a<x id="PH" equiv-text="placeholder + 1"/>b<x id="PH_1"/>c</source>`,
             `      </trans-unit>`,
             `      <trans-unit id="67890" datatype="html">`,
             `        <source>a<x id="START_TAG_SPAN"/><x id="CLOSE_TAG_SPAN"/>c</source>`,

packages/localize/src/tools/test/extract/translation_files/xliff2_translation_serializer_spec.ts

@@ -5,10 +5,9 @@
  * Use of this source code is governed by an MIT-style license that can be
  * found in the LICENSE file at https://angular.io/license
  */
-import {computeMsgId} from '@angular/compiler';
 import {absoluteFrom} from '@angular/compiler-cli/src/ngtsc/file_system';
 import {runInEachFileSystem} from '@angular/compiler-cli/src/ngtsc/file_system/testing';
-import {ɵParsedMessage} from '@angular/localize';
+import {ɵParsedMessage, ɵSourceLocation} from '@angular/localize';
 
 import {Xliff2TranslationSerializer} from '../../../src/extract/translation_files/xliff2_translation_serializer';
 
@@ -19,6 +18,18 @@ runInEachFileSystem(() => {
     [false, true].forEach(useLegacyIds => {
       describe(`renderFile() [using ${useLegacyIds ? 'legacy' : 'canonical'} ids]`, () => {
         it('should convert a set of parsed messages into an XML string', () => {
+          const phLocation: ɵSourceLocation = {
+            start: {line: 0, column: 10},
+            end: {line: 1, column: 15},
+            file: absoluteFrom('/project/file.ts'),
+            text: 'placeholder + 1'
+          };
+          const messagePartLocation: ɵSourceLocation = {
+            start: {line: 0, column: 5},
+            end: {line: 0, column: 10},
+            file: absoluteFrom('/project/file.ts'),
+            text: 'message part'
+          };
           const messages: ɵParsedMessage[] = [
             mockMessage('12345', ['a', 'b', 'c'], ['PH', 'PH_1'], {
               meaning: 'some meaning',
@@ -32,6 +43,8 @@ runInEachFileSystem(() => {
             mockMessage('54321', ['a', 'b', 'c'], ['PH', 'PH_1'], {
               customId: 'someId',
               legacyIds: ['87654321FEDCBA0987654321FEDCBA0987654321', '563965274788097516'],
+              messagePartLocations: [undefined, messagePartLocation, undefined],
+              substitutionLocations: {'PH': phLocation, 'PH_1': undefined},
             }),
             mockMessage('67890', ['a', '', 'c'], ['START_TAG_SPAN', 'CLOSE_TAG_SPAN'], {
               description: 'some description',
@@ -76,7 +89,7 @@ runInEachFileSystem(() => {
             `    </unit>`,
             `    <unit id="someId">`,
             `      <segment>`,
-            `        <source>a<ph id="0" equiv="PH"/>b<ph id="1" equiv="PH_1"/>c</source>`,
+            `        <source>a<ph id="0" equiv="PH" disp="placeholder + 1"/>b<ph id="1" equiv="PH_1"/>c</source>`,
             `      </segment>`,
             `    </unit>`,
             `    <unit id="67890">`,

packages/localize/src/tools/test/extract/translation_files/xmb_translation_serializer_spec.ts

@@ -7,7 +7,7 @@
  */
 import {absoluteFrom} from '@angular/compiler-cli/src/ngtsc/file_system';
 import {runInEachFileSystem} from '@angular/compiler-cli/src/ngtsc/file_system/testing';
-import {ɵParsedMessage} from '@angular/localize';
+import {ɵParsedMessage, ɵSourceLocation} from '@angular/localize';
 
 import {XmbTranslationSerializer} from '../../../src/extract/translation_files/xmb_translation_serializer';
 
@@ -18,6 +18,18 @@ runInEachFileSystem(() => {
     [false, true].forEach(useLegacyIds => {
       describe(`renderFile() [using ${useLegacyIds ? 'legacy' : 'canonical'} ids]`, () => {
         it('should convert a set of parsed messages into an XML string', () => {
+          const phLocation: ɵSourceLocation = {
+            start: {line: 0, column: 10},
+            end: {line: 1, column: 15},
+            file: absoluteFrom('/project/file.ts'),
+            text: 'placeholder + 1'
+          };
+          const messagePartLocation: ɵSourceLocation = {
+            start: {line: 0, column: 5},
+            end: {line: 0, column: 10},
+            file: absoluteFrom('/project/file.ts'),
+            text: 'message part'
+          };
           const messages: ɵParsedMessage[] = [
             mockMessage('12345', ['a', 'b', 'c'], ['PH', 'PH_1'], {
               meaning: 'some meaning',
@@ -26,6 +38,8 @@ runInEachFileSystem(() => {
             mockMessage('54321', ['a', 'b', 'c'], ['PH', 'PH_1'], {
               customId: 'someId',
               legacyIds: ['87654321FEDCBA0987654321FEDCBA0987654321', '563965274788097516'],
+              messagePartLocations: [undefined, messagePartLocation, undefined],
+              substitutionLocations: {'PH': phLocation, 'PH_1': undefined},
             }),
             mockMessage(
                 '67890', ['a', '', 'c'], ['START_TAG_SPAN', 'CLOSE_TAG_SPAN'],