fix(localize): render placeholder types in extracted XLIFF files (#39398)

The previous ViewEngine extraction tooling added `ctype` and `type`
attributes to XLIFF 1.2 and 2.0 translation files, respectively.

This commit adds this to the new $localize based extraction tooling.
Since the new extraction tooling works from the compiled output rather
than having direct access to the template content, the placeholder types
must be inferred from the name of the placeholder.  This is considered
reasonable, since it already does this to compute opening and closing
tag placeholders.

Fixes #38791

PR Close #39398

packages/localize/src/tools/src/extract/translation_files/xliff1_translation_serializer.ts

@@ -100,6 +100,10 @@ export class Xliff1TranslationSerializer implements TranslationSerializer {
 
   private serializePlaceholder(xml: XmlFile, id: string, text: string|undefined): void {
     const attrs: Record<string, string> = {id};
+    const ctype = getCtypeForPlaceholder(id);
+    if (ctype !== null) {
+      attrs.ctype = ctype;
+    }
     if (text !== undefined) {
       attrs['equiv-text'] = text;
     }
@@ -148,3 +152,68 @@ export class Xliff1TranslationSerializer implements TranslationSerializer {
         message.id;
   }
 }
+
+/**
+ * Compute the value of the `ctype` attribute from the `placeholder` name.
+ *
+ * The placeholder can take the following forms:
+ *
+ * - `START_BOLD_TEXT`/`END_BOLD_TEXT`
+ * - `TAG_<ELEMENT_NAME>`
+ * - `START_TAG_<ELEMENT_NAME>`
+ * - `CLOSE_TAG_<ELEMENT_NAME>`
+ *
+ * In these cases the element name of the tag is extracted from the placeholder name and returned as
+ * `x-<element_name>`.
+ *
+ * Line breaks and images are special cases.
+ */
+function getCtypeForPlaceholder(placeholder: string): string|null {
+  const tag = placeholder.replace(/^(START_|CLOSE_)/, '');
+  switch (tag) {
+    case 'LINE_BREAK':
+      return 'lb';
+    case 'TAG_IMG':
+      return 'image';
+    default:
+      const element = tag.startsWith('TAG_') ?
+          tag.replace(/^TAG_(.+)/, (_, tagName: string) => tagName.toLowerCase()) :
+          TAG_MAP[tag];
+      if (element === undefined) {
+        return null;
+      }
+      return `x-${element}`;
+  }
+}
+
+const TAG_MAP: Record<string, string> = {
+  'LINK': 'a',
+  'BOLD_TEXT': 'b',
+  'EMPHASISED_TEXT': 'em',
+  'HEADING_LEVEL1': 'h1',
+  'HEADING_LEVEL2': 'h2',
+  'HEADING_LEVEL3': 'h3',
+  'HEADING_LEVEL4': 'h4',
+  'HEADING_LEVEL5': 'h5',
+  'HEADING_LEVEL6': 'h6',
+  'HORIZONTAL_RULE': 'hr',
+  'ITALIC_TEXT': 'i',
+  'LIST_ITEM': 'li',
+  'MEDIA_LINK': 'link',
+  'ORDERED_LIST': 'ol',
+  'PARAGRAPH': 'p',
+  'QUOTATION': 'q',
+  'STRIKETHROUGH_TEXT': 's',
+  'SMALL_TEXT': 'small',
+  'SUBSTRIPT': 'sub',
+  'SUPERSCRIPT': 'sup',
+  'TABLE_BODY': 'tbody',
+  'TABLE_CELL': 'td',
+  'TABLE_FOOTER': 'tfoot',
+  'TABLE_HEADER_CELL': 'th',
+  'TABLE_HEADER': 'thead',
+  'TABLE_ROW': 'tr',
+  'MONOSPACED_TEXT': 'tt',
+  'UNDERLINED_TEXT': 'u',
+  'UNORDERED_LIST': 'ul',
+};

packages/localize/src/tools/src/extract/translation_files/xliff2_translation_serializer.ts

@@ -119,6 +119,10 @@ export class Xliff2TranslationSerializer implements TranslationSerializer {
         equivStart: placeholderName,
         equivEnd: closingPlaceholderName,
       };
+      const type = getTypeForPlaceholder(placeholderName);
+      if (type !== null) {
+        attrs.type = type;
+      }
       if (text !== undefined) {
         attrs.dispStart = text;
       }
@@ -129,8 +133,14 @@ export class Xliff2TranslationSerializer implements TranslationSerializer {
     } else if (placeholderName.startsWith('CLOSE_')) {
       xml.endTag('pc');
     } else {
-      const attrs:
+      const attrs: Record<string, string> = {
-          Record<string, string> = {id: `${this.currentPlaceholderId++}`, equiv: placeholderName};
+        id: `${this.currentPlaceholderId++}`,
+        equiv: placeholderName,
+      };
+      const type = getTypeForPlaceholder(placeholderName);
+      if (type !== null) {
+        attrs.type = type;
+      }
       if (text !== undefined) {
         attrs.disp = text;
       }
@@ -166,3 +176,29 @@ export class Xliff2TranslationSerializer implements TranslationSerializer {
         message.id;
   }
 }
+
+/**
+ * Compute the value of the `type` attribute from the `placeholder` name.
+ *
+ * If the tag is not known but starts with `TAG_`, `START_TAG_` or `CLOSE_TAG_` then the type is
+ * `other`. Certain formatting tags (e.g. bold, italic, etc) have type `fmt`. Line-breaks, images
+ * and links are special cases.
+ */
+function getTypeForPlaceholder(placeholder: string): string|null {
+  const tag = placeholder.replace(/^(START_|CLOSE_)/, '');
+  switch (tag) {
+    case 'BOLD_TEXT':
+    case 'EMPHASISED_TEXT':
+    case 'ITALIC_TEXT':
+    case 'LINE_BREAK':
+    case 'STRIKETHROUGH_TEXT':
+    case 'UNDERLINED_TEXT':
+      return 'fmt';
+    case 'TAG_IMG':
+      return 'image';
+    case 'LINK':
+      return 'link';
+    default:
+      return /^(START_|CLOSE_)/.test(placeholder) ? 'other' : null;
+  }
+}

packages/localize/src/tools/test/extract/integration/main_spec.ts

@@ -202,9 +202,9 @@ runInEachFileSystem(() => {
                  `        </context-group>`,
                  `      </trans-unit>`,
                  `      <trans-unit id="2932901491976224757" datatype="html">`,
-                 `        <source>pre<x id="START_TAG_SPAN" equiv-text="&apos;&lt;span&gt;&apos;"/>` +
+                 `        <source>pre<x id="START_TAG_SPAN" ctype="x-span" equiv-text="&apos;&lt;span&gt;&apos;"/>` +
-                     `inner-pre<x id="START_BOLD_TEXT" equiv-text="&apos;&lt;b&gt;&apos;"/>bold<x id="CLOSE_BOLD_TEXT" equiv-text="&apos;&lt;/b&gt;&apos;"/>` +
+                     `inner-pre<x id="START_BOLD_TEXT" ctype="x-b" equiv-text="&apos;&lt;b&gt;&apos;"/>bold<x id="CLOSE_BOLD_TEXT" ctype="x-b" equiv-text="&apos;&lt;/b&gt;&apos;"/>` +
-                     `inner-post<x id="CLOSE_TAG_SPAN" equiv-text="&apos;&lt;/span&gt;&apos;"/>post</source>`,
+                     `inner-post<x id="CLOSE_TAG_SPAN" ctype="x-span" equiv-text="&apos;&lt;/span&gt;&apos;"/>post</source>`,
                  `        <context-group purpose="location">`,
                  `          <context context-type="sourcefile">test_files/test.js</context>`,
                  `          <context context-type="linenumber">9,10</context>`,
@@ -279,8 +279,8 @@ runInEachFileSystem(() => {
               `        <note category="location">test_files/test.js:9,10</note>`,
               `      </notes>`,
               `      <segment>`,
-              `        <source>pre<pc id="0" equivStart="START_TAG_SPAN" equivEnd="CLOSE_TAG_SPAN" dispStart="&apos;&lt;span&gt;&apos;" dispEnd="&apos;&lt;/span&gt;&apos;">` +
+              `        <source>pre<pc id="0" equivStart="START_TAG_SPAN" equivEnd="CLOSE_TAG_SPAN" type="other" dispStart="&apos;&lt;span&gt;&apos;" dispEnd="&apos;&lt;/span&gt;&apos;">` +
-                  `inner-pre<pc id="1" equivStart="START_BOLD_TEXT" equivEnd="CLOSE_BOLD_TEXT" dispStart="&apos;&lt;b&gt;&apos;" dispEnd="&apos;&lt;/b&gt;&apos;">bold</pc>` +
+                  `inner-pre<pc id="1" equivStart="START_BOLD_TEXT" equivEnd="CLOSE_BOLD_TEXT" type="fmt" dispStart="&apos;&lt;b&gt;&apos;" dispEnd="&apos;&lt;/b&gt;&apos;">bold</pc>` +
                   `inner-post</pc>post</source>`,
               `      </segment>`,
               `    </unit>`,

packages/localize/src/tools/test/extract/translation_files/xliff1_translation_serializer_spec.ts

@@ -99,18 +99,18 @@ runInEachFileSystem(() => {
               `        <source>a<x id="PH" equiv-text="placeholder + 1"/>b<x id="PH_1"/>c</source>`,
               `      </trans-unit>`,
               `      <trans-unit id="67890" datatype="html">`,
-              `        <source>a<x id="START_TAG_SPAN"/><x id="CLOSE_TAG_SPAN"/>c</source>`,
+              `        <source>a<x id="START_TAG_SPAN" ctype="x-span"/><x id="CLOSE_TAG_SPAN" ctype="x-span"/>c</source>`,
               `        <note priority="1" from="description">some description</note>`,
               `      </trans-unit>`,
               `      <trans-unit id="38705" datatype="html">`,
-              `        <source>a<x id="START_TAG_SPAN"/><x id="CLOSE_TAG_SPAN"/>c</source>`,
+              `        <source>a<x id="START_TAG_SPAN" ctype="x-span"/><x id="CLOSE_TAG_SPAN" ctype="x-span"/>c</source>`,
               `        <context-group purpose="location">`,
               `          <context context-type="sourcefile">file.ts</context>`,
               `          <context context-type="linenumber">3,4</context>`,
               `        </context-group>`,
               `      </trans-unit>`,
               `      <trans-unit id="13579" datatype="html">`,
-              `        <source><x id="START_BOLD_TEXT"/>b<x id="CLOSE_BOLD_TEXT"/></source>`,
+              `        <source><x id="START_BOLD_TEXT" ctype="x-b"/>b<x id="CLOSE_BOLD_TEXT" ctype="x-b"/></source>`,
               `      </trans-unit>`,
               `      <trans-unit id="24680" datatype="html">`,
               `        <source>a</source>`,
@@ -128,7 +128,7 @@ runInEachFileSystem(() => {
               `        <source>pre-ICU {VAR_SELECT, select, a {a} b {<x id="INTERPOLATION"/>} c {pre <x id="INTERPOLATION_1"/> post}} post-ICU</source>`,
               `      </trans-unit>`,
               `      <trans-unit id="100001" datatype="html">`,
-              `        <source>{VAR_PLURAL, plural, one {<x id="START_BOLD_TEXT"/>something bold<x id="CLOSE_BOLD_TEXT"/>} other {pre <x id="START_TAG_SPAN"/>middle<x id="CLOSE_TAG_SPAN"/> post}}</source>`,
+              `        <source>{VAR_PLURAL, plural, one {<x id="START_BOLD_TEXT" ctype="x-b"/>something bold<x id="CLOSE_BOLD_TEXT" ctype="x-b"/>} other {pre <x id="START_TAG_SPAN" ctype="x-span"/>middle<x id="CLOSE_TAG_SPAN" ctype="x-span"/> post}}</source>`,
               `      </trans-unit>`,
               `    </body>`,
               `  </file>`,
@@ -230,6 +230,64 @@ runInEachFileSystem(() => {
               `</xliff>\n`,
             ].join('\n'));
           });
+
+          it('should render the "ctype" for line breaks', () => {
+            const serializer = new Xliff1TranslationSerializer(
+                'xx', absoluteFrom('/project'), useLegacyIds, options);
+            const output = serializer.serialize([mockMessage('1', ['a', 'b'], ['LINE_BREAK'], {})]);
+            expect(output).toContain(
+                '<source>a<x id="LINE_BREAK" ctype="lb"/>b</source>',
+            );
+          });
+
+          it('should render the "ctype" for images', () => {
+            const serializer = new Xliff1TranslationSerializer(
+                'xx', absoluteFrom('/project'), useLegacyIds, options);
+            const output = serializer.serialize([mockMessage('2', ['a', 'b'], ['TAG_IMG'], {})]);
+            expect(output).toContain(
+                '<source>a<x id="TAG_IMG" ctype="image"/>b</source>',
+            );
+          });
+
+          it('should render the "ctype" for bold elements', () => {
+            const serializer = new Xliff1TranslationSerializer(
+                'xx', absoluteFrom('/project'), useLegacyIds, options);
+            const output = serializer.serialize(
+                [mockMessage('3', ['a', 'b', 'c'], ['START_BOLD_TEXT', 'CLOSE_BOLD_TEXT'], {})]);
+            expect(output).toContain(
+                '<source>a<x id="START_BOLD_TEXT" ctype="x-b"/>b<x id="CLOSE_BOLD_TEXT" ctype="x-b"/>c</source>',
+            );
+          });
+
+          it('should render the "ctype" for headings', () => {
+            const serializer = new Xliff1TranslationSerializer(
+                'xx', absoluteFrom('/project'), useLegacyIds, options);
+            const output = serializer.serialize([mockMessage(
+                '4', ['a', 'b', 'c'], ['START_HEADING_LEVEL1', 'CLOSE_HEADING_LEVEL1'], {})]);
+            expect(output).toContain(
+                '<source>a<x id="START_HEADING_LEVEL1" ctype="x-h1"/>b<x id="CLOSE_HEADING_LEVEL1" ctype="x-h1"/>c</source>',
+            );
+          });
+
+          it('should render the "ctype" for span elements', () => {
+            const serializer = new Xliff1TranslationSerializer(
+                'xx', absoluteFrom('/project'), useLegacyIds, options);
+            const output = serializer.serialize(
+                [mockMessage('5', ['a', 'b', 'c'], ['START_TAG_SPAN', 'CLOSE_TAG_SPAN'], {})]);
+            expect(output).toContain(
+                '<source>a<x id="START_TAG_SPAN" ctype="x-span"/>b<x id="CLOSE_TAG_SPAN" ctype="x-span"/>c</source>',
+            );
+          });
+
+          it('should render the "ctype" for div elements', () => {
+            const serializer = new Xliff1TranslationSerializer(
+                'xx', absoluteFrom('/project'), useLegacyIds, options);
+            const output = serializer.serialize(
+                [mockMessage('6', ['a', 'b', 'c'], ['START_TAG_DIV', 'CLOSE_TAG_DIV'], {})]);
+            expect(output).toContain(
+                '<source>a<x id="START_TAG_DIV" ctype="x-div"/>b<x id="CLOSE_TAG_DIV" ctype="x-div"/>c</source>',
+            );
+          });
         });
       });
     });

packages/localize/src/tools/test/extract/translation_files/xliff2_translation_serializer_spec.ts

@@ -108,7 +108,7 @@ runInEachFileSystem(() => {
               `        <note category="description">some description</note>`,
               `      </notes>`,
               `      <segment>`,
-              `        <source>a<pc id="0" equivStart="START_TAG_SPAN" equivEnd="CLOSE_TAG_SPAN"></pc>c</source>`,
+              `        <source>a<pc id="0" equivStart="START_TAG_SPAN" equivEnd="CLOSE_TAG_SPAN" type="other"></pc>c</source>`,
               `      </segment>`,
               `    </unit>`,
               `    <unit id="location-only">`,
@@ -116,12 +116,12 @@ runInEachFileSystem(() => {
               `        <note category="location">file.ts:3,4</note>`,
               `      </notes>`,
               `      <segment>`,
-              `        <source>a<pc id="0" equivStart="START_TAG_SPAN" equivEnd="CLOSE_TAG_SPAN"></pc>c</source>`,
+              `        <source>a<pc id="0" equivStart="START_TAG_SPAN" equivEnd="CLOSE_TAG_SPAN" type="other"></pc>c</source>`,
               `      </segment>`,
               `    </unit>`,
               `    <unit id="13579">`,
               `      <segment>`,
-              `        <source><pc id="0" equivStart="START_BOLD_TEXT" equivEnd="CLOSE_BOLD_TEXT">b</pc></source>`,
+              `        <source><pc id="0" equivStart="START_BOLD_TEXT" equivEnd="CLOSE_BOLD_TEXT" type="fmt">b</pc></source>`,
               `      </segment>`,
               `    </unit>`,
               `    <unit id="24680">`,
@@ -151,7 +151,7 @@ runInEachFileSystem(() => {
               `    </unit>`,
               `    <unit id="100001">`,
               `      <segment>`,
-              `        <source>{VAR_PLURAL, plural, one {<pc id="0" equivStart="START_BOLD_TEXT" equivEnd="CLOSE_BOLD_TEXT">something bold</pc>} other {pre <pc id="1" equivStart="START_TAG_SPAN" equivEnd="CLOSE_TAG_SPAN">middle</pc> post}}</source>`,
+              `        <source>{VAR_PLURAL, plural, one {<pc id="0" equivStart="START_BOLD_TEXT" equivEnd="CLOSE_BOLD_TEXT" type="fmt">something bold</pc>} other {pre <pc id="1" equivStart="START_TAG_SPAN" equivEnd="CLOSE_TAG_SPAN" type="other">middle</pc> post}}</source>`,
               `      </segment>`,
               `    </unit>`,
               `  </file>`,
@@ -243,6 +243,64 @@ runInEachFileSystem(() => {
               `</xliff>\n`,
             ].join('\n'));
           });
+
+          it('should render the "type" for line breaks', () => {
+            const serializer = new Xliff2TranslationSerializer(
+                'xx', absoluteFrom('/project'), useLegacyIds, options);
+            const output = serializer.serialize([mockMessage('1', ['a', 'b'], ['LINE_BREAK'], {})]);
+            expect(output).toContain(
+                '<source>a<ph id="0" equiv="LINE_BREAK" type="fmt"/>b</source>',
+            );
+          });
+
+          it('should render the "type" for images', () => {
+            const serializer = new Xliff2TranslationSerializer(
+                'xx', absoluteFrom('/project'), useLegacyIds, options);
+            const output = serializer.serialize([mockMessage('2', ['a', 'b'], ['TAG_IMG'], {})]);
+            expect(output).toContain(
+                '<source>a<ph id="0" equiv="TAG_IMG" type="image"/>b</source>',
+            );
+          });
+
+          it('should render the "type" for bold elements', () => {
+            const serializer = new Xliff2TranslationSerializer(
+                'xx', absoluteFrom('/project'), useLegacyIds, options);
+            const output = serializer.serialize(
+                [mockMessage('3', ['a', 'b', 'c'], ['START_BOLD_TEXT', 'CLOSE_BOLD_TEXT'], {})]);
+            expect(output).toContain(
+                '<source>a<pc id="0" equivStart="START_BOLD_TEXT" equivEnd="CLOSE_BOLD_TEXT" type="fmt">b</pc>c</source>',
+            );
+          });
+
+          it('should render the "type" for heading elements', () => {
+            const serializer = new Xliff2TranslationSerializer(
+                'xx', absoluteFrom('/project'), useLegacyIds, options);
+            const output = serializer.serialize([mockMessage(
+                '4', ['a', 'b', 'c'], ['START_HEADING_LEVEL1', 'CLOSE_HEADING_LEVEL1'], {})]);
+            expect(output).toContain(
+                '<source>a<pc id="0" equivStart="START_HEADING_LEVEL1" equivEnd="CLOSE_HEADING_LEVEL1" type="other">b</pc>c</source>',
+            );
+          });
+
+          it('should render the "type" for span elements', () => {
+            const serializer = new Xliff2TranslationSerializer(
+                'xx', absoluteFrom('/project'), useLegacyIds, options);
+            const output = serializer.serialize(
+                [mockMessage('5', ['a', 'b', 'c'], ['START_TAG_SPAN', 'CLOSE_TAG_SPAN'], {})]);
+            expect(output).toContain(
+                '<source>a<pc id="0" equivStart="START_TAG_SPAN" equivEnd="CLOSE_TAG_SPAN" type="other">b</pc>c</source>',
+            );
+          });
+
+          it('should render the "type" for div elements', () => {
+            const serializer = new Xliff2TranslationSerializer(
+                'xx', absoluteFrom('/project'), useLegacyIds, options);
+            const output = serializer.serialize(
+                [mockMessage('6', ['a', 'b', 'c'], ['START_TAG_DIV', 'CLOSE_TAG_DIV'], {})]);
+            expect(output).toContain(
+                '<source>a<pc id="0" equivStart="START_TAG_DIV" equivEnd="CLOSE_TAG_DIV" type="other">b</pc>c</source>',
+            );
+          });
         });
       });
     });