fix(localize): merge translation from all XLIFF `<file>` elements (#35936)

XLIFF translation files can contain multiple `<file>` elements, each of which contains translations. In ViewEngine all these translations are merged into a single translation bundle. Previously in Ivy only the translations from the last `<file>` element were being loaded. Now all the translations from each `<file>` are merged into a single translation bundle. Fixes #35839 PR Close #35936
2020-03-08 12:30:52 +00:00 · 2020-03-08 12:30:52 +00:00 · fc4c3c3eb5
parent 1882451ec0
commit fc4c3c3eb5
4 changed files with 283 additions and 35 deletions
--- a/packages/localize/src/tools/src/translate/translation_files/translation_parsers/xliff1_translation_parser.ts
+++ b/packages/localize/src/tools/src/translate/translation_files/translation_parsers/xliff1_translation_parser.ts
@ -59,12 +59,24 @@ export class Xliff1TranslationParser implements TranslationParser<XmlTranslation
          ParseErrorLevel.WARNING);
    }

-    const bundle = {
-      locale: getAttribute(files[0], 'target-language'),
-      translations: {}, diagnostics,
-    };
+    const bundle: ParsedTranslationBundle = {locale: undefined, translations: {}, diagnostics};
    const translationVisitor = new XliffTranslationVisitor();
-    visitAll(translationVisitor, files[0].children, bundle);
+    const localesFound = new Set<string>();
+    for (const file of files) {
+      const locale = getAttribute(file, 'target-language');
+      if (locale !== undefined) {
+        localesFound.add(locale);
+        bundle.locale = locale;
+      }
+      visitAll(translationVisitor, file.children, bundle);
+    }
+
+    if (localesFound.size > 1) {
+      addParseDiagnostic(
+          diagnostics, element.sourceSpan,
+          `More than one locale found in translation file: ${JSON.stringify(Array.from(localesFound))}. Using "${bundle.locale}"`,
+          ParseErrorLevel.WARNING);
+    }

    return bundle;
  }
--- a/packages/localize/src/tools/src/translate/translation_files/translation_parsers/xliff2_translation_parser.ts
+++ b/packages/localize/src/tools/src/translate/translation_files/translation_parsers/xliff2_translation_parser.ts
@ -40,13 +40,6 @@ export class Xliff2TranslationParser implements TranslationParser<XmlTranslation
    const diagnostics = new Diagnostics();
    errors.forEach(e => addParseError(diagnostics, e));

-    if (element.children.length === 0) {
-      addParseDiagnostic(
-          diagnostics, element.sourceSpan, 'Missing expected <file> element',
-          ParseErrorLevel.WARNING);
-      return {locale: undefined, translations: {}, diagnostics};
-    }
-
    const locale = getAttribute(element, 'trgLang');
    const files = element.children.filter(isFileElement);
    if (files.length === 0) {
@ -61,8 +54,9 @@ export class Xliff2TranslationParser implements TranslationParser<XmlTranslation

    const bundle = {locale, translations: {}, diagnostics};
    const translationVisitor = new Xliff2TranslationVisitor();
-    visitAll(translationVisitor, files[0].children, {bundle});
-
+    for (const file of files) {
+      visitAll(translationVisitor, file.children, {bundle});
+    }
    return bundle;
  }

--- a/packages/localize/src/tools/test/translate/translation_files/translation_parsers/xliff1_translation_parser_spec.ts
+++ b/packages/localize/src/tools/test/translate/translation_files/translation_parsers/xliff1_translation_parser_spec.ts
@ -31,17 +31,33 @@ describe('Xliff1TranslationParser', () => {
  });

  describe('parse() [without hint]', () => {
-    it('should extract the locale from the file contents', () => {
+    it('should extract the locale from the last `<file>` element to contain a `target-language` attribute',
+       () => {
         const XLIFF = `
      <xliff version="1.2" xmlns="urn:oasis:names:tc:xliff:document:1.2">
+        <file source-language="en" datatype="plaintext" original="ng2.template">
+          <body></body>
+        </file>
        <file source-language="en" target-language="fr" datatype="plaintext" original="ng2.template">
-          <body>
-          </body>
+          <body></body>
+        </file>
+        <file source-language="en" datatype="plaintext" original="ng2.template">
+          <body></body>
+        </file>
+        <file source-language="en" target-language="de" datatype="plaintext" original="ng2.template">
+          <body></body>
+        </file>
+        <file source-language="en" datatype="plaintext" original="ng2.template">
+          <body></body>
        </file>
      </xliff>`;
         const parser = new Xliff1TranslationParser();
-      const result = parser.parse('/some/file.xlf', XLIFF);
-      expect(result.locale).toEqual('fr');
+         const hint = parser.canParse('/some/file.xlf', XLIFF);
+         if (!hint) {
+           return fail('expected XLIFF to be valid');
+         }
+         const result = parser.parse('/some/file.xlf', XLIFF, hint);
+         expect(result.locale).toEqual('de');
       });

    it('should return an undefined locale if there is no locale in the file', () => {
@ -437,6 +453,58 @@ describe('Xliff1TranslationParser', () => {
          .toEqual(ɵmakeParsedTranslation(['Weiter']));
    });

+    it('should merge messages from each `<file>` element', () => {
+      /**
+       * Source HTML:
+       *
+       * ```
+       * <div i18n>translatable attribute</div>
+       * ```
+
+       * ```
+       * <div i18n>translatable element <b>with placeholders</b> {{ interpolation}}</div>
+       * ```
+       */
+      const XLIFF = `
+      <xliff version="1.2" xmlns="urn:oasis:names:tc:xliff:document:1.2">
+        <file source-language="en" target-language="fr" datatype="plaintext" original="file-1">
+          <body>
+            <trans-unit id="1933478729560469763" datatype="html">
+              <source>translatable attribute</source>
+              <target>etubirtta elbatalsnart</target>
+              <context-group purpose="location">
+                <context context-type="sourcefile">file.ts</context>
+                <context context-type="linenumber">1</context>
+              </context-group>
+            </trans-unit>
+          </body>
+        </file>
+        <file source-language="en" target-language="fr" datatype="plaintext" original="file-2">
+          <body>
+            <trans-unit id="5057824347511785081" datatype="html">
+              <source>translatable element <x id="START_BOLD_TEXT" ctype="b"/>with placeholders<x id="CLOSE_BOLD_TEXT" ctype="b"/> <x id="INTERPOLATION"/></source>
+              <target><x id="INTERPOLATION"/> tnemele elbatalsnart <x id="START_BOLD_TEXT" ctype="x-b"/>sredlohecalp htiw<x id="CLOSE_BOLD_TEXT" ctype="x-b"/></target>
+              <context-group purpose="location">
+                <context context-type="sourcefile">file.ts</context>
+                <context context-type="linenumber">2</context>
+              </context-group>
+            </trans-unit>
+          </body>
+        </file>
+      </xliff>`;
+      const parser = new Xliff1TranslationParser();
+      const result = parser.parse('/some/file.xlf', XLIFF);
+
+      expect(result.translations[ɵcomputeMsgId('translatable attribute')])
+          .toEqual(ɵmakeParsedTranslation(['etubirtta elbatalsnart']));
+      expect(
+          result.translations[ɵcomputeMsgId(
+              'translatable element {$START_BOLD_TEXT}with placeholders{$LOSE_BOLD_TEXT} {$INTERPOLATION}')])
+          .toEqual(ɵmakeParsedTranslation(
+              ['', ' tnemele elbatalsnart ', 'sredlohecalp htiw', ''],
+              ['INTERPOLATION', 'START_BOLD_TEXT', 'CLOSE_BOLD_TEXT']));
+    });
+
    describe('[structure errors]', () => {
      it('should throw when a trans-unit has no translation', () => {
        const XLIFF = `
@ -547,12 +615,24 @@ describe('Xliff1TranslationParser', () => {
  });

  describe('parse() [with hint]', () => {
-    it('should extract the locale from the file contents', () => {
+    it('should extract the locale from the last `<file>` element to contain a `target-language` attribute',
+       () => {
         const XLIFF = `
      <xliff version="1.2" xmlns="urn:oasis:names:tc:xliff:document:1.2">
+        <file source-language="en" datatype="plaintext" original="ng2.template">
+          <body></body>
+        </file>
        <file source-language="en" target-language="fr" datatype="plaintext" original="ng2.template">
-          <body>
-          </body>
+          <body></body>
+        </file>
+        <file source-language="en" datatype="plaintext" original="ng2.template">
+          <body></body>
+        </file>
+        <file source-language="en" target-language="de" datatype="plaintext" original="ng2.template">
+          <body></body>
+        </file>
+        <file source-language="en" datatype="plaintext" original="ng2.template">
+          <body></body>
        </file>
      </xliff>`;
         const parser = new Xliff1TranslationParser();
@ -561,7 +641,7 @@ describe('Xliff1TranslationParser', () => {
           return fail('expected XLIFF to be valid');
         }
         const result = parser.parse('/some/file.xlf', XLIFF, hint);
-      expect(result.locale).toEqual('fr');
+         expect(result.locale).toEqual('de');
       });

    it('should return an undefined locale if there is no locale in the file', () => {
@ -1005,6 +1085,62 @@ describe('Xliff1TranslationParser', () => {
          .toEqual(ɵmakeParsedTranslation(['Weiter']));
    });

+    it('should merge messages from each `<file>` element', () => {
+      /**
+       * Source HTML:
+       *
+       * ```
+       * <div i18n>translatable attribute</div>
+       * ```
+       *
+       * ```
+       * <div i18n>translatable element <b>with placeholders</b> {{ interpolation}}</div>
+       * ```
+       */
+      const XLIFF = `
+      <xliff version="1.2" xmlns="urn:oasis:names:tc:xliff:document:1.2">
+        <file source-language="en" target-language="fr" datatype="plaintext" original="file-1">
+          <body>
+            <trans-unit id="1933478729560469763" datatype="html">
+              <source>translatable attribute</source>
+              <target>etubirtta elbatalsnart</target>
+              <context-group purpose="location">
+                <context context-type="sourcefile">file.ts</context>
+                <context context-type="linenumber">1</context>
+              </context-group>
+            </trans-unit>
+          </body>
+        </file>
+        <file source-language="en" target-language="fr" datatype="plaintext" original="file-2">
+          <body>
+            <trans-unit id="5057824347511785081" datatype="html">
+              <source>translatable element <x id="START_BOLD_TEXT" ctype="b"/>with placeholders<x id="CLOSE_BOLD_TEXT" ctype="b"/> <x id="INTERPOLATION"/></source>
+              <target><x id="INTERPOLATION"/> tnemele elbatalsnart <x id="START_BOLD_TEXT" ctype="x-b"/>sredlohecalp htiw<x id="CLOSE_BOLD_TEXT" ctype="x-b"/></target>
+              <context-group purpose="location">
+                <context context-type="sourcefile">file.ts</context>
+                <context context-type="linenumber">2</context>
+              </context-group>
+            </trans-unit>
+          </body>
+        </file>
+      </xliff>`;
+      const parser = new Xliff1TranslationParser();
+      const hint = parser.canParse('/some/file.xlf', XLIFF);
+      if (!hint) {
+        return fail('expected XLIFF to be valid');
+      }
+      const result = parser.parse('/some/file.xlf', XLIFF, hint);
+
+      expect(result.translations[ɵcomputeMsgId('translatable attribute')])
+          .toEqual(ɵmakeParsedTranslation(['etubirtta elbatalsnart']));
+      expect(
+          result.translations[ɵcomputeMsgId(
+              'translatable element {$START_BOLD_TEXT}with placeholders{$LOSE_BOLD_TEXT} {$INTERPOLATION}')])
+          .toEqual(ɵmakeParsedTranslation(
+              ['', ' tnemele elbatalsnart ', 'sredlohecalp htiw', ''],
+              ['INTERPOLATION', 'START_BOLD_TEXT', 'CLOSE_BOLD_TEXT']));
+    });
+
    describe('[structure errors]', () => {
      it('should provide a diagnostic error when a trans-unit has no translation', () => {
        const XLIFF = `
--- a/packages/localize/src/tools/test/translate/translation_files/translation_parsers/xliff2_translation_parser_spec.ts
+++ b/packages/localize/src/tools/test/translate/translation_files/translation_parsers/xliff2_translation_parser_spec.ts
@ -87,7 +87,7 @@ describe('Xliff2TranslationParser', () => {
       * Source HTML:
       *
       * ```
-       * <div i18n>translatable element <b>>with placeholders</b> {{ interpolation}}</div>
+       * <div i18n>translatable element <b>with placeholders</b> {{ interpolation}}</div>
       * ```
       */
      const XLIFF = `
@ -373,6 +373,57 @@ describe('Xliff2TranslationParser', () => {
          .toEqual(ɵmakeParsedTranslation(['Translated first sentence.']));
    });

+    it('should merge messages from each `<file>` element', () => {
+      /**
+       * Source HTML:
+       *
+       * ```
+       * <div i18n>translatable attribute</div>
+       * ```
+       *
+       * ```
+       * <div i18n>translatable element <b>with placeholders</b> {{ interpolation}}</div>
+       * ```
+       */
+      const XLIFF = `
+      <xliff version="2.0" xmlns="urn:oasis:names:tc:xliff:document:2.0" srcLang="en" trgLang="fr">
+        <file original="ng.template" id="file-1">
+          <unit id="1933478729560469763">
+            <notes>
+              <note category="location">file.ts:2</note>
+            </notes>
+            <segment>
+              <source>translatable attribute</source>
+              <target>etubirtta elbatalsnart</target>
+            </segment>
+          </unit>
+        </file>
+        <file original="ng.template" id="file-2">
+          <unit id="5057824347511785081">
+            <notes>
+              <note category="location">file.ts:3</note>
+            </notes>
+            <segment>
+              <source>translatable element <pc id="0" equivStart="START_BOLD_TEXT" equivEnd="CLOSE_BOLD_TEXT" type="fmt" dispStart="&lt;b&gt;" dispEnd="&lt;/b&gt;">with placeholders</pc> <ph id="1" equiv="INTERPOLATION" disp="{{ interpolation}}"/></source>
+              <target><ph id="1" equiv="INTERPOLATION" disp="{{ interpolation}}"/> tnemele elbatalsnart <pc id="0" equivStart="START_BOLD_TEXT" equivEnd="CLOSE_BOLD_TEXT" type="fmt" dispStart="&lt;b&gt;" dispEnd="&lt;/b&gt;">sredlohecalp htiw</pc></target>
+            </segment>
+          </unit>
+        </file>
+      </xliff>`;
+      const parser = new Xliff2TranslationParser();
+      const result = parser.parse('/some/file.xlf', XLIFF);
+
+      expect(result.translations[ɵcomputeMsgId('translatable attribute', '')])
+          .toEqual(ɵmakeParsedTranslation(['etubirtta elbatalsnart']));
+      expect(
+          result.translations[ɵcomputeMsgId(
+              'translatable element {$START_BOLD_TEXT}with placeholders{$LOSE_BOLD_TEXT} {$INTERPOLATION}')])
+          .toEqual(ɵmakeParsedTranslation(
+              ['', ' tnemele elbatalsnart ', 'sredlohecalp htiw', ''],
+              ['INTERPOLATION', 'START_BOLD_TEXT', 'CLOSE_BOLD_TEXT']));
+
+    });
+
    describe('[structure errors]', () => {
      it('should throw when a trans-unit has no translation', () => {
        const XLIFF = `<?xml version="1.0" encoding="UTF-8" ?>
@ -866,6 +917,61 @@ describe('Xliff2TranslationParser', () => {
          .toEqual(ɵmakeParsedTranslation(['Translated first sentence.']));
    });

+    it('should merge messages from each `<file>` element', () => {
+      /**
+       * Source HTML:
+       *
+       * ```
+       * <div i18n>translatable attribute</div>
+       * ```
+       *
+       * ```
+       * <div i18n>translatable element <b>with placeholders</b> {{ interpolation}}</div>
+       * ```
+       */
+      const XLIFF = `
+      <xliff version="2.0" xmlns="urn:oasis:names:tc:xliff:document:2.0" srcLang="en" trgLang="fr">
+        <file original="ng.template" id="file-1">
+          <unit id="1933478729560469763">
+            <notes>
+              <note category="location">file.ts:2</note>
+            </notes>
+            <segment>
+              <source>translatable attribute</source>
+              <target>etubirtta elbatalsnart</target>
+            </segment>
+          </unit>
+        </file>
+        <file original="ng.template" id="file-2">
+          <unit id="5057824347511785081">
+            <notes>
+              <note category="location">file.ts:3</note>
+            </notes>
+            <segment>
+              <source>translatable element <pc id="0" equivStart="START_BOLD_TEXT" equivEnd="CLOSE_BOLD_TEXT" type="fmt" dispStart="&lt;b&gt;" dispEnd="&lt;/b&gt;">with placeholders</pc> <ph id="1" equiv="INTERPOLATION" disp="{{ interpolation}}"/></source>
+              <target><ph id="1" equiv="INTERPOLATION" disp="{{ interpolation}}"/> tnemele elbatalsnart <pc id="0" equivStart="START_BOLD_TEXT" equivEnd="CLOSE_BOLD_TEXT" type="fmt" dispStart="&lt;b&gt;" dispEnd="&lt;/b&gt;">sredlohecalp htiw</pc></target>
+            </segment>
+          </unit>
+        </file>
+      </xliff>`;
+      const parser = new Xliff2TranslationParser();
+      const hint = parser.canParse('/some/file.xlf', XLIFF);
+      if (!hint) {
+        return fail('expected XLIFF to be valid');
+      }
+      const result = parser.parse('/some/file.xlf', XLIFF, hint);
+
+      expect(result.translations[ɵcomputeMsgId('translatable attribute', '')])
+          .toEqual(ɵmakeParsedTranslation(['etubirtta elbatalsnart']));
+      expect(
+          result.translations[ɵcomputeMsgId(
+              'translatable element {$START_BOLD_TEXT}with placeholders{$LOSE_BOLD_TEXT} {$INTERPOLATION}')])
+          .toEqual(ɵmakeParsedTranslation(
+              ['', ' tnemele elbatalsnart ', 'sredlohecalp htiw', ''],
+              ['INTERPOLATION', 'START_BOLD_TEXT', 'CLOSE_BOLD_TEXT']));
+
+    });
+
    describe('[structure errors]', () => {
      it('should provide a diagnostic error when a trans-unit has no translation', () => {
        const XLIFF = `<?xml version="1.0" encoding="UTF-8" ?>