feat(localize): support Application Resource Bundle (ARB) translation file format (#36795)

The ARB format is a JSON file containing an object where the keys are the
message ids and the values are the translations.

It is extensible because it can also contain metadata about each message.

For example:

```
{
  "@@locale": "...",
  "message-id": "Translated message string",
  "@message-id": {
    "type": "text",
    "description": "Some description text",
    "x-locations": [{ "start": {"line": 23, "column": 145}, "file": "some/file.ts" }]
  },
}
```

For more information, see:
https://github.com/google/app-resource-bundle/wiki/ApplicationResourceBundleSpecification

PR Close #36795

packages/localize/src/tools/src/extract/main.ts

@@ -17,6 +17,7 @@ import {DiagnosticHandlingStrategy} from '../diagnostics';
 import {checkDuplicateMessages} from './duplicates';
 import {MessageExtractor} from './extraction';
 import {TranslationSerializer} from './translation_files/translation_serializer';
+import {ArbTranslationSerializer} from './translation_files/arb_translation_serializer';
 import {SimpleJsonTranslationSerializer} from './translation_files/json_translation_serializer';
 import {Xliff1TranslationSerializer} from './translation_files/xliff1_translation_serializer';
 import {Xliff2TranslationSerializer} from './translation_files/xliff2_translation_serializer';
@@ -229,6 +230,8 @@ export function getSerializer(
       return new XmbTranslationSerializer(rootPath, useLegacyIds, fs);
     case 'json':
       return new SimpleJsonTranslationSerializer(sourceLocale);
+    case 'arb':
+      return new ArbTranslationSerializer(sourceLocale, rootPath, fs);
   }
   throw new Error(`No translation serializer can handle the provided format: ${format}`);
 }

packages/localize/src/tools/src/extract/translation_files/arb_translation_serializer.ts

@@ -0,0 +1,100 @@
+/**
+ * @license
+ * Copyright Google LLC All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+import {AbsoluteFsPath, FileSystem} from '@angular/compiler-cli/src/ngtsc/file_system';
+import {ɵParsedMessage, ɵSourceLocation} from '@angular/localize';
+import {ArbJsonObject, ArbLocation, ArbMetadata} from '../../translate/translation_files/translation_parsers/arb_translation_parser';
+import {TranslationSerializer} from './translation_serializer';
+import {consolidateMessages, hasLocation} from './utils';
+
+/**
+ * A translation serializer that can render JSON formatted as an Application Resource Bundle (ARB).
+ *
+ * See https://github.com/google/app-resource-bundle/wiki/ApplicationResourceBundleSpecification
+ *
+ * ```
+ * {
+ *   "@@locale": "en-US",
+ *   "message-id": "Target message string",
+ *   "@message-id": {
+ *     "type": "text",
+ *     "description": "Some description text",
+ *     "x-locations": [
+ *       {
+ *         "start": {"line": 23, "column": 145},
+ *         "end": {"line": 24, "column": 53},
+ *         "file": "some/file.ts"
+ *       },
+ *       ...
+ *     ]
+ *   },
+ *   ...
+ * }
+ * ```
+ */
+
+/**
+ * This is a semi-public bespoke serialization format that is used for testing and sometimes as a
+ * format for storing translations that will be inlined at runtime.
+ *
+ * @see ArbTranslationParser
+ */
+export class ArbTranslationSerializer implements TranslationSerializer {
+  constructor(
+      private sourceLocale: string, private basePath: AbsoluteFsPath, private fs: FileSystem) {}
+
+  serialize(messages: ɵParsedMessage[]): string {
+    const messageMap = consolidateMessages(messages, message => message.customId || message.id);
+
+    let output = `{\n  "@@locale": ${JSON.stringify(this.sourceLocale)}`;
+
+    for (const [id, duplicateMessages] of messageMap.entries()) {
+      const message = duplicateMessages[0];
+      output += this.serializeMessage(id, message);
+      output += this.serializeMeta(
+          id, message.description, duplicateMessages.filter(hasLocation).map(m => m.location));
+    }
+
+    output += '\n}';
+
+    return output;
+  }
+
+  private serializeMessage(id: string, message: ɵParsedMessage): string {
+    return `,\n  ${JSON.stringify(id)}: ${JSON.stringify(message.text)}`;
+  }
+
+  private serializeMeta(id: string, description: string|undefined, locations: ɵSourceLocation[]):
+      string {
+    const meta: string[] = [];
+
+    if (description) {
+      meta.push(`\n    "description": ${JSON.stringify(description)}`);
+    }
+
+    if (locations.length > 0) {
+      let locationStr = `\n    "x-locations": [`;
+      for (let i = 0; i < locations.length; i++) {
+        locationStr += (i > 0 ? ',\n' : '\n') + this.serializeLocation(locations[i]);
+      }
+      locationStr += '\n    ]';
+      meta.push(locationStr);
+    }
+
+    return meta.length > 0 ? `,\n  ${JSON.stringify('@' + id)}: {${meta.join(',')}\n  }` : '';
+  }
+
+  private serializeLocation({file, start, end}: ɵSourceLocation): string {
+    return [
+      `      {`,
+      `        "file": ${JSON.stringify(this.fs.relative(this.basePath, file))},`,
+      `        "start": { "line": "${start.line}", "column": "${start.column}" },`,
+      `        "end": { "line": "${end.line}", "column": "${end.column}" }`,
+      `      }`,
+    ].join('\n');
+  }
+}

packages/localize/src/tools/src/translate/main.ts

@@ -15,6 +15,7 @@ import {AssetTranslationHandler} from './asset_files/asset_translation_handler';
 import {getOutputPathFn, OutputPathFn} from './output_path';
 import {SourceFileTranslationHandler} from './source_files/source_file_translation_handler';
 import {TranslationLoader} from './translation_files/translation_loader';
+import {ArbTranslationParser} from './translation_files/translation_parsers/arb_translation_parser';
 import {SimpleJsonTranslationParser} from './translation_files/translation_parsers/simple_json_translation_parser';
 import {Xliff1TranslationParser} from './translation_files/translation_parsers/xliff1_translation_parser';
 import {Xliff2TranslationParser} from './translation_files/translation_parsers/xliff2_translation_parser';
@@ -209,6 +210,7 @@ export function translateFiles({
         new Xliff1TranslationParser(),
         new XtbTranslationParser(),
         new SimpleJsonTranslationParser(),
+        new ArbTranslationParser(),
       ],
       duplicateTranslation, diagnostics);
 

packages/localize/src/tools/src/translate/translation_files/translation_parsers/arb_translation_parser.ts

@@ -0,0 +1,102 @@
+/**
+ * @license
+ * Copyright Google LLC All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+import {ɵMessageId, ɵparseTranslation, ɵSourceLocation, ɵSourceMessage} from '@angular/localize';
+import {Diagnostics} from '../../../diagnostics';
+import {ParseAnalysis, ParsedTranslationBundle, TranslationParser} from './translation_parser';
+
+export interface ArbJsonObject extends Record<ɵMessageId, ɵSourceMessage|ArbMetadata> {
+  '@@locale': string;
+}
+
+export interface ArbMetadata {
+  type?: 'text'|'image'|'css';
+  description?: string;
+  ['x-locations']?: ArbLocation[];
+}
+
+export interface ArbLocation {
+  start: {line: number, column: number};
+  end: {line: number, column: number};
+  file: string;
+}
+
+/**
+ * A translation parser that can parse JSON formatted as an Application Resource Bundle (ARB).
+ *
+ * See https://github.com/google/app-resource-bundle/wiki/ApplicationResourceBundleSpecification
+ *
+ * ```
+ * {
+ *   "@@locale": "en-US",
+ *   "message-id": "Target message string",
+ *   "@message-id": {
+ *     "type": "text",
+ *     "description": "Some description text",
+ *     "x-locations": [
+ *       {
+ *         "start": {"line": 23, "column": 145},
+ *         "end": {"line": 24, "column": 53},
+ *         "file": "some/file.ts"
+ *       },
+ *       ...
+ *     ]
+ *   },
+ *   ...
+ * }
+ * ```
+ */
+export class ArbTranslationParser implements TranslationParser<ArbJsonObject> {
+  /**
+   * @deprecated
+   */
+  canParse(filePath: string, contents: string): ArbJsonObject|false {
+    const result = this.analyze(filePath, contents);
+    return result.canParse && result.hint;
+  }
+
+  analyze(_filePath: string, contents: string): ParseAnalysis<ArbJsonObject> {
+    const diagnostics = new Diagnostics();
+    if (!contents.includes('"@@locale"')) {
+      return {canParse: false, diagnostics};
+    }
+    try {
+      // We can parse this file if it is valid JSON and contains the `"@@locale"` property.
+      return {canParse: true, diagnostics, hint: this.tryParseArbFormat(contents)};
+    } catch {
+      diagnostics.warn('File is not valid JSON.');
+      return {canParse: false, diagnostics};
+    }
+  }
+
+  parse(_filePath: string, contents: string, arb: ArbJsonObject = this.tryParseArbFormat(contents)):
+      ParsedTranslationBundle {
+    const bundle: ParsedTranslationBundle = {
+      locale: arb['@@locale'],
+      translations: {},
+      diagnostics: new Diagnostics()
+    };
+
+    for (const messageId of Object.keys(arb)) {
+      if (messageId.startsWith('@')) {
+        // Skip metadata keys
+        continue;
+      }
+      const targetMessage = arb[messageId] as string;
+      bundle.translations[messageId] = ɵparseTranslation(targetMessage);
+    }
+    return bundle;
+  }
+
+  private tryParseArbFormat(contents: string): ArbJsonObject {
+    const json = JSON.parse(contents);
+    if (typeof json['@@locale'] !== 'string') {
+      throw new Error('Missing @@locale property.');
+    }
+    return json;
+  }
+}

packages/localize/src/tools/src/translate/translation_files/translation_parsers/simple_json_translation_parser.ts

@@ -38,7 +38,10 @@ export class SimpleJsonTranslationParser implements TranslationParser<Object> {
 
   analyze(filePath: string, contents: string): ParseAnalysis<Object> {
     const diagnostics = new Diagnostics();
-    if (extname(filePath) !== '.json') {
+    // For this to be parsable, the extension must be `.json` and the contents must include "locale"
+    // and "translations" keys.
+    if (extname(filePath) !== '.json' ||
+        !(contents.includes('"locale"') && contents.includes('"translations"'))) {
       diagnostics.warn('File does not have .json extension.');
       return {canParse: false, diagnostics};
     }

packages/localize/src/tools/test/extract/integration/main_spec.ts

@@ -62,7 +62,7 @@ runInEachFileSystem(() => {
 
     for (const useLegacyIds of [true, false]) {
       describe(useLegacyIds ? '[using legacy ids]' : '', () => {
-        it('should extract translations from source code, and write as JSON format', () => {
+        it('should extract translations from source code, and write as simple JSON format', () => {
           extractTranslations({
             rootPath,
             sourceLocale: 'en-GB',
@@ -90,6 +90,86 @@ runInEachFileSystem(() => {
           ].join('\n'));
         });
 
+        it('should extract translations from source code, and write as ARB format', () => {
+          extractTranslations({
+            rootPath,
+            sourceLocale: 'en-GB',
+            sourceFilePaths: [sourceFilePath],
+            format: 'arb',
+            outputPath,
+            logger,
+            useSourceMaps: false,
+            useLegacyIds,
+            duplicateMessageHandling: 'ignore',
+            fileSystem: fs,
+          });
+          expect(fs.readFile(outputPath)).toEqual([
+            '{',
+            '  "@@locale": "en-GB",',
+            '  "3291030485717846467": "Hello, {$PH}!",',
+            '  "@3291030485717846467": {',
+            '    "x-locations": [',
+            '      {',
+            '        "file": "test_files/test.js",',
+            '        "start": { "line": "1", "column": "23" },',
+            '        "end": { "line": "1", "column": "40" }',
+            '      }',
+            '    ]',
+            '  },',
+            '  "8669027859022295761": "try{$PH}me",',
+            '  "@8669027859022295761": {',
+            '    "x-locations": [',
+            '      {',
+            '        "file": "test_files/test.js",',
+            '        "start": { "line": "2", "column": "22" },',
+            '        "end": { "line": "2", "column": "80" }',
+            '      }',
+            '    ]',
+            '  },',
+            '  "custom-id": "Custom id message",',
+            '  "@custom-id": {',
+            '    "x-locations": [',
+            '      {',
+            '        "file": "test_files/test.js",',
+            '        "start": { "line": "3", "column": "29" },',
+            '        "end": { "line": "3", "column": "61" }',
+            '      }',
+            '    ]',
+            '  },',
+            '  "273296103957933077": "Legacy id message",',
+            '  "@273296103957933077": {',
+            '    "x-locations": [',
+            '      {',
+            '        "file": "test_files/test.js",',
+            '        "start": { "line": "5", "column": "13" },',
+            '        "end": { "line": "5", "column": "96" }',
+            '      }',
+            '    ]',
+            '  },',
+            '  "custom-id-2": "Custom and legacy message",',
+            '  "@custom-id-2": {',
+            '    "x-locations": [',
+            '      {',
+            '        "file": "test_files/test.js",',
+            '        "start": { "line": "7", "column": "13" },',
+            '        "end": { "line": "7", "column": "117" }',
+            '      }',
+            '    ]',
+            '  },',
+            '  "2932901491976224757": "pre{$START_TAG_SPAN}inner-pre{$START_BOLD_TEXT}bold{$CLOSE_BOLD_TEXT}inner-post{$CLOSE_TAG_SPAN}post",',
+            '  "@2932901491976224757": {',
+            '    "x-locations": [',
+            '      {',
+            '        "file": "test_files/test.js",',
+            '        "start": { "line": "8", "column": "26" },',
+            '        "end": { "line": "9", "column": "93" }',
+            '      }',
+            '    ]',
+            '  }',
+            '}',
+          ].join('\n'));
+        });
+
         it('should extract translations from source code, and write as xmb format', () => {
           extractTranslations({
             rootPath,

packages/localize/src/tools/test/extract/translation_files/arb_translation_serializer_spec.ts

@@ -0,0 +1,116 @@
+/**
+ * @license
+ * Copyright Google LLC All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+import {absoluteFrom, FileSystem, getFileSystem} from '@angular/compiler-cli/src/ngtsc/file_system';
+import {runInEachFileSystem} from '@angular/compiler-cli/src/ngtsc/file_system/testing';
+import {ɵParsedMessage, ɵSourceLocation} from '@angular/localize';
+
+import {ArbTranslationSerializer} from '../../../src/extract/translation_files/arb_translation_serializer';
+
+import {mockMessage} from './mock_message';
+
+runInEachFileSystem(() => {
+  let fs: FileSystem;
+  describe('ArbTranslationSerializer', () => {
+    beforeEach(() => {
+      fs = getFileSystem();
+    });
+
+    describe('renderFile()', () => {
+      it('should convert a set of parsed messages into a JSON string', () => {
+        const messages: ɵParsedMessage[] = [
+          mockMessage('12345', ['a', 'b', 'c'], ['PH', 'PH_1'], {
+            meaning: 'some meaning',
+            location: {
+              file: absoluteFrom('/project/file.ts'),
+              start: {line: 5, column: 10},
+              end: {line: 5, column: 12}
+            },
+          }),
+          mockMessage('54321', ['a', 'b', 'c'], ['PH', 'PH_1'], {
+            customId: 'someId',
+          }),
+          mockMessage('67890', ['a', '', 'c'], ['START_TAG_SPAN', 'CLOSE_TAG_SPAN'], {
+            description: 'some description',
+            location: {
+              file: absoluteFrom('/project/file.ts'),
+              start: {line: 5, column: 10},
+              end: {line: 5, column: 12}
+            },
+          }),
+          mockMessage('67890', ['a', '', 'c'], ['START_TAG_SPAN', 'CLOSE_TAG_SPAN'], {
+            description: 'some description',
+            location: {
+              file: absoluteFrom('/project/other.ts'),
+              start: {line: 2, column: 10},
+              end: {line: 3, column: 12}
+            },
+          }),
+          mockMessage('13579', ['', 'b', ''], ['START_BOLD_TEXT', 'CLOSE_BOLD_TEXT'], {}),
+          mockMessage('24680', ['a'], [], {meaning: 'meaning', description: 'and description'}),
+          mockMessage('80808', ['multi\nlines'], [], {}),
+          mockMessage('90000', ['<escape', 'me>'], ['double-quotes-"'], {}),
+          mockMessage(
+              '100000',
+              [
+                'pre-ICU {VAR_SELECT, select, a {a} b {{INTERPOLATION}} c {pre {INTERPOLATION_1} post}} post-ICU'
+              ],
+              [], {}),
+          mockMessage(
+              '100001',
+              [
+                '{VAR_PLURAL, plural, one {{START_BOLD_TEXT}something bold{CLOSE_BOLD_TEXT}} other {pre {START_TAG_SPAN}middle{CLOSE_TAG_SPAN} post}}'
+              ],
+              [], {}),
+        ];
+        const serializer = new ArbTranslationSerializer('xx', fs.resolve('/project'), fs);
+        const output = serializer.serialize(messages);
+        expect(output.split('\n')).toEqual([
+          '{',
+          '  "@@locale": "xx",',
+          '  "12345": "a{$PH}b{$PH_1}c",',
+          '  "@12345": {',
+          '    "x-locations": [',
+          '      {',
+          '        "file": "file.ts",',
+          '        "start": { "line": "5", "column": "10" },',
+          '        "end": { "line": "5", "column": "12" }',
+          '      }',
+          '    ]',
+          '  },',
+          '  "someId": "a{$PH}b{$PH_1}c",',
+          '  "67890": "a{$START_TAG_SPAN}{$CLOSE_TAG_SPAN}c",',
+          '  "@67890": {',
+          '    "description": "some description",',
+          '    "x-locations": [',
+          '      {',
+          '        "file": "file.ts",',
+          '        "start": { "line": "5", "column": "10" },',
+          '        "end": { "line": "5", "column": "12" }',
+          '      },',
+          '      {',
+          '        "file": "other.ts",',
+          '        "start": { "line": "2", "column": "10" },',
+          '        "end": { "line": "3", "column": "12" }',
+          '      }',
+          '    ]',
+          '  },',
+          '  "13579": "{$START_BOLD_TEXT}b{$CLOSE_BOLD_TEXT}",',
+          '  "24680": "a",',
+          '  "@24680": {',
+          '    "description": "and description"',
+          '  },',
+          '  "80808": "multi\\nlines",',
+          '  "90000": "<escape{$double-quotes-\\"}me>",',
+          '  "100000": "pre-ICU {VAR_SELECT, select, a {a} b {{INTERPOLATION}} c {pre {INTERPOLATION_1} post}} post-ICU",',
+          '  "100001": "{VAR_PLURAL, plural, one {{START_BOLD_TEXT}something bold{CLOSE_BOLD_TEXT}} other {pre {START_TAG_SPAN}middle{CLOSE_TAG_SPAN} post}}"',
+          '}',
+        ]);
+      });
+    });
+  });
+});

packages/localize/src/tools/test/translate/translation_files/translation_parsers/arb_translation_parser_spec.ts

@@ -0,0 +1,50 @@
+/**
+ * @license
+ * Copyright Google LLC All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+import {ɵmakeTemplateObject} from '@angular/localize';
+import {ArbTranslationParser} from '../../../../src/translate/translation_files/translation_parsers/arb_translation_parser';
+
+describe('SimpleArbTranslationParser', () => {
+  describe('canParse()', () => {
+    it('should return true if the file extension  is `.json` and contains `@@locale` property',
+       () => {
+         const parser = new ArbTranslationParser();
+         expect(parser.canParse('/some/file.xlf', '')).toBe(false);
+         expect(parser.canParse('/some/file.json', 'xxx')).toBe(false);
+         expect(parser.canParse('/some/file.json', '{ "someKey": "someValue" }')).toBe(false);
+         expect(parser.canParse('/some/file.json', '{ "@@locale": "en", "someKey": "someValue" }'))
+             .toBeTruthy();
+       });
+  });
+
+  describe('parse()', () => {
+    it('should extract the locale from the JSON contents', () => {
+      const parser = new ArbTranslationParser();
+      const result = parser.parse('/some/file.json', '{"@@locale": "en"}');
+      expect(result.locale).toEqual('en');
+    });
+
+    it('should extract and process the translations from the JSON contents', () => {
+      const parser = new ArbTranslationParser();
+      const result = parser.parse('/some/file.json', `{
+        "@@locale": "fr",
+        "customId": "Bonjour, {$ph_1}!",
+        "@customId": {
+          "type": "text",
+          "description": "Some description"
+        }
+      }`);
+      expect(result.translations).toEqual({
+        'customId': {
+          text: 'Bonjour, {$ph_1}!',
+          messageParts: ɵmakeTemplateObject(['Bonjour, ', '!'], ['Bonjour, ', '!']),
+          placeholderNames: ['ph_1'],
+        },
+      });
+    });
+  });
+});