From 5684ac5e34fd702e1cc73c760084ffc1ca5a3141 Mon Sep 17 00:00:00 2001 From: Pete Bacon Darwin Date: Mon, 20 Apr 2020 11:52:47 +0100 Subject: [PATCH] feat(localize): support Application Resource Bundle (ARB) translation file format (#36795) The ARB format is a JSON file containing an object where the keys are the message ids and the values are the translations. It is extensible because it can also contain metadata about each message. For example: ``` { "@@locale": "...", "message-id": "Translated message string", "@message-id": { "type": "text", "description": "Some description text", "x-locations": [{ "start": {"line": 23, "column": 145}, "file": "some/file.ts" }] }, } ``` For more information, see: https://github.com/google/app-resource-bundle/wiki/ApplicationResourceBundleSpecification PR Close #36795 --- .../localize/src/tools/src/extract/main.ts | 3 + .../arb_translation_serializer.ts | 100 +++++++++++++++ .../localize/src/tools/src/translate/main.ts | 2 + .../arb_translation_parser.ts | 102 +++++++++++++++ .../simple_json_translation_parser.ts | 5 +- .../test/extract/integration/main_spec.ts | 82 ++++++++++++- .../arb_translation_serializer_spec.ts | 116 ++++++++++++++++++ .../arb_translation_parser_spec.ts | 50 ++++++++ 8 files changed, 458 insertions(+), 2 deletions(-) create mode 100644 packages/localize/src/tools/src/extract/translation_files/arb_translation_serializer.ts create mode 100644 packages/localize/src/tools/src/translate/translation_files/translation_parsers/arb_translation_parser.ts create mode 100644 packages/localize/src/tools/test/extract/translation_files/arb_translation_serializer_spec.ts create mode 100644 packages/localize/src/tools/test/translate/translation_files/translation_parsers/arb_translation_parser_spec.ts diff --git a/packages/localize/src/tools/src/extract/main.ts b/packages/localize/src/tools/src/extract/main.ts index 899a994a42..e68335b4ed 100644 --- a/packages/localize/src/tools/src/extract/main.ts +++ b/packages/localize/src/tools/src/extract/main.ts @@ -17,6 +17,7 @@ import {DiagnosticHandlingStrategy} from '../diagnostics'; import {checkDuplicateMessages} from './duplicates'; import {MessageExtractor} from './extraction'; import {TranslationSerializer} from './translation_files/translation_serializer'; +import {ArbTranslationSerializer} from './translation_files/arb_translation_serializer'; import {SimpleJsonTranslationSerializer} from './translation_files/json_translation_serializer'; import {Xliff1TranslationSerializer} from './translation_files/xliff1_translation_serializer'; import {Xliff2TranslationSerializer} from './translation_files/xliff2_translation_serializer'; @@ -229,6 +230,8 @@ export function getSerializer( return new XmbTranslationSerializer(rootPath, useLegacyIds, fs); case 'json': return new SimpleJsonTranslationSerializer(sourceLocale); + case 'arb': + return new ArbTranslationSerializer(sourceLocale, rootPath, fs); } throw new Error(`No translation serializer can handle the provided format: ${format}`); } diff --git a/packages/localize/src/tools/src/extract/translation_files/arb_translation_serializer.ts b/packages/localize/src/tools/src/extract/translation_files/arb_translation_serializer.ts new file mode 100644 index 0000000000..8d303048e0 --- /dev/null +++ b/packages/localize/src/tools/src/extract/translation_files/arb_translation_serializer.ts @@ -0,0 +1,100 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ +import {AbsoluteFsPath, FileSystem} from '@angular/compiler-cli/src/ngtsc/file_system'; +import {ɵParsedMessage, ɵSourceLocation} from '@angular/localize'; +import {ArbJsonObject, ArbLocation, ArbMetadata} from '../../translate/translation_files/translation_parsers/arb_translation_parser'; +import {TranslationSerializer} from './translation_serializer'; +import {consolidateMessages, hasLocation} from './utils'; + +/** + * A translation serializer that can render JSON formatted as an Application Resource Bundle (ARB). + * + * See https://github.com/google/app-resource-bundle/wiki/ApplicationResourceBundleSpecification + * + * ``` + * { + * "@@locale": "en-US", + * "message-id": "Target message string", + * "@message-id": { + * "type": "text", + * "description": "Some description text", + * "x-locations": [ + * { + * "start": {"line": 23, "column": 145}, + * "end": {"line": 24, "column": 53}, + * "file": "some/file.ts" + * }, + * ... + * ] + * }, + * ... + * } + * ``` + */ + +/** + * This is a semi-public bespoke serialization format that is used for testing and sometimes as a + * format for storing translations that will be inlined at runtime. + * + * @see ArbTranslationParser + */ +export class ArbTranslationSerializer implements TranslationSerializer { + constructor( + private sourceLocale: string, private basePath: AbsoluteFsPath, private fs: FileSystem) {} + + serialize(messages: ɵParsedMessage[]): string { + const messageMap = consolidateMessages(messages, message => message.customId || message.id); + + let output = `{\n "@@locale": ${JSON.stringify(this.sourceLocale)}`; + + for (const [id, duplicateMessages] of messageMap.entries()) { + const message = duplicateMessages[0]; + output += this.serializeMessage(id, message); + output += this.serializeMeta( + id, message.description, duplicateMessages.filter(hasLocation).map(m => m.location)); + } + + output += '\n}'; + + return output; + } + + private serializeMessage(id: string, message: ɵParsedMessage): string { + return `,\n ${JSON.stringify(id)}: ${JSON.stringify(message.text)}`; + } + + private serializeMeta(id: string, description: string|undefined, locations: ɵSourceLocation[]): + string { + const meta: string[] = []; + + if (description) { + meta.push(`\n "description": ${JSON.stringify(description)}`); + } + + if (locations.length > 0) { + let locationStr = `\n "x-locations": [`; + for (let i = 0; i < locations.length; i++) { + locationStr += (i > 0 ? ',\n' : '\n') + this.serializeLocation(locations[i]); + } + locationStr += '\n ]'; + meta.push(locationStr); + } + + return meta.length > 0 ? `,\n ${JSON.stringify('@' + id)}: {${meta.join(',')}\n }` : ''; + } + + private serializeLocation({file, start, end}: ɵSourceLocation): string { + return [ + ` {`, + ` "file": ${JSON.stringify(this.fs.relative(this.basePath, file))},`, + ` "start": { "line": "${start.line}", "column": "${start.column}" },`, + ` "end": { "line": "${end.line}", "column": "${end.column}" }`, + ` }`, + ].join('\n'); + } +} diff --git a/packages/localize/src/tools/src/translate/main.ts b/packages/localize/src/tools/src/translate/main.ts index 590d00822f..8782d0f534 100644 --- a/packages/localize/src/tools/src/translate/main.ts +++ b/packages/localize/src/tools/src/translate/main.ts @@ -15,6 +15,7 @@ import {AssetTranslationHandler} from './asset_files/asset_translation_handler'; import {getOutputPathFn, OutputPathFn} from './output_path'; import {SourceFileTranslationHandler} from './source_files/source_file_translation_handler'; import {TranslationLoader} from './translation_files/translation_loader'; +import {ArbTranslationParser} from './translation_files/translation_parsers/arb_translation_parser'; import {SimpleJsonTranslationParser} from './translation_files/translation_parsers/simple_json_translation_parser'; import {Xliff1TranslationParser} from './translation_files/translation_parsers/xliff1_translation_parser'; import {Xliff2TranslationParser} from './translation_files/translation_parsers/xliff2_translation_parser'; @@ -209,6 +210,7 @@ export function translateFiles({ new Xliff1TranslationParser(), new XtbTranslationParser(), new SimpleJsonTranslationParser(), + new ArbTranslationParser(), ], duplicateTranslation, diagnostics); diff --git a/packages/localize/src/tools/src/translate/translation_files/translation_parsers/arb_translation_parser.ts b/packages/localize/src/tools/src/translate/translation_files/translation_parsers/arb_translation_parser.ts new file mode 100644 index 0000000000..990b16a74b --- /dev/null +++ b/packages/localize/src/tools/src/translate/translation_files/translation_parsers/arb_translation_parser.ts @@ -0,0 +1,102 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ +import {ɵMessageId, ɵparseTranslation, ɵSourceLocation, ɵSourceMessage} from '@angular/localize'; +import {Diagnostics} from '../../../diagnostics'; +import {ParseAnalysis, ParsedTranslationBundle, TranslationParser} from './translation_parser'; + +export interface ArbJsonObject extends Record<ɵMessageId, ɵSourceMessage|ArbMetadata> { + '@@locale': string; +} + +export interface ArbMetadata { + type?: 'text'|'image'|'css'; + description?: string; + ['x-locations']?: ArbLocation[]; +} + +export interface ArbLocation { + start: {line: number, column: number}; + end: {line: number, column: number}; + file: string; +} + +/** + * A translation parser that can parse JSON formatted as an Application Resource Bundle (ARB). + * + * See https://github.com/google/app-resource-bundle/wiki/ApplicationResourceBundleSpecification + * + * ``` + * { + * "@@locale": "en-US", + * "message-id": "Target message string", + * "@message-id": { + * "type": "text", + * "description": "Some description text", + * "x-locations": [ + * { + * "start": {"line": 23, "column": 145}, + * "end": {"line": 24, "column": 53}, + * "file": "some/file.ts" + * }, + * ... + * ] + * }, + * ... + * } + * ``` + */ +export class ArbTranslationParser implements TranslationParser { + /** + * @deprecated + */ + canParse(filePath: string, contents: string): ArbJsonObject|false { + const result = this.analyze(filePath, contents); + return result.canParse && result.hint; + } + + analyze(_filePath: string, contents: string): ParseAnalysis { + const diagnostics = new Diagnostics(); + if (!contents.includes('"@@locale"')) { + return {canParse: false, diagnostics}; + } + try { + // We can parse this file if it is valid JSON and contains the `"@@locale"` property. + return {canParse: true, diagnostics, hint: this.tryParseArbFormat(contents)}; + } catch { + diagnostics.warn('File is not valid JSON.'); + return {canParse: false, diagnostics}; + } + } + + parse(_filePath: string, contents: string, arb: ArbJsonObject = this.tryParseArbFormat(contents)): + ParsedTranslationBundle { + const bundle: ParsedTranslationBundle = { + locale: arb['@@locale'], + translations: {}, + diagnostics: new Diagnostics() + }; + + for (const messageId of Object.keys(arb)) { + if (messageId.startsWith('@')) { + // Skip metadata keys + continue; + } + const targetMessage = arb[messageId] as string; + bundle.translations[messageId] = ɵparseTranslation(targetMessage); + } + return bundle; + } + + private tryParseArbFormat(contents: string): ArbJsonObject { + const json = JSON.parse(contents); + if (typeof json['@@locale'] !== 'string') { + throw new Error('Missing @@locale property.'); + } + return json; + } +} diff --git a/packages/localize/src/tools/src/translate/translation_files/translation_parsers/simple_json_translation_parser.ts b/packages/localize/src/tools/src/translate/translation_files/translation_parsers/simple_json_translation_parser.ts index 92cd6f1072..6f695c4c9d 100644 --- a/packages/localize/src/tools/src/translate/translation_files/translation_parsers/simple_json_translation_parser.ts +++ b/packages/localize/src/tools/src/translate/translation_files/translation_parsers/simple_json_translation_parser.ts @@ -38,7 +38,10 @@ export class SimpleJsonTranslationParser implements TranslationParser { analyze(filePath: string, contents: string): ParseAnalysis { const diagnostics = new Diagnostics(); - if (extname(filePath) !== '.json') { + // For this to be parsable, the extension must be `.json` and the contents must include "locale" + // and "translations" keys. + if (extname(filePath) !== '.json' || + !(contents.includes('"locale"') && contents.includes('"translations"'))) { diagnostics.warn('File does not have .json extension.'); return {canParse: false, diagnostics}; } diff --git a/packages/localize/src/tools/test/extract/integration/main_spec.ts b/packages/localize/src/tools/test/extract/integration/main_spec.ts index ad19fec321..6663b785ce 100644 --- a/packages/localize/src/tools/test/extract/integration/main_spec.ts +++ b/packages/localize/src/tools/test/extract/integration/main_spec.ts @@ -62,7 +62,7 @@ runInEachFileSystem(() => { for (const useLegacyIds of [true, false]) { describe(useLegacyIds ? '[using legacy ids]' : '', () => { - it('should extract translations from source code, and write as JSON format', () => { + it('should extract translations from source code, and write as simple JSON format', () => { extractTranslations({ rootPath, sourceLocale: 'en-GB', @@ -90,6 +90,86 @@ runInEachFileSystem(() => { ].join('\n')); }); + it('should extract translations from source code, and write as ARB format', () => { + extractTranslations({ + rootPath, + sourceLocale: 'en-GB', + sourceFilePaths: [sourceFilePath], + format: 'arb', + outputPath, + logger, + useSourceMaps: false, + useLegacyIds, + duplicateMessageHandling: 'ignore', + fileSystem: fs, + }); + expect(fs.readFile(outputPath)).toEqual([ + '{', + ' "@@locale": "en-GB",', + ' "3291030485717846467": "Hello, {$PH}!",', + ' "@3291030485717846467": {', + ' "x-locations": [', + ' {', + ' "file": "test_files/test.js",', + ' "start": { "line": "1", "column": "23" },', + ' "end": { "line": "1", "column": "40" }', + ' }', + ' ]', + ' },', + ' "8669027859022295761": "try{$PH}me",', + ' "@8669027859022295761": {', + ' "x-locations": [', + ' {', + ' "file": "test_files/test.js",', + ' "start": { "line": "2", "column": "22" },', + ' "end": { "line": "2", "column": "80" }', + ' }', + ' ]', + ' },', + ' "custom-id": "Custom id message",', + ' "@custom-id": {', + ' "x-locations": [', + ' {', + ' "file": "test_files/test.js",', + ' "start": { "line": "3", "column": "29" },', + ' "end": { "line": "3", "column": "61" }', + ' }', + ' ]', + ' },', + ' "273296103957933077": "Legacy id message",', + ' "@273296103957933077": {', + ' "x-locations": [', + ' {', + ' "file": "test_files/test.js",', + ' "start": { "line": "5", "column": "13" },', + ' "end": { "line": "5", "column": "96" }', + ' }', + ' ]', + ' },', + ' "custom-id-2": "Custom and legacy message",', + ' "@custom-id-2": {', + ' "x-locations": [', + ' {', + ' "file": "test_files/test.js",', + ' "start": { "line": "7", "column": "13" },', + ' "end": { "line": "7", "column": "117" }', + ' }', + ' ]', + ' },', + ' "2932901491976224757": "pre{$START_TAG_SPAN}inner-pre{$START_BOLD_TEXT}bold{$CLOSE_BOLD_TEXT}inner-post{$CLOSE_TAG_SPAN}post",', + ' "@2932901491976224757": {', + ' "x-locations": [', + ' {', + ' "file": "test_files/test.js",', + ' "start": { "line": "8", "column": "26" },', + ' "end": { "line": "9", "column": "93" }', + ' }', + ' ]', + ' }', + '}', + ].join('\n')); + }); + it('should extract translations from source code, and write as xmb format', () => { extractTranslations({ rootPath, diff --git a/packages/localize/src/tools/test/extract/translation_files/arb_translation_serializer_spec.ts b/packages/localize/src/tools/test/extract/translation_files/arb_translation_serializer_spec.ts new file mode 100644 index 0000000000..5017151a1e --- /dev/null +++ b/packages/localize/src/tools/test/extract/translation_files/arb_translation_serializer_spec.ts @@ -0,0 +1,116 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ +import {absoluteFrom, FileSystem, getFileSystem} from '@angular/compiler-cli/src/ngtsc/file_system'; +import {runInEachFileSystem} from '@angular/compiler-cli/src/ngtsc/file_system/testing'; +import {ɵParsedMessage, ɵSourceLocation} from '@angular/localize'; + +import {ArbTranslationSerializer} from '../../../src/extract/translation_files/arb_translation_serializer'; + +import {mockMessage} from './mock_message'; + +runInEachFileSystem(() => { + let fs: FileSystem; + describe('ArbTranslationSerializer', () => { + beforeEach(() => { + fs = getFileSystem(); + }); + + describe('renderFile()', () => { + it('should convert a set of parsed messages into a JSON string', () => { + const messages: ɵParsedMessage[] = [ + mockMessage('12345', ['a', 'b', 'c'], ['PH', 'PH_1'], { + meaning: 'some meaning', + location: { + file: absoluteFrom('/project/file.ts'), + start: {line: 5, column: 10}, + end: {line: 5, column: 12} + }, + }), + mockMessage('54321', ['a', 'b', 'c'], ['PH', 'PH_1'], { + customId: 'someId', + }), + mockMessage('67890', ['a', '', 'c'], ['START_TAG_SPAN', 'CLOSE_TAG_SPAN'], { + description: 'some description', + location: { + file: absoluteFrom('/project/file.ts'), + start: {line: 5, column: 10}, + end: {line: 5, column: 12} + }, + }), + mockMessage('67890', ['a', '', 'c'], ['START_TAG_SPAN', 'CLOSE_TAG_SPAN'], { + description: 'some description', + location: { + file: absoluteFrom('/project/other.ts'), + start: {line: 2, column: 10}, + end: {line: 3, column: 12} + }, + }), + mockMessage('13579', ['', 'b', ''], ['START_BOLD_TEXT', 'CLOSE_BOLD_TEXT'], {}), + mockMessage('24680', ['a'], [], {meaning: 'meaning', description: 'and description'}), + mockMessage('80808', ['multi\nlines'], [], {}), + mockMessage('90000', [''], ['double-quotes-"'], {}), + mockMessage( + '100000', + [ + 'pre-ICU {VAR_SELECT, select, a {a} b {{INTERPOLATION}} c {pre {INTERPOLATION_1} post}} post-ICU' + ], + [], {}), + mockMessage( + '100001', + [ + '{VAR_PLURAL, plural, one {{START_BOLD_TEXT}something bold{CLOSE_BOLD_TEXT}} other {pre {START_TAG_SPAN}middle{CLOSE_TAG_SPAN} post}}' + ], + [], {}), + ]; + const serializer = new ArbTranslationSerializer('xx', fs.resolve('/project'), fs); + const output = serializer.serialize(messages); + expect(output.split('\n')).toEqual([ + '{', + ' "@@locale": "xx",', + ' "12345": "a{$PH}b{$PH_1}c",', + ' "@12345": {', + ' "x-locations": [', + ' {', + ' "file": "file.ts",', + ' "start": { "line": "5", "column": "10" },', + ' "end": { "line": "5", "column": "12" }', + ' }', + ' ]', + ' },', + ' "someId": "a{$PH}b{$PH_1}c",', + ' "67890": "a{$START_TAG_SPAN}{$CLOSE_TAG_SPAN}c",', + ' "@67890": {', + ' "description": "some description",', + ' "x-locations": [', + ' {', + ' "file": "file.ts",', + ' "start": { "line": "5", "column": "10" },', + ' "end": { "line": "5", "column": "12" }', + ' },', + ' {', + ' "file": "other.ts",', + ' "start": { "line": "2", "column": "10" },', + ' "end": { "line": "3", "column": "12" }', + ' }', + ' ]', + ' },', + ' "13579": "{$START_BOLD_TEXT}b{$CLOSE_BOLD_TEXT}",', + ' "24680": "a",', + ' "@24680": {', + ' "description": "and description"', + ' },', + ' "80808": "multi\\nlines",', + ' "90000": "",', + ' "100000": "pre-ICU {VAR_SELECT, select, a {a} b {{INTERPOLATION}} c {pre {INTERPOLATION_1} post}} post-ICU",', + ' "100001": "{VAR_PLURAL, plural, one {{START_BOLD_TEXT}something bold{CLOSE_BOLD_TEXT}} other {pre {START_TAG_SPAN}middle{CLOSE_TAG_SPAN} post}}"', + '}', + ]); + }); + }); + }); +}); diff --git a/packages/localize/src/tools/test/translate/translation_files/translation_parsers/arb_translation_parser_spec.ts b/packages/localize/src/tools/test/translate/translation_files/translation_parsers/arb_translation_parser_spec.ts new file mode 100644 index 0000000000..54778105a9 --- /dev/null +++ b/packages/localize/src/tools/test/translate/translation_files/translation_parsers/arb_translation_parser_spec.ts @@ -0,0 +1,50 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ +import {ɵmakeTemplateObject} from '@angular/localize'; +import {ArbTranslationParser} from '../../../../src/translate/translation_files/translation_parsers/arb_translation_parser'; + +describe('SimpleArbTranslationParser', () => { + describe('canParse()', () => { + it('should return true if the file extension is `.json` and contains `@@locale` property', + () => { + const parser = new ArbTranslationParser(); + expect(parser.canParse('/some/file.xlf', '')).toBe(false); + expect(parser.canParse('/some/file.json', 'xxx')).toBe(false); + expect(parser.canParse('/some/file.json', '{ "someKey": "someValue" }')).toBe(false); + expect(parser.canParse('/some/file.json', '{ "@@locale": "en", "someKey": "someValue" }')) + .toBeTruthy(); + }); + }); + + describe('parse()', () => { + it('should extract the locale from the JSON contents', () => { + const parser = new ArbTranslationParser(); + const result = parser.parse('/some/file.json', '{"@@locale": "en"}'); + expect(result.locale).toEqual('en'); + }); + + it('should extract and process the translations from the JSON contents', () => { + const parser = new ArbTranslationParser(); + const result = parser.parse('/some/file.json', `{ + "@@locale": "fr", + "customId": "Bonjour, {$ph_1}!", + "@customId": { + "type": "text", + "description": "Some description" + } + }`); + expect(result.translations).toEqual({ + 'customId': { + text: 'Bonjour, {$ph_1}!', + messageParts: ɵmakeTemplateObject(['Bonjour, ', '!'], ['Bonjour, ', '!']), + placeholderNames: ['ph_1'], + }, + }); + }); + }); +});