diff --git a/packages/localize/src/tools/src/extract/main.ts b/packages/localize/src/tools/src/extract/main.ts index 899a994a42..e68335b4ed 100644 --- a/packages/localize/src/tools/src/extract/main.ts +++ b/packages/localize/src/tools/src/extract/main.ts @@ -17,6 +17,7 @@ import {DiagnosticHandlingStrategy} from '../diagnostics'; import {checkDuplicateMessages} from './duplicates'; import {MessageExtractor} from './extraction'; import {TranslationSerializer} from './translation_files/translation_serializer'; +import {ArbTranslationSerializer} from './translation_files/arb_translation_serializer'; import {SimpleJsonTranslationSerializer} from './translation_files/json_translation_serializer'; import {Xliff1TranslationSerializer} from './translation_files/xliff1_translation_serializer'; import {Xliff2TranslationSerializer} from './translation_files/xliff2_translation_serializer'; @@ -229,6 +230,8 @@ export function getSerializer( return new XmbTranslationSerializer(rootPath, useLegacyIds, fs); case 'json': return new SimpleJsonTranslationSerializer(sourceLocale); + case 'arb': + return new ArbTranslationSerializer(sourceLocale, rootPath, fs); } throw new Error(`No translation serializer can handle the provided format: ${format}`); } diff --git a/packages/localize/src/tools/src/extract/translation_files/arb_translation_serializer.ts b/packages/localize/src/tools/src/extract/translation_files/arb_translation_serializer.ts new file mode 100644 index 0000000000..8d303048e0 --- /dev/null +++ b/packages/localize/src/tools/src/extract/translation_files/arb_translation_serializer.ts @@ -0,0 +1,100 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ +import {AbsoluteFsPath, FileSystem} from '@angular/compiler-cli/src/ngtsc/file_system'; +import {ɵParsedMessage, ɵSourceLocation} from '@angular/localize'; +import {ArbJsonObject, ArbLocation, ArbMetadata} from '../../translate/translation_files/translation_parsers/arb_translation_parser'; +import {TranslationSerializer} from './translation_serializer'; +import {consolidateMessages, hasLocation} from './utils'; + +/** + * A translation serializer that can render JSON formatted as an Application Resource Bundle (ARB). + * + * See https://github.com/google/app-resource-bundle/wiki/ApplicationResourceBundleSpecification + * + * ``` + * { + * "@@locale": "en-US", + * "message-id": "Target message string", + * "@message-id": { + * "type": "text", + * "description": "Some description text", + * "x-locations": [ + * { + * "start": {"line": 23, "column": 145}, + * "end": {"line": 24, "column": 53}, + * "file": "some/file.ts" + * }, + * ... + * ] + * }, + * ... + * } + * ``` + */ + +/** + * This is a semi-public bespoke serialization format that is used for testing and sometimes as a + * format for storing translations that will be inlined at runtime. + * + * @see ArbTranslationParser + */ +export class ArbTranslationSerializer implements TranslationSerializer { + constructor( + private sourceLocale: string, private basePath: AbsoluteFsPath, private fs: FileSystem) {} + + serialize(messages: ɵParsedMessage[]): string { + const messageMap = consolidateMessages(messages, message => message.customId || message.id); + + let output = `{\n "@@locale": ${JSON.stringify(this.sourceLocale)}`; + + for (const [id, duplicateMessages] of messageMap.entries()) { + const message = duplicateMessages[0]; + output += this.serializeMessage(id, message); + output += this.serializeMeta( + id, message.description, duplicateMessages.filter(hasLocation).map(m => m.location)); + } + + output += '\n}'; + + return output; + } + + private serializeMessage(id: string, message: ɵParsedMessage): string { + return `,\n ${JSON.stringify(id)}: ${JSON.stringify(message.text)}`; + } + + private serializeMeta(id: string, description: string|undefined, locations: ɵSourceLocation[]): + string { + const meta: string[] = []; + + if (description) { + meta.push(`\n "description": ${JSON.stringify(description)}`); + } + + if (locations.length > 0) { + let locationStr = `\n "x-locations": [`; + for (let i = 0; i < locations.length; i++) { + locationStr += (i > 0 ? ',\n' : '\n') + this.serializeLocation(locations[i]); + } + locationStr += '\n ]'; + meta.push(locationStr); + } + + return meta.length > 0 ? `,\n ${JSON.stringify('@' + id)}: {${meta.join(',')}\n }` : ''; + } + + private serializeLocation({file, start, end}: ɵSourceLocation): string { + return [ + ` {`, + ` "file": ${JSON.stringify(this.fs.relative(this.basePath, file))},`, + ` "start": { "line": "${start.line}", "column": "${start.column}" },`, + ` "end": { "line": "${end.line}", "column": "${end.column}" }`, + ` }`, + ].join('\n'); + } +} diff --git a/packages/localize/src/tools/src/translate/main.ts b/packages/localize/src/tools/src/translate/main.ts index 590d00822f..8782d0f534 100644 --- a/packages/localize/src/tools/src/translate/main.ts +++ b/packages/localize/src/tools/src/translate/main.ts @@ -15,6 +15,7 @@ import {AssetTranslationHandler} from './asset_files/asset_translation_handler'; import {getOutputPathFn, OutputPathFn} from './output_path'; import {SourceFileTranslationHandler} from './source_files/source_file_translation_handler'; import {TranslationLoader} from './translation_files/translation_loader'; +import {ArbTranslationParser} from './translation_files/translation_parsers/arb_translation_parser'; import {SimpleJsonTranslationParser} from './translation_files/translation_parsers/simple_json_translation_parser'; import {Xliff1TranslationParser} from './translation_files/translation_parsers/xliff1_translation_parser'; import {Xliff2TranslationParser} from './translation_files/translation_parsers/xliff2_translation_parser'; @@ -209,6 +210,7 @@ export function translateFiles({ new Xliff1TranslationParser(), new XtbTranslationParser(), new SimpleJsonTranslationParser(), + new ArbTranslationParser(), ], duplicateTranslation, diagnostics); diff --git a/packages/localize/src/tools/src/translate/translation_files/translation_parsers/arb_translation_parser.ts b/packages/localize/src/tools/src/translate/translation_files/translation_parsers/arb_translation_parser.ts new file mode 100644 index 0000000000..990b16a74b --- /dev/null +++ b/packages/localize/src/tools/src/translate/translation_files/translation_parsers/arb_translation_parser.ts @@ -0,0 +1,102 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ +import {ɵMessageId, ɵparseTranslation, ɵSourceLocation, ɵSourceMessage} from '@angular/localize'; +import {Diagnostics} from '../../../diagnostics'; +import {ParseAnalysis, ParsedTranslationBundle, TranslationParser} from './translation_parser'; + +export interface ArbJsonObject extends Record<ɵMessageId, ɵSourceMessage|ArbMetadata> { + '@@locale': string; +} + +export interface ArbMetadata { + type?: 'text'|'image'|'css'; + description?: string; + ['x-locations']?: ArbLocation[]; +} + +export interface ArbLocation { + start: {line: number, column: number}; + end: {line: number, column: number}; + file: string; +} + +/** + * A translation parser that can parse JSON formatted as an Application Resource Bundle (ARB). + * + * See https://github.com/google/app-resource-bundle/wiki/ApplicationResourceBundleSpecification + * + * ``` + * { + * "@@locale": "en-US", + * "message-id": "Target message string", + * "@message-id": { + * "type": "text", + * "description": "Some description text", + * "x-locations": [ + * { + * "start": {"line": 23, "column": 145}, + * "end": {"line": 24, "column": 53}, + * "file": "some/file.ts" + * }, + * ... + * ] + * }, + * ... + * } + * ``` + */ +export class ArbTranslationParser implements TranslationParser { + /** + * @deprecated + */ + canParse(filePath: string, contents: string): ArbJsonObject|false { + const result = this.analyze(filePath, contents); + return result.canParse && result.hint; + } + + analyze(_filePath: string, contents: string): ParseAnalysis { + const diagnostics = new Diagnostics(); + if (!contents.includes('"@@locale"')) { + return {canParse: false, diagnostics}; + } + try { + // We can parse this file if it is valid JSON and contains the `"@@locale"` property. + return {canParse: true, diagnostics, hint: this.tryParseArbFormat(contents)}; + } catch { + diagnostics.warn('File is not valid JSON.'); + return {canParse: false, diagnostics}; + } + } + + parse(_filePath: string, contents: string, arb: ArbJsonObject = this.tryParseArbFormat(contents)): + ParsedTranslationBundle { + const bundle: ParsedTranslationBundle = { + locale: arb['@@locale'], + translations: {}, + diagnostics: new Diagnostics() + }; + + for (const messageId of Object.keys(arb)) { + if (messageId.startsWith('@')) { + // Skip metadata keys + continue; + } + const targetMessage = arb[messageId] as string; + bundle.translations[messageId] = ɵparseTranslation(targetMessage); + } + return bundle; + } + + private tryParseArbFormat(contents: string): ArbJsonObject { + const json = JSON.parse(contents); + if (typeof json['@@locale'] !== 'string') { + throw new Error('Missing @@locale property.'); + } + return json; + } +} diff --git a/packages/localize/src/tools/src/translate/translation_files/translation_parsers/simple_json_translation_parser.ts b/packages/localize/src/tools/src/translate/translation_files/translation_parsers/simple_json_translation_parser.ts index 92cd6f1072..6f695c4c9d 100644 --- a/packages/localize/src/tools/src/translate/translation_files/translation_parsers/simple_json_translation_parser.ts +++ b/packages/localize/src/tools/src/translate/translation_files/translation_parsers/simple_json_translation_parser.ts @@ -38,7 +38,10 @@ export class SimpleJsonTranslationParser implements TranslationParser { analyze(filePath: string, contents: string): ParseAnalysis { const diagnostics = new Diagnostics(); - if (extname(filePath) !== '.json') { + // For this to be parsable, the extension must be `.json` and the contents must include "locale" + // and "translations" keys. + if (extname(filePath) !== '.json' || + !(contents.includes('"locale"') && contents.includes('"translations"'))) { diagnostics.warn('File does not have .json extension.'); return {canParse: false, diagnostics}; } diff --git a/packages/localize/src/tools/test/extract/integration/main_spec.ts b/packages/localize/src/tools/test/extract/integration/main_spec.ts index ad19fec321..6663b785ce 100644 --- a/packages/localize/src/tools/test/extract/integration/main_spec.ts +++ b/packages/localize/src/tools/test/extract/integration/main_spec.ts @@ -62,7 +62,7 @@ runInEachFileSystem(() => { for (const useLegacyIds of [true, false]) { describe(useLegacyIds ? '[using legacy ids]' : '', () => { - it('should extract translations from source code, and write as JSON format', () => { + it('should extract translations from source code, and write as simple JSON format', () => { extractTranslations({ rootPath, sourceLocale: 'en-GB', @@ -90,6 +90,86 @@ runInEachFileSystem(() => { ].join('\n')); }); + it('should extract translations from source code, and write as ARB format', () => { + extractTranslations({ + rootPath, + sourceLocale: 'en-GB', + sourceFilePaths: [sourceFilePath], + format: 'arb', + outputPath, + logger, + useSourceMaps: false, + useLegacyIds, + duplicateMessageHandling: 'ignore', + fileSystem: fs, + }); + expect(fs.readFile(outputPath)).toEqual([ + '{', + ' "@@locale": "en-GB",', + ' "3291030485717846467": "Hello, {$PH}!",', + ' "@3291030485717846467": {', + ' "x-locations": [', + ' {', + ' "file": "test_files/test.js",', + ' "start": { "line": "1", "column": "23" },', + ' "end": { "line": "1", "column": "40" }', + ' }', + ' ]', + ' },', + ' "8669027859022295761": "try{$PH}me",', + ' "@8669027859022295761": {', + ' "x-locations": [', + ' {', + ' "file": "test_files/test.js",', + ' "start": { "line": "2", "column": "22" },', + ' "end": { "line": "2", "column": "80" }', + ' }', + ' ]', + ' },', + ' "custom-id": "Custom id message",', + ' "@custom-id": {', + ' "x-locations": [', + ' {', + ' "file": "test_files/test.js",', + ' "start": { "line": "3", "column": "29" },', + ' "end": { "line": "3", "column": "61" }', + ' }', + ' ]', + ' },', + ' "273296103957933077": "Legacy id message",', + ' "@273296103957933077": {', + ' "x-locations": [', + ' {', + ' "file": "test_files/test.js",', + ' "start": { "line": "5", "column": "13" },', + ' "end": { "line": "5", "column": "96" }', + ' }', + ' ]', + ' },', + ' "custom-id-2": "Custom and legacy message",', + ' "@custom-id-2": {', + ' "x-locations": [', + ' {', + ' "file": "test_files/test.js",', + ' "start": { "line": "7", "column": "13" },', + ' "end": { "line": "7", "column": "117" }', + ' }', + ' ]', + ' },', + ' "2932901491976224757": "pre{$START_TAG_SPAN}inner-pre{$START_BOLD_TEXT}bold{$CLOSE_BOLD_TEXT}inner-post{$CLOSE_TAG_SPAN}post",', + ' "@2932901491976224757": {', + ' "x-locations": [', + ' {', + ' "file": "test_files/test.js",', + ' "start": { "line": "8", "column": "26" },', + ' "end": { "line": "9", "column": "93" }', + ' }', + ' ]', + ' }', + '}', + ].join('\n')); + }); + it('should extract translations from source code, and write as xmb format', () => { extractTranslations({ rootPath, diff --git a/packages/localize/src/tools/test/extract/translation_files/arb_translation_serializer_spec.ts b/packages/localize/src/tools/test/extract/translation_files/arb_translation_serializer_spec.ts new file mode 100644 index 0000000000..5017151a1e --- /dev/null +++ b/packages/localize/src/tools/test/extract/translation_files/arb_translation_serializer_spec.ts @@ -0,0 +1,116 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ +import {absoluteFrom, FileSystem, getFileSystem} from '@angular/compiler-cli/src/ngtsc/file_system'; +import {runInEachFileSystem} from '@angular/compiler-cli/src/ngtsc/file_system/testing'; +import {ɵParsedMessage, ɵSourceLocation} from '@angular/localize'; + +import {ArbTranslationSerializer} from '../../../src/extract/translation_files/arb_translation_serializer'; + +import {mockMessage} from './mock_message'; + +runInEachFileSystem(() => { + let fs: FileSystem; + describe('ArbTranslationSerializer', () => { + beforeEach(() => { + fs = getFileSystem(); + }); + + describe('renderFile()', () => { + it('should convert a set of parsed messages into a JSON string', () => { + const messages: ɵParsedMessage[] = [ + mockMessage('12345', ['a', 'b', 'c'], ['PH', 'PH_1'], { + meaning: 'some meaning', + location: { + file: absoluteFrom('/project/file.ts'), + start: {line: 5, column: 10}, + end: {line: 5, column: 12} + }, + }), + mockMessage('54321', ['a', 'b', 'c'], ['PH', 'PH_1'], { + customId: 'someId', + }), + mockMessage('67890', ['a', '', 'c'], ['START_TAG_SPAN', 'CLOSE_TAG_SPAN'], { + description: 'some description', + location: { + file: absoluteFrom('/project/file.ts'), + start: {line: 5, column: 10}, + end: {line: 5, column: 12} + }, + }), + mockMessage('67890', ['a', '', 'c'], ['START_TAG_SPAN', 'CLOSE_TAG_SPAN'], { + description: 'some description', + location: { + file: absoluteFrom('/project/other.ts'), + start: {line: 2, column: 10}, + end: {line: 3, column: 12} + }, + }), + mockMessage('13579', ['', 'b', ''], ['START_BOLD_TEXT', 'CLOSE_BOLD_TEXT'], {}), + mockMessage('24680', ['a'], [], {meaning: 'meaning', description: 'and description'}), + mockMessage('80808', ['multi\nlines'], [], {}), + mockMessage('90000', [''], ['double-quotes-"'], {}), + mockMessage( + '100000', + [ + 'pre-ICU {VAR_SELECT, select, a {a} b {{INTERPOLATION}} c {pre {INTERPOLATION_1} post}} post-ICU' + ], + [], {}), + mockMessage( + '100001', + [ + '{VAR_PLURAL, plural, one {{START_BOLD_TEXT}something bold{CLOSE_BOLD_TEXT}} other {pre {START_TAG_SPAN}middle{CLOSE_TAG_SPAN} post}}' + ], + [], {}), + ]; + const serializer = new ArbTranslationSerializer('xx', fs.resolve('/project'), fs); + const output = serializer.serialize(messages); + expect(output.split('\n')).toEqual([ + '{', + ' "@@locale": "xx",', + ' "12345": "a{$PH}b{$PH_1}c",', + ' "@12345": {', + ' "x-locations": [', + ' {', + ' "file": "file.ts",', + ' "start": { "line": "5", "column": "10" },', + ' "end": { "line": "5", "column": "12" }', + ' }', + ' ]', + ' },', + ' "someId": "a{$PH}b{$PH_1}c",', + ' "67890": "a{$START_TAG_SPAN}{$CLOSE_TAG_SPAN}c",', + ' "@67890": {', + ' "description": "some description",', + ' "x-locations": [', + ' {', + ' "file": "file.ts",', + ' "start": { "line": "5", "column": "10" },', + ' "end": { "line": "5", "column": "12" }', + ' },', + ' {', + ' "file": "other.ts",', + ' "start": { "line": "2", "column": "10" },', + ' "end": { "line": "3", "column": "12" }', + ' }', + ' ]', + ' },', + ' "13579": "{$START_BOLD_TEXT}b{$CLOSE_BOLD_TEXT}",', + ' "24680": "a",', + ' "@24680": {', + ' "description": "and description"', + ' },', + ' "80808": "multi\\nlines",', + ' "90000": "",', + ' "100000": "pre-ICU {VAR_SELECT, select, a {a} b {{INTERPOLATION}} c {pre {INTERPOLATION_1} post}} post-ICU",', + ' "100001": "{VAR_PLURAL, plural, one {{START_BOLD_TEXT}something bold{CLOSE_BOLD_TEXT}} other {pre {START_TAG_SPAN}middle{CLOSE_TAG_SPAN} post}}"', + '}', + ]); + }); + }); + }); +}); diff --git a/packages/localize/src/tools/test/translate/translation_files/translation_parsers/arb_translation_parser_spec.ts b/packages/localize/src/tools/test/translate/translation_files/translation_parsers/arb_translation_parser_spec.ts new file mode 100644 index 0000000000..54778105a9 --- /dev/null +++ b/packages/localize/src/tools/test/translate/translation_files/translation_parsers/arb_translation_parser_spec.ts @@ -0,0 +1,50 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ +import {ɵmakeTemplateObject} from '@angular/localize'; +import {ArbTranslationParser} from '../../../../src/translate/translation_files/translation_parsers/arb_translation_parser'; + +describe('SimpleArbTranslationParser', () => { + describe('canParse()', () => { + it('should return true if the file extension is `.json` and contains `@@locale` property', + () => { + const parser = new ArbTranslationParser(); + expect(parser.canParse('/some/file.xlf', '')).toBe(false); + expect(parser.canParse('/some/file.json', 'xxx')).toBe(false); + expect(parser.canParse('/some/file.json', '{ "someKey": "someValue" }')).toBe(false); + expect(parser.canParse('/some/file.json', '{ "@@locale": "en", "someKey": "someValue" }')) + .toBeTruthy(); + }); + }); + + describe('parse()', () => { + it('should extract the locale from the JSON contents', () => { + const parser = new ArbTranslationParser(); + const result = parser.parse('/some/file.json', '{"@@locale": "en"}'); + expect(result.locale).toEqual('en'); + }); + + it('should extract and process the translations from the JSON contents', () => { + const parser = new ArbTranslationParser(); + const result = parser.parse('/some/file.json', `{ + "@@locale": "fr", + "customId": "Bonjour, {$ph_1}!", + "@customId": { + "type": "text", + "description": "Some description" + } + }`); + expect(result.translations).toEqual({ + 'customId': { + text: 'Bonjour, {$ph_1}!', + messageParts: ɵmakeTemplateObject(['Bonjour, ', '!'], ['Bonjour, ', '!']), + placeholderNames: ['ph_1'], + }, + }); + }); + }); +});