diff --git a/packages/localize/package.json b/packages/localize/package.json index 27aa1dd296..37a802f06c 100644 --- a/packages/localize/package.json +++ b/packages/localize/package.json @@ -3,7 +3,8 @@ "version": "0.0.0-PLACEHOLDER", "description": "Angular - library for localizing messages", "bin": { - "localize-translate": "./src/tools/src/translate/main.js" + "localize-translate": "./src/tools/src/translate/main.js", + "localize-extract": "./src/tools/src/extract/main.js" }, "author": "angular", "license": "MIT", diff --git a/packages/localize/src/tools/BUILD.bazel b/packages/localize/src/tools/BUILD.bazel index 6f37b1b7c2..c13422f9d8 100644 --- a/packages/localize/src/tools/BUILD.bazel +++ b/packages/localize/src/tools/BUILD.bazel @@ -20,6 +20,8 @@ ts_library( deps = [ "//packages/compiler", "//packages/compiler-cli/src/ngtsc/file_system", + "//packages/compiler-cli/src/ngtsc/logging", + "//packages/compiler-cli/src/ngtsc/sourcemaps", "//packages/localize", "@npm//@babel/core", "@npm//@babel/types", diff --git a/packages/localize/src/tools/src/extract/extraction.ts b/packages/localize/src/tools/src/extract/extraction.ts new file mode 100644 index 0000000000..242e88df9e --- /dev/null +++ b/packages/localize/src/tools/src/extract/extraction.ts @@ -0,0 +1,108 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ +import {AbsoluteFsPath, FileSystem, PathSegment} from '@angular/compiler-cli/src/ngtsc/file_system'; +import {Logger} from '@angular/compiler-cli/src/ngtsc/logging'; +import {SourceFile, SourceFileLoader} from '@angular/compiler-cli/src/ngtsc/sourcemaps'; +import {ɵParsedMessage, ɵSourceLocation} from '@angular/localize'; +import {transformSync} from '@babel/core'; + +import {makeEs2015ExtractPlugin} from './source_files/es2015_extract_plugin'; +import {makeEs5ExtractPlugin} from './source_files/es5_extract_plugin'; + +export interface ExtractionOptions { + basePath: AbsoluteFsPath; + useSourceMaps?: boolean; + localizeName?: string; +} + +/** + * Extracts parsed messages from file contents, by parsing the contents as JavaScript + * and looking for occurrences of `$localize` in the source code. + */ +export class MessageExtractor { + private basePath: AbsoluteFsPath; + private useSourceMaps: boolean; + private localizeName: string; + private loader: SourceFileLoader; + + constructor( + private fs: FileSystem, private logger: Logger, + {basePath, useSourceMaps = true, localizeName = '$localize'}: ExtractionOptions) { + this.basePath = basePath; + this.useSourceMaps = useSourceMaps; + this.localizeName = localizeName; + this.loader = new SourceFileLoader(this.fs, this.logger, {webpack: basePath}); + } + + extractMessages( + filename: string, + ): ɵParsedMessage[] { + const messages: ɵParsedMessage[] = []; + const sourceCode = this.fs.readFile(this.fs.resolve(this.basePath, filename)); + if (sourceCode.includes(this.localizeName)) { + // Only bother to parse the file if it contains a reference to `$localize`. + transformSync(sourceCode, { + sourceRoot: this.basePath, + filename, + plugins: [ + makeEs2015ExtractPlugin(messages, this.localizeName), + makeEs5ExtractPlugin(messages, this.localizeName), + ], + code: false, + ast: false + }); + } + if (this.useSourceMaps) { + this.updateSourceLocations(filename, sourceCode, messages); + } + return messages; + } + + /** + * Update the location of each message to point to the source-mapped original source location, if + * available. + */ + private updateSourceLocations(filename: string, contents: string, messages: ɵParsedMessage[]): + void { + const sourceFile = + this.loader.loadSourceFile(this.fs.resolve(this.basePath, filename), contents); + if (sourceFile === null) { + return; + } + for (const message of messages) { + if (message.location !== undefined) { + message.location = this.getOriginalLocation(sourceFile, message.location); + } + } + } + + /** + * Find the original location using source-maps if available. + * + * @param sourceFile The generated `sourceFile` that contains the `location`. + * @param location The location within the generated `sourceFile` that needs mapping. + * + * @returns A new location that refers to the original source location mapped from the given + * `location` in the generated `sourceFile`. + */ + private getOriginalLocation(sourceFile: SourceFile, location: ɵSourceLocation): ɵSourceLocation { + const originalStart = + sourceFile.getOriginalLocation(location.start.line, location.start.column); + if (originalStart === null) { + return location; + } + const originalEnd = sourceFile.getOriginalLocation(location.end.line, location.end.column); + const start = {line: originalStart.line, column: originalStart.column}; + // We check whether the files are the same, since the returned location can only have a single + // `file` and it would not make sense to store the end position from a different source file. + const end = (originalEnd !== null && originalEnd.file === originalStart.file) ? + {line: originalEnd.line, column: originalEnd.column} : + start; + return {file: originalStart.file, start, end}; + } +} diff --git a/packages/localize/src/tools/src/extract/main.ts b/packages/localize/src/tools/src/extract/main.ts new file mode 100644 index 0000000000..517146945b --- /dev/null +++ b/packages/localize/src/tools/src/extract/main.ts @@ -0,0 +1,178 @@ +#!/usr/bin/env node +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ +import {getFileSystem, setFileSystem, NodeJSFileSystem, AbsoluteFsPath} from '@angular/compiler-cli/src/ngtsc/file_system'; +import {ConsoleLogger, Logger, LogLevel} from '@angular/compiler-cli/src/ngtsc/logging'; +import {ɵParsedMessage} from '@angular/localize'; +import * as glob from 'glob'; +import * as yargs from 'yargs'; +import {MessageExtractor} from './extraction'; +import {TranslationSerializer} from './translation_files/translation_serializer'; +import {SimpleJsonTranslationSerializer} from './translation_files/json_translation_serializer'; +import {Xliff1TranslationSerializer} from './translation_files/xliff1_translation_serializer'; +import {Xliff2TranslationSerializer} from './translation_files/xliff2_translation_serializer'; +import {XmbTranslationSerializer} from './translation_files/xmb_translation_serializer'; + +if (require.main === module) { + const args = process.argv.slice(2); + const options = + yargs + .option('l', { + alias: 'locale', + describe: 'The locale of the source being processed', + default: 'en', + }) + .option('r', { + alias: 'root', + default: '.', + describe: 'The root path for other paths provided in these options.\n' + + 'This should either be absolute or relative to the current working directory.' + }) + .option('s', { + alias: 'source', + required: true, + describe: + 'A glob pattern indicating what files to search for translations, e.g. `./dist/**/*.js`.\n' + + 'This should be relative to the root path.', + }) + .option('f', { + alias: 'format', + required: true, + choices: ['xmb', 'xlf', 'xlif', 'xliff', 'xlf2', 'xlif2', 'xliff2', 'json'], + describe: 'The format of the translation file.', + }) + .option('o', { + alias: 'outputPath', + required: true, + describe: + 'A path to where the translation file will be written. This should be relative to the root path.' + }) + .option('loglevel', { + describe: 'The lowest severity logging message that should be output.', + choices: ['debug', 'info', 'warn', 'error'], + }) + .option('useSourceMaps', { + type: 'boolean', + default: true, + describe: + 'Whether to generate source information in the output files by following source-map mappings found in the source files' + }) + .option('useLegacyIds', { + type: 'boolean', + default: true, + describe: + 'Whether to use the legacy id format for messages that were extracted from Angular templates.' + }) + .strict() + .help() + .parse(args); + + const fs = new NodeJSFileSystem(); + setFileSystem(fs); + + const rootPath = options['root']; + const sourceFilePaths = glob.sync(options['source'], {cwd: rootPath, nodir: true}); + const logLevel = options['loglevel'] as (keyof typeof LogLevel) | undefined; + const logger = new ConsoleLogger(logLevel ? LogLevel[logLevel] : LogLevel.warn); + + + extractTranslations({ + rootPath, + sourceFilePaths, + sourceLocale: options['locale'], + format: options['format'], + outputPath: options['outputPath'], + logger, + useSourceMaps: options['useSourceMaps'], + useLegacyIds: options['useLegacyIds'], + }); +} + +export interface ExtractTranslationsOptions { + /** + * The locale of the source being processed. + */ + sourceLocale: string; + /** + * The base path for other paths provided in these options. + * This should either be absolute or relative to the current working directory. + */ + rootPath: string; + /** + * An array of paths to files to search for translations. These should be relative to the + * rootPath. + */ + sourceFilePaths: string[]; + /** + * The format of the translation file. + */ + format: string; + /** + * A path to where the translation file will be written. This should be relative to the rootPath. + */ + outputPath: string; + /** + * The logger to use for diagnostic messages. + */ + logger: Logger; + /** + * Whether to generate source information in the output files by following source-map mappings + * found in the source file. + */ + useSourceMaps: boolean; + /** + * Whether to use the legacy id format for messages that were extracted from Angular templates + */ + useLegacyIds: boolean; +} + +export function extractTranslations({ + rootPath, + sourceFilePaths, + sourceLocale, + format, + outputPath: output, + logger, + useSourceMaps, + useLegacyIds +}: ExtractTranslationsOptions) { + const fs = getFileSystem(); + const extractor = + new MessageExtractor(fs, logger, {basePath: fs.resolve(rootPath), useSourceMaps}); + + const messages: ɵParsedMessage[] = []; + for (const file of sourceFilePaths) { + messages.push(...extractor.extractMessages(file)); + } + + const outputPath = fs.resolve(rootPath, output); + const serializer = getSerializer(format, sourceLocale, fs.dirname(outputPath), useLegacyIds); + const translationFile = serializer.serialize(messages); + fs.ensureDir(fs.dirname(outputPath)); + fs.writeFile(outputPath, translationFile); +} + +export function getSerializer( + format: string, sourceLocale: string, rootPath: AbsoluteFsPath, + useLegacyIds: boolean): TranslationSerializer { + switch (format) { + case 'xlf': + case 'xlif': + case 'xliff': + return new Xliff1TranslationSerializer(sourceLocale, rootPath, useLegacyIds); + case 'xlf2': + case 'xlif2': + case 'xliff2': + return new Xliff2TranslationSerializer(sourceLocale, rootPath, useLegacyIds); + case 'xmb': + return new XmbTranslationSerializer(rootPath, useLegacyIds); + case 'json': + return new SimpleJsonTranslationSerializer(sourceLocale); + } + throw new Error(`No translation serializer can handle the provided format: ${format}`); +} \ No newline at end of file diff --git a/packages/localize/src/tools/src/extract/source_files/es2015_extract_plugin.ts b/packages/localize/src/tools/src/extract/source_files/es2015_extract_plugin.ts new file mode 100644 index 0000000000..cada054ef4 --- /dev/null +++ b/packages/localize/src/tools/src/extract/source_files/es2015_extract_plugin.ts @@ -0,0 +1,29 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ +import {ɵParsedMessage, ɵparseMessage} from '@angular/localize'; +import {NodePath, PluginObj} from '@babel/core'; +import {TaggedTemplateExpression} from '@babel/types'; + +import {getLocation, isGlobalIdentifier, isNamedIdentifier, unwrapMessagePartsFromTemplateLiteral} from '../../source_file_utils'; + +export function makeEs2015ExtractPlugin( + messages: ɵParsedMessage[], localizeName = '$localize'): PluginObj { + return { + visitor: { + TaggedTemplateExpression(path: NodePath) { + const tag = path.get('tag'); + if (isNamedIdentifier(tag, localizeName) && isGlobalIdentifier(tag)) { + const messageParts = unwrapMessagePartsFromTemplateLiteral(path.node.quasi.quasis); + const location = getLocation(path.get('quasi')); + const message = ɵparseMessage(messageParts, path.node.quasi.expressions, location); + messages.push(message); + } + } + } + }; +} diff --git a/packages/localize/src/tools/src/extract/source_files/es5_extract_plugin.ts b/packages/localize/src/tools/src/extract/source_files/es5_extract_plugin.ts new file mode 100644 index 0000000000..4312f1e3c3 --- /dev/null +++ b/packages/localize/src/tools/src/extract/source_files/es5_extract_plugin.ts @@ -0,0 +1,30 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ +import {ɵParsedMessage, ɵparseMessage} from '@angular/localize'; +import {NodePath, PluginObj} from '@babel/core'; +import {CallExpression} from '@babel/types'; + +import {getLocation, isGlobalIdentifier, isNamedIdentifier, unwrapMessagePartsFromLocalizeCall, unwrapSubstitutionsFromLocalizeCall} from '../../source_file_utils'; + +export function makeEs5ExtractPlugin( + messages: ɵParsedMessage[], localizeName = '$localize'): PluginObj { + return { + visitor: { + CallExpression(callPath: NodePath) { + const calleePath = callPath.get('callee'); + if (isNamedIdentifier(calleePath, localizeName) && isGlobalIdentifier(calleePath)) { + const messageParts = unwrapMessagePartsFromLocalizeCall(callPath); + const expressions = unwrapSubstitutionsFromLocalizeCall(callPath.node); + const location = getLocation(callPath); + const message = ɵparseMessage(messageParts, expressions, location); + messages.push(message); + } + } + } + }; +} diff --git a/packages/localize/src/tools/src/extract/translation_files/json_translation_serializer.ts b/packages/localize/src/tools/src/extract/translation_files/json_translation_serializer.ts new file mode 100644 index 0000000000..9c27ef8b70 --- /dev/null +++ b/packages/localize/src/tools/src/extract/translation_files/json_translation_serializer.ts @@ -0,0 +1,32 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ +import {ɵMessageId, ɵParsedMessage, ɵSourceMessage} from '@angular/localize'; +import {TranslationSerializer} from './translation_serializer'; + + +interface SimpleJsonTranslationFile { + locale: string; + translations: Record<ɵMessageId, ɵSourceMessage>; +} + +/** + * This is a semi-public bespoke serialization format that is used for testing and sometimes as a + * format for storing translations that will be inlined at runtime. + * + * @see SimpleJsonTranslationParser + */ +export class SimpleJsonTranslationSerializer implements TranslationSerializer { + constructor(private sourceLocale: string) {} + serialize(messages: ɵParsedMessage[]): string { + const fileObj: SimpleJsonTranslationFile = {locale: this.sourceLocale, translations: {}}; + for (const message of messages) { + fileObj.translations[message.id] = message.text; + } + return JSON.stringify(fileObj, null, 2); + } +} diff --git a/packages/localize/src/tools/src/extract/translation_files/translation_serializer.ts b/packages/localize/src/tools/src/extract/translation_files/translation_serializer.ts new file mode 100644 index 0000000000..a313966bbe --- /dev/null +++ b/packages/localize/src/tools/src/extract/translation_files/translation_serializer.ts @@ -0,0 +1,21 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ +import {ɵParsedMessage} from '@angular/localize'; + +/** + * Implement this interface to provide a class that can render messages into a translation file. + */ +export interface TranslationSerializer { + /** + * Serialize the contents of a translation file containing the given `messages`. + * + * @param messages The messages to render to the file. + * @returns The contents of the serialized file. + */ + serialize(messages: ɵParsedMessage[]): string; +} diff --git a/packages/localize/src/tools/src/extract/translation_files/xliff1_translation_serializer.ts b/packages/localize/src/tools/src/extract/translation_files/xliff1_translation_serializer.ts new file mode 100644 index 0000000000..8c1afb69f2 --- /dev/null +++ b/packages/localize/src/tools/src/extract/translation_files/xliff1_translation_serializer.ts @@ -0,0 +1,111 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ +import {AbsoluteFsPath, relative} from '@angular/compiler-cli/src/ngtsc/file_system'; +import {ɵParsedMessage, ɵSourceLocation} from '@angular/localize'; + +import {TranslationSerializer} from './translation_serializer'; +import {XmlFile} from './xml_file'; + +/** This is the number of characters that a legacy Xliff 1.2 message id has. */ +const LEGACY_XLIFF_MESSAGE_LENGTH = 40; + +/** + * A translation serializer that can write XLIFF 1.2 formatted files. + * + * http://docs.oasis-open.org/xliff/v1.2/os/xliff-core.html + * http://docs.oasis-open.org/xliff/v1.2/xliff-profile-html/xliff-profile-html-1.2.html + * + * @see Xliff1TranslationParser + */ +export class Xliff1TranslationSerializer implements TranslationSerializer { + constructor( + private sourceLocale: string, private basePath: AbsoluteFsPath, + private useLegacyIds: boolean) {} + + serialize(messages: ɵParsedMessage[]): string { + const ids = new Set(); + const xml = new XmlFile(); + xml.startTag('xliff', {'version': '1.2', 'xmlns': 'urn:oasis:names:tc:xliff:document:1.2'}); + xml.startTag('file', {'source-language': this.sourceLocale, 'datatype': 'plaintext'}); + xml.startTag('body'); + for (const message of messages) { + const id = this.getMessageId(message); + if (ids.has(id)) { + // Do not render the same message more than once + continue; + } + ids.add(id); + + xml.startTag('trans-unit', {id, datatype: 'html'}); + xml.startTag('source', {}, {preserveWhitespace: true}); + this.serializeMessage(xml, message); + xml.endTag('source', {preserveWhitespace: false}); + if (message.location) { + this.serializeLocation(xml, message.location); + } + if (message.description) { + this.serializeNote(xml, 'description', message.description); + } + if (message.meaning) { + this.serializeNote(xml, 'meaning', message.meaning); + } + xml.endTag('trans-unit'); + } + xml.endTag('body'); + xml.endTag('file'); + xml.endTag('xliff'); + return xml.toString(); + } + + private serializeMessage(xml: XmlFile, message: ɵParsedMessage): void { + xml.text(message.messageParts[0]); + for (let i = 1; i < message.messageParts.length; i++) { + xml.startTag('x', {id: message.placeholderNames[i - 1]}, {selfClosing: true}); + xml.text(message.messageParts[i]); + } + } + + private serializeNote(xml: XmlFile, name: string, value: string): void { + xml.startTag('note', {priority: '1', from: name}, {preserveWhitespace: true}); + xml.text(value); + xml.endTag('note', {preserveWhitespace: false}); + } + + private serializeLocation(xml: XmlFile, location: ɵSourceLocation): void { + xml.startTag('context-group', {purpose: 'location'}); + this.renderContext(xml, 'sourcefile', relative(this.basePath, location.file)); + const endLineString = location.end !== undefined && location.end.line !== location.start.line ? + `,${location.end.line + 1}` : + ''; + this.renderContext(xml, 'linenumber', `${location.start.line + 1}${endLineString}`); + xml.endTag('context-group'); + } + + private renderContext(xml: XmlFile, type: string, value: string): void { + xml.startTag('context', {'context-type': type}, {preserveWhitespace: true}); + xml.text(value); + xml.endTag('context', {preserveWhitespace: false}); + } + + /** + * Get the id for the given `message`. + * + * If we have requested legacy message ids, then try to return the appropriate id + * from the list of legacy ids that were extracted. + * + * Otherwise return the canonical message id. + * + * An Xliff 1.2 legacy message id is a hex encoded SHA-1 string, which is 40 characters long. See + * https://csrc.nist.gov/csrc/media/publications/fips/180/4/final/documents/fips180-4-draft-aug2014.pdf + */ + private getMessageId(message: ɵParsedMessage): string { + return this.useLegacyIds && message.legacyIds !== undefined && + message.legacyIds.find(id => id.length === LEGACY_XLIFF_MESSAGE_LENGTH) || + message.id; + } +} diff --git a/packages/localize/src/tools/src/extract/translation_files/xliff2_translation_serializer.ts b/packages/localize/src/tools/src/extract/translation_files/xliff2_translation_serializer.ts new file mode 100644 index 0000000000..31f2a42eeb --- /dev/null +++ b/packages/localize/src/tools/src/extract/translation_files/xliff2_translation_serializer.ts @@ -0,0 +1,119 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ +import {AbsoluteFsPath, relative} from '@angular/compiler-cli/src/ngtsc/file_system'; +import {ɵParsedMessage} from '@angular/localize'; + +import {TranslationSerializer} from './translation_serializer'; +import {XmlFile} from './xml_file'; + +/** This is the maximum number of characters that can appear in a legacy XLIFF 2.0 message id. */ +const MAX_LEGACY_XLIFF_2_MESSAGE_LENGTH = 20; + +/** + * A translation serializer that can write translations in XLIFF 2 format. + * + * http://docs.oasis-open.org/xliff/xliff-core/v2.0/os/xliff-core-v2.0-os.html + * + * @see Xliff2TranslationParser + */ +export class Xliff2TranslationSerializer implements TranslationSerializer { + constructor( + private sourceLocale: string, private basePath: AbsoluteFsPath, + private useLegacyIds: boolean) {} + + serialize(messages: ɵParsedMessage[]): string { + const ids = new Set(); + const xml = new XmlFile(); + xml.startTag('xliff', { + 'version': '2.0', + 'xmlns': 'urn:oasis:names:tc:xliff:document:2.0', + 'srcLang': this.sourceLocale + }); + xml.startTag('file'); + for (const message of messages) { + const id = this.getMessageId(message); + if (ids.has(id)) { + // Do not render the same message more than once + continue; + } + ids.add(id); + xml.startTag('unit', {id}); + if (message.meaning || message.description) { + xml.startTag('notes'); + if (message.location) { + const {file, start, end} = message.location; + const endLineString = + end !== undefined && end.line !== start.line ? `,${end.line + 1}` : ''; + this.serializeNote( + xml, 'location', + `${relative(this.basePath, file)}:${start.line + 1}${endLineString}`); + } + if (message.description) { + this.serializeNote(xml, 'description', message.description); + } + if (message.meaning) { + this.serializeNote(xml, 'meaning', message.meaning); + } + xml.endTag('notes'); + } + xml.startTag('segment'); + xml.startTag('source', {}, {preserveWhitespace: true}); + this.serializeMessage(xml, message); + xml.endTag('source', {preserveWhitespace: false}); + xml.endTag('segment'); + xml.endTag('unit'); + } + xml.endTag('file'); + xml.endTag('xliff'); + return xml.toString(); + } + + private serializeMessage(xml: XmlFile, message: ɵParsedMessage): void { + xml.text(message.messageParts[0]); + for (let i = 1; i < message.messageParts.length; i++) { + const placeholderName = message.placeholderNames[i - 1]; + if (placeholderName.startsWith('START_')) { + xml.startTag('pc', { + id: `${i}`, + equivStart: placeholderName, + equivEnd: placeholderName.replace(/^START/, 'CLOSE') + }); + } else if (placeholderName.startsWith('CLOSE_')) { + xml.endTag('pc'); + } else { + xml.startTag('ph', {id: `${i}`, equiv: placeholderName}, {selfClosing: true}); + } + xml.text(message.messageParts[i]); + } + } + + private serializeNote(xml: XmlFile, name: string, value: string) { + xml.startTag('note', {category: name}, {preserveWhitespace: true}); + xml.text(value); + xml.endTag('note', {preserveWhitespace: false}); + } + + /** + * Get the id for the given `message`. + * + * If we have requested legacy message ids, then try to return the appropriate id + * from the list of legacy ids that were extracted. + * + * Otherwise return the canonical message id. + * + * An Xliff 2.0 legacy message id is a 64 bit number encoded as a decimal string, which will have + * at most 20 digits, since 2^65-1 = 36,893,488,147,419,103,231. This digest is based on: + * https://github.com/google/closure-compiler/blob/master/src/com/google/javascript/jscomp/GoogleJsMessageIdGenerator.java + */ + private getMessageId(message: ɵParsedMessage): string { + return this.useLegacyIds && message.legacyIds !== undefined && + message.legacyIds.find( + id => id.length <= MAX_LEGACY_XLIFF_2_MESSAGE_LENGTH && !/[^0-9]/.test(id)) || + message.id; + } +} diff --git a/packages/localize/src/tools/src/extract/translation_files/xmb_translation_serializer.ts b/packages/localize/src/tools/src/extract/translation_files/xmb_translation_serializer.ts new file mode 100644 index 0000000000..ffb4383409 --- /dev/null +++ b/packages/localize/src/tools/src/extract/translation_files/xmb_translation_serializer.ts @@ -0,0 +1,104 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ +import {AbsoluteFsPath, relative} from '@angular/compiler-cli/src/ngtsc/file_system'; +import {ɵParsedMessage, ɵSourceLocation} from '@angular/localize'; + +import {TranslationSerializer} from './translation_serializer'; +import {XmlFile} from './xml_file'; + +/** + * A translation serializer that can write files in XMB format. + * + * http://cldr.unicode.org/development/development-process/design-proposals/xmb + * + * @see XmbTranslationParser + */ +export class XmbTranslationSerializer implements TranslationSerializer { + constructor(private basePath: AbsoluteFsPath, private useLegacyIds: boolean) {} + + serialize(messages: ɵParsedMessage[]): string { + const ids = new Set(); + const xml = new XmlFile(); + xml.rawText( + `\n` + + `\n` + + `\n` + + `\n` + + `\n` + + `\n` + + `\n` + + `\n` + + `\n` + + `\n` + + `\n` + + `\n` + + `\n` + + `\n` + + `\n` + + `\n` + + `\n` + + `\n` + + `\n` + + `]>\n`); + xml.startTag('messagebundle'); + for (const message of messages) { + const id = this.getMessageId(message); + if (ids.has(id)) { + // Do not render the same message more than once + continue; + } + ids.add(id); + xml.startTag( + 'msg', {id, desc: message.description, meaning: message.meaning}, + {preserveWhitespace: true}); + if (message.location) { + this.serializeLocation(xml, message.location); + } + this.serializeMessage(xml, message); + xml.endTag('msg', {preserveWhitespace: false}); + } + xml.endTag('messagebundle'); + return xml.toString(); + } + + private serializeLocation(xml: XmlFile, location: ɵSourceLocation): void { + xml.startTag('source'); + const endLineString = location.end !== undefined && location.end.line !== location.start.line ? + `,${location.end.line + 1}` : + ''; + xml.text(`${relative(this.basePath, location.file)}:${location.start.line}${endLineString}`); + xml.endTag('source'); + } + + private serializeMessage(xml: XmlFile, message: ɵParsedMessage): void { + xml.text(message.messageParts[0]); + for (let i = 1; i < message.messageParts.length; i++) { + xml.startTag('ph', {name: message.placeholderNames[i - 1]}, {selfClosing: true}); + xml.text(message.messageParts[i]); + } + } + + /** + * Get the id for the given `message`. + * + * If we have requested legacy message ids, then try to return the appropriate id + * from the list of legacy ids that were extracted. + * + * Otherwise return the canonical message id. + * + * An XMB legacy message id is a 64 bit number encoded as a decimal string, which will have + * at most 20 digits, since 2^65-1 = 36,893,488,147,419,103,231. This digest is based on: + * https://github.com/google/closure-compiler/blob/master/src/com/google/javascript/jscomp/GoogleJsMessageIdGenerator.java + */ + private getMessageId(message: ɵParsedMessage): string { + return this.useLegacyIds && message.legacyIds !== undefined && + message.legacyIds.find(id => id.length <= 20 && !/[^0-9]/.test(id)) || + message.id; + } +} diff --git a/packages/localize/src/tools/src/extract/translation_files/xml_file.ts b/packages/localize/src/tools/src/extract/translation_files/xml_file.ts new file mode 100644 index 0000000000..a38ca3b11e --- /dev/null +++ b/packages/localize/src/tools/src/extract/translation_files/xml_file.ts @@ -0,0 +1,106 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +interface Options { + selfClosing?: boolean; + preserveWhitespace?: boolean; +} + +export class XmlFile { + private output = '\n'; + private indent = ''; + private elements: string[] = []; + private preservingWhitespace = false; + toString() { + return this.output; + } + + startTag( + name: string, attributes: Record = {}, + {selfClosing = false, preserveWhitespace}: Options = {}): this { + if (!this.preservingWhitespace) { + this.output += this.indent; + } + + this.output += `<${name}`; + + for (const [attrName, attrValue] of Object.entries(attributes)) { + if (attrValue) { + this.output += ` ${attrName}="${escapeXml(attrValue)}"`; + } + } + + if (selfClosing) { + this.output += '/>'; + } else { + this.output += '>'; + this.elements.push(name); + this.incIndent(); + } + + if (preserveWhitespace !== undefined) { + this.preservingWhitespace = preserveWhitespace; + } + if (!this.preservingWhitespace) { + this.output += `\n`; + } + return this; + } + + endTag(name: string, {preserveWhitespace}: Options = {}): this { + const expectedTag = this.elements.pop(); + if (expectedTag !== name) { + throw new Error(`Unexpected closing tag: "${name}", expected: "${expectedTag}"`); + } + + this.decIndent(); + + if (!this.preservingWhitespace) { + this.output += this.indent; + } + this.output += ``; + + if (preserveWhitespace !== undefined) { + this.preservingWhitespace = preserveWhitespace; + } + if (!this.preservingWhitespace) { + this.output += `\n`; + } + return this; + } + + text(str: string): this { + this.output += escapeXml(str); + return this; + } + + rawText(str: string): this { + this.output += str; + return this; + } + + private incIndent() { + this.indent = this.indent + ' '; + } + private decIndent() { + this.indent = this.indent.slice(0, -2); + } +} + +const _ESCAPED_CHARS: [RegExp, string][] = [ + [/&/g, '&'], + [/"/g, '"'], + [/'/g, '''], + [//g, '>'], +]; + +function escapeXml(text: string): string { + return _ESCAPED_CHARS.reduce( + (text: string, entry: [RegExp, string]) => text.replace(entry[0], entry[1]), text); +} \ No newline at end of file diff --git a/packages/localize/src/tools/src/translate/translation_files/translation_parsers/simple_json_translation_parser.ts b/packages/localize/src/tools/src/translate/translation_files/translation_parsers/simple_json_translation_parser.ts index e8cc0bc618..898cd1330a 100644 --- a/packages/localize/src/tools/src/translate/translation_files/translation_parsers/simple_json_translation_parser.ts +++ b/packages/localize/src/tools/src/translate/translation_files/translation_parsers/simple_json_translation_parser.ts @@ -22,6 +22,8 @@ import {ParsedTranslationBundle, TranslationParser} from './translation_parser'; * } * } * ``` + * + * @see SimpleJsonTranslationSerializer */ export class SimpleJsonTranslationParser implements TranslationParser { canParse(filePath: string, contents: string): Object|false { diff --git a/packages/localize/src/tools/src/translate/translation_files/translation_parsers/xliff1_translation_parser.ts b/packages/localize/src/tools/src/translate/translation_files/translation_parsers/xliff1_translation_parser.ts index 04c92e9d1f..664ec1c354 100644 --- a/packages/localize/src/tools/src/translate/translation_files/translation_parsers/xliff1_translation_parser.ts +++ b/packages/localize/src/tools/src/translate/translation_files/translation_parsers/xliff1_translation_parser.ts @@ -22,6 +22,7 @@ import {addParseDiagnostic, addParseError, canParseXml, getAttribute, isNamedEle * http://docs.oasis-open.org/xliff/v1.2/os/xliff-core.html * http://docs.oasis-open.org/xliff/v1.2/xliff-profile-html/xliff-profile-html-1.2.html * + * @see Xliff1TranslationSerializer */ export class Xliff1TranslationParser implements TranslationParser { canParse(filePath: string, contents: string): XmlTranslationParserHint|false { diff --git a/packages/localize/src/tools/src/translate/translation_files/translation_parsers/xliff2_translation_parser.ts b/packages/localize/src/tools/src/translate/translation_files/translation_parsers/xliff2_translation_parser.ts index 460a84592e..7c83283e19 100644 --- a/packages/localize/src/tools/src/translate/translation_files/translation_parsers/xliff2_translation_parser.ts +++ b/packages/localize/src/tools/src/translate/translation_files/translation_parsers/xliff2_translation_parser.ts @@ -21,6 +21,7 @@ import {addParseDiagnostic, addParseError, canParseXml, getAttribute, isNamedEle * * http://docs.oasis-open.org/xliff/xliff-core/v2.0/os/xliff-core-v2.0-os.html * + * @see Xliff2TranslationSerializer */ export class Xliff2TranslationParser implements TranslationParser { canParse(filePath: string, contents: string): XmlTranslationParserHint|false { diff --git a/packages/localize/src/tools/src/translate/translation_files/translation_parsers/xtb_translation_parser.ts b/packages/localize/src/tools/src/translate/translation_files/translation_parsers/xtb_translation_parser.ts index 1def01ac55..494158dd8c 100644 --- a/packages/localize/src/tools/src/translate/translation_files/translation_parsers/xtb_translation_parser.ts +++ b/packages/localize/src/tools/src/translate/translation_files/translation_parsers/xtb_translation_parser.ts @@ -19,7 +19,11 @@ import {addParseDiagnostic, addParseError, canParseXml, getAttribute, parseInner /** - * A translation parser that can load XB files. + * A translation parser that can load XTB files. + * + * http://cldr.unicode.org/development/development-process/design-proposals/xmb + * + * @see XmbTranslationSerializer */ export class XtbTranslationParser implements TranslationParser { canParse(filePath: string, contents: string): XmlTranslationParserHint|false { diff --git a/packages/localize/src/tools/test/BUILD.bazel b/packages/localize/src/tools/test/BUILD.bazel index 0ebf350f39..c441be3b2e 100644 --- a/packages/localize/src/tools/test/BUILD.bazel +++ b/packages/localize/src/tools/test/BUILD.bazel @@ -4,15 +4,17 @@ ts_library( name = "test_lib", testonly = True, srcs = glob( - ["**/*_spec.ts"], + ["**/*.ts"], ), deps = [ "//packages:types", "//packages/compiler", "//packages/compiler-cli/src/ngtsc/file_system", "//packages/compiler-cli/src/ngtsc/file_system/testing", + "//packages/compiler-cli/src/ngtsc/logging/testing", "//packages/localize", "//packages/localize/src/tools", + "//packages/localize/src/utils", "@npm//@babel/core", "@npm//@babel/generator", "@npm//@babel/template", diff --git a/packages/localize/src/tools/test/extract/extractor_spec.ts b/packages/localize/src/tools/test/extract/extractor_spec.ts new file mode 100644 index 0000000000..7c4c4d4f36 --- /dev/null +++ b/packages/localize/src/tools/test/extract/extractor_spec.ts @@ -0,0 +1,57 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ +import {absoluteFrom, getFileSystem, relativeFrom} from '@angular/compiler-cli/src/ngtsc/file_system'; +import {runInEachFileSystem} from '@angular/compiler-cli/src/ngtsc/file_system/testing'; +import {MockLogger} from '@angular/compiler-cli/src/ngtsc/logging/testing'; + +import {MessageExtractor} from '../../src/extract/extraction'; + +runInEachFileSystem(() => { + describe('extractMessages', () => { + it('should extract a message for each $localize template tag', () => { + const fs = getFileSystem(); + const logger = new MockLogger(); + const basePath = absoluteFrom('/root/path/'); + const filename = 'relative/path.js'; + const file = fs.resolve(basePath, filename); + const extractor = new MessageExtractor(fs, logger, {basePath}); + fs.ensureDir(absoluteFrom('/root/path/relative')); + fs.writeFile(file, [ + '$localize`:meaning|description:a${1}b${2}c`;', + '$localize(__makeTemplateObject(["a", ":custom-placeholder:b", "c"], ["a", ":custom-placeholder:b", "c"]), 1, 2);' + ].join('\n')); + const messages = extractor.extractMessages(filename); + + expect(messages.length).toEqual(2); + + expect(messages[0]).toEqual({ + id: '2714330828844000684', + description: 'description', + meaning: 'meaning', + messageParts: ['a', 'b', 'c'], + text: 'a{$PH}b{$PH_1}c', + placeholderNames: ['PH', 'PH_1'], + substitutions: jasmine.any(Object), + legacyIds: [], + location: {start: {line: 0, column: 9}, end: {line: 0, column: 43}, file}, + }); + + expect(messages[1]).toEqual({ + id: '5692770902395945649', + description: '', + meaning: '', + messageParts: ['a', 'b', 'c'], + text: 'a{$custom-placeholder}b{$PH_1}c', + placeholderNames: ['custom-placeholder', 'PH_1'], + substitutions: jasmine.any(Object), + legacyIds: [], + location: {start: {line: 1, column: 0}, end: {line: 1, column: 111}, file}, + }); + }); + }); +}); diff --git a/packages/localize/src/tools/test/extract/integration/BUILD.bazel b/packages/localize/src/tools/test/extract/integration/BUILD.bazel new file mode 100644 index 0000000000..bad294d09d --- /dev/null +++ b/packages/localize/src/tools/test/extract/integration/BUILD.bazel @@ -0,0 +1,33 @@ +load("//tools:defaults.bzl", "jasmine_node_test", "ts_library") + +ts_library( + name = "test_lib", + testonly = True, + srcs = glob( + ["**/*_spec.ts"], + ), + deps = [ + "//packages:types", + "//packages/compiler-cli/src/ngtsc/file_system", + "//packages/compiler-cli/src/ngtsc/file_system/testing", + "//packages/compiler-cli/src/ngtsc/logging", + "//packages/compiler-cli/src/ngtsc/logging/testing", + "//packages/compiler-cli/test/helpers", + "//packages/localize/src/tools", + ], +) + +jasmine_node_test( + name = "integration", + bootstrap = ["//tools/testing:node_no_angular_es5"], + data = [ + "//packages/localize/src/tools/test/extract/integration/test_files", + "//packages/localize/src/tools/test/extract/integration/test_files:compile_es2015", + "//packages/localize/src/tools/test/extract/integration/test_files:compile_es5", + ], + deps = [ + ":test_lib", + "@npm//glob", + "@npm//yargs", + ], +) diff --git a/packages/localize/src/tools/test/extract/integration/main_spec.ts b/packages/localize/src/tools/test/extract/integration/main_spec.ts new file mode 100644 index 0000000000..bb7d19c2ee --- /dev/null +++ b/packages/localize/src/tools/test/extract/integration/main_spec.ts @@ -0,0 +1,228 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ +import {absoluteFrom, AbsoluteFsPath, FileSystem, getFileSystem} from '@angular/compiler-cli/src/ngtsc/file_system'; +import {runInEachFileSystem} from '@angular/compiler-cli/src/ngtsc/file_system/testing'; +import {Logger} from '@angular/compiler-cli/src/ngtsc/logging'; +import {MockLogger} from '@angular/compiler-cli/src/ngtsc/logging/testing'; +import {loadTestDirectory} from '@angular/compiler-cli/test/helpers'; + +import {extractTranslations} from '../../../src/extract/main'; + +runInEachFileSystem(() => { + let fs: FileSystem; + let logger: Logger; + let rootPath: AbsoluteFsPath; + let outputPath: AbsoluteFsPath; + let sourceFilePath: AbsoluteFsPath; + let textFile1: AbsoluteFsPath; + let textFile2: AbsoluteFsPath; + + beforeEach(() => { + fs = getFileSystem(); + logger = new MockLogger(); + rootPath = absoluteFrom('/project'); + outputPath = fs.resolve(rootPath, 'extracted-message-file'); + sourceFilePath = fs.resolve(rootPath, 'test_files/test.js'); + textFile1 = fs.resolve(rootPath, 'test_files/test-1.txt'); + textFile2 = fs.resolve(rootPath, 'test_files/test-2.txt'); + + fs.ensureDir(fs.dirname(sourceFilePath)); + loadTestDirectory(fs, __dirname + '/test_files', absoluteFrom('/project/test_files')); + }); + + describe('extractTranslations()', () => { + it('should ignore non-code files', () => { + extractTranslations({ + rootPath, + sourceLocale: 'en', + sourceFilePaths: [], + format: 'json', + outputPath, + logger, + useSourceMaps: false, + useLegacyIds: false, + }); + expect(fs.readFile(outputPath)).toEqual([ + `{`, + ` "locale": "en",`, + ` "translations": {}`, + `}`, + ].join('\n')); + }); + + it('should extract translations from source code, and write as JSON format', () => { + extractTranslations({ + rootPath, + sourceLocale: 'en-GB', + sourceFilePaths: [sourceFilePath], + format: 'json', + outputPath, + logger, + useSourceMaps: false, + useLegacyIds: false, + }); + expect(fs.readFile(outputPath)).toEqual([ + `{`, + ` "locale": "en-GB",`, + ` "translations": {`, + ` "3291030485717846467": "Hello, {$PH}!",`, + ` "8669027859022295761": "try{$PH}me"`, + ` }`, + `}`, + ].join('\n')); + }); + + it('should extract translations from source code, and write as xmb format', () => { + extractTranslations({ + rootPath, + sourceLocale: 'en', + sourceFilePaths: [sourceFilePath], + format: 'xmb', + outputPath, + logger, + useSourceMaps: false, + useLegacyIds: false, + }); + expect(fs.readFile(outputPath)).toEqual([ + ``, + ``, + ``, + ``, + ``, + ``, + ``, + ``, + ``, + ``, + ``, + ``, + ``, + ``, + ``, + ``, + ``, + ``, + ``, + ``, + `]>`, + ``, + ` test_files/test.js:1Hello, !`, + ` test_files/test.js:2tryme`, + `\n`, + ].join('\n')); + }); + + it('should extract translations from source code, and write as XLIFF 1.2 format', () => { + extractTranslations({ + rootPath, + sourceLocale: 'en-CA', + sourceFilePaths: [sourceFilePath], + format: 'xliff', + outputPath, + logger, + useSourceMaps: false, + useLegacyIds: false, + }); + expect(fs.readFile(outputPath)).toEqual([ + ``, + ``, + ` `, + ` `, + ` `, + ` Hello, !`, + ` `, + ` test_files/test.js`, + ` 2`, + ` `, + ` `, + ` `, + ` tryme`, + ` `, + ` test_files/test.js`, + ` 3`, + ` `, + ` `, + ` `, + ` `, + `\n`, + ].join('\n')); + }); + + it('should extract translations from source code, and write as XLIFF 2 format', () => { + extractTranslations({ + rootPath, + sourceLocale: 'en-AU', + sourceFilePaths: [sourceFilePath], + format: 'xliff2', + outputPath, + logger, + useSourceMaps: false, + useLegacyIds: false, + }); + expect(fs.readFile(outputPath)).toEqual([ + ``, + ``, + ` `, + ` `, + ` `, + ` Hello, !`, + ` `, + ` `, + ` `, + ` `, + ` tryme`, + ` `, + ` `, + ` `, + `\n`, + ].join('\n')); + }); + + for (const target of ['es2015', 'es5']) { + it(`should render the original location of translations, when processing an ${ + target} bundle with source-maps`, + () => { + extractTranslations({ + rootPath, + sourceLocale: 'en-CA', + sourceFilePaths: [fs.resolve(rootPath, `test_files/dist_${target}/index.js`)], + format: 'xliff', + outputPath, + logger, + useSourceMaps: true, + useLegacyIds: false, + }); + expect(fs.readFile(outputPath)).toEqual([ + ``, + ``, + ` `, + ` `, + ` `, + ` Message in !`, + ` `, + // These source file paths are due to how Bazel TypeScript compilation source-maps work + ` ../packages/localize/src/tools/test/extract/integration/test_files/src/a.ts`, + ` 3`, + ` `, + ` `, + ` `, + ` Message in !`, + ` `, + ` ../packages/localize/src/tools/test/extract/integration/test_files/src/b.ts`, + ` 3`, + ` `, + ` `, + ` `, + ` `, + `\n`, + ].join('\n')); + }); + } + }); +}); diff --git a/packages/localize/src/tools/test/extract/integration/test_files/BUILD.bazel b/packages/localize/src/tools/test/extract/integration/test_files/BUILD.bazel new file mode 100644 index 0000000000..3abde62d10 --- /dev/null +++ b/packages/localize/src/tools/test/extract/integration/test_files/BUILD.bazel @@ -0,0 +1,54 @@ +package(default_visibility = ["//packages/localize/src/tools/test/extract/integration:__pkg__"]) + +load("@npm//typescript:index.bzl", "tsc") + +tsc( + name = "compile_es5", + outs = [ + "dist_es5/index.js", + "dist_es5/index.js.map", + ], + args = [ + "--target", + "es5", + "--module", + "amd", + "--outFile", + "$(execpath dist_es5/index.js)", + "--skipLibCheck", + "--sourceMap", + "--inlineSources", + "$(execpath src/index.ts)", + ], + data = glob(["src/*.ts"]), +) + +tsc( + name = "compile_es2015", + outs = [ + "dist_es2015/index.js", + "dist_es2015/index.js.map", + ], + args = [ + "--target", + "es2015", + "--module", + "amd", + "--outFile", + "$(execpath dist_es2015/index.js)", + "--skipLibCheck", + "--sourceMap", + "--inlineSources", + "$(execpath src/index.ts)", + ], + data = glob(["src/*.ts"]), +) + +filegroup( + name = "test_files", + srcs = glob([ + "**/*.js", + "**/*.txt", + "**/*.ts", + ]), +) diff --git a/packages/localize/src/tools/test/extract/integration/test_files/src/a.ts b/packages/localize/src/tools/test/extract/integration/test_files/src/a.ts new file mode 100644 index 0000000000..47da6f3f06 --- /dev/null +++ b/packages/localize/src/tools/test/extract/integration/test_files/src/a.ts @@ -0,0 +1,3 @@ +declare const $localize: any; +const file = 'a.ts'; +export const messageA = $localize`Message in ${file}:a-file:!`; diff --git a/packages/localize/src/tools/test/extract/integration/test_files/src/b.ts b/packages/localize/src/tools/test/extract/integration/test_files/src/b.ts new file mode 100644 index 0000000000..430df79309 --- /dev/null +++ b/packages/localize/src/tools/test/extract/integration/test_files/src/b.ts @@ -0,0 +1,3 @@ +declare const $localize: any; +const file = 'b.ts'; +export const messageB = $localize`Message in ${file}:b-file:!`; diff --git a/packages/localize/src/tools/test/extract/integration/test_files/src/index.ts b/packages/localize/src/tools/test/extract/integration/test_files/src/index.ts new file mode 100644 index 0000000000..b8e0ef47e6 --- /dev/null +++ b/packages/localize/src/tools/test/extract/integration/test_files/src/index.ts @@ -0,0 +1,2 @@ +export * from './a'; +export * from './b'; diff --git a/packages/localize/src/tools/test/extract/integration/test_files/test-1.txt b/packages/localize/src/tools/test/extract/integration/test_files/test-1.txt new file mode 100644 index 0000000000..46a9dd3a2d --- /dev/null +++ b/packages/localize/src/tools/test/extract/integration/test_files/test-1.txt @@ -0,0 +1 @@ +Contents of test-1.txt \ No newline at end of file diff --git a/packages/localize/src/tools/test/extract/integration/test_files/test-2.txt b/packages/localize/src/tools/test/extract/integration/test_files/test-2.txt new file mode 100644 index 0000000000..b48d58aff5 --- /dev/null +++ b/packages/localize/src/tools/test/extract/integration/test_files/test-2.txt @@ -0,0 +1 @@ +Contents of test-2.txt \ No newline at end of file diff --git a/packages/localize/src/tools/test/extract/integration/test_files/test.js b/packages/localize/src/tools/test/extract/integration/test_files/test.js new file mode 100644 index 0000000000..cef3d96032 --- /dev/null +++ b/packages/localize/src/tools/test/extract/integration/test_files/test.js @@ -0,0 +1,3 @@ +var name = 'World'; +var message = $localize`Hello, ${name}!`; +var other = $localize(__makeTemplateObject(['try', 'me'], ['try', 'me']), 40 + 2); \ No newline at end of file diff --git a/packages/localize/src/tools/test/extract/translation_files/json_translation_serializer_spec.ts b/packages/localize/src/tools/test/extract/translation_files/json_translation_serializer_spec.ts new file mode 100644 index 0000000000..bcfa7bd36e --- /dev/null +++ b/packages/localize/src/tools/test/extract/translation_files/json_translation_serializer_spec.ts @@ -0,0 +1,42 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ +import {ɵParsedMessage} from '@angular/localize'; + +import {SimpleJsonTranslationSerializer} from '../../../src/extract/translation_files/json_translation_serializer'; + +import {mockMessage} from './mock_message'; + +describe('JsonTranslationSerializer', () => { + describe('renderFile()', () => { + it('should convert a set of parsed messages into a JSON string', () => { + const messages: ɵParsedMessage[] = [ + mockMessage('12345', ['a', 'b', 'c'], ['PH', 'PH_1'], {meaning: 'some meaning'}), + mockMessage( + '67890', ['a', '', 'c'], ['START_TAG_SPAN', 'CLOSE_TAG_SPAN'], + {description: 'some description'}), + mockMessage('13579', ['', 'b', ''], ['START_BOLD_TEXT', 'CLOSE_BOLD_TEXT'], {}), + mockMessage('24680', ['a'], [], {meaning: 'meaning', description: 'and description'}), + mockMessage('80808', ['multi\nlines'], [], {}), + ]; + const serializer = new SimpleJsonTranslationSerializer('xx'); + const output = serializer.serialize(messages); + expect(output).toEqual([ + `{`, + ` "locale": "xx",`, + ` "translations": {`, + ` "12345": "a{$PH}b{$PH_1}c",`, + ` "13579": "{$START_BOLD_TEXT}b{$CLOSE_BOLD_TEXT}",`, + ` "24680": "a",`, + ` "67890": "a{$START_TAG_SPAN}{$CLOSE_TAG_SPAN}c",`, + ` "80808": "multi\\nlines"`, + ` }`, + `}`, + ].join('\n')); + }); + }); +}); diff --git a/packages/localize/src/tools/test/extract/translation_files/mock_message.ts b/packages/localize/src/tools/test/extract/translation_files/mock_message.ts new file mode 100644 index 0000000000..ec8a747173 --- /dev/null +++ b/packages/localize/src/tools/test/extract/translation_files/mock_message.ts @@ -0,0 +1,40 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ +import {ɵParsedMessage} from '@angular/localize'; +import {SourceLocation} from '@angular/localize/src/utils'; + +export interface MockMessageOptions { + meaning?: string; + description?: string; + location?: SourceLocation; + legacyIds?: string[]; +} +/** + * This helper is used to create `ParsedMessage` objects to be rendered in the + * `TranslationSerializer` tests. + */ +export function mockMessage( + id: string, messageParts: string[], placeholderNames: string[], + {meaning = '', description = '', location, legacyIds = []}: MockMessageOptions): + ɵParsedMessage { + let text = messageParts[0]; + for (let i = 1; i < messageParts.length; i++) { + text += `{$${placeholderNames[i - 1]}}${messageParts[i]}`; + } + return { + id, + text, + messageParts, + placeholderNames, + description, + meaning, + substitutions: [], + legacyIds, + location, + }; +} diff --git a/packages/localize/src/tools/test/extract/translation_files/xliff1_translation_serializer_spec.ts b/packages/localize/src/tools/test/extract/translation_files/xliff1_translation_serializer_spec.ts new file mode 100644 index 0000000000..2b89404fe2 --- /dev/null +++ b/packages/localize/src/tools/test/extract/translation_files/xliff1_translation_serializer_spec.ts @@ -0,0 +1,84 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ +import {absoluteFrom} from '@angular/compiler-cli/src/ngtsc/file_system'; +import {runInEachFileSystem} from '@angular/compiler-cli/src/ngtsc/file_system/testing'; +import {ɵParsedMessage} from '@angular/localize'; + +import {Xliff1TranslationSerializer} from '../../../src/extract/translation_files/xliff1_translation_serializer'; + +import {mockMessage} from './mock_message'; + +runInEachFileSystem(() => { + describe('Xliff1TranslationSerializer', () => { + [false, true].forEach(useLegacyIds => { + describe(`renderFile() [using ${useLegacyIds ? 'legacy' : 'canonical'} ids]`, () => { + it('should convert a set of parsed messages into an XML string', () => { + const messages: ɵParsedMessage[] = [ + mockMessage('12345', ['a', 'b', 'c'], ['PH', 'PH_1'], { + meaning: 'some meaning', + location: { + file: absoluteFrom('/project/file.ts'), + start: {line: 5, column: 10}, + end: {line: 5, column: 12} + }, + legacyIds: ['1234567890ABCDEF1234567890ABCDEF12345678', '615790887472569365'], + }), + mockMessage( + '67890', ['a', '', 'c'], ['START_TAG_SPAN', 'CLOSE_TAG_SPAN'], + {description: 'some description'}), + mockMessage('13579', ['', 'b', ''], ['START_BOLD_TEXT', 'CLOSE_BOLD_TEXT'], {}), + mockMessage('24680', ['a'], [], {meaning: 'meaning', description: 'and description'}), + mockMessage('80808', ['multi\nlines'], [], {}), + mockMessage('90000', [''], ['double-quotes-"'], {}) + ]; + const serializer = + new Xliff1TranslationSerializer('xx', absoluteFrom('/project'), useLegacyIds); + const output = serializer.serialize(messages); + expect(output).toEqual([ + ``, + ``, + ` `, + ` `, + ` `, + ` abc`, + ` `, + ` file.ts`, + ` 6`, + ` `, + ` some meaning`, + ` `, + ` `, + ` ac`, + ` some description`, + ` `, + ` `, + ` b`, + ` `, + ` `, + ` a`, + ` and description`, + ` meaning`, + ` `, + ` `, + ` multi`, + `lines`, + ` `, + ` `, + ` <escapeme>`, + ` `, + ` `, + ` `, + `\n`, + ].join('\n')); + }); + }); + }); + }); +}); diff --git a/packages/localize/src/tools/test/extract/translation_files/xliff2_translation_serializer_spec.ts b/packages/localize/src/tools/test/extract/translation_files/xliff2_translation_serializer_spec.ts new file mode 100644 index 0000000000..dbbf0fe84d --- /dev/null +++ b/packages/localize/src/tools/test/extract/translation_files/xliff2_translation_serializer_spec.ts @@ -0,0 +1,102 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ +import {computeMsgId} from '@angular/compiler'; +import {absoluteFrom} from '@angular/compiler-cli/src/ngtsc/file_system'; +import {runInEachFileSystem} from '@angular/compiler-cli/src/ngtsc/file_system/testing'; +import {ɵParsedMessage} from '@angular/localize'; + +import {Xliff2TranslationSerializer} from '../../../src/extract/translation_files/xliff2_translation_serializer'; + +import {mockMessage} from './mock_message'; + +runInEachFileSystem(() => { + describe('Xliff2TranslationSerializer', () => { + [false, true].forEach(useLegacyIds => { + describe(`renderFile() [using ${useLegacyIds ? 'legacy' : 'canonical'} ids]`, () => { + it('should convert a set of parsed messages into an XML string', () => { + const messages: ɵParsedMessage[] = [ + mockMessage('12345', ['a', 'b', 'c'], ['PH', 'PH_1'], { + meaning: 'some meaning', + location: { + file: absoluteFrom('/project/file.ts'), + start: {line: 5, column: 0}, + end: {line: 5, column: 3} + }, + legacyIds: ['1234567890ABCDEF1234567890ABCDEF12345678', '615790887472569365'], + }), + mockMessage('67890', ['a', '', 'c'], ['START_TAG_SPAN', 'CLOSE_TAG_SPAN'], { + description: 'some description', + location: { + file: absoluteFrom('/project/file.ts'), + start: {line: 2, column: 7}, + end: {line: 3, column: 2} + } + }), + mockMessage('13579', ['', 'b', ''], ['START_BOLD_TEXT', 'CLOSE_BOLD_TEXT'], {}), + mockMessage('24680', ['a'], [], {meaning: 'meaning', description: 'and description'}), + mockMessage('80808', ['multi\nlines'], [], {}), + mockMessage('90000', [''], ['double-quotes-"'], {}) + ]; + const serializer = + new Xliff2TranslationSerializer('xx', absoluteFrom('/project'), useLegacyIds); + const output = serializer.serialize(messages); + expect(output).toEqual([ + ``, + ``, + ` `, + ` `, + ` `, + ` file.ts:6`, + ` some meaning`, + ` `, + ` `, + ` abc`, + ` `, + ` `, + ` `, + ` `, + ` file.ts:3,4`, + ` some description`, + ` `, + ` `, + ` ac`, + ` `, + ` `, + ` `, + ` `, + ` b`, + ` `, + ` `, + ` `, + ` `, + ` and description`, + ` meaning`, + ` `, + ` `, + ` a`, + ` `, + ` `, + ` `, + ` `, + ` multi`, + `lines`, + ` `, + ` `, + ` `, + ` `, + ` <escapeme>`, + ` `, + ` `, + ` `, + `\n`, + ].join('\n')); + }); + }); + }); + }); +}); diff --git a/packages/localize/src/tools/test/extract/translation_files/xmb_translation_serializer_spec.ts b/packages/localize/src/tools/test/extract/translation_files/xmb_translation_serializer_spec.ts new file mode 100644 index 0000000000..45a1608334 --- /dev/null +++ b/packages/localize/src/tools/test/extract/translation_files/xmb_translation_serializer_spec.ts @@ -0,0 +1,53 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ +import {absoluteFrom} from '@angular/compiler-cli/src/ngtsc/file_system'; +import {runInEachFileSystem} from '@angular/compiler-cli/src/ngtsc/file_system/testing'; +import {ɵParsedMessage} from '@angular/localize'; + +import {XmbTranslationSerializer} from '../../../src/extract/translation_files/xmb_translation_serializer'; + +import {mockMessage} from './mock_message'; + +runInEachFileSystem(() => { + describe('XmbTranslationSerializer', () => { + [false, true].forEach(useLegacyIds => { + describe(`renderFile() [using ${useLegacyIds ? 'legacy' : 'canonical'} ids]`, () => { + it('should convert a set of parsed messages into an XML string', () => { + const messages: ɵParsedMessage[] = [ + mockMessage('12345', ['a', 'b', 'c'], ['PH', 'PH_1'], { + meaning: 'some meaning', + legacyIds: ['1234567890ABCDEF1234567890ABCDEF12345678', '615790887472569365'], + }), + mockMessage( + '67890', ['a', '', 'c'], ['START_TAG_SPAN', 'CLOSE_TAG_SPAN'], + {description: 'some description'}), + mockMessage('13579', ['', 'b', ''], ['START_BOLD_TEXT', 'CLOSE_BOLD_TEXT'], {}), + mockMessage('24680', ['a'], [], {meaning: 'meaning', description: 'and description'}), + mockMessage('80808', ['multi\nlines'], [], {}), + mockMessage('90000', [''], ['double-quotes-"'], {}), + ]; + const serializer = new XmbTranslationSerializer(absoluteFrom('/project'), useLegacyIds); + const output = serializer.serialize(messages); + expect(output).toContain([ + ``, + ` abc`, + ` ac`, + ` b`, + ` a`, + ` multi`, `lines`, + ` <escapeme>`, + `\n` + ].join('\n')); + }); + }); + }); + }); +}); diff --git a/packages/localize/src/utils/src/messages.ts b/packages/localize/src/utils/src/messages.ts index b735841bef..46663cf90f 100644 --- a/packages/localize/src/utils/src/messages.ts +++ b/packages/localize/src/utils/src/messages.ts @@ -152,7 +152,7 @@ export function parseMessage( cleanedMessageParts.push(messagePart); } const messageId = metadata.id || computeMsgId(messageString, metadata.meaning || ''); - const legacyIds = metadata.legacyIds && metadata.legacyIds.filter(id => id !== messageId); + const legacyIds = metadata.legacyIds ? metadata.legacyIds.filter(id => id !== messageId) : []; return { id: messageId, legacyIds,