feat(ivy): i18n - support inlining of XTB formatted translation files (#33444)

This commit implements the `XtbTranslationParser`, which can read XTB
formatted files.

PR Close #33444
This commit is contained in:
Pete Bacon Darwin 2019-10-28 12:51:45 +00:00 committed by Andrew Kushnir
parent c2f13a1e3a
commit 2c623fde16
6 changed files with 424 additions and 4 deletions

View File

@ -18,6 +18,7 @@ import {TranslationLoader} from './translation_files/translation_loader';
import {SimpleJsonTranslationParser} from './translation_files/translation_parsers/simple_json_translation_parser';
import {Xliff1TranslationParser} from './translation_files/translation_parsers/xliff1_translation_parser';
import {Xliff2TranslationParser} from './translation_files/translation_parsers/xliff2_translation_parser';
import {XtbTranslationParser} from './translation_files/translation_parsers/xtb_translation_parser';
import {Translator} from './translator';
import {Diagnostics} from '../diagnostics';
@ -141,6 +142,7 @@ export function translateFiles({sourceRootPath, sourceFilePaths, translationFile
[
new Xliff2TranslationParser(),
new Xliff1TranslationParser(),
new XtbTranslationParser(diagnostics),
new SimpleJsonTranslationParser(),
],
diagnostics);

View File

@ -27,6 +27,9 @@ export function parseInnerRange(element: Element): Node[] {
const xml = xmlParser.parse(
element.sourceSpan.start.file.content, element.sourceSpan.start.file.url,
{tokenizeExpansionForms: true, range: getInnerRange(element)});
if (xml.errors.length) {
throw xml.errors.map(e => new TranslationParseError(e.span, e.msg).toString()).join('\n');
}
return xml.rootNodes;
}

View File

@ -0,0 +1,103 @@
/**
* @license
* Copyright Google Inc. All Rights Reserved.
*
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/
import {Element, Node, XmlParser, visitAll} from '@angular/compiler';
import {ɵParsedTranslation} from '@angular/localize';
import {extname} from 'path';
import {Diagnostics} from '../../../diagnostics';
import {BaseVisitor} from '../base_visitor';
import {MessageSerializer} from '../message_serialization/message_serializer';
import {TargetMessageRenderer} from '../message_serialization/target_message_renderer';
import {TranslationParseError} from './translation_parse_error';
import {ParsedTranslationBundle, TranslationParser} from './translation_parser';
import {getAttrOrThrow, parseInnerRange} from './translation_utils';
/**
* A translation parser that can load XB files.
*/
export class XtbTranslationParser implements TranslationParser {
constructor(private diagnostics: Diagnostics) {}
canParse(filePath: string, contents: string): boolean {
const extension = extname(filePath);
return (extension === '.xtb' || extension === '.xmb') &&
contents.includes('<translationbundle');
}
parse(filePath: string, contents: string): ParsedTranslationBundle {
const xmlParser = new XmlParser();
const xml = xmlParser.parse(contents, filePath);
const bundle = XtbVisitor.extractBundle(this.diagnostics, xml.rootNodes);
if (bundle === undefined) {
throw new Error(`Unable to parse "${filePath}" as XTB/XMB format.`);
}
return bundle;
}
}
class XtbVisitor extends BaseVisitor {
static extractBundle(diagnostics: Diagnostics, messageBundles: Node[]): ParsedTranslationBundle
|undefined {
const visitor = new this(diagnostics);
const bundles: ParsedTranslationBundle[] = visitAll(visitor, messageBundles, undefined);
return bundles[0];
}
constructor(private diagnostics: Diagnostics) { super(); }
visitElement(element: Element, bundle: ParsedTranslationBundle|undefined): any {
switch (element.name) {
case 'translationbundle':
if (bundle) {
throw new TranslationParseError(
element.sourceSpan, '<translationbundle> elements can not be nested');
}
const langAttr = element.attrs.find((attr) => attr.name === 'lang');
bundle = {locale: langAttr && langAttr.value, translations: {}};
visitAll(this, element.children, bundle);
return bundle;
case 'translation':
if (!bundle) {
throw new TranslationParseError(
element.sourceSpan, '<translation> must be inside a <translationbundle>');
}
const id = getAttrOrThrow(element, 'id');
if (bundle.translations.hasOwnProperty(id)) {
throw new TranslationParseError(
element.sourceSpan, `Duplicated translations for message "${id}"`);
} else {
try {
bundle.translations[id] = serializeTargetMessage(element);
} catch (error) {
if (typeof error === 'string') {
this.diagnostics.warn(
`Could not parse message with id "${id}" - perhaps it has an unrecognised ICU format?\n` +
error);
} else {
throw error;
}
}
}
break;
default:
throw new TranslationParseError(element.sourceSpan, 'Unexpected tag');
}
}
}
function serializeTargetMessage(source: Element): ɵParsedTranslation {
const serializer = new MessageSerializer(
new TargetMessageRenderer(),
{inlineElements: [], placeholder: {elementName: 'ph', nameAttribute: 'name'}});
return serializer.serialize(parseInnerRange(source));
}

View File

@ -0,0 +1,13 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE translationbundle [<!ELEMENT translationbundle (translation)*>
<!ATTLIST translationbundle lang CDATA #REQUIRED>
<!ELEMENT translation (#PCDATA|ph)*>
<!ATTLIST translation id CDATA #REQUIRED>
<!ELEMENT ph EMPTY>
<!ATTLIST ph name CDATA #REQUIRED>
]>
<translationbundle lang="it">
<translation id="3291030485717846467">Ciao, <ph name="PH"/>!</translation>
</translationbundle>

View File

@ -29,7 +29,8 @@ describe('translateFiles()', () => {
sourceFilePaths: resolveAll(__dirname + '/test_files', ['test-1.txt', 'test-2.txt']),
outputPathFn,
translationFilePaths: resolveAll(
__dirname + '/locales', ['messages.de.json', 'messages.es.xlf', 'messages.fr.xlf']),
__dirname + '/locales',
['messages.de.json', 'messages.es.xlf', 'messages.fr.xlf', 'messages.it.xtb']),
translationFileLocales: [], diagnostics,
missingTranslation: 'error'
});
@ -48,6 +49,10 @@ describe('translateFiles()', () => {
.toEqual('Contents of test-1.txt');
expect(FileUtils.readFile(resolve(testDir, 'es', 'test-2.txt')))
.toEqual('Contents of test-2.txt');
expect(FileUtils.readFile(resolve(testDir, 'it', 'test-1.txt')))
.toEqual('Contents of test-1.txt');
expect(FileUtils.readFile(resolve(testDir, 'it', 'test-2.txt')))
.toEqual('Contents of test-2.txt');
});
it('should translate and copy source-code files to the destination folders', () => {
@ -57,7 +62,8 @@ describe('translateFiles()', () => {
sourceRootPath: resolve(__dirname, 'test_files'),
sourceFilePaths: resolveAll(__dirname + '/test_files', ['test.js']), outputPathFn,
translationFilePaths: resolveAll(
__dirname + '/locales', ['messages.de.json', 'messages.es.xlf', 'messages.fr.xlf']),
__dirname + '/locales',
['messages.de.json', 'messages.es.xlf', 'messages.fr.xlf', 'messages.it.xtb']),
translationFileLocales: [], diagnostics,
missingTranslation: 'error',
});
@ -70,6 +76,8 @@ describe('translateFiles()', () => {
.toEqual(`var name="World";var message="Guten Tag, "+name+"!";`);
expect(FileUtils.readFile(resolve(testDir, 'es', 'test.js')))
.toEqual(`var name="World";var message="Hola, "+name+"!";`);
expect(FileUtils.readFile(resolve(testDir, 'it', 'test.js')))
.toEqual(`var name="World";var message="Ciao, "+name+"!";`);
});
it('should translate and copy source-code files overriding the locales', () => {
@ -79,7 +87,8 @@ describe('translateFiles()', () => {
sourceRootPath: resolve(__dirname, 'test_files'),
sourceFilePaths: resolveAll(__dirname + '/test_files', ['test.js']), outputPathFn,
translationFilePaths: resolveAll(
__dirname + '/locales', ['messages.de.json', 'messages.es.xlf', 'messages.fr.xlf']),
__dirname + '/locales',
['messages.de.json', 'messages.es.xlf', 'messages.fr.xlf', 'messages.it.xtb']),
translationFileLocales: ['xde', undefined, 'fr'], diagnostics,
missingTranslation: 'error',
});
@ -97,6 +106,8 @@ describe('translateFiles()', () => {
.toEqual(`var name="World";var message="Hola, "+name+"!";`);
expect(FileUtils.readFile(resolve(testDir, 'fr', 'test.js')))
.toEqual(`var name="World";var message="Bonjour, "+name+"!";`);
expect(FileUtils.readFile(resolve(testDir, 'it', 'test.js')))
.toEqual(`var name="World";var message="Ciao, "+name+"!";`);
});
it('should transform and/or copy files to the destination folders', () => {
@ -108,7 +119,8 @@ describe('translateFiles()', () => {
resolveAll(__dirname + '/test_files', ['test-1.txt', 'test-2.txt', 'test.js']),
outputPathFn,
translationFilePaths: resolveAll(
__dirname + '/locales', ['messages.de.json', 'messages.es.xlf', 'messages.fr.xlf']),
__dirname + '/locales',
['messages.de.json', 'messages.es.xlf', 'messages.fr.xlf', 'messages.it.xtb']),
translationFileLocales: [], diagnostics,
missingTranslation: 'error',
});
@ -127,6 +139,10 @@ describe('translateFiles()', () => {
.toEqual('Contents of test-1.txt');
expect(FileUtils.readFile(resolve(testDir, 'es', 'test-2.txt')))
.toEqual('Contents of test-2.txt');
expect(FileUtils.readFile(resolve(testDir, 'it', 'test-1.txt')))
.toEqual('Contents of test-1.txt');
expect(FileUtils.readFile(resolve(testDir, 'it', 'test-2.txt')))
.toEqual('Contents of test-2.txt');
expect(FileUtils.readFile(resolve(testDir, 'fr', 'test.js')))
.toEqual(`var name="World";var message="Bonjour, "+name+"!";`);
@ -134,6 +150,8 @@ describe('translateFiles()', () => {
.toEqual(`var name="World";var message="Guten Tag, "+name+"!";`);
expect(FileUtils.readFile(resolve(testDir, 'es', 'test.js')))
.toEqual(`var name="World";var message="Hola, "+name+"!";`);
expect(FileUtils.readFile(resolve(testDir, 'it', 'test.js')))
.toEqual(`var name="World";var message="Ciao, "+name+"!";`);
});
});

View File

@ -0,0 +1,281 @@
/**
* @license
* Copyright Google Inc. All Rights Reserved.
*
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/
import {ɵcomputeMsgId, ɵmakeParsedTranslation} from '@angular/localize';
import {Diagnostics} from '../../../../src/diagnostics';
import {XtbTranslationParser} from '../../../../src/translate/translation_files/translation_parsers/xtb_translation_parser';
describe('XtbTranslationParser', () => {
describe('canParse()', () => {
it('should return true if the file extension is `.xtb` or `.xmb` and it contains the `<translationbundle>` tag',
() => {
const parser = new XtbTranslationParser(new Diagnostics());
expect(parser.canParse('/some/file.xtb', '<translationbundle>')).toBe(true);
expect(parser.canParse('/some/file.xmb', '<translationbundle>')).toBe(true);
expect(parser.canParse('/some/file.xtb', '<translationbundle lang="en">')).toBe(true);
expect(parser.canParse('/some/file.xmb', '<translationbundle lang="en">')).toBe(true);
expect(parser.canParse('/some/file.json', '<translationbundle>')).toBe(false);
expect(parser.canParse('/some/file.xmb', '')).toBe(false);
expect(parser.canParse('/some/file.xtb', '')).toBe(false);
});
});
describe('parse()', () => {
it('should extract the locale from the file contents', () => {
const XTB = `<?xml version="1.0" encoding="UTF-8"?>
<translationbundle lang='fr'>
<translation id="8841459487341224498">rab</translation>
</translationbundle>`;
const parser = new XtbTranslationParser(new Diagnostics());
const result = parser.parse('/some/file.xtb', XTB);
expect(result.locale).toEqual('fr');
});
it('should extract basic messages', () => {
const XTB = `<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE translationbundle [<!ELEMENT translationbundle (translation)*>
<!ATTLIST translationbundle lang CDATA #REQUIRED>
<!ELEMENT translation (#PCDATA|ph)*>
<!ATTLIST translation id CDATA #REQUIRED>
<!ELEMENT ph EMPTY>
<!ATTLIST ph name CDATA #REQUIRED>
]>
<translationbundle>
<translation id="8841459487341224498">rab</translation>
</translationbundle>`;
const parser = new XtbTranslationParser(new Diagnostics());
const result = parser.parse('/some/file.xtb', XTB);
expect(result.translations['8841459487341224498']).toEqual(ɵmakeParsedTranslation(['rab']));
});
it('should extract translations with simple placeholders', () => {
const XTB = `<?xml version="1.0" encoding="UTF-8"?>
<translationbundle>
<translation id="8877975308926375834"><ph name="START_PARAGRAPH"/>rab<ph name="CLOSE_PARAGRAPH"/></translation>
</translationbundle>`;
const parser = new XtbTranslationParser(new Diagnostics());
const result = parser.parse('/some/file.xtb', XTB);
expect(result.translations['8877975308926375834'])
.toEqual(ɵmakeParsedTranslation(['', 'rab', ''], ['START_PARAGRAPH', 'CLOSE_PARAGRAPH']));
});
it('should extract translations with simple ICU expressions', () => {
const XTB = `<?xml version="1.0" encoding="UTF-8" ?>
<translationbundle>
<translation id="7717087045075616176">*<ph name="ICU"/>*</translation>
<translation id="5115002811911870583">{VAR_PLURAL, plural, =1 {<ph name="START_PARAGRAPH"/>rab<ph name="CLOSE_PARAGRAPH"/>}}</translation>
</translationbundle>`;
const parser = new XtbTranslationParser(new Diagnostics());
const result = parser.parse('/some/file.xtb', XTB);
expect(result.translations['7717087045075616176'])
.toEqual(ɵmakeParsedTranslation(['*', '*'], ['ICU']));
expect(result.translations['5115002811911870583'])
.toEqual(ɵmakeParsedTranslation(
['{VAR_PLURAL, plural, =1 {{START_PARAGRAPH}rab{CLOSE_PARAGRAPH}}}'], []));
});
it('should extract translations with duplicate source messages', () => {
const XTB = `
<translationbundle>
<translation id="9205907420411818817">oof</translation>
<translation id="i">toto</translation>
<translation id="bar">tata</translation>
</translationbundle>`;
const parser = new XtbTranslationParser(new Diagnostics());
const result = parser.parse('/some/file.xtb', XTB);
expect(result.translations[ɵcomputeMsgId('foo')]).toEqual(ɵmakeParsedTranslation(['oof']));
expect(result.translations['i']).toEqual(ɵmakeParsedTranslation(['toto']));
expect(result.translations['bar']).toEqual(ɵmakeParsedTranslation(['tata']));
});
it('should extract translations with only placeholders, which are re-ordered', () => {
const XTB = `
<translationbundle>
<translation id="7118057989405618448"><ph name="TAG_IMG_1"/><ph name="TAG_IMG"/><ph name="LINE_BREAK"/></translation>
</translationbundle>`;
const parser = new XtbTranslationParser(new Diagnostics());
const result = parser.parse('/some/file.xtb', XTB);
expect(result.translations[ɵcomputeMsgId('{$LINE_BREAK}{$TAG_IMG}{$TAG_IMG_1}')])
.toEqual(
ɵmakeParsedTranslation(['', '', '', ''], ['TAG_IMG_1', 'TAG_IMG', 'LINE_BREAK']));
});
it('should extract translations with empty target', () => {
/**
* Source HTML:
*
* ```
* <div i18n>hello <span></span></div>
* ```
*/
const XTB = `
<translationbundle>
<translation id="2826198357052921524"></translation>
</translationbundle>`;
const parser = new XtbTranslationParser(new Diagnostics());
const result = parser.parse('/some/file.xtb', XTB);
expect(result.translations[ɵcomputeMsgId('hello {$START_TAG_SPAN}{$CLOSE_TAG_SPAN}')])
.toEqual(ɵmakeParsedTranslation(['']));
});
it('should extract translations with deeply nested ICUs', () => {
/**
* Source HTML:
*
* ```
* Test: { count, plural, =0 { { sex, select, other {<p>deeply nested</p>}} } =other {a lot}}
* ```
*
* Note that the message gets split into two translation units:
* * The first one contains the outer message with an `ICU` placeholder
* * The second one is the ICU expansion itself
*
* Note that special markers `VAR_PLURAL` and `VAR_SELECT` are added, which are then
replaced by IVY at runtime with the actual values being rendered by the ICU expansion.
*/
const XTB = `
<translationbundle>
<translation id="980940425376233536">Le test: <ph name="ICU" equiv-text="{ count, plural, =0 {...} =other {...}}"/></translation>
<translation id="5207293143089349404">{VAR_PLURAL, plural, =0 {{VAR_SELECT, select, other {<ph name="START_PARAGRAPH"/>profondément imbriqué<ph name="CLOSE_PARAGRAPH"/>}}} =other {beaucoup}}</translation>
</translationbundle>`;
const parser = new XtbTranslationParser(new Diagnostics());
const result = parser.parse('/some/file.xtb', XTB);
expect(result.translations[ɵcomputeMsgId('Test: {$ICU}')])
.toEqual(ɵmakeParsedTranslation(['Le test: ', ''], ['ICU']));
expect(
result.translations[ɵcomputeMsgId(
'{VAR_PLURAL, plural, =0 {{VAR_SELECT, select, other {{START_PARAGRAPH}deeply nested{CLOSE_PARAGRAPH}}}} =other {beaucoup}}')])
.toEqual(ɵmakeParsedTranslation([
'{VAR_PLURAL, plural, =0 {{VAR_SELECT, select, other {{START_PARAGRAPH}profondément imbriqué{CLOSE_PARAGRAPH}}}} =other {beaucoup}}'
]));
});
it('should extract translations containing multiple lines', () => {
/**
* Source HTML:
*
* ```
* <div i18n>multi
* lines</div>
* ```
*/
const XTB = `
<translationbundle>
<translation id="2340165783990709777">multi\nlignes</translation>
</translationbundle>`;
const parser = new XtbTranslationParser(new Diagnostics());
const result = parser.parse('/some/file.xtb', XTB);
expect(result.translations[ɵcomputeMsgId('multi\nlines')])
.toEqual(ɵmakeParsedTranslation(['multi\nlignes']));
});
it('should warn on unrecognised ICU messages', () => {
// See https://github.com/angular/angular/issues/14046
const XTB = `
<translationbundle>
<translation id="valid">This is a valid message</translation>
<translation id="invalid">{REGION_COUNT_1, plural, =0 {unused plural form} =1 {1 region} other {{REGION_COUNT_2} regions}}</translation>
</translationbundle>`;
// Parsing the file should not fail
const diagnostics = new Diagnostics();
const parser = new XtbTranslationParser(diagnostics);
const result = parser.parse('/some/file.xtb', XTB);
// We should be able to read the valid message
expect(result.translations['valid'])
.toEqual(ɵmakeParsedTranslation(['This is a valid message']));
// Trying to access the invalid message should fail
expect(result.translations['invalid']).toBeUndefined();
expect(diagnostics.messages).toContain({
type: 'warning',
message:
`Could not parse message with id "invalid" - perhaps it has an unrecognised ICU format?\n` +
`Error: Unexpected character "EOF" (Do you have an unescaped "{" in your template? Use "{{ '{' }}") to escape it.)\n` +
`Error: Invalid ICU message. Missing '}'.`
});
});
describe('[structure errors]', () => {
it('should throw when there are nested translationbundle tags', () => {
const XTB =
'<translationbundle><translationbundle></translationbundle></translationbundle>';
expect(() => {
const parser = new XtbTranslationParser(new Diagnostics());
parser.parse('/some/file.xtb', XTB);
}).toThrowError(/<translationbundle> elements can not be nested/);
});
it('should throw when a translation has no id attribute', () => {
const XTB = `
<translationbundle>
<translation></translation>
</translationbundle>`;
expect(() => {
const parser = new XtbTranslationParser(new Diagnostics());
parser.parse('/some/file.xtb', XTB);
}).toThrowError(/Missing required "id" attribute/);
});
it('should throw on duplicate translation id', () => {
const XTB = `
<translationbundle>
<translation id="deadbeef"></translation>
<translation id="deadbeef"></translation>
</translationbundle>`;
expect(() => {
const parser = new XtbTranslationParser(new Diagnostics());
parser.parse('/some/file.xtb', XTB);
}).toThrowError(/Duplicated translations for message "deadbeef"/);
});
});
describe('[message errors]', () => {
it('should throw on unknown message tags', () => {
const XTB = `
<translationbundle>
<translation id="deadbeef">
<source/>
</translation>
</translationbundle>`;
expect(() => {
const parser = new XtbTranslationParser(new Diagnostics());
parser.parse('/some/file.xtb', XTB);
}).toThrowError(/Invalid element found in message/);
});
it('should throw when a placeholder misses a name attribute', () => {
const XTB = `
<translationbundle>
<translation id="deadbeef"><ph/></translation>
</translationbundle>`;
expect(() => {
const parser = new XtbTranslationParser(new Diagnostics());
parser.parse('/some/file.xtb', XTB);
}).toThrowError(/required "name" attribute/gi);
});
});
});
});