iSharkFly-Docs
/
angular-docs-cn
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(compiler): updates hash algo for xmb/xtb files

This commit is contained in:
Victor Berchet 2016-11-07 16:20:05 -08:00
parent 76e4911e8b
commit 2f14415836
5 changed files with 103 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
modules/@angular/compiler-cli/integrationtest/test/i18n_spec.ts
View File

@ -34,9 +34,9 @@ const EXPECTED_XMB = `<?xml version="1.0" encoding="UTF-8" ?>
<!ELEMENT ex (#PCDATA)> <!ELEMENT ex (#PCDATA)>
]> ]>
<messagebundle> <messagebundle>
<msg id="252798779920123642">other-3rdP-component</msg> <msg id="3772663375917578720">other-3rdP-component</msg>
<msg id="7281825156779575080" desc="desc" meaning="meaning">translate me</msg> <msg id="8136548302122759730" desc="desc" meaning="meaning">translate me</msg>
<msg id="1325493959242906696">Welcome</msg> <msg id="3492007542396725315">Welcome</msg>
</messagebundle> </messagebundle>
`; `;

39
modules/@angular/compiler/src/i18n/digest.ts
View File

@ -15,7 +15,7 @@ export function digest(message: i18n.Message): string {
export function decimalDigest(message: i18n.Message): string { export function decimalDigest(message: i18n.Message): string {
const visitor = new _SerializerIgnoreIcuExpVisitor(); const visitor = new _SerializerIgnoreIcuExpVisitor();
const parts = message.nodes.map(a => a.visit(visitor, null)); const parts = message.nodes.map(a => a.visit(visitor, null));
return fingerprint(parts.join('') + `[${message.meaning}]`); return computeMsgId(parts.join(''), message.meaning);
} }
/** /**
@ -138,7 +138,7 @@ function fk(index: number, b: number, c: number, d: number): [number, number] {
* based on: * based on:
* https://github.com/google/closure-compiler/blob/master/src/com/google/javascript/jscomp/GoogleJsMessageIdGenerator.java * https://github.com/google/closure-compiler/blob/master/src/com/google/javascript/jscomp/GoogleJsMessageIdGenerator.java
*/ */
export function fingerprint(str: string): string { export function fingerprint(str: string): [number, number] {
const utf8 = utf8Encode(str); const utf8 = utf8Encode(str);
let [hi, lo] = [hash32(utf8, 0), hash32(utf8, 102072)]; let [hi, lo] = [hash32(utf8, 0), hash32(utf8, 102072)];
@ -148,9 +148,18 @@ export function fingerprint(str: string): string {
lo = lo ^ -0x6b5f56d8; lo = lo ^ -0x6b5f56d8;
} }
hi = hi & 0x7fffffff; return [hi, lo];
}
return byteStringToDecString(words32ToByteString([hi, lo])); export function computeMsgId(msg: string, meaning: string): string {
let [hi, lo] = fingerprint(msg);
if (meaning) {
const [him, lom] = fingerprint(meaning);
[hi, lo] = add64(rol64([hi, lo], 1), [him, lom]);
}
return byteStringToDecString(words32ToByteString([hi & 0x7fffffff, lo]));
} }
function hash32(str: string, c: number): number { function hash32(str: string, c: number): number {
@ -239,9 +248,19 @@ function decodeSurrogatePairs(str: string, index: number): number {
} }
function add32(a: number, b: number): number { function add32(a: number, b: number): number {
return add32to64(a, b)[1];
}
function add32to64(a: number, b: number): [number, number] {
const low = (a & 0xffff) + (b & 0xffff); const low = (a & 0xffff) + (b & 0xffff);
const high = (a >> 16) + (b >> 16) + (low >> 16); const high = (a >>> 16) + (b >>> 16) + (low >>> 16);
return (high << 16) | (low & 0xffff); return [high >>> 16, (high << 16) | (low & 0xffff)];
}
function add64([ah, al]: [number, number], [bh, bl]: [number, number]): [number, number] {
const [carry, l] = add32to64(al, bl);
const h = add32(add32(ah, bh), carry);
return [h, l];
} }
function sub32(a: number, b: number): number { function sub32(a: number, b: number): number {
@ -255,6 +274,13 @@ function rol32(a: number, count: number): number {
return (a << count) | (a >>> (32 - count)); return (a << count) | (a >>> (32 - count));
} }
// Rotate a 64b number left `count` position
function rol64([hi, lo]: [number, number], count: number): [number, number] {
const h = (hi << count) | (lo >>> (32 - count));
const l = (lo << count) | (hi >>> (32 - count));
return [h, l];
}
function stringToWords32(str: string, endian: Endian): number[] { function stringToWords32(str: string, endian: Endian): number[] {
const words32 = Array((str.length + 3) >>> 2); const words32 = Array((str.length + 3) >>> 2);
@ -317,6 +343,7 @@ function byteStringToDecString(str: string): string {
return decimal.split('').reverse().join(''); return decimal.split('').reverse().join('');
} }
// x and y decimal, lowest significant digit first
function addBigInt(x: string, y: string): string { function addBigInt(x: string, y: string): string {
let sum = ''; let sum = '';
const len = Math.max(x.length, y.length); const len = Math.max(x.length, y.length);

24
modules/@angular/compiler/test/i18n/digest_spec.ts
View File

@ -6,7 +6,7 @@
* found in the LICENSE file at https://angular.io/license * found in the LICENSE file at https://angular.io/license
*/ */
import {fingerprint, sha1} from '../../src/i18n/digest'; import {computeMsgId, sha1} from '../../src/i18n/digest';
export function main(): void { export function main(): void {
describe('digest', () => { describe('digest', () => {
@ -56,6 +56,7 @@ export function main(): void {
}); });
describe('decimal fingerprint', () => { describe('decimal fingerprint', () => {
it('should work on well known inputs w/o meaning', () => {
const fixtures: {[msg: string]: string} = { const fixtures: {[msg: string]: string} = {
' Spaced Out ': '3976450302996657536', ' Spaced Out ': '3976450302996657536',
'Last Name': '4407559560004943843', 'Last Name': '4407559560004943843',
@ -72,21 +73,32 @@ export function main(): void {
'': '4416290763660062288', '': '4416290763660062288',
}; };
it('should work on well known inputs', () => { Object.keys(fixtures).forEach(
Object.keys(fixtures).forEach(msg => { expect(fingerprint(msg)).toEqual(fixtures[msg]); }); msg => { expect(computeMsgId(msg, '')).toEqual(fixtures[msg]); });
});
it('should work on well known inputs with meaning', () => {
const fixtures: {[msg: string]: [string, string]} = {
'7790835225175622807': ['Last Name', 'Gmail UI'],
'1809086297585054940': ['First Name', 'Gmail UI'],
'3993998469942805487': ['View', 'Gmail UI'],
};
Object.keys(fixtures).forEach(
id => { expect(computeMsgId(fixtures[id][0], fixtures[id][1])).toEqual(id); });
}); });
it('should support arbitrary string size', () => { it('should support arbitrary string size', () => {
const prefix = `你好，世界`; const prefix = `你好，世界`;
let result = fingerprint(prefix); let result = computeMsgId(prefix, '');
for (let size = prefix.length; size < 5000; size += 101) { for (let size = prefix.length; size < 5000; size += 101) {
result = prefix + fingerprint(result); result = prefix + computeMsgId(result, '');
while (result.length < size) { while (result.length < size) {
result += result; result += result;
} }
result = result.slice(-size); result = result.slice(-size);
} }
expect(fingerprint(result)).toEqual('2122606631351252558'); expect(computeMsgId(result, '')).toEqual('2122606631351252558');
}); });
}); });

60
modules/@angular/compiler/test/i18n/integration_spec.ts
View File

@ -163,25 +163,25 @@ class FrLocalization extends NgLocalization {
const XTB = ` const XTB = `
<translationbundle> <translationbundle>
<translation id="7613717798286137988">attributs i18n sur les balises</translation> <translation id="615790887472569365">attributs i18n sur les balises</translation>
<translation id="496143996034957490">imbriqué</translation> <translation id="3707494640264351337">imbriqué</translation>
<translation id="4275167479475215567">imbriqué</translation> <translation id="5539162898278769904">imbriqué</translation>
<translation id="7210334813789040330"><ph name="START_ITALIC_TEXT"/>avec des espaces réservés<ph name="CLOSE_ITALIC_TEXT"/></translation> <translation id="3780349238193953556"><ph name="START_ITALIC_TEXT"/>avec des espaces réservés<ph name="CLOSE_ITALIC_TEXT"/></translation>
<translation id="4769680004784140786">sur des balises non traductibles</translation> <translation id="5525133077318024839">sur des balises non traductibles</translation>
<translation id="4033143013932333681">sur des balises traductibles</translation> <translation id="8670732454866344690">sur des balises traductibles</translation>
<translation id="6162642997206060264">{VAR_PLURAL, plural, =0 {zero} =1 {un} =2 {deux} other {<ph name="START_BOLD_TEXT"/>beaucoup<ph name="CLOSE_BOLD_TEXT"/>}}</translation> <translation id="4593805537723189714">{VAR_PLURAL, plural, =0 {zero} =1 {un} =2 {deux} other {<ph name="START_BOLD_TEXT"/>beaucoup<ph name="CLOSE_BOLD_TEXT"/>}}</translation>
<translation id="1882489820012923152"><ph name="ICU"/></translation> <translation id="1746565782635215"><ph name="ICU"/></translation>
<translation id="4822972059757846302">{VAR_SELECT, select, m {homme} f {femme}}</translation> <translation id="5868084092545682515">{VAR_SELECT, select, m {homme} f {femme}}</translation>
<translation id="5917557396782931034"><ph name="INTERPOLATION"/></translation> <translation id="4851788426695310455"><ph name="INTERPOLATION"/></translation>
<translation id="4687596778889597732">sexe = <ph name="INTERPOLATION"/></translation> <translation id="9013357158046221374">sexe = <ph name="INTERPOLATION"/></translation>
<translation id="2505882222003102347"><ph name="CUSTOM_NAME"/></translation> <translation id="8324617391167353662"><ph name="CUSTOM_NAME"/></translation>
<translation id="5340176214595489533">dans une section traductible</translation> <translation id="7685649297917455806">dans une section traductible</translation>
<translation id="4120782520649528473"> <translation id="2387287228265107305">
<ph name="START_HEADING_LEVEL1"/>Balises dans les commentaires html<ph name="CLOSE_HEADING_LEVEL1"/> <ph name="START_HEADING_LEVEL1"/>Balises dans les commentaires html<ph name="CLOSE_HEADING_LEVEL1"/>
<ph name="START_TAG_DIV"/><ph name="CLOSE_TAG_DIV"/> <ph name="START_TAG_DIV"/><ph name="CLOSE_TAG_DIV"/>
<ph name="START_TAG_DIV_1"/><ph name="ICU"/><ph name="CLOSE_TAG_DIV"></ph> <ph name="START_TAG_DIV_1"/><ph name="ICU"/><ph name="CLOSE_TAG_DIV"></ph>
</translation> </translation>
<translation id="1309478472899123444">ca <ph name="START_BOLD_TEXT"/>devrait<ph name="CLOSE_BOLD_TEXT"/> marcher</translation> <translation id="1491627405349178954">ca <ph name="START_BOLD_TEXT"/>devrait<ph name="CLOSE_BOLD_TEXT"/> marcher</translation>
</translationbundle>`; </translationbundle>`;
// unused, for reference only // unused, for reference only
@ -189,26 +189,26 @@ const XTB = `
// `fit('extract xmb', () => { console.log(toXmb(HTML)); });` // `fit('extract xmb', () => { console.log(toXmb(HTML)); });`
const XMB = ` const XMB = `
<messagebundle> <messagebundle>
<msg id="7613717798286137988">i18n attribute on tags</msg> <msg id="615790887472569365">i18n attribute on tags</msg>
<msg id="496143996034957490">nested</msg> <msg id="3707494640264351337">nested</msg>
<msg id="4275167479475215567" meaning="different meaning">nested</msg> <msg id="5539162898278769904" meaning="different meaning">nested</msg>
<msg id="7210334813789040330"><ph name="START_ITALIC_TEXT"><ex>&lt;i&gt;</ex></ph>with placeholders<ph name="CLOSE_ITALIC_TEXT"><ex>&lt;/i&gt;</ex></ph></msg> <msg id="3780349238193953556"><ph name="START_ITALIC_TEXT"><ex>&lt;i&gt;</ex></ph>with placeholders<ph name="CLOSE_ITALIC_TEXT"><ex>&lt;/i&gt;</ex></ph></msg>
<msg id="4769680004784140786">on not translatable node</msg> <msg id="5525133077318024839">on not translatable node</msg>
<msg id="4033143013932333681">on translatable node</msg> <msg id="8670732454866344690">on translatable node</msg>
<msg id="6162642997206060264">{VAR_PLURAL, plural, =0 {zero} =1 {one} =2 {two} other {<ph name="START_BOLD_TEXT"><ex>&lt;b&gt;</ex></ph>many<ph name="CLOSE_BOLD_TEXT"><ex>&lt;/b&gt;</ex></ph>} }</msg> <msg id="4593805537723189714">{VAR_PLURAL, plural, =0 {zero} =1 {one} =2 {two} other {<ph name="START_BOLD_TEXT"><ex>&lt;b&gt;</ex></ph>many<ph name="CLOSE_BOLD_TEXT"><ex>&lt;/b&gt;</ex></ph>} }</msg>
<msg id="1882489820012923152"> <msg id="1746565782635215">
<ph name="ICU"/> <ph name="ICU"/>
</msg> </msg>
<msg id="4822972059757846302">{VAR_SELECT, select, m {male} f {female} }</msg> <msg id="5868084092545682515">{VAR_SELECT, select, m {male} f {female} }</msg>
<msg id="5917557396782931034"><ph name="INTERPOLATION"/></msg> <msg id="4851788426695310455"><ph name="INTERPOLATION"/></msg>
<msg id="4687596778889597732">sex = <ph name="INTERPOLATION"/></msg> <msg id="9013357158046221374">sex = <ph name="INTERPOLATION"/></msg>
<msg id="2505882222003102347"><ph name="CUSTOM_NAME"/></msg> <msg id="8324617391167353662"><ph name="CUSTOM_NAME"/></msg>
<msg id="5340176214595489533">in a translatable section</msg> <msg id="7685649297917455806">in a translatable section</msg>
<msg id="4120782520649528473"> <msg id="2387287228265107305">
<ph name="START_HEADING_LEVEL1"><ex>&lt;h1&gt;</ex></ph>Markers in html comments<ph name="CLOSE_HEADING_LEVEL1"><ex>&lt;/h1&gt;</ex></ph> <ph name="START_HEADING_LEVEL1"><ex>&lt;h1&gt;</ex></ph>Markers in html comments<ph name="CLOSE_HEADING_LEVEL1"><ex>&lt;/h1&gt;</ex></ph>
<ph name="START_TAG_DIV"><ex>&lt;div&gt;</ex></ph><ph name="CLOSE_TAG_DIV"><ex>&lt;/div&gt;</ex></ph> <ph name="START_TAG_DIV"><ex>&lt;div&gt;</ex></ph><ph name="CLOSE_TAG_DIV"><ex>&lt;/div&gt;</ex></ph>
<ph name="START_TAG_DIV_1"><ex>&lt;div&gt;</ex></ph><ph name="ICU"/><ph name="CLOSE_TAG_DIV"><ex>&lt;/div&gt;</ex></ph> <ph name="START_TAG_DIV_1"><ex>&lt;div&gt;</ex></ph><ph name="ICU"/><ph name="CLOSE_TAG_DIV"><ex>&lt;/div&gt;</ex></ph>
</msg> </msg>
<msg id="1309478472899123444">it <ph name="START_BOLD_TEXT"><ex>&lt;b&gt;</ex></ph>should<ph name="CLOSE_BOLD_TEXT"><ex>&lt;/b&gt;</ex></ph> work</msg> <msg id="1491627405349178954">it <ph name="START_BOLD_TEXT"><ex>&lt;b&gt;</ex></ph>should<ph name="CLOSE_BOLD_TEXT"><ex>&lt;/b&gt;</ex></ph> work</msg>
</messagebundle> </messagebundle>
`; `;

8
modules/@angular/compiler/test/i18n/serializers/xmb_spec.ts
View File

@ -43,10 +43,10 @@ export function main(): void {
<!ELEMENT ex (#PCDATA)> <!ELEMENT ex (#PCDATA)>
]> ]>
<messagebundle> <messagebundle>
<msg id="2348600990161399314">translatable element <ph name="START_BOLD_TEXT"><ex>&lt;b&gt;</ex></ph>with placeholders<ph name="CLOSE_BOLD_TEXT"><ex>&lt;/b&gt;</ex></ph> <ph name="INTERPOLATION"/></msg> <msg id="7056919470098446707">translatable element <ph name="START_BOLD_TEXT"><ex>&lt;b&gt;</ex></ph>with placeholders<ph name="CLOSE_BOLD_TEXT"><ex>&lt;/b&gt;</ex></ph> <ph name="INTERPOLATION"/></msg>
<msg id="5525949440406338075">{VAR_PLURAL, plural, =0 {<ph name="START_PARAGRAPH"><ex>&lt;p&gt;</ex></ph>test<ph name="CLOSE_PARAGRAPH"><ex>&lt;/p&gt;</ex></ph>} }</msg> <msg id="2981514368455622387">{VAR_PLURAL, plural, =0 {<ph name="START_PARAGRAPH"><ex>&lt;p&gt;</ex></ph>test<ph name="CLOSE_PARAGRAPH"><ex>&lt;/p&gt;</ex></ph>} }</msg>
<msg id="130772889486467622" desc="d" meaning="m">foo</msg> <msg id="7999024498831672133" desc="d" meaning="m">foo</msg>
<msg id="9095788995532341072">{VAR_PLURAL, plural, =0 {{VAR_GENDER, gender, other {<ph name="START_PARAGRAPH"><ex>&lt;p&gt;</ex></ph>deeply nested<ph name="CLOSE_PARAGRAPH"><ex>&lt;/p&gt;</ex></ph>} } } }</msg> <msg id="2015957479576096115">{VAR_PLURAL, plural, =0 {{VAR_SELECT, select, other {<ph name="START_PARAGRAPH"><ex>&lt;p&gt;</ex></ph>deeply nested<ph name="CLOSE_PARAGRAPH"><ex>&lt;/p&gt;</ex></ph>} } } }</msg>
</messagebundle> </messagebundle>
`; `;