From 2f14415836888577bfab823bc3e99f4769da5dd5 Mon Sep 17 00:00:00 2001 From: Victor Berchet Date: Mon, 7 Nov 2016 16:20:05 -0800 Subject: [PATCH] fix(compiler): updates hash algo for xmb/xtb files --- .../integrationtest/test/i18n_spec.ts | 6 +- modules/@angular/compiler/src/i18n/digest.ts | 39 ++++++++++-- .../compiler/test/i18n/digest_spec.ts | 54 ++++++++++------- .../compiler/test/i18n/integration_spec.ts | 60 +++++++++---------- .../test/i18n/serializers/xmb_spec.ts | 8 +-- 5 files changed, 103 insertions(+), 64 deletions(-) diff --git a/modules/@angular/compiler-cli/integrationtest/test/i18n_spec.ts b/modules/@angular/compiler-cli/integrationtest/test/i18n_spec.ts index fc3fd6e61a..d46ad0d994 100644 --- a/modules/@angular/compiler-cli/integrationtest/test/i18n_spec.ts +++ b/modules/@angular/compiler-cli/integrationtest/test/i18n_spec.ts @@ -34,9 +34,9 @@ const EXPECTED_XMB = ` ]> - other-3rdP-component - translate me - Welcome + other-3rdP-component + translate me + Welcome `; diff --git a/modules/@angular/compiler/src/i18n/digest.ts b/modules/@angular/compiler/src/i18n/digest.ts index 895e7a167a..d00bf80354 100644 --- a/modules/@angular/compiler/src/i18n/digest.ts +++ b/modules/@angular/compiler/src/i18n/digest.ts @@ -15,7 +15,7 @@ export function digest(message: i18n.Message): string { export function decimalDigest(message: i18n.Message): string { const visitor = new _SerializerIgnoreIcuExpVisitor(); const parts = message.nodes.map(a => a.visit(visitor, null)); - return fingerprint(parts.join('') + `[${message.meaning}]`); + return computeMsgId(parts.join(''), message.meaning); } /** @@ -138,7 +138,7 @@ function fk(index: number, b: number, c: number, d: number): [number, number] { * based on: * https://github.com/google/closure-compiler/blob/master/src/com/google/javascript/jscomp/GoogleJsMessageIdGenerator.java */ -export function fingerprint(str: string): string { +export function fingerprint(str: string): [number, number] { const utf8 = utf8Encode(str); let [hi, lo] = [hash32(utf8, 0), hash32(utf8, 102072)]; @@ -148,9 +148,18 @@ export function fingerprint(str: string): string { lo = lo ^ -0x6b5f56d8; } - hi = hi & 0x7fffffff; + return [hi, lo]; +} - return byteStringToDecString(words32ToByteString([hi, lo])); +export function computeMsgId(msg: string, meaning: string): string { + let [hi, lo] = fingerprint(msg); + + if (meaning) { + const [him, lom] = fingerprint(meaning); + [hi, lo] = add64(rol64([hi, lo], 1), [him, lom]); + } + + return byteStringToDecString(words32ToByteString([hi & 0x7fffffff, lo])); } function hash32(str: string, c: number): number { @@ -239,9 +248,19 @@ function decodeSurrogatePairs(str: string, index: number): number { } function add32(a: number, b: number): number { + return add32to64(a, b)[1]; +} + +function add32to64(a: number, b: number): [number, number] { const low = (a & 0xffff) + (b & 0xffff); - const high = (a >> 16) + (b >> 16) + (low >> 16); - return (high << 16) | (low & 0xffff); + const high = (a >>> 16) + (b >>> 16) + (low >>> 16); + return [high >>> 16, (high << 16) | (low & 0xffff)]; +} + +function add64([ah, al]: [number, number], [bh, bl]: [number, number]): [number, number] { + const [carry, l] = add32to64(al, bl); + const h = add32(add32(ah, bh), carry); + return [h, l]; } function sub32(a: number, b: number): number { @@ -255,6 +274,13 @@ function rol32(a: number, count: number): number { return (a << count) | (a >>> (32 - count)); } +// Rotate a 64b number left `count` position +function rol64([hi, lo]: [number, number], count: number): [number, number] { + const h = (hi << count) | (lo >>> (32 - count)); + const l = (lo << count) | (hi >>> (32 - count)); + return [h, l]; +} + function stringToWords32(str: string, endian: Endian): number[] { const words32 = Array((str.length + 3) >>> 2); @@ -317,6 +343,7 @@ function byteStringToDecString(str: string): string { return decimal.split('').reverse().join(''); } +// x and y decimal, lowest significant digit first function addBigInt(x: string, y: string): string { let sum = ''; const len = Math.max(x.length, y.length); diff --git a/modules/@angular/compiler/test/i18n/digest_spec.ts b/modules/@angular/compiler/test/i18n/digest_spec.ts index 3fdc817e59..07b7bbe390 100644 --- a/modules/@angular/compiler/test/i18n/digest_spec.ts +++ b/modules/@angular/compiler/test/i18n/digest_spec.ts @@ -6,7 +6,7 @@ * found in the LICENSE file at https://angular.io/license */ -import {fingerprint, sha1} from '../../src/i18n/digest'; +import {computeMsgId, sha1} from '../../src/i18n/digest'; export function main(): void { describe('digest', () => { @@ -56,37 +56,49 @@ export function main(): void { }); describe('decimal fingerprint', () => { - const fixtures: {[msg: string]: string} = { - ' Spaced Out ': '3976450302996657536', - 'Last Name': '4407559560004943843', - 'First Name': '6028371114637047813', - 'View': '2509141182388535183', - 'START_BOLDNUMEND_BOLD of START_BOLDmillionsEND_BOLD': '29997634073898638', - 'The customer\'s credit card was authorized for AMOUNT and passed all risk checks.': - '6836487644149622036', - 'Hello world!': '3022994926184248873', - 'Jalape\u00f1o': '8054366208386598941', - 'The set of SET_NAME is {XXX, ...}.': '135956960462609535', - 'NAME took a trip to DESTINATION.': '768490705511913603', - 'by AUTHOR (YEAR)': '7036633296476174078', - '': '4416290763660062288', - }; + it('should work on well known inputs w/o meaning', () => { + const fixtures: {[msg: string]: string} = { + ' Spaced Out ': '3976450302996657536', + 'Last Name': '4407559560004943843', + 'First Name': '6028371114637047813', + 'View': '2509141182388535183', + 'START_BOLDNUMEND_BOLD of START_BOLDmillionsEND_BOLD': '29997634073898638', + 'The customer\'s credit card was authorized for AMOUNT and passed all risk checks.': + '6836487644149622036', + 'Hello world!': '3022994926184248873', + 'Jalape\u00f1o': '8054366208386598941', + 'The set of SET_NAME is {XXX, ...}.': '135956960462609535', + 'NAME took a trip to DESTINATION.': '768490705511913603', + 'by AUTHOR (YEAR)': '7036633296476174078', + '': '4416290763660062288', + }; - it('should work on well known inputs', () => { - Object.keys(fixtures).forEach(msg => { expect(fingerprint(msg)).toEqual(fixtures[msg]); }); + Object.keys(fixtures).forEach( + msg => { expect(computeMsgId(msg, '')).toEqual(fixtures[msg]); }); + }); + + it('should work on well known inputs with meaning', () => { + const fixtures: {[msg: string]: [string, string]} = { + '7790835225175622807': ['Last Name', 'Gmail UI'], + '1809086297585054940': ['First Name', 'Gmail UI'], + '3993998469942805487': ['View', 'Gmail UI'], + }; + + Object.keys(fixtures).forEach( + id => { expect(computeMsgId(fixtures[id][0], fixtures[id][1])).toEqual(id); }); }); it('should support arbitrary string size', () => { const prefix = `你好，世界`; - let result = fingerprint(prefix); + let result = computeMsgId(prefix, ''); for (let size = prefix.length; size < 5000; size += 101) { - result = prefix + fingerprint(result); + result = prefix + computeMsgId(result, ''); while (result.length < size) { result += result; } result = result.slice(-size); } - expect(fingerprint(result)).toEqual('2122606631351252558'); + expect(computeMsgId(result, '')).toEqual('2122606631351252558'); }); }); diff --git a/modules/@angular/compiler/test/i18n/integration_spec.ts b/modules/@angular/compiler/test/i18n/integration_spec.ts index 27381a0efc..ade32405dd 100644 --- a/modules/@angular/compiler/test/i18n/integration_spec.ts +++ b/modules/@angular/compiler/test/i18n/integration_spec.ts @@ -163,25 +163,25 @@ class FrLocalization extends NgLocalization { const XTB = ` - attributs i18n sur les balises - imbriqué - imbriqué - avec des espaces réservés - sur des balises non traductibles - sur des balises traductibles - {VAR_PLURAL, plural, =0 {zero} =1 {un} =2 {deux} other {beaucoup}} - - {VAR_SELECT, select, m {homme} f {femme}} - - sexe = - - dans une section traductible - + attributs i18n sur les balises + imbriqué + imbriqué + avec des espaces réservés + sur des balises non traductibles + sur des balises traductibles + {VAR_PLURAL, plural, =0 {zero} =1 {un} =2 {deux} other {beaucoup}} + + {VAR_SELECT, select, m {homme} f {femme}} + + sexe = + + dans une section traductible + Balises dans les commentaires html - ca devrait marcher + ca devrait marcher `; // unused, for reference only @@ -189,26 +189,26 @@ const XTB = ` // `fit('extract xmb', () => { console.log(toXmb(HTML)); });` const XMB = ` - i18n attribute on tags - nested - nested - <i>with placeholders</i> - on not translatable node - on translatable node - {VAR_PLURAL, plural, =0 {zero} =1 {one} =2 {two} other {<b>many</b>} } - + i18n attribute on tags + nested + nested + <i>with placeholders</i> + on not translatable node + on translatable node + {VAR_PLURAL, plural, =0 {zero} =1 {one} =2 {two} other {<b>many</b>} } + - {VAR_SELECT, select, m {male} f {female} } - - sex = - - in a translatable section - + {VAR_SELECT, select, m {male} f {female} } + + sex = + + in a translatable section + <h1>Markers in html comments</h1> <div></div> <div></div> - it <b>should</b> work + it <b>should</b> work `; diff --git a/modules/@angular/compiler/test/i18n/serializers/xmb_spec.ts b/modules/@angular/compiler/test/i18n/serializers/xmb_spec.ts index 0b00577c0a..319370102e 100644 --- a/modules/@angular/compiler/test/i18n/serializers/xmb_spec.ts +++ b/modules/@angular/compiler/test/i18n/serializers/xmb_spec.ts @@ -43,10 +43,10 @@ export function main(): void { ]> - translatable element <b>with placeholders</b> - {VAR_PLURAL, plural, =0 {<p>test</p>} } - foo - {VAR_PLURAL, plural, =0 {{VAR_GENDER, gender, other {<p>deeply nested</p>} } } } + translatable element <b>with placeholders</b> + {VAR_PLURAL, plural, =0 {<p>test</p>} } + foo + {VAR_PLURAL, plural, =0 {{VAR_SELECT, select, other {<p>deeply nested</p>} } } } `;