fix(compiler): updates hash algo for xmb/xtb files

2016-11-07 16:20:05 -08:00 · 2016-11-07 16:20:05 -08:00 · 2f14415836
parent 76e4911e8b
commit 2f14415836
5 changed files with 103 additions and 64 deletions
--- a/modules/@angular/compiler-cli/integrationtest/test/i18n_spec.ts
+++ b/modules/@angular/compiler-cli/integrationtest/test/i18n_spec.ts
@ -34,9 +34,9 @@ const EXPECTED_XMB = `<?xml version="1.0" encoding="UTF-8" ?>
 <!ELEMENT ex (#PCDATA)>
 ]>
 <messagebundle>
-  <msg id="252798779920123642">other-3rdP-component</msg>
-  <msg id="7281825156779575080" desc="desc" meaning="meaning">translate me</msg>
-  <msg id="1325493959242906696">Welcome</msg>
+  <msg id="3772663375917578720">other-3rdP-component</msg>
+  <msg id="8136548302122759730" desc="desc" meaning="meaning">translate me</msg>
+  <msg id="3492007542396725315">Welcome</msg>
 </messagebundle>
 `;

--- a/modules/@angular/compiler/src/i18n/digest.ts
+++ b/modules/@angular/compiler/src/i18n/digest.ts
@ -15,7 +15,7 @@ export function digest(message: i18n.Message): string {
 export function decimalDigest(message: i18n.Message): string {
  const visitor = new _SerializerIgnoreIcuExpVisitor();
  const parts = message.nodes.map(a => a.visit(visitor, null));
-  return fingerprint(parts.join('') + `[${message.meaning}]`);
+  return computeMsgId(parts.join(''), message.meaning);
 }

 /**
@ -138,7 +138,7 @@ function fk(index: number, b: number, c: number, d: number): [number, number] {
 * based on:
 * https://github.com/google/closure-compiler/blob/master/src/com/google/javascript/jscomp/GoogleJsMessageIdGenerator.java
 */
-export function fingerprint(str: string): string {
+export function fingerprint(str: string): [number, number] {
  const utf8 = utf8Encode(str);

  let [hi, lo] = [hash32(utf8, 0), hash32(utf8, 102072)];
@ -148,9 +148,18 @@ export function fingerprint(str: string): string {
    lo = lo ^ -0x6b5f56d8;
  }

-  hi = hi & 0x7fffffff;
+  return [hi, lo];
+}

-  return byteStringToDecString(words32ToByteString([hi, lo]));
+export function computeMsgId(msg: string, meaning: string): string {
+  let [hi, lo] = fingerprint(msg);
+
+  if (meaning) {
+    const [him, lom] = fingerprint(meaning);
+    [hi, lo] = add64(rol64([hi, lo], 1), [him, lom]);
+  }
+
+  return byteStringToDecString(words32ToByteString([hi & 0x7fffffff, lo]));
 }

 function hash32(str: string, c: number): number {
@ -239,9 +248,19 @@ function decodeSurrogatePairs(str: string, index: number): number {
 }

 function add32(a: number, b: number): number {
+  return add32to64(a, b)[1];
+}
+
+function add32to64(a: number, b: number): [number, number] {
  const low = (a & 0xffff) + (b & 0xffff);
-  const high = (a >> 16) + (b >> 16) + (low >> 16);
-  return (high << 16) | (low & 0xffff);
+  const high = (a >>> 16) + (b >>> 16) + (low >>> 16);
+  return [high >>> 16, (high << 16) | (low & 0xffff)];
+}
+
+function add64([ah, al]: [number, number], [bh, bl]: [number, number]): [number, number] {
+  const [carry, l] = add32to64(al, bl);
+  const h = add32(add32(ah, bh), carry);
+  return [h, l];
 }

 function sub32(a: number, b: number): number {
@ -255,6 +274,13 @@ function rol32(a: number, count: number): number {
  return (a << count) | (a >>> (32 - count));
 }

+// Rotate a 64b number left `count` position
+function rol64([hi, lo]: [number, number], count: number): [number, number] {
+  const h = (hi << count) | (lo >>> (32 - count));
+  const l = (lo << count) | (hi >>> (32 - count));
+  return [h, l];
+}
+
 function stringToWords32(str: string, endian: Endian): number[] {
  const words32 = Array((str.length + 3) >>> 2);

@ -317,6 +343,7 @@ function byteStringToDecString(str: string): string {
  return decimal.split('').reverse().join('');
 }

+// x and y decimal, lowest significant digit first
 function addBigInt(x: string, y: string): string {
  let sum = '';
  const len = Math.max(x.length, y.length);
--- a/modules/@angular/compiler/test/i18n/digest_spec.ts
+++ b/modules/@angular/compiler/test/i18n/digest_spec.ts
@ -6,7 +6,7 @@
 * found in the LICENSE file at https://angular.io/license
 */

-import {fingerprint, sha1} from '../../src/i18n/digest';
+import {computeMsgId, sha1} from '../../src/i18n/digest';

 export function main(): void {
  describe('digest', () => {
@ -56,37 +56,49 @@ export function main(): void {
    });

    describe('decimal fingerprint', () => {
-      const fixtures: {[msg: string]: string} = {
-        '  Spaced  Out  ': '3976450302996657536',
-        'Last Name': '4407559560004943843',
-        'First Name': '6028371114637047813',
-        'View': '2509141182388535183',
-        'START_BOLDNUMEND_BOLD of START_BOLDmillionsEND_BOLD': '29997634073898638',
-        'The customer\'s credit card was authorized for AMOUNT and passed all risk checks.':
-            '6836487644149622036',
-        'Hello world!': '3022994926184248873',
-        'Jalape\u00f1o': '8054366208386598941',
-        'The set of SET_NAME is {XXX, ...}.': '135956960462609535',
-        'NAME took a trip to DESTINATION.': '768490705511913603',
-        'by AUTHOR (YEAR)': '7036633296476174078',
-        '': '4416290763660062288',
-      };
+      it('should work on well known inputs w/o meaning', () => {
+        const fixtures: {[msg: string]: string} = {
+          '  Spaced  Out  ': '3976450302996657536',
+          'Last Name': '4407559560004943843',
+          'First Name': '6028371114637047813',
+          'View': '2509141182388535183',
+          'START_BOLDNUMEND_BOLD of START_BOLDmillionsEND_BOLD': '29997634073898638',
+          'The customer\'s credit card was authorized for AMOUNT and passed all risk checks.':
+              '6836487644149622036',
+          'Hello world!': '3022994926184248873',
+          'Jalape\u00f1o': '8054366208386598941',
+          'The set of SET_NAME is {XXX, ...}.': '135956960462609535',
+          'NAME took a trip to DESTINATION.': '768490705511913603',
+          'by AUTHOR (YEAR)': '7036633296476174078',
+          '': '4416290763660062288',
+        };

-      it('should work on well known inputs', () => {
-        Object.keys(fixtures).forEach(msg => { expect(fingerprint(msg)).toEqual(fixtures[msg]); });
+        Object.keys(fixtures).forEach(
+            msg => { expect(computeMsgId(msg, '')).toEqual(fixtures[msg]); });
+      });
+
+      it('should work on well known inputs with meaning', () => {
+        const fixtures: {[msg: string]: [string, string]} = {
+          '7790835225175622807': ['Last Name', 'Gmail UI'],
+          '1809086297585054940': ['First Name', 'Gmail UI'],
+          '3993998469942805487': ['View', 'Gmail UI'],
+        };
+
+        Object.keys(fixtures).forEach(
+            id => { expect(computeMsgId(fixtures[id][0], fixtures[id][1])).toEqual(id); });
      });

      it('should support arbitrary string size', () => {
        const prefix = `你好，世界`;
-        let result = fingerprint(prefix);
+        let result = computeMsgId(prefix, '');
        for (let size = prefix.length; size < 5000; size += 101) {
-          result = prefix + fingerprint(result);
+          result = prefix + computeMsgId(result, '');
          while (result.length < size) {
            result += result;
          }
          result = result.slice(-size);
        }
-        expect(fingerprint(result)).toEqual('2122606631351252558');
+        expect(computeMsgId(result, '')).toEqual('2122606631351252558');
      });

    });
--- a/modules/@angular/compiler/test/i18n/integration_spec.ts
+++ b/modules/@angular/compiler/test/i18n/integration_spec.ts
@ -163,25 +163,25 @@ class FrLocalization extends NgLocalization {

 const XTB = `
 <translationbundle>
-  <translation id="7613717798286137988">attributs i18n sur les balises</translation>
-  <translation id="496143996034957490">imbriqué</translation>
-  <translation id="4275167479475215567">imbriqué</translation>
-  <translation id="7210334813789040330"><ph name="START_ITALIC_TEXT"/>avec des espaces réservés<ph name="CLOSE_ITALIC_TEXT"/></translation>
-  <translation id="4769680004784140786">sur des balises non traductibles</translation>
-  <translation id="4033143013932333681">sur des balises traductibles</translation>
-  <translation id="6162642997206060264">{VAR_PLURAL, plural, =0 {zero} =1 {un} =2 {deux} other {<ph name="START_BOLD_TEXT"/>beaucoup<ph name="CLOSE_BOLD_TEXT"/>}}</translation>
-  <translation id="1882489820012923152"><ph name="ICU"/></translation>
-  <translation id="4822972059757846302">{VAR_SELECT, select, m {homme} f {femme}}</translation>
-  <translation id="5917557396782931034"><ph name="INTERPOLATION"/></translation>
-  <translation id="4687596778889597732">sexe = <ph name="INTERPOLATION"/></translation>
-  <translation id="2505882222003102347"><ph name="CUSTOM_NAME"/></translation>
-  <translation id="5340176214595489533">dans une section traductible</translation>
-  <translation id="4120782520649528473">
+  <translation id="615790887472569365">attributs i18n sur les balises</translation>
+  <translation id="3707494640264351337">imbriqué</translation>
+  <translation id="5539162898278769904">imbriqué</translation>
+  <translation id="3780349238193953556"><ph name="START_ITALIC_TEXT"/>avec des espaces réservés<ph name="CLOSE_ITALIC_TEXT"/></translation>
+  <translation id="5525133077318024839">sur des balises non traductibles</translation>
+  <translation id="8670732454866344690">sur des balises traductibles</translation>
+  <translation id="4593805537723189714">{VAR_PLURAL, plural, =0 {zero} =1 {un} =2 {deux} other {<ph name="START_BOLD_TEXT"/>beaucoup<ph name="CLOSE_BOLD_TEXT"/>}}</translation>
+  <translation id="1746565782635215"><ph name="ICU"/></translation>
+  <translation id="5868084092545682515">{VAR_SELECT, select, m {homme} f {femme}}</translation>
+  <translation id="4851788426695310455"><ph name="INTERPOLATION"/></translation>
+  <translation id="9013357158046221374">sexe = <ph name="INTERPOLATION"/></translation>
+  <translation id="8324617391167353662"><ph name="CUSTOM_NAME"/></translation>
+  <translation id="7685649297917455806">dans une section traductible</translation>
+  <translation id="2387287228265107305">
    <ph name="START_HEADING_LEVEL1"/>Balises dans les commentaires html<ph name="CLOSE_HEADING_LEVEL1"/>   
    <ph name="START_TAG_DIV"/><ph name="CLOSE_TAG_DIV"/>
    <ph name="START_TAG_DIV_1"/><ph name="ICU"/><ph name="CLOSE_TAG_DIV"></ph>
 </translation>
-  <translation id="1309478472899123444">ca <ph name="START_BOLD_TEXT"/>devrait<ph name="CLOSE_BOLD_TEXT"/> marcher</translation>
+  <translation id="1491627405349178954">ca <ph name="START_BOLD_TEXT"/>devrait<ph name="CLOSE_BOLD_TEXT"/> marcher</translation>
 </translationbundle>`;

 // unused, for reference only
@ -189,26 +189,26 @@ const XTB = `
 // `fit('extract xmb', () => { console.log(toXmb(HTML)); });`
 const XMB = `
 <messagebundle>
-  <msg id="7613717798286137988">i18n attribute on tags</msg>
-  <msg id="496143996034957490">nested</msg>
-  <msg id="4275167479475215567" meaning="different meaning">nested</msg>
-  <msg id="7210334813789040330"><ph name="START_ITALIC_TEXT"><ex>&lt;i&gt;</ex></ph>with placeholders<ph name="CLOSE_ITALIC_TEXT"><ex>&lt;/i&gt;</ex></ph></msg>
-  <msg id="4769680004784140786">on not translatable node</msg>
-  <msg id="4033143013932333681">on translatable node</msg>
-  <msg id="6162642997206060264">{VAR_PLURAL, plural, =0 {zero} =1 {one} =2 {two} other {<ph name="START_BOLD_TEXT"><ex>&lt;b&gt;</ex></ph>many<ph name="CLOSE_BOLD_TEXT"><ex>&lt;/b&gt;</ex></ph>} }</msg>
-  <msg id="1882489820012923152">
+  <msg id="615790887472569365">i18n attribute on tags</msg>
+  <msg id="3707494640264351337">nested</msg>
+  <msg id="5539162898278769904" meaning="different meaning">nested</msg>
+  <msg id="3780349238193953556"><ph name="START_ITALIC_TEXT"><ex>&lt;i&gt;</ex></ph>with placeholders<ph name="CLOSE_ITALIC_TEXT"><ex>&lt;/i&gt;</ex></ph></msg>
+  <msg id="5525133077318024839">on not translatable node</msg>
+  <msg id="8670732454866344690">on translatable node</msg>
+  <msg id="4593805537723189714">{VAR_PLURAL, plural, =0 {zero} =1 {one} =2 {two} other {<ph name="START_BOLD_TEXT"><ex>&lt;b&gt;</ex></ph>many<ph name="CLOSE_BOLD_TEXT"><ex>&lt;/b&gt;</ex></ph>} }</msg>
+  <msg id="1746565782635215">
        <ph name="ICU"/>
    </msg>
-  <msg id="4822972059757846302">{VAR_SELECT, select, m {male} f {female} }</msg>
-  <msg id="5917557396782931034"><ph name="INTERPOLATION"/></msg>
-  <msg id="4687596778889597732">sex = <ph name="INTERPOLATION"/></msg>
-  <msg id="2505882222003102347"><ph name="CUSTOM_NAME"/></msg>
-  <msg id="5340176214595489533">in a translatable section</msg>
-  <msg id="4120782520649528473">
+  <msg id="5868084092545682515">{VAR_SELECT, select, m {male} f {female} }</msg>
+  <msg id="4851788426695310455"><ph name="INTERPOLATION"/></msg>
+  <msg id="9013357158046221374">sex = <ph name="INTERPOLATION"/></msg>
+  <msg id="8324617391167353662"><ph name="CUSTOM_NAME"/></msg>
+  <msg id="7685649297917455806">in a translatable section</msg>
+  <msg id="2387287228265107305">
    <ph name="START_HEADING_LEVEL1"><ex>&lt;h1&gt;</ex></ph>Markers in html comments<ph name="CLOSE_HEADING_LEVEL1"><ex>&lt;/h1&gt;</ex></ph>   
    <ph name="START_TAG_DIV"><ex>&lt;div&gt;</ex></ph><ph name="CLOSE_TAG_DIV"><ex>&lt;/div&gt;</ex></ph>
    <ph name="START_TAG_DIV_1"><ex>&lt;div&gt;</ex></ph><ph name="ICU"/><ph name="CLOSE_TAG_DIV"><ex>&lt;/div&gt;</ex></ph>
 </msg>
-  <msg id="1309478472899123444">it <ph name="START_BOLD_TEXT"><ex>&lt;b&gt;</ex></ph>should<ph name="CLOSE_BOLD_TEXT"><ex>&lt;/b&gt;</ex></ph> work</msg>
+  <msg id="1491627405349178954">it <ph name="START_BOLD_TEXT"><ex>&lt;b&gt;</ex></ph>should<ph name="CLOSE_BOLD_TEXT"><ex>&lt;/b&gt;</ex></ph> work</msg>
 </messagebundle>
 `;
--- a/modules/@angular/compiler/test/i18n/serializers/xmb_spec.ts
+++ b/modules/@angular/compiler/test/i18n/serializers/xmb_spec.ts
@ -43,10 +43,10 @@ export function main(): void {
 <!ELEMENT ex (#PCDATA)>
 ]>
 <messagebundle>
-  <msg id="2348600990161399314">translatable element <ph name="START_BOLD_TEXT"><ex>&lt;b&gt;</ex></ph>with placeholders<ph name="CLOSE_BOLD_TEXT"><ex>&lt;/b&gt;</ex></ph> <ph name="INTERPOLATION"/></msg>
-  <msg id="5525949440406338075">{VAR_PLURAL, plural, =0 {<ph name="START_PARAGRAPH"><ex>&lt;p&gt;</ex></ph>test<ph name="CLOSE_PARAGRAPH"><ex>&lt;/p&gt;</ex></ph>} }</msg>
-  <msg id="130772889486467622" desc="d" meaning="m">foo</msg>
-  <msg id="9095788995532341072">{VAR_PLURAL, plural, =0 {{VAR_GENDER, gender, other {<ph name="START_PARAGRAPH"><ex>&lt;p&gt;</ex></ph>deeply nested<ph name="CLOSE_PARAGRAPH"><ex>&lt;/p&gt;</ex></ph>} } } }</msg>
+  <msg id="7056919470098446707">translatable element <ph name="START_BOLD_TEXT"><ex>&lt;b&gt;</ex></ph>with placeholders<ph name="CLOSE_BOLD_TEXT"><ex>&lt;/b&gt;</ex></ph> <ph name="INTERPOLATION"/></msg>
+  <msg id="2981514368455622387">{VAR_PLURAL, plural, =0 {<ph name="START_PARAGRAPH"><ex>&lt;p&gt;</ex></ph>test<ph name="CLOSE_PARAGRAPH"><ex>&lt;/p&gt;</ex></ph>} }</msg>
+  <msg id="7999024498831672133" desc="d" meaning="m">foo</msg>
+  <msg id="2015957479576096115">{VAR_PLURAL, plural, =0 {{VAR_SELECT, select, other {<ph name="START_PARAGRAPH"><ex>&lt;p&gt;</ex></ph>deeply nested<ph name="CLOSE_PARAGRAPH"><ex>&lt;/p&gt;</ex></ph>} } } }</msg>
 </messagebundle>
 `;