refactor(compiler): expose token parts in Text nodes (#42062)

When it was tokenized, text content is split into parts that can include
interpolations and encoded entities tokens.

To make this information available to downstream processing, this commit
adds these tokens to the `Text` AST nodes, with suitable processing.

PR Close #42062

packages/compiler/src/i18n/extractor_merger.ts

@@ -396,12 +396,12 @@ class _Visitor implements html.Visitor {
           if (nodes.length == 0) {
             translatedAttributes.push(new html.Attribute(
                 attr.name, '', attr.sourceSpan, undefined /* keySpan */, undefined /* valueSpan */,
-                undefined /* i18n */));
+                undefined /* valueTokens */, undefined /* i18n */));
           } else if (nodes[0] instanceof html.Text) {
             const value = (nodes[0] as html.Text).value;
             translatedAttributes.push(new html.Attribute(
                 attr.name, value, attr.sourceSpan, undefined /* keySpan */,
-                undefined /* valueSpan */, undefined /* i18n */));
+                undefined /* valueSpan */, undefined /* valueTokens */, undefined /* i18n */));
           } else {
             this._reportError(
                 el,

packages/compiler/src/ml_parser/ast.ts

@@ -9,6 +9,7 @@
 import {AstPath} from '../ast_path';
 import {I18nMeta} from '../i18n/i18n_ast';
 import {ParseSourceSpan} from '../parse_util';
+import {Token} from './lexer';
 
 interface BaseNode {
   sourceSpan: ParseSourceSpan;
@@ -23,7 +24,8 @@ export abstract class NodeWithI18n implements BaseNode {
 }
 
 export class Text extends NodeWithI18n {
-  constructor(public value: string, sourceSpan: ParseSourceSpan, i18n?: I18nMeta) {
+  constructor(
+      public value: string, sourceSpan: ParseSourceSpan, public tokens: Token[], i18n?: I18nMeta) {
     super(sourceSpan, i18n);
   }
   override visit(visitor: Visitor, context: any): any {
@@ -55,8 +57,8 @@ export class ExpansionCase implements BaseNode {
 export class Attribute extends NodeWithI18n {
   constructor(
       public name: string, public value: string, sourceSpan: ParseSourceSpan,
-      readonly keySpan: ParseSourceSpan|undefined, public valueSpan?: ParseSourceSpan,
+      readonly keySpan: ParseSourceSpan|undefined, public valueSpan: ParseSourceSpan|undefined,
-      i18n?: I18nMeta) {
+      public valueTokens: Token[]|undefined, i18n: I18nMeta|undefined) {
     super(sourceSpan, i18n);
   }
   override visit(visitor: Visitor, context: any): any {

packages/compiler/src/ml_parser/html_whitespaces.ts

@@ -8,6 +8,7 @@
 
 import * as html from './ast';
 import {NGSP_UNICODE} from './entities';
+import {Token, TokenType} from './lexer';
 import {ParseTreeResult} from './parser';
 
 export const PRESERVE_WS_ATTR_NAME = 'ngPreserveWhitespaces';
@@ -74,8 +75,13 @@ export class WhitespaceVisitor implements html.Visitor {
         (context.prev instanceof html.Expansion || context.next instanceof html.Expansion);
 
     if (isNotBlank || hasExpansionSibling) {
-      return new html.Text(
+      // Process the whitespace in the tokens of this Text node
-          replaceNgsp(text.value).replace(WS_REPLACE_REGEXP, ' '), text.sourceSpan, text.i18n);
+      const tokens = text.tokens.map(
+          token => token.type === TokenType.TEXT ? createTextTokenAfterWhitespaceProcessing(token) :
+                                                   token);
+      // Process the whitespace of the value of this Text node
+      const value = processWhitespace(text.value);
+      return new html.Text(value, text.sourceSpan, tokens, text.i18n);
     }
 
     return null;
@@ -94,6 +100,14 @@ export class WhitespaceVisitor implements html.Visitor {
   }
 }
 
+function createTextTokenAfterWhitespaceProcessing(token: Token): Token {
+  return new Token(token.type, [processWhitespace(token.parts[0])], token.sourceSpan);
+}
+
+function processWhitespace(text: string): string {
+  return replaceNgsp(text).replace(WS_REPLACE_REGEXP, ' ');
+}
+
 export function removeWhitespaces(htmlAstWithErrors: ParseTreeResult): ParseTreeResult {
   return new ParseTreeResult(
       html.visitAll(new WhitespaceVisitor(), htmlAstWithErrors.rootNodes),

packages/compiler/src/ml_parser/icu_ast_expander.ts

@@ -102,14 +102,15 @@ function _expandPluralForm(ast: html.Expansion, errors: ParseError[]): html.Elem
     errors.push(...expansionResult.errors);
 
     return new html.Element(
-        `ng-template`, [new html.Attribute(
+        `ng-template`,
+        [new html.Attribute(
             'ngPluralCase', `${c.value}`, c.valueSourceSpan, undefined /* keySpan */,
-                           undefined /* valueSpan */, undefined /* i18n */)],
+            undefined /* valueSpan */, undefined /* valueTokens */, undefined /* i18n */)],
         expansionResult.nodes, c.sourceSpan, c.sourceSpan, c.sourceSpan);
   });
   const switchAttr = new html.Attribute(
       '[ngPlural]', ast.switchValue, ast.switchValueSourceSpan, undefined /* keySpan */,
-      undefined /* valueSpan */, undefined /* i18n */);
+      undefined /* valueSpan */, undefined /* valueTokens */, undefined /* i18n */);
   return new html.Element(
       'ng-container', [switchAttr], children, ast.sourceSpan, ast.sourceSpan, ast.sourceSpan);
 }
@@ -123,21 +124,23 @@ function _expandDefaultForm(ast: html.Expansion, errors: ParseError[]): html.Ele
     if (c.value === 'other') {
       // other is the default case when no values match
       return new html.Element(
-          `ng-template`, [new html.Attribute(
+          `ng-template`,
+          [new html.Attribute(
               'ngSwitchDefault', '', c.valueSourceSpan, undefined /* keySpan */,
-                             undefined /* valueSpan */, undefined /* i18n */)],
+              undefined /* valueSpan */, undefined /* valueTokens */, undefined /* i18n */)],
           expansionResult.nodes, c.sourceSpan, c.sourceSpan, c.sourceSpan);
     }
 
     return new html.Element(
-        `ng-template`, [new html.Attribute(
+        `ng-template`,
+        [new html.Attribute(
             'ngSwitchCase', `${c.value}`, c.valueSourceSpan, undefined /* keySpan */,
-                           undefined /* valueSpan */, undefined /* i18n */)],
+            undefined /* valueSpan */, undefined /* valueTokens */, undefined /* i18n */)],
         expansionResult.nodes, c.sourceSpan, c.sourceSpan, c.sourceSpan);
   });
   const switchAttr = new html.Attribute(
       '[ngSwitch]', ast.switchValue, ast.switchValueSourceSpan, undefined /* keySpan */,
-      undefined /* valueSpan */, undefined /* i18n */);
+      undefined /* valueSpan */, undefined /* valueTokens */, undefined /* i18n */);
   return new html.Element(
       'ng-container', [switchAttr], children, ast.sourceSpan, ast.sourceSpan, ast.sourceSpan);
 }

packages/compiler/src/ml_parser/parser.ts

@@ -216,6 +216,7 @@ class _TreeBuilder {
   }
 
   private _consumeText(token: lex.Token) {
+    const tokens = [token];
     const startSpan = token.sourceSpan;
     let text = token.parts[0];
     if (text.length > 0 && text[0] == '\n') {
@@ -223,14 +224,15 @@ class _TreeBuilder {
       if (parent != null && parent.children.length == 0 &&
           this.getTagDefinition(parent.name).ignoreFirstLf) {
         text = text.substring(1);
+        tokens[0] = {type: token.type, sourceSpan: token.sourceSpan, parts: [text]};
       }
     }
 
-    // For now recombine text, interpolation and entity tokens
     while (this._peek.type === lex.TokenType.INTERPOLATION ||
            this._peek.type === lex.TokenType.TEXT ||
            this._peek.type === lex.TokenType.ENCODED_ENTITY) {
       token = this._advance();
+      tokens.push(token);
       if (token.type === lex.TokenType.INTERPOLATION) {
         // For backward compatibility we decode HTML entities that appear in interpolation
         // expressions. This is arguably a bug, but it could be a considerable breaking change to
@@ -248,8 +250,8 @@ class _TreeBuilder {
       const endSpan = token.sourceSpan;
       this._addToParent(new html.Text(
           text,
-          new ParseSourceSpan(
+          new ParseSourceSpan(startSpan.start, endSpan.end, startSpan.fullStart, startSpan.details),
-              startSpan.start, endSpan.end, startSpan.fullStart, startSpan.details)));
+          tokens));
     }
   }
 
@@ -372,16 +374,17 @@ class _TreeBuilder {
 
     // Consume the attribute value
     let value = '';
+    const valueTokens: lex.Token[] = [];
     let valueStartSpan: ParseSourceSpan|undefined = undefined;
     let valueEnd: ParseLocation|undefined = undefined;
     if (this._peek.type === lex.TokenType.ATTR_VALUE_TEXT) {
       valueStartSpan = this._peek.sourceSpan;
       valueEnd = this._peek.sourceSpan.end;
-      // For now recombine text, interpolation and entity tokens
       while (this._peek.type === lex.TokenType.ATTR_VALUE_TEXT ||
              this._peek.type === lex.TokenType.ATTR_VALUE_INTERPOLATION ||
              this._peek.type === lex.TokenType.ENCODED_ENTITY) {
-        let valueToken = this._advance();
+        const valueToken = this._advance();
+        valueTokens.push(valueToken);
         if (valueToken.type === lex.TokenType.ATTR_VALUE_INTERPOLATION) {
           // For backward compatibility we decode HTML entities that appear in interpolation
           // expressions. This is arguably a bug, but it could be a considerable breaking change to
@@ -408,7 +411,8 @@ class _TreeBuilder {
     return new html.Attribute(
         fullName, value,
         new ParseSourceSpan(attrName.sourceSpan.start, attrEnd, attrName.sourceSpan.fullStart),
-        attrName.sourceSpan, valueSpan);
+        attrName.sourceSpan, valueSpan, valueTokens.length > 0 ? valueTokens : undefined,
+        undefined);
   }
 
   private _getParentElement(): html.Element|null {

packages/compiler/test/ml_parser/ast_spec_utils.ts

@@ -52,12 +52,16 @@ class _Humanizer implements html.Visitor {
   }
 
   visitAttribute(attribute: html.Attribute, context: any): any {
-    const res = this._appendContext(attribute, [html.Attribute, attribute.name, attribute.value]);
+    const valueTokens = attribute.valueTokens ?? [];
+    const res = this._appendContext(attribute, [
+      html.Attribute, attribute.name, attribute.value, ...valueTokens.map(token => token.parts)
+    ]);
     this.result.push(res);
   }
 
   visitText(text: html.Text, context: any): any {
-    const res = this._appendContext(text, [html.Text, text.value, this.elDepth]);
+    const res = this._appendContext(
+        text, [html.Text, text.value, this.elDepth, ...text.tokens.map(token => token.parts)]);
     this.result.push(res);
   }
 

packages/compiler/test/ml_parser/html_parser_spec.ts

@@ -24,31 +24,31 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes}
     describe('parse', () => {
       describe('text nodes', () => {
         it('should parse root level text nodes', () => {
-          expect(humanizeDom(parser.parse('a', 'TestComp'))).toEqual([[html.Text, 'a', 0]]);
+          expect(humanizeDom(parser.parse('a', 'TestComp'))).toEqual([[html.Text, 'a', 0, ['a']]]);
         });
 
         it('should parse text nodes inside regular elements', () => {
           expect(humanizeDom(parser.parse('<div>a</div>', 'TestComp'))).toEqual([
-            [html.Element, 'div', 0], [html.Text, 'a', 1]
+            [html.Element, 'div', 0], [html.Text, 'a', 1, ['a']]
           ]);
         });
 
         it('should parse text nodes inside <ng-template> elements', () => {
           expect(humanizeDom(parser.parse('<ng-template>a</ng-template>', 'TestComp'))).toEqual([
-            [html.Element, 'ng-template', 0], [html.Text, 'a', 1]
+            [html.Element, 'ng-template', 0], [html.Text, 'a', 1, ['a']]
           ]);
         });
 
         it('should parse CDATA', () => {
           expect(humanizeDom(parser.parse('<![CDATA[text]]>', 'TestComp'))).toEqual([
-            [html.Text, 'text', 0]
+            [html.Text, 'text', 0, ['text']]
           ]);
         });
 
         it('should normalize line endings within CDATA', () => {
           const parsed = parser.parse('<![CDATA[ line 1 \r\n line 2 ]]>', 'TestComp');
           expect(humanizeDom(parsed)).toEqual([
-            [html.Text, ' line 1 \n line 2 ', 0],
+            [html.Text, ' line 1 \n line 2 ', 0, [' line 1 \n line 2 ']],
           ]);
           expect(parsed.errors).toEqual([]);
         });
@@ -76,8 +76,8 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes}
           expect(humanizeDom(parser.parse('<link rel="author license" href="/about">', 'TestComp')))
               .toEqual([
                 [html.Element, 'link', 0],
-                [html.Attribute, 'rel', 'author license'],
+                [html.Attribute, 'rel', 'author license', ['author license']],
-                [html.Attribute, 'href', '/about'],
+                [html.Attribute, 'href', '/about', ['/about']],
               ]);
         });
 
@@ -106,9 +106,9 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes}
         it('should close void elements on text nodes', () => {
           expect(humanizeDom(parser.parse('<p>before<br>after</p>', 'TestComp'))).toEqual([
             [html.Element, 'p', 0],
-            [html.Text, 'before', 1],
+            [html.Text, 'before', 1, ['before']],
             [html.Element, 'br', 1],
-            [html.Text, 'after', 1],
+            [html.Text, 'after', 1, ['after']],
           ]);
         });
 
@@ -116,9 +116,9 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes}
           expect(humanizeDom(parser.parse('<div><p>1<p>2</div>', 'TestComp'))).toEqual([
             [html.Element, 'div', 0],
             [html.Element, 'p', 1],
-            [html.Text, '1', 2],
+            [html.Text, '1', 2, ['1']],
             [html.Element, 'p', 1],
-            [html.Text, '2', 2],
+            [html.Text, '2', 2, ['2']],
           ]);
         });
 
@@ -200,12 +200,12 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes}
                      'TestComp')))
               .toEqual([
                 [html.Element, 'p', 0],
-                [html.Text, '\n', 1],
+                [html.Text, '\n', 1, ['\n']],
                 [html.Element, 'textarea', 0],
                 [html.Element, 'pre', 0],
-                [html.Text, '\n', 1],
+                [html.Text, '\n', 1, ['\n']],
                 [html.Element, 'listing', 0],
-                [html.Text, '\n', 1],
+                [html.Text, '\n', 1, ['\n']],
               ]);
         });
 
@@ -214,28 +214,28 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes}
           parsed = parser.parse('<title> line 1 \r\n line 2 </title>', 'TestComp');
           expect(humanizeDom(parsed)).toEqual([
             [html.Element, 'title', 0],
-            [html.Text, ' line 1 \n line 2 ', 1],
+            [html.Text, ' line 1 \n line 2 ', 1, [' line 1 \n line 2 ']],
           ]);
           expect(parsed.errors).toEqual([]);
 
           parsed = parser.parse('<script> line 1 \r\n line 2 </script>', 'TestComp');
           expect(humanizeDom(parsed)).toEqual([
             [html.Element, 'script', 0],
-            [html.Text, ' line 1 \n line 2 ', 1],
+            [html.Text, ' line 1 \n line 2 ', 1, [' line 1 \n line 2 ']],
           ]);
           expect(parsed.errors).toEqual([]);
 
           parsed = parser.parse('<div> line 1 \r\n line 2 </div>', 'TestComp');
           expect(humanizeDom(parsed)).toEqual([
             [html.Element, 'div', 0],
-            [html.Text, ' line 1 \n line 2 ', 1],
+            [html.Text, ' line 1 \n line 2 ', 1, [' line 1 \n line 2 ']],
           ]);
           expect(parsed.errors).toEqual([]);
 
           parsed = parser.parse('<span> line 1 \r\n line 2 </span>', 'TestComp');
           expect(humanizeDom(parsed)).toEqual([
             [html.Element, 'span', 0],
-            [html.Text, ' line 1 \n line 2 ', 1],
+            [html.Text, ' line 1 \n line 2 ', 1, [' line 1 \n line 2 ']],
           ]);
           expect(parsed.errors).toEqual([]);
         });
@@ -245,8 +245,22 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes}
         it('should parse attributes on regular elements case sensitive', () => {
           expect(humanizeDom(parser.parse('<div kEy="v" key2=v2></div>', 'TestComp'))).toEqual([
             [html.Element, 'div', 0],
-            [html.Attribute, 'kEy', 'v'],
+            [html.Attribute, 'kEy', 'v', ['v']],
-            [html.Attribute, 'key2', 'v2'],
+            [html.Attribute, 'key2', 'v2', ['v2']],
+          ]);
+        });
+
+        it('should parse attributes containing interpolation', () => {
+          expect(humanizeDom(parser.parse('<div foo="1{{message}}2"></div>', 'TestComp'))).toEqual([
+            [html.Element, 'div', 0],
+            [html.Attribute, 'foo', '1{{message}}2', ['1'], ['{{', 'message', '}}'], ['2']]
+          ]);
+        });
+
+        it('should parse attributes containing encoded entities', () => {
+          expect(humanizeDom(parser.parse('<div foo="&amp;"></div>', 'TestComp'))).toEqual([
+            [html.Element, 'div', 0],
+            [html.Attribute, 'foo', '&', [''], ['&', '&amp;'], ['']],
           ]);
         });
 
@@ -259,7 +273,10 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes}
                   html.Element, 'div', 0, '<div foo="{{&amp;}}"></div>', '<div foo="{{&amp;}}">',
                   '</div>'
                 ],
-                [html.Attribute, 'foo', '{{&}}', 'foo="{{&amp;}}"']
+                [
+                  html.Attribute, 'foo', '{{&}}', [''], ['{{', '&amp;', '}}'], [''],
+                  'foo="{{&amp;}}"'
+                ]
               ]);
         });
 
@@ -268,7 +285,7 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes}
               parser.parse('<div key="  \r\n line 1 \r\n   line 2  "></div>', 'TestComp');
           expect(humanizeDom(result)).toEqual([
             [html.Element, 'div', 0],
-            [html.Attribute, 'key', '  \n line 1 \n   line 2  '],
+            [html.Attribute, 'key', '  \n line 1 \n   line 2  ', ['  \n line 1 \n   line 2  ']],
           ]);
           expect(result.errors).toEqual([]);
         });
@@ -283,7 +300,7 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes}
         it('should parse attributes on svg elements case sensitive', () => {
           expect(humanizeDom(parser.parse('<svg viewBox="0"></svg>', 'TestComp'))).toEqual([
             [html.Element, ':svg:svg', 0],
-            [html.Attribute, 'viewBox', '0'],
+            [html.Attribute, 'viewBox', '0', ['0']],
           ]);
         });
 
@@ -291,14 +308,14 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes}
           expect(humanizeDom(parser.parse('<ng-template k="v"></ng-template>', 'TestComp')))
               .toEqual([
                 [html.Element, 'ng-template', 0],
-                [html.Attribute, 'k', 'v'],
+                [html.Attribute, 'k', 'v', ['v']],
               ]);
         });
 
         it('should support namespace', () => {
           expect(humanizeDom(parser.parse('<svg:use xlink:href="Port" />', 'TestComp'))).toEqual([
             [html.Element, ':svg:use', 0],
-            [html.Attribute, ':xlink:href', 'Port'],
+            [html.Attribute, ':xlink:href', 'Port', ['Port']],
           ]);
         });
       });
@@ -325,23 +342,23 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes}
 
           expect(humanizeDom(parsed)).toEqual([
             [html.Element, 'div', 0],
-            [html.Text, 'before', 1],
+            [html.Text, 'before', 1, ['before']],
             [html.Expansion, 'messages.length', 'plural', 1],
             [html.ExpansionCase, '=0', 2],
             [html.ExpansionCase, '=1', 2],
-            [html.Text, 'after', 1],
+            [html.Text, 'after', 1, ['after']],
           ]);
           const cases = (<any>parsed.rootNodes[0]).children[1].cases;
 
           expect(humanizeDom(new ParseTreeResult(cases[0].expression, []))).toEqual([
-            [html.Text, 'You have ', 0],
+            [html.Text, 'You have ', 0, ['You have ']],
             [html.Element, 'b', 0],
-            [html.Text, 'no', 1],
+            [html.Text, 'no', 1, ['no']],
-            [html.Text, ' messages', 0],
+            [html.Text, ' messages', 0, [' messages']],
           ]);
 
           expect(humanizeDom(new ParseTreeResult(cases[1].expression, []))).toEqual([
-            [html.Text, 'One {{message}}', 0]
+            [html.Text, 'One {{message}}', 0, ['One '], ['{{', 'message', '}}'], ['']]
           ]);
         });
 
@@ -363,20 +380,20 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes}
 
              expect(humanizeDom(parsed)).toEqual([
                [html.Element, 'div', 0],
-               [html.Text, '\n  ', 1],
+               [html.Text, '\n  ', 1, ['\n  ']],
                [html.Expansion, '\n    messages.length', 'plural', 1],
                [html.ExpansionCase, '=0', 2],
                [html.ExpansionCase, '=1', 2],
-               [html.Text, '\n', 1],
+               [html.Text, '\n', 1, ['\n']],
              ]);
              const cases = (<any>parsed.rootNodes[0]).children[1].cases;
 
              expect(humanizeDom(new ParseTreeResult(cases[0].expression, []))).toEqual([
-               [html.Text, 'You have \nno\n messages', 0],
+               [html.Text, 'You have \nno\n messages', 0, ['You have \nno\n messages']],
              ]);
 
              expect(humanizeDom(new ParseTreeResult(cases[1].expression, []))).toEqual([
-               [html.Text, 'One {{message}}', 0]
+               [html.Text, 'One {{message}}', 0, ['One '], ['{{', 'message', '}}'], ['']]
              ]);
 
              expect(parsed.errors).toEqual([]);
@@ -396,20 +413,20 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes}
 
              expect(humanizeDom(parsed)).toEqual([
                [html.Element, 'div', 0],
-               [html.Text, '\n  ', 1],
+               [html.Text, '\n  ', 1, ['\n  ']],
                [html.Expansion, '\r\n    messages.length', 'plural', 1],
                [html.ExpansionCase, '=0', 2],
                [html.ExpansionCase, '=1', 2],
-               [html.Text, '\n', 1],
+               [html.Text, '\n', 1, ['\n']],
              ]);
              const cases = (<any>parsed.rootNodes[0]).children[1].cases;
 
              expect(humanizeDom(new ParseTreeResult(cases[0].expression, []))).toEqual([
-               [html.Text, 'You have \nno\n messages', 0],
+               [html.Text, 'You have \nno\n messages', 0, ['You have \nno\n messages']],
              ]);
 
              expect(humanizeDom(new ParseTreeResult(cases[1].expression, []))).toEqual([
-               [html.Text, 'One {{message}}', 0]
+               [html.Text, 'One {{message}}', 0, ['One '], ['{{', 'message', '}}'], ['']]
              ]);
 
              expect(parsed.errors).toEqual([]);
@@ -433,20 +450,20 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes}
 
              expect(humanizeDom(parsed)).toEqual([
                [html.Element, 'div', 0],
-               [html.Text, '\n  ', 1],
+               [html.Text, '\n  ', 1, ['\n  ']],
                [html.Expansion, '\n    messages.length', 'plural', 1],
                [html.ExpansionCase, '=0', 2],
                [html.ExpansionCase, '=1', 2],
-               [html.Text, '\n', 1],
+               [html.Text, '\n', 1, ['\n']],
              ]);
              const cases = (<any>parsed.rootNodes[0]).children[1].cases;
 
              expect(humanizeDom(new ParseTreeResult(cases[0].expression, []))).toEqual([
-               [html.Text, 'You have \nno\n messages', 0],
+               [html.Text, 'You have \nno\n messages', 0, ['You have \nno\n messages']],
              ]);
 
              expect(humanizeDom(new ParseTreeResult(cases[1].expression, []))).toEqual([
-               [html.Text, 'One {{message}}', 0]
+               [html.Text, 'One {{message}}', 0, ['One '], ['{{', 'message', '}}'], ['']]
              ]);
 
              expect(parsed.errors).toEqual([]);
@@ -466,20 +483,20 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes}
 
              expect(humanizeDom(parsed)).toEqual([
                [html.Element, 'div', 0],
-               [html.Text, '\n  ', 1],
+               [html.Text, '\n  ', 1, ['\n  ']],
                [html.Expansion, '\r\n    messages.length', 'plural', 1],
                [html.ExpansionCase, '=0', 2],
                [html.ExpansionCase, '=1', 2],
-               [html.Text, '\n', 1],
+               [html.Text, '\n', 1, ['\n']],
              ]);
              const cases = (<any>parsed.rootNodes[0]).children[1].cases;
 
              expect(humanizeDom(new ParseTreeResult(cases[0].expression, []))).toEqual([
-               [html.Text, 'You have \nno\n messages', 0],
+               [html.Text, 'You have \nno\n messages', 0, ['You have \nno\n messages']],
              ]);
 
              expect(humanizeDom(new ParseTreeResult(cases[1].expression, []))).toEqual([
-               [html.Text, 'One {{message}}', 0]
+               [html.Text, 'One {{message}}', 0, ['One '], ['{{', 'message', '}}'], ['']]
              ]);
 
              expect(parsed.errors).toEqual([]);
@@ -512,7 +529,7 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes}
           expect(humanizeDom(new ParseTreeResult(firstCase.expression, []))).toEqual([
             [html.Expansion, 'p.gender', 'select', 0],
             [html.ExpansionCase, 'male', 1],
-            [html.Text, ' ', 0],
+            [html.Text, ' ', 0, [' ']],
           ]);
         });
 
@@ -540,10 +557,10 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes}
 
              const expansion = parsed.rootNodes[0] as html.Expansion;
              expect(humanizeDom(new ParseTreeResult(expansion.cases[0].expression, []))).toEqual([
-               [html.Text, 'zero \n       ', 0],
+               [html.Text, 'zero \n       ', 0, ['zero \n       ']],
                [html.Expansion, '\n         p.gender', 'select', 0],
                [html.ExpansionCase, 'male', 1],
-               [html.Text, '\n     ', 0],
+               [html.Text, '\n     ', 0, ['\n     ']],
              ]);
 
              expect(parsed.errors).toEqual([]);
@@ -569,10 +586,10 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes}
 
              const expansion = parsed.rootNodes[0] as html.Expansion;
              expect(humanizeDom(new ParseTreeResult(expansion.cases[0].expression, []))).toEqual([
-               [html.Text, 'zero \n       ', 0],
+               [html.Text, 'zero \n       ', 0, ['zero \n       ']],
                [html.Expansion, '\r\n         p.gender', 'select', 0],
                [html.ExpansionCase, 'male', 1],
-               [html.Text, '\n     ', 0],
+               [html.Text, '\n     ', 0, ['\n     ']],
              ]);
 
              expect(parsed.errors).toEqual([]);
@@ -598,10 +615,10 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes}
 
              const expansion = parsed.rootNodes[0] as html.Expansion;
              expect(humanizeDom(new ParseTreeResult(expansion.cases[0].expression, []))).toEqual([
-               [html.Text, 'zero \n       ', 0],
+               [html.Text, 'zero \n       ', 0, ['zero \n       ']],
                [html.Expansion, '\r\n         p.gender', 'select', 0],
                [html.ExpansionCase, 'male', 1],
-               [html.Text, '\n     ', 0],
+               [html.Text, '\n     ', 0, ['\n     ']],
              ]);
 
 
@@ -670,11 +687,11 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes}
                   '<div [prop]="v1" (e)="do()" attr="v2" noValue>\na\n</div>',
                   '<div [prop]="v1" (e)="do()" attr="v2" noValue>', '</div>'
                 ],
-                [html.Attribute, '[prop]', 'v1', '[prop]="v1"'],
+                [html.Attribute, '[prop]', 'v1', ['v1'], '[prop]="v1"'],
-                [html.Attribute, '(e)', 'do()', '(e)="do()"'],
+                [html.Attribute, '(e)', 'do()', ['do()'], '(e)="do()"'],
-                [html.Attribute, 'attr', 'v2', 'attr="v2"'],
+                [html.Attribute, 'attr', 'v2', ['v2'], 'attr="v2"'],
                 [html.Attribute, 'noValue', '', 'noValue'],
-                [html.Text, '\na\n', 1, '\na\n'],
+                [html.Text, '\na\n', 1, ['\na\n'], '\na\n'],
               ]);
         });
 
@@ -695,7 +712,9 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes}
                      '{{&amp;}}' +
                          '{{&#x25BE;}}' +
                          '{{&#9662;}}' +
+                         '{{&unknown;}}' +
                          '{{&amp (no semi-colon)}}' +
+                         '{{&#xyz; (invalid hex)}}' +
                          '{{&#25BE; (invalid decimal)}}',
                      'TestComp')))
               .toEqual([[
@@ -703,17 +722,48 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes}
                 '{{&}}' +
                     '{{\u25BE}}' +
                     '{{\u25BE}}' +
+                    '{{&unknown;}}' +
                     '{{&amp (no semi-colon)}}' +
+                    '{{&#xyz; (invalid hex)}}' +
                     '{{&#25BE; (invalid decimal)}}',
                 0,
+                [''],
+                ['{{', '&amp;', '}}'],
+                [''],
+                ['{{', '&#x25BE;', '}}'],
+                [''],
+                ['{{', '&#9662;', '}}'],
+                [''],
+                ['{{', '&unknown;', '}}'],
+                [''],
+                ['{{', '&amp (no semi-colon)', '}}'],
+                [''],
+                ['{{', '&#xyz; (invalid hex)', '}}'],
+                [''],
+                ['{{', '&#25BE; (invalid decimal)', '}}'],
+                [''],
                 '{{&amp;}}' +
                     '{{&#x25BE;}}' +
                     '{{&#9662;}}' +
+                    '{{&unknown;}}' +
                     '{{&amp (no semi-colon)}}' +
+                    '{{&#xyz; (invalid hex)}}' +
                     '{{&#25BE; (invalid decimal)}}',
               ]]);
         });
 
+        it('should support interpolations in text', () => {
+          expect(
+              humanizeDomSourceSpans(parser.parse('<div> pre {{ value }} post </div>', 'TestComp')))
+              .toEqual([
+                [html.Element, 'div', 0, '<div> pre {{ value }} post </div>', '<div>', '</div>'],
+                [
+                  html.Text, ' pre {{ value }} post ', 1, [' pre '], ['{{', ' value ', '}}'],
+                  [' post '], ' pre {{ value }} post '
+                ],
+              ]);
+        });
+
         it('should not set the end source span for void elements', () => {
           expect(humanizeDomSourceSpans(parser.parse('<div><br></div>', 'TestComp'))).toEqual([
             [html.Element, 'div', 0, '<div><br></div>', '<div>', '</div>'],
@@ -758,10 +808,10 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes}
                   html.Element, 'input', 0, '<input type="text" />', '<input type="text" />',
                   '<input type="text" />'
                 ],
-                [html.Attribute, 'type', 'text', 'type="text"'],
+                [html.Attribute, 'type', 'text', ['text'], 'type="text"'],
-                [html.Text, '\n\n\n  ', 0, ''],
+                [html.Text, '\n\n\n  ', 0, ['\n\n\n  '], ''],
                 [html.Element, 'span', 0, '<span>\n</span>', '<span>', '</span>'],
-                [html.Text, '\n', 1, ''],
+                [html.Text, '\n', 1, ['\n'], ''],
               ]);
         });
 
@@ -774,9 +824,9 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes}
               .toEqual([
                 [html.Element, 'div', 0, '<div><li>A<li>B</div>', '<div>', '</div>'],
                 [html.Element, 'li', 1, '<li>', '<li>', null],
-                [html.Text, 'A', 2, 'A'],
+                [html.Text, 'A', 2, ['A'], 'A'],
                 [html.Element, 'li', 1, '<li>', '<li>', null],
-                [html.Text, 'B', 2, 'B'],
+                [html.Text, 'B', 2, ['B'], 'B'],
               ]);
         });
 
@@ -814,7 +864,7 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes}
       describe('visitor', () => {
         it('should visit text nodes', () => {
           const result = humanizeDom(parser.parse('text', 'TestComp'));
-          expect(result).toEqual([[html.Text, 'text', 0]]);
+          expect(result).toEqual([[html.Text, 'text', 0, ['text']]]);
         });
 
         it('should visit element nodes', () => {
@@ -824,7 +874,7 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes}
 
         it('should visit attribute nodes', () => {
           const result = humanizeDom(parser.parse('<div id="foo"></div>', 'TestComp'));
-          expect(result).toContain([html.Attribute, 'id', 'foo']);
+          expect(result).toContain([html.Attribute, 'id', 'foo', ['foo']]);
         });
 
         it('should visit all nodes', () => {
@@ -932,7 +982,7 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes}
 
             expect(humanizeNodes(rootNodes, true)).toEqual([
               [html.Element, 'div', 0, '<div class="hi" sty', '<div class="hi" sty', null],
-              [html.Attribute, 'class', 'hi', 'class="hi"'],
+              [html.Attribute, 'class', 'hi', ['hi'], 'class="hi"'],
               [html.Attribute, 'sty', '', 'sty'],
               [html.Element, 'span', 0, '<span></span>', '<span>', '</span>'],
             ]);
@@ -947,7 +997,7 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes}
 
             expect(humanizeNodes(rootNodes, true)).toEqual([
               [html.Element, 'div', 0, '<div ', '<div ', null],
-              [html.Text, '"', 0, '"'],
+              [html.Text, '"', 0, ['"'], '"'],
               [html.Element, 'span', 0, '<span></span>', '<span>', '</span>'],
             ]);
 

packages/compiler/test/ml_parser/html_whitespaces_spec.ts

@@ -7,6 +7,7 @@
  */
 
 import * as html from '../../src/ml_parser/ast';
+import {NGSP_UNICODE} from '../../src/ml_parser/entities';
 import {HtmlParser} from '../../src/ml_parser/html_parser';
 import {PRESERVE_WS_ATTR_NAME, removeWhitespaces} from '../../src/ml_parser/html_whitespaces';
 import {TokenizeOptions} from '../../src/ml_parser/lexer';
@@ -52,48 +53,86 @@ import {humanizeDom} from './ast_spec_utils';
       expect(parseAndRemoveWS('<div><span>foo</span>&ngsp;<span>bar</span></div>')).toEqual([
         [html.Element, 'div', 0],
         [html.Element, 'span', 1],
-        [html.Text, 'foo', 2],
+        [html.Text, 'foo', 2, ['foo']],
-        [html.Text, ' ', 1],
+        [html.Text, ' ', 1, [''], [NGSP_UNICODE, '&ngsp;'], ['']],
         [html.Element, 'span', 1],
-        [html.Text, 'bar', 2],
+        [html.Text, 'bar', 2, ['bar']],
       ]);
     });
 
     it('should replace multiple whitespaces with one space', () => {
-      expect(parseAndRemoveWS('\n\n\nfoo\t\t\t')).toEqual([[html.Text, ' foo ', 0]]);
+      expect(parseAndRemoveWS('\n\n\nfoo\t\t\t')).toEqual([[html.Text, ' foo ', 0, [' foo ']]]);
-      expect(parseAndRemoveWS('   \n foo  \t ')).toEqual([[html.Text, ' foo ', 0]]);
+      expect(parseAndRemoveWS('   \n foo  \t ')).toEqual([[html.Text, ' foo ', 0, [' foo ']]]);
     });
 
     it('should not replace &nbsp;', () => {
-      expect(parseAndRemoveWS('&nbsp;')).toEqual([[html.Text, '\u00a0', 0]]);
+      expect(parseAndRemoveWS('&nbsp;')).toEqual([
+        [html.Text, '\u00a0', 0, [''], ['\u00a0', '&nbsp;'], ['']]
+      ]);
     });
 
     it('should not replace sequences of &nbsp;', () => {
-      expect(parseAndRemoveWS('&nbsp;&nbsp;foo&nbsp;&nbsp;')).toEqual([
+      expect(parseAndRemoveWS('&nbsp;&nbsp;foo&nbsp;&nbsp;')).toEqual([[
-        [html.Text, '\u00a0\u00a0foo\u00a0\u00a0', 0]
+        html.Text,
-      ]);
+        '\u00a0\u00a0foo\u00a0\u00a0',
+        0,
+        [''],
+        ['\u00a0', '&nbsp;'],
+        [''],
+        ['\u00a0', '&nbsp;'],
+        ['foo'],
+        ['\u00a0', '&nbsp;'],
+        [''],
+        ['\u00a0', '&nbsp;'],
+        [''],
+      ]]);
     });
 
     it('should not replace single tab and newline with spaces', () => {
-      expect(parseAndRemoveWS('\nfoo')).toEqual([[html.Text, '\nfoo', 0]]);
+      expect(parseAndRemoveWS('\nfoo')).toEqual([[html.Text, '\nfoo', 0, ['\nfoo']]]);
-      expect(parseAndRemoveWS('\tfoo')).toEqual([[html.Text, '\tfoo', 0]]);
+      expect(parseAndRemoveWS('\tfoo')).toEqual([[html.Text, '\tfoo', 0, ['\tfoo']]]);
     });
 
     it('should preserve single whitespaces between interpolations', () => {
-      expect(parseAndRemoveWS(`{{fooExp}} {{barExp}}`)).toEqual([
+      expect(parseAndRemoveWS(`{{fooExp}} {{barExp}}`)).toEqual([[
-        [html.Text, '{{fooExp}} {{barExp}}', 0],
+        html.Text,
-      ]);
+        '{{fooExp}} {{barExp}}',
+        0,
+        [''],
+        ['{{', 'fooExp', '}}'],
+        [' '],
+        ['{{', 'barExp', '}}'],
+        [''],
+      ]]);
       expect(parseAndRemoveWS(`{{fooExp}}\t{{barExp}}`)).toEqual([
-        [html.Text, '{{fooExp}}\t{{barExp}}', 0],
+        [
+          html.Text,
+          '{{fooExp}}\t{{barExp}}',
+          0,
+          [''],
+          ['{{', 'fooExp', '}}'],
+          ['\t'],
+          ['{{', 'barExp', '}}'],
+          [''],
+        ],
       ]);
       expect(parseAndRemoveWS(`{{fooExp}}\n{{barExp}}`)).toEqual([
-        [html.Text, '{{fooExp}}\n{{barExp}}', 0],
+        [
+          html.Text,
+          '{{fooExp}}\n{{barExp}}',
+          0,
+          [''],
+          ['{{', 'fooExp', '}}'],
+          ['\n'],
+          ['{{', 'barExp', '}}'],
+          [''],
+        ],
       ]);
     });
 
     it('should preserve whitespaces around interpolations', () => {
       expect(parseAndRemoveWS(` {{exp}} `)).toEqual([
-        [html.Text, ' {{exp}} ', 0],
+        [html.Text, ' {{exp}} ', 0, [' '], ['{{', 'exp', '}}'], [' ']]
       ]);
     });
 
@@ -101,10 +140,10 @@ import {humanizeDom} from './ast_spec_utils';
       expect(parseAndRemoveWS(`<span> {a, b, =4 {c}} </span>`, {tokenizeExpansionForms: true}))
           .toEqual([
             [html.Element, 'span', 0],
-            [html.Text, ' ', 1],
+            [html.Text, ' ', 1, [' ']],
             [html.Expansion, 'a', 'b', 1],
             [html.ExpansionCase, '=4', 2],
-            [html.Text, ' ', 1],
+            [html.Text, ' ', 1, [' ']],
           ]);
     });
 
@@ -112,17 +151,17 @@ import {humanizeDom} from './ast_spec_utils';
       expect(parseAndRemoveWS(`<pre><strong>foo</strong>\n<strong>bar</strong></pre>`)).toEqual([
         [html.Element, 'pre', 0],
         [html.Element, 'strong', 1],
-        [html.Text, 'foo', 2],
+        [html.Text, 'foo', 2, ['foo']],
-        [html.Text, '\n', 1],
+        [html.Text, '\n', 1, ['\n']],
         [html.Element, 'strong', 1],
-        [html.Text, 'bar', 2],
+        [html.Text, 'bar', 2, ['bar']],
       ]);
     });
 
     it('should skip whitespace trimming in <textarea>', () => {
       expect(parseAndRemoveWS(`<textarea>foo\n\n  bar</textarea>`)).toEqual([
         [html.Element, 'textarea', 0],
-        [html.Text, 'foo\n\n  bar', 1],
+        [html.Text, 'foo\n\n  bar', 1, ['foo\n\n  bar']],
       ]);
     });
 
@@ -131,7 +170,7 @@ import {humanizeDom} from './ast_spec_utils';
          expect(parseAndRemoveWS(`<div ${PRESERVE_WS_ATTR_NAME}><img> <img></div>`)).toEqual([
            [html.Element, 'div', 0],
            [html.Element, 'img', 1],
-           [html.Text, ' ', 1],
+           [html.Text, ' ', 1, [' ']],
            [html.Element, 'img', 1],
          ]);
        });

packages/compiler/test/ml_parser/icu_ast_expander_spec.ts

@@ -29,9 +29,9 @@ import {humanizeNodes} from './ast_spec_utils';
         [html.Attribute, '[ngPlural]', 'messages.length'],
         [html.Element, 'ng-template', 1],
         [html.Attribute, 'ngPluralCase', '=0'],
-        [html.Text, 'zero', 2],
+        [html.Text, 'zero', 2, ['zero']],
         [html.Element, 'b', 2],
-        [html.Text, 'bold', 3],
+        [html.Text, 'bold', 3, ['bold']],
       ]);
     });
 
@@ -47,8 +47,8 @@ import {humanizeNodes} from './ast_spec_utils';
         [html.Attribute, '[ngSwitch]', 'p.gender'],
         [html.Element, 'ng-template', 3],
         [html.Attribute, 'ngSwitchCase', 'male'],
-        [html.Text, 'm', 4],
+        [html.Text, 'm', 4, ['m']],
-        [html.Text, ' ', 2],
+        [html.Text, ' ', 2, [' ']],
       ]);
     });
 
@@ -88,10 +88,10 @@ import {humanizeNodes} from './ast_spec_utils';
         [html.Attribute, '[ngSwitch]', 'person.gender'],
         [html.Element, 'ng-template', 1],
         [html.Attribute, 'ngSwitchCase', 'male'],
-        [html.Text, 'm', 2],
+        [html.Text, 'm', 2, ['m']],
         [html.Element, 'ng-template', 1],
         [html.Attribute, 'ngSwitchDefault', ''],
-        [html.Text, 'default', 2],
+        [html.Text, 'default', 2, ['default']],
       ]);
     });
 
@@ -105,7 +105,7 @@ import {humanizeNodes} from './ast_spec_utils';
         [html.Attribute, '[ngSwitch]', 'a'],
         [html.Element, 'ng-template', 3],
         [html.Attribute, 'ngSwitchCase', '=4'],
-        [html.Text, 'c', 4],
+        [html.Text, 'c', 4, ['c']],
       ]);
     });