From 8a54896a91512131fbfb0b3d2f5440aec792cbfd Mon Sep 17 00:00:00 2001 From: Pete Bacon Darwin Date: Sat, 15 May 2021 20:37:08 +0100 Subject: [PATCH] refactor(compiler): expose token parts in Text nodes (#42062) When it was tokenized, text content is split into parts that can include interpolations and encoded entities tokens. To make this information available to downstream processing, this commit adds these tokens to the `Text` AST nodes, with suitable processing. PR Close #42062 --- .../compiler/src/i18n/extractor_merger.ts | 4 +- packages/compiler/src/ml_parser/ast.ts | 8 +- .../src/ml_parser/html_whitespaces.ts | 18 +- .../src/ml_parser/icu_ast_expander.ts | 25 +-- packages/compiler/src/ml_parser/parser.ts | 16 +- .../compiler/test/ml_parser/ast_spec_utils.ts | 8 +- .../test/ml_parser/html_parser_spec.ts | 184 +++++++++++------- .../test/ml_parser/html_whitespaces_spec.ts | 87 ++++++--- .../test/ml_parser/icu_ast_expander_spec.ts | 14 +- 9 files changed, 240 insertions(+), 124 deletions(-) diff --git a/packages/compiler/src/i18n/extractor_merger.ts b/packages/compiler/src/i18n/extractor_merger.ts index b0da1c372b..7bb07f9b7b 100644 --- a/packages/compiler/src/i18n/extractor_merger.ts +++ b/packages/compiler/src/i18n/extractor_merger.ts @@ -396,12 +396,12 @@ class _Visitor implements html.Visitor { if (nodes.length == 0) { translatedAttributes.push(new html.Attribute( attr.name, '', attr.sourceSpan, undefined /* keySpan */, undefined /* valueSpan */, - undefined /* i18n */)); + undefined /* valueTokens */, undefined /* i18n */)); } else if (nodes[0] instanceof html.Text) { const value = (nodes[0] as html.Text).value; translatedAttributes.push(new html.Attribute( attr.name, value, attr.sourceSpan, undefined /* keySpan */, - undefined /* valueSpan */, undefined /* i18n */)); + undefined /* valueSpan */, undefined /* valueTokens */, undefined /* i18n */)); } else { this._reportError( el, diff --git a/packages/compiler/src/ml_parser/ast.ts b/packages/compiler/src/ml_parser/ast.ts index 7b71ffe9af..a8abcc80a6 100644 --- a/packages/compiler/src/ml_parser/ast.ts +++ b/packages/compiler/src/ml_parser/ast.ts @@ -9,6 +9,7 @@ import {AstPath} from '../ast_path'; import {I18nMeta} from '../i18n/i18n_ast'; import {ParseSourceSpan} from '../parse_util'; +import {Token} from './lexer'; interface BaseNode { sourceSpan: ParseSourceSpan; @@ -23,7 +24,8 @@ export abstract class NodeWithI18n implements BaseNode { } export class Text extends NodeWithI18n { - constructor(public value: string, sourceSpan: ParseSourceSpan, i18n?: I18nMeta) { + constructor( + public value: string, sourceSpan: ParseSourceSpan, public tokens: Token[], i18n?: I18nMeta) { super(sourceSpan, i18n); } override visit(visitor: Visitor, context: any): any { @@ -55,8 +57,8 @@ export class ExpansionCase implements BaseNode { export class Attribute extends NodeWithI18n { constructor( public name: string, public value: string, sourceSpan: ParseSourceSpan, - readonly keySpan: ParseSourceSpan|undefined, public valueSpan?: ParseSourceSpan, - i18n?: I18nMeta) { + readonly keySpan: ParseSourceSpan|undefined, public valueSpan: ParseSourceSpan|undefined, + public valueTokens: Token[]|undefined, i18n: I18nMeta|undefined) { super(sourceSpan, i18n); } override visit(visitor: Visitor, context: any): any { diff --git a/packages/compiler/src/ml_parser/html_whitespaces.ts b/packages/compiler/src/ml_parser/html_whitespaces.ts index f416523476..57045ee8dc 100644 --- a/packages/compiler/src/ml_parser/html_whitespaces.ts +++ b/packages/compiler/src/ml_parser/html_whitespaces.ts @@ -8,6 +8,7 @@ import * as html from './ast'; import {NGSP_UNICODE} from './entities'; +import {Token, TokenType} from './lexer'; import {ParseTreeResult} from './parser'; export const PRESERVE_WS_ATTR_NAME = 'ngPreserveWhitespaces'; @@ -74,8 +75,13 @@ export class WhitespaceVisitor implements html.Visitor { (context.prev instanceof html.Expansion || context.next instanceof html.Expansion); if (isNotBlank || hasExpansionSibling) { - return new html.Text( - replaceNgsp(text.value).replace(WS_REPLACE_REGEXP, ' '), text.sourceSpan, text.i18n); + // Process the whitespace in the tokens of this Text node + const tokens = text.tokens.map( + token => token.type === TokenType.TEXT ? createTextTokenAfterWhitespaceProcessing(token) : + token); + // Process the whitespace of the value of this Text node + const value = processWhitespace(text.value); + return new html.Text(value, text.sourceSpan, tokens, text.i18n); } return null; @@ -94,6 +100,14 @@ export class WhitespaceVisitor implements html.Visitor { } } +function createTextTokenAfterWhitespaceProcessing(token: Token): Token { + return new Token(token.type, [processWhitespace(token.parts[0])], token.sourceSpan); +} + +function processWhitespace(text: string): string { + return replaceNgsp(text).replace(WS_REPLACE_REGEXP, ' '); +} + export function removeWhitespaces(htmlAstWithErrors: ParseTreeResult): ParseTreeResult { return new ParseTreeResult( html.visitAll(new WhitespaceVisitor(), htmlAstWithErrors.rootNodes), diff --git a/packages/compiler/src/ml_parser/icu_ast_expander.ts b/packages/compiler/src/ml_parser/icu_ast_expander.ts index 9e08c954dd..f52affd590 100644 --- a/packages/compiler/src/ml_parser/icu_ast_expander.ts +++ b/packages/compiler/src/ml_parser/icu_ast_expander.ts @@ -102,14 +102,15 @@ function _expandPluralForm(ast: html.Expansion, errors: ParseError[]): html.Elem errors.push(...expansionResult.errors); return new html.Element( - `ng-template`, [new html.Attribute( - 'ngPluralCase', `${c.value}`, c.valueSourceSpan, undefined /* keySpan */, - undefined /* valueSpan */, undefined /* i18n */)], + `ng-template`, + [new html.Attribute( + 'ngPluralCase', `${c.value}`, c.valueSourceSpan, undefined /* keySpan */, + undefined /* valueSpan */, undefined /* valueTokens */, undefined /* i18n */)], expansionResult.nodes, c.sourceSpan, c.sourceSpan, c.sourceSpan); }); const switchAttr = new html.Attribute( '[ngPlural]', ast.switchValue, ast.switchValueSourceSpan, undefined /* keySpan */, - undefined /* valueSpan */, undefined /* i18n */); + undefined /* valueSpan */, undefined /* valueTokens */, undefined /* i18n */); return new html.Element( 'ng-container', [switchAttr], children, ast.sourceSpan, ast.sourceSpan, ast.sourceSpan); } @@ -123,21 +124,23 @@ function _expandDefaultForm(ast: html.Expansion, errors: ParseError[]): html.Ele if (c.value === 'other') { // other is the default case when no values match return new html.Element( - `ng-template`, [new html.Attribute( - 'ngSwitchDefault', '', c.valueSourceSpan, undefined /* keySpan */, - undefined /* valueSpan */, undefined /* i18n */)], + `ng-template`, + [new html.Attribute( + 'ngSwitchDefault', '', c.valueSourceSpan, undefined /* keySpan */, + undefined /* valueSpan */, undefined /* valueTokens */, undefined /* i18n */)], expansionResult.nodes, c.sourceSpan, c.sourceSpan, c.sourceSpan); } return new html.Element( - `ng-template`, [new html.Attribute( - 'ngSwitchCase', `${c.value}`, c.valueSourceSpan, undefined /* keySpan */, - undefined /* valueSpan */, undefined /* i18n */)], + `ng-template`, + [new html.Attribute( + 'ngSwitchCase', `${c.value}`, c.valueSourceSpan, undefined /* keySpan */, + undefined /* valueSpan */, undefined /* valueTokens */, undefined /* i18n */)], expansionResult.nodes, c.sourceSpan, c.sourceSpan, c.sourceSpan); }); const switchAttr = new html.Attribute( '[ngSwitch]', ast.switchValue, ast.switchValueSourceSpan, undefined /* keySpan */, - undefined /* valueSpan */, undefined /* i18n */); + undefined /* valueSpan */, undefined /* valueTokens */, undefined /* i18n */); return new html.Element( 'ng-container', [switchAttr], children, ast.sourceSpan, ast.sourceSpan, ast.sourceSpan); } diff --git a/packages/compiler/src/ml_parser/parser.ts b/packages/compiler/src/ml_parser/parser.ts index 4d5f18d440..d9f5d17bd3 100644 --- a/packages/compiler/src/ml_parser/parser.ts +++ b/packages/compiler/src/ml_parser/parser.ts @@ -216,6 +216,7 @@ class _TreeBuilder { } private _consumeText(token: lex.Token) { + const tokens = [token]; const startSpan = token.sourceSpan; let text = token.parts[0]; if (text.length > 0 && text[0] == '\n') { @@ -223,14 +224,15 @@ class _TreeBuilder { if (parent != null && parent.children.length == 0 && this.getTagDefinition(parent.name).ignoreFirstLf) { text = text.substring(1); + tokens[0] = {type: token.type, sourceSpan: token.sourceSpan, parts: [text]}; } } - // For now recombine text, interpolation and entity tokens while (this._peek.type === lex.TokenType.INTERPOLATION || this._peek.type === lex.TokenType.TEXT || this._peek.type === lex.TokenType.ENCODED_ENTITY) { token = this._advance(); + tokens.push(token); if (token.type === lex.TokenType.INTERPOLATION) { // For backward compatibility we decode HTML entities that appear in interpolation // expressions. This is arguably a bug, but it could be a considerable breaking change to @@ -248,8 +250,8 @@ class _TreeBuilder { const endSpan = token.sourceSpan; this._addToParent(new html.Text( text, - new ParseSourceSpan( - startSpan.start, endSpan.end, startSpan.fullStart, startSpan.details))); + new ParseSourceSpan(startSpan.start, endSpan.end, startSpan.fullStart, startSpan.details), + tokens)); } } @@ -372,16 +374,17 @@ class _TreeBuilder { // Consume the attribute value let value = ''; + const valueTokens: lex.Token[] = []; let valueStartSpan: ParseSourceSpan|undefined = undefined; let valueEnd: ParseLocation|undefined = undefined; if (this._peek.type === lex.TokenType.ATTR_VALUE_TEXT) { valueStartSpan = this._peek.sourceSpan; valueEnd = this._peek.sourceSpan.end; - // For now recombine text, interpolation and entity tokens while (this._peek.type === lex.TokenType.ATTR_VALUE_TEXT || this._peek.type === lex.TokenType.ATTR_VALUE_INTERPOLATION || this._peek.type === lex.TokenType.ENCODED_ENTITY) { - let valueToken = this._advance(); + const valueToken = this._advance(); + valueTokens.push(valueToken); if (valueToken.type === lex.TokenType.ATTR_VALUE_INTERPOLATION) { // For backward compatibility we decode HTML entities that appear in interpolation // expressions. This is arguably a bug, but it could be a considerable breaking change to @@ -408,7 +411,8 @@ class _TreeBuilder { return new html.Attribute( fullName, value, new ParseSourceSpan(attrName.sourceSpan.start, attrEnd, attrName.sourceSpan.fullStart), - attrName.sourceSpan, valueSpan); + attrName.sourceSpan, valueSpan, valueTokens.length > 0 ? valueTokens : undefined, + undefined); } private _getParentElement(): html.Element|null { diff --git a/packages/compiler/test/ml_parser/ast_spec_utils.ts b/packages/compiler/test/ml_parser/ast_spec_utils.ts index a6f7ca65df..818ae726ce 100644 --- a/packages/compiler/test/ml_parser/ast_spec_utils.ts +++ b/packages/compiler/test/ml_parser/ast_spec_utils.ts @@ -52,12 +52,16 @@ class _Humanizer implements html.Visitor { } visitAttribute(attribute: html.Attribute, context: any): any { - const res = this._appendContext(attribute, [html.Attribute, attribute.name, attribute.value]); + const valueTokens = attribute.valueTokens ?? []; + const res = this._appendContext(attribute, [ + html.Attribute, attribute.name, attribute.value, ...valueTokens.map(token => token.parts) + ]); this.result.push(res); } visitText(text: html.Text, context: any): any { - const res = this._appendContext(text, [html.Text, text.value, this.elDepth]); + const res = this._appendContext( + text, [html.Text, text.value, this.elDepth, ...text.tokens.map(token => token.parts)]); this.result.push(res); } diff --git a/packages/compiler/test/ml_parser/html_parser_spec.ts b/packages/compiler/test/ml_parser/html_parser_spec.ts index 4fbfa75692..57171d5f40 100644 --- a/packages/compiler/test/ml_parser/html_parser_spec.ts +++ b/packages/compiler/test/ml_parser/html_parser_spec.ts @@ -24,31 +24,31 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes} describe('parse', () => { describe('text nodes', () => { it('should parse root level text nodes', () => { - expect(humanizeDom(parser.parse('a', 'TestComp'))).toEqual([[html.Text, 'a', 0]]); + expect(humanizeDom(parser.parse('a', 'TestComp'))).toEqual([[html.Text, 'a', 0, ['a']]]); }); it('should parse text nodes inside regular elements', () => { expect(humanizeDom(parser.parse('
a
', 'TestComp'))).toEqual([ - [html.Element, 'div', 0], [html.Text, 'a', 1] + [html.Element, 'div', 0], [html.Text, 'a', 1, ['a']] ]); }); it('should parse text nodes inside elements', () => { expect(humanizeDom(parser.parse('a', 'TestComp'))).toEqual([ - [html.Element, 'ng-template', 0], [html.Text, 'a', 1] + [html.Element, 'ng-template', 0], [html.Text, 'a', 1, ['a']] ]); }); it('should parse CDATA', () => { expect(humanizeDom(parser.parse('', 'TestComp'))).toEqual([ - [html.Text, 'text', 0] + [html.Text, 'text', 0, ['text']] ]); }); it('should normalize line endings within CDATA', () => { const parsed = parser.parse('', 'TestComp'); expect(humanizeDom(parsed)).toEqual([ - [html.Text, ' line 1 \n line 2 ', 0], + [html.Text, ' line 1 \n line 2 ', 0, [' line 1 \n line 2 ']], ]); expect(parsed.errors).toEqual([]); }); @@ -76,8 +76,8 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes} expect(humanizeDom(parser.parse('', 'TestComp'))) .toEqual([ [html.Element, 'link', 0], - [html.Attribute, 'rel', 'author license'], - [html.Attribute, 'href', '/about'], + [html.Attribute, 'rel', 'author license', ['author license']], + [html.Attribute, 'href', '/about', ['/about']], ]); }); @@ -106,9 +106,9 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes} it('should close void elements on text nodes', () => { expect(humanizeDom(parser.parse('

before
after

', 'TestComp'))).toEqual([ [html.Element, 'p', 0], - [html.Text, 'before', 1], + [html.Text, 'before', 1, ['before']], [html.Element, 'br', 1], - [html.Text, 'after', 1], + [html.Text, 'after', 1, ['after']], ]); }); @@ -116,9 +116,9 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes} expect(humanizeDom(parser.parse('

1

2

', 'TestComp'))).toEqual([ [html.Element, 'div', 0], [html.Element, 'p', 1], - [html.Text, '1', 2], + [html.Text, '1', 2, ['1']], [html.Element, 'p', 1], - [html.Text, '2', 2], + [html.Text, '2', 2, ['2']], ]); }); @@ -200,12 +200,12 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes} 'TestComp'))) .toEqual([ [html.Element, 'p', 0], - [html.Text, '\n', 1], + [html.Text, '\n', 1, ['\n']], [html.Element, 'textarea', 0], [html.Element, 'pre', 0], - [html.Text, '\n', 1], + [html.Text, '\n', 1, ['\n']], [html.Element, 'listing', 0], - [html.Text, '\n', 1], + [html.Text, '\n', 1, ['\n']], ]); }); @@ -214,28 +214,28 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes} parsed = parser.parse(' line 1 \r\n line 2 ', 'TestComp'); expect(humanizeDom(parsed)).toEqual([ [html.Element, 'title', 0], - [html.Text, ' line 1 \n line 2 ', 1], + [html.Text, ' line 1 \n line 2 ', 1, [' line 1 \n line 2 ']], ]); expect(parsed.errors).toEqual([]); parsed = parser.parse('', 'TestComp'); expect(humanizeDom(parsed)).toEqual([ [html.Element, 'script', 0], - [html.Text, ' line 1 \n line 2 ', 1], + [html.Text, ' line 1 \n line 2 ', 1, [' line 1 \n line 2 ']], ]); expect(parsed.errors).toEqual([]); parsed = parser.parse('
line 1 \r\n line 2
', 'TestComp'); expect(humanizeDom(parsed)).toEqual([ [html.Element, 'div', 0], - [html.Text, ' line 1 \n line 2 ', 1], + [html.Text, ' line 1 \n line 2 ', 1, [' line 1 \n line 2 ']], ]); expect(parsed.errors).toEqual([]); parsed = parser.parse(' line 1 \r\n line 2 ', 'TestComp'); expect(humanizeDom(parsed)).toEqual([ [html.Element, 'span', 0], - [html.Text, ' line 1 \n line 2 ', 1], + [html.Text, ' line 1 \n line 2 ', 1, [' line 1 \n line 2 ']], ]); expect(parsed.errors).toEqual([]); }); @@ -245,8 +245,22 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes} it('should parse attributes on regular elements case sensitive', () => { expect(humanizeDom(parser.parse('
', 'TestComp'))).toEqual([ [html.Element, 'div', 0], - [html.Attribute, 'kEy', 'v'], - [html.Attribute, 'key2', 'v2'], + [html.Attribute, 'kEy', 'v', ['v']], + [html.Attribute, 'key2', 'v2', ['v2']], + ]); + }); + + it('should parse attributes containing interpolation', () => { + expect(humanizeDom(parser.parse('
', 'TestComp'))).toEqual([ + [html.Element, 'div', 0], + [html.Attribute, 'foo', '1{{message}}2', ['1'], ['{{', 'message', '}}'], ['2']] + ]); + }); + + it('should parse attributes containing encoded entities', () => { + expect(humanizeDom(parser.parse('
', 'TestComp'))).toEqual([ + [html.Element, 'div', 0], + [html.Attribute, 'foo', '&', [''], ['&', '&'], ['']], ]); }); @@ -259,7 +273,10 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes} html.Element, 'div', 0, '
', '
', '
' ], - [html.Attribute, 'foo', '{{&}}', 'foo="{{&}}"'] + [ + html.Attribute, 'foo', '{{&}}', [''], ['{{', '&', '}}'], [''], + 'foo="{{&}}"' + ] ]); }); @@ -268,7 +285,7 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes} parser.parse('
', 'TestComp'); expect(humanizeDom(result)).toEqual([ [html.Element, 'div', 0], - [html.Attribute, 'key', ' \n line 1 \n line 2 '], + [html.Attribute, 'key', ' \n line 1 \n line 2 ', [' \n line 1 \n line 2 ']], ]); expect(result.errors).toEqual([]); }); @@ -283,7 +300,7 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes} it('should parse attributes on svg elements case sensitive', () => { expect(humanizeDom(parser.parse('', 'TestComp'))).toEqual([ [html.Element, ':svg:svg', 0], - [html.Attribute, 'viewBox', '0'], + [html.Attribute, 'viewBox', '0', ['0']], ]); }); @@ -291,14 +308,14 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes} expect(humanizeDom(parser.parse('', 'TestComp'))) .toEqual([ [html.Element, 'ng-template', 0], - [html.Attribute, 'k', 'v'], + [html.Attribute, 'k', 'v', ['v']], ]); }); it('should support namespace', () => { expect(humanizeDom(parser.parse('', 'TestComp'))).toEqual([ [html.Element, ':svg:use', 0], - [html.Attribute, ':xlink:href', 'Port'], + [html.Attribute, ':xlink:href', 'Port', ['Port']], ]); }); }); @@ -325,23 +342,23 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes} expect(humanizeDom(parsed)).toEqual([ [html.Element, 'div', 0], - [html.Text, 'before', 1], + [html.Text, 'before', 1, ['before']], [html.Expansion, 'messages.length', 'plural', 1], [html.ExpansionCase, '=0', 2], [html.ExpansionCase, '=1', 2], - [html.Text, 'after', 1], + [html.Text, 'after', 1, ['after']], ]); const cases = (parsed.rootNodes[0]).children[1].cases; expect(humanizeDom(new ParseTreeResult(cases[0].expression, []))).toEqual([ - [html.Text, 'You have ', 0], + [html.Text, 'You have ', 0, ['You have ']], [html.Element, 'b', 0], - [html.Text, 'no', 1], - [html.Text, ' messages', 0], + [html.Text, 'no', 1, ['no']], + [html.Text, ' messages', 0, [' messages']], ]); expect(humanizeDom(new ParseTreeResult(cases[1].expression, []))).toEqual([ - [html.Text, 'One {{message}}', 0] + [html.Text, 'One {{message}}', 0, ['One '], ['{{', 'message', '}}'], ['']] ]); }); @@ -363,20 +380,20 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes} expect(humanizeDom(parsed)).toEqual([ [html.Element, 'div', 0], - [html.Text, '\n ', 1], + [html.Text, '\n ', 1, ['\n ']], [html.Expansion, '\n messages.length', 'plural', 1], [html.ExpansionCase, '=0', 2], [html.ExpansionCase, '=1', 2], - [html.Text, '\n', 1], + [html.Text, '\n', 1, ['\n']], ]); const cases = (parsed.rootNodes[0]).children[1].cases; expect(humanizeDom(new ParseTreeResult(cases[0].expression, []))).toEqual([ - [html.Text, 'You have \nno\n messages', 0], + [html.Text, 'You have \nno\n messages', 0, ['You have \nno\n messages']], ]); expect(humanizeDom(new ParseTreeResult(cases[1].expression, []))).toEqual([ - [html.Text, 'One {{message}}', 0] + [html.Text, 'One {{message}}', 0, ['One '], ['{{', 'message', '}}'], ['']] ]); expect(parsed.errors).toEqual([]); @@ -396,20 +413,20 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes} expect(humanizeDom(parsed)).toEqual([ [html.Element, 'div', 0], - [html.Text, '\n ', 1], + [html.Text, '\n ', 1, ['\n ']], [html.Expansion, '\r\n messages.length', 'plural', 1], [html.ExpansionCase, '=0', 2], [html.ExpansionCase, '=1', 2], - [html.Text, '\n', 1], + [html.Text, '\n', 1, ['\n']], ]); const cases = (parsed.rootNodes[0]).children[1].cases; expect(humanizeDom(new ParseTreeResult(cases[0].expression, []))).toEqual([ - [html.Text, 'You have \nno\n messages', 0], + [html.Text, 'You have \nno\n messages', 0, ['You have \nno\n messages']], ]); expect(humanizeDom(new ParseTreeResult(cases[1].expression, []))).toEqual([ - [html.Text, 'One {{message}}', 0] + [html.Text, 'One {{message}}', 0, ['One '], ['{{', 'message', '}}'], ['']] ]); expect(parsed.errors).toEqual([]); @@ -433,20 +450,20 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes} expect(humanizeDom(parsed)).toEqual([ [html.Element, 'div', 0], - [html.Text, '\n ', 1], + [html.Text, '\n ', 1, ['\n ']], [html.Expansion, '\n messages.length', 'plural', 1], [html.ExpansionCase, '=0', 2], [html.ExpansionCase, '=1', 2], - [html.Text, '\n', 1], + [html.Text, '\n', 1, ['\n']], ]); const cases = (parsed.rootNodes[0]).children[1].cases; expect(humanizeDom(new ParseTreeResult(cases[0].expression, []))).toEqual([ - [html.Text, 'You have \nno\n messages', 0], + [html.Text, 'You have \nno\n messages', 0, ['You have \nno\n messages']], ]); expect(humanizeDom(new ParseTreeResult(cases[1].expression, []))).toEqual([ - [html.Text, 'One {{message}}', 0] + [html.Text, 'One {{message}}', 0, ['One '], ['{{', 'message', '}}'], ['']] ]); expect(parsed.errors).toEqual([]); @@ -466,20 +483,20 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes} expect(humanizeDom(parsed)).toEqual([ [html.Element, 'div', 0], - [html.Text, '\n ', 1], + [html.Text, '\n ', 1, ['\n ']], [html.Expansion, '\r\n messages.length', 'plural', 1], [html.ExpansionCase, '=0', 2], [html.ExpansionCase, '=1', 2], - [html.Text, '\n', 1], + [html.Text, '\n', 1, ['\n']], ]); const cases = (parsed.rootNodes[0]).children[1].cases; expect(humanizeDom(new ParseTreeResult(cases[0].expression, []))).toEqual([ - [html.Text, 'You have \nno\n messages', 0], + [html.Text, 'You have \nno\n messages', 0, ['You have \nno\n messages']], ]); expect(humanizeDom(new ParseTreeResult(cases[1].expression, []))).toEqual([ - [html.Text, 'One {{message}}', 0] + [html.Text, 'One {{message}}', 0, ['One '], ['{{', 'message', '}}'], ['']] ]); expect(parsed.errors).toEqual([]); @@ -512,7 +529,7 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes} expect(humanizeDom(new ParseTreeResult(firstCase.expression, []))).toEqual([ [html.Expansion, 'p.gender', 'select', 0], [html.ExpansionCase, 'male', 1], - [html.Text, ' ', 0], + [html.Text, ' ', 0, [' ']], ]); }); @@ -540,10 +557,10 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes} const expansion = parsed.rootNodes[0] as html.Expansion; expect(humanizeDom(new ParseTreeResult(expansion.cases[0].expression, []))).toEqual([ - [html.Text, 'zero \n ', 0], + [html.Text, 'zero \n ', 0, ['zero \n ']], [html.Expansion, '\n p.gender', 'select', 0], [html.ExpansionCase, 'male', 1], - [html.Text, '\n ', 0], + [html.Text, '\n ', 0, ['\n ']], ]); expect(parsed.errors).toEqual([]); @@ -569,10 +586,10 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes} const expansion = parsed.rootNodes[0] as html.Expansion; expect(humanizeDom(new ParseTreeResult(expansion.cases[0].expression, []))).toEqual([ - [html.Text, 'zero \n ', 0], + [html.Text, 'zero \n ', 0, ['zero \n ']], [html.Expansion, '\r\n p.gender', 'select', 0], [html.ExpansionCase, 'male', 1], - [html.Text, '\n ', 0], + [html.Text, '\n ', 0, ['\n ']], ]); expect(parsed.errors).toEqual([]); @@ -598,10 +615,10 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes} const expansion = parsed.rootNodes[0] as html.Expansion; expect(humanizeDom(new ParseTreeResult(expansion.cases[0].expression, []))).toEqual([ - [html.Text, 'zero \n ', 0], + [html.Text, 'zero \n ', 0, ['zero \n ']], [html.Expansion, '\r\n p.gender', 'select', 0], [html.ExpansionCase, 'male', 1], - [html.Text, '\n ', 0], + [html.Text, '\n ', 0, ['\n ']], ]); @@ -670,11 +687,11 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes} '
\na\n
', '
', '
' ], - [html.Attribute, '[prop]', 'v1', '[prop]="v1"'], - [html.Attribute, '(e)', 'do()', '(e)="do()"'], - [html.Attribute, 'attr', 'v2', 'attr="v2"'], + [html.Attribute, '[prop]', 'v1', ['v1'], '[prop]="v1"'], + [html.Attribute, '(e)', 'do()', ['do()'], '(e)="do()"'], + [html.Attribute, 'attr', 'v2', ['v2'], 'attr="v2"'], [html.Attribute, 'noValue', '', 'noValue'], - [html.Text, '\na\n', 1, '\na\n'], + [html.Text, '\na\n', 1, ['\na\n'], '\na\n'], ]); }); @@ -695,7 +712,9 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes} '{{&}}' + '{{▾}}' + '{{▾}}' + + '{{&unknown;}}' + '{{& (no semi-colon)}}' + + '{{&#xyz; (invalid hex)}}' + '{{BE; (invalid decimal)}}', 'TestComp'))) .toEqual([[ @@ -703,17 +722,48 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes} '{{&}}' + '{{\u25BE}}' + '{{\u25BE}}' + + '{{&unknown;}}' + '{{& (no semi-colon)}}' + + '{{&#xyz; (invalid hex)}}' + '{{BE; (invalid decimal)}}', 0, + [''], + ['{{', '&', '}}'], + [''], + ['{{', '▾', '}}'], + [''], + ['{{', '▾', '}}'], + [''], + ['{{', '&unknown;', '}}'], + [''], + ['{{', '& (no semi-colon)', '}}'], + [''], + ['{{', '&#xyz; (invalid hex)', '}}'], + [''], + ['{{', 'BE; (invalid decimal)', '}}'], + [''], '{{&}}' + '{{▾}}' + '{{▾}}' + + '{{&unknown;}}' + '{{& (no semi-colon)}}' + + '{{&#xyz; (invalid hex)}}' + '{{BE; (invalid decimal)}}', ]]); }); + it('should support interpolations in text', () => { + expect( + humanizeDomSourceSpans(parser.parse('
pre {{ value }} post
', 'TestComp'))) + .toEqual([ + [html.Element, 'div', 0, '
pre {{ value }} post
', '
', '
'], + [ + html.Text, ' pre {{ value }} post ', 1, [' pre '], ['{{', ' value ', '}}'], + [' post '], ' pre {{ value }} post ' + ], + ]); + }); + it('should not set the end source span for void elements', () => { expect(humanizeDomSourceSpans(parser.parse('

', 'TestComp'))).toEqual([ [html.Element, 'div', 0, '

', '
', '
'], @@ -758,10 +808,10 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes} html.Element, 'input', 0, '', '', '' ], - [html.Attribute, 'type', 'text', 'type="text"'], - [html.Text, '\n\n\n ', 0, ''], + [html.Attribute, 'type', 'text', ['text'], 'type="text"'], + [html.Text, '\n\n\n ', 0, ['\n\n\n '], ''], [html.Element, 'span', 0, '\n', '', ''], - [html.Text, '\n', 1, ''], + [html.Text, '\n', 1, ['\n'], ''], ]); }); @@ -774,9 +824,9 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes} .toEqual([ [html.Element, 'div', 0, '
  • A
  • B
    • ', '
    ', '
    '], [html.Element, 'li', 1, '
  • ', '
  • ', null], - [html.Text, 'A', 2, 'A'], + [html.Text, 'A', 2, ['A'], 'A'], [html.Element, 'li', 1, '
  • ', '
  • ', null], - [html.Text, 'B', 2, 'B'], + [html.Text, 'B', 2, ['B'], 'B'], ]); }); @@ -814,7 +864,7 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes} describe('visitor', () => { it('should visit text nodes', () => { const result = humanizeDom(parser.parse('text', 'TestComp')); - expect(result).toEqual([[html.Text, 'text', 0]]); + expect(result).toEqual([[html.Text, 'text', 0, ['text']]]); }); it('should visit element nodes', () => { @@ -824,7 +874,7 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes} it('should visit attribute nodes', () => { const result = humanizeDom(parser.parse('
    ', 'TestComp')); - expect(result).toContain([html.Attribute, 'id', 'foo']); + expect(result).toContain([html.Attribute, 'id', 'foo', ['foo']]); }); it('should visit all nodes', () => { @@ -932,7 +982,7 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes} expect(humanizeNodes(rootNodes, true)).toEqual([ [html.Element, 'div', 0, '
    ', '', ''], ]); @@ -947,7 +997,7 @@ import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes} expect(humanizeNodes(rootNodes, true)).toEqual([ [html.Element, 'div', 0, '
    ', '', ''], ]); diff --git a/packages/compiler/test/ml_parser/html_whitespaces_spec.ts b/packages/compiler/test/ml_parser/html_whitespaces_spec.ts index 117be970d1..4d1062a9c8 100644 --- a/packages/compiler/test/ml_parser/html_whitespaces_spec.ts +++ b/packages/compiler/test/ml_parser/html_whitespaces_spec.ts @@ -7,6 +7,7 @@ */ import * as html from '../../src/ml_parser/ast'; +import {NGSP_UNICODE} from '../../src/ml_parser/entities'; import {HtmlParser} from '../../src/ml_parser/html_parser'; import {PRESERVE_WS_ATTR_NAME, removeWhitespaces} from '../../src/ml_parser/html_whitespaces'; import {TokenizeOptions} from '../../src/ml_parser/lexer'; @@ -52,48 +53,86 @@ import {humanizeDom} from './ast_spec_utils'; expect(parseAndRemoveWS('
    foo&ngsp;bar
    ')).toEqual([ [html.Element, 'div', 0], [html.Element, 'span', 1], - [html.Text, 'foo', 2], - [html.Text, ' ', 1], + [html.Text, 'foo', 2, ['foo']], + [html.Text, ' ', 1, [''], [NGSP_UNICODE, '&ngsp;'], ['']], [html.Element, 'span', 1], - [html.Text, 'bar', 2], + [html.Text, 'bar', 2, ['bar']], ]); }); it('should replace multiple whitespaces with one space', () => { - expect(parseAndRemoveWS('\n\n\nfoo\t\t\t')).toEqual([[html.Text, ' foo ', 0]]); - expect(parseAndRemoveWS(' \n foo \t ')).toEqual([[html.Text, ' foo ', 0]]); + expect(parseAndRemoveWS('\n\n\nfoo\t\t\t')).toEqual([[html.Text, ' foo ', 0, [' foo ']]]); + expect(parseAndRemoveWS(' \n foo \t ')).toEqual([[html.Text, ' foo ', 0, [' foo ']]]); }); it('should not replace  ', () => { - expect(parseAndRemoveWS(' ')).toEqual([[html.Text, '\u00a0', 0]]); + expect(parseAndRemoveWS(' ')).toEqual([ + [html.Text, '\u00a0', 0, [''], ['\u00a0', ' '], ['']] + ]); }); it('should not replace sequences of  ', () => { - expect(parseAndRemoveWS('  foo  ')).toEqual([ - [html.Text, '\u00a0\u00a0foo\u00a0\u00a0', 0] - ]); + expect(parseAndRemoveWS('  foo  ')).toEqual([[ + html.Text, + '\u00a0\u00a0foo\u00a0\u00a0', + 0, + [''], + ['\u00a0', ' '], + [''], + ['\u00a0', ' '], + ['foo'], + ['\u00a0', ' '], + [''], + ['\u00a0', ' '], + [''], + ]]); }); it('should not replace single tab and newline with spaces', () => { - expect(parseAndRemoveWS('\nfoo')).toEqual([[html.Text, '\nfoo', 0]]); - expect(parseAndRemoveWS('\tfoo')).toEqual([[html.Text, '\tfoo', 0]]); + expect(parseAndRemoveWS('\nfoo')).toEqual([[html.Text, '\nfoo', 0, ['\nfoo']]]); + expect(parseAndRemoveWS('\tfoo')).toEqual([[html.Text, '\tfoo', 0, ['\tfoo']]]); }); it('should preserve single whitespaces between interpolations', () => { - expect(parseAndRemoveWS(`{{fooExp}} {{barExp}}`)).toEqual([ - [html.Text, '{{fooExp}} {{barExp}}', 0], - ]); + expect(parseAndRemoveWS(`{{fooExp}} {{barExp}}`)).toEqual([[ + html.Text, + '{{fooExp}} {{barExp}}', + 0, + [''], + ['{{', 'fooExp', '}}'], + [' '], + ['{{', 'barExp', '}}'], + [''], + ]]); expect(parseAndRemoveWS(`{{fooExp}}\t{{barExp}}`)).toEqual([ - [html.Text, '{{fooExp}}\t{{barExp}}', 0], + [ + html.Text, + '{{fooExp}}\t{{barExp}}', + 0, + [''], + ['{{', 'fooExp', '}}'], + ['\t'], + ['{{', 'barExp', '}}'], + [''], + ], ]); expect(parseAndRemoveWS(`{{fooExp}}\n{{barExp}}`)).toEqual([ - [html.Text, '{{fooExp}}\n{{barExp}}', 0], + [ + html.Text, + '{{fooExp}}\n{{barExp}}', + 0, + [''], + ['{{', 'fooExp', '}}'], + ['\n'], + ['{{', 'barExp', '}}'], + [''], + ], ]); }); it('should preserve whitespaces around interpolations', () => { expect(parseAndRemoveWS(` {{exp}} `)).toEqual([ - [html.Text, ' {{exp}} ', 0], + [html.Text, ' {{exp}} ', 0, [' '], ['{{', 'exp', '}}'], [' ']] ]); }); @@ -101,10 +140,10 @@ import {humanizeDom} from './ast_spec_utils'; expect(parseAndRemoveWS(` {a, b, =4 {c}} `, {tokenizeExpansionForms: true})) .toEqual([ [html.Element, 'span', 0], - [html.Text, ' ', 1], + [html.Text, ' ', 1, [' ']], [html.Expansion, 'a', 'b', 1], [html.ExpansionCase, '=4', 2], - [html.Text, ' ', 1], + [html.Text, ' ', 1, [' ']], ]); }); @@ -112,17 +151,17 @@ import {humanizeDom} from './ast_spec_utils'; expect(parseAndRemoveWS(`
    foo\nbar
    `)).toEqual([ [html.Element, 'pre', 0], [html.Element, 'strong', 1], - [html.Text, 'foo', 2], - [html.Text, '\n', 1], + [html.Text, 'foo', 2, ['foo']], + [html.Text, '\n', 1, ['\n']], [html.Element, 'strong', 1], - [html.Text, 'bar', 2], + [html.Text, 'bar', 2, ['bar']], ]); }); it('should skip whitespace trimming in `)).toEqual([ [html.Element, 'textarea', 0], - [html.Text, 'foo\n\n bar', 1], + [html.Text, 'foo\n\n bar', 1, ['foo\n\n bar']], ]); }); @@ -131,7 +170,7 @@ import {humanizeDom} from './ast_spec_utils'; expect(parseAndRemoveWS(`
    `)).toEqual([ [html.Element, 'div', 0], [html.Element, 'img', 1], - [html.Text, ' ', 1], + [html.Text, ' ', 1, [' ']], [html.Element, 'img', 1], ]); }); diff --git a/packages/compiler/test/ml_parser/icu_ast_expander_spec.ts b/packages/compiler/test/ml_parser/icu_ast_expander_spec.ts index d9e8ed2407..cbd51c1c6c 100644 --- a/packages/compiler/test/ml_parser/icu_ast_expander_spec.ts +++ b/packages/compiler/test/ml_parser/icu_ast_expander_spec.ts @@ -29,9 +29,9 @@ import {humanizeNodes} from './ast_spec_utils'; [html.Attribute, '[ngPlural]', 'messages.length'], [html.Element, 'ng-template', 1], [html.Attribute, 'ngPluralCase', '=0'], - [html.Text, 'zero', 2], + [html.Text, 'zero', 2, ['zero']], [html.Element, 'b', 2], - [html.Text, 'bold', 3], + [html.Text, 'bold', 3, ['bold']], ]); }); @@ -47,8 +47,8 @@ import {humanizeNodes} from './ast_spec_utils'; [html.Attribute, '[ngSwitch]', 'p.gender'], [html.Element, 'ng-template', 3], [html.Attribute, 'ngSwitchCase', 'male'], - [html.Text, 'm', 4], - [html.Text, ' ', 2], + [html.Text, 'm', 4, ['m']], + [html.Text, ' ', 2, [' ']], ]); }); @@ -88,10 +88,10 @@ import {humanizeNodes} from './ast_spec_utils'; [html.Attribute, '[ngSwitch]', 'person.gender'], [html.Element, 'ng-template', 1], [html.Attribute, 'ngSwitchCase', 'male'], - [html.Text, 'm', 2], + [html.Text, 'm', 2, ['m']], [html.Element, 'ng-template', 1], [html.Attribute, 'ngSwitchDefault', ''], - [html.Text, 'default', 2], + [html.Text, 'default', 2, ['default']], ]); }); @@ -105,7 +105,7 @@ import {humanizeNodes} from './ast_spec_utils'; [html.Attribute, '[ngSwitch]', 'a'], [html.Element, 'ng-template', 3], [html.Attribute, 'ngSwitchCase', '=4'], - [html.Text, 'c', 4], + [html.Text, 'c', 4, ['c']], ]); });