From 9b3d4f5575bfccfbbfb943dc9689f3915dd63752 Mon Sep 17 00:00:00 2001 From: Pete Bacon Darwin Date: Sat, 12 Jun 2021 20:56:05 +0100 Subject: [PATCH] refactor(compiler): define interfaces for each lexer token (#42062) These token interfaces will make it easier to reason about tokens in the parser and in specs. Previously, it was never clear what items could appear in the `parts` array of a token given a particular `TokenType`. Now, each token interface declares a labelled tuple for the parts, which helps to document the token better. PR Close #42062 --- packages/compiler/src/i18n/i18n_parser.ts | 10 +- packages/compiler/src/ml_parser/ast.ts | 7 +- .../src/ml_parser/html_whitespaces.ts | 10 +- packages/compiler/src/ml_parser/lexer.ts | 59 +- packages/compiler/src/ml_parser/parser.ts | 159 +- packages/compiler/src/ml_parser/tokens.ts | 172 ++ .../test/ml_parser/html_parser_spec.ts | 2 +- .../compiler/test/ml_parser/lexer_spec.ts | 1631 ++++++++--------- 8 files changed, 1099 insertions(+), 951 deletions(-) create mode 100644 packages/compiler/src/ml_parser/tokens.ts diff --git a/packages/compiler/src/i18n/i18n_parser.ts b/packages/compiler/src/i18n/i18n_parser.ts index dda395f873..5b4ad20e2b 100644 --- a/packages/compiler/src/i18n/i18n_parser.ts +++ b/packages/compiler/src/i18n/i18n_parser.ts @@ -11,7 +11,7 @@ import {Parser as ExpressionParser} from '../expression_parser/parser'; import * as html from '../ml_parser/ast'; import {getHtmlTagDefinition} from '../ml_parser/html_tags'; import {InterpolationConfig} from '../ml_parser/interpolation_config'; -import {Token, TokenType} from '../ml_parser/lexer'; +import {InterpolatedAttributeToken, InterpolatedTextToken, TokenType} from '../ml_parser/tokens'; import {ParseSourceSpan} from '../parse_util'; import * as i18n from './i18n_ast'; @@ -163,16 +163,16 @@ class _I18nVisitor implements html.Visitor { } /** - * Split the, potentially interpolated, text up into text and placeholder pieces. + * Convert, text and interpolated tokens up into text and placeholder pieces. * - * @param text The potentially interpolated string to be split. + * @param tokens The text and interpolated tokens. * @param sourceSpan The span of the whole of the `text` string. * @param context The current context of the visitor, used to compute and store placeholders. * @param previousI18n Any i18n metadata associated with this `text` from a previous pass. */ private _visitTextWithInterpolation( - tokens: Token[], sourceSpan: ParseSourceSpan, context: I18nMessageVisitorContext, - previousI18n: i18n.I18nMeta|undefined): i18n.Node { + tokens: (InterpolatedTextToken|InterpolatedAttributeToken)[], sourceSpan: ParseSourceSpan, + context: I18nMessageVisitorContext, previousI18n: i18n.I18nMeta|undefined): i18n.Node { // Return a sequence of `Text` and `Placeholder` nodes grouped in a `Container`. const nodes: i18n.Node[] = []; for (const token of tokens) { diff --git a/packages/compiler/src/ml_parser/ast.ts b/packages/compiler/src/ml_parser/ast.ts index a8abcc80a6..0d9c45f0ed 100644 --- a/packages/compiler/src/ml_parser/ast.ts +++ b/packages/compiler/src/ml_parser/ast.ts @@ -9,7 +9,7 @@ import {AstPath} from '../ast_path'; import {I18nMeta} from '../i18n/i18n_ast'; import {ParseSourceSpan} from '../parse_util'; -import {Token} from './lexer'; +import {InterpolatedAttributeToken, InterpolatedTextToken} from './tokens'; interface BaseNode { sourceSpan: ParseSourceSpan; @@ -25,7 +25,8 @@ export abstract class NodeWithI18n implements BaseNode { export class Text extends NodeWithI18n { constructor( - public value: string, sourceSpan: ParseSourceSpan, public tokens: Token[], i18n?: I18nMeta) { + public value: string, sourceSpan: ParseSourceSpan, public tokens: InterpolatedTextToken[], + i18n?: I18nMeta) { super(sourceSpan, i18n); } override visit(visitor: Visitor, context: any): any { @@ -58,7 +59,7 @@ export class Attribute extends NodeWithI18n { constructor( public name: string, public value: string, sourceSpan: ParseSourceSpan, readonly keySpan: ParseSourceSpan|undefined, public valueSpan: ParseSourceSpan|undefined, - public valueTokens: Token[]|undefined, i18n: I18nMeta|undefined) { + public valueTokens: InterpolatedAttributeToken[]|undefined, i18n: I18nMeta|undefined) { super(sourceSpan, i18n); } override visit(visitor: Visitor, context: any): any { diff --git a/packages/compiler/src/ml_parser/html_whitespaces.ts b/packages/compiler/src/ml_parser/html_whitespaces.ts index 57045ee8dc..2fa193ec44 100644 --- a/packages/compiler/src/ml_parser/html_whitespaces.ts +++ b/packages/compiler/src/ml_parser/html_whitespaces.ts @@ -8,8 +8,8 @@ import * as html from './ast'; import {NGSP_UNICODE} from './entities'; -import {Token, TokenType} from './lexer'; import {ParseTreeResult} from './parser'; +import {TextToken, TokenType} from './tokens'; export const PRESERVE_WS_ATTR_NAME = 'ngPreserveWhitespaces'; @@ -77,8 +77,8 @@ export class WhitespaceVisitor implements html.Visitor { if (isNotBlank || hasExpansionSibling) { // Process the whitespace in the tokens of this Text node const tokens = text.tokens.map( - token => token.type === TokenType.TEXT ? createTextTokenAfterWhitespaceProcessing(token) : - token); + token => + token.type === TokenType.TEXT ? createWhitespaceProcessedTextToken(token) : token); // Process the whitespace of the value of this Text node const value = processWhitespace(text.value); return new html.Text(value, text.sourceSpan, tokens, text.i18n); @@ -100,8 +100,8 @@ export class WhitespaceVisitor implements html.Visitor { } } -function createTextTokenAfterWhitespaceProcessing(token: Token): Token { - return new Token(token.type, [processWhitespace(token.parts[0])], token.sourceSpan); +function createWhitespaceProcessedTextToken({type, parts, sourceSpan}: TextToken): TextToken { + return {type, parts: [processWhitespace(parts[0])], sourceSpan}; } function processWhitespace(text: string): string { diff --git a/packages/compiler/src/ml_parser/lexer.ts b/packages/compiler/src/ml_parser/lexer.ts index 3c95825e34..34b16ab423 100644 --- a/packages/compiler/src/ml_parser/lexer.ts +++ b/packages/compiler/src/ml_parser/lexer.ts @@ -12,39 +12,7 @@ import {NAMED_ENTITIES} from './entities'; import {DEFAULT_INTERPOLATION_CONFIG, InterpolationConfig} from './interpolation_config'; import {TagContentType, TagDefinition} from './tags'; - -export enum TokenType { - TAG_OPEN_START, - TAG_OPEN_END, - TAG_OPEN_END_VOID, - TAG_CLOSE, - INCOMPLETE_TAG_OPEN, - TEXT, - ESCAPABLE_RAW_TEXT, - RAW_TEXT, - INTERPOLATION, - ENCODED_ENTITY, - COMMENT_START, - COMMENT_END, - CDATA_START, - CDATA_END, - ATTR_NAME, - ATTR_QUOTE, - ATTR_VALUE_TEXT, - ATTR_VALUE_INTERPOLATION, - DOC_TYPE, - EXPANSION_FORM_START, - EXPANSION_CASE_VALUE, - EXPANSION_CASE_EXP_START, - EXPANSION_CASE_EXP_END, - EXPANSION_FORM_END, - EOF -} - -export class Token { - constructor( - public type: TokenType|null, public parts: string[], public sourceSpan: ParseSourceSpan) {} -} +import {IncompleteTagOpenToken, TagOpenStartToken, Token, TokenType} from './tokens'; export class TokenError extends ParseError { constructor(errorMsg: string, public tokenType: TokenType|null, span: ParseSourceSpan) { @@ -290,9 +258,12 @@ class _Tokenizer { 'Programming error - attempted to end a token which has no token type', null, this._cursor.getSpan(this._currentTokenStart)); } - const token = new Token( - this._currentTokenType, parts, - (end ?? this._cursor).getSpan(this._currentTokenStart, this._leadingTriviaCodePoints)); + const token = { + type: this._currentTokenType, + parts, + sourceSpan: + (end ?? this._cursor).getSpan(this._currentTokenStart, this._leadingTriviaCodePoints), + } as Token; this.tokens.push(token); this._currentTokenStart = null; this._currentTokenType = null; @@ -527,7 +498,7 @@ class _Tokenizer { private _consumeTagOpen(start: CharacterCursor) { let tagName: string; let prefix: string; - let openTagToken: Token|undefined; + let openTagToken: TagOpenStartToken|IncompleteTagOpenToken|undefined; try { if (!chars.isAsciiLetter(this._cursor.peek())) { throw this._createError( @@ -590,10 +561,10 @@ class _Tokenizer { this._endToken([prefix, tagName]); } - private _consumeTagOpenStart(start: CharacterCursor) { + private _consumeTagOpenStart(start: CharacterCursor): TagOpenStartToken { this._beginToken(TokenType.TAG_OPEN_START, start); const parts = this._consumePrefixAndName(); - return this._endToken(parts); + return this._endToken(parts) as TagOpenStartToken; } private _consumeAttributeName() { @@ -764,7 +735,7 @@ class _Tokenizer { */ private _consumeInterpolation( interpolationTokenType: TokenType, interpolationStart: CharacterCursor, - prematureEndPredicate: (() => boolean)|null) { + prematureEndPredicate: (() => boolean)|null): void { const parts: string[] = []; this._beginToken(interpolationTokenType, interpolationStart); parts.push(this._interpolationConfig.start); @@ -783,7 +754,8 @@ class _Tokenizer { // (This is actually wrong but here for backward compatibility). this._cursor = current; parts.push(this._getProcessedChars(expressionStart, current)); - return this._endToken(parts); + this._endToken(parts); + return; } if (inQuote === null) { @@ -791,7 +763,8 @@ class _Tokenizer { // We are not in a string, and we hit the end interpolation marker parts.push(this._getProcessedChars(expressionStart, current)); parts.push(this._interpolationConfig.end); - return this._endToken(parts); + this._endToken(parts); + return; } else if (this._attemptStr('//')) { // Once we are in a comment we ignore any quotes inComment = true; @@ -814,7 +787,7 @@ class _Tokenizer { // We hit EOF without finding a closing interpolation marker parts.push(this._getProcessedChars(expressionStart, this._cursor)); - return this._endToken(parts); + this._endToken(parts); } private _getProcessedChars(start: CharacterCursor, end: CharacterCursor): string { diff --git a/packages/compiler/src/ml_parser/parser.ts b/packages/compiler/src/ml_parser/parser.ts index d9f5d17bd3..ec1157c01d 100644 --- a/packages/compiler/src/ml_parser/parser.ts +++ b/packages/compiler/src/ml_parser/parser.ts @@ -10,8 +10,9 @@ import {ParseError, ParseLocation, ParseSourceSpan} from '../parse_util'; import * as html from './ast'; import {NAMED_ENTITIES} from './entities'; -import * as lex from './lexer'; +import {tokenize, TokenizeOptions} from './lexer'; import {getNsPrefix, mergeNsAndName, splitNsName, TagDefinition} from './tags'; +import {AttributeNameToken, AttributeQuoteToken, CdataStartToken, CommentStartToken, ExpansionCaseExpressionEndToken, ExpansionCaseExpressionStartToken, ExpansionCaseValueToken, ExpansionFormStartToken, IncompleteTagOpenToken, InterpolatedAttributeToken, InterpolatedTextToken, TagCloseToken, TagOpenStartToken, TextToken, Token, TokenType} from './tokens'; export class TreeError extends ParseError { static create(elementName: string|null, span: ParseSourceSpan, msg: string): TreeError { @@ -30,8 +31,8 @@ export class ParseTreeResult { export class Parser { constructor(public getTagDefinition: (tagName: string) => TagDefinition) {} - parse(source: string, url: string, options?: lex.TokenizeOptions): ParseTreeResult { - const tokenizeResult = lex.tokenize(source, url, this.getTagDefinition, options); + parse(source: string, url: string, options?: TokenizeOptions): ParseTreeResult { + const tokenizeResult = tokenize(source, url, this.getTagDefinition, options); const parser = new _TreeBuilder(tokenizeResult.tokens, this.getTagDefinition); parser.build(); return new ParseTreeResult( @@ -43,38 +44,38 @@ export class Parser { class _TreeBuilder { private _index: number = -1; - // `_peek` will be initialized by the call to `advance()` in the constructor. - private _peek!: lex.Token; + // `_peek` will be initialized by the call to `_advance()` in the constructor. + private _peek!: Token; private _elementStack: html.Element[] = []; rootNodes: html.Node[] = []; errors: TreeError[] = []; constructor( - private tokens: lex.Token[], private getTagDefinition: (tagName: string) => TagDefinition) { + private tokens: Token[], private getTagDefinition: (tagName: string) => TagDefinition) { this._advance(); } build(): void { - while (this._peek.type !== lex.TokenType.EOF) { - if (this._peek.type === lex.TokenType.TAG_OPEN_START || - this._peek.type === lex.TokenType.INCOMPLETE_TAG_OPEN) { - this._consumeStartTag(this._advance()); - } else if (this._peek.type === lex.TokenType.TAG_CLOSE) { - this._consumeEndTag(this._advance()); - } else if (this._peek.type === lex.TokenType.CDATA_START) { + while (this._peek.type !== TokenType.EOF) { + if (this._peek.type === TokenType.TAG_OPEN_START || + this._peek.type === TokenType.INCOMPLETE_TAG_OPEN) { + this._consumeStartTag(this._advance()); + } else if (this._peek.type === TokenType.TAG_CLOSE) { + this._consumeEndTag(this._advance()); + } else if (this._peek.type === TokenType.CDATA_START) { this._closeVoidElement(); - this._consumeCdata(this._advance()); - } else if (this._peek.type === lex.TokenType.COMMENT_START) { + this._consumeCdata(this._advance()); + } else if (this._peek.type === TokenType.COMMENT_START) { this._closeVoidElement(); - this._consumeComment(this._advance()); + this._consumeComment(this._advance()); } else if ( - this._peek.type === lex.TokenType.TEXT || this._peek.type === lex.TokenType.RAW_TEXT || - this._peek.type === lex.TokenType.ESCAPABLE_RAW_TEXT) { + this._peek.type === TokenType.TEXT || this._peek.type === TokenType.RAW_TEXT || + this._peek.type === TokenType.ESCAPABLE_RAW_TEXT) { this._closeVoidElement(); - this._consumeText(this._advance()); - } else if (this._peek.type === lex.TokenType.EXPANSION_FORM_START) { - this._consumeExpansion(this._advance()); + this._consumeText(this._advance()); + } else if (this._peek.type === TokenType.EXPANSION_FORM_START) { + this._consumeExpansion(this._advance()); } else { // Skip all other tokens... this._advance(); @@ -82,50 +83,50 @@ class _TreeBuilder { } } - private _advance(): lex.Token { + private _advance(): T { const prev = this._peek; if (this._index < this.tokens.length - 1) { // Note: there is always an EOF token at the end this._index++; } this._peek = this.tokens[this._index]; - return prev; + return prev as T; } - private _advanceIf(type: lex.TokenType): lex.Token|null { + private _advanceIf(type: T): (Token&{type: T})|null { if (this._peek.type === type) { - return this._advance(); + return this._advance(); } return null; } - private _consumeCdata(_startToken: lex.Token) { - this._consumeText(this._advance()); - this._advanceIf(lex.TokenType.CDATA_END); + private _consumeCdata(_startToken: CdataStartToken) { + this._consumeText(this._advance()); + this._advanceIf(TokenType.CDATA_END); } - private _consumeComment(token: lex.Token) { - const text = this._advanceIf(lex.TokenType.RAW_TEXT); - this._advanceIf(lex.TokenType.COMMENT_END); + private _consumeComment(token: CommentStartToken) { + const text = this._advanceIf(TokenType.RAW_TEXT); + this._advanceIf(TokenType.COMMENT_END); const value = text != null ? text.parts[0].trim() : null; this._addToParent(new html.Comment(value, token.sourceSpan)); } - private _consumeExpansion(token: lex.Token) { - const switchValue = this._advance(); + private _consumeExpansion(token: ExpansionFormStartToken) { + const switchValue = this._advance(); - const type = this._advance(); + const type = this._advance(); const cases: html.ExpansionCase[] = []; // read = - while (this._peek.type === lex.TokenType.EXPANSION_CASE_VALUE) { + while (this._peek.type === TokenType.EXPANSION_CASE_VALUE) { const expCase = this._parseExpansionCase(); if (!expCase) return; // error cases.push(expCase); } // read the final } - if (this._peek.type !== lex.TokenType.EXPANSION_FORM_END) { + if (this._peek.type !== TokenType.EXPANSION_FORM_END) { this.errors.push( TreeError.create(null, this._peek.sourceSpan, `Invalid ICU message. Missing '}'.`)); return; @@ -139,23 +140,23 @@ class _TreeBuilder { } private _parseExpansionCase(): html.ExpansionCase|null { - const value = this._advance(); + const value = this._advance(); // read { - if (this._peek.type !== lex.TokenType.EXPANSION_CASE_EXP_START) { + if (this._peek.type !== TokenType.EXPANSION_CASE_EXP_START) { this.errors.push( TreeError.create(null, this._peek.sourceSpan, `Invalid ICU message. Missing '{'.`)); return null; } // read until } - const start = this._advance(); + const start = this._advance(); const exp = this._collectExpansionExpTokens(start); if (!exp) return null; - const end = this._advance(); - exp.push(new lex.Token(lex.TokenType.EOF, [], end.sourceSpan)); + const end = this._advance(); + exp.push({type: TokenType.EOF, parts: [], sourceSpan: end.sourceSpan}); // parse everything in between { and } const expansionCaseParser = new _TreeBuilder(exp, this.getTagDefinition); @@ -173,18 +174,18 @@ class _TreeBuilder { value.parts[0], expansionCaseParser.rootNodes, sourceSpan, value.sourceSpan, expSourceSpan); } - private _collectExpansionExpTokens(start: lex.Token): lex.Token[]|null { - const exp: lex.Token[] = []; - const expansionFormStack = [lex.TokenType.EXPANSION_CASE_EXP_START]; + private _collectExpansionExpTokens(start: Token): Token[]|null { + const exp: Token[] = []; + const expansionFormStack = [TokenType.EXPANSION_CASE_EXP_START]; while (true) { - if (this._peek.type === lex.TokenType.EXPANSION_FORM_START || - this._peek.type === lex.TokenType.EXPANSION_CASE_EXP_START) { + if (this._peek.type === TokenType.EXPANSION_FORM_START || + this._peek.type === TokenType.EXPANSION_CASE_EXP_START) { expansionFormStack.push(this._peek.type); } - if (this._peek.type === lex.TokenType.EXPANSION_CASE_EXP_END) { - if (lastOnStack(expansionFormStack, lex.TokenType.EXPANSION_CASE_EXP_START)) { + if (this._peek.type === TokenType.EXPANSION_CASE_EXP_END) { + if (lastOnStack(expansionFormStack, TokenType.EXPANSION_CASE_EXP_START)) { expansionFormStack.pop(); if (expansionFormStack.length == 0) return exp; @@ -195,8 +196,8 @@ class _TreeBuilder { } } - if (this._peek.type === lex.TokenType.EXPANSION_FORM_END) { - if (lastOnStack(expansionFormStack, lex.TokenType.EXPANSION_FORM_START)) { + if (this._peek.type === TokenType.EXPANSION_FORM_END) { + if (lastOnStack(expansionFormStack, TokenType.EXPANSION_FORM_START)) { expansionFormStack.pop(); } else { this.errors.push( @@ -205,7 +206,7 @@ class _TreeBuilder { } } - if (this._peek.type === lex.TokenType.EOF) { + if (this._peek.type === TokenType.EOF) { this.errors.push( TreeError.create(null, start.sourceSpan, `Invalid ICU message. Missing '}'.`)); return null; @@ -215,7 +216,7 @@ class _TreeBuilder { } } - private _consumeText(token: lex.Token) { + private _consumeText(token: InterpolatedTextToken) { const tokens = [token]; const startSpan = token.sourceSpan; let text = token.parts[0]; @@ -224,22 +225,21 @@ class _TreeBuilder { if (parent != null && parent.children.length == 0 && this.getTagDefinition(parent.name).ignoreFirstLf) { text = text.substring(1); - tokens[0] = {type: token.type, sourceSpan: token.sourceSpan, parts: [text]}; + tokens[0] = {type: token.type, sourceSpan: token.sourceSpan, parts: [text]} as typeof token; } } - while (this._peek.type === lex.TokenType.INTERPOLATION || - this._peek.type === lex.TokenType.TEXT || - this._peek.type === lex.TokenType.ENCODED_ENTITY) { + while (this._peek.type === TokenType.INTERPOLATION || this._peek.type === TokenType.TEXT || + this._peek.type === TokenType.ENCODED_ENTITY) { token = this._advance(); tokens.push(token); - if (token.type === lex.TokenType.INTERPOLATION) { + if (token.type === TokenType.INTERPOLATION) { // For backward compatibility we decode HTML entities that appear in interpolation // expressions. This is arguably a bug, but it could be a considerable breaking change to // fix it. It should be addressed in a larger project to refactor the entire parser/lexer // chain after View Engine has been removed. text += token.parts.join('').replace(/&([^;]+);/g, decodeEntity); - } else if (token.type === lex.TokenType.ENCODED_ENTITY) { + } else if (token.type === TokenType.ENCODED_ENTITY) { text += token.parts[0]; } else { text += token.parts.join(''); @@ -262,17 +262,17 @@ class _TreeBuilder { } } - private _consumeStartTag(startTagToken: lex.Token) { + private _consumeStartTag(startTagToken: TagOpenStartToken|IncompleteTagOpenToken) { const [prefix, name] = startTagToken.parts; const attrs: html.Attribute[] = []; - while (this._peek.type === lex.TokenType.ATTR_NAME) { - attrs.push(this._consumeAttr(this._advance())); + while (this._peek.type === TokenType.ATTR_NAME) { + attrs.push(this._consumeAttr(this._advance())); } const fullName = this._getElementFullName(prefix, name, this._getParentElement()); let selfClosing = false; // Note: There could have been a tokenizer error // so that we don't get a token for the end tag... - if (this._peek.type === lex.TokenType.TAG_OPEN_END_VOID) { + if (this._peek.type === TokenType.TAG_OPEN_END_VOID) { this._advance(); selfClosing = true; const tagDef = this.getTagDefinition(fullName); @@ -281,7 +281,7 @@ class _TreeBuilder { fullName, startTagToken.sourceSpan, `Only void and foreign elements can be self closed "${startTagToken.parts[1]}"`)); } - } else if (this._peek.type === lex.TokenType.TAG_OPEN_END) { + } else if (this._peek.type === TokenType.TAG_OPEN_END) { this._advance(); selfClosing = false; } @@ -297,7 +297,7 @@ class _TreeBuilder { // Elements that are self-closed have their `endSourceSpan` set to the full span, as the // element start tag also represents the end tag. this._popElement(fullName, span); - } else if (startTagToken.type === lex.TokenType.INCOMPLETE_TAG_OPEN) { + } else if (startTagToken.type === TokenType.INCOMPLETE_TAG_OPEN) { // We already know the opening tag is not complete, so it is unlikely it has a corresponding // close tag. Let's optimistically parse it as a full element and emit an error. this._popElement(fullName, null); @@ -317,7 +317,7 @@ class _TreeBuilder { this._elementStack.push(el); } - private _consumeEndTag(endTagToken: lex.Token) { + private _consumeEndTag(endTagToken: TagCloseToken) { const fullName = this._getElementFullName( endTagToken.parts[0], endTagToken.parts[1], this._getParentElement()); @@ -363,35 +363,40 @@ class _TreeBuilder { return false; } - private _consumeAttr(attrName: lex.Token): html.Attribute { + private _consumeAttr(attrName: AttributeNameToken): html.Attribute { const fullName = mergeNsAndName(attrName.parts[0], attrName.parts[1]); let attrEnd = attrName.sourceSpan.end; // Consume any quote - if (this._peek.type === lex.TokenType.ATTR_QUOTE) { + if (this._peek.type === TokenType.ATTR_QUOTE) { this._advance(); } // Consume the attribute value let value = ''; - const valueTokens: lex.Token[] = []; + const valueTokens: InterpolatedAttributeToken[] = []; let valueStartSpan: ParseSourceSpan|undefined = undefined; let valueEnd: ParseLocation|undefined = undefined; - if (this._peek.type === lex.TokenType.ATTR_VALUE_TEXT) { + // NOTE: We need to use a new variable `nextTokenType` here to hide the actual type of + // `_peek.type` from TS. Otherwise TS will narrow the type of `_peek.type` preventing it from + // being able to consider `ATTR_VALUE_INTERPOLATION` as an option. This is because TS is not + // able to see that `_advance()` will actually mutate `_peek`. + const nextTokenType = this._peek.type; + if (nextTokenType === TokenType.ATTR_VALUE_TEXT) { valueStartSpan = this._peek.sourceSpan; valueEnd = this._peek.sourceSpan.end; - while (this._peek.type === lex.TokenType.ATTR_VALUE_TEXT || - this._peek.type === lex.TokenType.ATTR_VALUE_INTERPOLATION || - this._peek.type === lex.TokenType.ENCODED_ENTITY) { - const valueToken = this._advance(); + while (this._peek.type === TokenType.ATTR_VALUE_TEXT || + this._peek.type === TokenType.ATTR_VALUE_INTERPOLATION || + this._peek.type === TokenType.ENCODED_ENTITY) { + const valueToken = this._advance(); valueTokens.push(valueToken); - if (valueToken.type === lex.TokenType.ATTR_VALUE_INTERPOLATION) { + if (valueToken.type === TokenType.ATTR_VALUE_INTERPOLATION) { // For backward compatibility we decode HTML entities that appear in interpolation // expressions. This is arguably a bug, but it could be a considerable breaking change to // fix it. It should be addressed in a larger project to refactor the entire parser/lexer // chain after View Engine has been removed. value += valueToken.parts.join('').replace(/&([^;]+);/g, decodeEntity); - } else if (valueToken.type === lex.TokenType.ENCODED_ENTITY) { + } else if (valueToken.type === TokenType.ENCODED_ENTITY) { value += valueToken.parts[0]; } else { value += valueToken.parts.join(''); @@ -401,8 +406,8 @@ class _TreeBuilder { } // Consume any quote - if (this._peek.type === lex.TokenType.ATTR_QUOTE) { - const quoteToken = this._advance(); + if (this._peek.type === TokenType.ATTR_QUOTE) { + const quoteToken = this._advance(); attrEnd = quoteToken.sourceSpan.end; } diff --git a/packages/compiler/src/ml_parser/tokens.ts b/packages/compiler/src/ml_parser/tokens.ts new file mode 100644 index 0000000000..643dc25003 --- /dev/null +++ b/packages/compiler/src/ml_parser/tokens.ts @@ -0,0 +1,172 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +import {ParseSourceSpan} from '../parse_util'; + +export const enum TokenType { + TAG_OPEN_START, + TAG_OPEN_END, + TAG_OPEN_END_VOID, + TAG_CLOSE, + INCOMPLETE_TAG_OPEN, + TEXT, + ESCAPABLE_RAW_TEXT, + RAW_TEXT, + INTERPOLATION, + ENCODED_ENTITY, + COMMENT_START, + COMMENT_END, + CDATA_START, + CDATA_END, + ATTR_NAME, + ATTR_QUOTE, + ATTR_VALUE_TEXT, + ATTR_VALUE_INTERPOLATION, + DOC_TYPE, + EXPANSION_FORM_START, + EXPANSION_CASE_VALUE, + EXPANSION_CASE_EXP_START, + EXPANSION_CASE_EXP_END, + EXPANSION_FORM_END, + EOF +} + +export type Token = TagOpenStartToken|TagOpenEndToken|TagOpenEndVoidToken|TagCloseToken| + IncompleteTagOpenToken|TextToken|InterpolationToken|EncodedEntityToken|CommentStartToken| + CommentEndToken|CdataStartToken|CdataEndToken|AttributeNameToken|AttributeQuoteToken| + AttributeValueTextToken|AttributeValueInterpolationToken|DocTypeToken|ExpansionFormStartToken| + ExpansionCaseValueToken|ExpansionCaseExpressionStartToken|ExpansionCaseExpressionEndToken| + ExpansionFormEndToken|EndOfFileToken; + +export type InterpolatedTextToken = TextToken|InterpolationToken|EncodedEntityToken; + +export type InterpolatedAttributeToken = + AttributeValueTextToken|AttributeValueInterpolationToken|EncodedEntityToken; + +export interface TokenBase { + type: TokenType; + parts: string[]; + sourceSpan: ParseSourceSpan; +} + +export interface TagOpenStartToken extends TokenBase { + type: TokenType.TAG_OPEN_START; + parts: [prefix: string, name: string]; +} + +export interface TagOpenEndToken extends TokenBase { + type: TokenType.TAG_OPEN_END; + parts: []; +} + +export interface TagOpenEndVoidToken extends TokenBase { + type: TokenType.TAG_OPEN_END_VOID; + parts: []; +} + +export interface TagCloseToken extends TokenBase { + type: TokenType.TAG_CLOSE; + parts: [prefix: string, name: string]; +} + +export interface IncompleteTagOpenToken extends TokenBase { + type: TokenType.INCOMPLETE_TAG_OPEN; + parts: [prefix: string, name: string]; +} + +export interface TextToken extends TokenBase { + type: TokenType.TEXT|TokenType.ESCAPABLE_RAW_TEXT|TokenType.RAW_TEXT; + parts: [text: string]; +} + +export interface InterpolationToken extends TokenBase { + type: TokenType.INTERPOLATION; + parts: [startMarker: string, expression: string, endMarker: string]| + [startMarker: string, expression: string]; +} + +export interface EncodedEntityToken extends TokenBase { + type: TokenType.ENCODED_ENTITY; + parts: [decoded: string, encoded: string]; +} + +export interface CommentStartToken extends TokenBase { + type: TokenType.COMMENT_START; + parts: []; +} + +export interface CommentEndToken extends TokenBase { + type: TokenType.COMMENT_END; + parts: []; +} + +export interface CdataStartToken extends TokenBase { + type: TokenType.CDATA_START; + parts: []; +} + +export interface CdataEndToken extends TokenBase { + type: TokenType.CDATA_END; + parts: []; +} + +export interface AttributeNameToken extends TokenBase { + type: TokenType.ATTR_NAME; + parts: [prefix: string, name: string]; +} + +export interface AttributeQuoteToken extends TokenBase { + type: TokenType.ATTR_QUOTE; + parts: [quote: '\''|'"']; +} + +export interface AttributeValueTextToken extends TokenBase { + type: TokenType.ATTR_VALUE_TEXT; + parts: [value: string]; +} + +export interface AttributeValueInterpolationToken extends TokenBase { + type: TokenType.ATTR_VALUE_INTERPOLATION; + parts: [startMarker: string, expression: string, endMarker: string]| + [startMarker: string, expression: string]; +} + +export interface DocTypeToken extends TokenBase { + type: TokenType.DOC_TYPE; + parts: [content: string]; +} + +export interface ExpansionFormStartToken extends TokenBase { + type: TokenType.EXPANSION_FORM_START; + parts: []; +} + +export interface ExpansionCaseValueToken extends TokenBase { + type: TokenType.EXPANSION_CASE_VALUE; + parts: [value: string]; +} + +export interface ExpansionCaseExpressionStartToken extends TokenBase { + type: TokenType.EXPANSION_CASE_EXP_START; + parts: []; +} + +export interface ExpansionCaseExpressionEndToken extends TokenBase { + type: TokenType.EXPANSION_CASE_EXP_END; + parts: []; +} + +export interface ExpansionFormEndToken extends TokenBase { + type: TokenType.EXPANSION_FORM_END; + parts: []; +} + +export interface EndOfFileToken extends TokenBase { + type: TokenType.EOF; + parts: []; +} diff --git a/packages/compiler/test/ml_parser/html_parser_spec.ts b/packages/compiler/test/ml_parser/html_parser_spec.ts index 01562f53cf..45dcc1d957 100644 --- a/packages/compiler/test/ml_parser/html_parser_spec.ts +++ b/packages/compiler/test/ml_parser/html_parser_spec.ts @@ -8,7 +8,7 @@ import * as html from '../../src/ml_parser/ast'; import {HtmlParser, ParseTreeResult, TreeError} from '../../src/ml_parser/html_parser'; -import {TokenType} from '../../src/ml_parser/lexer'; +import {TokenType} from '../../src/ml_parser/tokens'; import {ParseError} from '../../src/parse_util'; import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes} from './ast_spec_utils'; diff --git a/packages/compiler/test/ml_parser/lexer_spec.ts b/packages/compiler/test/ml_parser/lexer_spec.ts index c0302a3345..5f6e7c317d 100644 --- a/packages/compiler/test/ml_parser/lexer_spec.ts +++ b/packages/compiler/test/ml_parser/lexer_spec.ts @@ -7,7 +7,8 @@ */ import {getHtmlTagDefinition} from '../../src/ml_parser/html_tags'; -import * as lex from '../../src/ml_parser/lexer'; +import {TokenError, tokenize, TokenizeOptions, TokenizeResult} from '../../src/ml_parser/lexer'; +import {Token, TokenType} from '../../src/ml_parser/tokens'; import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_util'; { @@ -15,41 +16,41 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u describe('line/column numbers', () => { it('should work without newlines', () => { expect(tokenizeAndHumanizeLineColumn('a')).toEqual([ - [lex.TokenType.TAG_OPEN_START, '0:0'], - [lex.TokenType.TAG_OPEN_END, '0:2'], - [lex.TokenType.TEXT, '0:3'], - [lex.TokenType.TAG_CLOSE, '0:4'], - [lex.TokenType.EOF, '0:8'], + [TokenType.TAG_OPEN_START, '0:0'], + [TokenType.TAG_OPEN_END, '0:2'], + [TokenType.TEXT, '0:3'], + [TokenType.TAG_CLOSE, '0:4'], + [TokenType.EOF, '0:8'], ]); }); it('should work with one newline', () => { expect(tokenizeAndHumanizeLineColumn('\na')).toEqual([ - [lex.TokenType.TAG_OPEN_START, '0:0'], - [lex.TokenType.TAG_OPEN_END, '0:2'], - [lex.TokenType.TEXT, '0:3'], - [lex.TokenType.TAG_CLOSE, '1:1'], - [lex.TokenType.EOF, '1:5'], + [TokenType.TAG_OPEN_START, '0:0'], + [TokenType.TAG_OPEN_END, '0:2'], + [TokenType.TEXT, '0:3'], + [TokenType.TAG_CLOSE, '1:1'], + [TokenType.EOF, '1:5'], ]); }); it('should work with multiple newlines', () => { expect(tokenizeAndHumanizeLineColumn('\na')).toEqual([ - [lex.TokenType.TAG_OPEN_START, '0:0'], - [lex.TokenType.TAG_OPEN_END, '1:0'], - [lex.TokenType.TEXT, '1:1'], - [lex.TokenType.TAG_CLOSE, '2:1'], - [lex.TokenType.EOF, '2:5'], + [TokenType.TAG_OPEN_START, '0:0'], + [TokenType.TAG_OPEN_END, '1:0'], + [TokenType.TEXT, '1:1'], + [TokenType.TAG_CLOSE, '2:1'], + [TokenType.EOF, '2:5'], ]); }); it('should work with CR and LF', () => { expect(tokenizeAndHumanizeLineColumn('\r\na\r')).toEqual([ - [lex.TokenType.TAG_OPEN_START, '0:0'], - [lex.TokenType.TAG_OPEN_END, '1:0'], - [lex.TokenType.TEXT, '1:1'], - [lex.TokenType.TAG_CLOSE, '2:1'], - [lex.TokenType.EOF, '2:5'], + [TokenType.TAG_OPEN_START, '0:0'], + [TokenType.TAG_OPEN_END, '1:0'], + [TokenType.TEXT, '1:1'], + [TokenType.TAG_CLOSE, '2:1'], + [TokenType.EOF, '2:5'], ]); }); @@ -57,11 +58,11 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u expect( tokenizeAndHumanizeFullStart('\n \t a', {leadingTriviaChars: ['\n', ' ', '\t']})) .toEqual([ - [lex.TokenType.TAG_OPEN_START, '0:0', '0:0'], - [lex.TokenType.TAG_OPEN_END, '0:2', '0:2'], - [lex.TokenType.TEXT, '1:3', '0:3'], - [lex.TokenType.TAG_CLOSE, '1:4', '1:4'], - [lex.TokenType.EOF, '1:8', '1:8'], + [TokenType.TAG_OPEN_START, '0:0', '0:0'], + [TokenType.TAG_OPEN_END, '0:2', '0:2'], + [TokenType.TEXT, '1:3', '0:3'], + [TokenType.TAG_CLOSE, '1:4', '1:4'], + [TokenType.EOF, '1:8', '1:8'], ]); }); }); @@ -72,8 +73,8 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u 'pre 1\npre 2\npre 3 `line 1\nline 2\nline 3` post 1\n post 2\n post 3', {range: {startPos: 19, startLine: 2, startCol: 7, endPos: 39}})) .toEqual([ - [lex.TokenType.TEXT, 'line 1\nline 2\nline 3'], - [lex.TokenType.EOF, ''], + [TokenType.TEXT, 'line 1\nline 2\nline 3'], + [TokenType.EOF, ''], ]); }); @@ -82,8 +83,8 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u 'pre 1\npre 2\npre 3 `line 1\nline 2\nline 3` post 1\n post 2\n post 3', {range: {startPos: 19, startLine: 2, startCol: 7, endPos: 39}})) .toEqual([ - [lex.TokenType.TEXT, '2:7'], - [lex.TokenType.EOF, '4:6'], + [TokenType.TEXT, '2:7'], + [TokenType.EOF, '4:6'], ]); }); }); @@ -91,49 +92,49 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u describe('comments', () => { it('should parse comments', () => { expect(tokenizeAndHumanizeParts('')).toEqual([ - [lex.TokenType.COMMENT_START], - [lex.TokenType.RAW_TEXT, 't\ne\ns\nt'], - [lex.TokenType.COMMENT_END], - [lex.TokenType.EOF], + [TokenType.COMMENT_START], + [TokenType.RAW_TEXT, 't\ne\ns\nt'], + [TokenType.COMMENT_END], + [TokenType.EOF], ]); }); it('should store the locations', () => { expect(tokenizeAndHumanizeSourceSpans('')).toEqual([ - [lex.TokenType.COMMENT_START, ''], - [lex.TokenType.EOF, ''], + [TokenType.COMMENT_START, ''], + [TokenType.EOF, ''], ]); }); it('should report { expect(tokenizeAndHumanizeErrors(' { expect(tokenizeAndHumanizeErrors('')).toEqual([ - [lex.TokenType.COMMENT_START, ''], - [lex.TokenType.EOF, ''], + [TokenType.COMMENT_START, ''], + [TokenType.EOF, ''], ]); }); it('should accept comments finishing by too many dashes (odd number)', () => { expect(tokenizeAndHumanizeSourceSpans('')).toEqual([ - [lex.TokenType.COMMENT_START, ''], - [lex.TokenType.EOF, ''], + [TokenType.COMMENT_START, ''], + [TokenType.EOF, ''], ]); }); }); @@ -141,21 +142,21 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u describe('doctype', () => { it('should parse doctypes', () => { expect(tokenizeAndHumanizeParts('')).toEqual([ - [lex.TokenType.DOC_TYPE, 'doctype html'], - [lex.TokenType.EOF], + [TokenType.DOC_TYPE, 'doctype html'], + [TokenType.EOF], ]); }); it('should store the locations', () => { expect(tokenizeAndHumanizeSourceSpans('')).toEqual([ - [lex.TokenType.DOC_TYPE, ''], - [lex.TokenType.EOF, ''], + [TokenType.DOC_TYPE, ''], + [TokenType.EOF, ''], ]); }); it('should report missing end doctype', () => { expect(tokenizeAndHumanizeErrors(' { it('should parse CDATA', () => { expect(tokenizeAndHumanizeParts('')).toEqual([ - [lex.TokenType.CDATA_START], - [lex.TokenType.RAW_TEXT, 't\ne\ns\nt'], - [lex.TokenType.CDATA_END], - [lex.TokenType.EOF], + [TokenType.CDATA_START], + [TokenType.RAW_TEXT, 't\ne\ns\nt'], + [TokenType.CDATA_END], + [TokenType.EOF], ]); }); it('should store the locations', () => { expect(tokenizeAndHumanizeSourceSpans('')).toEqual([ - [lex.TokenType.CDATA_START, ''], - [lex.TokenType.EOF, ''], + [TokenType.CDATA_START, ''], + [TokenType.EOF, ''], ]); }); it('should report { expect(tokenizeAndHumanizeErrors(' { expect(tokenizeAndHumanizeErrors(' { it('should parse open tags without prefix', () => { expect(tokenizeAndHumanizeParts('')).toEqual([ - [lex.TokenType.TAG_OPEN_START, '', 'test'], - [lex.TokenType.TAG_OPEN_END], - [lex.TokenType.EOF], + [TokenType.TAG_OPEN_START, '', 'test'], + [TokenType.TAG_OPEN_END], + [TokenType.EOF], ]); }); it('should parse namespace prefix', () => { expect(tokenizeAndHumanizeParts('')).toEqual([ - [lex.TokenType.TAG_OPEN_START, 'ns1', 'test'], - [lex.TokenType.TAG_OPEN_END], - [lex.TokenType.EOF], + [TokenType.TAG_OPEN_START, 'ns1', 'test'], + [TokenType.TAG_OPEN_END], + [TokenType.EOF], ]); }); it('should parse void tags', () => { expect(tokenizeAndHumanizeParts('')).toEqual([ - [lex.TokenType.TAG_OPEN_START, '', 'test'], - [lex.TokenType.TAG_OPEN_END_VOID], - [lex.TokenType.EOF], + [TokenType.TAG_OPEN_START, '', 'test'], + [TokenType.TAG_OPEN_END_VOID], + [TokenType.EOF], ]); }); it('should allow whitespace after the tag name', () => { expect(tokenizeAndHumanizeParts('')).toEqual([ - [lex.TokenType.TAG_OPEN_START, '', 'test'], - [lex.TokenType.TAG_OPEN_END], - [lex.TokenType.EOF], + [TokenType.TAG_OPEN_START, '', 'test'], + [TokenType.TAG_OPEN_END], + [TokenType.EOF], ]); }); it('should store the locations', () => { expect(tokenizeAndHumanizeSourceSpans('')).toEqual([ - [lex.TokenType.TAG_OPEN_START, ''], - [lex.TokenType.EOF, ''], + [TokenType.TAG_OPEN_START, ''], + [TokenType.EOF, ''], ]); }); describe('tags', () => { it('terminated with EOF', () => { expect(tokenizeAndHumanizeSourceSpans(' { expect(tokenizeAndHumanizeSourceSpans('')).toEqual([ - [lex.TokenType.INCOMPLETE_TAG_OPEN, ''], - [lex.TokenType.INCOMPLETE_TAG_OPEN, ''], - [lex.TokenType.EOF, ''], + [TokenType.INCOMPLETE_TAG_OPEN, ''], + [TokenType.INCOMPLETE_TAG_OPEN, ''], + [TokenType.EOF, ''], ]); }); it('in attribute', () => { expect(tokenizeAndHumanizeSourceSpans('
')).toEqual([ - [lex.TokenType.INCOMPLETE_TAG_OPEN, ''], - [lex.TokenType.TAG_CLOSE, ''], - [lex.TokenType.EOF, ''], + [TokenType.INCOMPLETE_TAG_OPEN, ''], + [TokenType.TAG_CLOSE, ''], + [TokenType.EOF, ''], ]); }); it('after quote', () => { expect(tokenizeAndHumanizeSourceSpans('
')).toEqual([ - [lex.TokenType.INCOMPLETE_TAG_OPEN, ''], - [lex.TokenType.TAG_CLOSE, ''], - [lex.TokenType.EOF, ''], + [TokenType.INCOMPLETE_TAG_OPEN, ''], + [TokenType.TAG_CLOSE, ''], + [TokenType.EOF, ''], ]); }); }); @@ -283,226 +284,226 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u describe('attributes', () => { it('should parse attributes without prefix', () => { expect(tokenizeAndHumanizeParts('')).toEqual([ - [lex.TokenType.TAG_OPEN_START, '', 't'], - [lex.TokenType.ATTR_NAME, '', 'a'], - [lex.TokenType.TAG_OPEN_END], - [lex.TokenType.EOF], + [TokenType.TAG_OPEN_START, '', 't'], + [TokenType.ATTR_NAME, '', 'a'], + [TokenType.TAG_OPEN_END], + [TokenType.EOF], ]); }); it('should parse attributes with interpolation', () => { expect(tokenizeAndHumanizeParts('')).toEqual([ - [lex.TokenType.TAG_OPEN_START, '', 't'], - [lex.TokenType.ATTR_NAME, '', 'a'], - [lex.TokenType.ATTR_QUOTE, '"'], - [lex.TokenType.ATTR_VALUE_TEXT, ''], - [lex.TokenType.ATTR_VALUE_INTERPOLATION, '{{', 'v', '}}'], - [lex.TokenType.ATTR_VALUE_TEXT, ''], - [lex.TokenType.ATTR_QUOTE, '"'], - [lex.TokenType.ATTR_NAME, '', 'b'], - [lex.TokenType.ATTR_QUOTE, '"'], - [lex.TokenType.ATTR_VALUE_TEXT, 's'], - [lex.TokenType.ATTR_VALUE_INTERPOLATION, '{{', 'm', '}}'], - [lex.TokenType.ATTR_VALUE_TEXT, 'e'], - [lex.TokenType.ATTR_QUOTE, '"'], - [lex.TokenType.ATTR_NAME, '', 'c'], - [lex.TokenType.ATTR_QUOTE, '"'], - [lex.TokenType.ATTR_VALUE_TEXT, 's'], - [lex.TokenType.ATTR_VALUE_INTERPOLATION, '{{', 'm//c', '}}'], - [lex.TokenType.ATTR_VALUE_TEXT, 'e'], - [lex.TokenType.ATTR_QUOTE, '"'], - [lex.TokenType.TAG_OPEN_END], - [lex.TokenType.EOF], + [TokenType.TAG_OPEN_START, '', 't'], + [TokenType.ATTR_NAME, '', 'a'], + [TokenType.ATTR_QUOTE, '"'], + [TokenType.ATTR_VALUE_TEXT, ''], + [TokenType.ATTR_VALUE_INTERPOLATION, '{{', 'v', '}}'], + [TokenType.ATTR_VALUE_TEXT, ''], + [TokenType.ATTR_QUOTE, '"'], + [TokenType.ATTR_NAME, '', 'b'], + [TokenType.ATTR_QUOTE, '"'], + [TokenType.ATTR_VALUE_TEXT, 's'], + [TokenType.ATTR_VALUE_INTERPOLATION, '{{', 'm', '}}'], + [TokenType.ATTR_VALUE_TEXT, 'e'], + [TokenType.ATTR_QUOTE, '"'], + [TokenType.ATTR_NAME, '', 'c'], + [TokenType.ATTR_QUOTE, '"'], + [TokenType.ATTR_VALUE_TEXT, 's'], + [TokenType.ATTR_VALUE_INTERPOLATION, '{{', 'm//c', '}}'], + [TokenType.ATTR_VALUE_TEXT, 'e'], + [TokenType.ATTR_QUOTE, '"'], + [TokenType.TAG_OPEN_END], + [TokenType.EOF], ]); }); it('should end interpolation on an unescaped matching quote', () => { expect(tokenizeAndHumanizeParts('')).toEqual([ - [lex.TokenType.TAG_OPEN_START, '', 't'], - [lex.TokenType.ATTR_NAME, '', 'a'], - [lex.TokenType.ATTR_QUOTE, '"'], - [lex.TokenType.ATTR_VALUE_TEXT, ''], - [lex.TokenType.ATTR_VALUE_INTERPOLATION, '{{', ' a \\" \' b '], - [lex.TokenType.ATTR_VALUE_TEXT, ''], - [lex.TokenType.ATTR_QUOTE, '"'], - [lex.TokenType.TAG_OPEN_END], - [lex.TokenType.EOF], + [TokenType.TAG_OPEN_START, '', 't'], + [TokenType.ATTR_NAME, '', 'a'], + [TokenType.ATTR_QUOTE, '"'], + [TokenType.ATTR_VALUE_TEXT, ''], + [TokenType.ATTR_VALUE_INTERPOLATION, '{{', ' a \\" \' b '], + [TokenType.ATTR_VALUE_TEXT, ''], + [TokenType.ATTR_QUOTE, '"'], + [TokenType.TAG_OPEN_END], + [TokenType.EOF], ]); expect(tokenizeAndHumanizeParts('')).toEqual([ - [lex.TokenType.TAG_OPEN_START, '', 't'], - [lex.TokenType.ATTR_NAME, '', 'a'], - [lex.TokenType.ATTR_QUOTE, '\''], - [lex.TokenType.ATTR_VALUE_TEXT, ''], - [lex.TokenType.ATTR_VALUE_INTERPOLATION, '{{', ' a " \\\' b '], - [lex.TokenType.ATTR_VALUE_TEXT, ''], - [lex.TokenType.ATTR_QUOTE, '\''], - [lex.TokenType.TAG_OPEN_END], - [lex.TokenType.EOF], + [TokenType.TAG_OPEN_START, '', 't'], + [TokenType.ATTR_NAME, '', 'a'], + [TokenType.ATTR_QUOTE, '\''], + [TokenType.ATTR_VALUE_TEXT, ''], + [TokenType.ATTR_VALUE_INTERPOLATION, '{{', ' a " \\\' b '], + [TokenType.ATTR_VALUE_TEXT, ''], + [TokenType.ATTR_QUOTE, '\''], + [TokenType.TAG_OPEN_END], + [TokenType.EOF], ]); }); it('should parse attributes with prefix', () => { expect(tokenizeAndHumanizeParts('')).toEqual([ - [lex.TokenType.TAG_OPEN_START, '', 't'], - [lex.TokenType.ATTR_NAME, 'ns1', 'a'], - [lex.TokenType.TAG_OPEN_END], - [lex.TokenType.EOF], + [TokenType.TAG_OPEN_START, '', 't'], + [TokenType.ATTR_NAME, 'ns1', 'a'], + [TokenType.TAG_OPEN_END], + [TokenType.EOF], ]); }); it('should parse attributes whose prefix is not valid', () => { expect(tokenizeAndHumanizeParts('')).toEqual([ - [lex.TokenType.TAG_OPEN_START, '', 't'], - [lex.TokenType.ATTR_NAME, '', '(ns1:a)'], - [lex.TokenType.TAG_OPEN_END], - [lex.TokenType.EOF], + [TokenType.TAG_OPEN_START, '', 't'], + [TokenType.ATTR_NAME, '', '(ns1:a)'], + [TokenType.TAG_OPEN_END], + [TokenType.EOF], ]); }); it('should parse attributes with single quote value', () => { expect(tokenizeAndHumanizeParts('')).toEqual([ - [lex.TokenType.TAG_OPEN_START, '', 't'], - [lex.TokenType.ATTR_NAME, '', 'a'], - [lex.TokenType.ATTR_QUOTE, '\''], - [lex.TokenType.ATTR_VALUE_TEXT, 'b'], - [lex.TokenType.ATTR_QUOTE, '\''], - [lex.TokenType.TAG_OPEN_END], - [lex.TokenType.EOF], + [TokenType.TAG_OPEN_START, '', 't'], + [TokenType.ATTR_NAME, '', 'a'], + [TokenType.ATTR_QUOTE, '\''], + [TokenType.ATTR_VALUE_TEXT, 'b'], + [TokenType.ATTR_QUOTE, '\''], + [TokenType.TAG_OPEN_END], + [TokenType.EOF], ]); }); it('should parse attributes with double quote value', () => { expect(tokenizeAndHumanizeParts('')).toEqual([ - [lex.TokenType.TAG_OPEN_START, '', 't'], - [lex.TokenType.ATTR_NAME, '', 'a'], - [lex.TokenType.ATTR_QUOTE, '"'], - [lex.TokenType.ATTR_VALUE_TEXT, 'b'], - [lex.TokenType.ATTR_QUOTE, '"'], - [lex.TokenType.TAG_OPEN_END], - [lex.TokenType.EOF], + [TokenType.TAG_OPEN_START, '', 't'], + [TokenType.ATTR_NAME, '', 'a'], + [TokenType.ATTR_QUOTE, '"'], + [TokenType.ATTR_VALUE_TEXT, 'b'], + [TokenType.ATTR_QUOTE, '"'], + [TokenType.TAG_OPEN_END], + [TokenType.EOF], ]); }); it('should parse attributes with unquoted value', () => { expect(tokenizeAndHumanizeParts('')).toEqual([ - [lex.TokenType.TAG_OPEN_START, '', 't'], - [lex.TokenType.ATTR_NAME, '', 'a'], - [lex.TokenType.ATTR_VALUE_TEXT, 'b'], - [lex.TokenType.TAG_OPEN_END], - [lex.TokenType.EOF], + [TokenType.TAG_OPEN_START, '', 't'], + [TokenType.ATTR_NAME, '', 'a'], + [TokenType.ATTR_VALUE_TEXT, 'b'], + [TokenType.TAG_OPEN_END], + [TokenType.EOF], ]); }); it('should parse attributes with unquoted interpolation value', () => { expect(tokenizeAndHumanizeParts('')).toEqual([ - [lex.TokenType.TAG_OPEN_START, '', 'a'], - [lex.TokenType.ATTR_NAME, '', 'a'], - [lex.TokenType.ATTR_VALUE_TEXT, ''], - [lex.TokenType.ATTR_VALUE_INTERPOLATION, '{{', 'link.text', '}}'], - [lex.TokenType.ATTR_VALUE_TEXT, ''], - [lex.TokenType.TAG_OPEN_END], - [lex.TokenType.EOF], + [TokenType.TAG_OPEN_START, '', 'a'], + [TokenType.ATTR_NAME, '', 'a'], + [TokenType.ATTR_VALUE_TEXT, ''], + [TokenType.ATTR_VALUE_INTERPOLATION, '{{', 'link.text', '}}'], + [TokenType.ATTR_VALUE_TEXT, ''], + [TokenType.TAG_OPEN_END], + [TokenType.EOF], ]); }); it('should parse attributes with empty quoted value', () => { expect(tokenizeAndHumanizeParts('')).toEqual([ - [lex.TokenType.TAG_OPEN_START, '', 't'], - [lex.TokenType.ATTR_NAME, '', 'a'], - [lex.TokenType.ATTR_QUOTE, '"'], - [lex.TokenType.ATTR_VALUE_TEXT, ''], - [lex.TokenType.ATTR_QUOTE, '"'], - [lex.TokenType.TAG_OPEN_END], - [lex.TokenType.EOF], + [TokenType.TAG_OPEN_START, '', 't'], + [TokenType.ATTR_NAME, '', 'a'], + [TokenType.ATTR_QUOTE, '"'], + [TokenType.ATTR_VALUE_TEXT, ''], + [TokenType.ATTR_QUOTE, '"'], + [TokenType.TAG_OPEN_END], + [TokenType.EOF], ]); }); it('should allow whitespace', () => { expect(tokenizeAndHumanizeParts('')).toEqual([ - [lex.TokenType.TAG_OPEN_START, '', 't'], - [lex.TokenType.ATTR_NAME, '', 'a'], - [lex.TokenType.ATTR_VALUE_TEXT, 'b'], - [lex.TokenType.TAG_OPEN_END], - [lex.TokenType.EOF], + [TokenType.TAG_OPEN_START, '', 't'], + [TokenType.ATTR_NAME, '', 'a'], + [TokenType.ATTR_VALUE_TEXT, 'b'], + [TokenType.TAG_OPEN_END], + [TokenType.EOF], ]); }); it('should parse attributes with entities in values', () => { expect(tokenizeAndHumanizeParts('')).toEqual([ - [lex.TokenType.TAG_OPEN_START, '', 't'], - [lex.TokenType.ATTR_NAME, '', 'a'], - [lex.TokenType.ATTR_QUOTE, '"'], - [lex.TokenType.ATTR_VALUE_TEXT, ''], - [lex.TokenType.ENCODED_ENTITY, 'A', 'A'], - [lex.TokenType.ATTR_VALUE_TEXT, ''], - [lex.TokenType.ENCODED_ENTITY, 'A', 'A'], - [lex.TokenType.ATTR_VALUE_TEXT, ''], - [lex.TokenType.ATTR_QUOTE, '"'], - [lex.TokenType.TAG_OPEN_END], - [lex.TokenType.EOF], + [TokenType.TAG_OPEN_START, '', 't'], + [TokenType.ATTR_NAME, '', 'a'], + [TokenType.ATTR_QUOTE, '"'], + [TokenType.ATTR_VALUE_TEXT, ''], + [TokenType.ENCODED_ENTITY, 'A', 'A'], + [TokenType.ATTR_VALUE_TEXT, ''], + [TokenType.ENCODED_ENTITY, 'A', 'A'], + [TokenType.ATTR_VALUE_TEXT, ''], + [TokenType.ATTR_QUOTE, '"'], + [TokenType.TAG_OPEN_END], + [TokenType.EOF], ]); }); it('should not decode entities without trailing ";"', () => { expect(tokenizeAndHumanizeParts('')).toEqual([ - [lex.TokenType.TAG_OPEN_START, '', 't'], - [lex.TokenType.ATTR_NAME, '', 'a'], - [lex.TokenType.ATTR_QUOTE, '"'], - [lex.TokenType.ATTR_VALUE_TEXT, '&'], - [lex.TokenType.ATTR_QUOTE, '"'], - [lex.TokenType.ATTR_NAME, '', 'b'], - [lex.TokenType.ATTR_QUOTE, '"'], - [lex.TokenType.ATTR_VALUE_TEXT, 'c&&d'], - [lex.TokenType.ATTR_QUOTE, '"'], - [lex.TokenType.TAG_OPEN_END], - [lex.TokenType.EOF], + [TokenType.TAG_OPEN_START, '', 't'], + [TokenType.ATTR_NAME, '', 'a'], + [TokenType.ATTR_QUOTE, '"'], + [TokenType.ATTR_VALUE_TEXT, '&'], + [TokenType.ATTR_QUOTE, '"'], + [TokenType.ATTR_NAME, '', 'b'], + [TokenType.ATTR_QUOTE, '"'], + [TokenType.ATTR_VALUE_TEXT, 'c&&d'], + [TokenType.ATTR_QUOTE, '"'], + [TokenType.TAG_OPEN_END], + [TokenType.EOF], ]); }); it('should parse attributes with "&" in values', () => { expect(tokenizeAndHumanizeParts('')).toEqual([ - [lex.TokenType.TAG_OPEN_START, '', 't'], - [lex.TokenType.ATTR_NAME, '', 'a'], - [lex.TokenType.ATTR_QUOTE, '"'], - [lex.TokenType.ATTR_VALUE_TEXT, 'b && c &'], - [lex.TokenType.ATTR_QUOTE, '"'], - [lex.TokenType.TAG_OPEN_END], - [lex.TokenType.EOF], + [TokenType.TAG_OPEN_START, '', 't'], + [TokenType.ATTR_NAME, '', 'a'], + [TokenType.ATTR_QUOTE, '"'], + [TokenType.ATTR_VALUE_TEXT, 'b && c &'], + [TokenType.ATTR_QUOTE, '"'], + [TokenType.TAG_OPEN_END], + [TokenType.EOF], ]); }); it('should parse values with CR and LF', () => { expect(tokenizeAndHumanizeParts('')).toEqual([ - [lex.TokenType.TAG_OPEN_START, '', 't'], - [lex.TokenType.ATTR_NAME, '', 'a'], - [lex.TokenType.ATTR_QUOTE, '\''], - [lex.TokenType.ATTR_VALUE_TEXT, 't\ne\ns\nt'], - [lex.TokenType.ATTR_QUOTE, '\''], - [lex.TokenType.TAG_OPEN_END], - [lex.TokenType.EOF], + [TokenType.TAG_OPEN_START, '', 't'], + [TokenType.ATTR_NAME, '', 'a'], + [TokenType.ATTR_QUOTE, '\''], + [TokenType.ATTR_VALUE_TEXT, 't\ne\ns\nt'], + [TokenType.ATTR_QUOTE, '\''], + [TokenType.TAG_OPEN_END], + [TokenType.EOF], ]); }); it('should store the locations', () => { expect(tokenizeAndHumanizeSourceSpans('')).toEqual([ - [lex.TokenType.TAG_OPEN_START, ''], - [lex.TokenType.EOF, ''], + [TokenType.TAG_OPEN_START, ''], + [TokenType.EOF, ''], ]); }); it('should report missing closing single quote', () => { expect(tokenizeAndHumanizeErrors('')).toEqual([ - [lex.TokenType.ATTR_VALUE_TEXT, 'Unexpected character "EOF"', '0:8'], + [TokenType.ATTR_VALUE_TEXT, 'Unexpected character "EOF"', '0:8'], ]); }); it('should report missing closing double quote', () => { expect(tokenizeAndHumanizeErrors(' { it('should parse closing tags without prefix', () => { expect(tokenizeAndHumanizeParts('')).toEqual([ - [lex.TokenType.TAG_CLOSE, '', 'test'], - [lex.TokenType.EOF], + [TokenType.TAG_CLOSE, '', 'test'], + [TokenType.EOF], ]); }); it('should parse closing tags with prefix', () => { expect(tokenizeAndHumanizeParts('')).toEqual([ - [lex.TokenType.TAG_CLOSE, 'ns1', 'test'], - [lex.TokenType.EOF], + [TokenType.TAG_CLOSE, 'ns1', 'test'], + [TokenType.EOF], ]); }); it('should allow whitespace', () => { expect(tokenizeAndHumanizeParts('')).toEqual([ - [lex.TokenType.TAG_CLOSE, '', 'test'], - [lex.TokenType.EOF], + [TokenType.TAG_CLOSE, '', 'test'], + [TokenType.EOF], ]); }); it('should store the locations', () => { expect(tokenizeAndHumanizeSourceSpans('')).toEqual([ - [lex.TokenType.TAG_CLOSE, ''], - [lex.TokenType.EOF, ''], + [TokenType.TAG_CLOSE, ''], + [TokenType.EOF, ''], ]); }); it('should report missing name after { expect(tokenizeAndHumanizeErrors('', () => { expect(tokenizeAndHumanizeErrors(' { it('should parse named entities', () => { expect(tokenizeAndHumanizeParts('a&b')).toEqual([ - [lex.TokenType.TEXT, 'a'], - [lex.TokenType.ENCODED_ENTITY, '&', '&'], - [lex.TokenType.TEXT, 'b'], - [lex.TokenType.EOF], + [TokenType.TEXT, 'a'], + [TokenType.ENCODED_ENTITY, '&', '&'], + [TokenType.TEXT, 'b'], + [TokenType.EOF], ]); }); it('should parse hexadecimal entities', () => { expect(tokenizeAndHumanizeParts('AA')).toEqual([ - [lex.TokenType.TEXT, ''], - [lex.TokenType.ENCODED_ENTITY, 'A', 'A'], - [lex.TokenType.TEXT, ''], - [lex.TokenType.ENCODED_ENTITY, 'A', 'A'], - [lex.TokenType.TEXT, ''], - [lex.TokenType.EOF], + [TokenType.TEXT, ''], + [TokenType.ENCODED_ENTITY, 'A', 'A'], + [TokenType.TEXT, ''], + [TokenType.ENCODED_ENTITY, 'A', 'A'], + [TokenType.TEXT, ''], + [TokenType.EOF], ]); }); it('should parse decimal entities', () => { expect(tokenizeAndHumanizeParts('A')).toEqual([ - [lex.TokenType.TEXT, ''], - [lex.TokenType.ENCODED_ENTITY, 'A', 'A'], - [lex.TokenType.TEXT, ''], - [lex.TokenType.EOF], + [TokenType.TEXT, ''], + [TokenType.ENCODED_ENTITY, 'A', 'A'], + [TokenType.TEXT, ''], + [TokenType.EOF], ]); }); it('should store the locations', () => { expect(tokenizeAndHumanizeSourceSpans('a&b')).toEqual([ - [lex.TokenType.TEXT, 'a'], - [lex.TokenType.ENCODED_ENTITY, '&'], - [lex.TokenType.TEXT, 'b'], - [lex.TokenType.EOF, ''], + [TokenType.TEXT, 'a'], + [TokenType.ENCODED_ENTITY, '&'], + [TokenType.TEXT, 'b'], + [TokenType.EOF, ''], ]); }); it('should report malformed/unknown entities', () => { expect(tokenizeAndHumanizeErrors('&tbo;')).toEqual([[ - lex.TokenType.ENCODED_ENTITY, + TokenType.ENCODED_ENTITY, 'Unknown entity "tbo" - use the "&#;" or "&#x;" syntax', '0:0' ]]); expect(tokenizeAndHumanizeErrors('sdf;')).toEqual([[ - lex.TokenType.ENCODED_ENTITY, + TokenType.ENCODED_ENTITY, 'Unable to parse entity "s" - decimal character reference entities must end with ";"', '0:4' ]]); expect(tokenizeAndHumanizeErrors(' sdf;')).toEqual([[ - lex.TokenType.ENCODED_ENTITY, + TokenType.ENCODED_ENTITY, 'Unable to parse entity " s" - hexadecimal character reference entities must end with ";"', '0:5' ]]); expect(tokenizeAndHumanizeErrors('઼')).toEqual([ - [lex.TokenType.ENCODED_ENTITY, 'Unexpected character "EOF"', '0:6'] + [TokenType.ENCODED_ENTITY, 'Unexpected character "EOF"', '0:6'] ]); }); }); @@ -613,8 +614,8 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u describe('regular text', () => { it('should parse text', () => { expect(tokenizeAndHumanizeParts('a')).toEqual([ - [lex.TokenType.TEXT, 'a'], - [lex.TokenType.EOF], + [TokenType.TEXT, 'a'], + [TokenType.EOF], ]); }); @@ -622,168 +623,168 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u expect(tokenizeAndHumanizeParts( '{{ a }}b{{ c // comment }}d{{ e "}} \' " f }}g{{ h // " i }}')) .toEqual([ - [lex.TokenType.TEXT, ''], - [lex.TokenType.INTERPOLATION, '{{', ' a ', '}}'], - [lex.TokenType.TEXT, 'b'], - [lex.TokenType.INTERPOLATION, '{{', ' c // comment ', '}}'], - [lex.TokenType.TEXT, 'd'], - [lex.TokenType.INTERPOLATION, '{{', ' e "}} \' " f ', '}}'], - [lex.TokenType.TEXT, 'g'], - [lex.TokenType.INTERPOLATION, '{{', ' h // " i ', '}}'], - [lex.TokenType.TEXT, ''], - [lex.TokenType.EOF], + [TokenType.TEXT, ''], + [TokenType.INTERPOLATION, '{{', ' a ', '}}'], + [TokenType.TEXT, 'b'], + [TokenType.INTERPOLATION, '{{', ' c // comment ', '}}'], + [TokenType.TEXT, 'd'], + [TokenType.INTERPOLATION, '{{', ' e "}} \' " f ', '}}'], + [TokenType.TEXT, 'g'], + [TokenType.INTERPOLATION, '{{', ' h // " i ', '}}'], + [TokenType.TEXT, ''], + [TokenType.EOF], ]); expect(tokenizeAndHumanizeSourceSpans('{{ a }}b{{ c // comment }}')).toEqual([ - [lex.TokenType.TEXT, ''], - [lex.TokenType.INTERPOLATION, '{{ a }}'], - [lex.TokenType.TEXT, 'b'], - [lex.TokenType.INTERPOLATION, '{{ c // comment }}'], - [lex.TokenType.TEXT, ''], - [lex.TokenType.EOF, ''], + [TokenType.TEXT, ''], + [TokenType.INTERPOLATION, '{{ a }}'], + [TokenType.TEXT, 'b'], + [TokenType.INTERPOLATION, '{{ c // comment }}'], + [TokenType.TEXT, ''], + [TokenType.EOF, ''], ]); }); it('should parse interpolation with custom markers', () => { expect(tokenizeAndHumanizeParts('{% a %}', {interpolationConfig: {start: '{%', end: '%}'}})) .toEqual([ - [lex.TokenType.TEXT, ''], - [lex.TokenType.INTERPOLATION, '{%', ' a ', '%}'], - [lex.TokenType.TEXT, ''], - [lex.TokenType.EOF], + [TokenType.TEXT, ''], + [TokenType.INTERPOLATION, '{%', ' a ', '%}'], + [TokenType.TEXT, ''], + [TokenType.EOF], ]); }); it('should handle CR & LF in text', () => { expect(tokenizeAndHumanizeParts('t\ne\rs\r\nt')).toEqual([ - [lex.TokenType.TEXT, 't\ne\ns\nt'], - [lex.TokenType.EOF], + [TokenType.TEXT, 't\ne\ns\nt'], + [TokenType.EOF], ]); expect(tokenizeAndHumanizeSourceSpans('t\ne\rs\r\nt')).toEqual([ - [lex.TokenType.TEXT, 't\ne\rs\r\nt'], - [lex.TokenType.EOF, ''], + [TokenType.TEXT, 't\ne\rs\r\nt'], + [TokenType.EOF, ''], ]); }); it('should handle CR & LF in interpolation', () => { expect(tokenizeAndHumanizeParts('{{t\ne\rs\r\nt}}')).toEqual([ - [lex.TokenType.TEXT, ''], - [lex.TokenType.INTERPOLATION, '{{', 't\ne\ns\nt', '}}'], - [lex.TokenType.TEXT, ''], - [lex.TokenType.EOF], + [TokenType.TEXT, ''], + [TokenType.INTERPOLATION, '{{', 't\ne\ns\nt', '}}'], + [TokenType.TEXT, ''], + [TokenType.EOF], ]); expect(tokenizeAndHumanizeSourceSpans('{{t\ne\rs\r\nt}}')).toEqual([ - [lex.TokenType.TEXT, ''], - [lex.TokenType.INTERPOLATION, '{{t\ne\rs\r\nt}}'], - [lex.TokenType.TEXT, ''], - [lex.TokenType.EOF, ''], + [TokenType.TEXT, ''], + [TokenType.INTERPOLATION, '{{t\ne\rs\r\nt}}'], + [TokenType.TEXT, ''], + [TokenType.EOF, ''], ]); }); it('should parse entities', () => { expect(tokenizeAndHumanizeParts('a&b')).toEqual([ - [lex.TokenType.TEXT, 'a'], - [lex.TokenType.ENCODED_ENTITY, '&', '&'], - [lex.TokenType.TEXT, 'b'], - [lex.TokenType.EOF], + [TokenType.TEXT, 'a'], + [TokenType.ENCODED_ENTITY, '&', '&'], + [TokenType.TEXT, 'b'], + [TokenType.EOF], ]); expect(tokenizeAndHumanizeSourceSpans('a&b')).toEqual([ - [lex.TokenType.TEXT, 'a'], - [lex.TokenType.ENCODED_ENTITY, '&'], - [lex.TokenType.TEXT, 'b'], - [lex.TokenType.EOF, ''], + [TokenType.TEXT, 'a'], + [TokenType.ENCODED_ENTITY, '&'], + [TokenType.TEXT, 'b'], + [TokenType.EOF, ''], ]); }); it('should parse text starting with "&"', () => { expect(tokenizeAndHumanizeParts('a && b &')).toEqual([ - [lex.TokenType.TEXT, 'a && b &'], - [lex.TokenType.EOF], + [TokenType.TEXT, 'a && b &'], + [TokenType.EOF], ]); }); it('should store the locations', () => { expect(tokenizeAndHumanizeSourceSpans('a')).toEqual([ - [lex.TokenType.TEXT, 'a'], - [lex.TokenType.EOF, ''], + [TokenType.TEXT, 'a'], + [TokenType.EOF, ''], ]); }); it('should allow "<" in text nodes', () => { expect(tokenizeAndHumanizeParts('{{ a < b ? c : d }}')).toEqual([ - [lex.TokenType.TEXT, ''], - [lex.TokenType.INTERPOLATION, '{{', ' a < b ? c : d ', '}}'], - [lex.TokenType.TEXT, ''], - [lex.TokenType.EOF], + [TokenType.TEXT, ''], + [TokenType.INTERPOLATION, '{{', ' a < b ? c : d ', '}}'], + [TokenType.TEXT, ''], + [TokenType.EOF], ]); expect(tokenizeAndHumanizeSourceSpans('

a')).toEqual([ - [lex.TokenType.TAG_OPEN_START, ''], - [lex.TokenType.TEXT, 'a'], - [lex.TokenType.INCOMPLETE_TAG_OPEN, ''], - [lex.TokenType.EOF, ''], + [TokenType.TAG_OPEN_START, ''], + [TokenType.TEXT, 'a'], + [TokenType.INCOMPLETE_TAG_OPEN, ''], + [TokenType.EOF, ''], ]); expect(tokenizeAndHumanizeParts('< a>')).toEqual([ - [lex.TokenType.TEXT, '< a>'], - [lex.TokenType.EOF], + [TokenType.TEXT, '< a>'], + [TokenType.EOF], ]); }); it('should break out of interpolation in text token on valid start tag', () => { expect(tokenizeAndHumanizeParts('{{ a d }}')).toEqual([ - [lex.TokenType.TEXT, ''], - [lex.TokenType.INTERPOLATION, '{{', ' a '], - [lex.TokenType.TEXT, ''], - [lex.TokenType.TAG_OPEN_START, '', 'b'], - [lex.TokenType.ATTR_NAME, '', '&&'], - [lex.TokenType.ATTR_NAME, '', 'c'], - [lex.TokenType.TAG_OPEN_END], - [lex.TokenType.TEXT, ' d }}'], - [lex.TokenType.EOF], + [TokenType.TEXT, ''], + [TokenType.INTERPOLATION, '{{', ' a '], + [TokenType.TEXT, ''], + [TokenType.TAG_OPEN_START, '', 'b'], + [TokenType.ATTR_NAME, '', '&&'], + [TokenType.ATTR_NAME, '', 'c'], + [TokenType.TAG_OPEN_END], + [TokenType.TEXT, ' d }}'], + [TokenType.EOF], ]); }); it('should break out of interpolation in text token on valid comment', () => { expect(tokenizeAndHumanizeParts('{{ a }}')).toEqual([ - [lex.TokenType.TEXT, ''], - [lex.TokenType.INTERPOLATION, '{{', ' a }'], - [lex.TokenType.TEXT, ''], - [lex.TokenType.COMMENT_START], - [lex.TokenType.RAW_TEXT, ''], - [lex.TokenType.COMMENT_END], - [lex.TokenType.TEXT, '}'], - [lex.TokenType.EOF], + [TokenType.TEXT, ''], + [TokenType.INTERPOLATION, '{{', ' a }'], + [TokenType.TEXT, ''], + [TokenType.COMMENT_START], + [TokenType.RAW_TEXT, ''], + [TokenType.COMMENT_END], + [TokenType.TEXT, '}'], + [TokenType.EOF], ]); }); it('should end interpolation on a valid closing tag', () => { expect(tokenizeAndHumanizeParts('

{{ a

')).toEqual([ - [lex.TokenType.TAG_OPEN_START, '', 'p'], - [lex.TokenType.TAG_OPEN_END], - [lex.TokenType.TEXT, ''], - [lex.TokenType.INTERPOLATION, '{{', ' a '], - [lex.TokenType.TEXT, ''], - [lex.TokenType.TAG_CLOSE, '', 'p'], - [lex.TokenType.EOF], + [TokenType.TAG_OPEN_START, '', 'p'], + [TokenType.TAG_OPEN_END], + [TokenType.TEXT, ''], + [TokenType.INTERPOLATION, '{{', ' a '], + [TokenType.TEXT, ''], + [TokenType.TAG_CLOSE, '', 'p'], + [TokenType.EOF], ]); }); it('should break out of interpolation in text token on valid CDATA', () => { expect(tokenizeAndHumanizeParts('{{ a }}')).toEqual([ - [lex.TokenType.TEXT, ''], - [lex.TokenType.INTERPOLATION, '{{', ' a }'], - [lex.TokenType.TEXT, ''], - [lex.TokenType.CDATA_START], - [lex.TokenType.RAW_TEXT, ''], - [lex.TokenType.CDATA_END], - [lex.TokenType.TEXT, '}'], - [lex.TokenType.EOF], + [TokenType.TEXT, ''], + [TokenType.INTERPOLATION, '{{', ' a }'], + [TokenType.TEXT, ''], + [TokenType.CDATA_START], + [TokenType.RAW_TEXT, ''], + [TokenType.CDATA_END], + [TokenType.TEXT, '}'], + [TokenType.EOF], ]); }); @@ -792,82 +793,82 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u // incorrectly be considered part of an ICU. expect(tokenizeAndHumanizeParts(`{{'<={'}}`, {tokenizeExpansionForms: true})) .toEqual([ - [lex.TokenType.TAG_OPEN_START, '', 'code'], - [lex.TokenType.TAG_OPEN_END], - [lex.TokenType.TEXT, ''], - [lex.TokenType.INTERPOLATION, '{{', '\'<={\'', '}}'], - [lex.TokenType.TEXT, ''], - [lex.TokenType.TAG_CLOSE, '', 'code'], - [lex.TokenType.EOF], + [TokenType.TAG_OPEN_START, '', 'code'], + [TokenType.TAG_OPEN_END], + [TokenType.TEXT, ''], + [TokenType.INTERPOLATION, '{{', '\'<={\'', '}}'], + [TokenType.TEXT, ''], + [TokenType.TAG_CLOSE, '', 'code'], + [TokenType.EOF], ]); }); it('should parse start tags quotes in place of an attribute name as text', () => { expect(tokenizeAndHumanizeParts('')).toEqual([ - [lex.TokenType.INCOMPLETE_TAG_OPEN, '', 't'], - [lex.TokenType.TEXT, '">'], - [lex.TokenType.EOF], + [TokenType.INCOMPLETE_TAG_OPEN, '', 't'], + [TokenType.TEXT, '">'], + [TokenType.EOF], ]); expect(tokenizeAndHumanizeParts('')).toEqual([ - [lex.TokenType.INCOMPLETE_TAG_OPEN, '', 't'], - [lex.TokenType.TEXT, '\'>'], - [lex.TokenType.EOF], + [TokenType.INCOMPLETE_TAG_OPEN, '', 't'], + [TokenType.TEXT, '\'>'], + [TokenType.EOF], ]); }); it('should parse start tags quotes in place of an attribute name (after a valid attribute)', () => { expect(tokenizeAndHumanizeParts('')).toEqual([ - [lex.TokenType.INCOMPLETE_TAG_OPEN, '', 't'], - [lex.TokenType.ATTR_NAME, '', 'a'], - [lex.TokenType.ATTR_QUOTE, '"'], - [lex.TokenType.ATTR_VALUE_TEXT, 'b'], - [lex.TokenType.ATTR_QUOTE, '"'], + [TokenType.INCOMPLETE_TAG_OPEN, '', 't'], + [TokenType.ATTR_NAME, '', 'a'], + [TokenType.ATTR_QUOTE, '"'], + [TokenType.ATTR_VALUE_TEXT, 'b'], + [TokenType.ATTR_QUOTE, '"'], // TODO(ayazhafiz): the " symbol should be a synthetic attribute, // allowing us to complete the opening tag correctly. - [lex.TokenType.TEXT, '">'], - [lex.TokenType.EOF], + [TokenType.TEXT, '">'], + [TokenType.EOF], ]); expect(tokenizeAndHumanizeParts('')).toEqual([ - [lex.TokenType.INCOMPLETE_TAG_OPEN, '', 't'], - [lex.TokenType.ATTR_NAME, '', 'a'], - [lex.TokenType.ATTR_QUOTE, '\''], - [lex.TokenType.ATTR_VALUE_TEXT, 'b'], - [lex.TokenType.ATTR_QUOTE, '\''], + [TokenType.INCOMPLETE_TAG_OPEN, '', 't'], + [TokenType.ATTR_NAME, '', 'a'], + [TokenType.ATTR_QUOTE, '\''], + [TokenType.ATTR_VALUE_TEXT, 'b'], + [TokenType.ATTR_QUOTE, '\''], // TODO(ayazhafiz): the ' symbol should be a synthetic attribute, // allowing us to complete the opening tag correctly. - [lex.TokenType.TEXT, '\'>'], - [lex.TokenType.EOF], + [TokenType.TEXT, '\'>'], + [TokenType.EOF], ]); }); it('should be able to escape {', () => { expect(tokenizeAndHumanizeParts('{{ "{" }}')).toEqual([ - [lex.TokenType.TEXT, ''], - [lex.TokenType.INTERPOLATION, '{{', ' "{" ', '}}'], - [lex.TokenType.TEXT, ''], - [lex.TokenType.EOF], + [TokenType.TEXT, ''], + [TokenType.INTERPOLATION, '{{', ' "{" ', '}}'], + [TokenType.TEXT, ''], + [TokenType.EOF], ]); }); it('should be able to escape {{', () => { expect(tokenizeAndHumanizeParts('{{ "{{" }}')).toEqual([ - [lex.TokenType.TEXT, ''], - [lex.TokenType.INTERPOLATION, '{{', ' "{{" ', '}}'], - [lex.TokenType.TEXT, ''], - [lex.TokenType.EOF], + [TokenType.TEXT, ''], + [TokenType.INTERPOLATION, '{{', ' "{{" ', '}}'], + [TokenType.TEXT, ''], + [TokenType.EOF], ]); }); it('should capture everything up to the end of file in the interpolation expression part if there are mismatched quotes', () => { expect(tokenizeAndHumanizeParts('{{ "{{a}}\' }}')).toEqual([ - [lex.TokenType.TEXT, ''], - [lex.TokenType.INTERPOLATION, '{{', ' "{{a}}\' }}'], - [lex.TokenType.TEXT, ''], - [lex.TokenType.EOF], + [TokenType.TEXT, ''], + [TokenType.INTERPOLATION, '{{', ' "{{a}}\' }}'], + [TokenType.TEXT, ''], + [TokenType.EOF], ]); }); @@ -875,11 +876,11 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u expect(tokenizeAndHumanizeParts( '{a, b, =4 {c}}', {tokenizeExpansionForms: false})) .toEqual([ - [lex.TokenType.TAG_OPEN_START, '', 'span'], - [lex.TokenType.TAG_OPEN_END], - [lex.TokenType.TEXT, '{a, b, =4 {c}}'], - [lex.TokenType.TAG_CLOSE, '', 'span'], - [lex.TokenType.EOF], + [TokenType.TAG_OPEN_START, '', 'span'], + [TokenType.TAG_OPEN_END], + [TokenType.TEXT, '{a, b, =4 {c}}'], + [TokenType.TAG_CLOSE, '', 'span'], + [TokenType.EOF], ]); }); }); @@ -887,51 +888,51 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u describe('raw text', () => { it('should parse text', () => { expect(tokenizeAndHumanizeParts(``)).toEqual([ - [lex.TokenType.TAG_OPEN_START, '', 'script'], - [lex.TokenType.TAG_OPEN_END], - [lex.TokenType.RAW_TEXT, 't\ne\ns\nt'], - [lex.TokenType.TAG_CLOSE, '', 'script'], - [lex.TokenType.EOF], + [TokenType.TAG_OPEN_START, '', 'script'], + [TokenType.TAG_OPEN_END], + [TokenType.RAW_TEXT, 't\ne\ns\nt'], + [TokenType.TAG_CLOSE, '', 'script'], + [TokenType.EOF], ]); }); it('should not detect entities', () => { expect(tokenizeAndHumanizeParts(``)).toEqual([ - [lex.TokenType.TAG_OPEN_START, '', 'script'], - [lex.TokenType.TAG_OPEN_END], - [lex.TokenType.RAW_TEXT, '&'], - [lex.TokenType.TAG_CLOSE, '', 'script'], - [lex.TokenType.EOF], + [TokenType.TAG_OPEN_START, '', 'script'], + [TokenType.TAG_OPEN_END], + [TokenType.RAW_TEXT, '&'], + [TokenType.TAG_CLOSE, '', 'script'], + [TokenType.EOF], ]); }); it('should ignore other opening tags', () => { expect(tokenizeAndHumanizeParts(``)).toEqual([ - [lex.TokenType.TAG_OPEN_START, '', 'script'], - [lex.TokenType.TAG_OPEN_END], - [lex.TokenType.RAW_TEXT, 'a
'], - [lex.TokenType.TAG_CLOSE, '', 'script'], - [lex.TokenType.EOF], + [TokenType.TAG_OPEN_START, '', 'script'], + [TokenType.TAG_OPEN_END], + [TokenType.RAW_TEXT, 'a
'], + [TokenType.TAG_CLOSE, '', 'script'], + [TokenType.EOF], ]); }); it('should ignore other closing tags', () => { expect(tokenizeAndHumanizeParts(``)).toEqual([ - [lex.TokenType.TAG_OPEN_START, '', 'script'], - [lex.TokenType.TAG_OPEN_END], - [lex.TokenType.RAW_TEXT, 'a'], - [lex.TokenType.TAG_CLOSE, '', 'script'], - [lex.TokenType.EOF], + [TokenType.TAG_OPEN_START, '', 'script'], + [TokenType.TAG_OPEN_END], + [TokenType.RAW_TEXT, 'a'], + [TokenType.TAG_CLOSE, '', 'script'], + [TokenType.EOF], ]); }); it('should store the locations', () => { expect(tokenizeAndHumanizeSourceSpans(``)).toEqual([ - [lex.TokenType.TAG_OPEN_START, ''], - [lex.TokenType.RAW_TEXT, 'a'], - [lex.TokenType.TAG_CLOSE, ''], - [lex.TokenType.EOF, ''], + [TokenType.TAG_OPEN_START, ''], + [TokenType.RAW_TEXT, 'a'], + [TokenType.TAG_CLOSE, ''], + [TokenType.EOF, ''], ]); }); }); @@ -939,53 +940,53 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u describe('escapable raw text', () => { it('should parse text', () => { expect(tokenizeAndHumanizeParts(`t\ne\rs\r\nt`)).toEqual([ - [lex.TokenType.TAG_OPEN_START, '', 'title'], - [lex.TokenType.TAG_OPEN_END], - [lex.TokenType.ESCAPABLE_RAW_TEXT, 't\ne\ns\nt'], - [lex.TokenType.TAG_CLOSE, '', 'title'], - [lex.TokenType.EOF], + [TokenType.TAG_OPEN_START, '', 'title'], + [TokenType.TAG_OPEN_END], + [TokenType.ESCAPABLE_RAW_TEXT, 't\ne\ns\nt'], + [TokenType.TAG_CLOSE, '', 'title'], + [TokenType.EOF], ]); }); it('should detect entities', () => { expect(tokenizeAndHumanizeParts(`&`)).toEqual([ - [lex.TokenType.TAG_OPEN_START, '', 'title'], - [lex.TokenType.TAG_OPEN_END], - [lex.TokenType.ESCAPABLE_RAW_TEXT, ''], - [lex.TokenType.ENCODED_ENTITY, '&', '&'], - [lex.TokenType.ESCAPABLE_RAW_TEXT, ''], - [lex.TokenType.TAG_CLOSE, '', 'title'], - [lex.TokenType.EOF], + [TokenType.TAG_OPEN_START, '', 'title'], + [TokenType.TAG_OPEN_END], + [TokenType.ESCAPABLE_RAW_TEXT, ''], + [TokenType.ENCODED_ENTITY, '&', '&'], + [TokenType.ESCAPABLE_RAW_TEXT, ''], + [TokenType.TAG_CLOSE, '', 'title'], + [TokenType.EOF], ]); }); it('should ignore other opening tags', () => { expect(tokenizeAndHumanizeParts(`a<div>`)).toEqual([ - [lex.TokenType.TAG_OPEN_START, '', 'title'], - [lex.TokenType.TAG_OPEN_END], - [lex.TokenType.ESCAPABLE_RAW_TEXT, 'a
'], - [lex.TokenType.TAG_CLOSE, '', 'title'], - [lex.TokenType.EOF], + [TokenType.TAG_OPEN_START, '', 'title'], + [TokenType.TAG_OPEN_END], + [TokenType.ESCAPABLE_RAW_TEXT, 'a
'], + [TokenType.TAG_CLOSE, '', 'title'], + [TokenType.EOF], ]); }); it('should ignore other closing tags', () => { expect(tokenizeAndHumanizeParts(`a</test>`)).toEqual([ - [lex.TokenType.TAG_OPEN_START, '', 'title'], - [lex.TokenType.TAG_OPEN_END], - [lex.TokenType.ESCAPABLE_RAW_TEXT, 'a'], - [lex.TokenType.TAG_CLOSE, '', 'title'], - [lex.TokenType.EOF], + [TokenType.TAG_OPEN_START, '', 'title'], + [TokenType.TAG_OPEN_END], + [TokenType.ESCAPABLE_RAW_TEXT, 'a'], + [TokenType.TAG_CLOSE, '', 'title'], + [TokenType.EOF], ]); }); it('should store the locations', () => { expect(tokenizeAndHumanizeSourceSpans(`a`)).toEqual([ - [lex.TokenType.TAG_OPEN_START, ''], - [lex.TokenType.ESCAPABLE_RAW_TEXT, 'a'], - [lex.TokenType.TAG_CLOSE, ''], - [lex.TokenType.EOF, ''], + [TokenType.TAG_OPEN_START, ''], + [TokenType.ESCAPABLE_RAW_TEXT, 'a'], + [TokenType.TAG_CLOSE, ''], + [TokenType.EOF, ''], ]); }); }); @@ -993,24 +994,24 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u describe('parsable data', () => { it('should parse an SVG tag', () => { expect(tokenizeAndHumanizeParts(`<svg:title>test</svg:title>`)).toEqual([ - [lex.TokenType.TAG_OPEN_START, 'svg', 'title'], - [lex.TokenType.TAG_OPEN_END], - [lex.TokenType.TEXT, 'test'], - [lex.TokenType.TAG_CLOSE, 'svg', 'title'], - [lex.TokenType.EOF], + [TokenType.TAG_OPEN_START, 'svg', 'title'], + [TokenType.TAG_OPEN_END], + [TokenType.TEXT, 'test'], + [TokenType.TAG_CLOSE, 'svg', 'title'], + [TokenType.EOF], ]); }); it('should parse an SVG <title> tag with children', () => { expect(tokenizeAndHumanizeParts(`<svg:title><f>test</f></svg:title>`)).toEqual([ - [lex.TokenType.TAG_OPEN_START, 'svg', 'title'], - [lex.TokenType.TAG_OPEN_END], - [lex.TokenType.TAG_OPEN_START, '', 'f'], - [lex.TokenType.TAG_OPEN_END], - [lex.TokenType.TEXT, 'test'], - [lex.TokenType.TAG_CLOSE, '', 'f'], - [lex.TokenType.TAG_CLOSE, 'svg', 'title'], - [lex.TokenType.EOF], + [TokenType.TAG_OPEN_START, 'svg', 'title'], + [TokenType.TAG_OPEN_END], + [TokenType.TAG_OPEN_START, '', 'f'], + [TokenType.TAG_OPEN_END], + [TokenType.TEXT, 'test'], + [TokenType.TAG_CLOSE, '', 'f'], + [TokenType.TAG_CLOSE, 'svg', 'title'], + [TokenType.EOF], ]); }); }); @@ -1021,23 +1022,23 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u tokenizeAndHumanizeParts( '{one.two, three, =4 {four} =5 {five} foo {bar} }', {tokenizeExpansionForms: true})) .toEqual([ - [lex.TokenType.EXPANSION_FORM_START], - [lex.TokenType.RAW_TEXT, 'one.two'], - [lex.TokenType.RAW_TEXT, 'three'], - [lex.TokenType.EXPANSION_CASE_VALUE, '=4'], - [lex.TokenType.EXPANSION_CASE_EXP_START], - [lex.TokenType.TEXT, 'four'], - [lex.TokenType.EXPANSION_CASE_EXP_END], - [lex.TokenType.EXPANSION_CASE_VALUE, '=5'], - [lex.TokenType.EXPANSION_CASE_EXP_START], - [lex.TokenType.TEXT, 'five'], - [lex.TokenType.EXPANSION_CASE_EXP_END], - [lex.TokenType.EXPANSION_CASE_VALUE, 'foo'], - [lex.TokenType.EXPANSION_CASE_EXP_START], - [lex.TokenType.TEXT, 'bar'], - [lex.TokenType.EXPANSION_CASE_EXP_END], - [lex.TokenType.EXPANSION_FORM_END], - [lex.TokenType.EOF], + [TokenType.EXPANSION_FORM_START], + [TokenType.RAW_TEXT, 'one.two'], + [TokenType.RAW_TEXT, 'three'], + [TokenType.EXPANSION_CASE_VALUE, '=4'], + [TokenType.EXPANSION_CASE_EXP_START], + [TokenType.TEXT, 'four'], + [TokenType.EXPANSION_CASE_EXP_END], + [TokenType.EXPANSION_CASE_VALUE, '=5'], + [TokenType.EXPANSION_CASE_EXP_START], + [TokenType.TEXT, 'five'], + [TokenType.EXPANSION_CASE_EXP_END], + [TokenType.EXPANSION_CASE_VALUE, 'foo'], + [TokenType.EXPANSION_CASE_EXP_START], + [TokenType.TEXT, 'bar'], + [TokenType.EXPANSION_CASE_EXP_END], + [TokenType.EXPANSION_FORM_END], + [TokenType.EOF], ]); }); @@ -1045,17 +1046,17 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u expect(tokenizeAndHumanizeParts( 'before{one.two, three, =4 {four}}after', {tokenizeExpansionForms: true})) .toEqual([ - [lex.TokenType.TEXT, 'before'], - [lex.TokenType.EXPANSION_FORM_START], - [lex.TokenType.RAW_TEXT, 'one.two'], - [lex.TokenType.RAW_TEXT, 'three'], - [lex.TokenType.EXPANSION_CASE_VALUE, '=4'], - [lex.TokenType.EXPANSION_CASE_EXP_START], - [lex.TokenType.TEXT, 'four'], - [lex.TokenType.EXPANSION_CASE_EXP_END], - [lex.TokenType.EXPANSION_FORM_END], - [lex.TokenType.TEXT, 'after'], - [lex.TokenType.EOF], + [TokenType.TEXT, 'before'], + [TokenType.EXPANSION_FORM_START], + [TokenType.RAW_TEXT, 'one.two'], + [TokenType.RAW_TEXT, 'three'], + [TokenType.EXPANSION_CASE_VALUE, '=4'], + [TokenType.EXPANSION_CASE_EXP_START], + [TokenType.TEXT, 'four'], + [TokenType.EXPANSION_CASE_EXP_END], + [TokenType.EXPANSION_FORM_END], + [TokenType.TEXT, 'after'], + [TokenType.EOF], ]); }); @@ -1063,21 +1064,21 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u expect(tokenizeAndHumanizeParts( '<div><span>{a, b, =4 {c}}</span></div>', {tokenizeExpansionForms: true})) .toEqual([ - [lex.TokenType.TAG_OPEN_START, '', 'div'], - [lex.TokenType.TAG_OPEN_END], - [lex.TokenType.TAG_OPEN_START, '', 'span'], - [lex.TokenType.TAG_OPEN_END], - [lex.TokenType.EXPANSION_FORM_START], - [lex.TokenType.RAW_TEXT, 'a'], - [lex.TokenType.RAW_TEXT, 'b'], - [lex.TokenType.EXPANSION_CASE_VALUE, '=4'], - [lex.TokenType.EXPANSION_CASE_EXP_START], - [lex.TokenType.TEXT, 'c'], - [lex.TokenType.EXPANSION_CASE_EXP_END], - [lex.TokenType.EXPANSION_FORM_END], - [lex.TokenType.TAG_CLOSE, '', 'span'], - [lex.TokenType.TAG_CLOSE, '', 'div'], - [lex.TokenType.EOF], + [TokenType.TAG_OPEN_START, '', 'div'], + [TokenType.TAG_OPEN_END], + [TokenType.TAG_OPEN_START, '', 'span'], + [TokenType.TAG_OPEN_END], + [TokenType.EXPANSION_FORM_START], + [TokenType.RAW_TEXT, 'a'], + [TokenType.RAW_TEXT, 'b'], + [TokenType.EXPANSION_CASE_VALUE, '=4'], + [TokenType.EXPANSION_CASE_EXP_START], + [TokenType.TEXT, 'c'], + [TokenType.EXPANSION_CASE_EXP_END], + [TokenType.EXPANSION_FORM_END], + [TokenType.TAG_CLOSE, '', 'span'], + [TokenType.TAG_CLOSE, '', 'div'], + [TokenType.EOF], ]); }); @@ -1085,23 +1086,23 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u expect(tokenizeAndHumanizeParts( '<div><span> {a, b, =4 {c}} </span></div>', {tokenizeExpansionForms: true})) .toEqual([ - [lex.TokenType.TAG_OPEN_START, '', 'div'], - [lex.TokenType.TAG_OPEN_END], - [lex.TokenType.TAG_OPEN_START, '', 'span'], - [lex.TokenType.TAG_OPEN_END], - [lex.TokenType.TEXT, ' '], - [lex.TokenType.EXPANSION_FORM_START], - [lex.TokenType.RAW_TEXT, 'a'], - [lex.TokenType.RAW_TEXT, 'b'], - [lex.TokenType.EXPANSION_CASE_VALUE, '=4'], - [lex.TokenType.EXPANSION_CASE_EXP_START], - [lex.TokenType.TEXT, 'c'], - [lex.TokenType.EXPANSION_CASE_EXP_END], - [lex.TokenType.EXPANSION_FORM_END], - [lex.TokenType.TEXT, ' '], - [lex.TokenType.TAG_CLOSE, '', 'span'], - [lex.TokenType.TAG_CLOSE, '', 'div'], - [lex.TokenType.EOF], + [TokenType.TAG_OPEN_START, '', 'div'], + [TokenType.TAG_OPEN_END], + [TokenType.TAG_OPEN_START, '', 'span'], + [TokenType.TAG_OPEN_END], + [TokenType.TEXT, ' '], + [TokenType.EXPANSION_FORM_START], + [TokenType.RAW_TEXT, 'a'], + [TokenType.RAW_TEXT, 'b'], + [TokenType.EXPANSION_CASE_VALUE, '=4'], + [TokenType.EXPANSION_CASE_EXP_START], + [TokenType.TEXT, 'c'], + [TokenType.EXPANSION_CASE_EXP_END], + [TokenType.EXPANSION_FORM_END], + [TokenType.TEXT, ' '], + [TokenType.TAG_CLOSE, '', 'span'], + [TokenType.TAG_CLOSE, '', 'div'], + [TokenType.EOF], ]); }); @@ -1109,19 +1110,19 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u expect(tokenizeAndHumanizeParts( '{one.two, three, =4 {four <b>a</b>}}', {tokenizeExpansionForms: true})) .toEqual([ - [lex.TokenType.EXPANSION_FORM_START], - [lex.TokenType.RAW_TEXT, 'one.two'], - [lex.TokenType.RAW_TEXT, 'three'], - [lex.TokenType.EXPANSION_CASE_VALUE, '=4'], - [lex.TokenType.EXPANSION_CASE_EXP_START], - [lex.TokenType.TEXT, 'four '], - [lex.TokenType.TAG_OPEN_START, '', 'b'], - [lex.TokenType.TAG_OPEN_END], - [lex.TokenType.TEXT, 'a'], - [lex.TokenType.TAG_CLOSE, '', 'b'], - [lex.TokenType.EXPANSION_CASE_EXP_END], - [lex.TokenType.EXPANSION_FORM_END], - [lex.TokenType.EOF], + [TokenType.EXPANSION_FORM_START], + [TokenType.RAW_TEXT, 'one.two'], + [TokenType.RAW_TEXT, 'three'], + [TokenType.EXPANSION_CASE_VALUE, '=4'], + [TokenType.EXPANSION_CASE_EXP_START], + [TokenType.TEXT, 'four '], + [TokenType.TAG_OPEN_START, '', 'b'], + [TokenType.TAG_OPEN_END], + [TokenType.TEXT, 'a'], + [TokenType.TAG_CLOSE, '', 'b'], + [TokenType.EXPANSION_CASE_EXP_END], + [TokenType.EXPANSION_FORM_END], + [TokenType.EOF], ]); }); @@ -1129,17 +1130,17 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u expect(tokenizeAndHumanizeParts( '{one.two, three, =4 {four {{a}}}}', {tokenizeExpansionForms: true})) .toEqual([ - [lex.TokenType.EXPANSION_FORM_START], - [lex.TokenType.RAW_TEXT, 'one.two'], - [lex.TokenType.RAW_TEXT, 'three'], - [lex.TokenType.EXPANSION_CASE_VALUE, '=4'], - [lex.TokenType.EXPANSION_CASE_EXP_START], - [lex.TokenType.TEXT, 'four '], - [lex.TokenType.INTERPOLATION, '{{', 'a', '}}'], - [lex.TokenType.TEXT, ''], - [lex.TokenType.EXPANSION_CASE_EXP_END], - [lex.TokenType.EXPANSION_FORM_END], - [lex.TokenType.EOF], + [TokenType.EXPANSION_FORM_START], + [TokenType.RAW_TEXT, 'one.two'], + [TokenType.RAW_TEXT, 'three'], + [TokenType.EXPANSION_CASE_VALUE, '=4'], + [TokenType.EXPANSION_CASE_EXP_START], + [TokenType.TEXT, 'four '], + [TokenType.INTERPOLATION, '{{', 'a', '}}'], + [TokenType.TEXT, ''], + [TokenType.EXPANSION_CASE_EXP_END], + [TokenType.EXPANSION_FORM_END], + [TokenType.EOF], ]); }); @@ -1147,23 +1148,23 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u expect(tokenizeAndHumanizeParts( `{one.two, three, =4 { {xx, yy, =x {one}} }}`, {tokenizeExpansionForms: true})) .toEqual([ - [lex.TokenType.EXPANSION_FORM_START], - [lex.TokenType.RAW_TEXT, 'one.two'], - [lex.TokenType.RAW_TEXT, 'three'], - [lex.TokenType.EXPANSION_CASE_VALUE, '=4'], - [lex.TokenType.EXPANSION_CASE_EXP_START], - [lex.TokenType.EXPANSION_FORM_START], - [lex.TokenType.RAW_TEXT, 'xx'], - [lex.TokenType.RAW_TEXT, 'yy'], - [lex.TokenType.EXPANSION_CASE_VALUE, '=x'], - [lex.TokenType.EXPANSION_CASE_EXP_START], - [lex.TokenType.TEXT, 'one'], - [lex.TokenType.EXPANSION_CASE_EXP_END], - [lex.TokenType.EXPANSION_FORM_END], - [lex.TokenType.TEXT, ' '], - [lex.TokenType.EXPANSION_CASE_EXP_END], - [lex.TokenType.EXPANSION_FORM_END], - [lex.TokenType.EOF], + [TokenType.EXPANSION_FORM_START], + [TokenType.RAW_TEXT, 'one.two'], + [TokenType.RAW_TEXT, 'three'], + [TokenType.EXPANSION_CASE_VALUE, '=4'], + [TokenType.EXPANSION_CASE_EXP_START], + [TokenType.EXPANSION_FORM_START], + [TokenType.RAW_TEXT, 'xx'], + [TokenType.RAW_TEXT, 'yy'], + [TokenType.EXPANSION_CASE_VALUE, '=x'], + [TokenType.EXPANSION_CASE_EXP_START], + [TokenType.TEXT, 'one'], + [TokenType.EXPANSION_CASE_EXP_END], + [TokenType.EXPANSION_FORM_END], + [TokenType.TEXT, ' '], + [TokenType.EXPANSION_CASE_EXP_END], + [TokenType.EXPANSION_FORM_END], + [TokenType.EOF], ]); }); @@ -1184,22 +1185,22 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u }); expect(humanizeParts(result.tokens)).toEqual([ - [lex.TokenType.EXPANSION_FORM_START], - [lex.TokenType.RAW_TEXT, '\n messages.length'], - [lex.TokenType.RAW_TEXT, 'plural'], - [lex.TokenType.EXPANSION_CASE_VALUE, '=0'], - [lex.TokenType.EXPANSION_CASE_EXP_START], - [lex.TokenType.TEXT, 'You have \nno\n messages'], - [lex.TokenType.EXPANSION_CASE_EXP_END], - [lex.TokenType.EXPANSION_CASE_VALUE, '=1'], - [lex.TokenType.EXPANSION_CASE_EXP_START], - [lex.TokenType.TEXT, 'One '], - [lex.TokenType.INTERPOLATION, '{{', 'message', '}}'], - [lex.TokenType.TEXT, ''], - [lex.TokenType.EXPANSION_CASE_EXP_END], - [lex.TokenType.EXPANSION_FORM_END], - [lex.TokenType.TEXT, '\n'], - [lex.TokenType.EOF], + [TokenType.EXPANSION_FORM_START], + [TokenType.RAW_TEXT, '\n messages.length'], + [TokenType.RAW_TEXT, 'plural'], + [TokenType.EXPANSION_CASE_VALUE, '=0'], + [TokenType.EXPANSION_CASE_EXP_START], + [TokenType.TEXT, 'You have \nno\n messages'], + [TokenType.EXPANSION_CASE_EXP_END], + [TokenType.EXPANSION_CASE_VALUE, '=1'], + [TokenType.EXPANSION_CASE_EXP_START], + [TokenType.TEXT, 'One '], + [TokenType.INTERPOLATION, '{{', 'message', '}}'], + [TokenType.TEXT, ''], + [TokenType.EXPANSION_CASE_EXP_END], + [TokenType.EXPANSION_FORM_END], + [TokenType.TEXT, '\n'], + [TokenType.EOF], ]); expect(result.nonNormalizedIcuExpressions).toEqual([]); @@ -1216,22 +1217,22 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u {tokenizeExpansionForms: true, escapedString: true}); expect(humanizeParts(result.tokens)).toEqual([ - [lex.TokenType.EXPANSION_FORM_START], - [lex.TokenType.RAW_TEXT, '\r\n messages.length'], - [lex.TokenType.RAW_TEXT, 'plural'], - [lex.TokenType.EXPANSION_CASE_VALUE, '=0'], - [lex.TokenType.EXPANSION_CASE_EXP_START], - [lex.TokenType.TEXT, 'You have \nno\n messages'], - [lex.TokenType.EXPANSION_CASE_EXP_END], - [lex.TokenType.EXPANSION_CASE_VALUE, '=1'], - [lex.TokenType.EXPANSION_CASE_EXP_START], - [lex.TokenType.TEXT, 'One '], - [lex.TokenType.INTERPOLATION, '{{', 'message', '}}'], - [lex.TokenType.TEXT, ''], - [lex.TokenType.EXPANSION_CASE_EXP_END], - [lex.TokenType.EXPANSION_FORM_END], - [lex.TokenType.TEXT, '\n'], - [lex.TokenType.EOF], + [TokenType.EXPANSION_FORM_START], + [TokenType.RAW_TEXT, '\r\n messages.length'], + [TokenType.RAW_TEXT, 'plural'], + [TokenType.EXPANSION_CASE_VALUE, '=0'], + [TokenType.EXPANSION_CASE_EXP_START], + [TokenType.TEXT, 'You have \nno\n messages'], + [TokenType.EXPANSION_CASE_EXP_END], + [TokenType.EXPANSION_CASE_VALUE, '=1'], + [TokenType.EXPANSION_CASE_EXP_START], + [TokenType.TEXT, 'One '], + [TokenType.INTERPOLATION, '{{', 'message', '}}'], + [TokenType.TEXT, ''], + [TokenType.EXPANSION_CASE_EXP_END], + [TokenType.EXPANSION_FORM_END], + [TokenType.TEXT, '\n'], + [TokenType.EOF], ]); expect(result.nonNormalizedIcuExpressions!.length).toBe(1); @@ -1253,26 +1254,26 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u `}`, {tokenizeExpansionForms: true, escapedString: true}); expect(humanizeParts(result.tokens)).toEqual([ - [lex.TokenType.EXPANSION_FORM_START], - [lex.TokenType.RAW_TEXT, '\r\n messages.length'], - [lex.TokenType.RAW_TEXT, 'plural'], - [lex.TokenType.EXPANSION_CASE_VALUE, '=0'], - [lex.TokenType.EXPANSION_CASE_EXP_START], - [lex.TokenType.TEXT, 'zero \n '], + [TokenType.EXPANSION_FORM_START], + [TokenType.RAW_TEXT, '\r\n messages.length'], + [TokenType.RAW_TEXT, 'plural'], + [TokenType.EXPANSION_CASE_VALUE, '=0'], + [TokenType.EXPANSION_CASE_EXP_START], + [TokenType.TEXT, 'zero \n '], - [lex.TokenType.EXPANSION_FORM_START], - [lex.TokenType.RAW_TEXT, '\r\n p.gender'], - [lex.TokenType.RAW_TEXT, 'select'], - [lex.TokenType.EXPANSION_CASE_VALUE, 'male'], - [lex.TokenType.EXPANSION_CASE_EXP_START], - [lex.TokenType.TEXT, 'm'], - [lex.TokenType.EXPANSION_CASE_EXP_END], - [lex.TokenType.EXPANSION_FORM_END], + [TokenType.EXPANSION_FORM_START], + [TokenType.RAW_TEXT, '\r\n p.gender'], + [TokenType.RAW_TEXT, 'select'], + [TokenType.EXPANSION_CASE_VALUE, 'male'], + [TokenType.EXPANSION_CASE_EXP_START], + [TokenType.TEXT, 'm'], + [TokenType.EXPANSION_CASE_EXP_END], + [TokenType.EXPANSION_FORM_END], - [lex.TokenType.TEXT, '\n '], - [lex.TokenType.EXPANSION_CASE_EXP_END], - [lex.TokenType.EXPANSION_FORM_END], - [lex.TokenType.EOF], + [TokenType.TEXT, '\n '], + [TokenType.EXPANSION_CASE_EXP_END], + [TokenType.EXPANSION_FORM_END], + [TokenType.EOF], ]); expect(result.nonNormalizedIcuExpressions!.length).toBe(2); @@ -1299,22 +1300,22 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u }); expect(humanizeParts(result.tokens)).toEqual([ - [lex.TokenType.EXPANSION_FORM_START], - [lex.TokenType.RAW_TEXT, '\n messages.length'], - [lex.TokenType.RAW_TEXT, 'plural'], - [lex.TokenType.EXPANSION_CASE_VALUE, '=0'], - [lex.TokenType.EXPANSION_CASE_EXP_START], - [lex.TokenType.TEXT, 'You have \nno\n messages'], - [lex.TokenType.EXPANSION_CASE_EXP_END], - [lex.TokenType.EXPANSION_CASE_VALUE, '=1'], - [lex.TokenType.EXPANSION_CASE_EXP_START], - [lex.TokenType.TEXT, 'One '], - [lex.TokenType.INTERPOLATION, '{{', 'message', '}}'], - [lex.TokenType.TEXT, ''], - [lex.TokenType.EXPANSION_CASE_EXP_END], - [lex.TokenType.EXPANSION_FORM_END], - [lex.TokenType.TEXT, '\n'], - [lex.TokenType.EOF], + [TokenType.EXPANSION_FORM_START], + [TokenType.RAW_TEXT, '\n messages.length'], + [TokenType.RAW_TEXT, 'plural'], + [TokenType.EXPANSION_CASE_VALUE, '=0'], + [TokenType.EXPANSION_CASE_EXP_START], + [TokenType.TEXT, 'You have \nno\n messages'], + [TokenType.EXPANSION_CASE_EXP_END], + [TokenType.EXPANSION_CASE_VALUE, '=1'], + [TokenType.EXPANSION_CASE_EXP_START], + [TokenType.TEXT, 'One '], + [TokenType.INTERPOLATION, '{{', 'message', '}}'], + [TokenType.TEXT, ''], + [TokenType.EXPANSION_CASE_EXP_END], + [TokenType.EXPANSION_FORM_END], + [TokenType.TEXT, '\n'], + [TokenType.EOF], ]); expect(result.nonNormalizedIcuExpressions).toEqual([]); @@ -1331,22 +1332,22 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u {tokenizeExpansionForms: true, escapedString: false}); expect(humanizeParts(result.tokens)).toEqual([ - [lex.TokenType.EXPANSION_FORM_START], - [lex.TokenType.RAW_TEXT, '\r\n messages.length'], - [lex.TokenType.RAW_TEXT, 'plural'], - [lex.TokenType.EXPANSION_CASE_VALUE, '=0'], - [lex.TokenType.EXPANSION_CASE_EXP_START], - [lex.TokenType.TEXT, 'You have \nno\n messages'], - [lex.TokenType.EXPANSION_CASE_EXP_END], - [lex.TokenType.EXPANSION_CASE_VALUE, '=1'], - [lex.TokenType.EXPANSION_CASE_EXP_START], - [lex.TokenType.TEXT, 'One '], - [lex.TokenType.INTERPOLATION, '{{', 'message', '}}'], - [lex.TokenType.TEXT, ''], - [lex.TokenType.EXPANSION_CASE_EXP_END], - [lex.TokenType.EXPANSION_FORM_END], - [lex.TokenType.TEXT, '\n'], - [lex.TokenType.EOF], + [TokenType.EXPANSION_FORM_START], + [TokenType.RAW_TEXT, '\r\n messages.length'], + [TokenType.RAW_TEXT, 'plural'], + [TokenType.EXPANSION_CASE_VALUE, '=0'], + [TokenType.EXPANSION_CASE_EXP_START], + [TokenType.TEXT, 'You have \nno\n messages'], + [TokenType.EXPANSION_CASE_EXP_END], + [TokenType.EXPANSION_CASE_VALUE, '=1'], + [TokenType.EXPANSION_CASE_EXP_START], + [TokenType.TEXT, 'One '], + [TokenType.INTERPOLATION, '{{', 'message', '}}'], + [TokenType.TEXT, ''], + [TokenType.EXPANSION_CASE_EXP_END], + [TokenType.EXPANSION_FORM_END], + [TokenType.TEXT, '\n'], + [TokenType.EOF], ]); expect(result.nonNormalizedIcuExpressions!.length).toBe(1); @@ -1369,26 +1370,26 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u {tokenizeExpansionForms: true}); expect(humanizeParts(result.tokens)).toEqual([ - [lex.TokenType.EXPANSION_FORM_START], - [lex.TokenType.RAW_TEXT, '\r\n messages.length'], - [lex.TokenType.RAW_TEXT, 'plural'], - [lex.TokenType.EXPANSION_CASE_VALUE, '=0'], - [lex.TokenType.EXPANSION_CASE_EXP_START], - [lex.TokenType.TEXT, 'zero \n '], + [TokenType.EXPANSION_FORM_START], + [TokenType.RAW_TEXT, '\r\n messages.length'], + [TokenType.RAW_TEXT, 'plural'], + [TokenType.EXPANSION_CASE_VALUE, '=0'], + [TokenType.EXPANSION_CASE_EXP_START], + [TokenType.TEXT, 'zero \n '], - [lex.TokenType.EXPANSION_FORM_START], - [lex.TokenType.RAW_TEXT, '\r\n p.gender'], - [lex.TokenType.RAW_TEXT, 'select'], - [lex.TokenType.EXPANSION_CASE_VALUE, 'male'], - [lex.TokenType.EXPANSION_CASE_EXP_START], - [lex.TokenType.TEXT, 'm'], - [lex.TokenType.EXPANSION_CASE_EXP_END], - [lex.TokenType.EXPANSION_FORM_END], + [TokenType.EXPANSION_FORM_START], + [TokenType.RAW_TEXT, '\r\n p.gender'], + [TokenType.RAW_TEXT, 'select'], + [TokenType.EXPANSION_CASE_VALUE, 'male'], + [TokenType.EXPANSION_CASE_EXP_START], + [TokenType.TEXT, 'm'], + [TokenType.EXPANSION_CASE_EXP_END], + [TokenType.EXPANSION_FORM_END], - [lex.TokenType.TEXT, '\n '], - [lex.TokenType.EXPANSION_CASE_EXP_END], - [lex.TokenType.EXPANSION_FORM_END], - [lex.TokenType.EOF], + [TokenType.TEXT, '\n '], + [TokenType.EXPANSION_CASE_EXP_END], + [TokenType.EXPANSION_FORM_END], + [TokenType.EOF], ]); expect(result.nonNormalizedIcuExpressions!.length).toBe(2); @@ -1406,7 +1407,7 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u it('should report unescaped "{" on error', () => { expect(tokenizeAndHumanizeErrors(`<p>before { after</p>`, {tokenizeExpansionForms: true})) .toEqual([[ - lex.TokenType.RAW_TEXT, + TokenType.RAW_TEXT, `Unexpected character "EOF" (Do you have an unescaped "{" in your template? Use "{{ '{' }}") to escape it.)`, '0:21', ]]); @@ -1418,7 +1419,7 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u `<code>{{b}<!---->}</code><pre>import {a} from 'a';</pre>`, {tokenizeExpansionForms: true})) .toEqual([[ - lex.TokenType.RAW_TEXT, + TokenType.RAW_TEXT, `Unexpected character "EOF" (Do you have an unescaped "{" in your template? Use "{{ '{' }}") to escape it.)`, '0:56', ]]); @@ -1429,7 +1430,7 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u const file = new ParseSourceFile(src, 'file://'); const location = new ParseLocation(file, 12, 123, 456); const span = new ParseSourceSpan(location, location); - const error = new lex.TokenError('**ERROR**', null!, span); + const error = new TokenError('**ERROR**', null!, span); expect(error.toString()) .toEqual(`**ERROR** ("\n222\n333\n[ERROR ->]E\n444\n555\n"): file://@123:456`); }); @@ -1438,11 +1439,11 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u describe('unicode characters', () => { it('should support unicode characters', () => { expect(tokenizeAndHumanizeSourceSpans(`<p>İ</p>`)).toEqual([ - [lex.TokenType.TAG_OPEN_START, '<p'], - [lex.TokenType.TAG_OPEN_END, '>'], - [lex.TokenType.TEXT, 'İ'], - [lex.TokenType.TAG_CLOSE, '</p>'], - [lex.TokenType.EOF, ''], + [TokenType.TAG_OPEN_START, '<p'], + [TokenType.TAG_OPEN_END, '>'], + [TokenType.TEXT, 'İ'], + [TokenType.TAG_CLOSE, '</p>'], + [TokenType.EOF, ''], ]); }); }); @@ -1450,63 +1451,63 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u describe('(processing escaped strings)', () => { it('should unescape standard escape sequences', () => { expect(tokenizeAndHumanizeParts('\\\' \\\' \\\'', {escapedString: true})).toEqual([ - [lex.TokenType.TEXT, '\' \' \''], - [lex.TokenType.EOF], + [TokenType.TEXT, '\' \' \''], + [TokenType.EOF], ]); expect(tokenizeAndHumanizeParts('\\" \\" \\"', {escapedString: true})).toEqual([ - [lex.TokenType.TEXT, '\" \" \"'], - [lex.TokenType.EOF], + [TokenType.TEXT, '\" \" \"'], + [TokenType.EOF], ]); expect(tokenizeAndHumanizeParts('\\` \\` \\`', {escapedString: true})).toEqual([ - [lex.TokenType.TEXT, '\` \` \`'], - [lex.TokenType.EOF], + [TokenType.TEXT, '\` \` \`'], + [TokenType.EOF], ]); expect(tokenizeAndHumanizeParts('\\\\ \\\\ \\\\', {escapedString: true})).toEqual([ - [lex.TokenType.TEXT, '\\ \\ \\'], - [lex.TokenType.EOF], + [TokenType.TEXT, '\\ \\ \\'], + [TokenType.EOF], ]); expect(tokenizeAndHumanizeParts('\\n \\n \\n', {escapedString: true})).toEqual([ - [lex.TokenType.TEXT, '\n \n \n'], - [lex.TokenType.EOF], + [TokenType.TEXT, '\n \n \n'], + [TokenType.EOF], ]); expect(tokenizeAndHumanizeParts('\\r{{\\r}}\\r', {escapedString: true})).toEqual([ // post processing converts `\r` to `\n` - [lex.TokenType.TEXT, '\n'], - [lex.TokenType.INTERPOLATION, '{{', '\n', '}}'], - [lex.TokenType.TEXT, '\n'], - [lex.TokenType.EOF], + [TokenType.TEXT, '\n'], + [TokenType.INTERPOLATION, '{{', '\n', '}}'], + [TokenType.TEXT, '\n'], + [TokenType.EOF], ]); expect(tokenizeAndHumanizeParts('\\v \\v \\v', {escapedString: true})).toEqual([ - [lex.TokenType.TEXT, '\v \v \v'], - [lex.TokenType.EOF], + [TokenType.TEXT, '\v \v \v'], + [TokenType.EOF], ]); expect(tokenizeAndHumanizeParts('\\t \\t \\t', {escapedString: true})).toEqual([ - [lex.TokenType.TEXT, '\t \t \t'], - [lex.TokenType.EOF], + [TokenType.TEXT, '\t \t \t'], + [TokenType.EOF], ]); expect(tokenizeAndHumanizeParts('\\b \\b \\b', {escapedString: true})).toEqual([ - [lex.TokenType.TEXT, '\b \b \b'], - [lex.TokenType.EOF], + [TokenType.TEXT, '\b \b \b'], + [TokenType.EOF], ]); expect(tokenizeAndHumanizeParts('\\f \\f \\f', {escapedString: true})).toEqual([ - [lex.TokenType.TEXT, '\f \f \f'], - [lex.TokenType.EOF], + [TokenType.TEXT, '\f \f \f'], + [TokenType.EOF], ]); expect(tokenizeAndHumanizeParts( '\\\' \\" \\` \\\\ \\n \\r \\v \\t \\b \\f', {escapedString: true})) .toEqual([ - [lex.TokenType.TEXT, '\' \" \` \\ \n \n \v \t \b \f'], - [lex.TokenType.EOF], + [TokenType.TEXT, '\' \" \` \\ \n \n \v \t \b \f'], + [TokenType.EOF], ]); }); it('should unescape null sequences', () => { expect(tokenizeAndHumanizeParts('\\0', {escapedString: true})).toEqual([ - [lex.TokenType.EOF], + [TokenType.EOF], ]); // \09 is not an octal number so the \0 is taken as EOF expect(tokenizeAndHumanizeParts('\\09', {escapedString: true})).toEqual([ - [lex.TokenType.EOF], + [TokenType.EOF], ]); }); @@ -1517,15 +1518,15 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u expect(tokenizeAndHumanizeParts( '\\001 \\01 \\1 \\12 \\223 \\19 \\2234 \\999', {escapedString: true})) .toEqual([ - [lex.TokenType.TEXT, '\x01 \x01 \x01 \x0A \x93 \x019 \x934 999'], - [lex.TokenType.EOF], + [TokenType.TEXT, '\x01 \x01 \x01 \x0A \x93 \x019 \x934 999'], + [TokenType.EOF], ]); }); it('should unescape hex sequences', () => { expect(tokenizeAndHumanizeParts('\\x12 \\x4F \\xDC', {escapedString: true})).toEqual([ - [lex.TokenType.TEXT, '\x12 \x4F \xDC'], - [lex.TokenType.EOF], + [TokenType.TEXT, '\x12 \x4F \xDC'], + [TokenType.EOF], ]); }); @@ -1535,18 +1536,18 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u ]); expect(tokenizeAndHumanizeErrors('abc \\x xyz', {escapedString: true})).toEqual([ - [lex.TokenType.TEXT, 'Invalid hexadecimal escape sequence', '0:6'] + [TokenType.TEXT, 'Invalid hexadecimal escape sequence', '0:6'] ]); expect(tokenizeAndHumanizeErrors('abc\\x', {escapedString: true})).toEqual([ - [lex.TokenType.TEXT, 'Unexpected character "EOF"', '0:5'] + [TokenType.TEXT, 'Unexpected character "EOF"', '0:5'] ]); }); it('should unescape fixed length Unicode sequences', () => { expect(tokenizeAndHumanizeParts('\\u0123 \\uABCD', {escapedString: true})).toEqual([ - [lex.TokenType.TEXT, '\u0123 \uABCD'], - [lex.TokenType.EOF], + [TokenType.TEXT, '\u0123 \uABCD'], + [TokenType.EOF], ]); }); @@ -1560,8 +1561,8 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u expect(tokenizeAndHumanizeParts( '\\u{01} \\u{ABC} \\u{1234} \\u{123AB}', {escapedString: true})) .toEqual([ - [lex.TokenType.TEXT, '\u{01} \u{ABC} \u{1234} \u{123AB}'], - [lex.TokenType.EOF], + [TokenType.TEXT, '\u{01} \u{ABC} \u{1234} \u{123AB}'], + [TokenType.EOF], ]); }); @@ -1573,27 +1574,27 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u it('should unescape line continuations', () => { expect(tokenizeAndHumanizeParts('abc\\\ndef', {escapedString: true})).toEqual([ - [lex.TokenType.TEXT, 'abcdef'], - [lex.TokenType.EOF], + [TokenType.TEXT, 'abcdef'], + [TokenType.EOF], ]); expect(tokenizeAndHumanizeParts('\\\nx\\\ny\\\n', {escapedString: true})).toEqual([ - [lex.TokenType.TEXT, 'xy'], - [lex.TokenType.EOF], + [TokenType.TEXT, 'xy'], + [TokenType.EOF], ]); }); it('should remove backslash from "non-escape" sequences', () => { expect(tokenizeAndHumanizeParts('\a \g \~', {escapedString: true})).toEqual([ - [lex.TokenType.TEXT, 'a g ~'], - [lex.TokenType.EOF], + [TokenType.TEXT, 'a g ~'], + [TokenType.EOF], ]); }); it('should unescape sequences in plain text', () => { expect(tokenizeAndHumanizeParts('abc\ndef\\nghi\\tjkl\\`\\\'\\"mno', {escapedString: true})) .toEqual([ - [lex.TokenType.TEXT, 'abc\ndef\nghi\tjkl`\'"mno'], - [lex.TokenType.EOF], + [TokenType.TEXT, 'abc\ndef\nghi\tjkl`\'"mno'], + [TokenType.EOF], ]); }); @@ -1601,11 +1602,11 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u expect(tokenizeAndHumanizeParts( '<script>abc\ndef\\nghi\\tjkl\\`\\\'\\"mno</script>', {escapedString: true})) .toEqual([ - [lex.TokenType.TAG_OPEN_START, '', 'script'], - [lex.TokenType.TAG_OPEN_END], - [lex.TokenType.RAW_TEXT, 'abc\ndef\nghi\tjkl`\'"mno'], - [lex.TokenType.TAG_CLOSE, '', 'script'], - [lex.TokenType.EOF], + [TokenType.TAG_OPEN_START, '', 'script'], + [TokenType.TAG_OPEN_END], + [TokenType.RAW_TEXT, 'abc\ndef\nghi\tjkl`\'"mno'], + [TokenType.TAG_CLOSE, '', 'script'], + [TokenType.EOF], ]); }); @@ -1613,90 +1614,90 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u expect(tokenizeAndHumanizeParts( '<title>abc\ndef\\nghi\\tjkl\\`\\\'\\"mno', {escapedString: true})) .toEqual([ - [lex.TokenType.TAG_OPEN_START, '', 'title'], - [lex.TokenType.TAG_OPEN_END], - [lex.TokenType.ESCAPABLE_RAW_TEXT, 'abc\ndef\nghi\tjkl`\'"mno'], - [lex.TokenType.TAG_CLOSE, '', 'title'], - [lex.TokenType.EOF], + [TokenType.TAG_OPEN_START, '', 'title'], + [TokenType.TAG_OPEN_END], + [TokenType.ESCAPABLE_RAW_TEXT, 'abc\ndef\nghi\tjkl`\'"mno'], + [TokenType.TAG_CLOSE, '', 'title'], + [TokenType.EOF], ]); }); it('should parse over escape sequences in tag definitions', () => { expect(tokenizeAndHumanizeParts('', {escapedString: true})) .toEqual([ - [lex.TokenType.TAG_OPEN_START, '', 't'], - [lex.TokenType.ATTR_NAME, '', 'a'], - [lex.TokenType.ATTR_QUOTE, '"'], - [lex.TokenType.ATTR_VALUE_TEXT, 'b'], - [lex.TokenType.ATTR_QUOTE, '"'], - [lex.TokenType.ATTR_NAME, '', 'c'], - [lex.TokenType.ATTR_QUOTE, '\''], - [lex.TokenType.ATTR_VALUE_TEXT, 'd'], - [lex.TokenType.ATTR_QUOTE, '\''], - [lex.TokenType.TAG_OPEN_END], - [lex.TokenType.EOF], + [TokenType.TAG_OPEN_START, '', 't'], + [TokenType.ATTR_NAME, '', 'a'], + [TokenType.ATTR_QUOTE, '"'], + [TokenType.ATTR_VALUE_TEXT, 'b'], + [TokenType.ATTR_QUOTE, '"'], + [TokenType.ATTR_NAME, '', 'c'], + [TokenType.ATTR_QUOTE, '\''], + [TokenType.ATTR_VALUE_TEXT, 'd'], + [TokenType.ATTR_QUOTE, '\''], + [TokenType.TAG_OPEN_END], + [TokenType.EOF], ]); }); it('should parse over escaped new line in tag definitions', () => { const text = ''; expect(tokenizeAndHumanizeParts(text, {escapedString: true})).toEqual([ - [lex.TokenType.TAG_OPEN_START, '', 't'], - [lex.TokenType.TAG_OPEN_END], - [lex.TokenType.TAG_CLOSE, '', 't'], - [lex.TokenType.EOF], + [TokenType.TAG_OPEN_START, '', 't'], + [TokenType.TAG_OPEN_END], + [TokenType.TAG_CLOSE, '', 't'], + [TokenType.EOF], ]); }); it('should parse over escaped characters in tag definitions', () => { const text = ''; expect(tokenizeAndHumanizeParts(text, {escapedString: true})).toEqual([ - [lex.TokenType.TAG_OPEN_START, '', 't'], - [lex.TokenType.TAG_OPEN_END], - [lex.TokenType.TAG_CLOSE, '', 't'], - [lex.TokenType.EOF], + [TokenType.TAG_OPEN_START, '', 't'], + [TokenType.TAG_OPEN_END], + [TokenType.TAG_CLOSE, '', 't'], + [TokenType.EOF], ]); }); it('should unescape characters in tag names', () => { const text = ''; expect(tokenizeAndHumanizeParts(text, {escapedString: true})).toEqual([ - [lex.TokenType.TAG_OPEN_START, '', 'td'], - [lex.TokenType.TAG_OPEN_END], - [lex.TokenType.TAG_CLOSE, '', 'td'], - [lex.TokenType.EOF], + [TokenType.TAG_OPEN_START, '', 'td'], + [TokenType.TAG_OPEN_END], + [TokenType.TAG_CLOSE, '', 'td'], + [TokenType.EOF], ]); expect(tokenizeAndHumanizeSourceSpans(text, {escapedString: true})).toEqual([ - [lex.TokenType.TAG_OPEN_START, ''], - [lex.TokenType.TAG_CLOSE, ''], - [lex.TokenType.EOF, ''], + [TokenType.TAG_OPEN_START, ''], + [TokenType.TAG_CLOSE, ''], + [TokenType.EOF, ''], ]); }); it('should unescape characters in attributes', () => { const text = ''; expect(tokenizeAndHumanizeParts(text, {escapedString: true})).toEqual([ - [lex.TokenType.TAG_OPEN_START, '', 't'], - [lex.TokenType.ATTR_NAME, '', 'd'], - [lex.TokenType.ATTR_QUOTE, '"'], - [lex.TokenType.ATTR_VALUE_TEXT, 'e'], - [lex.TokenType.ATTR_QUOTE, '"'], - [lex.TokenType.TAG_OPEN_END], - [lex.TokenType.TAG_CLOSE, '', 't'], - [lex.TokenType.EOF], + [TokenType.TAG_OPEN_START, '', 't'], + [TokenType.ATTR_NAME, '', 'd'], + [TokenType.ATTR_QUOTE, '"'], + [TokenType.ATTR_VALUE_TEXT, 'e'], + [TokenType.ATTR_QUOTE, '"'], + [TokenType.TAG_OPEN_END], + [TokenType.TAG_CLOSE, '', 't'], + [TokenType.EOF], ]); }); it('should parse over escaped new line in attribute values', () => { const text = ''; expect(tokenizeAndHumanizeParts(text, {escapedString: true})).toEqual([ - [lex.TokenType.TAG_OPEN_START, '', 't'], - [lex.TokenType.ATTR_NAME, '', 'a'], - [lex.TokenType.ATTR_VALUE_TEXT, 'b'], - [lex.TokenType.TAG_OPEN_END], - [lex.TokenType.TAG_CLOSE, '', 't'], - [lex.TokenType.EOF], + [TokenType.TAG_OPEN_START, '', 't'], + [TokenType.ATTR_NAME, '', 'a'], + [TokenType.ATTR_VALUE_TEXT, 'b'], + [TokenType.TAG_OPEN_END], + [TokenType.TAG_CLOSE, '', 't'], + [TokenType.EOF], ]); }); @@ -1710,12 +1711,12 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u endPos: 59, }; expect(tokenizeAndHumanizeParts(text, {range, escapedString: true})).toEqual([ - [lex.TokenType.TEXT, 'line 1\n"line 2"\nline 3'], - [lex.TokenType.EOF], + [TokenType.TEXT, 'line 1\n"line 2"\nline 3'], + [TokenType.EOF], ]); expect(tokenizeAndHumanizeSourceSpans(text, {range, escapedString: true})).toEqual([ - [lex.TokenType.TEXT, 'line 1\\n\\"line 2\\"\\nline 3'], - [lex.TokenType.EOF, ''], + [TokenType.TEXT, 'line 1\\n\\"line 2\\"\\nline 3'], + [TokenType.EOF, ''], ]); }); @@ -1725,61 +1726,57 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u 'line 3\\\n' + // <- line continuation ''; expect(tokenizeAndHumanizeParts(text, {escapedString: true})).toEqual([ - [lex.TokenType.TAG_OPEN_START, '', 't'], [lex.TokenType.TAG_OPEN_END], - [lex.TokenType.TEXT, 'line 1'], [lex.TokenType.TAG_CLOSE, '', 't'], - [lex.TokenType.TEXT, '\n'], + [TokenType.TAG_OPEN_START, '', 't'], [TokenType.TAG_OPEN_END], [TokenType.TEXT, 'line 1'], + [TokenType.TAG_CLOSE, '', 't'], [TokenType.TEXT, '\n'], - [lex.TokenType.TAG_OPEN_START, '', 't'], [lex.TokenType.TAG_OPEN_END], - [lex.TokenType.TEXT, 'line 2'], [lex.TokenType.TAG_CLOSE, '', 't'], - [lex.TokenType.TEXT, '\n'], + [TokenType.TAG_OPEN_START, '', 't'], [TokenType.TAG_OPEN_END], [TokenType.TEXT, 'line 2'], + [TokenType.TAG_CLOSE, '', 't'], [TokenType.TEXT, '\n'], - [lex.TokenType.TAG_OPEN_START, '', 't'], [lex.TokenType.TAG_OPEN_END], - [lex.TokenType.TEXT, 'line 3'], // <- line continuation does not appear in token - [lex.TokenType.TAG_CLOSE, '', 't'], + [TokenType.TAG_OPEN_START, '', 't'], [TokenType.TAG_OPEN_END], + [TokenType.TEXT, 'line 3'], // <- line continuation does not appear in token + [TokenType.TAG_CLOSE, '', 't'], - [lex.TokenType.EOF] + [TokenType.EOF] ]); expect(tokenizeAndHumanizeLineColumn(text, {escapedString: true})).toEqual([ - [lex.TokenType.TAG_OPEN_START, '0:0'], - [lex.TokenType.TAG_OPEN_END, '0:2'], - [lex.TokenType.TEXT, '0:3'], - [lex.TokenType.TAG_CLOSE, '0:9'], - [lex.TokenType.TEXT, '0:13'], // <- real newline increments the row + [TokenType.TAG_OPEN_START, '0:0'], + [TokenType.TAG_OPEN_END, '0:2'], + [TokenType.TEXT, '0:3'], + [TokenType.TAG_CLOSE, '0:9'], + [TokenType.TEXT, '0:13'], // <- real newline increments the row - [lex.TokenType.TAG_OPEN_START, '1:0'], - [lex.TokenType.TAG_OPEN_END, '1:2'], - [lex.TokenType.TEXT, '1:3'], - [lex.TokenType.TAG_CLOSE, '1:9'], - [lex.TokenType.TEXT, '1:13'], // <- escaped newline does not increment the row + [TokenType.TAG_OPEN_START, '1:0'], + [TokenType.TAG_OPEN_END, '1:2'], + [TokenType.TEXT, '1:3'], + [TokenType.TAG_CLOSE, '1:9'], + [TokenType.TEXT, '1:13'], // <- escaped newline does not increment the row - [lex.TokenType.TAG_OPEN_START, '1:15'], - [lex.TokenType.TAG_OPEN_END, '1:17'], - [lex.TokenType.TEXT, '1:18'], // <- the line continuation increments the row - [lex.TokenType.TAG_CLOSE, '2:0'], + [TokenType.TAG_OPEN_START, '1:15'], + [TokenType.TAG_OPEN_END, '1:17'], + [TokenType.TEXT, '1:18'], // <- the line continuation increments the row + [TokenType.TAG_CLOSE, '2:0'], - [lex.TokenType.EOF, '2:4'], + [TokenType.EOF, '2:4'], ]); expect(tokenizeAndHumanizeSourceSpans(text, {escapedString: true})).toEqual([ - [lex.TokenType.TAG_OPEN_START, ''], - [lex.TokenType.TEXT, 'line 1'], [lex.TokenType.TAG_CLOSE, ''], - [lex.TokenType.TEXT, '\n'], + [TokenType.TAG_OPEN_START, ''], + [TokenType.TEXT, 'line 1'], [TokenType.TAG_CLOSE, ''], [TokenType.TEXT, '\n'], - [lex.TokenType.TAG_OPEN_START, ''], - [lex.TokenType.TEXT, 'line 2'], [lex.TokenType.TAG_CLOSE, ''], - [lex.TokenType.TEXT, '\\n'], + [TokenType.TAG_OPEN_START, ''], + [TokenType.TEXT, 'line 2'], [TokenType.TAG_CLOSE, ''], [TokenType.TEXT, '\\n'], - [lex.TokenType.TAG_OPEN_START, ''], - [lex.TokenType.TEXT, 'line 3\\\n'], [lex.TokenType.TAG_CLOSE, ''], + [TokenType.TAG_OPEN_START, ''], + [TokenType.TEXT, 'line 3\\\n'], [TokenType.TAG_CLOSE, ''], - [lex.TokenType.EOF, ''] + [TokenType.EOF, ''] ]); }); }); }); } -function tokenizeWithoutErrors(input: string, options?: lex.TokenizeOptions): lex.TokenizeResult { - const tokenizeResult = lex.tokenize(input, 'someUrl', getHtmlTagDefinition, options); +function tokenizeWithoutErrors(input: string, options?: TokenizeOptions): TokenizeResult { + const tokenizeResult = tokenize(input, 'someUrl', getHtmlTagDefinition, options); if (tokenizeResult.errors.length > 0) { const errorString = tokenizeResult.errors.join('\n'); @@ -1789,15 +1786,15 @@ function tokenizeWithoutErrors(input: string, options?: lex.TokenizeOptions): le return tokenizeResult; } -function humanizeParts(tokens: lex.Token[]) { - return tokens.map(token => [token.type, ...token.parts] as [lex.TokenType, ...string[]]); +function humanizeParts(tokens: Token[]) { + return tokens.map(token => [token.type, ...token.parts]); } -function tokenizeAndHumanizeParts(input: string, options?: lex.TokenizeOptions): any[] { +function tokenizeAndHumanizeParts(input: string, options?: TokenizeOptions): any[] { return humanizeParts(tokenizeWithoutErrors(input, options).tokens); } -function tokenizeAndHumanizeSourceSpans(input: string, options?: lex.TokenizeOptions): any[] { +function tokenizeAndHumanizeSourceSpans(input: string, options?: TokenizeOptions): any[] { return tokenizeWithoutErrors(input, options) .tokens.map(token => [token.type, token.sourceSpan.toString()]); } @@ -1806,12 +1803,12 @@ function humanizeLineColumn(location: ParseLocation): string { return `${location.line}:${location.col}`; } -function tokenizeAndHumanizeLineColumn(input: string, options?: lex.TokenizeOptions): any[] { +function tokenizeAndHumanizeLineColumn(input: string, options?: TokenizeOptions): any[] { return tokenizeWithoutErrors(input, options) .tokens.map(token => [token.type, humanizeLineColumn(token.sourceSpan.start)]); } -function tokenizeAndHumanizeFullStart(input: string, options?: lex.TokenizeOptions): any[] { +function tokenizeAndHumanizeFullStart(input: string, options?: TokenizeOptions): any[] { return tokenizeWithoutErrors(input, options) .tokens.map( token => @@ -1819,7 +1816,7 @@ function tokenizeAndHumanizeFullStart(input: string, options?: lex.TokenizeOptio humanizeLineColumn(token.sourceSpan.fullStart)]); } -function tokenizeAndHumanizeErrors(input: string, options?: lex.TokenizeOptions): any[] { - return lex.tokenize(input, 'someUrl', getHtmlTagDefinition, options) +function tokenizeAndHumanizeErrors(input: string, options?: TokenizeOptions): any[] { + return tokenize(input, 'someUrl', getHtmlTagDefinition, options) .errors.map(e => [e.tokenType, e.msg, humanizeLineColumn(e.span.start)]); }