From 8b6f7ac36b03d6e5565f989ee3d8ee046b0f2f72 Mon Sep 17 00:00:00 2001 From: atscott Date: Tue, 3 Aug 2021 14:48:49 -0700 Subject: [PATCH] Revert "refactor(compiler): define interfaces for each lexer token (#42062)" (#43033) This reverts commit 9b3d4f5575bfccfbbfb943dc9689f3915dd63752. PR Close #43033 --- packages/compiler/src/i18n/i18n_parser.ts | 10 +- packages/compiler/src/ml_parser/ast.ts | 7 +- .../src/ml_parser/html_whitespaces.ts | 10 +- packages/compiler/src/ml_parser/lexer.ts | 59 +- packages/compiler/src/ml_parser/parser.ts | 159 +- packages/compiler/src/ml_parser/tokens.ts | 172 -- .../test/ml_parser/html_parser_spec.ts | 2 +- .../compiler/test/ml_parser/lexer_spec.ts | 1631 +++++++++-------- 8 files changed, 951 insertions(+), 1099 deletions(-) delete mode 100644 packages/compiler/src/ml_parser/tokens.ts diff --git a/packages/compiler/src/i18n/i18n_parser.ts b/packages/compiler/src/i18n/i18n_parser.ts index 5b4ad20e2b..dda395f873 100644 --- a/packages/compiler/src/i18n/i18n_parser.ts +++ b/packages/compiler/src/i18n/i18n_parser.ts @@ -11,7 +11,7 @@ import {Parser as ExpressionParser} from '../expression_parser/parser'; import * as html from '../ml_parser/ast'; import {getHtmlTagDefinition} from '../ml_parser/html_tags'; import {InterpolationConfig} from '../ml_parser/interpolation_config'; -import {InterpolatedAttributeToken, InterpolatedTextToken, TokenType} from '../ml_parser/tokens'; +import {Token, TokenType} from '../ml_parser/lexer'; import {ParseSourceSpan} from '../parse_util'; import * as i18n from './i18n_ast'; @@ -163,16 +163,16 @@ class _I18nVisitor implements html.Visitor { } /** - * Convert, text and interpolated tokens up into text and placeholder pieces. + * Split the, potentially interpolated, text up into text and placeholder pieces. * - * @param tokens The text and interpolated tokens. + * @param text The potentially interpolated string to be split. * @param sourceSpan The span of the whole of the `text` string. * @param context The current context of the visitor, used to compute and store placeholders. * @param previousI18n Any i18n metadata associated with this `text` from a previous pass. */ private _visitTextWithInterpolation( - tokens: (InterpolatedTextToken|InterpolatedAttributeToken)[], sourceSpan: ParseSourceSpan, - context: I18nMessageVisitorContext, previousI18n: i18n.I18nMeta|undefined): i18n.Node { + tokens: Token[], sourceSpan: ParseSourceSpan, context: I18nMessageVisitorContext, + previousI18n: i18n.I18nMeta|undefined): i18n.Node { // Return a sequence of `Text` and `Placeholder` nodes grouped in a `Container`. const nodes: i18n.Node[] = []; for (const token of tokens) { diff --git a/packages/compiler/src/ml_parser/ast.ts b/packages/compiler/src/ml_parser/ast.ts index 0d9c45f0ed..a8abcc80a6 100644 --- a/packages/compiler/src/ml_parser/ast.ts +++ b/packages/compiler/src/ml_parser/ast.ts @@ -9,7 +9,7 @@ import {AstPath} from '../ast_path'; import {I18nMeta} from '../i18n/i18n_ast'; import {ParseSourceSpan} from '../parse_util'; -import {InterpolatedAttributeToken, InterpolatedTextToken} from './tokens'; +import {Token} from './lexer'; interface BaseNode { sourceSpan: ParseSourceSpan; @@ -25,8 +25,7 @@ export abstract class NodeWithI18n implements BaseNode { export class Text extends NodeWithI18n { constructor( - public value: string, sourceSpan: ParseSourceSpan, public tokens: InterpolatedTextToken[], - i18n?: I18nMeta) { + public value: string, sourceSpan: ParseSourceSpan, public tokens: Token[], i18n?: I18nMeta) { super(sourceSpan, i18n); } override visit(visitor: Visitor, context: any): any { @@ -59,7 +58,7 @@ export class Attribute extends NodeWithI18n { constructor( public name: string, public value: string, sourceSpan: ParseSourceSpan, readonly keySpan: ParseSourceSpan|undefined, public valueSpan: ParseSourceSpan|undefined, - public valueTokens: InterpolatedAttributeToken[]|undefined, i18n: I18nMeta|undefined) { + public valueTokens: Token[]|undefined, i18n: I18nMeta|undefined) { super(sourceSpan, i18n); } override visit(visitor: Visitor, context: any): any { diff --git a/packages/compiler/src/ml_parser/html_whitespaces.ts b/packages/compiler/src/ml_parser/html_whitespaces.ts index 2fa193ec44..57045ee8dc 100644 --- a/packages/compiler/src/ml_parser/html_whitespaces.ts +++ b/packages/compiler/src/ml_parser/html_whitespaces.ts @@ -8,8 +8,8 @@ import * as html from './ast'; import {NGSP_UNICODE} from './entities'; +import {Token, TokenType} from './lexer'; import {ParseTreeResult} from './parser'; -import {TextToken, TokenType} from './tokens'; export const PRESERVE_WS_ATTR_NAME = 'ngPreserveWhitespaces'; @@ -77,8 +77,8 @@ export class WhitespaceVisitor implements html.Visitor { if (isNotBlank || hasExpansionSibling) { // Process the whitespace in the tokens of this Text node const tokens = text.tokens.map( - token => - token.type === TokenType.TEXT ? createWhitespaceProcessedTextToken(token) : token); + token => token.type === TokenType.TEXT ? createTextTokenAfterWhitespaceProcessing(token) : + token); // Process the whitespace of the value of this Text node const value = processWhitespace(text.value); return new html.Text(value, text.sourceSpan, tokens, text.i18n); @@ -100,8 +100,8 @@ export class WhitespaceVisitor implements html.Visitor { } } -function createWhitespaceProcessedTextToken({type, parts, sourceSpan}: TextToken): TextToken { - return {type, parts: [processWhitespace(parts[0])], sourceSpan}; +function createTextTokenAfterWhitespaceProcessing(token: Token): Token { + return new Token(token.type, [processWhitespace(token.parts[0])], token.sourceSpan); } function processWhitespace(text: string): string { diff --git a/packages/compiler/src/ml_parser/lexer.ts b/packages/compiler/src/ml_parser/lexer.ts index 34b16ab423..3c95825e34 100644 --- a/packages/compiler/src/ml_parser/lexer.ts +++ b/packages/compiler/src/ml_parser/lexer.ts @@ -12,7 +12,39 @@ import {NAMED_ENTITIES} from './entities'; import {DEFAULT_INTERPOLATION_CONFIG, InterpolationConfig} from './interpolation_config'; import {TagContentType, TagDefinition} from './tags'; -import {IncompleteTagOpenToken, TagOpenStartToken, Token, TokenType} from './tokens'; + +export enum TokenType { + TAG_OPEN_START, + TAG_OPEN_END, + TAG_OPEN_END_VOID, + TAG_CLOSE, + INCOMPLETE_TAG_OPEN, + TEXT, + ESCAPABLE_RAW_TEXT, + RAW_TEXT, + INTERPOLATION, + ENCODED_ENTITY, + COMMENT_START, + COMMENT_END, + CDATA_START, + CDATA_END, + ATTR_NAME, + ATTR_QUOTE, + ATTR_VALUE_TEXT, + ATTR_VALUE_INTERPOLATION, + DOC_TYPE, + EXPANSION_FORM_START, + EXPANSION_CASE_VALUE, + EXPANSION_CASE_EXP_START, + EXPANSION_CASE_EXP_END, + EXPANSION_FORM_END, + EOF +} + +export class Token { + constructor( + public type: TokenType|null, public parts: string[], public sourceSpan: ParseSourceSpan) {} +} export class TokenError extends ParseError { constructor(errorMsg: string, public tokenType: TokenType|null, span: ParseSourceSpan) { @@ -258,12 +290,9 @@ class _Tokenizer { 'Programming error - attempted to end a token which has no token type', null, this._cursor.getSpan(this._currentTokenStart)); } - const token = { - type: this._currentTokenType, - parts, - sourceSpan: - (end ?? this._cursor).getSpan(this._currentTokenStart, this._leadingTriviaCodePoints), - } as Token; + const token = new Token( + this._currentTokenType, parts, + (end ?? this._cursor).getSpan(this._currentTokenStart, this._leadingTriviaCodePoints)); this.tokens.push(token); this._currentTokenStart = null; this._currentTokenType = null; @@ -498,7 +527,7 @@ class _Tokenizer { private _consumeTagOpen(start: CharacterCursor) { let tagName: string; let prefix: string; - let openTagToken: TagOpenStartToken|IncompleteTagOpenToken|undefined; + let openTagToken: Token|undefined; try { if (!chars.isAsciiLetter(this._cursor.peek())) { throw this._createError( @@ -561,10 +590,10 @@ class _Tokenizer { this._endToken([prefix, tagName]); } - private _consumeTagOpenStart(start: CharacterCursor): TagOpenStartToken { + private _consumeTagOpenStart(start: CharacterCursor) { this._beginToken(TokenType.TAG_OPEN_START, start); const parts = this._consumePrefixAndName(); - return this._endToken(parts) as TagOpenStartToken; + return this._endToken(parts); } private _consumeAttributeName() { @@ -735,7 +764,7 @@ class _Tokenizer { */ private _consumeInterpolation( interpolationTokenType: TokenType, interpolationStart: CharacterCursor, - prematureEndPredicate: (() => boolean)|null): void { + prematureEndPredicate: (() => boolean)|null) { const parts: string[] = []; this._beginToken(interpolationTokenType, interpolationStart); parts.push(this._interpolationConfig.start); @@ -754,8 +783,7 @@ class _Tokenizer { // (This is actually wrong but here for backward compatibility). this._cursor = current; parts.push(this._getProcessedChars(expressionStart, current)); - this._endToken(parts); - return; + return this._endToken(parts); } if (inQuote === null) { @@ -763,8 +791,7 @@ class _Tokenizer { // We are not in a string, and we hit the end interpolation marker parts.push(this._getProcessedChars(expressionStart, current)); parts.push(this._interpolationConfig.end); - this._endToken(parts); - return; + return this._endToken(parts); } else if (this._attemptStr('//')) { // Once we are in a comment we ignore any quotes inComment = true; @@ -787,7 +814,7 @@ class _Tokenizer { // We hit EOF without finding a closing interpolation marker parts.push(this._getProcessedChars(expressionStart, this._cursor)); - this._endToken(parts); + return this._endToken(parts); } private _getProcessedChars(start: CharacterCursor, end: CharacterCursor): string { diff --git a/packages/compiler/src/ml_parser/parser.ts b/packages/compiler/src/ml_parser/parser.ts index ec1157c01d..d9f5d17bd3 100644 --- a/packages/compiler/src/ml_parser/parser.ts +++ b/packages/compiler/src/ml_parser/parser.ts @@ -10,9 +10,8 @@ import {ParseError, ParseLocation, ParseSourceSpan} from '../parse_util'; import * as html from './ast'; import {NAMED_ENTITIES} from './entities'; -import {tokenize, TokenizeOptions} from './lexer'; +import * as lex from './lexer'; import {getNsPrefix, mergeNsAndName, splitNsName, TagDefinition} from './tags'; -import {AttributeNameToken, AttributeQuoteToken, CdataStartToken, CommentStartToken, ExpansionCaseExpressionEndToken, ExpansionCaseExpressionStartToken, ExpansionCaseValueToken, ExpansionFormStartToken, IncompleteTagOpenToken, InterpolatedAttributeToken, InterpolatedTextToken, TagCloseToken, TagOpenStartToken, TextToken, Token, TokenType} from './tokens'; export class TreeError extends ParseError { static create(elementName: string|null, span: ParseSourceSpan, msg: string): TreeError { @@ -31,8 +30,8 @@ export class ParseTreeResult { export class Parser { constructor(public getTagDefinition: (tagName: string) => TagDefinition) {} - parse(source: string, url: string, options?: TokenizeOptions): ParseTreeResult { - const tokenizeResult = tokenize(source, url, this.getTagDefinition, options); + parse(source: string, url: string, options?: lex.TokenizeOptions): ParseTreeResult { + const tokenizeResult = lex.tokenize(source, url, this.getTagDefinition, options); const parser = new _TreeBuilder(tokenizeResult.tokens, this.getTagDefinition); parser.build(); return new ParseTreeResult( @@ -44,38 +43,38 @@ export class Parser { class _TreeBuilder { private _index: number = -1; - // `_peek` will be initialized by the call to `_advance()` in the constructor. - private _peek!: Token; + // `_peek` will be initialized by the call to `advance()` in the constructor. + private _peek!: lex.Token; private _elementStack: html.Element[] = []; rootNodes: html.Node[] = []; errors: TreeError[] = []; constructor( - private tokens: Token[], private getTagDefinition: (tagName: string) => TagDefinition) { + private tokens: lex.Token[], private getTagDefinition: (tagName: string) => TagDefinition) { this._advance(); } build(): void { - while (this._peek.type !== TokenType.EOF) { - if (this._peek.type === TokenType.TAG_OPEN_START || - this._peek.type === TokenType.INCOMPLETE_TAG_OPEN) { - this._consumeStartTag(this._advance()); - } else if (this._peek.type === TokenType.TAG_CLOSE) { - this._consumeEndTag(this._advance()); - } else if (this._peek.type === TokenType.CDATA_START) { + while (this._peek.type !== lex.TokenType.EOF) { + if (this._peek.type === lex.TokenType.TAG_OPEN_START || + this._peek.type === lex.TokenType.INCOMPLETE_TAG_OPEN) { + this._consumeStartTag(this._advance()); + } else if (this._peek.type === lex.TokenType.TAG_CLOSE) { + this._consumeEndTag(this._advance()); + } else if (this._peek.type === lex.TokenType.CDATA_START) { this._closeVoidElement(); - this._consumeCdata(this._advance()); - } else if (this._peek.type === TokenType.COMMENT_START) { + this._consumeCdata(this._advance()); + } else if (this._peek.type === lex.TokenType.COMMENT_START) { this._closeVoidElement(); - this._consumeComment(this._advance()); + this._consumeComment(this._advance()); } else if ( - this._peek.type === TokenType.TEXT || this._peek.type === TokenType.RAW_TEXT || - this._peek.type === TokenType.ESCAPABLE_RAW_TEXT) { + this._peek.type === lex.TokenType.TEXT || this._peek.type === lex.TokenType.RAW_TEXT || + this._peek.type === lex.TokenType.ESCAPABLE_RAW_TEXT) { this._closeVoidElement(); - this._consumeText(this._advance()); - } else if (this._peek.type === TokenType.EXPANSION_FORM_START) { - this._consumeExpansion(this._advance()); + this._consumeText(this._advance()); + } else if (this._peek.type === lex.TokenType.EXPANSION_FORM_START) { + this._consumeExpansion(this._advance()); } else { // Skip all other tokens... this._advance(); @@ -83,50 +82,50 @@ class _TreeBuilder { } } - private _advance(): T { + private _advance(): lex.Token { const prev = this._peek; if (this._index < this.tokens.length - 1) { // Note: there is always an EOF token at the end this._index++; } this._peek = this.tokens[this._index]; - return prev as T; + return prev; } - private _advanceIf(type: T): (Token&{type: T})|null { + private _advanceIf(type: lex.TokenType): lex.Token|null { if (this._peek.type === type) { - return this._advance(); + return this._advance(); } return null; } - private _consumeCdata(_startToken: CdataStartToken) { - this._consumeText(this._advance()); - this._advanceIf(TokenType.CDATA_END); + private _consumeCdata(_startToken: lex.Token) { + this._consumeText(this._advance()); + this._advanceIf(lex.TokenType.CDATA_END); } - private _consumeComment(token: CommentStartToken) { - const text = this._advanceIf(TokenType.RAW_TEXT); - this._advanceIf(TokenType.COMMENT_END); + private _consumeComment(token: lex.Token) { + const text = this._advanceIf(lex.TokenType.RAW_TEXT); + this._advanceIf(lex.TokenType.COMMENT_END); const value = text != null ? text.parts[0].trim() : null; this._addToParent(new html.Comment(value, token.sourceSpan)); } - private _consumeExpansion(token: ExpansionFormStartToken) { - const switchValue = this._advance(); + private _consumeExpansion(token: lex.Token) { + const switchValue = this._advance(); - const type = this._advance(); + const type = this._advance(); const cases: html.ExpansionCase[] = []; // read = - while (this._peek.type === TokenType.EXPANSION_CASE_VALUE) { + while (this._peek.type === lex.TokenType.EXPANSION_CASE_VALUE) { const expCase = this._parseExpansionCase(); if (!expCase) return; // error cases.push(expCase); } // read the final } - if (this._peek.type !== TokenType.EXPANSION_FORM_END) { + if (this._peek.type !== lex.TokenType.EXPANSION_FORM_END) { this.errors.push( TreeError.create(null, this._peek.sourceSpan, `Invalid ICU message. Missing '}'.`)); return; @@ -140,23 +139,23 @@ class _TreeBuilder { } private _parseExpansionCase(): html.ExpansionCase|null { - const value = this._advance(); + const value = this._advance(); // read { - if (this._peek.type !== TokenType.EXPANSION_CASE_EXP_START) { + if (this._peek.type !== lex.TokenType.EXPANSION_CASE_EXP_START) { this.errors.push( TreeError.create(null, this._peek.sourceSpan, `Invalid ICU message. Missing '{'.`)); return null; } // read until } - const start = this._advance(); + const start = this._advance(); const exp = this._collectExpansionExpTokens(start); if (!exp) return null; - const end = this._advance(); - exp.push({type: TokenType.EOF, parts: [], sourceSpan: end.sourceSpan}); + const end = this._advance(); + exp.push(new lex.Token(lex.TokenType.EOF, [], end.sourceSpan)); // parse everything in between { and } const expansionCaseParser = new _TreeBuilder(exp, this.getTagDefinition); @@ -174,18 +173,18 @@ class _TreeBuilder { value.parts[0], expansionCaseParser.rootNodes, sourceSpan, value.sourceSpan, expSourceSpan); } - private _collectExpansionExpTokens(start: Token): Token[]|null { - const exp: Token[] = []; - const expansionFormStack = [TokenType.EXPANSION_CASE_EXP_START]; + private _collectExpansionExpTokens(start: lex.Token): lex.Token[]|null { + const exp: lex.Token[] = []; + const expansionFormStack = [lex.TokenType.EXPANSION_CASE_EXP_START]; while (true) { - if (this._peek.type === TokenType.EXPANSION_FORM_START || - this._peek.type === TokenType.EXPANSION_CASE_EXP_START) { + if (this._peek.type === lex.TokenType.EXPANSION_FORM_START || + this._peek.type === lex.TokenType.EXPANSION_CASE_EXP_START) { expansionFormStack.push(this._peek.type); } - if (this._peek.type === TokenType.EXPANSION_CASE_EXP_END) { - if (lastOnStack(expansionFormStack, TokenType.EXPANSION_CASE_EXP_START)) { + if (this._peek.type === lex.TokenType.EXPANSION_CASE_EXP_END) { + if (lastOnStack(expansionFormStack, lex.TokenType.EXPANSION_CASE_EXP_START)) { expansionFormStack.pop(); if (expansionFormStack.length == 0) return exp; @@ -196,8 +195,8 @@ class _TreeBuilder { } } - if (this._peek.type === TokenType.EXPANSION_FORM_END) { - if (lastOnStack(expansionFormStack, TokenType.EXPANSION_FORM_START)) { + if (this._peek.type === lex.TokenType.EXPANSION_FORM_END) { + if (lastOnStack(expansionFormStack, lex.TokenType.EXPANSION_FORM_START)) { expansionFormStack.pop(); } else { this.errors.push( @@ -206,7 +205,7 @@ class _TreeBuilder { } } - if (this._peek.type === TokenType.EOF) { + if (this._peek.type === lex.TokenType.EOF) { this.errors.push( TreeError.create(null, start.sourceSpan, `Invalid ICU message. Missing '}'.`)); return null; @@ -216,7 +215,7 @@ class _TreeBuilder { } } - private _consumeText(token: InterpolatedTextToken) { + private _consumeText(token: lex.Token) { const tokens = [token]; const startSpan = token.sourceSpan; let text = token.parts[0]; @@ -225,21 +224,22 @@ class _TreeBuilder { if (parent != null && parent.children.length == 0 && this.getTagDefinition(parent.name).ignoreFirstLf) { text = text.substring(1); - tokens[0] = {type: token.type, sourceSpan: token.sourceSpan, parts: [text]} as typeof token; + tokens[0] = {type: token.type, sourceSpan: token.sourceSpan, parts: [text]}; } } - while (this._peek.type === TokenType.INTERPOLATION || this._peek.type === TokenType.TEXT || - this._peek.type === TokenType.ENCODED_ENTITY) { + while (this._peek.type === lex.TokenType.INTERPOLATION || + this._peek.type === lex.TokenType.TEXT || + this._peek.type === lex.TokenType.ENCODED_ENTITY) { token = this._advance(); tokens.push(token); - if (token.type === TokenType.INTERPOLATION) { + if (token.type === lex.TokenType.INTERPOLATION) { // For backward compatibility we decode HTML entities that appear in interpolation // expressions. This is arguably a bug, but it could be a considerable breaking change to // fix it. It should be addressed in a larger project to refactor the entire parser/lexer // chain after View Engine has been removed. text += token.parts.join('').replace(/&([^;]+);/g, decodeEntity); - } else if (token.type === TokenType.ENCODED_ENTITY) { + } else if (token.type === lex.TokenType.ENCODED_ENTITY) { text += token.parts[0]; } else { text += token.parts.join(''); @@ -262,17 +262,17 @@ class _TreeBuilder { } } - private _consumeStartTag(startTagToken: TagOpenStartToken|IncompleteTagOpenToken) { + private _consumeStartTag(startTagToken: lex.Token) { const [prefix, name] = startTagToken.parts; const attrs: html.Attribute[] = []; - while (this._peek.type === TokenType.ATTR_NAME) { - attrs.push(this._consumeAttr(this._advance())); + while (this._peek.type === lex.TokenType.ATTR_NAME) { + attrs.push(this._consumeAttr(this._advance())); } const fullName = this._getElementFullName(prefix, name, this._getParentElement()); let selfClosing = false; // Note: There could have been a tokenizer error // so that we don't get a token for the end tag... - if (this._peek.type === TokenType.TAG_OPEN_END_VOID) { + if (this._peek.type === lex.TokenType.TAG_OPEN_END_VOID) { this._advance(); selfClosing = true; const tagDef = this.getTagDefinition(fullName); @@ -281,7 +281,7 @@ class _TreeBuilder { fullName, startTagToken.sourceSpan, `Only void and foreign elements can be self closed "${startTagToken.parts[1]}"`)); } - } else if (this._peek.type === TokenType.TAG_OPEN_END) { + } else if (this._peek.type === lex.TokenType.TAG_OPEN_END) { this._advance(); selfClosing = false; } @@ -297,7 +297,7 @@ class _TreeBuilder { // Elements that are self-closed have their `endSourceSpan` set to the full span, as the // element start tag also represents the end tag. this._popElement(fullName, span); - } else if (startTagToken.type === TokenType.INCOMPLETE_TAG_OPEN) { + } else if (startTagToken.type === lex.TokenType.INCOMPLETE_TAG_OPEN) { // We already know the opening tag is not complete, so it is unlikely it has a corresponding // close tag. Let's optimistically parse it as a full element and emit an error. this._popElement(fullName, null); @@ -317,7 +317,7 @@ class _TreeBuilder { this._elementStack.push(el); } - private _consumeEndTag(endTagToken: TagCloseToken) { + private _consumeEndTag(endTagToken: lex.Token) { const fullName = this._getElementFullName( endTagToken.parts[0], endTagToken.parts[1], this._getParentElement()); @@ -363,40 +363,35 @@ class _TreeBuilder { return false; } - private _consumeAttr(attrName: AttributeNameToken): html.Attribute { + private _consumeAttr(attrName: lex.Token): html.Attribute { const fullName = mergeNsAndName(attrName.parts[0], attrName.parts[1]); let attrEnd = attrName.sourceSpan.end; // Consume any quote - if (this._peek.type === TokenType.ATTR_QUOTE) { + if (this._peek.type === lex.TokenType.ATTR_QUOTE) { this._advance(); } // Consume the attribute value let value = ''; - const valueTokens: InterpolatedAttributeToken[] = []; + const valueTokens: lex.Token[] = []; let valueStartSpan: ParseSourceSpan|undefined = undefined; let valueEnd: ParseLocation|undefined = undefined; - // NOTE: We need to use a new variable `nextTokenType` here to hide the actual type of - // `_peek.type` from TS. Otherwise TS will narrow the type of `_peek.type` preventing it from - // being able to consider `ATTR_VALUE_INTERPOLATION` as an option. This is because TS is not - // able to see that `_advance()` will actually mutate `_peek`. - const nextTokenType = this._peek.type; - if (nextTokenType === TokenType.ATTR_VALUE_TEXT) { + if (this._peek.type === lex.TokenType.ATTR_VALUE_TEXT) { valueStartSpan = this._peek.sourceSpan; valueEnd = this._peek.sourceSpan.end; - while (this._peek.type === TokenType.ATTR_VALUE_TEXT || - this._peek.type === TokenType.ATTR_VALUE_INTERPOLATION || - this._peek.type === TokenType.ENCODED_ENTITY) { - const valueToken = this._advance(); + while (this._peek.type === lex.TokenType.ATTR_VALUE_TEXT || + this._peek.type === lex.TokenType.ATTR_VALUE_INTERPOLATION || + this._peek.type === lex.TokenType.ENCODED_ENTITY) { + const valueToken = this._advance(); valueTokens.push(valueToken); - if (valueToken.type === TokenType.ATTR_VALUE_INTERPOLATION) { + if (valueToken.type === lex.TokenType.ATTR_VALUE_INTERPOLATION) { // For backward compatibility we decode HTML entities that appear in interpolation // expressions. This is arguably a bug, but it could be a considerable breaking change to // fix it. It should be addressed in a larger project to refactor the entire parser/lexer // chain after View Engine has been removed. value += valueToken.parts.join('').replace(/&([^;]+);/g, decodeEntity); - } else if (valueToken.type === TokenType.ENCODED_ENTITY) { + } else if (valueToken.type === lex.TokenType.ENCODED_ENTITY) { value += valueToken.parts[0]; } else { value += valueToken.parts.join(''); @@ -406,8 +401,8 @@ class _TreeBuilder { } // Consume any quote - if (this._peek.type === TokenType.ATTR_QUOTE) { - const quoteToken = this._advance(); + if (this._peek.type === lex.TokenType.ATTR_QUOTE) { + const quoteToken = this._advance(); attrEnd = quoteToken.sourceSpan.end; } diff --git a/packages/compiler/src/ml_parser/tokens.ts b/packages/compiler/src/ml_parser/tokens.ts deleted file mode 100644 index 643dc25003..0000000000 --- a/packages/compiler/src/ml_parser/tokens.ts +++ /dev/null @@ -1,172 +0,0 @@ -/** - * @license - * Copyright Google LLC All Rights Reserved. - * - * Use of this source code is governed by an MIT-style license that can be - * found in the LICENSE file at https://angular.io/license - */ - -import {ParseSourceSpan} from '../parse_util'; - -export const enum TokenType { - TAG_OPEN_START, - TAG_OPEN_END, - TAG_OPEN_END_VOID, - TAG_CLOSE, - INCOMPLETE_TAG_OPEN, - TEXT, - ESCAPABLE_RAW_TEXT, - RAW_TEXT, - INTERPOLATION, - ENCODED_ENTITY, - COMMENT_START, - COMMENT_END, - CDATA_START, - CDATA_END, - ATTR_NAME, - ATTR_QUOTE, - ATTR_VALUE_TEXT, - ATTR_VALUE_INTERPOLATION, - DOC_TYPE, - EXPANSION_FORM_START, - EXPANSION_CASE_VALUE, - EXPANSION_CASE_EXP_START, - EXPANSION_CASE_EXP_END, - EXPANSION_FORM_END, - EOF -} - -export type Token = TagOpenStartToken|TagOpenEndToken|TagOpenEndVoidToken|TagCloseToken| - IncompleteTagOpenToken|TextToken|InterpolationToken|EncodedEntityToken|CommentStartToken| - CommentEndToken|CdataStartToken|CdataEndToken|AttributeNameToken|AttributeQuoteToken| - AttributeValueTextToken|AttributeValueInterpolationToken|DocTypeToken|ExpansionFormStartToken| - ExpansionCaseValueToken|ExpansionCaseExpressionStartToken|ExpansionCaseExpressionEndToken| - ExpansionFormEndToken|EndOfFileToken; - -export type InterpolatedTextToken = TextToken|InterpolationToken|EncodedEntityToken; - -export type InterpolatedAttributeToken = - AttributeValueTextToken|AttributeValueInterpolationToken|EncodedEntityToken; - -export interface TokenBase { - type: TokenType; - parts: string[]; - sourceSpan: ParseSourceSpan; -} - -export interface TagOpenStartToken extends TokenBase { - type: TokenType.TAG_OPEN_START; - parts: [prefix: string, name: string]; -} - -export interface TagOpenEndToken extends TokenBase { - type: TokenType.TAG_OPEN_END; - parts: []; -} - -export interface TagOpenEndVoidToken extends TokenBase { - type: TokenType.TAG_OPEN_END_VOID; - parts: []; -} - -export interface TagCloseToken extends TokenBase { - type: TokenType.TAG_CLOSE; - parts: [prefix: string, name: string]; -} - -export interface IncompleteTagOpenToken extends TokenBase { - type: TokenType.INCOMPLETE_TAG_OPEN; - parts: [prefix: string, name: string]; -} - -export interface TextToken extends TokenBase { - type: TokenType.TEXT|TokenType.ESCAPABLE_RAW_TEXT|TokenType.RAW_TEXT; - parts: [text: string]; -} - -export interface InterpolationToken extends TokenBase { - type: TokenType.INTERPOLATION; - parts: [startMarker: string, expression: string, endMarker: string]| - [startMarker: string, expression: string]; -} - -export interface EncodedEntityToken extends TokenBase { - type: TokenType.ENCODED_ENTITY; - parts: [decoded: string, encoded: string]; -} - -export interface CommentStartToken extends TokenBase { - type: TokenType.COMMENT_START; - parts: []; -} - -export interface CommentEndToken extends TokenBase { - type: TokenType.COMMENT_END; - parts: []; -} - -export interface CdataStartToken extends TokenBase { - type: TokenType.CDATA_START; - parts: []; -} - -export interface CdataEndToken extends TokenBase { - type: TokenType.CDATA_END; - parts: []; -} - -export interface AttributeNameToken extends TokenBase { - type: TokenType.ATTR_NAME; - parts: [prefix: string, name: string]; -} - -export interface AttributeQuoteToken extends TokenBase { - type: TokenType.ATTR_QUOTE; - parts: [quote: '\''|'"']; -} - -export interface AttributeValueTextToken extends TokenBase { - type: TokenType.ATTR_VALUE_TEXT; - parts: [value: string]; -} - -export interface AttributeValueInterpolationToken extends TokenBase { - type: TokenType.ATTR_VALUE_INTERPOLATION; - parts: [startMarker: string, expression: string, endMarker: string]| - [startMarker: string, expression: string]; -} - -export interface DocTypeToken extends TokenBase { - type: TokenType.DOC_TYPE; - parts: [content: string]; -} - -export interface ExpansionFormStartToken extends TokenBase { - type: TokenType.EXPANSION_FORM_START; - parts: []; -} - -export interface ExpansionCaseValueToken extends TokenBase { - type: TokenType.EXPANSION_CASE_VALUE; - parts: [value: string]; -} - -export interface ExpansionCaseExpressionStartToken extends TokenBase { - type: TokenType.EXPANSION_CASE_EXP_START; - parts: []; -} - -export interface ExpansionCaseExpressionEndToken extends TokenBase { - type: TokenType.EXPANSION_CASE_EXP_END; - parts: []; -} - -export interface ExpansionFormEndToken extends TokenBase { - type: TokenType.EXPANSION_FORM_END; - parts: []; -} - -export interface EndOfFileToken extends TokenBase { - type: TokenType.EOF; - parts: []; -} diff --git a/packages/compiler/test/ml_parser/html_parser_spec.ts b/packages/compiler/test/ml_parser/html_parser_spec.ts index 45dcc1d957..01562f53cf 100644 --- a/packages/compiler/test/ml_parser/html_parser_spec.ts +++ b/packages/compiler/test/ml_parser/html_parser_spec.ts @@ -8,7 +8,7 @@ import * as html from '../../src/ml_parser/ast'; import {HtmlParser, ParseTreeResult, TreeError} from '../../src/ml_parser/html_parser'; -import {TokenType} from '../../src/ml_parser/tokens'; +import {TokenType} from '../../src/ml_parser/lexer'; import {ParseError} from '../../src/parse_util'; import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes} from './ast_spec_utils'; diff --git a/packages/compiler/test/ml_parser/lexer_spec.ts b/packages/compiler/test/ml_parser/lexer_spec.ts index 5f6e7c317d..c0302a3345 100644 --- a/packages/compiler/test/ml_parser/lexer_spec.ts +++ b/packages/compiler/test/ml_parser/lexer_spec.ts @@ -7,8 +7,7 @@ */ import {getHtmlTagDefinition} from '../../src/ml_parser/html_tags'; -import {TokenError, tokenize, TokenizeOptions, TokenizeResult} from '../../src/ml_parser/lexer'; -import {Token, TokenType} from '../../src/ml_parser/tokens'; +import * as lex from '../../src/ml_parser/lexer'; import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_util'; { @@ -16,41 +15,41 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u describe('line/column numbers', () => { it('should work without newlines', () => { expect(tokenizeAndHumanizeLineColumn('a')).toEqual([ - [TokenType.TAG_OPEN_START, '0:0'], - [TokenType.TAG_OPEN_END, '0:2'], - [TokenType.TEXT, '0:3'], - [TokenType.TAG_CLOSE, '0:4'], - [TokenType.EOF, '0:8'], + [lex.TokenType.TAG_OPEN_START, '0:0'], + [lex.TokenType.TAG_OPEN_END, '0:2'], + [lex.TokenType.TEXT, '0:3'], + [lex.TokenType.TAG_CLOSE, '0:4'], + [lex.TokenType.EOF, '0:8'], ]); }); it('should work with one newline', () => { expect(tokenizeAndHumanizeLineColumn('\na')).toEqual([ - [TokenType.TAG_OPEN_START, '0:0'], - [TokenType.TAG_OPEN_END, '0:2'], - [TokenType.TEXT, '0:3'], - [TokenType.TAG_CLOSE, '1:1'], - [TokenType.EOF, '1:5'], + [lex.TokenType.TAG_OPEN_START, '0:0'], + [lex.TokenType.TAG_OPEN_END, '0:2'], + [lex.TokenType.TEXT, '0:3'], + [lex.TokenType.TAG_CLOSE, '1:1'], + [lex.TokenType.EOF, '1:5'], ]); }); it('should work with multiple newlines', () => { expect(tokenizeAndHumanizeLineColumn('\na')).toEqual([ - [TokenType.TAG_OPEN_START, '0:0'], - [TokenType.TAG_OPEN_END, '1:0'], - [TokenType.TEXT, '1:1'], - [TokenType.TAG_CLOSE, '2:1'], - [TokenType.EOF, '2:5'], + [lex.TokenType.TAG_OPEN_START, '0:0'], + [lex.TokenType.TAG_OPEN_END, '1:0'], + [lex.TokenType.TEXT, '1:1'], + [lex.TokenType.TAG_CLOSE, '2:1'], + [lex.TokenType.EOF, '2:5'], ]); }); it('should work with CR and LF', () => { expect(tokenizeAndHumanizeLineColumn('\r\na\r')).toEqual([ - [TokenType.TAG_OPEN_START, '0:0'], - [TokenType.TAG_OPEN_END, '1:0'], - [TokenType.TEXT, '1:1'], - [TokenType.TAG_CLOSE, '2:1'], - [TokenType.EOF, '2:5'], + [lex.TokenType.TAG_OPEN_START, '0:0'], + [lex.TokenType.TAG_OPEN_END, '1:0'], + [lex.TokenType.TEXT, '1:1'], + [lex.TokenType.TAG_CLOSE, '2:1'], + [lex.TokenType.EOF, '2:5'], ]); }); @@ -58,11 +57,11 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u expect( tokenizeAndHumanizeFullStart('\n \t a', {leadingTriviaChars: ['\n', ' ', '\t']})) .toEqual([ - [TokenType.TAG_OPEN_START, '0:0', '0:0'], - [TokenType.TAG_OPEN_END, '0:2', '0:2'], - [TokenType.TEXT, '1:3', '0:3'], - [TokenType.TAG_CLOSE, '1:4', '1:4'], - [TokenType.EOF, '1:8', '1:8'], + [lex.TokenType.TAG_OPEN_START, '0:0', '0:0'], + [lex.TokenType.TAG_OPEN_END, '0:2', '0:2'], + [lex.TokenType.TEXT, '1:3', '0:3'], + [lex.TokenType.TAG_CLOSE, '1:4', '1:4'], + [lex.TokenType.EOF, '1:8', '1:8'], ]); }); }); @@ -73,8 +72,8 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u 'pre 1\npre 2\npre 3 `line 1\nline 2\nline 3` post 1\n post 2\n post 3', {range: {startPos: 19, startLine: 2, startCol: 7, endPos: 39}})) .toEqual([ - [TokenType.TEXT, 'line 1\nline 2\nline 3'], - [TokenType.EOF, ''], + [lex.TokenType.TEXT, 'line 1\nline 2\nline 3'], + [lex.TokenType.EOF, ''], ]); }); @@ -83,8 +82,8 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u 'pre 1\npre 2\npre 3 `line 1\nline 2\nline 3` post 1\n post 2\n post 3', {range: {startPos: 19, startLine: 2, startCol: 7, endPos: 39}})) .toEqual([ - [TokenType.TEXT, '2:7'], - [TokenType.EOF, '4:6'], + [lex.TokenType.TEXT, '2:7'], + [lex.TokenType.EOF, '4:6'], ]); }); }); @@ -92,49 +91,49 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u describe('comments', () => { it('should parse comments', () => { expect(tokenizeAndHumanizeParts('')).toEqual([ - [TokenType.COMMENT_START], - [TokenType.RAW_TEXT, 't\ne\ns\nt'], - [TokenType.COMMENT_END], - [TokenType.EOF], + [lex.TokenType.COMMENT_START], + [lex.TokenType.RAW_TEXT, 't\ne\ns\nt'], + [lex.TokenType.COMMENT_END], + [lex.TokenType.EOF], ]); }); it('should store the locations', () => { expect(tokenizeAndHumanizeSourceSpans('')).toEqual([ - [TokenType.COMMENT_START, ''], - [TokenType.EOF, ''], + [lex.TokenType.COMMENT_START, ''], + [lex.TokenType.EOF, ''], ]); }); it('should report { expect(tokenizeAndHumanizeErrors(' { expect(tokenizeAndHumanizeErrors('')).toEqual([ - [TokenType.COMMENT_START, ''], - [TokenType.EOF, ''], + [lex.TokenType.COMMENT_START, ''], + [lex.TokenType.EOF, ''], ]); }); it('should accept comments finishing by too many dashes (odd number)', () => { expect(tokenizeAndHumanizeSourceSpans('')).toEqual([ - [TokenType.COMMENT_START, ''], - [TokenType.EOF, ''], + [lex.TokenType.COMMENT_START, ''], + [lex.TokenType.EOF, ''], ]); }); }); @@ -142,21 +141,21 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u describe('doctype', () => { it('should parse doctypes', () => { expect(tokenizeAndHumanizeParts('')).toEqual([ - [TokenType.DOC_TYPE, 'doctype html'], - [TokenType.EOF], + [lex.TokenType.DOC_TYPE, 'doctype html'], + [lex.TokenType.EOF], ]); }); it('should store the locations', () => { expect(tokenizeAndHumanizeSourceSpans('')).toEqual([ - [TokenType.DOC_TYPE, ''], - [TokenType.EOF, ''], + [lex.TokenType.DOC_TYPE, ''], + [lex.TokenType.EOF, ''], ]); }); it('should report missing end doctype', () => { expect(tokenizeAndHumanizeErrors(' { it('should parse CDATA', () => { expect(tokenizeAndHumanizeParts('')).toEqual([ - [TokenType.CDATA_START], - [TokenType.RAW_TEXT, 't\ne\ns\nt'], - [TokenType.CDATA_END], - [TokenType.EOF], + [lex.TokenType.CDATA_START], + [lex.TokenType.RAW_TEXT, 't\ne\ns\nt'], + [lex.TokenType.CDATA_END], + [lex.TokenType.EOF], ]); }); it('should store the locations', () => { expect(tokenizeAndHumanizeSourceSpans('')).toEqual([ - [TokenType.CDATA_START, ''], - [TokenType.EOF, ''], + [lex.TokenType.CDATA_START, ''], + [lex.TokenType.EOF, ''], ]); }); it('should report { expect(tokenizeAndHumanizeErrors(' { expect(tokenizeAndHumanizeErrors(' { it('should parse open tags without prefix', () => { expect(tokenizeAndHumanizeParts('')).toEqual([ - [TokenType.TAG_OPEN_START, '', 'test'], - [TokenType.TAG_OPEN_END], - [TokenType.EOF], + [lex.TokenType.TAG_OPEN_START, '', 'test'], + [lex.TokenType.TAG_OPEN_END], + [lex.TokenType.EOF], ]); }); it('should parse namespace prefix', () => { expect(tokenizeAndHumanizeParts('')).toEqual([ - [TokenType.TAG_OPEN_START, 'ns1', 'test'], - [TokenType.TAG_OPEN_END], - [TokenType.EOF], + [lex.TokenType.TAG_OPEN_START, 'ns1', 'test'], + [lex.TokenType.TAG_OPEN_END], + [lex.TokenType.EOF], ]); }); it('should parse void tags', () => { expect(tokenizeAndHumanizeParts('')).toEqual([ - [TokenType.TAG_OPEN_START, '', 'test'], - [TokenType.TAG_OPEN_END_VOID], - [TokenType.EOF], + [lex.TokenType.TAG_OPEN_START, '', 'test'], + [lex.TokenType.TAG_OPEN_END_VOID], + [lex.TokenType.EOF], ]); }); it('should allow whitespace after the tag name', () => { expect(tokenizeAndHumanizeParts('')).toEqual([ - [TokenType.TAG_OPEN_START, '', 'test'], - [TokenType.TAG_OPEN_END], - [TokenType.EOF], + [lex.TokenType.TAG_OPEN_START, '', 'test'], + [lex.TokenType.TAG_OPEN_END], + [lex.TokenType.EOF], ]); }); it('should store the locations', () => { expect(tokenizeAndHumanizeSourceSpans('')).toEqual([ - [TokenType.TAG_OPEN_START, ''], - [TokenType.EOF, ''], + [lex.TokenType.TAG_OPEN_START, ''], + [lex.TokenType.EOF, ''], ]); }); describe('tags', () => { it('terminated with EOF', () => { expect(tokenizeAndHumanizeSourceSpans(' { expect(tokenizeAndHumanizeSourceSpans('')).toEqual([ - [TokenType.INCOMPLETE_TAG_OPEN, ''], - [TokenType.INCOMPLETE_TAG_OPEN, ''], - [TokenType.EOF, ''], + [lex.TokenType.INCOMPLETE_TAG_OPEN, ''], + [lex.TokenType.INCOMPLETE_TAG_OPEN, ''], + [lex.TokenType.EOF, ''], ]); }); it('in attribute', () => { expect(tokenizeAndHumanizeSourceSpans('
')).toEqual([ - [TokenType.INCOMPLETE_TAG_OPEN, ''], - [TokenType.TAG_CLOSE, ''], - [TokenType.EOF, ''], + [lex.TokenType.INCOMPLETE_TAG_OPEN, ''], + [lex.TokenType.TAG_CLOSE, ''], + [lex.TokenType.EOF, ''], ]); }); it('after quote', () => { expect(tokenizeAndHumanizeSourceSpans('
')).toEqual([ - [TokenType.INCOMPLETE_TAG_OPEN, ''], - [TokenType.TAG_CLOSE, ''], - [TokenType.EOF, ''], + [lex.TokenType.INCOMPLETE_TAG_OPEN, ''], + [lex.TokenType.TAG_CLOSE, ''], + [lex.TokenType.EOF, ''], ]); }); }); @@ -284,226 +283,226 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u describe('attributes', () => { it('should parse attributes without prefix', () => { expect(tokenizeAndHumanizeParts('')).toEqual([ - [TokenType.TAG_OPEN_START, '', 't'], - [TokenType.ATTR_NAME, '', 'a'], - [TokenType.TAG_OPEN_END], - [TokenType.EOF], + [lex.TokenType.TAG_OPEN_START, '', 't'], + [lex.TokenType.ATTR_NAME, '', 'a'], + [lex.TokenType.TAG_OPEN_END], + [lex.TokenType.EOF], ]); }); it('should parse attributes with interpolation', () => { expect(tokenizeAndHumanizeParts('')).toEqual([ - [TokenType.TAG_OPEN_START, '', 't'], - [TokenType.ATTR_NAME, '', 'a'], - [TokenType.ATTR_QUOTE, '"'], - [TokenType.ATTR_VALUE_TEXT, ''], - [TokenType.ATTR_VALUE_INTERPOLATION, '{{', 'v', '}}'], - [TokenType.ATTR_VALUE_TEXT, ''], - [TokenType.ATTR_QUOTE, '"'], - [TokenType.ATTR_NAME, '', 'b'], - [TokenType.ATTR_QUOTE, '"'], - [TokenType.ATTR_VALUE_TEXT, 's'], - [TokenType.ATTR_VALUE_INTERPOLATION, '{{', 'm', '}}'], - [TokenType.ATTR_VALUE_TEXT, 'e'], - [TokenType.ATTR_QUOTE, '"'], - [TokenType.ATTR_NAME, '', 'c'], - [TokenType.ATTR_QUOTE, '"'], - [TokenType.ATTR_VALUE_TEXT, 's'], - [TokenType.ATTR_VALUE_INTERPOLATION, '{{', 'm//c', '}}'], - [TokenType.ATTR_VALUE_TEXT, 'e'], - [TokenType.ATTR_QUOTE, '"'], - [TokenType.TAG_OPEN_END], - [TokenType.EOF], + [lex.TokenType.TAG_OPEN_START, '', 't'], + [lex.TokenType.ATTR_NAME, '', 'a'], + [lex.TokenType.ATTR_QUOTE, '"'], + [lex.TokenType.ATTR_VALUE_TEXT, ''], + [lex.TokenType.ATTR_VALUE_INTERPOLATION, '{{', 'v', '}}'], + [lex.TokenType.ATTR_VALUE_TEXT, ''], + [lex.TokenType.ATTR_QUOTE, '"'], + [lex.TokenType.ATTR_NAME, '', 'b'], + [lex.TokenType.ATTR_QUOTE, '"'], + [lex.TokenType.ATTR_VALUE_TEXT, 's'], + [lex.TokenType.ATTR_VALUE_INTERPOLATION, '{{', 'm', '}}'], + [lex.TokenType.ATTR_VALUE_TEXT, 'e'], + [lex.TokenType.ATTR_QUOTE, '"'], + [lex.TokenType.ATTR_NAME, '', 'c'], + [lex.TokenType.ATTR_QUOTE, '"'], + [lex.TokenType.ATTR_VALUE_TEXT, 's'], + [lex.TokenType.ATTR_VALUE_INTERPOLATION, '{{', 'm//c', '}}'], + [lex.TokenType.ATTR_VALUE_TEXT, 'e'], + [lex.TokenType.ATTR_QUOTE, '"'], + [lex.TokenType.TAG_OPEN_END], + [lex.TokenType.EOF], ]); }); it('should end interpolation on an unescaped matching quote', () => { expect(tokenizeAndHumanizeParts('')).toEqual([ - [TokenType.TAG_OPEN_START, '', 't'], - [TokenType.ATTR_NAME, '', 'a'], - [TokenType.ATTR_QUOTE, '"'], - [TokenType.ATTR_VALUE_TEXT, ''], - [TokenType.ATTR_VALUE_INTERPOLATION, '{{', ' a \\" \' b '], - [TokenType.ATTR_VALUE_TEXT, ''], - [TokenType.ATTR_QUOTE, '"'], - [TokenType.TAG_OPEN_END], - [TokenType.EOF], + [lex.TokenType.TAG_OPEN_START, '', 't'], + [lex.TokenType.ATTR_NAME, '', 'a'], + [lex.TokenType.ATTR_QUOTE, '"'], + [lex.TokenType.ATTR_VALUE_TEXT, ''], + [lex.TokenType.ATTR_VALUE_INTERPOLATION, '{{', ' a \\" \' b '], + [lex.TokenType.ATTR_VALUE_TEXT, ''], + [lex.TokenType.ATTR_QUOTE, '"'], + [lex.TokenType.TAG_OPEN_END], + [lex.TokenType.EOF], ]); expect(tokenizeAndHumanizeParts('')).toEqual([ - [TokenType.TAG_OPEN_START, '', 't'], - [TokenType.ATTR_NAME, '', 'a'], - [TokenType.ATTR_QUOTE, '\''], - [TokenType.ATTR_VALUE_TEXT, ''], - [TokenType.ATTR_VALUE_INTERPOLATION, '{{', ' a " \\\' b '], - [TokenType.ATTR_VALUE_TEXT, ''], - [TokenType.ATTR_QUOTE, '\''], - [TokenType.TAG_OPEN_END], - [TokenType.EOF], + [lex.TokenType.TAG_OPEN_START, '', 't'], + [lex.TokenType.ATTR_NAME, '', 'a'], + [lex.TokenType.ATTR_QUOTE, '\''], + [lex.TokenType.ATTR_VALUE_TEXT, ''], + [lex.TokenType.ATTR_VALUE_INTERPOLATION, '{{', ' a " \\\' b '], + [lex.TokenType.ATTR_VALUE_TEXT, ''], + [lex.TokenType.ATTR_QUOTE, '\''], + [lex.TokenType.TAG_OPEN_END], + [lex.TokenType.EOF], ]); }); it('should parse attributes with prefix', () => { expect(tokenizeAndHumanizeParts('')).toEqual([ - [TokenType.TAG_OPEN_START, '', 't'], - [TokenType.ATTR_NAME, 'ns1', 'a'], - [TokenType.TAG_OPEN_END], - [TokenType.EOF], + [lex.TokenType.TAG_OPEN_START, '', 't'], + [lex.TokenType.ATTR_NAME, 'ns1', 'a'], + [lex.TokenType.TAG_OPEN_END], + [lex.TokenType.EOF], ]); }); it('should parse attributes whose prefix is not valid', () => { expect(tokenizeAndHumanizeParts('')).toEqual([ - [TokenType.TAG_OPEN_START, '', 't'], - [TokenType.ATTR_NAME, '', '(ns1:a)'], - [TokenType.TAG_OPEN_END], - [TokenType.EOF], + [lex.TokenType.TAG_OPEN_START, '', 't'], + [lex.TokenType.ATTR_NAME, '', '(ns1:a)'], + [lex.TokenType.TAG_OPEN_END], + [lex.TokenType.EOF], ]); }); it('should parse attributes with single quote value', () => { expect(tokenizeAndHumanizeParts('')).toEqual([ - [TokenType.TAG_OPEN_START, '', 't'], - [TokenType.ATTR_NAME, '', 'a'], - [TokenType.ATTR_QUOTE, '\''], - [TokenType.ATTR_VALUE_TEXT, 'b'], - [TokenType.ATTR_QUOTE, '\''], - [TokenType.TAG_OPEN_END], - [TokenType.EOF], + [lex.TokenType.TAG_OPEN_START, '', 't'], + [lex.TokenType.ATTR_NAME, '', 'a'], + [lex.TokenType.ATTR_QUOTE, '\''], + [lex.TokenType.ATTR_VALUE_TEXT, 'b'], + [lex.TokenType.ATTR_QUOTE, '\''], + [lex.TokenType.TAG_OPEN_END], + [lex.TokenType.EOF], ]); }); it('should parse attributes with double quote value', () => { expect(tokenizeAndHumanizeParts('')).toEqual([ - [TokenType.TAG_OPEN_START, '', 't'], - [TokenType.ATTR_NAME, '', 'a'], - [TokenType.ATTR_QUOTE, '"'], - [TokenType.ATTR_VALUE_TEXT, 'b'], - [TokenType.ATTR_QUOTE, '"'], - [TokenType.TAG_OPEN_END], - [TokenType.EOF], + [lex.TokenType.TAG_OPEN_START, '', 't'], + [lex.TokenType.ATTR_NAME, '', 'a'], + [lex.TokenType.ATTR_QUOTE, '"'], + [lex.TokenType.ATTR_VALUE_TEXT, 'b'], + [lex.TokenType.ATTR_QUOTE, '"'], + [lex.TokenType.TAG_OPEN_END], + [lex.TokenType.EOF], ]); }); it('should parse attributes with unquoted value', () => { expect(tokenizeAndHumanizeParts('')).toEqual([ - [TokenType.TAG_OPEN_START, '', 't'], - [TokenType.ATTR_NAME, '', 'a'], - [TokenType.ATTR_VALUE_TEXT, 'b'], - [TokenType.TAG_OPEN_END], - [TokenType.EOF], + [lex.TokenType.TAG_OPEN_START, '', 't'], + [lex.TokenType.ATTR_NAME, '', 'a'], + [lex.TokenType.ATTR_VALUE_TEXT, 'b'], + [lex.TokenType.TAG_OPEN_END], + [lex.TokenType.EOF], ]); }); it('should parse attributes with unquoted interpolation value', () => { expect(tokenizeAndHumanizeParts('')).toEqual([ - [TokenType.TAG_OPEN_START, '', 'a'], - [TokenType.ATTR_NAME, '', 'a'], - [TokenType.ATTR_VALUE_TEXT, ''], - [TokenType.ATTR_VALUE_INTERPOLATION, '{{', 'link.text', '}}'], - [TokenType.ATTR_VALUE_TEXT, ''], - [TokenType.TAG_OPEN_END], - [TokenType.EOF], + [lex.TokenType.TAG_OPEN_START, '', 'a'], + [lex.TokenType.ATTR_NAME, '', 'a'], + [lex.TokenType.ATTR_VALUE_TEXT, ''], + [lex.TokenType.ATTR_VALUE_INTERPOLATION, '{{', 'link.text', '}}'], + [lex.TokenType.ATTR_VALUE_TEXT, ''], + [lex.TokenType.TAG_OPEN_END], + [lex.TokenType.EOF], ]); }); it('should parse attributes with empty quoted value', () => { expect(tokenizeAndHumanizeParts('')).toEqual([ - [TokenType.TAG_OPEN_START, '', 't'], - [TokenType.ATTR_NAME, '', 'a'], - [TokenType.ATTR_QUOTE, '"'], - [TokenType.ATTR_VALUE_TEXT, ''], - [TokenType.ATTR_QUOTE, '"'], - [TokenType.TAG_OPEN_END], - [TokenType.EOF], + [lex.TokenType.TAG_OPEN_START, '', 't'], + [lex.TokenType.ATTR_NAME, '', 'a'], + [lex.TokenType.ATTR_QUOTE, '"'], + [lex.TokenType.ATTR_VALUE_TEXT, ''], + [lex.TokenType.ATTR_QUOTE, '"'], + [lex.TokenType.TAG_OPEN_END], + [lex.TokenType.EOF], ]); }); it('should allow whitespace', () => { expect(tokenizeAndHumanizeParts('')).toEqual([ - [TokenType.TAG_OPEN_START, '', 't'], - [TokenType.ATTR_NAME, '', 'a'], - [TokenType.ATTR_VALUE_TEXT, 'b'], - [TokenType.TAG_OPEN_END], - [TokenType.EOF], + [lex.TokenType.TAG_OPEN_START, '', 't'], + [lex.TokenType.ATTR_NAME, '', 'a'], + [lex.TokenType.ATTR_VALUE_TEXT, 'b'], + [lex.TokenType.TAG_OPEN_END], + [lex.TokenType.EOF], ]); }); it('should parse attributes with entities in values', () => { expect(tokenizeAndHumanizeParts('')).toEqual([ - [TokenType.TAG_OPEN_START, '', 't'], - [TokenType.ATTR_NAME, '', 'a'], - [TokenType.ATTR_QUOTE, '"'], - [TokenType.ATTR_VALUE_TEXT, ''], - [TokenType.ENCODED_ENTITY, 'A', 'A'], - [TokenType.ATTR_VALUE_TEXT, ''], - [TokenType.ENCODED_ENTITY, 'A', 'A'], - [TokenType.ATTR_VALUE_TEXT, ''], - [TokenType.ATTR_QUOTE, '"'], - [TokenType.TAG_OPEN_END], - [TokenType.EOF], + [lex.TokenType.TAG_OPEN_START, '', 't'], + [lex.TokenType.ATTR_NAME, '', 'a'], + [lex.TokenType.ATTR_QUOTE, '"'], + [lex.TokenType.ATTR_VALUE_TEXT, ''], + [lex.TokenType.ENCODED_ENTITY, 'A', 'A'], + [lex.TokenType.ATTR_VALUE_TEXT, ''], + [lex.TokenType.ENCODED_ENTITY, 'A', 'A'], + [lex.TokenType.ATTR_VALUE_TEXT, ''], + [lex.TokenType.ATTR_QUOTE, '"'], + [lex.TokenType.TAG_OPEN_END], + [lex.TokenType.EOF], ]); }); it('should not decode entities without trailing ";"', () => { expect(tokenizeAndHumanizeParts('')).toEqual([ - [TokenType.TAG_OPEN_START, '', 't'], - [TokenType.ATTR_NAME, '', 'a'], - [TokenType.ATTR_QUOTE, '"'], - [TokenType.ATTR_VALUE_TEXT, '&'], - [TokenType.ATTR_QUOTE, '"'], - [TokenType.ATTR_NAME, '', 'b'], - [TokenType.ATTR_QUOTE, '"'], - [TokenType.ATTR_VALUE_TEXT, 'c&&d'], - [TokenType.ATTR_QUOTE, '"'], - [TokenType.TAG_OPEN_END], - [TokenType.EOF], + [lex.TokenType.TAG_OPEN_START, '', 't'], + [lex.TokenType.ATTR_NAME, '', 'a'], + [lex.TokenType.ATTR_QUOTE, '"'], + [lex.TokenType.ATTR_VALUE_TEXT, '&'], + [lex.TokenType.ATTR_QUOTE, '"'], + [lex.TokenType.ATTR_NAME, '', 'b'], + [lex.TokenType.ATTR_QUOTE, '"'], + [lex.TokenType.ATTR_VALUE_TEXT, 'c&&d'], + [lex.TokenType.ATTR_QUOTE, '"'], + [lex.TokenType.TAG_OPEN_END], + [lex.TokenType.EOF], ]); }); it('should parse attributes with "&" in values', () => { expect(tokenizeAndHumanizeParts('')).toEqual([ - [TokenType.TAG_OPEN_START, '', 't'], - [TokenType.ATTR_NAME, '', 'a'], - [TokenType.ATTR_QUOTE, '"'], - [TokenType.ATTR_VALUE_TEXT, 'b && c &'], - [TokenType.ATTR_QUOTE, '"'], - [TokenType.TAG_OPEN_END], - [TokenType.EOF], + [lex.TokenType.TAG_OPEN_START, '', 't'], + [lex.TokenType.ATTR_NAME, '', 'a'], + [lex.TokenType.ATTR_QUOTE, '"'], + [lex.TokenType.ATTR_VALUE_TEXT, 'b && c &'], + [lex.TokenType.ATTR_QUOTE, '"'], + [lex.TokenType.TAG_OPEN_END], + [lex.TokenType.EOF], ]); }); it('should parse values with CR and LF', () => { expect(tokenizeAndHumanizeParts('')).toEqual([ - [TokenType.TAG_OPEN_START, '', 't'], - [TokenType.ATTR_NAME, '', 'a'], - [TokenType.ATTR_QUOTE, '\''], - [TokenType.ATTR_VALUE_TEXT, 't\ne\ns\nt'], - [TokenType.ATTR_QUOTE, '\''], - [TokenType.TAG_OPEN_END], - [TokenType.EOF], + [lex.TokenType.TAG_OPEN_START, '', 't'], + [lex.TokenType.ATTR_NAME, '', 'a'], + [lex.TokenType.ATTR_QUOTE, '\''], + [lex.TokenType.ATTR_VALUE_TEXT, 't\ne\ns\nt'], + [lex.TokenType.ATTR_QUOTE, '\''], + [lex.TokenType.TAG_OPEN_END], + [lex.TokenType.EOF], ]); }); it('should store the locations', () => { expect(tokenizeAndHumanizeSourceSpans('')).toEqual([ - [TokenType.TAG_OPEN_START, ''], - [TokenType.EOF, ''], + [lex.TokenType.TAG_OPEN_START, ''], + [lex.TokenType.EOF, ''], ]); }); it('should report missing closing single quote', () => { expect(tokenizeAndHumanizeErrors('')).toEqual([ - [TokenType.ATTR_VALUE_TEXT, 'Unexpected character "EOF"', '0:8'], + [lex.TokenType.ATTR_VALUE_TEXT, 'Unexpected character "EOF"', '0:8'], ]); }); it('should report missing closing double quote', () => { expect(tokenizeAndHumanizeErrors(' { it('should parse closing tags without prefix', () => { expect(tokenizeAndHumanizeParts('')).toEqual([ - [TokenType.TAG_CLOSE, '', 'test'], - [TokenType.EOF], + [lex.TokenType.TAG_CLOSE, '', 'test'], + [lex.TokenType.EOF], ]); }); it('should parse closing tags with prefix', () => { expect(tokenizeAndHumanizeParts('')).toEqual([ - [TokenType.TAG_CLOSE, 'ns1', 'test'], - [TokenType.EOF], + [lex.TokenType.TAG_CLOSE, 'ns1', 'test'], + [lex.TokenType.EOF], ]); }); it('should allow whitespace', () => { expect(tokenizeAndHumanizeParts('')).toEqual([ - [TokenType.TAG_CLOSE, '', 'test'], - [TokenType.EOF], + [lex.TokenType.TAG_CLOSE, '', 'test'], + [lex.TokenType.EOF], ]); }); it('should store the locations', () => { expect(tokenizeAndHumanizeSourceSpans('')).toEqual([ - [TokenType.TAG_CLOSE, ''], - [TokenType.EOF, ''], + [lex.TokenType.TAG_CLOSE, ''], + [lex.TokenType.EOF, ''], ]); }); it('should report missing name after { expect(tokenizeAndHumanizeErrors('', () => { expect(tokenizeAndHumanizeErrors(' { it('should parse named entities', () => { expect(tokenizeAndHumanizeParts('a&b')).toEqual([ - [TokenType.TEXT, 'a'], - [TokenType.ENCODED_ENTITY, '&', '&'], - [TokenType.TEXT, 'b'], - [TokenType.EOF], + [lex.TokenType.TEXT, 'a'], + [lex.TokenType.ENCODED_ENTITY, '&', '&'], + [lex.TokenType.TEXT, 'b'], + [lex.TokenType.EOF], ]); }); it('should parse hexadecimal entities', () => { expect(tokenizeAndHumanizeParts('AA')).toEqual([ - [TokenType.TEXT, ''], - [TokenType.ENCODED_ENTITY, 'A', 'A'], - [TokenType.TEXT, ''], - [TokenType.ENCODED_ENTITY, 'A', 'A'], - [TokenType.TEXT, ''], - [TokenType.EOF], + [lex.TokenType.TEXT, ''], + [lex.TokenType.ENCODED_ENTITY, 'A', 'A'], + [lex.TokenType.TEXT, ''], + [lex.TokenType.ENCODED_ENTITY, 'A', 'A'], + [lex.TokenType.TEXT, ''], + [lex.TokenType.EOF], ]); }); it('should parse decimal entities', () => { expect(tokenizeAndHumanizeParts('A')).toEqual([ - [TokenType.TEXT, ''], - [TokenType.ENCODED_ENTITY, 'A', 'A'], - [TokenType.TEXT, ''], - [TokenType.EOF], + [lex.TokenType.TEXT, ''], + [lex.TokenType.ENCODED_ENTITY, 'A', 'A'], + [lex.TokenType.TEXT, ''], + [lex.TokenType.EOF], ]); }); it('should store the locations', () => { expect(tokenizeAndHumanizeSourceSpans('a&b')).toEqual([ - [TokenType.TEXT, 'a'], - [TokenType.ENCODED_ENTITY, '&'], - [TokenType.TEXT, 'b'], - [TokenType.EOF, ''], + [lex.TokenType.TEXT, 'a'], + [lex.TokenType.ENCODED_ENTITY, '&'], + [lex.TokenType.TEXT, 'b'], + [lex.TokenType.EOF, ''], ]); }); it('should report malformed/unknown entities', () => { expect(tokenizeAndHumanizeErrors('&tbo;')).toEqual([[ - TokenType.ENCODED_ENTITY, + lex.TokenType.ENCODED_ENTITY, 'Unknown entity "tbo" - use the "&#;" or "&#x;" syntax', '0:0' ]]); expect(tokenizeAndHumanizeErrors('sdf;')).toEqual([[ - TokenType.ENCODED_ENTITY, + lex.TokenType.ENCODED_ENTITY, 'Unable to parse entity "s" - decimal character reference entities must end with ";"', '0:4' ]]); expect(tokenizeAndHumanizeErrors(' sdf;')).toEqual([[ - TokenType.ENCODED_ENTITY, + lex.TokenType.ENCODED_ENTITY, 'Unable to parse entity " s" - hexadecimal character reference entities must end with ";"', '0:5' ]]); expect(tokenizeAndHumanizeErrors('઼')).toEqual([ - [TokenType.ENCODED_ENTITY, 'Unexpected character "EOF"', '0:6'] + [lex.TokenType.ENCODED_ENTITY, 'Unexpected character "EOF"', '0:6'] ]); }); }); @@ -614,8 +613,8 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u describe('regular text', () => { it('should parse text', () => { expect(tokenizeAndHumanizeParts('a')).toEqual([ - [TokenType.TEXT, 'a'], - [TokenType.EOF], + [lex.TokenType.TEXT, 'a'], + [lex.TokenType.EOF], ]); }); @@ -623,168 +622,168 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u expect(tokenizeAndHumanizeParts( '{{ a }}b{{ c // comment }}d{{ e "}} \' " f }}g{{ h // " i }}')) .toEqual([ - [TokenType.TEXT, ''], - [TokenType.INTERPOLATION, '{{', ' a ', '}}'], - [TokenType.TEXT, 'b'], - [TokenType.INTERPOLATION, '{{', ' c // comment ', '}}'], - [TokenType.TEXT, 'd'], - [TokenType.INTERPOLATION, '{{', ' e "}} \' " f ', '}}'], - [TokenType.TEXT, 'g'], - [TokenType.INTERPOLATION, '{{', ' h // " i ', '}}'], - [TokenType.TEXT, ''], - [TokenType.EOF], + [lex.TokenType.TEXT, ''], + [lex.TokenType.INTERPOLATION, '{{', ' a ', '}}'], + [lex.TokenType.TEXT, 'b'], + [lex.TokenType.INTERPOLATION, '{{', ' c // comment ', '}}'], + [lex.TokenType.TEXT, 'd'], + [lex.TokenType.INTERPOLATION, '{{', ' e "}} \' " f ', '}}'], + [lex.TokenType.TEXT, 'g'], + [lex.TokenType.INTERPOLATION, '{{', ' h // " i ', '}}'], + [lex.TokenType.TEXT, ''], + [lex.TokenType.EOF], ]); expect(tokenizeAndHumanizeSourceSpans('{{ a }}b{{ c // comment }}')).toEqual([ - [TokenType.TEXT, ''], - [TokenType.INTERPOLATION, '{{ a }}'], - [TokenType.TEXT, 'b'], - [TokenType.INTERPOLATION, '{{ c // comment }}'], - [TokenType.TEXT, ''], - [TokenType.EOF, ''], + [lex.TokenType.TEXT, ''], + [lex.TokenType.INTERPOLATION, '{{ a }}'], + [lex.TokenType.TEXT, 'b'], + [lex.TokenType.INTERPOLATION, '{{ c // comment }}'], + [lex.TokenType.TEXT, ''], + [lex.TokenType.EOF, ''], ]); }); it('should parse interpolation with custom markers', () => { expect(tokenizeAndHumanizeParts('{% a %}', {interpolationConfig: {start: '{%', end: '%}'}})) .toEqual([ - [TokenType.TEXT, ''], - [TokenType.INTERPOLATION, '{%', ' a ', '%}'], - [TokenType.TEXT, ''], - [TokenType.EOF], + [lex.TokenType.TEXT, ''], + [lex.TokenType.INTERPOLATION, '{%', ' a ', '%}'], + [lex.TokenType.TEXT, ''], + [lex.TokenType.EOF], ]); }); it('should handle CR & LF in text', () => { expect(tokenizeAndHumanizeParts('t\ne\rs\r\nt')).toEqual([ - [TokenType.TEXT, 't\ne\ns\nt'], - [TokenType.EOF], + [lex.TokenType.TEXT, 't\ne\ns\nt'], + [lex.TokenType.EOF], ]); expect(tokenizeAndHumanizeSourceSpans('t\ne\rs\r\nt')).toEqual([ - [TokenType.TEXT, 't\ne\rs\r\nt'], - [TokenType.EOF, ''], + [lex.TokenType.TEXT, 't\ne\rs\r\nt'], + [lex.TokenType.EOF, ''], ]); }); it('should handle CR & LF in interpolation', () => { expect(tokenizeAndHumanizeParts('{{t\ne\rs\r\nt}}')).toEqual([ - [TokenType.TEXT, ''], - [TokenType.INTERPOLATION, '{{', 't\ne\ns\nt', '}}'], - [TokenType.TEXT, ''], - [TokenType.EOF], + [lex.TokenType.TEXT, ''], + [lex.TokenType.INTERPOLATION, '{{', 't\ne\ns\nt', '}}'], + [lex.TokenType.TEXT, ''], + [lex.TokenType.EOF], ]); expect(tokenizeAndHumanizeSourceSpans('{{t\ne\rs\r\nt}}')).toEqual([ - [TokenType.TEXT, ''], - [TokenType.INTERPOLATION, '{{t\ne\rs\r\nt}}'], - [TokenType.TEXT, ''], - [TokenType.EOF, ''], + [lex.TokenType.TEXT, ''], + [lex.TokenType.INTERPOLATION, '{{t\ne\rs\r\nt}}'], + [lex.TokenType.TEXT, ''], + [lex.TokenType.EOF, ''], ]); }); it('should parse entities', () => { expect(tokenizeAndHumanizeParts('a&b')).toEqual([ - [TokenType.TEXT, 'a'], - [TokenType.ENCODED_ENTITY, '&', '&'], - [TokenType.TEXT, 'b'], - [TokenType.EOF], + [lex.TokenType.TEXT, 'a'], + [lex.TokenType.ENCODED_ENTITY, '&', '&'], + [lex.TokenType.TEXT, 'b'], + [lex.TokenType.EOF], ]); expect(tokenizeAndHumanizeSourceSpans('a&b')).toEqual([ - [TokenType.TEXT, 'a'], - [TokenType.ENCODED_ENTITY, '&'], - [TokenType.TEXT, 'b'], - [TokenType.EOF, ''], + [lex.TokenType.TEXT, 'a'], + [lex.TokenType.ENCODED_ENTITY, '&'], + [lex.TokenType.TEXT, 'b'], + [lex.TokenType.EOF, ''], ]); }); it('should parse text starting with "&"', () => { expect(tokenizeAndHumanizeParts('a && b &')).toEqual([ - [TokenType.TEXT, 'a && b &'], - [TokenType.EOF], + [lex.TokenType.TEXT, 'a && b &'], + [lex.TokenType.EOF], ]); }); it('should store the locations', () => { expect(tokenizeAndHumanizeSourceSpans('a')).toEqual([ - [TokenType.TEXT, 'a'], - [TokenType.EOF, ''], + [lex.TokenType.TEXT, 'a'], + [lex.TokenType.EOF, ''], ]); }); it('should allow "<" in text nodes', () => { expect(tokenizeAndHumanizeParts('{{ a < b ? c : d }}')).toEqual([ - [TokenType.TEXT, ''], - [TokenType.INTERPOLATION, '{{', ' a < b ? c : d ', '}}'], - [TokenType.TEXT, ''], - [TokenType.EOF], + [lex.TokenType.TEXT, ''], + [lex.TokenType.INTERPOLATION, '{{', ' a < b ? c : d ', '}}'], + [lex.TokenType.TEXT, ''], + [lex.TokenType.EOF], ]); expect(tokenizeAndHumanizeSourceSpans('

a')).toEqual([ - [TokenType.TAG_OPEN_START, ''], - [TokenType.TEXT, 'a'], - [TokenType.INCOMPLETE_TAG_OPEN, ''], - [TokenType.EOF, ''], + [lex.TokenType.TAG_OPEN_START, ''], + [lex.TokenType.TEXT, 'a'], + [lex.TokenType.INCOMPLETE_TAG_OPEN, ''], + [lex.TokenType.EOF, ''], ]); expect(tokenizeAndHumanizeParts('< a>')).toEqual([ - [TokenType.TEXT, '< a>'], - [TokenType.EOF], + [lex.TokenType.TEXT, '< a>'], + [lex.TokenType.EOF], ]); }); it('should break out of interpolation in text token on valid start tag', () => { expect(tokenizeAndHumanizeParts('{{ a d }}')).toEqual([ - [TokenType.TEXT, ''], - [TokenType.INTERPOLATION, '{{', ' a '], - [TokenType.TEXT, ''], - [TokenType.TAG_OPEN_START, '', 'b'], - [TokenType.ATTR_NAME, '', '&&'], - [TokenType.ATTR_NAME, '', 'c'], - [TokenType.TAG_OPEN_END], - [TokenType.TEXT, ' d }}'], - [TokenType.EOF], + [lex.TokenType.TEXT, ''], + [lex.TokenType.INTERPOLATION, '{{', ' a '], + [lex.TokenType.TEXT, ''], + [lex.TokenType.TAG_OPEN_START, '', 'b'], + [lex.TokenType.ATTR_NAME, '', '&&'], + [lex.TokenType.ATTR_NAME, '', 'c'], + [lex.TokenType.TAG_OPEN_END], + [lex.TokenType.TEXT, ' d }}'], + [lex.TokenType.EOF], ]); }); it('should break out of interpolation in text token on valid comment', () => { expect(tokenizeAndHumanizeParts('{{ a }}')).toEqual([ - [TokenType.TEXT, ''], - [TokenType.INTERPOLATION, '{{', ' a }'], - [TokenType.TEXT, ''], - [TokenType.COMMENT_START], - [TokenType.RAW_TEXT, ''], - [TokenType.COMMENT_END], - [TokenType.TEXT, '}'], - [TokenType.EOF], + [lex.TokenType.TEXT, ''], + [lex.TokenType.INTERPOLATION, '{{', ' a }'], + [lex.TokenType.TEXT, ''], + [lex.TokenType.COMMENT_START], + [lex.TokenType.RAW_TEXT, ''], + [lex.TokenType.COMMENT_END], + [lex.TokenType.TEXT, '}'], + [lex.TokenType.EOF], ]); }); it('should end interpolation on a valid closing tag', () => { expect(tokenizeAndHumanizeParts('

{{ a

')).toEqual([ - [TokenType.TAG_OPEN_START, '', 'p'], - [TokenType.TAG_OPEN_END], - [TokenType.TEXT, ''], - [TokenType.INTERPOLATION, '{{', ' a '], - [TokenType.TEXT, ''], - [TokenType.TAG_CLOSE, '', 'p'], - [TokenType.EOF], + [lex.TokenType.TAG_OPEN_START, '', 'p'], + [lex.TokenType.TAG_OPEN_END], + [lex.TokenType.TEXT, ''], + [lex.TokenType.INTERPOLATION, '{{', ' a '], + [lex.TokenType.TEXT, ''], + [lex.TokenType.TAG_CLOSE, '', 'p'], + [lex.TokenType.EOF], ]); }); it('should break out of interpolation in text token on valid CDATA', () => { expect(tokenizeAndHumanizeParts('{{ a }}')).toEqual([ - [TokenType.TEXT, ''], - [TokenType.INTERPOLATION, '{{', ' a }'], - [TokenType.TEXT, ''], - [TokenType.CDATA_START], - [TokenType.RAW_TEXT, ''], - [TokenType.CDATA_END], - [TokenType.TEXT, '}'], - [TokenType.EOF], + [lex.TokenType.TEXT, ''], + [lex.TokenType.INTERPOLATION, '{{', ' a }'], + [lex.TokenType.TEXT, ''], + [lex.TokenType.CDATA_START], + [lex.TokenType.RAW_TEXT, ''], + [lex.TokenType.CDATA_END], + [lex.TokenType.TEXT, '}'], + [lex.TokenType.EOF], ]); }); @@ -793,82 +792,82 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u // incorrectly be considered part of an ICU. expect(tokenizeAndHumanizeParts(`{{'<={'}}`, {tokenizeExpansionForms: true})) .toEqual([ - [TokenType.TAG_OPEN_START, '', 'code'], - [TokenType.TAG_OPEN_END], - [TokenType.TEXT, ''], - [TokenType.INTERPOLATION, '{{', '\'<={\'', '}}'], - [TokenType.TEXT, ''], - [TokenType.TAG_CLOSE, '', 'code'], - [TokenType.EOF], + [lex.TokenType.TAG_OPEN_START, '', 'code'], + [lex.TokenType.TAG_OPEN_END], + [lex.TokenType.TEXT, ''], + [lex.TokenType.INTERPOLATION, '{{', '\'<={\'', '}}'], + [lex.TokenType.TEXT, ''], + [lex.TokenType.TAG_CLOSE, '', 'code'], + [lex.TokenType.EOF], ]); }); it('should parse start tags quotes in place of an attribute name as text', () => { expect(tokenizeAndHumanizeParts('')).toEqual([ - [TokenType.INCOMPLETE_TAG_OPEN, '', 't'], - [TokenType.TEXT, '">'], - [TokenType.EOF], + [lex.TokenType.INCOMPLETE_TAG_OPEN, '', 't'], + [lex.TokenType.TEXT, '">'], + [lex.TokenType.EOF], ]); expect(tokenizeAndHumanizeParts('')).toEqual([ - [TokenType.INCOMPLETE_TAG_OPEN, '', 't'], - [TokenType.TEXT, '\'>'], - [TokenType.EOF], + [lex.TokenType.INCOMPLETE_TAG_OPEN, '', 't'], + [lex.TokenType.TEXT, '\'>'], + [lex.TokenType.EOF], ]); }); it('should parse start tags quotes in place of an attribute name (after a valid attribute)', () => { expect(tokenizeAndHumanizeParts('')).toEqual([ - [TokenType.INCOMPLETE_TAG_OPEN, '', 't'], - [TokenType.ATTR_NAME, '', 'a'], - [TokenType.ATTR_QUOTE, '"'], - [TokenType.ATTR_VALUE_TEXT, 'b'], - [TokenType.ATTR_QUOTE, '"'], + [lex.TokenType.INCOMPLETE_TAG_OPEN, '', 't'], + [lex.TokenType.ATTR_NAME, '', 'a'], + [lex.TokenType.ATTR_QUOTE, '"'], + [lex.TokenType.ATTR_VALUE_TEXT, 'b'], + [lex.TokenType.ATTR_QUOTE, '"'], // TODO(ayazhafiz): the " symbol should be a synthetic attribute, // allowing us to complete the opening tag correctly. - [TokenType.TEXT, '">'], - [TokenType.EOF], + [lex.TokenType.TEXT, '">'], + [lex.TokenType.EOF], ]); expect(tokenizeAndHumanizeParts('')).toEqual([ - [TokenType.INCOMPLETE_TAG_OPEN, '', 't'], - [TokenType.ATTR_NAME, '', 'a'], - [TokenType.ATTR_QUOTE, '\''], - [TokenType.ATTR_VALUE_TEXT, 'b'], - [TokenType.ATTR_QUOTE, '\''], + [lex.TokenType.INCOMPLETE_TAG_OPEN, '', 't'], + [lex.TokenType.ATTR_NAME, '', 'a'], + [lex.TokenType.ATTR_QUOTE, '\''], + [lex.TokenType.ATTR_VALUE_TEXT, 'b'], + [lex.TokenType.ATTR_QUOTE, '\''], // TODO(ayazhafiz): the ' symbol should be a synthetic attribute, // allowing us to complete the opening tag correctly. - [TokenType.TEXT, '\'>'], - [TokenType.EOF], + [lex.TokenType.TEXT, '\'>'], + [lex.TokenType.EOF], ]); }); it('should be able to escape {', () => { expect(tokenizeAndHumanizeParts('{{ "{" }}')).toEqual([ - [TokenType.TEXT, ''], - [TokenType.INTERPOLATION, '{{', ' "{" ', '}}'], - [TokenType.TEXT, ''], - [TokenType.EOF], + [lex.TokenType.TEXT, ''], + [lex.TokenType.INTERPOLATION, '{{', ' "{" ', '}}'], + [lex.TokenType.TEXT, ''], + [lex.TokenType.EOF], ]); }); it('should be able to escape {{', () => { expect(tokenizeAndHumanizeParts('{{ "{{" }}')).toEqual([ - [TokenType.TEXT, ''], - [TokenType.INTERPOLATION, '{{', ' "{{" ', '}}'], - [TokenType.TEXT, ''], - [TokenType.EOF], + [lex.TokenType.TEXT, ''], + [lex.TokenType.INTERPOLATION, '{{', ' "{{" ', '}}'], + [lex.TokenType.TEXT, ''], + [lex.TokenType.EOF], ]); }); it('should capture everything up to the end of file in the interpolation expression part if there are mismatched quotes', () => { expect(tokenizeAndHumanizeParts('{{ "{{a}}\' }}')).toEqual([ - [TokenType.TEXT, ''], - [TokenType.INTERPOLATION, '{{', ' "{{a}}\' }}'], - [TokenType.TEXT, ''], - [TokenType.EOF], + [lex.TokenType.TEXT, ''], + [lex.TokenType.INTERPOLATION, '{{', ' "{{a}}\' }}'], + [lex.TokenType.TEXT, ''], + [lex.TokenType.EOF], ]); }); @@ -876,11 +875,11 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u expect(tokenizeAndHumanizeParts( '{a, b, =4 {c}}', {tokenizeExpansionForms: false})) .toEqual([ - [TokenType.TAG_OPEN_START, '', 'span'], - [TokenType.TAG_OPEN_END], - [TokenType.TEXT, '{a, b, =4 {c}}'], - [TokenType.TAG_CLOSE, '', 'span'], - [TokenType.EOF], + [lex.TokenType.TAG_OPEN_START, '', 'span'], + [lex.TokenType.TAG_OPEN_END], + [lex.TokenType.TEXT, '{a, b, =4 {c}}'], + [lex.TokenType.TAG_CLOSE, '', 'span'], + [lex.TokenType.EOF], ]); }); }); @@ -888,51 +887,51 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u describe('raw text', () => { it('should parse text', () => { expect(tokenizeAndHumanizeParts(``)).toEqual([ - [TokenType.TAG_OPEN_START, '', 'script'], - [TokenType.TAG_OPEN_END], - [TokenType.RAW_TEXT, 't\ne\ns\nt'], - [TokenType.TAG_CLOSE, '', 'script'], - [TokenType.EOF], + [lex.TokenType.TAG_OPEN_START, '', 'script'], + [lex.TokenType.TAG_OPEN_END], + [lex.TokenType.RAW_TEXT, 't\ne\ns\nt'], + [lex.TokenType.TAG_CLOSE, '', 'script'], + [lex.TokenType.EOF], ]); }); it('should not detect entities', () => { expect(tokenizeAndHumanizeParts(``)).toEqual([ - [TokenType.TAG_OPEN_START, '', 'script'], - [TokenType.TAG_OPEN_END], - [TokenType.RAW_TEXT, '&'], - [TokenType.TAG_CLOSE, '', 'script'], - [TokenType.EOF], + [lex.TokenType.TAG_OPEN_START, '', 'script'], + [lex.TokenType.TAG_OPEN_END], + [lex.TokenType.RAW_TEXT, '&'], + [lex.TokenType.TAG_CLOSE, '', 'script'], + [lex.TokenType.EOF], ]); }); it('should ignore other opening tags', () => { expect(tokenizeAndHumanizeParts(``)).toEqual([ - [TokenType.TAG_OPEN_START, '', 'script'], - [TokenType.TAG_OPEN_END], - [TokenType.RAW_TEXT, 'a
'], - [TokenType.TAG_CLOSE, '', 'script'], - [TokenType.EOF], + [lex.TokenType.TAG_OPEN_START, '', 'script'], + [lex.TokenType.TAG_OPEN_END], + [lex.TokenType.RAW_TEXT, 'a
'], + [lex.TokenType.TAG_CLOSE, '', 'script'], + [lex.TokenType.EOF], ]); }); it('should ignore other closing tags', () => { expect(tokenizeAndHumanizeParts(``)).toEqual([ - [TokenType.TAG_OPEN_START, '', 'script'], - [TokenType.TAG_OPEN_END], - [TokenType.RAW_TEXT, 'a'], - [TokenType.TAG_CLOSE, '', 'script'], - [TokenType.EOF], + [lex.TokenType.TAG_OPEN_START, '', 'script'], + [lex.TokenType.TAG_OPEN_END], + [lex.TokenType.RAW_TEXT, 'a'], + [lex.TokenType.TAG_CLOSE, '', 'script'], + [lex.TokenType.EOF], ]); }); it('should store the locations', () => { expect(tokenizeAndHumanizeSourceSpans(``)).toEqual([ - [TokenType.TAG_OPEN_START, ''], - [TokenType.RAW_TEXT, 'a'], - [TokenType.TAG_CLOSE, ''], - [TokenType.EOF, ''], + [lex.TokenType.TAG_OPEN_START, ''], + [lex.TokenType.RAW_TEXT, 'a'], + [lex.TokenType.TAG_CLOSE, ''], + [lex.TokenType.EOF, ''], ]); }); }); @@ -940,53 +939,53 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u describe('escapable raw text', () => { it('should parse text', () => { expect(tokenizeAndHumanizeParts(`t\ne\rs\r\nt`)).toEqual([ - [TokenType.TAG_OPEN_START, '', 'title'], - [TokenType.TAG_OPEN_END], - [TokenType.ESCAPABLE_RAW_TEXT, 't\ne\ns\nt'], - [TokenType.TAG_CLOSE, '', 'title'], - [TokenType.EOF], + [lex.TokenType.TAG_OPEN_START, '', 'title'], + [lex.TokenType.TAG_OPEN_END], + [lex.TokenType.ESCAPABLE_RAW_TEXT, 't\ne\ns\nt'], + [lex.TokenType.TAG_CLOSE, '', 'title'], + [lex.TokenType.EOF], ]); }); it('should detect entities', () => { expect(tokenizeAndHumanizeParts(`&`)).toEqual([ - [TokenType.TAG_OPEN_START, '', 'title'], - [TokenType.TAG_OPEN_END], - [TokenType.ESCAPABLE_RAW_TEXT, ''], - [TokenType.ENCODED_ENTITY, '&', '&'], - [TokenType.ESCAPABLE_RAW_TEXT, ''], - [TokenType.TAG_CLOSE, '', 'title'], - [TokenType.EOF], + [lex.TokenType.TAG_OPEN_START, '', 'title'], + [lex.TokenType.TAG_OPEN_END], + [lex.TokenType.ESCAPABLE_RAW_TEXT, ''], + [lex.TokenType.ENCODED_ENTITY, '&', '&'], + [lex.TokenType.ESCAPABLE_RAW_TEXT, ''], + [lex.TokenType.TAG_CLOSE, '', 'title'], + [lex.TokenType.EOF], ]); }); it('should ignore other opening tags', () => { expect(tokenizeAndHumanizeParts(`a<div>`)).toEqual([ - [TokenType.TAG_OPEN_START, '', 'title'], - [TokenType.TAG_OPEN_END], - [TokenType.ESCAPABLE_RAW_TEXT, 'a
'], - [TokenType.TAG_CLOSE, '', 'title'], - [TokenType.EOF], + [lex.TokenType.TAG_OPEN_START, '', 'title'], + [lex.TokenType.TAG_OPEN_END], + [lex.TokenType.ESCAPABLE_RAW_TEXT, 'a
'], + [lex.TokenType.TAG_CLOSE, '', 'title'], + [lex.TokenType.EOF], ]); }); it('should ignore other closing tags', () => { expect(tokenizeAndHumanizeParts(`a</test>`)).toEqual([ - [TokenType.TAG_OPEN_START, '', 'title'], - [TokenType.TAG_OPEN_END], - [TokenType.ESCAPABLE_RAW_TEXT, 'a'], - [TokenType.TAG_CLOSE, '', 'title'], - [TokenType.EOF], + [lex.TokenType.TAG_OPEN_START, '', 'title'], + [lex.TokenType.TAG_OPEN_END], + [lex.TokenType.ESCAPABLE_RAW_TEXT, 'a'], + [lex.TokenType.TAG_CLOSE, '', 'title'], + [lex.TokenType.EOF], ]); }); it('should store the locations', () => { expect(tokenizeAndHumanizeSourceSpans(`a`)).toEqual([ - [TokenType.TAG_OPEN_START, ''], - [TokenType.ESCAPABLE_RAW_TEXT, 'a'], - [TokenType.TAG_CLOSE, ''], - [TokenType.EOF, ''], + [lex.TokenType.TAG_OPEN_START, ''], + [lex.TokenType.ESCAPABLE_RAW_TEXT, 'a'], + [lex.TokenType.TAG_CLOSE, ''], + [lex.TokenType.EOF, ''], ]); }); }); @@ -994,24 +993,24 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u describe('parsable data', () => { it('should parse an SVG tag', () => { expect(tokenizeAndHumanizeParts(`<svg:title>test</svg:title>`)).toEqual([ - [TokenType.TAG_OPEN_START, 'svg', 'title'], - [TokenType.TAG_OPEN_END], - [TokenType.TEXT, 'test'], - [TokenType.TAG_CLOSE, 'svg', 'title'], - [TokenType.EOF], + [lex.TokenType.TAG_OPEN_START, 'svg', 'title'], + [lex.TokenType.TAG_OPEN_END], + [lex.TokenType.TEXT, 'test'], + [lex.TokenType.TAG_CLOSE, 'svg', 'title'], + [lex.TokenType.EOF], ]); }); it('should parse an SVG <title> tag with children', () => { expect(tokenizeAndHumanizeParts(`<svg:title><f>test</f></svg:title>`)).toEqual([ - [TokenType.TAG_OPEN_START, 'svg', 'title'], - [TokenType.TAG_OPEN_END], - [TokenType.TAG_OPEN_START, '', 'f'], - [TokenType.TAG_OPEN_END], - [TokenType.TEXT, 'test'], - [TokenType.TAG_CLOSE, '', 'f'], - [TokenType.TAG_CLOSE, 'svg', 'title'], - [TokenType.EOF], + [lex.TokenType.TAG_OPEN_START, 'svg', 'title'], + [lex.TokenType.TAG_OPEN_END], + [lex.TokenType.TAG_OPEN_START, '', 'f'], + [lex.TokenType.TAG_OPEN_END], + [lex.TokenType.TEXT, 'test'], + [lex.TokenType.TAG_CLOSE, '', 'f'], + [lex.TokenType.TAG_CLOSE, 'svg', 'title'], + [lex.TokenType.EOF], ]); }); }); @@ -1022,23 +1021,23 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u tokenizeAndHumanizeParts( '{one.two, three, =4 {four} =5 {five} foo {bar} }', {tokenizeExpansionForms: true})) .toEqual([ - [TokenType.EXPANSION_FORM_START], - [TokenType.RAW_TEXT, 'one.two'], - [TokenType.RAW_TEXT, 'three'], - [TokenType.EXPANSION_CASE_VALUE, '=4'], - [TokenType.EXPANSION_CASE_EXP_START], - [TokenType.TEXT, 'four'], - [TokenType.EXPANSION_CASE_EXP_END], - [TokenType.EXPANSION_CASE_VALUE, '=5'], - [TokenType.EXPANSION_CASE_EXP_START], - [TokenType.TEXT, 'five'], - [TokenType.EXPANSION_CASE_EXP_END], - [TokenType.EXPANSION_CASE_VALUE, 'foo'], - [TokenType.EXPANSION_CASE_EXP_START], - [TokenType.TEXT, 'bar'], - [TokenType.EXPANSION_CASE_EXP_END], - [TokenType.EXPANSION_FORM_END], - [TokenType.EOF], + [lex.TokenType.EXPANSION_FORM_START], + [lex.TokenType.RAW_TEXT, 'one.two'], + [lex.TokenType.RAW_TEXT, 'three'], + [lex.TokenType.EXPANSION_CASE_VALUE, '=4'], + [lex.TokenType.EXPANSION_CASE_EXP_START], + [lex.TokenType.TEXT, 'four'], + [lex.TokenType.EXPANSION_CASE_EXP_END], + [lex.TokenType.EXPANSION_CASE_VALUE, '=5'], + [lex.TokenType.EXPANSION_CASE_EXP_START], + [lex.TokenType.TEXT, 'five'], + [lex.TokenType.EXPANSION_CASE_EXP_END], + [lex.TokenType.EXPANSION_CASE_VALUE, 'foo'], + [lex.TokenType.EXPANSION_CASE_EXP_START], + [lex.TokenType.TEXT, 'bar'], + [lex.TokenType.EXPANSION_CASE_EXP_END], + [lex.TokenType.EXPANSION_FORM_END], + [lex.TokenType.EOF], ]); }); @@ -1046,17 +1045,17 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u expect(tokenizeAndHumanizeParts( 'before{one.two, three, =4 {four}}after', {tokenizeExpansionForms: true})) .toEqual([ - [TokenType.TEXT, 'before'], - [TokenType.EXPANSION_FORM_START], - [TokenType.RAW_TEXT, 'one.two'], - [TokenType.RAW_TEXT, 'three'], - [TokenType.EXPANSION_CASE_VALUE, '=4'], - [TokenType.EXPANSION_CASE_EXP_START], - [TokenType.TEXT, 'four'], - [TokenType.EXPANSION_CASE_EXP_END], - [TokenType.EXPANSION_FORM_END], - [TokenType.TEXT, 'after'], - [TokenType.EOF], + [lex.TokenType.TEXT, 'before'], + [lex.TokenType.EXPANSION_FORM_START], + [lex.TokenType.RAW_TEXT, 'one.two'], + [lex.TokenType.RAW_TEXT, 'three'], + [lex.TokenType.EXPANSION_CASE_VALUE, '=4'], + [lex.TokenType.EXPANSION_CASE_EXP_START], + [lex.TokenType.TEXT, 'four'], + [lex.TokenType.EXPANSION_CASE_EXP_END], + [lex.TokenType.EXPANSION_FORM_END], + [lex.TokenType.TEXT, 'after'], + [lex.TokenType.EOF], ]); }); @@ -1064,21 +1063,21 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u expect(tokenizeAndHumanizeParts( '<div><span>{a, b, =4 {c}}</span></div>', {tokenizeExpansionForms: true})) .toEqual([ - [TokenType.TAG_OPEN_START, '', 'div'], - [TokenType.TAG_OPEN_END], - [TokenType.TAG_OPEN_START, '', 'span'], - [TokenType.TAG_OPEN_END], - [TokenType.EXPANSION_FORM_START], - [TokenType.RAW_TEXT, 'a'], - [TokenType.RAW_TEXT, 'b'], - [TokenType.EXPANSION_CASE_VALUE, '=4'], - [TokenType.EXPANSION_CASE_EXP_START], - [TokenType.TEXT, 'c'], - [TokenType.EXPANSION_CASE_EXP_END], - [TokenType.EXPANSION_FORM_END], - [TokenType.TAG_CLOSE, '', 'span'], - [TokenType.TAG_CLOSE, '', 'div'], - [TokenType.EOF], + [lex.TokenType.TAG_OPEN_START, '', 'div'], + [lex.TokenType.TAG_OPEN_END], + [lex.TokenType.TAG_OPEN_START, '', 'span'], + [lex.TokenType.TAG_OPEN_END], + [lex.TokenType.EXPANSION_FORM_START], + [lex.TokenType.RAW_TEXT, 'a'], + [lex.TokenType.RAW_TEXT, 'b'], + [lex.TokenType.EXPANSION_CASE_VALUE, '=4'], + [lex.TokenType.EXPANSION_CASE_EXP_START], + [lex.TokenType.TEXT, 'c'], + [lex.TokenType.EXPANSION_CASE_EXP_END], + [lex.TokenType.EXPANSION_FORM_END], + [lex.TokenType.TAG_CLOSE, '', 'span'], + [lex.TokenType.TAG_CLOSE, '', 'div'], + [lex.TokenType.EOF], ]); }); @@ -1086,23 +1085,23 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u expect(tokenizeAndHumanizeParts( '<div><span> {a, b, =4 {c}} </span></div>', {tokenizeExpansionForms: true})) .toEqual([ - [TokenType.TAG_OPEN_START, '', 'div'], - [TokenType.TAG_OPEN_END], - [TokenType.TAG_OPEN_START, '', 'span'], - [TokenType.TAG_OPEN_END], - [TokenType.TEXT, ' '], - [TokenType.EXPANSION_FORM_START], - [TokenType.RAW_TEXT, 'a'], - [TokenType.RAW_TEXT, 'b'], - [TokenType.EXPANSION_CASE_VALUE, '=4'], - [TokenType.EXPANSION_CASE_EXP_START], - [TokenType.TEXT, 'c'], - [TokenType.EXPANSION_CASE_EXP_END], - [TokenType.EXPANSION_FORM_END], - [TokenType.TEXT, ' '], - [TokenType.TAG_CLOSE, '', 'span'], - [TokenType.TAG_CLOSE, '', 'div'], - [TokenType.EOF], + [lex.TokenType.TAG_OPEN_START, '', 'div'], + [lex.TokenType.TAG_OPEN_END], + [lex.TokenType.TAG_OPEN_START, '', 'span'], + [lex.TokenType.TAG_OPEN_END], + [lex.TokenType.TEXT, ' '], + [lex.TokenType.EXPANSION_FORM_START], + [lex.TokenType.RAW_TEXT, 'a'], + [lex.TokenType.RAW_TEXT, 'b'], + [lex.TokenType.EXPANSION_CASE_VALUE, '=4'], + [lex.TokenType.EXPANSION_CASE_EXP_START], + [lex.TokenType.TEXT, 'c'], + [lex.TokenType.EXPANSION_CASE_EXP_END], + [lex.TokenType.EXPANSION_FORM_END], + [lex.TokenType.TEXT, ' '], + [lex.TokenType.TAG_CLOSE, '', 'span'], + [lex.TokenType.TAG_CLOSE, '', 'div'], + [lex.TokenType.EOF], ]); }); @@ -1110,19 +1109,19 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u expect(tokenizeAndHumanizeParts( '{one.two, three, =4 {four <b>a</b>}}', {tokenizeExpansionForms: true})) .toEqual([ - [TokenType.EXPANSION_FORM_START], - [TokenType.RAW_TEXT, 'one.two'], - [TokenType.RAW_TEXT, 'three'], - [TokenType.EXPANSION_CASE_VALUE, '=4'], - [TokenType.EXPANSION_CASE_EXP_START], - [TokenType.TEXT, 'four '], - [TokenType.TAG_OPEN_START, '', 'b'], - [TokenType.TAG_OPEN_END], - [TokenType.TEXT, 'a'], - [TokenType.TAG_CLOSE, '', 'b'], - [TokenType.EXPANSION_CASE_EXP_END], - [TokenType.EXPANSION_FORM_END], - [TokenType.EOF], + [lex.TokenType.EXPANSION_FORM_START], + [lex.TokenType.RAW_TEXT, 'one.two'], + [lex.TokenType.RAW_TEXT, 'three'], + [lex.TokenType.EXPANSION_CASE_VALUE, '=4'], + [lex.TokenType.EXPANSION_CASE_EXP_START], + [lex.TokenType.TEXT, 'four '], + [lex.TokenType.TAG_OPEN_START, '', 'b'], + [lex.TokenType.TAG_OPEN_END], + [lex.TokenType.TEXT, 'a'], + [lex.TokenType.TAG_CLOSE, '', 'b'], + [lex.TokenType.EXPANSION_CASE_EXP_END], + [lex.TokenType.EXPANSION_FORM_END], + [lex.TokenType.EOF], ]); }); @@ -1130,17 +1129,17 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u expect(tokenizeAndHumanizeParts( '{one.two, three, =4 {four {{a}}}}', {tokenizeExpansionForms: true})) .toEqual([ - [TokenType.EXPANSION_FORM_START], - [TokenType.RAW_TEXT, 'one.two'], - [TokenType.RAW_TEXT, 'three'], - [TokenType.EXPANSION_CASE_VALUE, '=4'], - [TokenType.EXPANSION_CASE_EXP_START], - [TokenType.TEXT, 'four '], - [TokenType.INTERPOLATION, '{{', 'a', '}}'], - [TokenType.TEXT, ''], - [TokenType.EXPANSION_CASE_EXP_END], - [TokenType.EXPANSION_FORM_END], - [TokenType.EOF], + [lex.TokenType.EXPANSION_FORM_START], + [lex.TokenType.RAW_TEXT, 'one.two'], + [lex.TokenType.RAW_TEXT, 'three'], + [lex.TokenType.EXPANSION_CASE_VALUE, '=4'], + [lex.TokenType.EXPANSION_CASE_EXP_START], + [lex.TokenType.TEXT, 'four '], + [lex.TokenType.INTERPOLATION, '{{', 'a', '}}'], + [lex.TokenType.TEXT, ''], + [lex.TokenType.EXPANSION_CASE_EXP_END], + [lex.TokenType.EXPANSION_FORM_END], + [lex.TokenType.EOF], ]); }); @@ -1148,23 +1147,23 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u expect(tokenizeAndHumanizeParts( `{one.two, three, =4 { {xx, yy, =x {one}} }}`, {tokenizeExpansionForms: true})) .toEqual([ - [TokenType.EXPANSION_FORM_START], - [TokenType.RAW_TEXT, 'one.two'], - [TokenType.RAW_TEXT, 'three'], - [TokenType.EXPANSION_CASE_VALUE, '=4'], - [TokenType.EXPANSION_CASE_EXP_START], - [TokenType.EXPANSION_FORM_START], - [TokenType.RAW_TEXT, 'xx'], - [TokenType.RAW_TEXT, 'yy'], - [TokenType.EXPANSION_CASE_VALUE, '=x'], - [TokenType.EXPANSION_CASE_EXP_START], - [TokenType.TEXT, 'one'], - [TokenType.EXPANSION_CASE_EXP_END], - [TokenType.EXPANSION_FORM_END], - [TokenType.TEXT, ' '], - [TokenType.EXPANSION_CASE_EXP_END], - [TokenType.EXPANSION_FORM_END], - [TokenType.EOF], + [lex.TokenType.EXPANSION_FORM_START], + [lex.TokenType.RAW_TEXT, 'one.two'], + [lex.TokenType.RAW_TEXT, 'three'], + [lex.TokenType.EXPANSION_CASE_VALUE, '=4'], + [lex.TokenType.EXPANSION_CASE_EXP_START], + [lex.TokenType.EXPANSION_FORM_START], + [lex.TokenType.RAW_TEXT, 'xx'], + [lex.TokenType.RAW_TEXT, 'yy'], + [lex.TokenType.EXPANSION_CASE_VALUE, '=x'], + [lex.TokenType.EXPANSION_CASE_EXP_START], + [lex.TokenType.TEXT, 'one'], + [lex.TokenType.EXPANSION_CASE_EXP_END], + [lex.TokenType.EXPANSION_FORM_END], + [lex.TokenType.TEXT, ' '], + [lex.TokenType.EXPANSION_CASE_EXP_END], + [lex.TokenType.EXPANSION_FORM_END], + [lex.TokenType.EOF], ]); }); @@ -1185,22 +1184,22 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u }); expect(humanizeParts(result.tokens)).toEqual([ - [TokenType.EXPANSION_FORM_START], - [TokenType.RAW_TEXT, '\n messages.length'], - [TokenType.RAW_TEXT, 'plural'], - [TokenType.EXPANSION_CASE_VALUE, '=0'], - [TokenType.EXPANSION_CASE_EXP_START], - [TokenType.TEXT, 'You have \nno\n messages'], - [TokenType.EXPANSION_CASE_EXP_END], - [TokenType.EXPANSION_CASE_VALUE, '=1'], - [TokenType.EXPANSION_CASE_EXP_START], - [TokenType.TEXT, 'One '], - [TokenType.INTERPOLATION, '{{', 'message', '}}'], - [TokenType.TEXT, ''], - [TokenType.EXPANSION_CASE_EXP_END], - [TokenType.EXPANSION_FORM_END], - [TokenType.TEXT, '\n'], - [TokenType.EOF], + [lex.TokenType.EXPANSION_FORM_START], + [lex.TokenType.RAW_TEXT, '\n messages.length'], + [lex.TokenType.RAW_TEXT, 'plural'], + [lex.TokenType.EXPANSION_CASE_VALUE, '=0'], + [lex.TokenType.EXPANSION_CASE_EXP_START], + [lex.TokenType.TEXT, 'You have \nno\n messages'], + [lex.TokenType.EXPANSION_CASE_EXP_END], + [lex.TokenType.EXPANSION_CASE_VALUE, '=1'], + [lex.TokenType.EXPANSION_CASE_EXP_START], + [lex.TokenType.TEXT, 'One '], + [lex.TokenType.INTERPOLATION, '{{', 'message', '}}'], + [lex.TokenType.TEXT, ''], + [lex.TokenType.EXPANSION_CASE_EXP_END], + [lex.TokenType.EXPANSION_FORM_END], + [lex.TokenType.TEXT, '\n'], + [lex.TokenType.EOF], ]); expect(result.nonNormalizedIcuExpressions).toEqual([]); @@ -1217,22 +1216,22 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u {tokenizeExpansionForms: true, escapedString: true}); expect(humanizeParts(result.tokens)).toEqual([ - [TokenType.EXPANSION_FORM_START], - [TokenType.RAW_TEXT, '\r\n messages.length'], - [TokenType.RAW_TEXT, 'plural'], - [TokenType.EXPANSION_CASE_VALUE, '=0'], - [TokenType.EXPANSION_CASE_EXP_START], - [TokenType.TEXT, 'You have \nno\n messages'], - [TokenType.EXPANSION_CASE_EXP_END], - [TokenType.EXPANSION_CASE_VALUE, '=1'], - [TokenType.EXPANSION_CASE_EXP_START], - [TokenType.TEXT, 'One '], - [TokenType.INTERPOLATION, '{{', 'message', '}}'], - [TokenType.TEXT, ''], - [TokenType.EXPANSION_CASE_EXP_END], - [TokenType.EXPANSION_FORM_END], - [TokenType.TEXT, '\n'], - [TokenType.EOF], + [lex.TokenType.EXPANSION_FORM_START], + [lex.TokenType.RAW_TEXT, '\r\n messages.length'], + [lex.TokenType.RAW_TEXT, 'plural'], + [lex.TokenType.EXPANSION_CASE_VALUE, '=0'], + [lex.TokenType.EXPANSION_CASE_EXP_START], + [lex.TokenType.TEXT, 'You have \nno\n messages'], + [lex.TokenType.EXPANSION_CASE_EXP_END], + [lex.TokenType.EXPANSION_CASE_VALUE, '=1'], + [lex.TokenType.EXPANSION_CASE_EXP_START], + [lex.TokenType.TEXT, 'One '], + [lex.TokenType.INTERPOLATION, '{{', 'message', '}}'], + [lex.TokenType.TEXT, ''], + [lex.TokenType.EXPANSION_CASE_EXP_END], + [lex.TokenType.EXPANSION_FORM_END], + [lex.TokenType.TEXT, '\n'], + [lex.TokenType.EOF], ]); expect(result.nonNormalizedIcuExpressions!.length).toBe(1); @@ -1254,26 +1253,26 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u `}`, {tokenizeExpansionForms: true, escapedString: true}); expect(humanizeParts(result.tokens)).toEqual([ - [TokenType.EXPANSION_FORM_START], - [TokenType.RAW_TEXT, '\r\n messages.length'], - [TokenType.RAW_TEXT, 'plural'], - [TokenType.EXPANSION_CASE_VALUE, '=0'], - [TokenType.EXPANSION_CASE_EXP_START], - [TokenType.TEXT, 'zero \n '], + [lex.TokenType.EXPANSION_FORM_START], + [lex.TokenType.RAW_TEXT, '\r\n messages.length'], + [lex.TokenType.RAW_TEXT, 'plural'], + [lex.TokenType.EXPANSION_CASE_VALUE, '=0'], + [lex.TokenType.EXPANSION_CASE_EXP_START], + [lex.TokenType.TEXT, 'zero \n '], - [TokenType.EXPANSION_FORM_START], - [TokenType.RAW_TEXT, '\r\n p.gender'], - [TokenType.RAW_TEXT, 'select'], - [TokenType.EXPANSION_CASE_VALUE, 'male'], - [TokenType.EXPANSION_CASE_EXP_START], - [TokenType.TEXT, 'm'], - [TokenType.EXPANSION_CASE_EXP_END], - [TokenType.EXPANSION_FORM_END], + [lex.TokenType.EXPANSION_FORM_START], + [lex.TokenType.RAW_TEXT, '\r\n p.gender'], + [lex.TokenType.RAW_TEXT, 'select'], + [lex.TokenType.EXPANSION_CASE_VALUE, 'male'], + [lex.TokenType.EXPANSION_CASE_EXP_START], + [lex.TokenType.TEXT, 'm'], + [lex.TokenType.EXPANSION_CASE_EXP_END], + [lex.TokenType.EXPANSION_FORM_END], - [TokenType.TEXT, '\n '], - [TokenType.EXPANSION_CASE_EXP_END], - [TokenType.EXPANSION_FORM_END], - [TokenType.EOF], + [lex.TokenType.TEXT, '\n '], + [lex.TokenType.EXPANSION_CASE_EXP_END], + [lex.TokenType.EXPANSION_FORM_END], + [lex.TokenType.EOF], ]); expect(result.nonNormalizedIcuExpressions!.length).toBe(2); @@ -1300,22 +1299,22 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u }); expect(humanizeParts(result.tokens)).toEqual([ - [TokenType.EXPANSION_FORM_START], - [TokenType.RAW_TEXT, '\n messages.length'], - [TokenType.RAW_TEXT, 'plural'], - [TokenType.EXPANSION_CASE_VALUE, '=0'], - [TokenType.EXPANSION_CASE_EXP_START], - [TokenType.TEXT, 'You have \nno\n messages'], - [TokenType.EXPANSION_CASE_EXP_END], - [TokenType.EXPANSION_CASE_VALUE, '=1'], - [TokenType.EXPANSION_CASE_EXP_START], - [TokenType.TEXT, 'One '], - [TokenType.INTERPOLATION, '{{', 'message', '}}'], - [TokenType.TEXT, ''], - [TokenType.EXPANSION_CASE_EXP_END], - [TokenType.EXPANSION_FORM_END], - [TokenType.TEXT, '\n'], - [TokenType.EOF], + [lex.TokenType.EXPANSION_FORM_START], + [lex.TokenType.RAW_TEXT, '\n messages.length'], + [lex.TokenType.RAW_TEXT, 'plural'], + [lex.TokenType.EXPANSION_CASE_VALUE, '=0'], + [lex.TokenType.EXPANSION_CASE_EXP_START], + [lex.TokenType.TEXT, 'You have \nno\n messages'], + [lex.TokenType.EXPANSION_CASE_EXP_END], + [lex.TokenType.EXPANSION_CASE_VALUE, '=1'], + [lex.TokenType.EXPANSION_CASE_EXP_START], + [lex.TokenType.TEXT, 'One '], + [lex.TokenType.INTERPOLATION, '{{', 'message', '}}'], + [lex.TokenType.TEXT, ''], + [lex.TokenType.EXPANSION_CASE_EXP_END], + [lex.TokenType.EXPANSION_FORM_END], + [lex.TokenType.TEXT, '\n'], + [lex.TokenType.EOF], ]); expect(result.nonNormalizedIcuExpressions).toEqual([]); @@ -1332,22 +1331,22 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u {tokenizeExpansionForms: true, escapedString: false}); expect(humanizeParts(result.tokens)).toEqual([ - [TokenType.EXPANSION_FORM_START], - [TokenType.RAW_TEXT, '\r\n messages.length'], - [TokenType.RAW_TEXT, 'plural'], - [TokenType.EXPANSION_CASE_VALUE, '=0'], - [TokenType.EXPANSION_CASE_EXP_START], - [TokenType.TEXT, 'You have \nno\n messages'], - [TokenType.EXPANSION_CASE_EXP_END], - [TokenType.EXPANSION_CASE_VALUE, '=1'], - [TokenType.EXPANSION_CASE_EXP_START], - [TokenType.TEXT, 'One '], - [TokenType.INTERPOLATION, '{{', 'message', '}}'], - [TokenType.TEXT, ''], - [TokenType.EXPANSION_CASE_EXP_END], - [TokenType.EXPANSION_FORM_END], - [TokenType.TEXT, '\n'], - [TokenType.EOF], + [lex.TokenType.EXPANSION_FORM_START], + [lex.TokenType.RAW_TEXT, '\r\n messages.length'], + [lex.TokenType.RAW_TEXT, 'plural'], + [lex.TokenType.EXPANSION_CASE_VALUE, '=0'], + [lex.TokenType.EXPANSION_CASE_EXP_START], + [lex.TokenType.TEXT, 'You have \nno\n messages'], + [lex.TokenType.EXPANSION_CASE_EXP_END], + [lex.TokenType.EXPANSION_CASE_VALUE, '=1'], + [lex.TokenType.EXPANSION_CASE_EXP_START], + [lex.TokenType.TEXT, 'One '], + [lex.TokenType.INTERPOLATION, '{{', 'message', '}}'], + [lex.TokenType.TEXT, ''], + [lex.TokenType.EXPANSION_CASE_EXP_END], + [lex.TokenType.EXPANSION_FORM_END], + [lex.TokenType.TEXT, '\n'], + [lex.TokenType.EOF], ]); expect(result.nonNormalizedIcuExpressions!.length).toBe(1); @@ -1370,26 +1369,26 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u {tokenizeExpansionForms: true}); expect(humanizeParts(result.tokens)).toEqual([ - [TokenType.EXPANSION_FORM_START], - [TokenType.RAW_TEXT, '\r\n messages.length'], - [TokenType.RAW_TEXT, 'plural'], - [TokenType.EXPANSION_CASE_VALUE, '=0'], - [TokenType.EXPANSION_CASE_EXP_START], - [TokenType.TEXT, 'zero \n '], + [lex.TokenType.EXPANSION_FORM_START], + [lex.TokenType.RAW_TEXT, '\r\n messages.length'], + [lex.TokenType.RAW_TEXT, 'plural'], + [lex.TokenType.EXPANSION_CASE_VALUE, '=0'], + [lex.TokenType.EXPANSION_CASE_EXP_START], + [lex.TokenType.TEXT, 'zero \n '], - [TokenType.EXPANSION_FORM_START], - [TokenType.RAW_TEXT, '\r\n p.gender'], - [TokenType.RAW_TEXT, 'select'], - [TokenType.EXPANSION_CASE_VALUE, 'male'], - [TokenType.EXPANSION_CASE_EXP_START], - [TokenType.TEXT, 'm'], - [TokenType.EXPANSION_CASE_EXP_END], - [TokenType.EXPANSION_FORM_END], + [lex.TokenType.EXPANSION_FORM_START], + [lex.TokenType.RAW_TEXT, '\r\n p.gender'], + [lex.TokenType.RAW_TEXT, 'select'], + [lex.TokenType.EXPANSION_CASE_VALUE, 'male'], + [lex.TokenType.EXPANSION_CASE_EXP_START], + [lex.TokenType.TEXT, 'm'], + [lex.TokenType.EXPANSION_CASE_EXP_END], + [lex.TokenType.EXPANSION_FORM_END], - [TokenType.TEXT, '\n '], - [TokenType.EXPANSION_CASE_EXP_END], - [TokenType.EXPANSION_FORM_END], - [TokenType.EOF], + [lex.TokenType.TEXT, '\n '], + [lex.TokenType.EXPANSION_CASE_EXP_END], + [lex.TokenType.EXPANSION_FORM_END], + [lex.TokenType.EOF], ]); expect(result.nonNormalizedIcuExpressions!.length).toBe(2); @@ -1407,7 +1406,7 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u it('should report unescaped "{" on error', () => { expect(tokenizeAndHumanizeErrors(`<p>before { after</p>`, {tokenizeExpansionForms: true})) .toEqual([[ - TokenType.RAW_TEXT, + lex.TokenType.RAW_TEXT, `Unexpected character "EOF" (Do you have an unescaped "{" in your template? Use "{{ '{' }}") to escape it.)`, '0:21', ]]); @@ -1419,7 +1418,7 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u `<code>{{b}<!---->}</code><pre>import {a} from 'a';</pre>`, {tokenizeExpansionForms: true})) .toEqual([[ - TokenType.RAW_TEXT, + lex.TokenType.RAW_TEXT, `Unexpected character "EOF" (Do you have an unescaped "{" in your template? Use "{{ '{' }}") to escape it.)`, '0:56', ]]); @@ -1430,7 +1429,7 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u const file = new ParseSourceFile(src, 'file://'); const location = new ParseLocation(file, 12, 123, 456); const span = new ParseSourceSpan(location, location); - const error = new TokenError('**ERROR**', null!, span); + const error = new lex.TokenError('**ERROR**', null!, span); expect(error.toString()) .toEqual(`**ERROR** ("\n222\n333\n[ERROR ->]E\n444\n555\n"): file://@123:456`); }); @@ -1439,11 +1438,11 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u describe('unicode characters', () => { it('should support unicode characters', () => { expect(tokenizeAndHumanizeSourceSpans(`<p>İ</p>`)).toEqual([ - [TokenType.TAG_OPEN_START, '<p'], - [TokenType.TAG_OPEN_END, '>'], - [TokenType.TEXT, 'İ'], - [TokenType.TAG_CLOSE, '</p>'], - [TokenType.EOF, ''], + [lex.TokenType.TAG_OPEN_START, '<p'], + [lex.TokenType.TAG_OPEN_END, '>'], + [lex.TokenType.TEXT, 'İ'], + [lex.TokenType.TAG_CLOSE, '</p>'], + [lex.TokenType.EOF, ''], ]); }); }); @@ -1451,63 +1450,63 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u describe('(processing escaped strings)', () => { it('should unescape standard escape sequences', () => { expect(tokenizeAndHumanizeParts('\\\' \\\' \\\'', {escapedString: true})).toEqual([ - [TokenType.TEXT, '\' \' \''], - [TokenType.EOF], + [lex.TokenType.TEXT, '\' \' \''], + [lex.TokenType.EOF], ]); expect(tokenizeAndHumanizeParts('\\" \\" \\"', {escapedString: true})).toEqual([ - [TokenType.TEXT, '\" \" \"'], - [TokenType.EOF], + [lex.TokenType.TEXT, '\" \" \"'], + [lex.TokenType.EOF], ]); expect(tokenizeAndHumanizeParts('\\` \\` \\`', {escapedString: true})).toEqual([ - [TokenType.TEXT, '\` \` \`'], - [TokenType.EOF], + [lex.TokenType.TEXT, '\` \` \`'], + [lex.TokenType.EOF], ]); expect(tokenizeAndHumanizeParts('\\\\ \\\\ \\\\', {escapedString: true})).toEqual([ - [TokenType.TEXT, '\\ \\ \\'], - [TokenType.EOF], + [lex.TokenType.TEXT, '\\ \\ \\'], + [lex.TokenType.EOF], ]); expect(tokenizeAndHumanizeParts('\\n \\n \\n', {escapedString: true})).toEqual([ - [TokenType.TEXT, '\n \n \n'], - [TokenType.EOF], + [lex.TokenType.TEXT, '\n \n \n'], + [lex.TokenType.EOF], ]); expect(tokenizeAndHumanizeParts('\\r{{\\r}}\\r', {escapedString: true})).toEqual([ // post processing converts `\r` to `\n` - [TokenType.TEXT, '\n'], - [TokenType.INTERPOLATION, '{{', '\n', '}}'], - [TokenType.TEXT, '\n'], - [TokenType.EOF], + [lex.TokenType.TEXT, '\n'], + [lex.TokenType.INTERPOLATION, '{{', '\n', '}}'], + [lex.TokenType.TEXT, '\n'], + [lex.TokenType.EOF], ]); expect(tokenizeAndHumanizeParts('\\v \\v \\v', {escapedString: true})).toEqual([ - [TokenType.TEXT, '\v \v \v'], - [TokenType.EOF], + [lex.TokenType.TEXT, '\v \v \v'], + [lex.TokenType.EOF], ]); expect(tokenizeAndHumanizeParts('\\t \\t \\t', {escapedString: true})).toEqual([ - [TokenType.TEXT, '\t \t \t'], - [TokenType.EOF], + [lex.TokenType.TEXT, '\t \t \t'], + [lex.TokenType.EOF], ]); expect(tokenizeAndHumanizeParts('\\b \\b \\b', {escapedString: true})).toEqual([ - [TokenType.TEXT, '\b \b \b'], - [TokenType.EOF], + [lex.TokenType.TEXT, '\b \b \b'], + [lex.TokenType.EOF], ]); expect(tokenizeAndHumanizeParts('\\f \\f \\f', {escapedString: true})).toEqual([ - [TokenType.TEXT, '\f \f \f'], - [TokenType.EOF], + [lex.TokenType.TEXT, '\f \f \f'], + [lex.TokenType.EOF], ]); expect(tokenizeAndHumanizeParts( '\\\' \\" \\` \\\\ \\n \\r \\v \\t \\b \\f', {escapedString: true})) .toEqual([ - [TokenType.TEXT, '\' \" \` \\ \n \n \v \t \b \f'], - [TokenType.EOF], + [lex.TokenType.TEXT, '\' \" \` \\ \n \n \v \t \b \f'], + [lex.TokenType.EOF], ]); }); it('should unescape null sequences', () => { expect(tokenizeAndHumanizeParts('\\0', {escapedString: true})).toEqual([ - [TokenType.EOF], + [lex.TokenType.EOF], ]); // \09 is not an octal number so the \0 is taken as EOF expect(tokenizeAndHumanizeParts('\\09', {escapedString: true})).toEqual([ - [TokenType.EOF], + [lex.TokenType.EOF], ]); }); @@ -1518,15 +1517,15 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u expect(tokenizeAndHumanizeParts( '\\001 \\01 \\1 \\12 \\223 \\19 \\2234 \\999', {escapedString: true})) .toEqual([ - [TokenType.TEXT, '\x01 \x01 \x01 \x0A \x93 \x019 \x934 999'], - [TokenType.EOF], + [lex.TokenType.TEXT, '\x01 \x01 \x01 \x0A \x93 \x019 \x934 999'], + [lex.TokenType.EOF], ]); }); it('should unescape hex sequences', () => { expect(tokenizeAndHumanizeParts('\\x12 \\x4F \\xDC', {escapedString: true})).toEqual([ - [TokenType.TEXT, '\x12 \x4F \xDC'], - [TokenType.EOF], + [lex.TokenType.TEXT, '\x12 \x4F \xDC'], + [lex.TokenType.EOF], ]); }); @@ -1536,18 +1535,18 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u ]); expect(tokenizeAndHumanizeErrors('abc \\x xyz', {escapedString: true})).toEqual([ - [TokenType.TEXT, 'Invalid hexadecimal escape sequence', '0:6'] + [lex.TokenType.TEXT, 'Invalid hexadecimal escape sequence', '0:6'] ]); expect(tokenizeAndHumanizeErrors('abc\\x', {escapedString: true})).toEqual([ - [TokenType.TEXT, 'Unexpected character "EOF"', '0:5'] + [lex.TokenType.TEXT, 'Unexpected character "EOF"', '0:5'] ]); }); it('should unescape fixed length Unicode sequences', () => { expect(tokenizeAndHumanizeParts('\\u0123 \\uABCD', {escapedString: true})).toEqual([ - [TokenType.TEXT, '\u0123 \uABCD'], - [TokenType.EOF], + [lex.TokenType.TEXT, '\u0123 \uABCD'], + [lex.TokenType.EOF], ]); }); @@ -1561,8 +1560,8 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u expect(tokenizeAndHumanizeParts( '\\u{01} \\u{ABC} \\u{1234} \\u{123AB}', {escapedString: true})) .toEqual([ - [TokenType.TEXT, '\u{01} \u{ABC} \u{1234} \u{123AB}'], - [TokenType.EOF], + [lex.TokenType.TEXT, '\u{01} \u{ABC} \u{1234} \u{123AB}'], + [lex.TokenType.EOF], ]); }); @@ -1574,27 +1573,27 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u it('should unescape line continuations', () => { expect(tokenizeAndHumanizeParts('abc\\\ndef', {escapedString: true})).toEqual([ - [TokenType.TEXT, 'abcdef'], - [TokenType.EOF], + [lex.TokenType.TEXT, 'abcdef'], + [lex.TokenType.EOF], ]); expect(tokenizeAndHumanizeParts('\\\nx\\\ny\\\n', {escapedString: true})).toEqual([ - [TokenType.TEXT, 'xy'], - [TokenType.EOF], + [lex.TokenType.TEXT, 'xy'], + [lex.TokenType.EOF], ]); }); it('should remove backslash from "non-escape" sequences', () => { expect(tokenizeAndHumanizeParts('\a \g \~', {escapedString: true})).toEqual([ - [TokenType.TEXT, 'a g ~'], - [TokenType.EOF], + [lex.TokenType.TEXT, 'a g ~'], + [lex.TokenType.EOF], ]); }); it('should unescape sequences in plain text', () => { expect(tokenizeAndHumanizeParts('abc\ndef\\nghi\\tjkl\\`\\\'\\"mno', {escapedString: true})) .toEqual([ - [TokenType.TEXT, 'abc\ndef\nghi\tjkl`\'"mno'], - [TokenType.EOF], + [lex.TokenType.TEXT, 'abc\ndef\nghi\tjkl`\'"mno'], + [lex.TokenType.EOF], ]); }); @@ -1602,11 +1601,11 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u expect(tokenizeAndHumanizeParts( '<script>abc\ndef\\nghi\\tjkl\\`\\\'\\"mno</script>', {escapedString: true})) .toEqual([ - [TokenType.TAG_OPEN_START, '', 'script'], - [TokenType.TAG_OPEN_END], - [TokenType.RAW_TEXT, 'abc\ndef\nghi\tjkl`\'"mno'], - [TokenType.TAG_CLOSE, '', 'script'], - [TokenType.EOF], + [lex.TokenType.TAG_OPEN_START, '', 'script'], + [lex.TokenType.TAG_OPEN_END], + [lex.TokenType.RAW_TEXT, 'abc\ndef\nghi\tjkl`\'"mno'], + [lex.TokenType.TAG_CLOSE, '', 'script'], + [lex.TokenType.EOF], ]); }); @@ -1614,90 +1613,90 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u expect(tokenizeAndHumanizeParts( '<title>abc\ndef\\nghi\\tjkl\\`\\\'\\"mno', {escapedString: true})) .toEqual([ - [TokenType.TAG_OPEN_START, '', 'title'], - [TokenType.TAG_OPEN_END], - [TokenType.ESCAPABLE_RAW_TEXT, 'abc\ndef\nghi\tjkl`\'"mno'], - [TokenType.TAG_CLOSE, '', 'title'], - [TokenType.EOF], + [lex.TokenType.TAG_OPEN_START, '', 'title'], + [lex.TokenType.TAG_OPEN_END], + [lex.TokenType.ESCAPABLE_RAW_TEXT, 'abc\ndef\nghi\tjkl`\'"mno'], + [lex.TokenType.TAG_CLOSE, '', 'title'], + [lex.TokenType.EOF], ]); }); it('should parse over escape sequences in tag definitions', () => { expect(tokenizeAndHumanizeParts('', {escapedString: true})) .toEqual([ - [TokenType.TAG_OPEN_START, '', 't'], - [TokenType.ATTR_NAME, '', 'a'], - [TokenType.ATTR_QUOTE, '"'], - [TokenType.ATTR_VALUE_TEXT, 'b'], - [TokenType.ATTR_QUOTE, '"'], - [TokenType.ATTR_NAME, '', 'c'], - [TokenType.ATTR_QUOTE, '\''], - [TokenType.ATTR_VALUE_TEXT, 'd'], - [TokenType.ATTR_QUOTE, '\''], - [TokenType.TAG_OPEN_END], - [TokenType.EOF], + [lex.TokenType.TAG_OPEN_START, '', 't'], + [lex.TokenType.ATTR_NAME, '', 'a'], + [lex.TokenType.ATTR_QUOTE, '"'], + [lex.TokenType.ATTR_VALUE_TEXT, 'b'], + [lex.TokenType.ATTR_QUOTE, '"'], + [lex.TokenType.ATTR_NAME, '', 'c'], + [lex.TokenType.ATTR_QUOTE, '\''], + [lex.TokenType.ATTR_VALUE_TEXT, 'd'], + [lex.TokenType.ATTR_QUOTE, '\''], + [lex.TokenType.TAG_OPEN_END], + [lex.TokenType.EOF], ]); }); it('should parse over escaped new line in tag definitions', () => { const text = ''; expect(tokenizeAndHumanizeParts(text, {escapedString: true})).toEqual([ - [TokenType.TAG_OPEN_START, '', 't'], - [TokenType.TAG_OPEN_END], - [TokenType.TAG_CLOSE, '', 't'], - [TokenType.EOF], + [lex.TokenType.TAG_OPEN_START, '', 't'], + [lex.TokenType.TAG_OPEN_END], + [lex.TokenType.TAG_CLOSE, '', 't'], + [lex.TokenType.EOF], ]); }); it('should parse over escaped characters in tag definitions', () => { const text = ''; expect(tokenizeAndHumanizeParts(text, {escapedString: true})).toEqual([ - [TokenType.TAG_OPEN_START, '', 't'], - [TokenType.TAG_OPEN_END], - [TokenType.TAG_CLOSE, '', 't'], - [TokenType.EOF], + [lex.TokenType.TAG_OPEN_START, '', 't'], + [lex.TokenType.TAG_OPEN_END], + [lex.TokenType.TAG_CLOSE, '', 't'], + [lex.TokenType.EOF], ]); }); it('should unescape characters in tag names', () => { const text = ''; expect(tokenizeAndHumanizeParts(text, {escapedString: true})).toEqual([ - [TokenType.TAG_OPEN_START, '', 'td'], - [TokenType.TAG_OPEN_END], - [TokenType.TAG_CLOSE, '', 'td'], - [TokenType.EOF], + [lex.TokenType.TAG_OPEN_START, '', 'td'], + [lex.TokenType.TAG_OPEN_END], + [lex.TokenType.TAG_CLOSE, '', 'td'], + [lex.TokenType.EOF], ]); expect(tokenizeAndHumanizeSourceSpans(text, {escapedString: true})).toEqual([ - [TokenType.TAG_OPEN_START, ''], - [TokenType.TAG_CLOSE, ''], - [TokenType.EOF, ''], + [lex.TokenType.TAG_OPEN_START, ''], + [lex.TokenType.TAG_CLOSE, ''], + [lex.TokenType.EOF, ''], ]); }); it('should unescape characters in attributes', () => { const text = ''; expect(tokenizeAndHumanizeParts(text, {escapedString: true})).toEqual([ - [TokenType.TAG_OPEN_START, '', 't'], - [TokenType.ATTR_NAME, '', 'd'], - [TokenType.ATTR_QUOTE, '"'], - [TokenType.ATTR_VALUE_TEXT, 'e'], - [TokenType.ATTR_QUOTE, '"'], - [TokenType.TAG_OPEN_END], - [TokenType.TAG_CLOSE, '', 't'], - [TokenType.EOF], + [lex.TokenType.TAG_OPEN_START, '', 't'], + [lex.TokenType.ATTR_NAME, '', 'd'], + [lex.TokenType.ATTR_QUOTE, '"'], + [lex.TokenType.ATTR_VALUE_TEXT, 'e'], + [lex.TokenType.ATTR_QUOTE, '"'], + [lex.TokenType.TAG_OPEN_END], + [lex.TokenType.TAG_CLOSE, '', 't'], + [lex.TokenType.EOF], ]); }); it('should parse over escaped new line in attribute values', () => { const text = ''; expect(tokenizeAndHumanizeParts(text, {escapedString: true})).toEqual([ - [TokenType.TAG_OPEN_START, '', 't'], - [TokenType.ATTR_NAME, '', 'a'], - [TokenType.ATTR_VALUE_TEXT, 'b'], - [TokenType.TAG_OPEN_END], - [TokenType.TAG_CLOSE, '', 't'], - [TokenType.EOF], + [lex.TokenType.TAG_OPEN_START, '', 't'], + [lex.TokenType.ATTR_NAME, '', 'a'], + [lex.TokenType.ATTR_VALUE_TEXT, 'b'], + [lex.TokenType.TAG_OPEN_END], + [lex.TokenType.TAG_CLOSE, '', 't'], + [lex.TokenType.EOF], ]); }); @@ -1711,12 +1710,12 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u endPos: 59, }; expect(tokenizeAndHumanizeParts(text, {range, escapedString: true})).toEqual([ - [TokenType.TEXT, 'line 1\n"line 2"\nline 3'], - [TokenType.EOF], + [lex.TokenType.TEXT, 'line 1\n"line 2"\nline 3'], + [lex.TokenType.EOF], ]); expect(tokenizeAndHumanizeSourceSpans(text, {range, escapedString: true})).toEqual([ - [TokenType.TEXT, 'line 1\\n\\"line 2\\"\\nline 3'], - [TokenType.EOF, ''], + [lex.TokenType.TEXT, 'line 1\\n\\"line 2\\"\\nline 3'], + [lex.TokenType.EOF, ''], ]); }); @@ -1726,57 +1725,61 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u 'line 3\\\n' + // <- line continuation ''; expect(tokenizeAndHumanizeParts(text, {escapedString: true})).toEqual([ - [TokenType.TAG_OPEN_START, '', 't'], [TokenType.TAG_OPEN_END], [TokenType.TEXT, 'line 1'], - [TokenType.TAG_CLOSE, '', 't'], [TokenType.TEXT, '\n'], + [lex.TokenType.TAG_OPEN_START, '', 't'], [lex.TokenType.TAG_OPEN_END], + [lex.TokenType.TEXT, 'line 1'], [lex.TokenType.TAG_CLOSE, '', 't'], + [lex.TokenType.TEXT, '\n'], - [TokenType.TAG_OPEN_START, '', 't'], [TokenType.TAG_OPEN_END], [TokenType.TEXT, 'line 2'], - [TokenType.TAG_CLOSE, '', 't'], [TokenType.TEXT, '\n'], + [lex.TokenType.TAG_OPEN_START, '', 't'], [lex.TokenType.TAG_OPEN_END], + [lex.TokenType.TEXT, 'line 2'], [lex.TokenType.TAG_CLOSE, '', 't'], + [lex.TokenType.TEXT, '\n'], - [TokenType.TAG_OPEN_START, '', 't'], [TokenType.TAG_OPEN_END], - [TokenType.TEXT, 'line 3'], // <- line continuation does not appear in token - [TokenType.TAG_CLOSE, '', 't'], + [lex.TokenType.TAG_OPEN_START, '', 't'], [lex.TokenType.TAG_OPEN_END], + [lex.TokenType.TEXT, 'line 3'], // <- line continuation does not appear in token + [lex.TokenType.TAG_CLOSE, '', 't'], - [TokenType.EOF] + [lex.TokenType.EOF] ]); expect(tokenizeAndHumanizeLineColumn(text, {escapedString: true})).toEqual([ - [TokenType.TAG_OPEN_START, '0:0'], - [TokenType.TAG_OPEN_END, '0:2'], - [TokenType.TEXT, '0:3'], - [TokenType.TAG_CLOSE, '0:9'], - [TokenType.TEXT, '0:13'], // <- real newline increments the row + [lex.TokenType.TAG_OPEN_START, '0:0'], + [lex.TokenType.TAG_OPEN_END, '0:2'], + [lex.TokenType.TEXT, '0:3'], + [lex.TokenType.TAG_CLOSE, '0:9'], + [lex.TokenType.TEXT, '0:13'], // <- real newline increments the row - [TokenType.TAG_OPEN_START, '1:0'], - [TokenType.TAG_OPEN_END, '1:2'], - [TokenType.TEXT, '1:3'], - [TokenType.TAG_CLOSE, '1:9'], - [TokenType.TEXT, '1:13'], // <- escaped newline does not increment the row + [lex.TokenType.TAG_OPEN_START, '1:0'], + [lex.TokenType.TAG_OPEN_END, '1:2'], + [lex.TokenType.TEXT, '1:3'], + [lex.TokenType.TAG_CLOSE, '1:9'], + [lex.TokenType.TEXT, '1:13'], // <- escaped newline does not increment the row - [TokenType.TAG_OPEN_START, '1:15'], - [TokenType.TAG_OPEN_END, '1:17'], - [TokenType.TEXT, '1:18'], // <- the line continuation increments the row - [TokenType.TAG_CLOSE, '2:0'], + [lex.TokenType.TAG_OPEN_START, '1:15'], + [lex.TokenType.TAG_OPEN_END, '1:17'], + [lex.TokenType.TEXT, '1:18'], // <- the line continuation increments the row + [lex.TokenType.TAG_CLOSE, '2:0'], - [TokenType.EOF, '2:4'], + [lex.TokenType.EOF, '2:4'], ]); expect(tokenizeAndHumanizeSourceSpans(text, {escapedString: true})).toEqual([ - [TokenType.TAG_OPEN_START, ''], - [TokenType.TEXT, 'line 1'], [TokenType.TAG_CLOSE, ''], [TokenType.TEXT, '\n'], + [lex.TokenType.TAG_OPEN_START, ''], + [lex.TokenType.TEXT, 'line 1'], [lex.TokenType.TAG_CLOSE, ''], + [lex.TokenType.TEXT, '\n'], - [TokenType.TAG_OPEN_START, ''], - [TokenType.TEXT, 'line 2'], [TokenType.TAG_CLOSE, ''], [TokenType.TEXT, '\\n'], + [lex.TokenType.TAG_OPEN_START, ''], + [lex.TokenType.TEXT, 'line 2'], [lex.TokenType.TAG_CLOSE, ''], + [lex.TokenType.TEXT, '\\n'], - [TokenType.TAG_OPEN_START, ''], - [TokenType.TEXT, 'line 3\\\n'], [TokenType.TAG_CLOSE, ''], + [lex.TokenType.TAG_OPEN_START, ''], + [lex.TokenType.TEXT, 'line 3\\\n'], [lex.TokenType.TAG_CLOSE, ''], - [TokenType.EOF, ''] + [lex.TokenType.EOF, ''] ]); }); }); }); } -function tokenizeWithoutErrors(input: string, options?: TokenizeOptions): TokenizeResult { - const tokenizeResult = tokenize(input, 'someUrl', getHtmlTagDefinition, options); +function tokenizeWithoutErrors(input: string, options?: lex.TokenizeOptions): lex.TokenizeResult { + const tokenizeResult = lex.tokenize(input, 'someUrl', getHtmlTagDefinition, options); if (tokenizeResult.errors.length > 0) { const errorString = tokenizeResult.errors.join('\n'); @@ -1786,15 +1789,15 @@ function tokenizeWithoutErrors(input: string, options?: TokenizeOptions): Tokeni return tokenizeResult; } -function humanizeParts(tokens: Token[]) { - return tokens.map(token => [token.type, ...token.parts]); +function humanizeParts(tokens: lex.Token[]) { + return tokens.map(token => [token.type, ...token.parts] as [lex.TokenType, ...string[]]); } -function tokenizeAndHumanizeParts(input: string, options?: TokenizeOptions): any[] { +function tokenizeAndHumanizeParts(input: string, options?: lex.TokenizeOptions): any[] { return humanizeParts(tokenizeWithoutErrors(input, options).tokens); } -function tokenizeAndHumanizeSourceSpans(input: string, options?: TokenizeOptions): any[] { +function tokenizeAndHumanizeSourceSpans(input: string, options?: lex.TokenizeOptions): any[] { return tokenizeWithoutErrors(input, options) .tokens.map(token => [token.type, token.sourceSpan.toString()]); } @@ -1803,12 +1806,12 @@ function humanizeLineColumn(location: ParseLocation): string { return `${location.line}:${location.col}`; } -function tokenizeAndHumanizeLineColumn(input: string, options?: TokenizeOptions): any[] { +function tokenizeAndHumanizeLineColumn(input: string, options?: lex.TokenizeOptions): any[] { return tokenizeWithoutErrors(input, options) .tokens.map(token => [token.type, humanizeLineColumn(token.sourceSpan.start)]); } -function tokenizeAndHumanizeFullStart(input: string, options?: TokenizeOptions): any[] { +function tokenizeAndHumanizeFullStart(input: string, options?: lex.TokenizeOptions): any[] { return tokenizeWithoutErrors(input, options) .tokens.map( token => @@ -1816,7 +1819,7 @@ function tokenizeAndHumanizeFullStart(input: string, options?: TokenizeOptions): humanizeLineColumn(token.sourceSpan.fullStart)]); } -function tokenizeAndHumanizeErrors(input: string, options?: TokenizeOptions): any[] { - return tokenize(input, 'someUrl', getHtmlTagDefinition, options) +function tokenizeAndHumanizeErrors(input: string, options?: lex.TokenizeOptions): any[] { + return lex.tokenize(input, 'someUrl', getHtmlTagDefinition, options) .errors.map(e => [e.tokenType, e.msg, humanizeLineColumn(e.span.start)]); }