2016-05-20 15:44:58 -07:00
|
|
|
import {ListWrapper} from './facade/collection';
|
2016-06-08 16:38:52 -07:00
|
|
|
import {NumberWrapper, StringWrapper, isBlank, isPresent} from './facade/lang';
|
|
|
|
import {HtmlTagContentType, NAMED_ENTITIES, getHtmlTagDefinition} from './html_tags';
|
|
|
|
import {ParseError, ParseLocation, ParseSourceFile, ParseSourceSpan} from './parse_util';
|
2015-10-07 09:34:21 -07:00
|
|
|
|
|
|
|
export enum HtmlTokenType {
|
|
|
|
TAG_OPEN_START,
|
|
|
|
TAG_OPEN_END,
|
|
|
|
TAG_OPEN_END_VOID,
|
|
|
|
TAG_CLOSE,
|
|
|
|
TEXT,
|
|
|
|
ESCAPABLE_RAW_TEXT,
|
|
|
|
RAW_TEXT,
|
|
|
|
COMMENT_START,
|
|
|
|
COMMENT_END,
|
|
|
|
CDATA_START,
|
|
|
|
CDATA_END,
|
|
|
|
ATTR_NAME,
|
|
|
|
ATTR_VALUE,
|
|
|
|
DOC_TYPE,
|
2016-04-12 11:46:28 -07:00
|
|
|
EXPANSION_FORM_START,
|
|
|
|
EXPANSION_CASE_VALUE,
|
|
|
|
EXPANSION_CASE_EXP_START,
|
|
|
|
EXPANSION_CASE_EXP_END,
|
|
|
|
EXPANSION_FORM_END,
|
2015-10-07 09:34:21 -07:00
|
|
|
EOF
|
|
|
|
}
|
|
|
|
|
|
|
|
export class HtmlToken {
|
2016-06-08 16:38:52 -07:00
|
|
|
constructor(
|
|
|
|
public type: HtmlTokenType, public parts: string[], public sourceSpan: ParseSourceSpan) {}
|
2015-10-07 09:34:21 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
export class HtmlTokenError extends ParseError {
|
2016-02-16 16:46:51 -08:00
|
|
|
constructor(errorMsg: string, public tokenType: HtmlTokenType, span: ParseSourceSpan) {
|
|
|
|
super(span, errorMsg);
|
2015-10-07 09:34:21 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
export class HtmlTokenizeResult {
|
|
|
|
constructor(public tokens: HtmlToken[], public errors: HtmlTokenError[]) {}
|
|
|
|
}
|
|
|
|
|
2016-06-08 16:38:52 -07:00
|
|
|
export function tokenizeHtml(
|
|
|
|
sourceContent: string, sourceUrl: string,
|
|
|
|
tokenizeExpansionForms: boolean = false): HtmlTokenizeResult {
|
2016-04-12 11:46:28 -07:00
|
|
|
return new _HtmlTokenizer(new ParseSourceFile(sourceContent, sourceUrl), tokenizeExpansionForms)
|
|
|
|
.tokenize();
|
2015-10-07 09:34:21 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
const $EOF = 0;
|
|
|
|
const $TAB = 9;
|
|
|
|
const $LF = 10;
|
2015-11-10 15:56:25 -08:00
|
|
|
const $FF = 12;
|
2015-10-07 09:34:21 -07:00
|
|
|
const $CR = 13;
|
|
|
|
|
|
|
|
const $SPACE = 32;
|
|
|
|
|
|
|
|
const $BANG = 33;
|
|
|
|
const $DQ = 34;
|
2015-11-10 15:56:25 -08:00
|
|
|
const $HASH = 35;
|
2015-10-07 09:34:21 -07:00
|
|
|
const $$ = 36;
|
|
|
|
const $AMPERSAND = 38;
|
|
|
|
const $SQ = 39;
|
|
|
|
const $MINUS = 45;
|
|
|
|
const $SLASH = 47;
|
|
|
|
const $0 = 48;
|
|
|
|
|
|
|
|
const $SEMICOLON = 59;
|
|
|
|
|
|
|
|
const $9 = 57;
|
|
|
|
const $COLON = 58;
|
|
|
|
const $LT = 60;
|
|
|
|
const $EQ = 61;
|
|
|
|
const $GT = 62;
|
|
|
|
const $QUESTION = 63;
|
|
|
|
const $LBRACKET = 91;
|
|
|
|
const $RBRACKET = 93;
|
2016-04-12 11:46:28 -07:00
|
|
|
const $LBRACE = 123;
|
|
|
|
const $RBRACE = 125;
|
|
|
|
const $COMMA = 44;
|
2015-12-21 11:32:58 -08:00
|
|
|
const $A = 65;
|
|
|
|
const $F = 70;
|
|
|
|
const $X = 88;
|
|
|
|
const $Z = 90;
|
|
|
|
|
2015-10-07 09:34:21 -07:00
|
|
|
const $a = 97;
|
2015-11-10 15:56:25 -08:00
|
|
|
const $f = 102;
|
2015-10-07 09:34:21 -07:00
|
|
|
const $z = 122;
|
2015-11-10 15:56:25 -08:00
|
|
|
const $x = 120;
|
2015-10-07 09:34:21 -07:00
|
|
|
|
|
|
|
const $NBSP = 160;
|
|
|
|
|
2015-12-13 17:40:06 +02:00
|
|
|
var CR_OR_CRLF_REGEXP = /\r\n?/g;
|
2015-12-04 23:12:31 -08:00
|
|
|
|
2015-10-07 09:34:21 -07:00
|
|
|
function unexpectedCharacterErrorMsg(charCode: number): string {
|
|
|
|
var char = charCode === $EOF ? 'EOF' : StringWrapper.fromCharCode(charCode);
|
|
|
|
return `Unexpected character "${char}"`;
|
|
|
|
}
|
|
|
|
|
|
|
|
function unknownEntityErrorMsg(entitySrc: string): string {
|
2015-11-10 15:56:25 -08:00
|
|
|
return `Unknown entity "${entitySrc}" - use the "&#<decimal>;" or "&#x<hex>;" syntax`;
|
2015-10-07 09:34:21 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
class ControlFlowError {
|
|
|
|
constructor(public error: HtmlTokenError) {}
|
|
|
|
}
|
|
|
|
|
|
|
|
// See http://www.w3.org/TR/html51/syntax.html#writing
|
|
|
|
class _HtmlTokenizer {
|
2016-06-09 13:48:53 -07:00
|
|
|
private _input: string;
|
|
|
|
private _length: number;
|
2015-10-07 09:34:21 -07:00
|
|
|
// Note: this is always lowercase!
|
2016-06-09 13:48:53 -07:00
|
|
|
private _peek: number = -1;
|
|
|
|
private _nextPeek: number = -1;
|
|
|
|
private _index: number = -1;
|
|
|
|
private _line: number = 0;
|
|
|
|
private _column: number = -1;
|
|
|
|
private _currentTokenStart: ParseLocation;
|
|
|
|
private _currentTokenType: HtmlTokenType;
|
|
|
|
private _expansionCaseStack: HtmlTokenType[] = [];
|
2016-04-12 11:46:28 -07:00
|
|
|
|
2015-10-07 09:34:21 -07:00
|
|
|
tokens: HtmlToken[] = [];
|
|
|
|
errors: HtmlTokenError[] = [];
|
|
|
|
|
2016-04-12 11:46:28 -07:00
|
|
|
constructor(private file: ParseSourceFile, private tokenizeExpansionForms: boolean) {
|
2016-06-09 13:48:53 -07:00
|
|
|
this._input = file.content;
|
|
|
|
this._length = file.content.length;
|
2015-10-07 09:34:21 -07:00
|
|
|
this._advance();
|
|
|
|
}
|
|
|
|
|
2015-12-04 23:12:31 -08:00
|
|
|
private _processCarriageReturns(content: string): string {
|
|
|
|
// http://www.w3.org/TR/html5/syntax.html#preprocessing-the-input-stream
|
2016-02-16 16:46:51 -08:00
|
|
|
// In order to keep the original position in the source, we can not
|
|
|
|
// pre-process it.
|
2015-12-04 23:12:31 -08:00
|
|
|
// Instead CRs are processed right before instantiating the tokens.
|
2015-12-13 17:40:06 +02:00
|
|
|
return StringWrapper.replaceAll(content, CR_OR_CRLF_REGEXP, '\n');
|
2015-12-04 23:12:31 -08:00
|
|
|
}
|
|
|
|
|
2015-10-07 09:34:21 -07:00
|
|
|
tokenize(): HtmlTokenizeResult {
|
2016-06-09 13:48:53 -07:00
|
|
|
while (this._peek !== $EOF) {
|
2015-10-07 09:34:21 -07:00
|
|
|
var start = this._getLocation();
|
|
|
|
try {
|
2015-12-21 11:32:58 -08:00
|
|
|
if (this._attemptCharCode($LT)) {
|
|
|
|
if (this._attemptCharCode($BANG)) {
|
|
|
|
if (this._attemptCharCode($LBRACKET)) {
|
2015-10-07 09:34:21 -07:00
|
|
|
this._consumeCdata(start);
|
2015-12-21 11:32:58 -08:00
|
|
|
} else if (this._attemptCharCode($MINUS)) {
|
2015-10-07 09:34:21 -07:00
|
|
|
this._consumeComment(start);
|
|
|
|
} else {
|
|
|
|
this._consumeDocType(start);
|
|
|
|
}
|
2015-12-21 11:32:58 -08:00
|
|
|
} else if (this._attemptCharCode($SLASH)) {
|
2015-10-07 09:34:21 -07:00
|
|
|
this._consumeTagClose(start);
|
|
|
|
} else {
|
|
|
|
this._consumeTagOpen(start);
|
|
|
|
}
|
2016-06-09 13:48:53 -07:00
|
|
|
} else if (
|
|
|
|
isExpansionFormStart(this._peek, this._nextPeek) && this.tokenizeExpansionForms) {
|
2016-04-12 11:46:28 -07:00
|
|
|
this._consumeExpansionFormStart();
|
|
|
|
|
2016-06-09 13:48:53 -07:00
|
|
|
} else if (
|
|
|
|
isExpansionCaseStart(this._peek) && this._isInExpansionForm() &&
|
|
|
|
this.tokenizeExpansionForms) {
|
2016-04-12 11:46:28 -07:00
|
|
|
this._consumeExpansionCaseStart();
|
|
|
|
|
2016-06-09 14:53:03 -07:00
|
|
|
} else if (
|
|
|
|
this._peek === $RBRACE && this._isInExpansionCase() && this.tokenizeExpansionForms) {
|
2016-04-12 11:46:28 -07:00
|
|
|
this._consumeExpansionCaseEnd();
|
|
|
|
|
2016-06-09 14:53:03 -07:00
|
|
|
} else if (
|
|
|
|
this._peek === $RBRACE && this._isInExpansionForm() && this.tokenizeExpansionForms) {
|
2016-04-12 11:46:28 -07:00
|
|
|
this._consumeExpansionFormEnd();
|
|
|
|
|
2015-10-07 09:34:21 -07:00
|
|
|
} else {
|
|
|
|
this._consumeText();
|
|
|
|
}
|
|
|
|
} catch (e) {
|
|
|
|
if (e instanceof ControlFlowError) {
|
|
|
|
this.errors.push(e.error);
|
|
|
|
} else {
|
|
|
|
throw e;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
this._beginToken(HtmlTokenType.EOF);
|
|
|
|
this._endToken([]);
|
2015-12-06 13:11:00 -08:00
|
|
|
return new HtmlTokenizeResult(mergeTextTokens(this.tokens), this.errors);
|
2015-10-07 09:34:21 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
private _getLocation(): ParseLocation {
|
2016-06-09 13:48:53 -07:00
|
|
|
return new ParseLocation(this.file, this._index, this._line, this._column);
|
2015-10-07 09:34:21 -07:00
|
|
|
}
|
|
|
|
|
2016-02-16 16:46:51 -08:00
|
|
|
private _getSpan(start?: ParseLocation, end?: ParseLocation): ParseSourceSpan {
|
|
|
|
if (isBlank(start)) {
|
|
|
|
start = this._getLocation();
|
|
|
|
}
|
|
|
|
if (isBlank(end)) {
|
|
|
|
end = this._getLocation();
|
|
|
|
}
|
|
|
|
return new ParseSourceSpan(start, end);
|
|
|
|
}
|
|
|
|
|
2015-10-07 09:34:21 -07:00
|
|
|
private _beginToken(type: HtmlTokenType, start: ParseLocation = null) {
|
|
|
|
if (isBlank(start)) {
|
|
|
|
start = this._getLocation();
|
|
|
|
}
|
2016-06-09 13:48:53 -07:00
|
|
|
this._currentTokenStart = start;
|
|
|
|
this._currentTokenType = type;
|
2015-10-07 09:34:21 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
private _endToken(parts: string[], end: ParseLocation = null): HtmlToken {
|
|
|
|
if (isBlank(end)) {
|
|
|
|
end = this._getLocation();
|
|
|
|
}
|
2016-06-09 14:53:03 -07:00
|
|
|
var token = new HtmlToken(
|
|
|
|
this._currentTokenType, parts, new ParseSourceSpan(this._currentTokenStart, end));
|
2015-10-07 09:34:21 -07:00
|
|
|
this.tokens.push(token);
|
2016-06-09 13:48:53 -07:00
|
|
|
this._currentTokenStart = null;
|
|
|
|
this._currentTokenType = null;
|
2015-10-07 09:34:21 -07:00
|
|
|
return token;
|
|
|
|
}
|
|
|
|
|
2016-02-16 16:46:51 -08:00
|
|
|
private _createError(msg: string, span: ParseSourceSpan): ControlFlowError {
|
2016-06-09 13:48:53 -07:00
|
|
|
var error = new HtmlTokenError(msg, this._currentTokenType, span);
|
|
|
|
this._currentTokenStart = null;
|
|
|
|
this._currentTokenType = null;
|
2015-10-07 09:34:21 -07:00
|
|
|
return new ControlFlowError(error);
|
|
|
|
}
|
|
|
|
|
|
|
|
private _advance() {
|
2016-06-09 13:48:53 -07:00
|
|
|
if (this._index >= this._length) {
|
2016-02-16 16:46:51 -08:00
|
|
|
throw this._createError(unexpectedCharacterErrorMsg($EOF), this._getSpan());
|
2015-10-07 09:34:21 -07:00
|
|
|
}
|
2016-06-09 13:48:53 -07:00
|
|
|
if (this._peek === $LF) {
|
|
|
|
this._line++;
|
|
|
|
this._column = 0;
|
|
|
|
} else if (this._peek !== $LF && this._peek !== $CR) {
|
|
|
|
this._column++;
|
2015-10-07 09:34:21 -07:00
|
|
|
}
|
2016-06-09 13:48:53 -07:00
|
|
|
this._index++;
|
2016-06-09 14:53:03 -07:00
|
|
|
this._peek =
|
|
|
|
this._index >= this._length ? $EOF : StringWrapper.charCodeAt(this._input, this._index);
|
|
|
|
this._nextPeek = this._index + 1 >= this._length ?
|
|
|
|
$EOF :
|
|
|
|
StringWrapper.charCodeAt(this._input, this._index + 1);
|
2015-10-07 09:34:21 -07:00
|
|
|
}
|
|
|
|
|
2015-12-21 11:32:58 -08:00
|
|
|
private _attemptCharCode(charCode: number): boolean {
|
2016-06-09 13:48:53 -07:00
|
|
|
if (this._peek === charCode) {
|
2015-10-07 09:34:21 -07:00
|
|
|
this._advance();
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2015-12-21 11:32:58 -08:00
|
|
|
private _attemptCharCodeCaseInsensitive(charCode: number): boolean {
|
2016-06-09 13:48:53 -07:00
|
|
|
if (compareCharCodeCaseInsensitive(this._peek, charCode)) {
|
2015-12-21 11:32:58 -08:00
|
|
|
this._advance();
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
private _requireCharCode(charCode: number) {
|
2015-10-07 09:34:21 -07:00
|
|
|
var location = this._getLocation();
|
2015-12-21 11:32:58 -08:00
|
|
|
if (!this._attemptCharCode(charCode)) {
|
2016-06-09 14:53:03 -07:00
|
|
|
throw this._createError(
|
|
|
|
unexpectedCharacterErrorMsg(this._peek), this._getSpan(location, location));
|
2015-10-07 09:34:21 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-12-21 11:32:58 -08:00
|
|
|
private _attemptStr(chars: string): boolean {
|
2016-06-09 13:48:53 -07:00
|
|
|
var indexBeforeAttempt = this._index;
|
|
|
|
var columnBeforeAttempt = this._column;
|
|
|
|
var lineBeforeAttempt = this._line;
|
2015-10-07 09:34:21 -07:00
|
|
|
for (var i = 0; i < chars.length; i++) {
|
2015-12-21 11:32:58 -08:00
|
|
|
if (!this._attemptCharCode(StringWrapper.charCodeAt(chars, i))) {
|
2016-02-17 14:52:51 +01:00
|
|
|
// If attempting to parse the string fails, we want to reset the parser
|
|
|
|
// to where it was before the attempt
|
2016-06-09 13:48:53 -07:00
|
|
|
this._index = indexBeforeAttempt;
|
|
|
|
this._column = columnBeforeAttempt;
|
|
|
|
this._line = lineBeforeAttempt;
|
2015-10-07 09:34:21 -07:00
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2015-12-21 11:32:58 -08:00
|
|
|
private _attemptStrCaseInsensitive(chars: string): boolean {
|
|
|
|
for (var i = 0; i < chars.length; i++) {
|
|
|
|
if (!this._attemptCharCodeCaseInsensitive(StringWrapper.charCodeAt(chars, i))) {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
private _requireStr(chars: string) {
|
2015-10-07 09:34:21 -07:00
|
|
|
var location = this._getLocation();
|
2015-12-21 11:32:58 -08:00
|
|
|
if (!this._attemptStr(chars)) {
|
2016-06-09 13:48:53 -07:00
|
|
|
throw this._createError(unexpectedCharacterErrorMsg(this._peek), this._getSpan(location));
|
2015-10-07 09:34:21 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-12-21 11:32:58 -08:00
|
|
|
private _attemptCharCodeUntilFn(predicate: Function) {
|
2016-06-09 13:48:53 -07:00
|
|
|
while (!predicate(this._peek)) {
|
2015-10-07 09:34:21 -07:00
|
|
|
this._advance();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-12-21 11:32:58 -08:00
|
|
|
private _requireCharCodeUntilFn(predicate: Function, len: number) {
|
2015-10-07 09:34:21 -07:00
|
|
|
var start = this._getLocation();
|
2015-12-21 11:32:58 -08:00
|
|
|
this._attemptCharCodeUntilFn(predicate);
|
2016-06-09 13:48:53 -07:00
|
|
|
if (this._index - start.offset < len) {
|
|
|
|
throw this._createError(unexpectedCharacterErrorMsg(this._peek), this._getSpan(start, start));
|
2015-10-07 09:34:21 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
private _attemptUntilChar(char: number) {
|
2016-06-09 13:48:53 -07:00
|
|
|
while (this._peek !== char) {
|
2015-10-07 09:34:21 -07:00
|
|
|
this._advance();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
private _readChar(decodeEntities: boolean): string {
|
2016-06-09 13:48:53 -07:00
|
|
|
if (decodeEntities && this._peek === $AMPERSAND) {
|
2015-11-10 15:56:25 -08:00
|
|
|
return this._decodeEntity();
|
2015-10-07 09:34:21 -07:00
|
|
|
} else {
|
2016-06-09 13:48:53 -07:00
|
|
|
var index = this._index;
|
2015-10-07 09:34:21 -07:00
|
|
|
this._advance();
|
2016-06-09 13:48:53 -07:00
|
|
|
return this._input[index];
|
2015-10-07 09:34:21 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-11-10 15:56:25 -08:00
|
|
|
private _decodeEntity(): string {
|
|
|
|
var start = this._getLocation();
|
|
|
|
this._advance();
|
2015-12-21 11:32:58 -08:00
|
|
|
if (this._attemptCharCode($HASH)) {
|
|
|
|
let isHex = this._attemptCharCode($x) || this._attemptCharCode($X);
|
2015-11-10 15:56:25 -08:00
|
|
|
let numberStart = this._getLocation().offset;
|
2015-12-21 11:32:58 -08:00
|
|
|
this._attemptCharCodeUntilFn(isDigitEntityEnd);
|
2016-06-09 13:48:53 -07:00
|
|
|
if (this._peek != $SEMICOLON) {
|
|
|
|
throw this._createError(unexpectedCharacterErrorMsg(this._peek), this._getSpan());
|
2015-11-10 15:56:25 -08:00
|
|
|
}
|
|
|
|
this._advance();
|
2016-06-09 13:48:53 -07:00
|
|
|
let strNum = this._input.substring(numberStart, this._index - 1);
|
2015-11-10 15:56:25 -08:00
|
|
|
try {
|
|
|
|
let charCode = NumberWrapper.parseInt(strNum, isHex ? 16 : 10);
|
|
|
|
return StringWrapper.fromCharCode(charCode);
|
|
|
|
} catch (e) {
|
2016-06-09 13:48:53 -07:00
|
|
|
let entity = this._input.substring(start.offset + 1, this._index - 1);
|
2016-02-16 16:46:51 -08:00
|
|
|
throw this._createError(unknownEntityErrorMsg(entity), this._getSpan(start));
|
2015-11-10 15:56:25 -08:00
|
|
|
}
|
|
|
|
} else {
|
|
|
|
let startPosition = this._savePosition();
|
2015-12-21 11:32:58 -08:00
|
|
|
this._attemptCharCodeUntilFn(isNamedEntityEnd);
|
2016-06-09 13:48:53 -07:00
|
|
|
if (this._peek != $SEMICOLON) {
|
2015-11-10 15:56:25 -08:00
|
|
|
this._restorePosition(startPosition);
|
|
|
|
return '&';
|
|
|
|
}
|
|
|
|
this._advance();
|
2016-06-09 13:48:53 -07:00
|
|
|
let name = this._input.substring(start.offset + 1, this._index - 1);
|
2016-06-11 21:23:37 -07:00
|
|
|
let char = (NAMED_ENTITIES as any)[name];
|
2015-11-10 15:56:25 -08:00
|
|
|
if (isBlank(char)) {
|
2016-02-16 16:46:51 -08:00
|
|
|
throw this._createError(unknownEntityErrorMsg(name), this._getSpan(start));
|
2015-11-10 15:56:25 -08:00
|
|
|
}
|
|
|
|
return char;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-06-08 16:38:52 -07:00
|
|
|
private _consumeRawText(
|
|
|
|
decodeEntities: boolean, firstCharOfEnd: number, attemptEndRest: Function): HtmlToken {
|
2016-06-11 21:23:37 -07:00
|
|
|
var tagCloseStart: ParseLocation;
|
2015-10-07 09:34:21 -07:00
|
|
|
var textStart = this._getLocation();
|
2016-06-08 16:38:52 -07:00
|
|
|
this._beginToken(
|
|
|
|
decodeEntities ? HtmlTokenType.ESCAPABLE_RAW_TEXT : HtmlTokenType.RAW_TEXT, textStart);
|
2016-06-11 21:23:37 -07:00
|
|
|
var parts: string[] = [];
|
2015-10-07 09:34:21 -07:00
|
|
|
while (true) {
|
|
|
|
tagCloseStart = this._getLocation();
|
2015-12-21 11:32:58 -08:00
|
|
|
if (this._attemptCharCode(firstCharOfEnd) && attemptEndRest()) {
|
2015-10-07 09:34:21 -07:00
|
|
|
break;
|
|
|
|
}
|
2016-06-09 13:48:53 -07:00
|
|
|
if (this._index > tagCloseStart.offset) {
|
|
|
|
parts.push(this._input.substring(tagCloseStart.offset, this._index));
|
2015-10-07 09:34:21 -07:00
|
|
|
}
|
2016-06-09 13:48:53 -07:00
|
|
|
while (this._peek !== firstCharOfEnd) {
|
2015-10-07 09:34:21 -07:00
|
|
|
parts.push(this._readChar(decodeEntities));
|
|
|
|
}
|
|
|
|
}
|
2015-12-04 23:12:31 -08:00
|
|
|
return this._endToken([this._processCarriageReturns(parts.join(''))], tagCloseStart);
|
2015-10-07 09:34:21 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
private _consumeComment(start: ParseLocation) {
|
|
|
|
this._beginToken(HtmlTokenType.COMMENT_START, start);
|
2015-12-21 11:32:58 -08:00
|
|
|
this._requireCharCode($MINUS);
|
2015-10-07 09:34:21 -07:00
|
|
|
this._endToken([]);
|
2015-12-21 11:32:58 -08:00
|
|
|
var textToken = this._consumeRawText(false, $MINUS, () => this._attemptStr('->'));
|
2015-10-07 09:34:21 -07:00
|
|
|
this._beginToken(HtmlTokenType.COMMENT_END, textToken.sourceSpan.end);
|
|
|
|
this._endToken([]);
|
|
|
|
}
|
|
|
|
|
|
|
|
private _consumeCdata(start: ParseLocation) {
|
|
|
|
this._beginToken(HtmlTokenType.CDATA_START, start);
|
2015-12-21 11:32:58 -08:00
|
|
|
this._requireStr('CDATA[');
|
2015-10-07 09:34:21 -07:00
|
|
|
this._endToken([]);
|
2015-12-21 11:32:58 -08:00
|
|
|
var textToken = this._consumeRawText(false, $RBRACKET, () => this._attemptStr(']>'));
|
2015-10-07 09:34:21 -07:00
|
|
|
this._beginToken(HtmlTokenType.CDATA_END, textToken.sourceSpan.end);
|
|
|
|
this._endToken([]);
|
|
|
|
}
|
|
|
|
|
|
|
|
private _consumeDocType(start: ParseLocation) {
|
|
|
|
this._beginToken(HtmlTokenType.DOC_TYPE, start);
|
|
|
|
this._attemptUntilChar($GT);
|
|
|
|
this._advance();
|
2016-06-09 13:48:53 -07:00
|
|
|
this._endToken([this._input.substring(start.offset + 2, this._index - 1)]);
|
2015-10-07 09:34:21 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
private _consumePrefixAndName(): string[] {
|
2016-06-09 13:48:53 -07:00
|
|
|
var nameOrPrefixStart = this._index;
|
2016-06-11 21:23:37 -07:00
|
|
|
var prefix: string = null;
|
2016-06-09 13:48:53 -07:00
|
|
|
while (this._peek !== $COLON && !isPrefixEnd(this._peek)) {
|
2015-10-07 09:34:21 -07:00
|
|
|
this._advance();
|
|
|
|
}
|
2016-06-11 21:23:37 -07:00
|
|
|
var nameStart: number;
|
2016-06-09 13:48:53 -07:00
|
|
|
if (this._peek === $COLON) {
|
2015-10-07 09:34:21 -07:00
|
|
|
this._advance();
|
2016-06-09 13:48:53 -07:00
|
|
|
prefix = this._input.substring(nameOrPrefixStart, this._index - 1);
|
|
|
|
nameStart = this._index;
|
2015-10-07 09:34:21 -07:00
|
|
|
} else {
|
|
|
|
nameStart = nameOrPrefixStart;
|
|
|
|
}
|
2016-06-09 13:48:53 -07:00
|
|
|
this._requireCharCodeUntilFn(isNameEnd, this._index === nameStart ? 1 : 0);
|
|
|
|
var name = this._input.substring(nameStart, this._index);
|
2015-10-07 09:34:21 -07:00
|
|
|
return [prefix, name];
|
|
|
|
}
|
|
|
|
|
|
|
|
private _consumeTagOpen(start: ParseLocation) {
|
2015-12-06 13:11:00 -08:00
|
|
|
let savedPos = this._savePosition();
|
2016-06-11 21:23:37 -07:00
|
|
|
let lowercaseTagName: string;
|
2015-12-06 13:11:00 -08:00
|
|
|
try {
|
2016-06-09 13:48:53 -07:00
|
|
|
if (!isAsciiLetter(this._peek)) {
|
|
|
|
throw this._createError(unexpectedCharacterErrorMsg(this._peek), this._getSpan());
|
2015-12-06 13:12:41 -08:00
|
|
|
}
|
2016-06-09 13:48:53 -07:00
|
|
|
var nameStart = this._index;
|
2015-12-06 13:11:00 -08:00
|
|
|
this._consumeTagOpenStart(start);
|
2016-06-09 13:48:53 -07:00
|
|
|
lowercaseTagName = this._input.substring(nameStart, this._index).toLowerCase();
|
2015-12-21 11:32:58 -08:00
|
|
|
this._attemptCharCodeUntilFn(isNotWhitespace);
|
2016-06-09 13:48:53 -07:00
|
|
|
while (this._peek !== $SLASH && this._peek !== $GT) {
|
2015-12-06 13:11:00 -08:00
|
|
|
this._consumeAttributeName();
|
2015-12-21 11:32:58 -08:00
|
|
|
this._attemptCharCodeUntilFn(isNotWhitespace);
|
|
|
|
if (this._attemptCharCode($EQ)) {
|
|
|
|
this._attemptCharCodeUntilFn(isNotWhitespace);
|
2015-12-06 13:11:00 -08:00
|
|
|
this._consumeAttributeValue();
|
|
|
|
}
|
2015-12-21 11:32:58 -08:00
|
|
|
this._attemptCharCodeUntilFn(isNotWhitespace);
|
2015-10-07 09:34:21 -07:00
|
|
|
}
|
2015-12-06 13:11:00 -08:00
|
|
|
this._consumeTagOpenEnd();
|
|
|
|
} catch (e) {
|
|
|
|
if (e instanceof ControlFlowError) {
|
|
|
|
// When the start tag is invalid, assume we want a "<"
|
|
|
|
this._restorePosition(savedPos);
|
|
|
|
// Back to back text tokens are merged at the end
|
|
|
|
this._beginToken(HtmlTokenType.TEXT, start);
|
|
|
|
this._endToken(['<']);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
throw e;
|
2015-10-07 09:34:21 -07:00
|
|
|
}
|
2015-12-06 13:11:00 -08:00
|
|
|
|
2015-10-07 09:34:21 -07:00
|
|
|
var contentTokenType = getHtmlTagDefinition(lowercaseTagName).contentType;
|
|
|
|
if (contentTokenType === HtmlTagContentType.RAW_TEXT) {
|
|
|
|
this._consumeRawTextWithTagClose(lowercaseTagName, false);
|
|
|
|
} else if (contentTokenType === HtmlTagContentType.ESCAPABLE_RAW_TEXT) {
|
|
|
|
this._consumeRawTextWithTagClose(lowercaseTagName, true);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
private _consumeRawTextWithTagClose(lowercaseTagName: string, decodeEntities: boolean) {
|
|
|
|
var textToken = this._consumeRawText(decodeEntities, $LT, () => {
|
2015-12-21 11:32:58 -08:00
|
|
|
if (!this._attemptCharCode($SLASH)) return false;
|
|
|
|
this._attemptCharCodeUntilFn(isNotWhitespace);
|
|
|
|
if (!this._attemptStrCaseInsensitive(lowercaseTagName)) return false;
|
|
|
|
this._attemptCharCodeUntilFn(isNotWhitespace);
|
|
|
|
if (!this._attemptCharCode($GT)) return false;
|
2015-10-07 09:34:21 -07:00
|
|
|
return true;
|
|
|
|
});
|
|
|
|
this._beginToken(HtmlTokenType.TAG_CLOSE, textToken.sourceSpan.end);
|
|
|
|
this._endToken([null, lowercaseTagName]);
|
|
|
|
}
|
|
|
|
|
|
|
|
private _consumeTagOpenStart(start: ParseLocation) {
|
|
|
|
this._beginToken(HtmlTokenType.TAG_OPEN_START, start);
|
|
|
|
var parts = this._consumePrefixAndName();
|
|
|
|
this._endToken(parts);
|
|
|
|
}
|
|
|
|
|
|
|
|
private _consumeAttributeName() {
|
|
|
|
this._beginToken(HtmlTokenType.ATTR_NAME);
|
|
|
|
var prefixAndName = this._consumePrefixAndName();
|
|
|
|
this._endToken(prefixAndName);
|
|
|
|
}
|
|
|
|
|
|
|
|
private _consumeAttributeValue() {
|
|
|
|
this._beginToken(HtmlTokenType.ATTR_VALUE);
|
2016-06-11 21:23:37 -07:00
|
|
|
var value: string;
|
2016-06-09 13:48:53 -07:00
|
|
|
if (this._peek === $SQ || this._peek === $DQ) {
|
|
|
|
var quoteChar = this._peek;
|
2015-10-07 09:34:21 -07:00
|
|
|
this._advance();
|
2016-06-11 21:23:37 -07:00
|
|
|
var parts: string[] = [];
|
2016-06-09 13:48:53 -07:00
|
|
|
while (this._peek !== quoteChar) {
|
2015-10-07 09:34:21 -07:00
|
|
|
parts.push(this._readChar(true));
|
|
|
|
}
|
|
|
|
value = parts.join('');
|
|
|
|
this._advance();
|
|
|
|
} else {
|
2016-06-09 13:48:53 -07:00
|
|
|
var valueStart = this._index;
|
2015-12-21 11:32:58 -08:00
|
|
|
this._requireCharCodeUntilFn(isNameEnd, 1);
|
2016-06-09 13:48:53 -07:00
|
|
|
value = this._input.substring(valueStart, this._index);
|
2015-10-07 09:34:21 -07:00
|
|
|
}
|
2015-12-04 23:12:31 -08:00
|
|
|
this._endToken([this._processCarriageReturns(value)]);
|
2015-10-07 09:34:21 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
private _consumeTagOpenEnd() {
|
2015-12-21 11:32:58 -08:00
|
|
|
var tokenType = this._attemptCharCode($SLASH) ? HtmlTokenType.TAG_OPEN_END_VOID :
|
|
|
|
HtmlTokenType.TAG_OPEN_END;
|
2015-10-07 09:34:21 -07:00
|
|
|
this._beginToken(tokenType);
|
2015-12-21 11:32:58 -08:00
|
|
|
this._requireCharCode($GT);
|
2015-10-07 09:34:21 -07:00
|
|
|
this._endToken([]);
|
|
|
|
}
|
|
|
|
|
|
|
|
private _consumeTagClose(start: ParseLocation) {
|
|
|
|
this._beginToken(HtmlTokenType.TAG_CLOSE, start);
|
2015-12-21 11:32:58 -08:00
|
|
|
this._attemptCharCodeUntilFn(isNotWhitespace);
|
2016-06-11 21:23:37 -07:00
|
|
|
let prefixAndName = this._consumePrefixAndName();
|
2015-12-21 11:32:58 -08:00
|
|
|
this._attemptCharCodeUntilFn(isNotWhitespace);
|
|
|
|
this._requireCharCode($GT);
|
2015-10-07 09:34:21 -07:00
|
|
|
this._endToken(prefixAndName);
|
|
|
|
}
|
|
|
|
|
2016-04-12 11:46:28 -07:00
|
|
|
private _consumeExpansionFormStart() {
|
|
|
|
this._beginToken(HtmlTokenType.EXPANSION_FORM_START, this._getLocation());
|
|
|
|
this._requireCharCode($LBRACE);
|
|
|
|
this._endToken([]);
|
|
|
|
|
|
|
|
this._beginToken(HtmlTokenType.RAW_TEXT, this._getLocation());
|
|
|
|
let condition = this._readUntil($COMMA);
|
|
|
|
this._endToken([condition], this._getLocation());
|
|
|
|
this._requireCharCode($COMMA);
|
|
|
|
this._attemptCharCodeUntilFn(isNotWhitespace);
|
|
|
|
|
|
|
|
this._beginToken(HtmlTokenType.RAW_TEXT, this._getLocation());
|
|
|
|
let type = this._readUntil($COMMA);
|
|
|
|
this._endToken([type], this._getLocation());
|
|
|
|
this._requireCharCode($COMMA);
|
|
|
|
this._attemptCharCodeUntilFn(isNotWhitespace);
|
|
|
|
|
2016-06-09 13:48:53 -07:00
|
|
|
this._expansionCaseStack.push(HtmlTokenType.EXPANSION_FORM_START);
|
2016-04-12 11:46:28 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
private _consumeExpansionCaseStart() {
|
|
|
|
this._beginToken(HtmlTokenType.EXPANSION_CASE_VALUE, this._getLocation());
|
|
|
|
let value = this._readUntil($LBRACE).trim();
|
|
|
|
this._endToken([value], this._getLocation());
|
|
|
|
this._attemptCharCodeUntilFn(isNotWhitespace);
|
|
|
|
|
|
|
|
this._beginToken(HtmlTokenType.EXPANSION_CASE_EXP_START, this._getLocation());
|
|
|
|
this._requireCharCode($LBRACE);
|
|
|
|
this._endToken([], this._getLocation());
|
|
|
|
this._attemptCharCodeUntilFn(isNotWhitespace);
|
|
|
|
|
2016-06-09 13:48:53 -07:00
|
|
|
this._expansionCaseStack.push(HtmlTokenType.EXPANSION_CASE_EXP_START);
|
2016-04-12 11:46:28 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
private _consumeExpansionCaseEnd() {
|
|
|
|
this._beginToken(HtmlTokenType.EXPANSION_CASE_EXP_END, this._getLocation());
|
|
|
|
this._requireCharCode($RBRACE);
|
|
|
|
this._endToken([], this._getLocation());
|
|
|
|
this._attemptCharCodeUntilFn(isNotWhitespace);
|
|
|
|
|
2016-06-09 13:48:53 -07:00
|
|
|
this._expansionCaseStack.pop();
|
2016-04-12 11:46:28 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
private _consumeExpansionFormEnd() {
|
|
|
|
this._beginToken(HtmlTokenType.EXPANSION_FORM_END, this._getLocation());
|
|
|
|
this._requireCharCode($RBRACE);
|
|
|
|
this._endToken([]);
|
|
|
|
|
2016-06-09 13:48:53 -07:00
|
|
|
this._expansionCaseStack.pop();
|
2016-04-12 11:46:28 -07:00
|
|
|
}
|
|
|
|
|
2015-10-07 09:34:21 -07:00
|
|
|
private _consumeText() {
|
|
|
|
var start = this._getLocation();
|
|
|
|
this._beginToken(HtmlTokenType.TEXT, start);
|
2016-04-12 11:46:28 -07:00
|
|
|
|
2016-06-11 21:23:37 -07:00
|
|
|
var parts: string[] = [];
|
2016-04-12 11:46:28 -07:00
|
|
|
let interpolation = false;
|
|
|
|
|
2016-06-09 13:48:53 -07:00
|
|
|
if (this._peek === $LBRACE && this._nextPeek === $LBRACE) {
|
2016-04-12 11:46:28 -07:00
|
|
|
parts.push(this._readChar(true));
|
|
|
|
parts.push(this._readChar(true));
|
|
|
|
interpolation = true;
|
|
|
|
} else {
|
2015-10-07 09:34:21 -07:00
|
|
|
parts.push(this._readChar(true));
|
|
|
|
}
|
2016-04-12 11:46:28 -07:00
|
|
|
|
2016-06-09 13:48:53 -07:00
|
|
|
while (!this._isTextEnd(interpolation)) {
|
|
|
|
if (this._peek === $LBRACE && this._nextPeek === $LBRACE) {
|
2016-04-12 11:46:28 -07:00
|
|
|
parts.push(this._readChar(true));
|
|
|
|
parts.push(this._readChar(true));
|
|
|
|
interpolation = true;
|
2016-06-09 13:48:53 -07:00
|
|
|
} else if (this._peek === $RBRACE && this._nextPeek === $RBRACE && interpolation) {
|
2016-04-12 11:46:28 -07:00
|
|
|
parts.push(this._readChar(true));
|
|
|
|
parts.push(this._readChar(true));
|
|
|
|
interpolation = false;
|
|
|
|
} else {
|
|
|
|
parts.push(this._readChar(true));
|
|
|
|
}
|
|
|
|
}
|
2015-12-04 23:12:31 -08:00
|
|
|
this._endToken([this._processCarriageReturns(parts.join(''))]);
|
2015-10-07 09:34:21 -07:00
|
|
|
}
|
2015-11-10 15:56:25 -08:00
|
|
|
|
2016-06-09 13:48:53 -07:00
|
|
|
private _isTextEnd(interpolation: boolean): boolean {
|
|
|
|
if (this._peek === $LT || this._peek === $EOF) return true;
|
2016-04-12 11:46:28 -07:00
|
|
|
if (this.tokenizeExpansionForms) {
|
2016-06-09 13:48:53 -07:00
|
|
|
if (isExpansionFormStart(this._peek, this._nextPeek)) return true;
|
|
|
|
if (this._peek === $RBRACE && !interpolation &&
|
|
|
|
(this._isInExpansionCase() || this._isInExpansionForm()))
|
2016-04-13 16:01:25 -07:00
|
|
|
return true;
|
2016-04-12 11:46:28 -07:00
|
|
|
}
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2015-12-06 13:11:00 -08:00
|
|
|
private _savePosition(): number[] {
|
2016-06-09 13:48:53 -07:00
|
|
|
return [this._peek, this._index, this._column, this._line, this.tokens.length];
|
2015-12-06 13:11:00 -08:00
|
|
|
}
|
2015-11-10 15:56:25 -08:00
|
|
|
|
2016-04-12 11:46:28 -07:00
|
|
|
private _readUntil(char: number): string {
|
2016-06-09 13:48:53 -07:00
|
|
|
let start = this._index;
|
2016-04-12 11:46:28 -07:00
|
|
|
this._attemptUntilChar(char);
|
2016-06-09 13:48:53 -07:00
|
|
|
return this._input.substring(start, this._index);
|
2016-04-12 11:46:28 -07:00
|
|
|
}
|
|
|
|
|
2015-11-10 15:56:25 -08:00
|
|
|
private _restorePosition(position: number[]): void {
|
2016-06-09 13:48:53 -07:00
|
|
|
this._peek = position[0];
|
|
|
|
this._index = position[1];
|
|
|
|
this._column = position[2];
|
|
|
|
this._line = position[3];
|
2015-12-06 13:11:00 -08:00
|
|
|
let nbTokens = position[4];
|
|
|
|
if (nbTokens < this.tokens.length) {
|
|
|
|
// remove any extra tokens
|
|
|
|
this.tokens = ListWrapper.slice(this.tokens, 0, nbTokens);
|
|
|
|
}
|
2015-11-10 15:56:25 -08:00
|
|
|
}
|
2016-04-13 16:01:25 -07:00
|
|
|
|
2016-06-09 13:48:53 -07:00
|
|
|
private _isInExpansionCase(): boolean {
|
|
|
|
return this._expansionCaseStack.length > 0 &&
|
2016-06-09 14:53:03 -07:00
|
|
|
this._expansionCaseStack[this._expansionCaseStack.length - 1] ===
|
|
|
|
HtmlTokenType.EXPANSION_CASE_EXP_START;
|
2016-04-13 16:01:25 -07:00
|
|
|
}
|
|
|
|
|
2016-06-09 13:48:53 -07:00
|
|
|
private _isInExpansionForm(): boolean {
|
|
|
|
return this._expansionCaseStack.length > 0 &&
|
2016-06-09 14:53:03 -07:00
|
|
|
this._expansionCaseStack[this._expansionCaseStack.length - 1] ===
|
|
|
|
HtmlTokenType.EXPANSION_FORM_START;
|
2016-04-13 16:01:25 -07:00
|
|
|
}
|
2015-10-07 09:34:21 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
function isNotWhitespace(code: number): boolean {
|
|
|
|
return !isWhitespace(code) || code === $EOF;
|
|
|
|
}
|
|
|
|
|
|
|
|
function isWhitespace(code: number): boolean {
|
|
|
|
return (code >= $TAB && code <= $SPACE) || (code === $NBSP);
|
|
|
|
}
|
|
|
|
|
|
|
|
function isNameEnd(code: number): boolean {
|
|
|
|
return isWhitespace(code) || code === $GT || code === $SLASH || code === $SQ || code === $DQ ||
|
2016-06-08 16:38:52 -07:00
|
|
|
code === $EQ;
|
2015-10-07 09:34:21 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
function isPrefixEnd(code: number): boolean {
|
|
|
|
return (code < $a || $z < code) && (code < $A || $Z < code) && (code < $0 || code > $9);
|
|
|
|
}
|
|
|
|
|
2015-11-10 15:56:25 -08:00
|
|
|
function isDigitEntityEnd(code: number): boolean {
|
|
|
|
return code == $SEMICOLON || code == $EOF || !isAsciiHexDigit(code);
|
|
|
|
}
|
|
|
|
|
|
|
|
function isNamedEntityEnd(code: number): boolean {
|
|
|
|
return code == $SEMICOLON || code == $EOF || !isAsciiLetter(code);
|
|
|
|
}
|
|
|
|
|
2016-06-09 13:48:53 -07:00
|
|
|
function isExpansionFormStart(peek: number, nextPeek: number): boolean {
|
2016-04-12 11:46:28 -07:00
|
|
|
return peek === $LBRACE && nextPeek != $LBRACE;
|
2015-10-07 09:34:21 -07:00
|
|
|
}
|
|
|
|
|
2016-06-09 13:48:53 -07:00
|
|
|
function isExpansionCaseStart(peek: number): boolean {
|
|
|
|
return peek === $EQ || isAsciiLetter(peek);
|
|
|
|
}
|
|
|
|
|
2015-11-10 15:56:25 -08:00
|
|
|
function isAsciiLetter(code: number): boolean {
|
2015-12-21 11:32:58 -08:00
|
|
|
return code >= $a && code <= $z || code >= $A && code <= $Z;
|
2015-11-10 15:56:25 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
function isAsciiHexDigit(code: number): boolean {
|
2015-12-21 11:32:58 -08:00
|
|
|
return code >= $a && code <= $f || code >= $A && code <= $F || code >= $0 && code <= $9;
|
|
|
|
}
|
|
|
|
|
|
|
|
function compareCharCodeCaseInsensitive(code1: number, code2: number): boolean {
|
|
|
|
return toUpperCaseCharCode(code1) == toUpperCaseCharCode(code2);
|
|
|
|
}
|
|
|
|
|
|
|
|
function toUpperCaseCharCode(code: number): number {
|
|
|
|
return code >= $a && code <= $z ? code - $a + $A : code;
|
2015-10-07 09:34:21 -07:00
|
|
|
}
|
2015-12-06 13:11:00 -08:00
|
|
|
|
|
|
|
function mergeTextTokens(srcTokens: HtmlToken[]): HtmlToken[] {
|
2016-06-11 21:23:37 -07:00
|
|
|
let dstTokens: HtmlToken[] = [];
|
2015-12-06 13:11:00 -08:00
|
|
|
let lastDstToken: HtmlToken;
|
|
|
|
for (let i = 0; i < srcTokens.length; i++) {
|
|
|
|
let token = srcTokens[i];
|
|
|
|
if (isPresent(lastDstToken) && lastDstToken.type == HtmlTokenType.TEXT &&
|
|
|
|
token.type == HtmlTokenType.TEXT) {
|
|
|
|
lastDstToken.parts[0] += token.parts[0];
|
|
|
|
lastDstToken.sourceSpan.end = token.sourceSpan.end;
|
|
|
|
} else {
|
|
|
|
lastDstToken = token;
|
|
|
|
dstTokens.push(lastDstToken);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return dstTokens;
|
|
|
|
}
|