iSharkFly-Docs
/
angular-docs-cn
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor(compiler): capture `fullStart` locations when tokenizing (#39486) 

This commit ensures that when leading whitespace is skipped by
the tokenizer, the original start location (before skipping) is captured
in the `fullStart` property of the token's source-span.

PR Close #39486
This commit is contained in:
Pete Bacon Darwin 2020-10-28 21:37:24 +00:00 committed by Misko Hevery
parent 8d90c1ad97
commit 43d8e9aad2
2 changed files with 27 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
packages/compiler/src/ml_parser/lexer.ts
View File

@ -917,19 +917,20 @@ class PlainCharacterCursor implements CharacterCursor {
getSpan(start?: this, leadingTriviaCodePoints?: number[]): ParseSourceSpan { getSpan(start?: this, leadingTriviaCodePoints?: number[]): ParseSourceSpan {
start = start || this; start = start || this;
let cloned = false; let fullStart = start;
if (leadingTriviaCodePoints) { if (leadingTriviaCodePoints) {
while (this.diff(start) > 0 && leadingTriviaCodePoints.indexOf(start.peek()) !== -1) { while (this.diff(start) > 0 && leadingTriviaCodePoints.indexOf(start.peek()) !== -1) {
if (!cloned) { if (fullStart === start) {
start = start.clone() as this; start = start.clone() as this;
cloned = true;
} }
start.advance(); start.advance();
} }
} }
return new ParseSourceSpan( const startLocation = this.locationFromCursor(start);
new ParseLocation(start.file, start.state.offset, start.state.line, start.state.column), const endLocation = this.locationFromCursor(this);
new ParseLocation(this.file, this.state.offset, this.state.line, this.state.column)); const fullStartLocation =
fullStart !== start ? this.locationFromCursor(fullStart) : startLocation;
return new ParseSourceSpan(startLocation, endLocation, fullStartLocation);
} }
getChars(start: this): string { getChars(start: this): string {
@ -959,6 +960,11 @@ class PlainCharacterCursor implements CharacterCursor {
protected updatePeek(state: CursorState): void { protected updatePeek(state: CursorState): void {
state.peek = state.offset >= this.end ? chars.$EOF : this.charAt(state.offset); state.peek = state.offset >= this.end ? chars.$EOF : this.charAt(state.offset);
} }
private locationFromCursor(cursor: this): ParseLocation {
return new ParseLocation(
cursor.file, cursor.state.offset, cursor.state.line, cursor.state.column);
}
} }
class EscapedCharacterCursor extends PlainCharacterCursor { class EscapedCharacterCursor extends PlainCharacterCursor {

22
packages/compiler/test/ml_parser/lexer_spec.ts
View File

@ -54,14 +54,14 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u
}); });
it('should skip over leading trivia for source-span start', () => { it('should skip over leading trivia for source-span start', () => {
expect(tokenizeAndHumanizeLineColumn( expect(
'<t>\n \t a</t>', {leadingTriviaChars: ['\n', ' ', '\t']})) tokenizeAndHumanizeFullStart('<t>\n \t a</t>', {leadingTriviaChars: ['\n', ' ', '\t']}))
.toEqual([ .toEqual([
[lex.TokenType.TAG_OPEN_START, '0:0'], [lex.TokenType.TAG_OPEN_START, '0:0', '0:0'],
[lex.TokenType.TAG_OPEN_END, '0:2'], [lex.TokenType.TAG_OPEN_END, '0:2', '0:2'],
[lex.TokenType.TEXT, '1:3'], [lex.TokenType.TEXT, '1:3', '0:3'],
[lex.TokenType.TAG_CLOSE, '1:4'], [lex.TokenType.TAG_CLOSE, '1:4', '1:4'],
[lex.TokenType.EOF, '1:8'], [lex.TokenType.EOF, '1:8', '1:8'],
]); ]);
}); });
}); });
@ -1560,6 +1560,14 @@ function tokenizeAndHumanizeLineColumn(input: string, options?: lex.TokenizeOpti
.tokens.map(token => [<any>token.type, humanizeLineColumn(token.sourceSpan.start)]); .tokens.map(token => [<any>token.type, humanizeLineColumn(token.sourceSpan.start)]);
} }
function tokenizeAndHumanizeFullStart(input: string, options?: lex.TokenizeOptions): any[] {
return tokenizeWithoutErrors(input, options)
.tokens.map(
token =>
[<any>token.type, humanizeLineColumn(token.sourceSpan.start),
humanizeLineColumn(token.sourceSpan.fullStart)]);
}
function tokenizeAndHumanizeErrors(input: string, options?: lex.TokenizeOptions): any[] { function tokenizeAndHumanizeErrors(input: string, options?: lex.TokenizeOptions): any[] {
return lex.tokenize(input, 'someUrl', getHtmlTagDefinition, options) return lex.tokenize(input, 'someUrl', getHtmlTagDefinition, options)
.errors.map(e => [<any>e.tokenType, e.msg, humanizeLineColumn(e.span.start)]); .errors.map(e => [<any>e.tokenType, e.msg, humanizeLineColumn(e.span.start)]);