test(scanner): port the rest of the lexer tests from AngularDart
Closes #64
This commit is contained in:
Chirayu Krishnappa
parent
5162b3c0ca
commit
e4ce69dcc7
|
|
@ -163,6 +163,15 @@ const $TILDE = 126;
|
|||
const $NBSP = 160;
|
||||
|
||||
|
||||
export class ScannerError extends Error {
|
||||
constructor(message) {
|
||||
this.message = message;
|
||||
}
|
||||
|
||||
toString() {
|
||||
return this.message;
|
||||
}
|
||||
}
|
||||
|
||||
export class Scanner {
|
||||
@FIELD('final input:String')
|
||||
|
|
@ -214,8 +223,8 @@ export class Scanner {
|
|||
switch (peek) {
|
||||
case $PERIOD:
|
||||
this.advance();
|
||||
return isDigit(peek) ? scanNumber(start) :
|
||||
newCharacterToken(start, $PERIOD);
|
||||
return isDigit(this.peek) ? this.scanNumber(start) :
|
||||
newCharacterToken(start, $PERIOD);
|
||||
case $LPAREN: case $RPAREN:
|
||||
case $LBRACE: case $RBRACE:
|
||||
case $LBRACKET: case $RBRACKET:
|
||||
|
|
@ -250,7 +259,7 @@ export class Scanner {
|
|||
return this.scanToken();
|
||||
}
|
||||
|
||||
this.error(`Unexpected character [${StringWrapper.fromCharCode(peek)}]`);
|
||||
this.error(`Unexpected character [${StringWrapper.fromCharCode(peek)}]`, 0);
|
||||
return null;
|
||||
}
|
||||
|
||||
|
|
@ -305,7 +314,7 @@ export class Scanner {
|
|||
} else if (isExponentStart(this.peek)) {
|
||||
this.advance();
|
||||
if (isExponentSign(this.peek)) this.advance();
|
||||
if (!isDigit(this.peek)) this.error('Invalid exponent');
|
||||
if (!isDigit(this.peek)) this.error('Invalid exponent', -1);
|
||||
simple = false;
|
||||
} else {
|
||||
break;
|
||||
|
|
@ -324,7 +333,7 @@ export class Scanner {
|
|||
var quote:int = this.peek;
|
||||
this.advance(); // Skip initial quote.
|
||||
|
||||
var buffer:StringJoiner; //ckck
|
||||
var buffer:StringJoiner;
|
||||
var marker:int = this.index;
|
||||
var input:string = this.input;
|
||||
|
||||
|
|
@ -337,7 +346,11 @@ export class Scanner {
|
|||
if (this.peek == $u) {
|
||||
// 4 character hex code for unicode character.
|
||||
var hex:string = input.substring(this.index + 1, this.index + 5);
|
||||
unescapedCode = NumberWrapper.parseInt(hex, 16);
|
||||
try {
|
||||
unescapedCode = NumberWrapper.parseInt(hex, 16);
|
||||
} catch (e) {
|
||||
this.error(`Invalid unicode escape [\\u${hex}]`, 0);
|
||||
}
|
||||
for (var i:int = 0; i < 5; i++) {
|
||||
this.advance();
|
||||
}
|
||||
|
|
@ -348,7 +361,7 @@ export class Scanner {
|
|||
buffer.add(StringWrapper.fromCharCode(unescapedCode));
|
||||
marker = this.index;
|
||||
} else if (this.peek == $EOF) {
|
||||
this.error('Unterminated quote');
|
||||
this.error('Unterminated quote', 0);
|
||||
} else {
|
||||
this.advance();
|
||||
}
|
||||
|
|
@ -367,9 +380,9 @@ export class Scanner {
|
|||
return newStringToken(start, unescaped);
|
||||
}
|
||||
|
||||
error(message:string) {
|
||||
var position:int = this.index;
|
||||
throw `Lexer Error: ${message} at column ${position} in expression [${input}]`;
|
||||
error(message:string, offset:int) {
|
||||
var position:int = this.index + offset;
|
||||
throw new ScannerError(`Lexer Error: ${message} at column ${position} in expression [${this.input}]`);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,108 +0,0 @@
|
|||
import {describe, it, expect} from 'test_lib/test_lib';
|
||||
import {Scanner, Token} from 'change_detection/parser/scanner';
|
||||
import {List, ListWrapper} from "facade/collection";
|
||||
import {StringWrapper} from "facade/lang";
|
||||
|
||||
function lex(text:string):List {
|
||||
var scanner:Scanner = new Scanner(text);
|
||||
var tokens:List<Token> = [];
|
||||
var token:Token = scanner.scanToken();
|
||||
while (token != null) {
|
||||
ListWrapper.push(tokens, token);
|
||||
token = scanner.scanToken();
|
||||
}
|
||||
return tokens;
|
||||
}
|
||||
|
||||
function expectToken(token, index) {
|
||||
expect(token instanceof Token).toBe(true);
|
||||
expect(token.index).toEqual(index);
|
||||
}
|
||||
|
||||
function expectCharacterToken(token, index, character) {
|
||||
expect(character.length).toBe(1);
|
||||
expectToken(token, index);
|
||||
expect(token.isCharacter(StringWrapper.charCodeAt(character, 0))).toBe(true);
|
||||
}
|
||||
|
||||
function expectOperatorToken(token, index, operator) {
|
||||
expectToken(token, index);
|
||||
expect(token.isOperator(operator)).toBe(true);
|
||||
}
|
||||
|
||||
function expectNumberToken(token, index, n) {
|
||||
expectToken(token, index);
|
||||
expect(token.isNumber()).toBe(true);
|
||||
expect(token.toNumber()).toEqual(n);
|
||||
}
|
||||
|
||||
function expectStringToken(token, index, str) {
|
||||
expectToken(token, index);
|
||||
expect(token.isString()).toBe(true);
|
||||
expect(token.toString()).toEqual(str);
|
||||
}
|
||||
|
||||
function expectIdentifierToken(token, index, identifier) {
|
||||
expectToken(token, index);
|
||||
expect(token.isIdentifier()).toBe(true);
|
||||
expect(token.toString()).toEqual(identifier);
|
||||
}
|
||||
|
||||
function expectKeywordToken(token, index, keyword) {
|
||||
expectToken(token, index);
|
||||
expect(token.isKeyword()).toBe(true);
|
||||
expect(token.toString()).toEqual(keyword);
|
||||
}
|
||||
|
||||
|
||||
export function main() {
|
||||
describe('scanner', function() {
|
||||
describe('token', function() {
|
||||
it('should tokenize a simple identifier', function() {
|
||||
var tokens:List<int> = lex("j");
|
||||
expect(tokens.length).toEqual(1);
|
||||
expectIdentifierToken(tokens[0], 0, 'j');
|
||||
});
|
||||
|
||||
it('should tokenize a dotted identifier', function() {
|
||||
var tokens:List<int> = lex("j.k");
|
||||
expect(tokens.length).toEqual(3);
|
||||
expectIdentifierToken(tokens[0], 0, 'j');
|
||||
expectCharacterToken (tokens[1], 1, '.');
|
||||
expectIdentifierToken(tokens[2], 2, 'k');
|
||||
});
|
||||
|
||||
it('should tokenize an operator', function() {
|
||||
var tokens:List<int> = lex("j-k");
|
||||
expect(tokens.length).toEqual(3);
|
||||
expectOperatorToken(tokens[1], 1, '-');
|
||||
});
|
||||
|
||||
it('should tokenize an indexed operator', function() {
|
||||
var tokens:List<int> = lex("j[k]");
|
||||
expect(tokens.length).toEqual(4);
|
||||
expectCharacterToken(tokens[1], 1, "[");
|
||||
expectCharacterToken(tokens[3], 3, "]");
|
||||
});
|
||||
|
||||
it('should tokenize numbers', function() {
|
||||
var tokens:List<int> = lex("88");
|
||||
expect(tokens.length).toEqual(1);
|
||||
expectNumberToken(tokens[0], 0, 88);
|
||||
});
|
||||
|
||||
it('should tokenize numbers within index ops', function() {
|
||||
expectNumberToken(lex("a[22]")[2], 2, 22);
|
||||
});
|
||||
|
||||
it('should tokenize simple quoted strings', function() {
|
||||
expectStringToken(lex('"a"')[0], 0, "a");
|
||||
});
|
||||
|
||||
it('should tokenize quoted strings with escaped quotes', function() {
|
||||
expectStringToken(lex('"a\\""')[0], 0, 'a"');
|
||||
});
|
||||
|
||||
});
|
||||
});
|
||||
}
|
||||
|
|
@ -0,0 +1,249 @@
|
|||
import {describe, it, expect} from 'test_lib/test_lib';
|
||||
import {Scanner, Token} from 'change_detection/parser/scanner';
|
||||
import {List, ListWrapper} from "facade/collection";
|
||||
import {StringWrapper} from "facade/lang";
|
||||
|
||||
function lex(text:string):List {
|
||||
var scanner:Scanner = new Scanner(text);
|
||||
var tokens:List<Token> = [];
|
||||
var token:Token = scanner.scanToken();
|
||||
while (token != null) {
|
||||
ListWrapper.push(tokens, token);
|
||||
token = scanner.scanToken();
|
||||
}
|
||||
return tokens;
|
||||
}
|
||||
|
||||
function expectToken(token, index) {
|
||||
expect(token instanceof Token).toBe(true);
|
||||
expect(token.index).toEqual(index);
|
||||
}
|
||||
|
||||
function expectCharacterToken(token, index, character) {
|
||||
expect(character.length).toBe(1);
|
||||
expectToken(token, index);
|
||||
expect(token.isCharacter(StringWrapper.charCodeAt(character, 0))).toBe(true);
|
||||
}
|
||||
|
||||
function expectOperatorToken(token, index, operator) {
|
||||
expectToken(token, index);
|
||||
expect(token.isOperator(operator)).toBe(true);
|
||||
}
|
||||
|
||||
function expectNumberToken(token, index, n) {
|
||||
expectToken(token, index);
|
||||
expect(token.isNumber()).toBe(true);
|
||||
expect(token.toNumber()).toEqual(n);
|
||||
}
|
||||
|
||||
function expectStringToken(token, index, str) {
|
||||
expectToken(token, index);
|
||||
expect(token.isString()).toBe(true);
|
||||
expect(token.toString()).toEqual(str);
|
||||
}
|
||||
|
||||
function expectIdentifierToken(token, index, identifier) {
|
||||
expectToken(token, index);
|
||||
expect(token.isIdentifier()).toBe(true);
|
||||
expect(token.toString()).toEqual(identifier);
|
||||
}
|
||||
|
||||
function expectKeywordToken(token, index, keyword) {
|
||||
expectToken(token, index);
|
||||
expect(token.isKeyword()).toBe(true);
|
||||
expect(token.toString()).toEqual(keyword);
|
||||
}
|
||||
|
||||
|
||||
export function main() {
|
||||
describe('scanner', function() {
|
||||
describe('token', function() {
|
||||
it('should tokenize a simple identifier', function() {
|
||||
var tokens:List<int> = lex("j");
|
||||
expect(tokens.length).toEqual(1);
|
||||
expectIdentifierToken(tokens[0], 0, 'j');
|
||||
});
|
||||
|
||||
it('should tokenize a dotted identifier', function() {
|
||||
var tokens:List<int> = lex("j.k");
|
||||
expect(tokens.length).toEqual(3);
|
||||
expectIdentifierToken(tokens[0], 0, 'j');
|
||||
expectCharacterToken (tokens[1], 1, '.');
|
||||
expectIdentifierToken(tokens[2], 2, 'k');
|
||||
});
|
||||
|
||||
it('should tokenize an operator', function() {
|
||||
var tokens:List<int> = lex("j-k");
|
||||
expect(tokens.length).toEqual(3);
|
||||
expectOperatorToken(tokens[1], 1, '-');
|
||||
});
|
||||
|
||||
it('should tokenize an indexed operator', function() {
|
||||
var tokens:List<int> = lex("j[k]");
|
||||
expect(tokens.length).toEqual(4);
|
||||
expectCharacterToken(tokens[1], 1, "[");
|
||||
expectCharacterToken(tokens[3], 3, "]");
|
||||
});
|
||||
|
||||
it('should tokenize numbers', function() {
|
||||
var tokens:List<int> = lex("88");
|
||||
expect(tokens.length).toEqual(1);
|
||||
expectNumberToken(tokens[0], 0, 88);
|
||||
});
|
||||
|
||||
it('should tokenize numbers within index ops', function() {
|
||||
expectNumberToken(lex("a[22]")[2], 2, 22);
|
||||
});
|
||||
|
||||
it('should tokenize simple quoted strings', function() {
|
||||
expectStringToken(lex('"a"')[0], 0, "a");
|
||||
});
|
||||
|
||||
it('should tokenize quoted strings with escaped quotes', function() {
|
||||
expectStringToken(lex('"a\\""')[0], 0, 'a"');
|
||||
});
|
||||
|
||||
it('should tokenize a string', function() {
|
||||
var tokens:List<Token> = lex("j-a.bc[22]+1.3|f:'a\\\'c':\"d\\\"e\"");
|
||||
expectIdentifierToken(tokens[0], 0, 'j');
|
||||
expectOperatorToken(tokens[1], 1, '-');
|
||||
expectIdentifierToken(tokens[2], 2, 'a');
|
||||
expectCharacterToken(tokens[3], 3, '.');
|
||||
expectIdentifierToken(tokens[4], 4, 'bc');
|
||||
expectCharacterToken(tokens[5], 6, '[');
|
||||
expectNumberToken(tokens[6], 7, 22);
|
||||
expectCharacterToken(tokens[7], 9, ']');
|
||||
expectOperatorToken(tokens[8], 10, '+');
|
||||
expectNumberToken(tokens[9], 11, 1.3);
|
||||
expectOperatorToken(tokens[10], 14, '|');
|
||||
expectIdentifierToken(tokens[11], 15, 'f');
|
||||
expectCharacterToken(tokens[12], 16, ':');
|
||||
expectStringToken(tokens[13], 17, "a'c");
|
||||
expectCharacterToken(tokens[14], 23, ':');
|
||||
expectStringToken(tokens[15], 24, 'd"e');
|
||||
});
|
||||
|
||||
it('should tokenize undefined', function() {
|
||||
var tokens:List<Token> = lex("undefined");
|
||||
expectKeywordToken(tokens[0], 0, "undefined");
|
||||
expect(tokens[0].isKeywordUndefined()).toBe(true);
|
||||
});
|
||||
|
||||
it('should ignore whitespace', function() {
|
||||
var tokens:List<Token> = lex("a \t \n \r b");
|
||||
expectIdentifierToken(tokens[0], 0, 'a');
|
||||
expectIdentifierToken(tokens[1], 8, 'b');
|
||||
});
|
||||
|
||||
it('should tokenize quoted string', function() {
|
||||
var str = "['\\'', \"\\\"\"]";
|
||||
var tokens:List<Token> = lex(str);
|
||||
expectStringToken(tokens[1], 1, "'");
|
||||
expectStringToken(tokens[3], 7, '"');
|
||||
});
|
||||
|
||||
it('should tokenize escaped quoted string', function() {
|
||||
var str = '"\\"\\n\\f\\r\\t\\v\\u00A0"';
|
||||
var tokens:List<Token> = lex(str);
|
||||
expect(tokens.length).toEqual(1);
|
||||
expect(tokens[0].toString()).toEqual('"\n\f\r\t\v\u00A0');
|
||||
});
|
||||
|
||||
it('should tokenize unicode', function() {
|
||||
var tokens:List<Token> = lex('"\\u00A0"');
|
||||
expect(tokens.length).toEqual(1);
|
||||
expect(tokens[0].toString()).toEqual('\u00a0');
|
||||
});
|
||||
|
||||
it('should tokenize relation', function() {
|
||||
var tokens:List<Token> = lex("! == != < > <= >=");
|
||||
expectOperatorToken(tokens[0], 0, '!');
|
||||
expectOperatorToken(tokens[1], 2, '==');
|
||||
expectOperatorToken(tokens[2], 5, '!=');
|
||||
expectOperatorToken(tokens[3], 8, '<');
|
||||
expectOperatorToken(tokens[4], 10, '>');
|
||||
expectOperatorToken(tokens[5], 12, '<=');
|
||||
expectOperatorToken(tokens[6], 15, '>=');
|
||||
});
|
||||
|
||||
it('should tokenize statements', function() {
|
||||
var tokens:List<Token> = lex("a;b;");
|
||||
expectIdentifierToken(tokens[0], 0, 'a');
|
||||
expectCharacterToken(tokens[1], 1, ';');
|
||||
expectIdentifierToken(tokens[2], 2, 'b');
|
||||
expectCharacterToken(tokens[3], 3, ';');
|
||||
});
|
||||
|
||||
it('should tokenize function invocation', function() {
|
||||
var tokens:List<Token> = lex("a()");
|
||||
expectIdentifierToken(tokens[0], 0, 'a');
|
||||
expectCharacterToken(tokens[1], 1, '(');
|
||||
expectCharacterToken(tokens[2], 2, ')');
|
||||
});
|
||||
|
||||
it('should tokenize simple method invocations', function() {
|
||||
var tokens:List<Token> = lex("a.method()");
|
||||
expectIdentifierToken(tokens[2], 2, 'method');
|
||||
});
|
||||
|
||||
it('should tokenize method invocation', function() {
|
||||
var tokens:List<Token> = lex("a.b.c (d) - e.f()");
|
||||
expectIdentifierToken(tokens[0], 0, 'a');
|
||||
expectCharacterToken(tokens[1], 1, '.');
|
||||
expectIdentifierToken(tokens[2], 2, 'b');
|
||||
expectCharacterToken(tokens[3], 3, '.');
|
||||
expectIdentifierToken(tokens[4], 4, 'c');
|
||||
expectCharacterToken(tokens[5], 6, '(');
|
||||
expectIdentifierToken(tokens[6], 7, 'd');
|
||||
expectCharacterToken(tokens[7], 8, ')');
|
||||
expectOperatorToken(tokens[8], 10, '-');
|
||||
expectIdentifierToken(tokens[9], 12, 'e');
|
||||
expectCharacterToken(tokens[10], 13, '.');
|
||||
expectIdentifierToken(tokens[11], 14, 'f');
|
||||
expectCharacterToken(tokens[12], 15, '(');
|
||||
expectCharacterToken(tokens[13], 16, ')');
|
||||
});
|
||||
|
||||
it('should tokenize number', function() {
|
||||
var tokens:List<Token> = lex("0.5");
|
||||
expectNumberToken(tokens[0], 0, 0.5);
|
||||
});
|
||||
|
||||
// NOTE(deboer): NOT A LEXER TEST
|
||||
// it('should tokenize negative number', function() {
|
||||
// var tokens:List<Token> = lex("-0.5");
|
||||
// expectNumberToken(tokens[0], 0, -0.5);
|
||||
// });
|
||||
|
||||
it('should tokenize number with exponent', function() {
|
||||
var tokens:List<Token> = lex("0.5E-10");
|
||||
expect(tokens.length).toEqual(1);
|
||||
expectNumberToken(tokens[0], 0, 0.5E-10);
|
||||
tokens = lex("0.5E+10");
|
||||
expectNumberToken(tokens[0], 0, 0.5E+10);
|
||||
});
|
||||
|
||||
it('should throws exception for invalid exponent', function() {
|
||||
expect(function() {
|
||||
lex("0.5E-");
|
||||
}).toThrowError('Lexer Error: Invalid exponent at column 4 in expression [0.5E-]');
|
||||
|
||||
expect(function() {
|
||||
lex("0.5E-A");
|
||||
}).toThrowError('Lexer Error: Invalid exponent at column 4 in expression [0.5E-A]');
|
||||
});
|
||||
|
||||
it('should tokenize number starting with a dot', function() {
|
||||
var tokens:List<Token> = lex(".5");
|
||||
expectNumberToken(tokens[0], 0, 0.5);
|
||||
});
|
||||
|
||||
it('should throw error on invalid unicode', function() {
|
||||
expect(function() {
|
||||
lex("'\\u1''bla'");
|
||||
}).toThrowError("Lexer Error: Invalid unicode escape [\\u1''b] at column 2 in expression ['\\u1''bla']");
|
||||
});
|
||||
|
||||
});
|
||||
});
|
||||
}
|
||||
|
|
@ -59,21 +59,42 @@ export class StringJoiner {
|
|||
}
|
||||
}
|
||||
|
||||
export class NumerParseError extends Error {
|
||||
constructor(message) {
|
||||
this.message = message;
|
||||
}
|
||||
|
||||
toString() {
|
||||
return this.message;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
export class NumberWrapper {
|
||||
static parseIntAutoRadix(text:string):int {
|
||||
var result:int = parseInt(text);
|
||||
if (isNaN(result)) {
|
||||
throw new Error("Invalid integer literal when parsing " + text);
|
||||
throw new NumerParseError("Invalid integer literal when parsing " + text);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
static parseInt(text:string, radix:int):int {
|
||||
var result:int = parseInt(text, radix);
|
||||
if (isNaN(result)) {
|
||||
throw new Error("Invalid integer literal when parsing " + text + " in base " + radix);
|
||||
if (radix == 10) {
|
||||
if (/^(\-|\+)?[0-9]+$/.test(text)) {
|
||||
return parseInt(text, radix);
|
||||
}
|
||||
} else if (radix == 16) {
|
||||
if (/^(\-|\+)?[0-9ABCDEFabcdef]+$/.test(text)) {
|
||||
return parseInt(text, radix);
|
||||
}
|
||||
} else {
|
||||
var result:int = parseInt(text, radix);
|
||||
if (!isNaN(result)) {
|
||||
return result;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
throw new NumerParseError("Invalid integer literal when parsing " + text + " in base " + radix);
|
||||
}
|
||||
|
||||
// TODO: NaN is a valid literal but is returned by parseFloat to indicate an error.
|
||||
|
|
|
|||
Loading…
Reference in New Issue