diff --git a/modules/change_detection/src/parser/scanner.js b/modules/change_detection/src/parser/scanner.js index 3ca4f60b96..bf7e8ef0ba 100644 --- a/modules/change_detection/src/parser/scanner.js +++ b/modules/change_detection/src/parser/scanner.js @@ -163,6 +163,15 @@ const $TILDE = 126; const $NBSP = 160; +export class ScannerError extends Error { + constructor(message) { + this.message = message; + } + + toString() { + return this.message; + } +} export class Scanner { @FIELD('final input:String') @@ -214,8 +223,8 @@ export class Scanner { switch (peek) { case $PERIOD: this.advance(); - return isDigit(peek) ? scanNumber(start) : - newCharacterToken(start, $PERIOD); + return isDigit(this.peek) ? this.scanNumber(start) : + newCharacterToken(start, $PERIOD); case $LPAREN: case $RPAREN: case $LBRACE: case $RBRACE: case $LBRACKET: case $RBRACKET: @@ -250,7 +259,7 @@ export class Scanner { return this.scanToken(); } - this.error(`Unexpected character [${StringWrapper.fromCharCode(peek)}]`); + this.error(`Unexpected character [${StringWrapper.fromCharCode(peek)}]`, 0); return null; } @@ -305,7 +314,7 @@ export class Scanner { } else if (isExponentStart(this.peek)) { this.advance(); if (isExponentSign(this.peek)) this.advance(); - if (!isDigit(this.peek)) this.error('Invalid exponent'); + if (!isDigit(this.peek)) this.error('Invalid exponent', -1); simple = false; } else { break; @@ -324,7 +333,7 @@ export class Scanner { var quote:int = this.peek; this.advance(); // Skip initial quote. - var buffer:StringJoiner; //ckck + var buffer:StringJoiner; var marker:int = this.index; var input:string = this.input; @@ -337,7 +346,11 @@ export class Scanner { if (this.peek == $u) { // 4 character hex code for unicode character. var hex:string = input.substring(this.index + 1, this.index + 5); - unescapedCode = NumberWrapper.parseInt(hex, 16); + try { + unescapedCode = NumberWrapper.parseInt(hex, 16); + } catch (e) { + this.error(`Invalid unicode escape [\\u${hex}]`, 0); + } for (var i:int = 0; i < 5; i++) { this.advance(); } @@ -348,7 +361,7 @@ export class Scanner { buffer.add(StringWrapper.fromCharCode(unescapedCode)); marker = this.index; } else if (this.peek == $EOF) { - this.error('Unterminated quote'); + this.error('Unterminated quote', 0); } else { this.advance(); } @@ -367,9 +380,9 @@ export class Scanner { return newStringToken(start, unescaped); } - error(message:string) { - var position:int = this.index; - throw `Lexer Error: ${message} at column ${position} in expression [${input}]`; + error(message:string, offset:int) { + var position:int = this.index + offset; + throw new ScannerError(`Lexer Error: ${message} at column ${position} in expression [${this.input}]`); } } diff --git a/modules/change_detection/test/parser/lexer_spec.js b/modules/change_detection/test/parser/lexer_spec.js deleted file mode 100644 index 1375cc3055..0000000000 --- a/modules/change_detection/test/parser/lexer_spec.js +++ /dev/null @@ -1,108 +0,0 @@ -import {describe, it, expect} from 'test_lib/test_lib'; -import {Scanner, Token} from 'change_detection/parser/scanner'; -import {List, ListWrapper} from "facade/collection"; -import {StringWrapper} from "facade/lang"; - -function lex(text:string):List { - var scanner:Scanner = new Scanner(text); - var tokens:List = []; - var token:Token = scanner.scanToken(); - while (token != null) { - ListWrapper.push(tokens, token); - token = scanner.scanToken(); - } - return tokens; -} - -function expectToken(token, index) { - expect(token instanceof Token).toBe(true); - expect(token.index).toEqual(index); -} - -function expectCharacterToken(token, index, character) { - expect(character.length).toBe(1); - expectToken(token, index); - expect(token.isCharacter(StringWrapper.charCodeAt(character, 0))).toBe(true); -} - -function expectOperatorToken(token, index, operator) { - expectToken(token, index); - expect(token.isOperator(operator)).toBe(true); -} - -function expectNumberToken(token, index, n) { - expectToken(token, index); - expect(token.isNumber()).toBe(true); - expect(token.toNumber()).toEqual(n); -} - -function expectStringToken(token, index, str) { - expectToken(token, index); - expect(token.isString()).toBe(true); - expect(token.toString()).toEqual(str); -} - -function expectIdentifierToken(token, index, identifier) { - expectToken(token, index); - expect(token.isIdentifier()).toBe(true); - expect(token.toString()).toEqual(identifier); -} - -function expectKeywordToken(token, index, keyword) { - expectToken(token, index); - expect(token.isKeyword()).toBe(true); - expect(token.toString()).toEqual(keyword); -} - - -export function main() { - describe('scanner', function() { - describe('token', function() { - it('should tokenize a simple identifier', function() { - var tokens:List = lex("j"); - expect(tokens.length).toEqual(1); - expectIdentifierToken(tokens[0], 0, 'j'); - }); - - it('should tokenize a dotted identifier', function() { - var tokens:List = lex("j.k"); - expect(tokens.length).toEqual(3); - expectIdentifierToken(tokens[0], 0, 'j'); - expectCharacterToken (tokens[1], 1, '.'); - expectIdentifierToken(tokens[2], 2, 'k'); - }); - - it('should tokenize an operator', function() { - var tokens:List = lex("j-k"); - expect(tokens.length).toEqual(3); - expectOperatorToken(tokens[1], 1, '-'); - }); - - it('should tokenize an indexed operator', function() { - var tokens:List = lex("j[k]"); - expect(tokens.length).toEqual(4); - expectCharacterToken(tokens[1], 1, "["); - expectCharacterToken(tokens[3], 3, "]"); - }); - - it('should tokenize numbers', function() { - var tokens:List = lex("88"); - expect(tokens.length).toEqual(1); - expectNumberToken(tokens[0], 0, 88); - }); - - it('should tokenize numbers within index ops', function() { - expectNumberToken(lex("a[22]")[2], 2, 22); - }); - - it('should tokenize simple quoted strings', function() { - expectStringToken(lex('"a"')[0], 0, "a"); - }); - - it('should tokenize quoted strings with escaped quotes', function() { - expectStringToken(lex('"a\\""')[0], 0, 'a"'); - }); - - }); - }); -} diff --git a/modules/change_detection/test/parser/scanner_spec.js b/modules/change_detection/test/parser/scanner_spec.js new file mode 100644 index 0000000000..e4a22b698c --- /dev/null +++ b/modules/change_detection/test/parser/scanner_spec.js @@ -0,0 +1,249 @@ +import {describe, it, expect} from 'test_lib/test_lib'; +import {Scanner, Token} from 'change_detection/parser/scanner'; +import {List, ListWrapper} from "facade/collection"; +import {StringWrapper} from "facade/lang"; + +function lex(text:string):List { + var scanner:Scanner = new Scanner(text); + var tokens:List = []; + var token:Token = scanner.scanToken(); + while (token != null) { + ListWrapper.push(tokens, token); + token = scanner.scanToken(); + } + return tokens; +} + +function expectToken(token, index) { + expect(token instanceof Token).toBe(true); + expect(token.index).toEqual(index); +} + +function expectCharacterToken(token, index, character) { + expect(character.length).toBe(1); + expectToken(token, index); + expect(token.isCharacter(StringWrapper.charCodeAt(character, 0))).toBe(true); +} + +function expectOperatorToken(token, index, operator) { + expectToken(token, index); + expect(token.isOperator(operator)).toBe(true); +} + +function expectNumberToken(token, index, n) { + expectToken(token, index); + expect(token.isNumber()).toBe(true); + expect(token.toNumber()).toEqual(n); +} + +function expectStringToken(token, index, str) { + expectToken(token, index); + expect(token.isString()).toBe(true); + expect(token.toString()).toEqual(str); +} + +function expectIdentifierToken(token, index, identifier) { + expectToken(token, index); + expect(token.isIdentifier()).toBe(true); + expect(token.toString()).toEqual(identifier); +} + +function expectKeywordToken(token, index, keyword) { + expectToken(token, index); + expect(token.isKeyword()).toBe(true); + expect(token.toString()).toEqual(keyword); +} + + +export function main() { + describe('scanner', function() { + describe('token', function() { + it('should tokenize a simple identifier', function() { + var tokens:List = lex("j"); + expect(tokens.length).toEqual(1); + expectIdentifierToken(tokens[0], 0, 'j'); + }); + + it('should tokenize a dotted identifier', function() { + var tokens:List = lex("j.k"); + expect(tokens.length).toEqual(3); + expectIdentifierToken(tokens[0], 0, 'j'); + expectCharacterToken (tokens[1], 1, '.'); + expectIdentifierToken(tokens[2], 2, 'k'); + }); + + it('should tokenize an operator', function() { + var tokens:List = lex("j-k"); + expect(tokens.length).toEqual(3); + expectOperatorToken(tokens[1], 1, '-'); + }); + + it('should tokenize an indexed operator', function() { + var tokens:List = lex("j[k]"); + expect(tokens.length).toEqual(4); + expectCharacterToken(tokens[1], 1, "["); + expectCharacterToken(tokens[3], 3, "]"); + }); + + it('should tokenize numbers', function() { + var tokens:List = lex("88"); + expect(tokens.length).toEqual(1); + expectNumberToken(tokens[0], 0, 88); + }); + + it('should tokenize numbers within index ops', function() { + expectNumberToken(lex("a[22]")[2], 2, 22); + }); + + it('should tokenize simple quoted strings', function() { + expectStringToken(lex('"a"')[0], 0, "a"); + }); + + it('should tokenize quoted strings with escaped quotes', function() { + expectStringToken(lex('"a\\""')[0], 0, 'a"'); + }); + + it('should tokenize a string', function() { + var tokens:List = lex("j-a.bc[22]+1.3|f:'a\\\'c':\"d\\\"e\""); + expectIdentifierToken(tokens[0], 0, 'j'); + expectOperatorToken(tokens[1], 1, '-'); + expectIdentifierToken(tokens[2], 2, 'a'); + expectCharacterToken(tokens[3], 3, '.'); + expectIdentifierToken(tokens[4], 4, 'bc'); + expectCharacterToken(tokens[5], 6, '['); + expectNumberToken(tokens[6], 7, 22); + expectCharacterToken(tokens[7], 9, ']'); + expectOperatorToken(tokens[8], 10, '+'); + expectNumberToken(tokens[9], 11, 1.3); + expectOperatorToken(tokens[10], 14, '|'); + expectIdentifierToken(tokens[11], 15, 'f'); + expectCharacterToken(tokens[12], 16, ':'); + expectStringToken(tokens[13], 17, "a'c"); + expectCharacterToken(tokens[14], 23, ':'); + expectStringToken(tokens[15], 24, 'd"e'); + }); + + it('should tokenize undefined', function() { + var tokens:List = lex("undefined"); + expectKeywordToken(tokens[0], 0, "undefined"); + expect(tokens[0].isKeywordUndefined()).toBe(true); + }); + + it('should ignore whitespace', function() { + var tokens:List = lex("a \t \n \r b"); + expectIdentifierToken(tokens[0], 0, 'a'); + expectIdentifierToken(tokens[1], 8, 'b'); + }); + + it('should tokenize quoted string', function() { + var str = "['\\'', \"\\\"\"]"; + var tokens:List = lex(str); + expectStringToken(tokens[1], 1, "'"); + expectStringToken(tokens[3], 7, '"'); + }); + + it('should tokenize escaped quoted string', function() { + var str = '"\\"\\n\\f\\r\\t\\v\\u00A0"'; + var tokens:List = lex(str); + expect(tokens.length).toEqual(1); + expect(tokens[0].toString()).toEqual('"\n\f\r\t\v\u00A0'); + }); + + it('should tokenize unicode', function() { + var tokens:List = lex('"\\u00A0"'); + expect(tokens.length).toEqual(1); + expect(tokens[0].toString()).toEqual('\u00a0'); + }); + + it('should tokenize relation', function() { + var tokens:List = lex("! == != < > <= >="); + expectOperatorToken(tokens[0], 0, '!'); + expectOperatorToken(tokens[1], 2, '=='); + expectOperatorToken(tokens[2], 5, '!='); + expectOperatorToken(tokens[3], 8, '<'); + expectOperatorToken(tokens[4], 10, '>'); + expectOperatorToken(tokens[5], 12, '<='); + expectOperatorToken(tokens[6], 15, '>='); + }); + + it('should tokenize statements', function() { + var tokens:List = lex("a;b;"); + expectIdentifierToken(tokens[0], 0, 'a'); + expectCharacterToken(tokens[1], 1, ';'); + expectIdentifierToken(tokens[2], 2, 'b'); + expectCharacterToken(tokens[3], 3, ';'); + }); + + it('should tokenize function invocation', function() { + var tokens:List = lex("a()"); + expectIdentifierToken(tokens[0], 0, 'a'); + expectCharacterToken(tokens[1], 1, '('); + expectCharacterToken(tokens[2], 2, ')'); + }); + + it('should tokenize simple method invocations', function() { + var tokens:List = lex("a.method()"); + expectIdentifierToken(tokens[2], 2, 'method'); + }); + + it('should tokenize method invocation', function() { + var tokens:List = lex("a.b.c (d) - e.f()"); + expectIdentifierToken(tokens[0], 0, 'a'); + expectCharacterToken(tokens[1], 1, '.'); + expectIdentifierToken(tokens[2], 2, 'b'); + expectCharacterToken(tokens[3], 3, '.'); + expectIdentifierToken(tokens[4], 4, 'c'); + expectCharacterToken(tokens[5], 6, '('); + expectIdentifierToken(tokens[6], 7, 'd'); + expectCharacterToken(tokens[7], 8, ')'); + expectOperatorToken(tokens[8], 10, '-'); + expectIdentifierToken(tokens[9], 12, 'e'); + expectCharacterToken(tokens[10], 13, '.'); + expectIdentifierToken(tokens[11], 14, 'f'); + expectCharacterToken(tokens[12], 15, '('); + expectCharacterToken(tokens[13], 16, ')'); + }); + + it('should tokenize number', function() { + var tokens:List = lex("0.5"); + expectNumberToken(tokens[0], 0, 0.5); + }); + + // NOTE(deboer): NOT A LEXER TEST + // it('should tokenize negative number', function() { + // var tokens:List = lex("-0.5"); + // expectNumberToken(tokens[0], 0, -0.5); + // }); + + it('should tokenize number with exponent', function() { + var tokens:List = lex("0.5E-10"); + expect(tokens.length).toEqual(1); + expectNumberToken(tokens[0], 0, 0.5E-10); + tokens = lex("0.5E+10"); + expectNumberToken(tokens[0], 0, 0.5E+10); + }); + + it('should throws exception for invalid exponent', function() { + expect(function() { + lex("0.5E-"); + }).toThrowError('Lexer Error: Invalid exponent at column 4 in expression [0.5E-]'); + + expect(function() { + lex("0.5E-A"); + }).toThrowError('Lexer Error: Invalid exponent at column 4 in expression [0.5E-A]'); + }); + + it('should tokenize number starting with a dot', function() { + var tokens:List = lex(".5"); + expectNumberToken(tokens[0], 0, 0.5); + }); + + it('should throw error on invalid unicode', function() { + expect(function() { + lex("'\\u1''bla'"); + }).toThrowError("Lexer Error: Invalid unicode escape [\\u1''b] at column 2 in expression ['\\u1''bla']"); + }); + + }); + }); +} diff --git a/modules/facade/src/lang.es6 b/modules/facade/src/lang.es6 index 89ac94fd21..d32b79f9f0 100644 --- a/modules/facade/src/lang.es6 +++ b/modules/facade/src/lang.es6 @@ -59,21 +59,42 @@ export class StringJoiner { } } +export class NumerParseError extends Error { + constructor(message) { + this.message = message; + } + + toString() { + return this.message; + } +} + + export class NumberWrapper { static parseIntAutoRadix(text:string):int { var result:int = parseInt(text); if (isNaN(result)) { - throw new Error("Invalid integer literal when parsing " + text); + throw new NumerParseError("Invalid integer literal when parsing " + text); } return result; } static parseInt(text:string, radix:int):int { - var result:int = parseInt(text, radix); - if (isNaN(result)) { - throw new Error("Invalid integer literal when parsing " + text + " in base " + radix); + if (radix == 10) { + if (/^(\-|\+)?[0-9]+$/.test(text)) { + return parseInt(text, radix); + } + } else if (radix == 16) { + if (/^(\-|\+)?[0-9ABCDEFabcdef]+$/.test(text)) { + return parseInt(text, radix); + } + } else { + var result:int = parseInt(text, radix); + if (!isNaN(result)) { + return result; + } } - return result; + throw new NumerParseError("Invalid integer literal when parsing " + text + " in base " + radix); } // TODO: NaN is a valid literal but is returned by parseFloat to indicate an error.