test(scanner): port the rest of the lexer tests from AngularDart

Closes #64

modules/change_detection/src/parser/scanner.js

@@ -163,6 +163,15 @@ const $TILDE  = 126;
 const $NBSP   = 160;
 
 
+export class ScannerError extends Error {
+  constructor(message) {
+    this.message = message;
+  }
+
+  toString() {
+    return this.message;
+  }
+}
 
 export class Scanner {
   @FIELD('final input:String')
@@ -214,7 +223,7 @@ export class Scanner {
     switch (peek) {
       case $PERIOD:
         this.advance();
-        return isDigit(peek) ? scanNumber(start) :
+        return isDigit(this.peek) ? this.scanNumber(start) :
                                     newCharacterToken(start, $PERIOD);
       case $LPAREN:   case $RPAREN:
       case $LBRACE:   case $RBRACE:
@@ -250,7 +259,7 @@ export class Scanner {
         return this.scanToken();
     }
 
-    this.error(`Unexpected character [${StringWrapper.fromCharCode(peek)}]`);
+    this.error(`Unexpected character [${StringWrapper.fromCharCode(peek)}]`, 0);
     return null;
   }
 
@@ -305,7 +314,7 @@ export class Scanner {
       } else if (isExponentStart(this.peek)) {
         this.advance();
         if (isExponentSign(this.peek)) this.advance();
-        if (!isDigit(this.peek)) this.error('Invalid exponent');
+        if (!isDigit(this.peek)) this.error('Invalid exponent', -1);
         simple = false;
       } else {
         break;
@@ -324,7 +333,7 @@ export class Scanner {
     var quote:int = this.peek;
     this.advance();  // Skip initial quote.
 
-    var buffer:StringJoiner; //ckck
+    var buffer:StringJoiner;
     var marker:int = this.index;
     var input:string = this.input;
 
@@ -337,7 +346,11 @@ export class Scanner {
         if (this.peek == $u) {
           // 4 character hex code for unicode character.
           var hex:string = input.substring(this.index + 1, this.index + 5);
+          try {
             unescapedCode = NumberWrapper.parseInt(hex, 16);
+          } catch (e) {
+            this.error(`Invalid unicode escape [\\u${hex}]`, 0);
+          }
           for (var i:int = 0; i < 5; i++) {
             this.advance();
           }
@@ -348,7 +361,7 @@ export class Scanner {
         buffer.add(StringWrapper.fromCharCode(unescapedCode));
         marker = this.index;
       } else if (this.peek == $EOF) {
-        this.error('Unterminated quote');
+        this.error('Unterminated quote', 0);
       } else {
         this.advance();
       }
@@ -367,9 +380,9 @@ export class Scanner {
     return newStringToken(start, unescaped);
   }
 
-  error(message:string) {
+  error(message:string, offset:int) {
-    var position:int = this.index;
+    var position:int = this.index + offset;
-    throw `Lexer Error: ${message} at column ${position} in expression [${input}]`;
+    throw new ScannerError(`Lexer Error: ${message} at column ${position} in expression [${this.input}]`);
   }
 }
 

modules/change_detection/test/parser/lexer_spec.js

@@ -1,108 +0,0 @@
-import {describe, it, expect} from 'test_lib/test_lib';
-import {Scanner, Token} from 'change_detection/parser/scanner';
-import {List, ListWrapper} from "facade/collection";
-import {StringWrapper} from "facade/lang";
-
-function lex(text:string):List {
-  var scanner:Scanner = new Scanner(text);
-  var tokens:List<Token> = [];
-  var token:Token = scanner.scanToken();
-  while (token != null) {
-    ListWrapper.push(tokens, token);
-    token = scanner.scanToken();
-  }
-  return tokens;
-}
-
-function expectToken(token, index) {
-  expect(token instanceof Token).toBe(true);
-  expect(token.index).toEqual(index);
-}
-
-function expectCharacterToken(token, index, character) {
-  expect(character.length).toBe(1);
-  expectToken(token, index);
-  expect(token.isCharacter(StringWrapper.charCodeAt(character, 0))).toBe(true);
-}
-
-function expectOperatorToken(token, index, operator) {
-  expectToken(token, index);
-  expect(token.isOperator(operator)).toBe(true);
-}
-
-function expectNumberToken(token, index, n) {
-  expectToken(token, index);
-  expect(token.isNumber()).toBe(true);
-  expect(token.toNumber()).toEqual(n);
-}
-
-function expectStringToken(token, index, str) {
-  expectToken(token, index);
-  expect(token.isString()).toBe(true);
-  expect(token.toString()).toEqual(str);
-}
-
-function expectIdentifierToken(token, index, identifier) {
-  expectToken(token, index);
-  expect(token.isIdentifier()).toBe(true);
-  expect(token.toString()).toEqual(identifier);
-}
-
-function expectKeywordToken(token, index, keyword) {
-  expectToken(token, index);
-  expect(token.isKeyword()).toBe(true);
-  expect(token.toString()).toEqual(keyword);
-}
-
-
-export function main() {
-  describe('scanner', function() {
-    describe('token', function() {
-      it('should tokenize a simple identifier', function() {
-        var tokens:List<int> = lex("j");
-        expect(tokens.length).toEqual(1);
-        expectIdentifierToken(tokens[0], 0, 'j');
-      });
-
-      it('should tokenize a dotted identifier', function() {
-        var tokens:List<int> = lex("j.k");
-        expect(tokens.length).toEqual(3);
-        expectIdentifierToken(tokens[0], 0, 'j');
-        expectCharacterToken (tokens[1], 1, '.');
-        expectIdentifierToken(tokens[2], 2, 'k');
-      });
-
-      it('should tokenize an operator', function() {
-        var tokens:List<int> = lex("j-k");
-        expect(tokens.length).toEqual(3);
-        expectOperatorToken(tokens[1], 1, '-');
-      });
-
-      it('should tokenize an indexed operator', function() {
-        var tokens:List<int> = lex("j[k]");
-        expect(tokens.length).toEqual(4);
-        expectCharacterToken(tokens[1], 1, "[");
-        expectCharacterToken(tokens[3], 3, "]");
-      });
-
-      it('should tokenize numbers', function() {
-        var tokens:List<int> = lex("88");
-        expect(tokens.length).toEqual(1);
-        expectNumberToken(tokens[0], 0, 88);
-      });
-
-      it('should tokenize numbers within index ops', function() {
-        expectNumberToken(lex("a[22]")[2], 2, 22);
-      });
-
-      it('should tokenize simple quoted strings', function() {
-        expectStringToken(lex('"a"')[0], 0, "a");
-      });
-
-      it('should tokenize quoted strings with escaped quotes', function() {
-        expectStringToken(lex('"a\\""')[0], 0, 'a"');
-      });
-
-    });
-  });
-}

modules/change_detection/test/parser/scanner_spec.js

@@ -0,0 +1,249 @@
+import {describe, it, expect} from 'test_lib/test_lib';
+import {Scanner, Token} from 'change_detection/parser/scanner';
+import {List, ListWrapper} from "facade/collection";
+import {StringWrapper} from "facade/lang";
+
+function lex(text:string):List {
+  var scanner:Scanner = new Scanner(text);
+  var tokens:List<Token> = [];
+  var token:Token = scanner.scanToken();
+  while (token != null) {
+    ListWrapper.push(tokens, token);
+    token = scanner.scanToken();
+  }
+  return tokens;
+}
+
+function expectToken(token, index) {
+  expect(token instanceof Token).toBe(true);
+  expect(token.index).toEqual(index);
+}
+
+function expectCharacterToken(token, index, character) {
+  expect(character.length).toBe(1);
+  expectToken(token, index);
+  expect(token.isCharacter(StringWrapper.charCodeAt(character, 0))).toBe(true);
+}
+
+function expectOperatorToken(token, index, operator) {
+  expectToken(token, index);
+  expect(token.isOperator(operator)).toBe(true);
+}
+
+function expectNumberToken(token, index, n) {
+  expectToken(token, index);
+  expect(token.isNumber()).toBe(true);
+  expect(token.toNumber()).toEqual(n);
+}
+
+function expectStringToken(token, index, str) {
+  expectToken(token, index);
+  expect(token.isString()).toBe(true);
+  expect(token.toString()).toEqual(str);
+}
+
+function expectIdentifierToken(token, index, identifier) {
+  expectToken(token, index);
+  expect(token.isIdentifier()).toBe(true);
+  expect(token.toString()).toEqual(identifier);
+}
+
+function expectKeywordToken(token, index, keyword) {
+  expectToken(token, index);
+  expect(token.isKeyword()).toBe(true);
+  expect(token.toString()).toEqual(keyword);
+}
+
+
+export function main() {
+  describe('scanner', function() {
+    describe('token', function() {
+      it('should tokenize a simple identifier', function() {
+        var tokens:List<int> = lex("j");
+        expect(tokens.length).toEqual(1);
+        expectIdentifierToken(tokens[0], 0, 'j');
+      });
+
+      it('should tokenize a dotted identifier', function() {
+        var tokens:List<int> = lex("j.k");
+        expect(tokens.length).toEqual(3);
+        expectIdentifierToken(tokens[0], 0, 'j');
+        expectCharacterToken (tokens[1], 1, '.');
+        expectIdentifierToken(tokens[2], 2, 'k');
+      });
+
+      it('should tokenize an operator', function() {
+        var tokens:List<int> = lex("j-k");
+        expect(tokens.length).toEqual(3);
+        expectOperatorToken(tokens[1], 1, '-');
+      });
+
+      it('should tokenize an indexed operator', function() {
+        var tokens:List<int> = lex("j[k]");
+        expect(tokens.length).toEqual(4);
+        expectCharacterToken(tokens[1], 1, "[");
+        expectCharacterToken(tokens[3], 3, "]");
+      });
+
+      it('should tokenize numbers', function() {
+        var tokens:List<int> = lex("88");
+        expect(tokens.length).toEqual(1);
+        expectNumberToken(tokens[0], 0, 88);
+      });
+
+      it('should tokenize numbers within index ops', function() {
+        expectNumberToken(lex("a[22]")[2], 2, 22);
+      });
+
+      it('should tokenize simple quoted strings', function() {
+        expectStringToken(lex('"a"')[0], 0, "a");
+      });
+
+      it('should tokenize quoted strings with escaped quotes', function() {
+        expectStringToken(lex('"a\\""')[0], 0, 'a"');
+      });
+
+      it('should tokenize a string', function() {
+        var tokens:List<Token> = lex("j-a.bc[22]+1.3|f:'a\\\'c':\"d\\\"e\"");
+        expectIdentifierToken(tokens[0], 0, 'j');
+        expectOperatorToken(tokens[1], 1, '-');
+        expectIdentifierToken(tokens[2], 2, 'a');
+        expectCharacterToken(tokens[3], 3, '.');
+        expectIdentifierToken(tokens[4], 4, 'bc');
+        expectCharacterToken(tokens[5], 6, '[');
+        expectNumberToken(tokens[6], 7, 22);
+        expectCharacterToken(tokens[7], 9, ']');
+        expectOperatorToken(tokens[8], 10, '+');
+        expectNumberToken(tokens[9], 11, 1.3);
+        expectOperatorToken(tokens[10], 14, '|');
+        expectIdentifierToken(tokens[11], 15, 'f');
+        expectCharacterToken(tokens[12], 16, ':');
+        expectStringToken(tokens[13], 17, "a'c");
+        expectCharacterToken(tokens[14], 23, ':');
+        expectStringToken(tokens[15], 24, 'd"e');
+      });
+
+      it('should tokenize undefined', function() {
+        var tokens:List<Token> = lex("undefined");
+        expectKeywordToken(tokens[0], 0, "undefined");
+        expect(tokens[0].isKeywordUndefined()).toBe(true);
+      });
+
+      it('should ignore whitespace', function() {
+        var tokens:List<Token> = lex("a \t \n \r b");
+        expectIdentifierToken(tokens[0], 0, 'a');
+        expectIdentifierToken(tokens[1], 8, 'b');
+      });
+
+      it('should tokenize quoted string', function() {
+        var str = "['\\'', \"\\\"\"]";
+        var tokens:List<Token> = lex(str);
+        expectStringToken(tokens[1], 1, "'");
+        expectStringToken(tokens[3], 7, '"');
+      });
+
+      it('should tokenize escaped quoted string', function() {
+        var str = '"\\"\\n\\f\\r\\t\\v\\u00A0"';
+        var tokens:List<Token> = lex(str);
+        expect(tokens.length).toEqual(1);
+        expect(tokens[0].toString()).toEqual('"\n\f\r\t\v\u00A0');
+      });
+
+      it('should tokenize unicode', function() {
+        var tokens:List<Token> = lex('"\\u00A0"');
+        expect(tokens.length).toEqual(1);
+        expect(tokens[0].toString()).toEqual('\u00a0');
+      });
+
+      it('should tokenize relation', function() {
+        var tokens:List<Token> = lex("! == != < > <= >=");
+        expectOperatorToken(tokens[0], 0, '!');
+        expectOperatorToken(tokens[1], 2, '==');
+        expectOperatorToken(tokens[2], 5, '!=');
+        expectOperatorToken(tokens[3], 8, '<');
+        expectOperatorToken(tokens[4], 10, '>');
+        expectOperatorToken(tokens[5], 12, '<=');
+        expectOperatorToken(tokens[6], 15, '>=');
+      });
+
+      it('should tokenize statements', function() {
+        var tokens:List<Token> = lex("a;b;");
+        expectIdentifierToken(tokens[0], 0, 'a');
+        expectCharacterToken(tokens[1], 1, ';');
+        expectIdentifierToken(tokens[2], 2, 'b');
+        expectCharacterToken(tokens[3], 3, ';');
+      });
+
+      it('should tokenize function invocation', function() {
+        var tokens:List<Token> = lex("a()");
+        expectIdentifierToken(tokens[0], 0, 'a');
+        expectCharacterToken(tokens[1], 1, '(');
+        expectCharacterToken(tokens[2], 2, ')');
+      });
+
+      it('should tokenize simple method invocations', function() {
+        var tokens:List<Token> = lex("a.method()");
+        expectIdentifierToken(tokens[2], 2, 'method');
+      });
+
+      it('should tokenize method invocation', function() {
+        var tokens:List<Token> = lex("a.b.c (d) - e.f()");
+        expectIdentifierToken(tokens[0], 0, 'a');
+        expectCharacterToken(tokens[1], 1, '.');
+        expectIdentifierToken(tokens[2], 2, 'b');
+        expectCharacterToken(tokens[3], 3, '.');
+        expectIdentifierToken(tokens[4], 4, 'c');
+        expectCharacterToken(tokens[5], 6, '(');
+        expectIdentifierToken(tokens[6], 7, 'd');
+        expectCharacterToken(tokens[7], 8, ')');
+        expectOperatorToken(tokens[8], 10, '-');
+        expectIdentifierToken(tokens[9], 12, 'e');
+        expectCharacterToken(tokens[10], 13, '.');
+        expectIdentifierToken(tokens[11], 14, 'f');
+        expectCharacterToken(tokens[12], 15, '(');
+        expectCharacterToken(tokens[13], 16, ')');
+      });
+
+      it('should tokenize number', function() {
+        var tokens:List<Token> = lex("0.5");
+        expectNumberToken(tokens[0], 0, 0.5);
+      });
+
+      // NOTE(deboer): NOT A LEXER TEST
+      //    it('should tokenize negative number', function() {
+      //      var tokens:List<Token> = lex("-0.5");
+      //      expectNumberToken(tokens[0], 0, -0.5);
+      //    });
+
+      it('should tokenize number with exponent', function() {
+        var tokens:List<Token> = lex("0.5E-10");
+        expect(tokens.length).toEqual(1);
+        expectNumberToken(tokens[0], 0, 0.5E-10);
+        tokens = lex("0.5E+10");
+        expectNumberToken(tokens[0], 0, 0.5E+10);
+      });
+
+      it('should throws exception for invalid exponent', function() {
+        expect(function() {
+          lex("0.5E-");
+        }).toThrowError('Lexer Error: Invalid exponent at column 4 in expression [0.5E-]');
+
+        expect(function() {
+          lex("0.5E-A");
+        }).toThrowError('Lexer Error: Invalid exponent at column 4 in expression [0.5E-A]');
+      });
+
+      it('should tokenize number starting with a dot', function() {
+        var tokens:List<Token> = lex(".5");
+        expectNumberToken(tokens[0], 0, 0.5);
+      });
+
+      it('should throw error on invalid unicode', function() {
+        expect(function() {
+          lex("'\\u1''bla'");
+        }).toThrowError("Lexer Error: Invalid unicode escape [\\u1''b] at column 2 in expression ['\\u1''bla']");
+      });
+
+    });
+  });
+}

modules/facade/src/lang.es6

@@ -59,22 +59,43 @@ export class StringJoiner {
   }
 }
 
+export class NumerParseError extends Error {
+  constructor(message) {
+    this.message = message;
+  }
+
+  toString() {
+    return this.message;
+  }
+}
+
+
 export class NumberWrapper {
   static parseIntAutoRadix(text:string):int {
     var result:int = parseInt(text);
     if (isNaN(result)) {
-      throw new Error("Invalid integer literal when parsing " + text);
+      throw new NumerParseError("Invalid integer literal when parsing " + text);
     }
     return result;
   }
 
   static parseInt(text:string, radix:int):int {
-    var result:int = parseInt(text, radix);
+    if (radix == 10) {
-    if (isNaN(result)) {
+      if (/^(\-|\+)?[0-9]+$/.test(text)) {
-      throw new Error("Invalid integer literal when parsing " + text + " in base " + radix);
+        return parseInt(text, radix);
       }
+    } else if (radix == 16) {
+      if (/^(\-|\+)?[0-9ABCDEFabcdef]+$/.test(text)) {
+        return parseInt(text, radix);
+      }
+    } else {
+      var result:int = parseInt(text, radix);
+      if (!isNaN(result)) {
         return result;
       }
+    }
+    throw new NumerParseError("Invalid integer literal when parsing " + text + " in base " + radix);
+  }
 
   // TODO: NaN is a valid literal but is returned by parseFloat to indicate an error.
   static parseFloat(text:string):number {