Add direct lexer tests

git-svn-id: https://svn.apache.org/repos/asf/commons/proper/csv/trunk@1303719 13f79535-47bb-0310-9956-ffa450edef68

src/test/java/org/apache/commons/csv/PerformanceTest.java

@@ -68,7 +68,7 @@ public class PerformanceTest {
                 tests[i-1]=args[i];
             }
         } else {
-            tests=new String[]{"file", "split", "extb", "exts", "csv"};
+            tests=new String[]{"file", "split", "extb", "exts", "csv", "lexreset", "lexnew"};
         }
         for(String p : PROPS) {
             System.out.println(p+"="+System.getProperty(p));
@@ -82,6 +82,10 @@ public class PerformanceTest {
                 testReadBigFile(true);
             } else if ("csv".equals(test)) {
                 testParseCommonsCSV();
+            } else if ("lexreset".equals(test)) {
+                testCSVLexer(false, test);
+            } else if ("lexnew".equals(test)) {
+                testCSVLexer(true, test);
             } else if ("extb".equals(test)) {
                 testExtendedBuffer(false);
             } else if ("exts".equals(test)) {
@@ -194,6 +198,43 @@ public class PerformanceTest {
        show();
    }
 
+   private static void testCSVLexer(final boolean newToken, String test) throws Exception {
+       Token token = new Token();
+       for (int i = 0; i < max; i++) {
+           final BufferedReader reader = getReader();
+           Lexer lexer = new CSVLexer(format, new ExtendedBufferedReader(reader));
+           int count = 0;
+           int fields = 0;
+           long t0 = System.currentTimeMillis();
+           do {
+               if (newToken) {
+                   token = new Token();
+               } else {
+                   token.reset();
+               }
+               lexer.nextToken(token);
+               switch(token.type) {
+               case EOF:
+                   break;
+               case EORECORD:
+                   fields++;
+                   count++;
+                   break;
+               case INVALID:
+                   throw new IOException("invalid parse sequence");
+               case TOKEN:
+                   fields++;
+                   break;
+              }
+
+           } while (!token.type.equals(Token.Type.EOF));
+           Stats s = new Stats(count, fields);
+           reader.close();
+           show(test, s, t0);
+       }
+       show();
+   }
+
    private static Stats iterate(Iterable<CSVRecord> it) {
        int count = 0;
        int fields = 0;