From 0a0f3bbd365c470c2ae86b1cf383838bfaab77e1 Mon Sep 17 00:00:00 2001
From: Sebastian Bazley <sebb@apache.org>
Date: Wed, 28 Mar 2012 00:01:43 +0000
Subject: [PATCH] CSV-70 Improve readability of CSVLexer Introduce COMMENT type

git-svn-id: https://svn.apache.org/repos/asf/commons/proper/csv/trunk@1306064 13f79535-47bb-0310-9956-ffa450edef68
---
 src/main/java/org/apache/commons/csv/CSVLexer.java     | 2 +-
 src/main/java/org/apache/commons/csv/CSVParser.java    | 2 ++
 src/main/java/org/apache/commons/csv/Token.java        | 5 ++++-
 src/test/java/org/apache/commons/csv/CSVLexerTest.java | 1 +
 4 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/apache/commons/csv/CSVLexer.java b/src/main/java/org/apache/commons/csv/CSVLexer.java
index 943e3675..c4554c29 100644
--- a/src/main/java/org/apache/commons/csv/CSVLexer.java
+++ b/src/main/java/org/apache/commons/csv/CSVLexer.java
@@ -92,7 +92,7 @@ class CSVLexer extends Lexer {
             if (isCommentStart(c)) { // TODO should only match at start of line
                 // ignore everything till end of line and continue (incr linecount)
                 in.readLine();
-                tkn = nextToken(tkn.reset());
+                tkn.type = COMMENT;
             } else if (isDelimiter(c)) {
                 // empty token return TOKEN("")
                 tkn.type = TOKEN;
diff --git a/src/main/java/org/apache/commons/csv/CSVParser.java b/src/main/java/org/apache/commons/csv/CSVParser.java
index f5eacc30..ec4c0791 100644
--- a/src/main/java/org/apache/commons/csv/CSVParser.java
+++ b/src/main/java/org/apache/commons/csv/CSVParser.java
@@ -154,6 +154,8 @@ public class CSVParser implements Iterable<CSVRecord> {
                     break;
                 case INVALID:
                     throw new IOException("(line " + getLineNumber() + ") invalid parse sequence");
+                case COMMENT: // Ignored currently
+                    break;
             }
         } while (reusableToken.type == TOKEN);
         
diff --git a/src/main/java/org/apache/commons/csv/Token.java b/src/main/java/org/apache/commons/csv/Token.java
index 5c7e8fbb..100f1e97 100644
--- a/src/main/java/org/apache/commons/csv/Token.java
+++ b/src/main/java/org/apache/commons/csv/Token.java
@@ -41,7 +41,10 @@ class Token {
         EOF,
         
         /** Token with content when end of a line is reached. */
-        EORECORD
+        EORECORD,
+        
+        /** Token is a comment line */
+        COMMENT
     }
     
     /** Token type */
diff --git a/src/test/java/org/apache/commons/csv/CSVLexerTest.java b/src/test/java/org/apache/commons/csv/CSVLexerTest.java
index 9db704ca..27680aff 100644
--- a/src/test/java/org/apache/commons/csv/CSVLexerTest.java
+++ b/src/test/java/org/apache/commons/csv/CSVLexerTest.java
@@ -76,6 +76,7 @@ public class CSVLexerTest {
         assertTokenEquals(TOKEN, "a", parser.nextToken(new Token()));
         assertTokenEquals(TOKEN, "b x", parser.nextToken(new Token()));
         assertTokenEquals(EORECORD, "c", parser.nextToken(new Token()));
+        assertTokenEquals(COMMENT, "", parser.nextToken(new Token()));
         assertTokenEquals(EORECORD, "", parser.nextToken(new Token()));
         assertTokenEquals(TOKEN, "d", parser.nextToken(new Token()));
         assertTokenEquals(TOKEN, "e", parser.nextToken(new Token()));