diff --git a/src/test/java/org/apache/commons/csv/CSVLexerTest.java b/src/test/java/org/apache/commons/csv/CSVLexerTest.java index 6c53c587..79812209 100644 --- a/src/test/java/org/apache/commons/csv/CSVLexerTest.java +++ b/src/test/java/org/apache/commons/csv/CSVLexerTest.java @@ -73,41 +73,60 @@ public class CSVLexerTest { @Test public void testIgnoreEmptyLines() throws IOException { final String code = - "1,2,3,\n"+ // 1 + "first,line,\n"+ "\n"+ "\n"+ - "a,b x,c#no-comment\n"+ // 2 + "second,line\n"+ "\n"+ "\n"+ - "# foo \n"+ // 3 - "\n"+ // 4 - "d,e,#no-comment\n"+ // 5 + "third line \n"+ "\n"+ "\n"+ - "# penultimate comment\n"+ // 6 + "last, line \n"+ "\n"+ "\n"+ - "# Final comment\n"; // 7 - final CSVFormat format = CSVFormat.newBuilder().withCommentStart('#').withIgnoreEmptyLines(true).build(); + "\n"; + final CSVFormat format = CSVFormat.newBuilder().withIgnoreEmptyLines(true).build(); final Lexer parser = getLexer(code, format); - assertThat(parser.nextToken(new Token()), matches(TOKEN, "1")); - assertThat(parser.nextToken(new Token()), matches(TOKEN, "2")); - assertThat(parser.nextToken(new Token()), matches(TOKEN, "3")); - assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); // 1 - assertThat(parser.nextToken(new Token()), matches(TOKEN, "a")); - assertThat(parser.nextToken(new Token()), matches(TOKEN, "b x")); - assertThat(parser.nextToken(new Token()), matches(EORECORD, "c#no-comment")); // 2 - assertThat(parser.nextToken(new Token()), matches(COMMENT, "foo")); // 3 - // 4 empty line, ignored // 4 - assertThat(parser.nextToken(new Token()), matches(TOKEN, "d")); - assertThat(parser.nextToken(new Token()), matches(TOKEN, "e")); - assertThat(parser.nextToken(new Token()), matches(EORECORD, "#no-comment")); // 5 - assertThat(parser.nextToken(new Token()), matches(COMMENT, "penultimate comment")); // 6 - assertThat(parser.nextToken(new Token()), matches(COMMENT, "Final comment")); // 7 + assertThat(parser.nextToken(new Token()), matches(TOKEN, "first")); + assertThat(parser.nextToken(new Token()), matches(TOKEN, "line")); + assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); + assertThat(parser.nextToken(new Token()), matches(TOKEN, "second")); + assertThat(parser.nextToken(new Token()), matches(EORECORD, "line")); + assertThat(parser.nextToken(new Token()), matches(EORECORD, "third line ")); + assertThat(parser.nextToken(new Token()), matches(TOKEN, "last")); + assertThat(parser.nextToken(new Token()), matches(EORECORD, " line ")); assertThat(parser.nextToken(new Token()), matches(EOF, "")); assertThat(parser.nextToken(new Token()), matches(EOF, "")); + } + @Test + public void testComments() throws IOException { + final String code = + "first,line,\n"+ + "second,line,tokenWith#no-comment\n"+ + "# comment line \n"+ + "third,line,#no-comment\n"+ + "# penultimate comment\n"+ + "# Final comment\n"; + final CSVFormat format = CSVFormat.newBuilder().withCommentStart('#').build(); + final Lexer parser = getLexer(code, format); + + assertThat(parser.nextToken(new Token()), matches(TOKEN, "first")); + assertThat(parser.nextToken(new Token()), matches(TOKEN, "line")); + assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); + assertThat(parser.nextToken(new Token()), matches(TOKEN, "second")); + assertThat(parser.nextToken(new Token()), matches(TOKEN, "line")); + assertThat(parser.nextToken(new Token()), matches(EORECORD, "tokenWith#no-comment")); + assertThat(parser.nextToken(new Token()), matches(COMMENT, "comment line")); + assertThat(parser.nextToken(new Token()), matches(TOKEN, "third")); + assertThat(parser.nextToken(new Token()), matches(TOKEN, "line")); + assertThat(parser.nextToken(new Token()), matches(EORECORD, "#no-comment")); + assertThat(parser.nextToken(new Token()), matches(COMMENT, "penultimate comment")); + assertThat(parser.nextToken(new Token()), matches(COMMENT, "Final comment")); + assertThat(parser.nextToken(new Token()), matches(EOF, "")); + assertThat(parser.nextToken(new Token()), matches(EOF, "")); } // multiline including comments (and empty lines)