Apache
/
commons-csv
mirror of https://github.com/apache/commons-csv.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Separate testing empty line handling from comment recognition 

git-svn-id: https://svn.apache.org/repos/asf/commons/proper/csv/trunk@1465722 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Benedikt Ritter 2013-04-08 18:55:58 +00:00
parent 449e9a8cf6
commit 5744ee8a16
1 changed files with 41 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
src/test/java/org/apache/commons/csv/CSVLexerTest.java
View File

@ -73,41 +73,60 @@ public class CSVLexerTest {
@Test @Test
public void testIgnoreEmptyLines() throws IOException { public void testIgnoreEmptyLines() throws IOException {
final String code = final String code =
"1,2,3,\n"+ // 1 "first,line,\n"+
"\n"+ "\n"+
"\n"+ "\n"+
"a,b x,c#no-comment\n"+ // 2 "second,line\n"+
"\n"+ "\n"+
"\n"+ "\n"+
"# foo \n"+ // 3 "third line \n"+
"\n"+ // 4
"d,e,#no-comment\n"+ // 5
"\n"+ "\n"+
"\n"+ "\n"+
"# penultimate comment\n"+ // 6 "last, line \n"+
"\n"+ "\n"+
"\n"+ "\n"+
"# Final comment\n"; // 7 "\n";
final CSVFormat format = CSVFormat.newBuilder().withCommentStart('#').withIgnoreEmptyLines(true).build(); final CSVFormat format = CSVFormat.newBuilder().withIgnoreEmptyLines(true).build();
final Lexer parser = getLexer(code, format); final Lexer parser = getLexer(code, format);
assertThat(parser.nextToken(new Token()), matches(TOKEN, "1")); assertThat(parser.nextToken(new Token()), matches(TOKEN, "first"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "2")); assertThat(parser.nextToken(new Token()), matches(TOKEN, "line"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "3")); assertThat(parser.nextToken(new Token()), matches(EORECORD, ""));
assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); // 1 assertThat(parser.nextToken(new Token()), matches(TOKEN, "second"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "a")); assertThat(parser.nextToken(new Token()), matches(EORECORD, "line"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "b x")); assertThat(parser.nextToken(new Token()), matches(EORECORD, "third line "));
assertThat(parser.nextToken(new Token()), matches(EORECORD, "c#no-comment")); // 2 assertThat(parser.nextToken(new Token()), matches(TOKEN, "last"));
assertThat(parser.nextToken(new Token()), matches(COMMENT, "foo")); // 3 assertThat(parser.nextToken(new Token()), matches(EORECORD, " line "));
// 4 empty line, ignored // 4
assertThat(parser.nextToken(new Token()), matches(TOKEN, "d"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "e"));
assertThat(parser.nextToken(new Token()), matches(EORECORD, "#no-comment")); // 5
assertThat(parser.nextToken(new Token()), matches(COMMENT, "penultimate comment")); // 6
assertThat(parser.nextToken(new Token()), matches(COMMENT, "Final comment")); // 7
assertThat(parser.nextToken(new Token()), matches(EOF, "")); assertThat(parser.nextToken(new Token()), matches(EOF, ""));
assertThat(parser.nextToken(new Token()), matches(EOF, "")); assertThat(parser.nextToken(new Token()), matches(EOF, ""));
}
@Test
public void testComments() throws IOException {
final String code =
"first,line,\n"+
"second,line,tokenWith#no-comment\n"+
"# comment line \n"+
"third,line,#no-comment\n"+
"# penultimate comment\n"+
"# Final comment\n";
final CSVFormat format = CSVFormat.newBuilder().withCommentStart('#').build();
final Lexer parser = getLexer(code, format);
assertThat(parser.nextToken(new Token()), matches(TOKEN, "first"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "line"));
assertThat(parser.nextToken(new Token()), matches(EORECORD, ""));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "second"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "line"));
assertThat(parser.nextToken(new Token()), matches(EORECORD, "tokenWith#no-comment"));
assertThat(parser.nextToken(new Token()), matches(COMMENT, "comment line"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "third"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "line"));
assertThat(parser.nextToken(new Token()), matches(EORECORD, "#no-comment"));
assertThat(parser.nextToken(new Token()), matches(COMMENT, "penultimate comment"));
assertThat(parser.nextToken(new Token()), matches(COMMENT, "Final comment"));
assertThat(parser.nextToken(new Token()), matches(EOF, ""));
assertThat(parser.nextToken(new Token()), matches(EOF, ""));
} }
// multiline including comments (and empty lines) // multiline including comments (and empty lines)