diff --git a/src/test/java/org/apache/commons/csv/CSVLexerTest.java b/src/test/java/org/apache/commons/csv/CSVLexerTest.java index 8e288448..139b87cd 100644 --- a/src/test/java/org/apache/commons/csv/CSVLexerTest.java +++ b/src/test/java/org/apache/commons/csv/CSVLexerTest.java @@ -89,6 +89,42 @@ public class CSVLexerTest { } + // multiline including comments (and empty lines) + @Test + public void testNextToken2EmptyLines() throws IOException { + final String code = + "1,2,3,\n"+ // 1 + "a,b x,c#no-comment\n"+ // 2 + "#foo\n"+ // 3 + "\n"+ // 4 + "d,e,#no-comment\n"+ // 5 + "# penultimate comment\n"+ // 6 + "# Final comment\n"; // 7 + CSVFormat format = CSVFormat.DEFAULT.withCommentStart('#').withEmptyLinesIgnored(false); + assertFalse("Should not ignore empty lines", format.isEmptyLinesIgnored()); + + Lexer parser = getLexer(code, format); + + + assertTokenEquals(TOKEN, "1", parser.nextToken(new Token())); + assertTokenEquals(TOKEN, "2", parser.nextToken(new Token())); + assertTokenEquals(TOKEN, "3", parser.nextToken(new Token())); + assertTokenEquals(EORECORD, "", parser.nextToken(new Token())); // 1 + assertTokenEquals(TOKEN, "a", parser.nextToken(new Token())); + assertTokenEquals(TOKEN, "b x", parser.nextToken(new Token())); + assertTokenEquals(EORECORD, "c#no-comment", parser.nextToken(new Token())); // 2 + assertTokenEquals(COMMENT, "", parser.nextToken(new Token())); // 3 + assertTokenEquals(EORECORD, "", parser.nextToken(new Token())); // 4 + assertTokenEquals(TOKEN, "d", parser.nextToken(new Token())); + assertTokenEquals(TOKEN, "e", parser.nextToken(new Token())); + assertTokenEquals(EORECORD, "#no-comment", parser.nextToken(new Token())); // 5 + assertTokenEquals(COMMENT, "", parser.nextToken(new Token())); // 6 + assertTokenEquals(COMMENT, "", parser.nextToken(new Token())); // 7 + assertTokenEquals(EOF, "", parser.nextToken(new Token())); + assertTokenEquals(EOF, "", parser.nextToken(new Token())); + + } + // simple token with escaping @Test public void testNextToken3() throws IOException {