Apache
/
commons-csv
mirror of https://github.com/apache/commons-csv.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Use try-with-resources. 

git-svn-id: https://svn.apache.org/repos/asf/commons/proper/csv/trunk@1748094 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Gary D. Gregory 2016-06-13 06:56:49 +00:00
parent ed6adc706e
commit 9daee9042c
12 changed files with 1131 additions and 1144 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/main/java/org/apache/commons/csv/CSVFormat.java
View File

@ -609,8 +609,8 @@ public final class CSVFormat implements Serializable {
*/
public String format(final Object... values) {
final StringWriter out = new StringWriter();
try {
new CSVPrinter(out, this).printRecord(values);
try (final CSVPrinter csvPrinter = new CSVPrinter(out, this)) {
csvPrinter.printRecord(values);
return out.toString().trim();
} catch (final IOException e) {
// should not happen because a StringWriter does not do IO.

40
src/test/java/org/apache/commons/csv/CSVFileParserTest.java
View File

@ -116,19 +116,19 @@ public class CSVFileParserTest {
// Now parse the file and compare against the expected results
// We use a buffered reader internally so no need to create one here.
final CSVParser parser = CSVParser.parse(new File(BASE, split[0]), Charset.defaultCharset(), format);
for (final CSVRecord record : parser) {
String parsed = Arrays.toString(record.values());
if (checkComments) {
final String comment = record.getComment().replace("\n", "\\n");
if (comment != null) {
parsed += "#" + comment;
try (final CSVParser parser = CSVParser.parse(new File(BASE, split[0]), Charset.defaultCharset(), format)) {
for (final CSVRecord record : parser) {
String parsed = Arrays.toString(record.values());
if (checkComments) {
final String comment = record.getComment().replace("\n", "\\n");
if (comment != null) {
parsed += "#" + comment;
}
}
final int count = record.size();
assertEquals(testName, readTestData(), count + ":" + parsed);
}
final int count = record.size();
assertEquals(testName, readTestData(), count + ":" + parsed);
}
parser.close();
}
@Test
@ -160,18 +160,18 @@ public class CSVFileParserTest {
// Now parse the file and compare against the expected results
final URL resource = ClassLoader.getSystemResource("CSVFileParser/" + split[0]);
final CSVParser parser = CSVParser.parse(resource, Charset.forName("UTF-8"), format);
for (final CSVRecord record : parser) {
String parsed = Arrays.toString(record.values());
if (checkComments) {
final String comment = record.getComment().replace("\n", "\\n");
if (comment != null) {
parsed += "#" + comment;
try (final CSVParser parser = CSVParser.parse(resource, Charset.forName("UTF-8"), format)) {
for (final CSVRecord record : parser) {
String parsed = Arrays.toString(record.values());
if (checkComments) {
final String comment = record.getComment().replace("\n", "\\n");
if (comment != null) {
parsed += "#" + comment;
}
}
final int count = record.size();
assertEquals(testName, readTestData(), count + ":" + parsed);
}
final int count = record.size();
assertEquals(testName, readTestData(), count + ":" + parsed);
}
parser.close();
}
}

8
src/test/java/org/apache/commons/csv/CSVFormatTest.java
View File

@ -327,10 +327,10 @@ public class CSVFormatTest {
public void testSerialization() throws Exception {
final ByteArrayOutputStream out = new ByteArrayOutputStream();
final ObjectOutputStream oos = new ObjectOutputStream(out);
oos.writeObject(CSVFormat.DEFAULT);
oos.flush();
oos.close();
try (final ObjectOutputStream oos = new ObjectOutputStream(out)) {
oos.writeObject(CSVFormat.DEFAULT);
oos.flush();
}
final ObjectInputStream in = new ObjectInputStream(new ByteArrayInputStream(out.toByteArray()));
final CSVFormat format = (CSVFormat) in.readObject();

551
src/test/java/org/apache/commons/csv/CSVParserTest.java
View File

@ -61,7 +61,7 @@ import org.junit.Test;
public class CSVParserTest {
private static final String CSV_INPUT = "a,b,c,d\n" + " a , b , 1 2 \n" + "\"foo baar\", b,\n"
// + " \"foo\n,,\n\"\",,\n\\\"\",d,e\n";
// + " \"foo\n,,\n\"\",,\n\\\"\",d,e\n";
+ " \"foo\n,,\n\"\",,\n\"\"\",d,e\n"; // changed to use standard CSV escaping
private static final String CSV_INPUT_1 = "a,b,c,d";
@ -79,7 +79,7 @@ public class CSVParserTest {
// quote as the encapsulator.
final String code = "one,two,three\n" // 0
+ "'',''\n" // 1) empty encapsulators
+ "'',''\n" // 1) empty encapsulators
+ "/',/'\n" // 2) single encapsulators
+ "'/'','/''\n" // 3) single encapsulators encapsulated via escape
+ "'''',''''\n" // 4) single encapsulators encapsulated via doubling
@ -102,12 +102,12 @@ public class CSVParserTest {
final CSVFormat format = CSVFormat.newFormat(',').withQuote('\'').withRecordSeparator(CRLF).withEscape('/')
.withIgnoreEmptyLines();
final CSVParser parser = CSVParser.parse(code, format);
final List<CSVRecord> records = parser.getRecords();
assertTrue(records.size() > 0);
try (final CSVParser parser = CSVParser.parse(code, format)) {
final List<CSVRecord> records = parser.getRecords();
assertTrue(records.size() > 0);
Utils.compare("Records do not match expected result", res, records);
parser.close();
Utils.compare("Records do not match expected result", res, records);
}
}
@Test
@ -129,104 +129,98 @@ public class CSVParserTest {
final CSVFormat format = CSVFormat.newFormat(',').withRecordSeparator(CRLF).withEscape('/')
.withIgnoreEmptyLines();
final CSVParser parser = CSVParser.parse(code, format);
final List<CSVRecord> records = parser.getRecords();
assertTrue(records.size() > 0);
try (final CSVParser parser = CSVParser.parse(code, format)) {
final List<CSVRecord> records = parser.getRecords();
assertTrue(records.size() > 0);
Utils.compare("", res, records);
parser.close();
Utils.compare("", res, records);
}
}
@Test
@Ignore
public void testBackslashEscapingOld() throws IOException {
final String code = "one,two,three\n" + "on\\\"e,two\n" + "on\"e,two\n" + "one,\"tw\\\"o\"\n"
+ "one,\"t\\,wo\"\n" + "one,two,\"th,ree\"\n" + "\"a\\\\\"\n" + "a\\,b\n" + "\"a\\\\,b\"";
final String code = "one,two,three\n" + "on\\\"e,two\n" + "on\"e,two\n" + "one,\"tw\\\"o\"\n" +
"one,\"t\\,wo\"\n" + "one,two,\"th,ree\"\n" + "\"a\\\\\"\n" + "a\\,b\n" + "\"a\\\\,b\"";
final String[][] res = { { "one", "two", "three" }, { "on\\\"e", "two" }, { "on\"e", "two" },
{ "one", "tw\"o" }, { "one", "t\\,wo" }, // backslash in quotes only escapes a delimiter (",")
{ "one", "two", "th,ree" }, { "a\\\\" }, // backslash in quotes only escapes a delimiter (",")
{ "a\\", "b" }, // a backslash must be returnd
{ "a\\\\,b" } // backslash in quotes only escapes a delimiter (",")
};
final CSVParser parser = CSVParser.parse(code, CSVFormat.DEFAULT);
final List<CSVRecord> records = parser.getRecords();
assertEquals(res.length, records.size());
assertTrue(records.size() > 0);
for (int i = 0; i < res.length; i++) {
assertArrayEquals(res[i], records.get(i).values());
try (final CSVParser parser = CSVParser.parse(code, CSVFormat.DEFAULT)) {
final List<CSVRecord> records = parser.getRecords();
assertEquals(res.length, records.size());
assertTrue(records.size() > 0);
for (int i = 0; i < res.length; i++) {
assertArrayEquals(res[i], records.get(i).values());
}
}
parser.close();
}
@Test
@Ignore("CSV-107")
public void testBOM() throws IOException {
final URL url = ClassLoader.getSystemClassLoader().getResource("CSVFileParser/bom.csv");
final CSVParser parser = CSVParser.parse(url, Charset.forName("UTF-8"), CSVFormat.EXCEL.withHeader());
try {
try (final CSVParser parser = CSVParser.parse(url, Charset.forName("UTF-8"), CSVFormat.EXCEL.withHeader())) {
for (final CSVRecord record : parser) {
final String string = record.get("Date");
Assert.assertNotNull(string);
// System.out.println("date: " + record.get("Date"));
}
} finally {
parser.close();
}
}
@Test
public void testBOMInputStream() throws IOException {
final URL url = ClassLoader.getSystemClassLoader().getResource("CSVFileParser/bom.csv");
final Reader reader = new InputStreamReader(new BOMInputStream(url.openStream()), "UTF-8");
final CSVParser parser = new CSVParser(reader, CSVFormat.EXCEL.withHeader());
try {
try (final Reader reader = new InputStreamReader(new BOMInputStream(url.openStream()), "UTF-8");
final CSVParser parser = new CSVParser(reader, CSVFormat.EXCEL.withHeader())) {
for (final CSVRecord record : parser) {
final String string = record.get("Date");
Assert.assertNotNull(string);
// System.out.println("date: " + record.get("Date"));
}
} finally {
parser.close();
reader.close();
}
}
@Test
public void testCarriageReturnEndings() throws IOException {
final String code = "foo\rbaar,\rhello,world\r,kanu";
final CSVParser parser = CSVParser.parse(code, CSVFormat.DEFAULT);
final List<CSVRecord> records = parser.getRecords();
assertEquals(4, records.size());
parser.close();
try (final CSVParser parser = CSVParser.parse(code, CSVFormat.DEFAULT)) {
final List<CSVRecord> records = parser.getRecords();
assertEquals(4, records.size());
}
}
@Test
public void testCarriageReturnLineFeedEndings() throws IOException {
final String code = "foo\r\nbaar,\r\nhello,world\r\n,kanu";
final CSVParser parser = CSVParser.parse(code, CSVFormat.DEFAULT);
final List<CSVRecord> records = parser.getRecords();
assertEquals(4, records.size());
parser.close();
try (final CSVParser parser = CSVParser.parse(code, CSVFormat.DEFAULT)) {
final List<CSVRecord> records = parser.getRecords();
assertEquals(4, records.size());
}
}
@Test(expected = NoSuchElementException.class)
public void testClose() throws Exception {
final Reader in = new StringReader("# comment\na,b,c\n1,2,3\nx,y,z");
final CSVParser parser = CSVFormat.DEFAULT.withCommentMarker('#').withHeader().parse(in);
final Iterator<CSVRecord> records = parser.iterator();
assertTrue(records.hasNext());
parser.close();
final Iterator<CSVRecord> records;
try (final CSVParser parser = CSVFormat.DEFAULT.withCommentMarker('#').withHeader().parse(in)) {
records = parser.iterator();
assertTrue(records.hasNext());
}
assertFalse(records.hasNext());
records.next();
}
@Test
public void testCSV57() throws Exception {
final CSVParser parser = CSVParser.parse("", CSVFormat.DEFAULT);
final List<CSVRecord> list = parser.getRecords();
assertNotNull(list);
assertEquals(0, list.size());
parser.close();
try (final CSVParser parser = CSVParser.parse("", CSVFormat.DEFAULT)) {
final List<CSVRecord> list = parser.getRecords();
assertNotNull(list);
assertEquals(0, list.size());
}
}
@Test
@ -235,27 +229,26 @@ public class CSVParserTest {
+ "\"\n\",\" \",#\n" // 2)
+ "#,\"\"\n" // 3)
+ "# Final comment\n"// 4)
;
;
final String[][] res = { { "a", "b#" }, { "\n", " ", "#" }, { "#", "" }, { "# Final comment" } };
CSVFormat format = CSVFormat.DEFAULT;
assertFalse(format.isCommentMarkerSet());
CSVParser parser = CSVParser.parse(code, format);
List<CSVRecord> records = parser.getRecords();
assertTrue(records.size() > 0);
Utils.compare("Failed to parse without comments", res, records);
final String[][] res_comments = { { "a", "b#" }, { "\n", " ", "#" }, };
format = CSVFormat.DEFAULT.withCommentMarker('#');
parser.close();
parser = CSVParser.parse(code, format);
records = parser.getRecords();
try (final CSVParser parser = CSVParser.parse(code, format)) {
List<CSVRecord> records = parser.getRecords();
assertTrue(records.size() > 0);
Utils.compare("Failed to parse with comments", res_comments, records);
parser.close();
Utils.compare("Failed to parse without comments", res, records);
format = CSVFormat.DEFAULT.withCommentMarker('#');
}
try (final CSVParser parser = CSVParser.parse(code, format)) {
List<CSVRecord> records = parser.getRecords();
Utils.compare("Failed to parse with comments", res_comments, records);
}
}
@Test(expected = IllegalArgumentException.class)
@ -265,9 +258,9 @@ public class CSVParserTest {
@Test
public void testEmptyFile() throws Exception {
final CSVParser parser = CSVParser.parse("", CSVFormat.DEFAULT);
assertNull(parser.nextRecord());
parser.close();
try (final CSVParser parser = CSVParser.parse("", CSVFormat.DEFAULT)) {
assertNull(parser.nextRecord());
}
}
@Test
@ -276,14 +269,14 @@ public class CSVParserTest {
final String[][] res = { { "hello", "" } // CSV format ignores empty lines
};
for (final String code : codes) {
final CSVParser parser = CSVParser.parse(code, CSVFormat.DEFAULT);
final List<CSVRecord> records = parser.getRecords();
assertEquals(res.length, records.size());
assertTrue(records.size() > 0);
for (int i = 0; i < res.length; i++) {
assertArrayEquals(res[i], records.get(i).values());
try (final CSVParser parser = CSVParser.parse(code, CSVFormat.DEFAULT)) {
final List<CSVRecord> records = parser.getRecords();
assertEquals(res.length, records.size());
assertTrue(records.size() > 0);
for (int i = 0; i < res.length; i++) {
assertArrayEquals(res[i], records.get(i).values());
}
}
parser.close();
}
}
@ -293,14 +286,14 @@ public class CSVParserTest {
final String[][] res = { { "hello", "" }, { "" }, // Excel format does not ignore empty lines
{ "" } };
for (final String code : codes) {
final CSVParser parser = CSVParser.parse(code, CSVFormat.EXCEL);
final List<CSVRecord> records = parser.getRecords();
assertEquals(res.length, records.size());
assertTrue(records.size() > 0);
for (int i = 0; i < res.length; i++) {
assertArrayEquals(res[i], records.get(i).values());
try (final CSVParser parser = CSVParser.parse(code, CSVFormat.EXCEL)) {
final List<CSVRecord> records = parser.getRecords();
assertEquals(res.length, records.size());
assertTrue(records.size() > 0);
for (int i = 0; i < res.length; i++) {
assertArrayEquals(res[i], records.get(i).values());
}
}
parser.close();
}
}
@ -312,14 +305,14 @@ public class CSVParserTest {
final String[][] res = { { "hello", "" }, // CSV format ignores empty lines
{ "world", "" } };
for (final String code : codes) {
final CSVParser parser = CSVParser.parse(code, CSVFormat.DEFAULT);
final List<CSVRecord> records = parser.getRecords();
assertEquals(res.length, records.size());
assertTrue(records.size() > 0);
for (int i = 0; i < res.length; i++) {
assertArrayEquals(res[i], records.get(i).values());
try (final CSVParser parser = CSVParser.parse(code, CSVFormat.DEFAULT)) {
final List<CSVRecord> records = parser.getRecords();
assertEquals(res.length, records.size());
assertTrue(records.size() > 0);
for (int i = 0; i < res.length; i++) {
assertArrayEquals(res[i], records.get(i).values());
}
}
parser.close();
}
}
@ -332,45 +325,45 @@ public class CSVParserTest {
{ "world", "" } };
for (final String code : codes) {
final CSVParser parser = CSVParser.parse(code, CSVFormat.EXCEL);
try (final CSVParser parser = CSVParser.parse(code, CSVFormat.EXCEL)) {
final List<CSVRecord> records = parser.getRecords();
assertEquals(res.length, records.size());
assertTrue(records.size() > 0);
for (int i = 0; i < res.length; i++) {
assertArrayEquals(res[i], records.get(i).values());
}
}
}
}
@Test
public void testExcelFormat1() throws IOException {
final String code = "value1,value2,value3,value4\r\na,b,c,d\r\n x,,," +
"\r\n\r\n\"\"\"hello\"\"\",\" \"\"world\"\"\",\"abc\ndef\",\r\n";
final String[][] res = { { "value1", "value2", "value3", "value4" }, { "a", "b", "c", "d" },
{ " x", "", "", "" }, { "" }, { "\"hello\"", " \"world\"", "abc\ndef", "" } };
try (final CSVParser parser = CSVParser.parse(code, CSVFormat.EXCEL)) {
final List<CSVRecord> records = parser.getRecords();
assertEquals(res.length, records.size());
assertTrue(records.size() > 0);
for (int i = 0; i < res.length; i++) {
assertArrayEquals(res[i], records.get(i).values());
}
parser.close();
}
}
@Test
public void testExcelFormat1() throws IOException {
final String code = "value1,value2,value3,value4\r\na,b,c,d\r\n x,,,"
+ "\r\n\r\n\"\"\"hello\"\"\",\" \"\"world\"\"\",\"abc\ndef\",\r\n";
final String[][] res = { { "value1", "value2", "value3", "value4" }, { "a", "b", "c", "d" },
{ " x", "", "", "" }, { "" }, { "\"hello\"", " \"world\"", "abc\ndef", "" } };
final CSVParser parser = CSVParser.parse(code, CSVFormat.EXCEL);
final List<CSVRecord> records = parser.getRecords();
assertEquals(res.length, records.size());
assertTrue(records.size() > 0);
for (int i = 0; i < res.length; i++) {
assertArrayEquals(res[i], records.get(i).values());
}
parser.close();
}
@Test
public void testExcelFormat2() throws Exception {
final String code = "foo,baar\r\n\r\nhello,\r\n\r\nworld,\r\n";
final String[][] res = { { "foo", "baar" }, { "" }, { "hello", "" }, { "" }, { "world", "" } };
final CSVParser parser = CSVParser.parse(code, CSVFormat.EXCEL);
final List<CSVRecord> records = parser.getRecords();
assertEquals(res.length, records.size());
assertTrue(records.size() > 0);
for (int i = 0; i < res.length; i++) {
assertArrayEquals(res[i], records.get(i).values());
try (final CSVParser parser = CSVParser.parse(code, CSVFormat.EXCEL)) {
final List<CSVRecord> records = parser.getRecords();
assertEquals(res.length, records.size());
assertTrue(records.size() > 0);
for (int i = 0; i < res.length; i++) {
assertArrayEquals(res[i], records.get(i).values());
}
}
parser.close();
}
/**
@ -379,67 +372,63 @@ public class CSVParserTest {
@Test
public void testExcelHeaderCountLessThanData() throws Exception {
final String code = "A,B,C,,\r\na,b,c,d,e\r\n";
final CSVParser parser = CSVParser.parse(code, CSVFormat.EXCEL.withHeader());
try {
try (final CSVParser parser = CSVParser.parse(code, CSVFormat.EXCEL.withHeader())) {
for (final CSVRecord record : parser.getRecords()) {
Assert.assertEquals("a", record.get("A"));
Assert.assertEquals("b", record.get("B"));
Assert.assertEquals("c", record.get("C"));
}
} finally {
parser.close();
}
}
@Test
public void testForEach() throws Exception {
final List<CSVRecord> records = new ArrayList<>();
final Reader in = new StringReader("a,b,c\n1,2,3\nx,y,z");
for (final CSVRecord record : CSVFormat.DEFAULT.parse(in)) {
records.add(record);
try (final Reader in = new StringReader("a,b,c\n1,2,3\nx,y,z")) {
for (final CSVRecord record : CSVFormat.DEFAULT.parse(in)) {
records.add(record);
}
assertEquals(3, records.size());
assertArrayEquals(new String[] { "a", "b", "c" }, records.get(0).values());
assertArrayEquals(new String[] { "1", "2", "3" }, records.get(1).values());
assertArrayEquals(new String[] { "x", "y", "z" }, records.get(2).values());
}
assertEquals(3, records.size());
assertArrayEquals(new String[] { "a", "b", "c" }, records.get(0).values());
assertArrayEquals(new String[] { "1", "2", "3" }, records.get(1).values());
assertArrayEquals(new String[] { "x", "y", "z" }, records.get(2).values());
}
@Test
public void testGetHeaderMap() throws Exception {
final CSVParser parser = CSVParser.parse("a,b,c\n1,2,3\nx,y,z", CSVFormat.DEFAULT.withHeader("A", "B", "C"));
final Map<String, Integer> headerMap = parser.getHeaderMap();
final Iterator<String> columnNames = headerMap.keySet().iterator();
// Headers are iterated in column order.
Assert.assertEquals("A", columnNames.next());
Assert.assertEquals("B", columnNames.next());
Assert.assertEquals("C", columnNames.next());
final Iterator<CSVRecord> records = parser.iterator();
try (final CSVParser parser = CSVParser.parse("a,b,c\n1,2,3\nx,y,z",
CSVFormat.DEFAULT.withHeader("A", "B", "C"))) {
final Map<String, Integer> headerMap = parser.getHeaderMap();
final Iterator<String> columnNames = headerMap.keySet().iterator();
// Headers are iterated in column order.
Assert.assertEquals("A", columnNames.next());
Assert.assertEquals("B", columnNames.next());
Assert.assertEquals("C", columnNames.next());
final Iterator<CSVRecord> records = parser.iterator();
// Parse to make sure getHeaderMap did not have a side-effect.
for (int i = 0; i < 3; i++) {
assertTrue(records.hasNext());
final CSVRecord record = records.next();
assertEquals(record.get(0), record.get("A"));
assertEquals(record.get(1), record.get("B"));
assertEquals(record.get(2), record.get("C"));
// Parse to make sure getHeaderMap did not have a side-effect.
for (int i = 0; i < 3; i++) {
assertTrue(records.hasNext());
final CSVRecord record = records.next();
assertEquals(record.get(0), record.get("A"));
assertEquals(record.get(1), record.get("B"));
assertEquals(record.get(2), record.get("C"));
}
assertFalse(records.hasNext());
}
assertFalse(records.hasNext());
parser.close();
}
@Test
public void testGetLine() throws IOException {
final CSVParser parser = CSVParser.parse(CSV_INPUT, CSVFormat.DEFAULT.withIgnoreSurroundingSpaces());
for (final String[] re : RESULT) {
assertArrayEquals(re, parser.nextRecord().values());
}
try (final CSVParser parser = CSVParser.parse(CSV_INPUT, CSVFormat.DEFAULT.withIgnoreSurroundingSpaces())) {
for (final String[] re : RESULT) {
assertArrayEquals(re, parser.nextRecord().values());
}
assertNull(parser.nextRecord());
parser.close();
assertNull(parser.nextRecord());
}
}
@Test
@ -459,10 +448,10 @@ public class CSVParserTest {
@Test
public void testGetOneLine() throws IOException {
final CSVParser parser = CSVParser.parse(CSV_INPUT_1, CSVFormat.DEFAULT);
final CSVRecord record = parser.getRecords().get(0);
assertArrayEquals(RESULT[0], record.values());
parser.close();
try (final CSVParser parser = CSVParser.parse(CSV_INPUT_1, CSVFormat.DEFAULT)) {
final CSVRecord record = parser.getRecords().get(0);
assertArrayEquals(RESULT[0], record.values());
}
}
/**
@ -472,11 +461,9 @@ public class CSVParserTest {
*/
@Test
public void testGetOneLineOneParser() throws IOException {
final PipedWriter writer = new PipedWriter();
final PipedReader reader = new PipedReader(writer);
final CSVFormat format = CSVFormat.DEFAULT;
final CSVParser parser = new CSVParser(reader, format);
try {
try (final PipedWriter writer = new PipedWriter();
final CSVParser parser = new CSVParser(new PipedReader(writer), format)) {
writer.append(CSV_INPUT_1);
writer.append(format.getRecordSeparator());
final CSVRecord record1 = parser.nextRecord();
@ -485,8 +472,6 @@ public class CSVParserTest {
writer.append(format.getRecordSeparator());
final CSVRecord record2 = parser.nextRecord();
assertArrayEquals(RESULT[1], record2.values());
} finally {
parser.close();
}
}
@ -517,39 +502,40 @@ public class CSVParserTest {
@Test
public void testGetRecords() throws IOException {
final CSVParser parser = CSVParser.parse(CSV_INPUT, CSVFormat.DEFAULT.withIgnoreSurroundingSpaces());
final List<CSVRecord> records = parser.getRecords();
assertEquals(RESULT.length, records.size());
assertTrue(records.size() > 0);
for (int i = 0; i < RESULT.length; i++) {
assertArrayEquals(RESULT[i], records.get(i).values());
try (final CSVParser parser = CSVParser.parse(CSV_INPUT, CSVFormat.DEFAULT.withIgnoreSurroundingSpaces())) {
final List<CSVRecord> records = parser.getRecords();
assertEquals(RESULT.length, records.size());
assertTrue(records.size() > 0);
for (int i = 0; i < RESULT.length; i++) {
assertArrayEquals(RESULT[i], records.get(i).values());
}
}
parser.close();
}
@Test
public void testGetRecordWithMultiLineValues() throws Exception {
final CSVParser parser = CSVParser.parse("\"a\r\n1\",\"a\r\n2\"" + CRLF + "\"b\r\n1\",\"b\r\n2\"" + CRLF +
"\"c\r\n1\",\"c\r\n2\"", CSVFormat.DEFAULT.withRecordSeparator(CRLF));
CSVRecord record;
assertEquals(0, parser.getRecordNumber());
assertEquals(0, parser.getCurrentLineNumber());
assertNotNull(record = parser.nextRecord());
assertEquals(3, parser.getCurrentLineNumber());
assertEquals(1, record.getRecordNumber());
assertEquals(1, parser.getRecordNumber());
assertNotNull(record = parser.nextRecord());
assertEquals(6, parser.getCurrentLineNumber());
assertEquals(2, record.getRecordNumber());
assertEquals(2, parser.getRecordNumber());
assertNotNull(record = parser.nextRecord());
assertEquals(8, parser.getCurrentLineNumber());
assertEquals(3, record.getRecordNumber());
assertEquals(3, parser.getRecordNumber());
assertNull(record = parser.nextRecord());
assertEquals(8, parser.getCurrentLineNumber());
assertEquals(3, parser.getRecordNumber());
parser.close();
try (final CSVParser parser = CSVParser.parse(
"\"a\r\n1\",\"a\r\n2\"" + CRLF + "\"b\r\n1\",\"b\r\n2\"" + CRLF + "\"c\r\n1\",\"c\r\n2\"",
CSVFormat.DEFAULT.withRecordSeparator(CRLF))) {
CSVRecord record;
assertEquals(0, parser.getRecordNumber());
assertEquals(0, parser.getCurrentLineNumber());
assertNotNull(record = parser.nextRecord());
assertEquals(3, parser.getCurrentLineNumber());
assertEquals(1, record.getRecordNumber());
assertEquals(1, parser.getRecordNumber());
assertNotNull(record = parser.nextRecord());
assertEquals(6, parser.getCurrentLineNumber());
assertEquals(2, record.getRecordNumber());
assertEquals(2, parser.getRecordNumber());
assertNotNull(record = parser.nextRecord());
assertEquals(8, parser.getCurrentLineNumber());
assertEquals(3, record.getRecordNumber());
assertEquals(3, parser.getRecordNumber());
assertNull(record = parser.nextRecord());
assertEquals(8, parser.getCurrentLineNumber());
assertEquals(3, parser.getRecordNumber());
}
}
@Test
@ -636,16 +622,18 @@ public class CSVParserTest {
final String code = "\nfoo,baar\n\r\n,\n\n,world\r\n\n";
// String code = "world\r\n\n";
// String code = "foo;baar\r\n\r\nhello;\r\n\r\nworld;\r\n";
final CSVParser parser = CSVParser.parse(code, CSVFormat.DEFAULT);
final List<CSVRecord> records = parser.getRecords();
assertEquals(3, records.size());
parser.close();
try (final CSVParser parser = CSVParser.parse(code, CSVFormat.DEFAULT)) {
final List<CSVRecord> records = parser.getRecords();
assertEquals(3, records.size());
}
}
@Test(expected = IllegalArgumentException.class)
public void testInvalidFormat() throws Exception {
final CSVFormat invalidFormat = CSVFormat.DEFAULT.withDelimiter(CR);
new CSVParser(null, invalidFormat).close();
try (final CSVParser parser = new CSVParser(null, invalidFormat)) {
Assert.fail("This test should have thrown an exception.");
}
}
@Test
@ -680,17 +668,17 @@ public class CSVParserTest {
@Test
public void testLineFeedEndings() throws IOException {
final String code = "foo\nbaar,\nhello,world\n,kanu";
final CSVParser parser = CSVParser.parse(code, CSVFormat.DEFAULT);
final List<CSVRecord> records = parser.getRecords();
assertEquals(4, records.size());
parser.close();
try (final CSVParser parser = CSVParser.parse(code, CSVFormat.DEFAULT)) {
final List<CSVRecord> records = parser.getRecords();
assertEquals(4, records.size());
}
}
@Test
public void testMappedButNotSetAsOutlook2007ContactExport() throws Exception {
final Reader in = new StringReader("a,b,c\n1,2\nx,y,z");
final Iterator<CSVRecord> records = CSVFormat.DEFAULT.withHeader("A", "B", "C").withSkipHeaderRecord()
.parse(in).iterator();
final Iterator<CSVRecord> records = CSVFormat.DEFAULT.withHeader("A", "B", "C").withSkipHeaderRecord().parse(in)
.iterator();
CSVRecord record;
// 1st record
@ -724,38 +712,41 @@ public class CSVParserTest {
@Test
// TODO this may lead to strange behavior, throw an exception if iterator() has already been called?
public void testMultipleIterators() throws Exception {
final CSVParser parser = CSVParser.parse("a,b,c" + CR + "d,e,f", CSVFormat.DEFAULT);
try (final CSVParser parser = CSVParser.parse("a,b,c" + CR + "d,e,f", CSVFormat.DEFAULT)) {
final Iterator<CSVRecord> itr1 = parser.iterator();
final Iterator<CSVRecord> itr2 = parser.iterator();
final Iterator<CSVRecord> itr1 = parser.iterator();
final Iterator<CSVRecord> itr2 = parser.iterator();
final CSVRecord first = itr1.next();
assertEquals("a", first.get(0));
assertEquals("b", first.get(1));
assertEquals("c", first.get(2));
final CSVRecord first = itr1.next();
assertEquals("a", first.get(0));
assertEquals("b", first.get(1));
assertEquals("c", first.get(2));
final CSVRecord second = itr2.next();
assertEquals("d", second.get(0));
assertEquals("e", second.get(1));
assertEquals("f", second.get(2));
parser.close();
final CSVRecord second = itr2.next();
assertEquals("d", second.get(0));
assertEquals("e", second.get(1));
assertEquals("f", second.get(2));
}
}
@Test(expected = IllegalArgumentException.class)
public void testNewCSVParserNullReaderFormat() throws Exception {
new CSVParser(null, CSVFormat.DEFAULT).close();
try (final CSVParser parser = new CSVParser(null, CSVFormat.DEFAULT)) {
Assert.fail("This test should have thrown an exception.");
}
}
@Test(expected = IllegalArgumentException.class)
public void testNewCSVParserReaderNullFormat() throws Exception {
new CSVParser(new StringReader(""), null).close();
try (final CSVParser parser = new CSVParser(new StringReader(""), null)) {
Assert.fail("This test should have thrown an exception.");
}
}
@Test
public void testNoHeaderMap() throws Exception {
final CSVParser parser = CSVParser.parse("a,b,c\n1,2,3\nx,y,z", CSVFormat.DEFAULT);
Assert.assertNull(parser.getHeaderMap());
parser.close();
try (final CSVParser parser = CSVParser.parse("a,b,c\n1,2,3\nx,y,z", CSVFormat.DEFAULT)) {
Assert.assertNull(parser.getHeaderMap());
}
}
@Test(expected = IllegalArgumentException.class)
@ -780,8 +771,9 @@ public class CSVParserTest {
@Test(expected = IllegalArgumentException.class)
public void testParserUrlNullCharsetFormat() throws Exception {
final CSVParser parser = CSVParser.parse(new URL("http://commons.apache.org"), null, CSVFormat.DEFAULT);
parser.close();
try (final CSVParser parser = CSVParser.parse(new URL("http://commons.apache.org"), null, CSVFormat.DEFAULT)) {
Assert.fail("This test should have thrown an exception.");
}
}
@Test(expected = IllegalArgumentException.class)
@ -791,8 +783,9 @@ public class CSVParserTest {
@Test(expected = IllegalArgumentException.class)
public void testParseUrlCharsetNullFormat() throws Exception {
final CSVParser parser = CSVParser.parse(new URL("http://commons.apache.org"), Charset.defaultCharset(), null);
parser.close();
try (final CSVParser parser = CSVParser.parse(new URL("http://commons.apache.org"), Charset.defaultCharset(), null)) {
Assert.fail("This test should have thrown an exception.");
}
}
@Test
@ -840,13 +833,13 @@ public class CSVParserTest {
@Test
public void testRoundtrip() throws Exception {
final StringWriter out = new StringWriter();
final CSVPrinter printer = new CSVPrinter(out, CSVFormat.DEFAULT);
final String input = "a,b,c\r\n1,2,3\r\nx,y,z\r\n";
for (final CSVRecord record : CSVParser.parse(input, CSVFormat.DEFAULT)) {
printer.printRecord(record);
try (final CSVPrinter printer = new CSVPrinter(out, CSVFormat.DEFAULT)) {
final String input = "a,b,c\r\n1,2,3\r\nx,y,z\r\n";
for (final CSVRecord record : CSVParser.parse(input, CSVFormat.DEFAULT)) {
printer.printRecord(record);
}
assertEquals(input, out.toString());
}
assertEquals(input, out.toString());
printer.close();
}
@Test
@ -858,12 +851,12 @@ public class CSVParserTest {
assertEquals("2", record.get("b"));
assertEquals("3", record.get("c"));
}
@Test
public void testSkipHeaderOverrideDuplicateHeaders() throws Exception {
final Reader in = new StringReader("a,a,a\n1,2,3\nx,y,z");
final Iterator<CSVRecord> records = CSVFormat.DEFAULT.withHeader("X", "Y", "Z").withSkipHeaderRecord()
.parse(in).iterator();
final Iterator<CSVRecord> records = CSVFormat.DEFAULT.withHeader("X", "Y", "Z").withSkipHeaderRecord().parse(in)
.iterator();
final CSVRecord record = records.next();
assertEquals("1", record.get("X"));
assertEquals("2", record.get("Y"));
@ -873,8 +866,8 @@ public class CSVParserTest {
@Test
public void testSkipSetAltHeaders() throws Exception {
final Reader in = new StringReader("a,b,c\n1,2,3\nx,y,z");
final Iterator<CSVRecord> records = CSVFormat.DEFAULT.withHeader("X", "Y", "Z").withSkipHeaderRecord()
.parse(in).iterator();
final Iterator<CSVRecord> records = CSVFormat.DEFAULT.withHeader("X", "Y", "Z").withSkipHeaderRecord().parse(in)
.iterator();
final CSVRecord record = records.next();
assertEquals("1", record.get("X"));
assertEquals("2", record.get("Y"));
@ -884,8 +877,8 @@ public class CSVParserTest {
@Test
public void testSkipSetHeader() throws Exception {
final Reader in = new StringReader("a,b,c\n1,2,3\nx,y,z");
final Iterator<CSVRecord> records = CSVFormat.DEFAULT.withHeader("a", "b", "c").withSkipHeaderRecord()
.parse(in).iterator();
final Iterator<CSVRecord> records = CSVFormat.DEFAULT.withHeader("a", "b", "c").withSkipHeaderRecord().parse(in)
.iterator();
final CSVRecord record = records.next();
assertEquals("1", record.get("a"));
assertEquals("2", record.get("b"));
@ -895,27 +888,27 @@ public class CSVParserTest {
@Test
@Ignore
public void testStartWithEmptyLinesThenHeaders() throws Exception {
final String[] codes = {"\r\n\r\n\r\nhello,\r\n\r\n\r\n", "hello,\n\n\n", "hello,\"\"\r\n\r\n\r\n",
"hello,\"\"\n\n\n"};
final String[][] res = {{"hello", ""}, {""}, // Excel format does not ignore empty lines
{""}};
final String[] codes = { "\r\n\r\n\r\nhello,\r\n\r\n\r\n", "hello,\n\n\n", "hello,\"\"\r\n\r\n\r\n",
"hello,\"\"\n\n\n" };
final String[][] res = { { "hello", "" }, { "" }, // Excel format does not ignore empty lines
{ "" } };
for (final String code : codes) {
final CSVParser parser = CSVParser.parse(code, CSVFormat.EXCEL);
final List<CSVRecord> records = parser.getRecords();
assertEquals(res.length, records.size());
assertTrue(records.size() > 0);
for (int i = 0; i < res.length; i++) {
assertArrayEquals(res[i], records.get(i).values());
try (final CSVParser parser = CSVParser.parse(code, CSVFormat.EXCEL)) {
final List<CSVRecord> records = parser.getRecords();
assertEquals(res.length, records.size());
assertTrue(records.size() > 0);
for (int i = 0; i < res.length; i++) {
assertArrayEquals(res[i], records.get(i).values());
}
}
parser.close();
}
}
@Test
public void testTrailingDelimiter() throws Exception {
final Reader in = new StringReader("a,a,a,\n\"1\",\"2\",\"3\",\nx,y,z,");
final Iterator<CSVRecord> records = CSVFormat.DEFAULT.withHeader("X", "Y", "Z").withSkipHeaderRecord().withTrailingDelimiter()
.parse(in).iterator();
final Iterator<CSVRecord> records = CSVFormat.DEFAULT.withHeader("X", "Y", "Z").withSkipHeaderRecord()
.withTrailingDelimiter().parse(in).iterator();
final CSVRecord record = records.next();
assertEquals("1", record.get("X"));
assertEquals("2", record.get("Y"));
@ -926,8 +919,8 @@ public class CSVParserTest {
@Test
public void testTrim() throws Exception {
final Reader in = new StringReader("a,a,a\n\" 1 \",\" 2 \",\" 3 \"\nx,y,z");
final Iterator<CSVRecord> records = CSVFormat.DEFAULT.withHeader("X", "Y", "Z").withSkipHeaderRecord().withTrim()
.parse(in).iterator();
final Iterator<CSVRecord> records = CSVFormat.DEFAULT.withHeader("X", "Y", "Z").withSkipHeaderRecord()
.withTrim().parse(in).iterator();
final CSVRecord record = records.next();
assertEquals("1", record.get("X"));
assertEquals("2", record.get("Y"));
@ -936,46 +929,46 @@ public class CSVParserTest {
}
private void validateLineNumbers(final String lineSeparator) throws IOException {
final CSVParser parser = CSVParser.parse("a" + lineSeparator + "b" + lineSeparator + "c",
CSVFormat.DEFAULT.withRecordSeparator(lineSeparator));
assertEquals(0, parser.getCurrentLineNumber());
assertNotNull(parser.nextRecord());
assertEquals(1, parser.getCurrentLineNumber());
assertNotNull(parser.nextRecord());
assertEquals(2, parser.getCurrentLineNumber());
assertNotNull(parser.nextRecord());
// Still 2 because the last line is does not have EOL chars
assertEquals(2, parser.getCurrentLineNumber());
assertNull(parser.nextRecord());
// Still 2 because the last line is does not have EOL chars
assertEquals(2, parser.getCurrentLineNumber());
parser.close();
try (final CSVParser parser = CSVParser.parse("a" + lineSeparator + "b" + lineSeparator + "c",
CSVFormat.DEFAULT.withRecordSeparator(lineSeparator))) {
assertEquals(0, parser.getCurrentLineNumber());
assertNotNull(parser.nextRecord());
assertEquals(1, parser.getCurrentLineNumber());
assertNotNull(parser.nextRecord());
assertEquals(2, parser.getCurrentLineNumber());
assertNotNull(parser.nextRecord());
// Still 2 because the last line is does not have EOL chars
assertEquals(2, parser.getCurrentLineNumber());
assertNull(parser.nextRecord());
// Still 2 because the last line is does not have EOL chars
assertEquals(2, parser.getCurrentLineNumber());
}
}
private void validateRecordNumbers(final String lineSeparator) throws IOException {
final CSVParser parser = CSVParser.parse("a" + lineSeparator + "b" + lineSeparator + "c",
CSVFormat.DEFAULT.withRecordSeparator(lineSeparator));
CSVRecord record;
assertEquals(0, parser.getRecordNumber());
assertNotNull(record = parser.nextRecord());
assertEquals(1, record.getRecordNumber());
assertEquals(1, parser.getRecordNumber());
assertNotNull(record = parser.nextRecord());
assertEquals(2, record.getRecordNumber());
assertEquals(2, parser.getRecordNumber());
assertNotNull(record = parser.nextRecord());
assertEquals(3, record.getRecordNumber());
assertEquals(3, parser.getRecordNumber());
assertNull(record = parser.nextRecord());
assertEquals(3, parser.getRecordNumber());
parser.close();
try (final CSVParser parser = CSVParser.parse("a" + lineSeparator + "b" + lineSeparator + "c",
CSVFormat.DEFAULT.withRecordSeparator(lineSeparator))) {
CSVRecord record;
assertEquals(0, parser.getRecordNumber());
assertNotNull(record = parser.nextRecord());
assertEquals(1, record.getRecordNumber());
assertEquals(1, parser.getRecordNumber());
assertNotNull(record = parser.nextRecord());
assertEquals(2, record.getRecordNumber());
assertEquals(2, parser.getRecordNumber());
assertNotNull(record = parser.nextRecord());
assertEquals(3, record.getRecordNumber());
assertEquals(3, parser.getRecordNumber());
assertNull(record = parser.nextRecord());
assertEquals(3, parser.getRecordNumber());
}
}
private void validateRecordPosition(final String lineSeparator) throws IOException {
final String nl = lineSeparator; // used as linebreak in values for better distinction
final String code = "a,b,c" + lineSeparator + "1,2,3" + lineSeparator +
// to see if recordPosition correctly points to the enclosing quote
// to see if recordPosition correctly points to the enclosing quote
"'A" + nl + "A','B" + nl + "B',CC" + lineSeparator +
// unicode test... not very relevant while operating on strings instead of bytes, but for
// completeness...

635
src/test/java/org/apache/commons/csv/CSVPrinterTest.java
View File

File diff suppressed because it is too large Load Diff

38
src/test/java/org/apache/commons/csv/CSVRecordTest.java
View File

@ -143,16 +143,16 @@ public class CSVRecordTest {
@Test
public void testRemoveAndAddColumns() throws IOException {
// do:
final CSVPrinter printer = new CSVPrinter(new StringBuilder(), CSVFormat.DEFAULT);
final Map<String, String> map = recordWithHeader.toMap();
map.remove("OldColumn");
map.put("ZColumn", "NewValue");
// check:
final ArrayList<String> list = new ArrayList<>(map.values());
Collections.sort(list);
printer.printRecord(list);
Assert.assertEquals("A,B,C,NewValue" + CSVFormat.DEFAULT.getRecordSeparator(), printer.getOut().toString());
printer.close();
try (final CSVPrinter printer = new CSVPrinter(new StringBuilder(), CSVFormat.DEFAULT)) {
final Map<String, String> map = recordWithHeader.toMap();
map.remove("OldColumn");
map.put("ZColumn", "NewValue");
// check:
final ArrayList<String> list = new ArrayList<>(map.values());
Collections.sort(list);
printer.printRecord(list);
Assert.assertEquals("A,B,C,NewValue" + CSVFormat.DEFAULT.getRecordSeparator(), printer.getOut().toString());
}
}
@Test
@ -163,18 +163,20 @@ public class CSVRecordTest {
@Test
public void testToMapWithShortRecord() throws Exception {
final CSVParser parser = CSVParser.parse("a,b", CSVFormat.DEFAULT.withHeader("A", "B", "C"));
final CSVRecord shortRec = parser.iterator().next();
shortRec.toMap();
try (final CSVParser parser = CSVParser.parse("a,b", CSVFormat.DEFAULT.withHeader("A", "B", "C"))) {
final CSVRecord shortRec = parser.iterator().next();
shortRec.toMap();
}
}
@Test
public void testToMapWithNoHeader() throws Exception {
final CSVParser parser = CSVParser.parse("a,b", CSVFormat.newFormat(','));
final CSVRecord shortRec = parser.iterator().next();
final Map<String, String> map = shortRec.toMap();
assertNotNull("Map is not null.", map);
assertTrue("Map is empty.", map.isEmpty());
try (final CSVParser parser = CSVParser.parse("a,b", CSVFormat.newFormat(','))) {
final CSVRecord shortRec = parser.iterator().next();
final Map<String, String> map = shortRec.toMap();
assertNotNull("Map is not null.", map);
assertTrue("Map is empty.", map.isEmpty());
}
}
private void validateMap(final Map<String, String> map, final boolean allowsNulls) {

273
src/test/java/org/apache/commons/csv/ExtendedBufferedReaderTest.java
View File

@ -36,72 +36,72 @@ public class ExtendedBufferedReaderTest {
@Test
public void testEmptyInput() throws Exception {
final ExtendedBufferedReader br = getBufferedReader("");
assertEquals(END_OF_STREAM, br.read());
assertEquals(END_OF_STREAM, br.lookAhead());
assertEquals(END_OF_STREAM, br.getLastChar());
assertNull(br.readLine());
assertEquals(0, br.read(new char[10], 0, 0));
br.close();
try (final ExtendedBufferedReader br = createBufferedReader("")) {
assertEquals(END_OF_STREAM, br.read());
assertEquals(END_OF_STREAM, br.lookAhead());
assertEquals(END_OF_STREAM, br.getLastChar());
assertNull(br.readLine());
assertEquals(0, br.read(new char[10], 0, 0));
}
}
@Test
public void testReadLookahead1() throws Exception {
final ExtendedBufferedReader br = getBufferedReader("1\n2\r3\n");
assertEquals(0, br.getCurrentLineNumber());
assertEquals('1', br.lookAhead());
assertEquals(UNDEFINED, br.getLastChar());
assertEquals(0, br.getCurrentLineNumber());
assertEquals('1', br.read()); // Start line 1
assertEquals('1', br.getLastChar());
try (final ExtendedBufferedReader br = createBufferedReader("1\n2\r3\n")) {
assertEquals(0, br.getCurrentLineNumber());
assertEquals('1', br.lookAhead());
assertEquals(UNDEFINED, br.getLastChar());
assertEquals(0, br.getCurrentLineNumber());
assertEquals('1', br.read()); // Start line 1
assertEquals('1', br.getLastChar());
assertEquals(1, br.getCurrentLineNumber());
assertEquals('\n', br.lookAhead());
assertEquals(1, br.getCurrentLineNumber());
assertEquals('1', br.getLastChar());
assertEquals('\n', br.read());
assertEquals(1, br.getCurrentLineNumber());
assertEquals('\n', br.getLastChar());
assertEquals(1, br.getCurrentLineNumber());
assertEquals(1, br.getCurrentLineNumber());
assertEquals('\n', br.lookAhead());
assertEquals(1, br.getCurrentLineNumber());
assertEquals('1', br.getLastChar());
assertEquals('\n', br.read());
assertEquals(1, br.getCurrentLineNumber());
assertEquals('\n', br.getLastChar());
assertEquals(1, br.getCurrentLineNumber());
assertEquals('2', br.lookAhead());
assertEquals(1, br.getCurrentLineNumber());
assertEquals('\n', br.getLastChar());
assertEquals(1, br.getCurrentLineNumber());
assertEquals('2', br.read()); // Start line 2
assertEquals(2, br.getCurrentLineNumber());
assertEquals('2', br.getLastChar());
assertEquals('2', br.lookAhead());
assertEquals(1, br.getCurrentLineNumber());
assertEquals('\n', br.getLastChar());
assertEquals(1, br.getCurrentLineNumber());
assertEquals('2', br.read()); // Start line 2
assertEquals(2, br.getCurrentLineNumber());
assertEquals('2', br.getLastChar());
assertEquals('\r', br.lookAhead());
assertEquals(2, br.getCurrentLineNumber());
assertEquals('2', br.getLastChar());
assertEquals('\r', br.read());
assertEquals('\r', br.getLastChar());
assertEquals(2, br.getCurrentLineNumber());
assertEquals('\r', br.lookAhead());
assertEquals(2, br.getCurrentLineNumber());
assertEquals('2', br.getLastChar());
assertEquals('\r', br.read());
assertEquals('\r', br.getLastChar());
assertEquals(2, br.getCurrentLineNumber());
assertEquals('3', br.lookAhead());
assertEquals('\r', br.getLastChar());
assertEquals('3', br.read()); // Start line 3
assertEquals('3', br.getLastChar());
assertEquals(3, br.getCurrentLineNumber());
assertEquals('3', br.lookAhead());
assertEquals('\r', br.getLastChar());
assertEquals('3', br.read()); // Start line 3
assertEquals('3', br.getLastChar());
assertEquals(3, br.getCurrentLineNumber());
assertEquals('\n', br.lookAhead());
assertEquals(3, br.getCurrentLineNumber());
assertEquals('3', br.getLastChar());
assertEquals('\n', br.read());
assertEquals(3, br.getCurrentLineNumber());
assertEquals('\n', br.getLastChar());
assertEquals(3, br.getCurrentLineNumber());
assertEquals('\n', br.lookAhead());
assertEquals(3, br.getCurrentLineNumber());
assertEquals('3', br.getLastChar());
assertEquals('\n', br.read());
assertEquals(3, br.getCurrentLineNumber());
assertEquals('\n', br.getLastChar());
assertEquals(3, br.getCurrentLineNumber());
assertEquals(END_OF_STREAM, br.lookAhead());
assertEquals('\n', br.getLastChar());
assertEquals(END_OF_STREAM, br.read());
assertEquals(END_OF_STREAM, br.getLastChar());
assertEquals(END_OF_STREAM, br.read());
assertEquals(END_OF_STREAM, br.lookAhead());
assertEquals(3, br.getCurrentLineNumber());
assertEquals(END_OF_STREAM, br.lookAhead());
assertEquals('\n', br.getLastChar());
assertEquals(END_OF_STREAM, br.read());
assertEquals(END_OF_STREAM, br.getLastChar());
assertEquals(END_OF_STREAM, br.read());
assertEquals(END_OF_STREAM, br.lookAhead());
assertEquals(3, br.getCurrentLineNumber());
br.close();
}
}
@Test
@ -109,109 +109,104 @@ public class ExtendedBufferedReaderTest {
final char[] ref = new char[5];
final char[] res = new char[5];
final ExtendedBufferedReader br = getBufferedReader("abcdefg");
ref[0] = 'a';
ref[1] = 'b';
ref[2] = 'c';
assertEquals(3, br.read(res, 0, 3));
assertArrayEquals(ref, res);
assertEquals('c', br.getLastChar());
try (final ExtendedBufferedReader br = createBufferedReader("abcdefg")) {
ref[0] = 'a';
ref[1] = 'b';
ref[2] = 'c';
assertEquals(3, br.read(res, 0, 3));
assertArrayEquals(ref, res);
assertEquals('c', br.getLastChar());
assertEquals('d', br.lookAhead());
ref[4] = 'd';
assertEquals(1, br.read(res, 4, 1));
assertArrayEquals(ref, res);
assertEquals('d', br.getLastChar());
br.close();
assertEquals('d', br.lookAhead());
ref[4] = 'd';
assertEquals(1, br.read(res, 4, 1));
assertArrayEquals(ref, res);
assertEquals('d', br.getLastChar());
}
}
@Test
public void testReadLine() throws Exception {
ExtendedBufferedReader br = getBufferedReader("");
assertNull(br.readLine());
br.close();
br = getBufferedReader("\n");
assertEquals("",br.readLine());
assertNull(br.readLine());
br.close();
br = getBufferedReader("foo\n\nhello");
assertEquals(0, br.getCurrentLineNumber());
assertEquals("foo",br.readLine());
assertEquals(1, br.getCurrentLineNumber());
assertEquals("",br.readLine());
assertEquals(2, br.getCurrentLineNumber());
assertEquals("hello",br.readLine());
assertEquals(3, br.getCurrentLineNumber());
assertNull(br.readLine());
assertEquals(3, br.getCurrentLineNumber());
br.close();
br = getBufferedReader("foo\n\nhello");
assertEquals('f', br.read());
assertEquals('o', br.lookAhead());
assertEquals("oo",br.readLine());
assertEquals(1, br.getCurrentLineNumber());
assertEquals('\n', br.lookAhead());
assertEquals("",br.readLine());
assertEquals(2, br.getCurrentLineNumber());
assertEquals('h', br.lookAhead());
assertEquals("hello",br.readLine());
assertNull(br.readLine());
assertEquals(3, br.getCurrentLineNumber());
br.close();
br = getBufferedReader("foo\rbaar\r\nfoo");
assertEquals("foo",br.readLine());
assertEquals('b', br.lookAhead());
assertEquals("baar",br.readLine());
assertEquals('f', br.lookAhead());
assertEquals("foo",br.readLine());
assertNull(br.readLine());
br.close();
try (final ExtendedBufferedReader br = createBufferedReader("")) {
assertNull(br.readLine());
}
try (final ExtendedBufferedReader br = createBufferedReader("\n")) {
assertEquals("", br.readLine());
assertNull(br.readLine());
}
try (final ExtendedBufferedReader br = createBufferedReader("foo\n\nhello")) {
assertEquals(0, br.getCurrentLineNumber());
assertEquals("foo", br.readLine());
assertEquals(1, br.getCurrentLineNumber());
assertEquals("", br.readLine());
assertEquals(2, br.getCurrentLineNumber());
assertEquals("hello", br.readLine());
assertEquals(3, br.getCurrentLineNumber());
assertNull(br.readLine());
assertEquals(3, br.getCurrentLineNumber());
}
try (final ExtendedBufferedReader br = createBufferedReader("foo\n\nhello")) {
assertEquals('f', br.read());
assertEquals('o', br.lookAhead());
assertEquals("oo", br.readLine());
assertEquals(1, br.getCurrentLineNumber());
assertEquals('\n', br.lookAhead());
assertEquals("", br.readLine());
assertEquals(2, br.getCurrentLineNumber());
assertEquals('h', br.lookAhead());
assertEquals("hello", br.readLine());
assertNull(br.readLine());
assertEquals(3, br.getCurrentLineNumber());
}
try (final ExtendedBufferedReader br = createBufferedReader("foo\rbaar\r\nfoo")) {
assertEquals("foo", br.readLine());
assertEquals('b', br.lookAhead());
assertEquals("baar", br.readLine());
assertEquals('f', br.lookAhead());
assertEquals("foo", br.readLine());
assertNull(br.readLine());
}
}
/*
* Test to illustrate https://issues.apache.org/jira/browse/CSV-75
* Test to illustrate https://issues.apache.org/jira/browse/CSV-75
*
*/
@Test
public void testReadChar() throws Exception {
final String LF="\n"; final String CR="\r"; final String CRLF=CR+LF; final String LFCR=LF+CR;// easier to read the string below
final String test="a" + LF + "b" + CR + "c" + LF + LF + "d" + CR + CR + "e" + LFCR + "f "+ CRLF;
// EOL eol EOL EOL eol eol EOL+CR EOL
final String LF = "\n";
final String CR = "\r";
final String CRLF = CR + LF;
final String LFCR = LF + CR;// easier to read the string below
final String test = "a" + LF + "b" + CR + "c" + LF + LF + "d" + CR + CR + "e" + LFCR + "f " + CRLF;
// EOL eol EOL EOL eol eol EOL+CR EOL
final int EOLeolct = 9;
ExtendedBufferedReader br;
br = getBufferedReader(test);
assertEquals(0, br.getCurrentLineNumber());
while (br.readLine() != null) {
// consume all
try (final ExtendedBufferedReader br = createBufferedReader(test)) {
assertEquals(0, br.getCurrentLineNumber());
while (br.readLine() != null) {
// consume all
}
assertEquals(EOLeolct, br.getCurrentLineNumber());
}
assertEquals(EOLeolct, br.getCurrentLineNumber());
br.close();
br = getBufferedReader(test);
assertEquals(0, br.getCurrentLineNumber());
while (br.read() != -1) {
// consume all
try (final ExtendedBufferedReader br = createBufferedReader(test)) {
assertEquals(0, br.getCurrentLineNumber());
while (br.read() != -1) {
// consume all
}
assertEquals(EOLeolct, br.getCurrentLineNumber());
}
assertEquals(EOLeolct, br.getCurrentLineNumber());
br.close();
br = getBufferedReader(test);
assertEquals(0, br.getCurrentLineNumber());
final char[] buff = new char[10];
while (br.read(buff, 0, 3) != -1) {
// consume all
try (final ExtendedBufferedReader br = createBufferedReader(test)) {
assertEquals(0, br.getCurrentLineNumber());
final char[] buff = new char[10];
while (br.read(buff, 0, 3) != -1) {
// consume all
}
assertEquals(EOLeolct, br.getCurrentLineNumber());
}
assertEquals(EOLeolct, br.getCurrentLineNumber());
br.close();
}
private ExtendedBufferedReader getBufferedReader(final String s) {
private ExtendedBufferedReader createBufferedReader(final String s) {
return new ExtendedBufferedReader(new StringReader(s));
}
}

412
src/test/java/org/apache/commons/csv/LexerTest.java
View File

@ -52,345 +52,341 @@ public class LexerTest {
formatWithEscaping = CSVFormat.DEFAULT.withEscape('\\');
}
private Lexer getLexer(final String input, final CSVFormat format) {
private Lexer createLexer(final String input, final CSVFormat format) {
return new Lexer(format, new ExtendedBufferedReader(new StringReader(input)));
}
@Test
public void testSurroundingSpacesAreDeleted() throws IOException {
final String code = "noSpaces, leadingSpaces,trailingSpaces , surroundingSpaces , ,,";
final Lexer parser = getLexer(code, CSVFormat.DEFAULT.withIgnoreSurroundingSpaces());
assertThat(parser.nextToken(new Token()), matches(TOKEN, "noSpaces"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "leadingSpaces"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "trailingSpaces"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "surroundingSpaces"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, ""));
assertThat(parser.nextToken(new Token()), matches(TOKEN, ""));
assertThat(parser.nextToken(new Token()), matches(EOF, ""));
try (final Lexer parser = createLexer(code, CSVFormat.DEFAULT.withIgnoreSurroundingSpaces())) {
assertThat(parser.nextToken(new Token()), matches(TOKEN, "noSpaces"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "leadingSpaces"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "trailingSpaces"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "surroundingSpaces"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, ""));
assertThat(parser.nextToken(new Token()), matches(TOKEN, ""));
assertThat(parser.nextToken(new Token()), matches(EOF, ""));
}
}
@Test
public void testSurroundingTabsAreDeleted() throws IOException {
final String code = "noTabs,\tleadingTab,trailingTab\t,\tsurroundingTabs\t,\t\t,,";
final Lexer parser = getLexer(code, CSVFormat.DEFAULT.withIgnoreSurroundingSpaces());
assertThat(parser.nextToken(new Token()), matches(TOKEN, "noTabs"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "leadingTab"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "trailingTab"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "surroundingTabs"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, ""));
assertThat(parser.nextToken(new Token()), matches(TOKEN, ""));
assertThat(parser.nextToken(new Token()), matches(EOF, ""));
try (final Lexer parser = createLexer(code, CSVFormat.DEFAULT.withIgnoreSurroundingSpaces())) {
assertThat(parser.nextToken(new Token()), matches(TOKEN, "noTabs"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "leadingTab"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "trailingTab"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "surroundingTabs"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, ""));
assertThat(parser.nextToken(new Token()), matches(TOKEN, ""));
assertThat(parser.nextToken(new Token()), matches(EOF, ""));
}
}
@Test
public void testIgnoreEmptyLines() throws IOException {
final String code =
"first,line,\n"+
"\n"+
"\n"+
"second,line\n"+
"\n"+
"\n"+
"third line \n"+
"\n"+
"\n"+
"last, line \n"+
"\n"+
"\n"+
"\n";
final String code = "first,line,\n" + "\n" + "\n" + "second,line\n" + "\n" + "\n" + "third line \n" + "\n" +
"\n" + "last, line \n" + "\n" + "\n" + "\n";
final CSVFormat format = CSVFormat.DEFAULT.withIgnoreEmptyLines();
final Lexer parser = getLexer(code, format);
assertThat(parser.nextToken(new Token()), matches(TOKEN, "first"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "line"));
assertThat(parser.nextToken(new Token()), matches(EORECORD, ""));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "second"));
assertThat(parser.nextToken(new Token()), matches(EORECORD, "line"));
assertThat(parser.nextToken(new Token()), matches(EORECORD, "third line "));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "last"));
assertThat(parser.nextToken(new Token()), matches(EORECORD, " line "));
assertThat(parser.nextToken(new Token()), matches(EOF, ""));
assertThat(parser.nextToken(new Token()), matches(EOF, ""));
try (final Lexer parser = createLexer(code, format)) {
assertThat(parser.nextToken(new Token()), matches(TOKEN, "first"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "line"));
assertThat(parser.nextToken(new Token()), matches(EORECORD, ""));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "second"));
assertThat(parser.nextToken(new Token()), matches(EORECORD, "line"));
assertThat(parser.nextToken(new Token()), matches(EORECORD, "third line "));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "last"));
assertThat(parser.nextToken(new Token()), matches(EORECORD, " line "));
assertThat(parser.nextToken(new Token()), matches(EOF, ""));
assertThat(parser.nextToken(new Token()), matches(EOF, ""));
}
}
@Test
public void testComments() throws IOException {
final String code =
"first,line,\n"+
"second,line,tokenWith#no-comment\n"+
"# comment line \n"+
"third,line,#no-comment\n"+
"# penultimate comment\n"+
"# Final comment\n";
final String code = "first,line,\n" + "second,line,tokenWith#no-comment\n" + "# comment line \n" +
"third,line,#no-comment\n" + "# penultimate comment\n" + "# Final comment\n";
final CSVFormat format = CSVFormat.DEFAULT.withCommentMarker('#');
final Lexer parser = getLexer(code, format);
assertThat(parser.nextToken(new Token()), matches(TOKEN, "first"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "line"));
assertThat(parser.nextToken(new Token()), matches(EORECORD, ""));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "second"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "line"));
assertThat(parser.nextToken(new Token()), matches(EORECORD, "tokenWith#no-comment"));
assertThat(parser.nextToken(new Token()), matches(COMMENT, "comment line"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "third"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "line"));
assertThat(parser.nextToken(new Token()), matches(EORECORD, "#no-comment"));
assertThat(parser.nextToken(new Token()), matches(COMMENT, "penultimate comment"));
assertThat(parser.nextToken(new Token()), matches(COMMENT, "Final comment"));
assertThat(parser.nextToken(new Token()), matches(EOF, ""));
assertThat(parser.nextToken(new Token()), matches(EOF, ""));
try (final Lexer parser = createLexer(code, format)) {
assertThat(parser.nextToken(new Token()), matches(TOKEN, "first"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "line"));
assertThat(parser.nextToken(new Token()), matches(EORECORD, ""));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "second"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "line"));
assertThat(parser.nextToken(new Token()), matches(EORECORD, "tokenWith#no-comment"));
assertThat(parser.nextToken(new Token()), matches(COMMENT, "comment line"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "third"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "line"));
assertThat(parser.nextToken(new Token()), matches(EORECORD, "#no-comment"));
assertThat(parser.nextToken(new Token()), matches(COMMENT, "penultimate comment"));
assertThat(parser.nextToken(new Token()), matches(COMMENT, "Final comment"));
assertThat(parser.nextToken(new Token()), matches(EOF, ""));
assertThat(parser.nextToken(new Token()), matches(EOF, ""));
}
}
@Test
public void testCommentsAndEmptyLines() throws IOException {
final String code =
"1,2,3,\n"+ // 1
"\n"+ // 1b
"\n"+ // 1c
"a,b x,c#no-comment\n"+ // 2
"#foo\n"+ // 3
"\n"+ // 4
"\n"+ // 4b
"d,e,#no-comment\n"+ // 5
"\n"+ // 5b
"\n"+ // 5c
"# penultimate comment\n"+ // 6
"\n"+ // 6b
"\n"+ // 6c
"# Final comment\n"; // 7
final String code = "1,2,3,\n" + // 1
"\n" + // 1b
"\n" + // 1c
"a,b x,c#no-comment\n" + // 2
"#foo\n" + // 3
"\n" + // 4
"\n" + // 4b
"d,e,#no-comment\n" + // 5
"\n" + // 5b
"\n" + // 5c
"# penultimate comment\n" + // 6
"\n" + // 6b
"\n" + // 6c
"# Final comment\n"; // 7
final CSVFormat format = CSVFormat.DEFAULT.withCommentMarker('#').withIgnoreEmptyLines(false);
assertFalse("Should not ignore empty lines", format.getIgnoreEmptyLines());
final Lexer parser = getLexer(code, format);
assertThat(parser.nextToken(new Token()), matches(TOKEN, "1"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "2"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "3"));
assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); // 1
assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); // 1b
assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); // 1c
assertThat(parser.nextToken(new Token()), matches(TOKEN, "a"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "b x"));
assertThat(parser.nextToken(new Token()), matches(EORECORD, "c#no-comment")); // 2
assertThat(parser.nextToken(new Token()), matches(COMMENT, "foo")); // 3
assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); // 4
assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); // 4b
assertThat(parser.nextToken(new Token()), matches(TOKEN, "d"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "e"));
assertThat(parser.nextToken(new Token()), matches(EORECORD, "#no-comment")); // 5
assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); // 5b
assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); // 5c
assertThat(parser.nextToken(new Token()), matches(COMMENT, "penultimate comment")); // 6
assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); // 6b
assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); // 6c
assertThat(parser.nextToken(new Token()), matches(COMMENT, "Final comment")); // 7
assertThat(parser.nextToken(new Token()), matches(EOF, ""));
assertThat(parser.nextToken(new Token()), matches(EOF, ""));
try (final Lexer parser = createLexer(code, format)) {
assertThat(parser.nextToken(new Token()), matches(TOKEN, "1"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "2"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "3"));
assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); // 1
assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); // 1b
assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); // 1c
assertThat(parser.nextToken(new Token()), matches(TOKEN, "a"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "b x"));
assertThat(parser.nextToken(new Token()), matches(EORECORD, "c#no-comment")); // 2
assertThat(parser.nextToken(new Token()), matches(COMMENT, "foo")); // 3
assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); // 4
assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); // 4b
assertThat(parser.nextToken(new Token()), matches(TOKEN, "d"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "e"));
assertThat(parser.nextToken(new Token()), matches(EORECORD, "#no-comment")); // 5
assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); // 5b
assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); // 5c
assertThat(parser.nextToken(new Token()), matches(COMMENT, "penultimate comment")); // 6
assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); // 6b
assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); // 6c
assertThat(parser.nextToken(new Token()), matches(COMMENT, "Final comment")); // 7
assertThat(parser.nextToken(new Token()), matches(EOF, ""));
assertThat(parser.nextToken(new Token()), matches(EOF, ""));
}
}
// simple token with escaping not enabled
@Test
public void testBackslashWithoutEscaping() throws IOException {
/* file: a,\,,b
* \,,
*/
/*
* file: a,\,,b \,,
*/
final String code = "a,\\,,b\\\n\\,,";
final CSVFormat format = CSVFormat.DEFAULT;
assertFalse(format.isEscapeCharacterSet());
final Lexer parser = getLexer(code, format);
assertThat(parser.nextToken(new Token()), matches(TOKEN, "a"));
// an unquoted single backslash is not an escape char
assertThat(parser.nextToken(new Token()), matches(TOKEN, "\\"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, ""));
assertThat(parser.nextToken(new Token()), matches(EORECORD, "b\\"));
// an unquoted single backslash is not an escape char
assertThat(parser.nextToken(new Token()), matches(TOKEN, "\\"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, ""));
assertThat(parser.nextToken(new Token()), matches(EOF, ""));
try (final Lexer parser = createLexer(code, format)) {
assertThat(parser.nextToken(new Token()), matches(TOKEN, "a"));
// an unquoted single backslash is not an escape char
assertThat(parser.nextToken(new Token()), matches(TOKEN, "\\"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, ""));
assertThat(parser.nextToken(new Token()), matches(EORECORD, "b\\"));
// an unquoted single backslash is not an escape char
assertThat(parser.nextToken(new Token()), matches(TOKEN, "\\"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, ""));
assertThat(parser.nextToken(new Token()), matches(EOF, ""));
}
}
// simple token with escaping enabled
@Test
public void testBackslashWithEscaping() throws IOException {
/* file: a,\,,b
* \,,
*/
/*
* file: a,\,,b \,,
*/
final String code = "a,\\,,b\\\\\n\\,,\\\nc,d\\\r\ne";
final CSVFormat format = formatWithEscaping.withIgnoreEmptyLines(false);
assertTrue(format.isEscapeCharacterSet());
final Lexer parser = getLexer(code, format);
assertThat(parser.nextToken(new Token()), matches(TOKEN, "a"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, ","));
assertThat(parser.nextToken(new Token()), matches(EORECORD, "b\\"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, ","));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "\nc"));
assertThat(parser.nextToken(new Token()), matches(EORECORD, "d\r"));
assertThat(parser.nextToken(new Token()), matches(EOF, "e"));
try (final Lexer parser = createLexer(code, format)) {
assertThat(parser.nextToken(new Token()), matches(TOKEN, "a"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, ","));
assertThat(parser.nextToken(new Token()), matches(EORECORD, "b\\"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, ","));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "\nc"));
assertThat(parser.nextToken(new Token()), matches(EORECORD, "d\r"));
assertThat(parser.nextToken(new Token()), matches(EOF, "e"));
}
}
// encapsulator tokenizer (single line)
@Test
public void testNextToken4() throws IOException {
/* file: a,"foo",b
* a, " foo",b
* a,"foo " ,b // whitespace after closing encapsulator
* a, " foo " ,b
*/
/*
* file: a,"foo",b a, " foo",b a,"foo " ,b // whitespace after closing encapsulator a, " foo " ,b
*/
final String code = "a,\"foo\",b\na, \" foo\",b\na,\"foo \" ,b\na, \" foo \" ,b";
final Lexer parser = getLexer(code, CSVFormat.DEFAULT.withIgnoreSurroundingSpaces());
assertThat(parser.nextToken(new Token()), matches(TOKEN, "a"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "foo"));
assertThat(parser.nextToken(new Token()), matches(EORECORD, "b"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "a"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, " foo"));
assertThat(parser.nextToken(new Token()), matches(EORECORD, "b"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "a"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "foo "));
assertThat(parser.nextToken(new Token()), matches(EORECORD, "b"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "a"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, " foo "));
// assertTokenEquals(EORECORD, "b", parser.nextToken(new Token()));
assertThat(parser.nextToken(new Token()), matches(EOF, "b"));
try (final Lexer parser = createLexer(code, CSVFormat.DEFAULT.withIgnoreSurroundingSpaces())) {
assertThat(parser.nextToken(new Token()), matches(TOKEN, "a"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "foo"));
assertThat(parser.nextToken(new Token()), matches(EORECORD, "b"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "a"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, " foo"));
assertThat(parser.nextToken(new Token()), matches(EORECORD, "b"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "a"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "foo "));
assertThat(parser.nextToken(new Token()), matches(EORECORD, "b"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "a"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, " foo "));
// assertTokenEquals(EORECORD, "b", parser.nextToken(new Token()));
assertThat(parser.nextToken(new Token()), matches(EOF, "b"));
}
}
// encapsulator tokenizer (multi line, delimiter in string)
@Test
public void testNextToken5() throws IOException {
final String code = "a,\"foo\n\",b\n\"foo\n baar ,,,\"\n\"\n\t \n\"";
final Lexer parser = getLexer(code, CSVFormat.DEFAULT);
assertThat(parser.nextToken(new Token()), matches(TOKEN, "a"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "foo\n"));
assertThat(parser.nextToken(new Token()), matches(EORECORD, "b"));
assertThat(parser.nextToken(new Token()), matches(EORECORD, "foo\n baar ,,,"));
assertThat(parser.nextToken(new Token()), matches(EOF, "\n\t \n"));
try (final Lexer parser = createLexer(code, CSVFormat.DEFAULT)) {
assertThat(parser.nextToken(new Token()), matches(TOKEN, "a"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "foo\n"));
assertThat(parser.nextToken(new Token()), matches(EORECORD, "b"));
assertThat(parser.nextToken(new Token()), matches(EORECORD, "foo\n baar ,,,"));
assertThat(parser.nextToken(new Token()), matches(EOF, "\n\t \n"));
}
}
// change delimiters, comment, encapsulater
@Test
public void testNextToken6() throws IOException {
/* file: a;'b and \' more
* '
* !comment;;;;
* ;;
*/
/*
* file: a;'b and \' more ' !comment;;;; ;;
*/
final String code = "a;'b and '' more\n'\n!comment;;;;\n;;";
final CSVFormat format = CSVFormat.DEFAULT.withQuote('\'').withCommentMarker('!').withDelimiter(';');
final Lexer parser = getLexer(code, format);
assertThat(parser.nextToken(new Token()), matches(TOKEN, "a"));
assertThat(parser.nextToken(new Token()), matches(EORECORD, "b and ' more\n"));
try (final Lexer parser = createLexer(code, format)) {
assertThat(parser.nextToken(new Token()), matches(TOKEN, "a"));
assertThat(parser.nextToken(new Token()), matches(EORECORD, "b and ' more\n"));
}
}
// From CSV-1
@Test
public void testDelimiterIsWhitespace() throws IOException {
final String code = "one\ttwo\t\tfour \t five\t six";
final Lexer parser = getLexer(code, CSVFormat.TDF);
assertThat(parser.nextToken(new Token()), matches(TOKEN, "one"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "two"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, ""));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "four"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "five"));
assertThat(parser.nextToken(new Token()), matches(EOF, "six"));
try (final Lexer parser = createLexer(code, CSVFormat.TDF)) {
assertThat(parser.nextToken(new Token()), matches(TOKEN, "one"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "two"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, ""));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "four"));
assertThat(parser.nextToken(new Token()), matches(TOKEN, "five"));
assertThat(parser.nextToken(new Token()), matches(EOF, "six"));
}
}
@Test
public void testEscapedCR() throws Exception {
final Lexer lexer = getLexer("character\\" + CR + "Escaped", formatWithEscaping);
assertThat(lexer.nextToken(new Token()), hasContent("character" + CR + "Escaped"));
try (final Lexer lexer = createLexer("character\\" + CR + "Escaped", formatWithEscaping)) {
assertThat(lexer.nextToken(new Token()), hasContent("character" + CR + "Escaped"));
}
}
@Test
public void testCR() throws Exception {
final Lexer lexer = getLexer("character" + CR + "NotEscaped", formatWithEscaping);
assertThat(lexer.nextToken(new Token()), hasContent("character"));
assertThat(lexer.nextToken(new Token()), hasContent("NotEscaped"));
try (final Lexer lexer = createLexer("character" + CR + "NotEscaped", formatWithEscaping)) {
assertThat(lexer.nextToken(new Token()), hasContent("character"));
assertThat(lexer.nextToken(new Token()), hasContent("NotEscaped"));
}
}
@Test
public void testEscapedLF() throws Exception {
final Lexer lexer = getLexer("character\\" + LF + "Escaped", formatWithEscaping);
assertThat(lexer.nextToken(new Token()), hasContent("character" + LF + "Escaped"));
try (final Lexer lexer = createLexer("character\\" + LF + "Escaped", formatWithEscaping)) {
assertThat(lexer.nextToken(new Token()), hasContent("character" + LF + "Escaped"));
}
}
@Test
public void testLF() throws Exception {
final Lexer lexer = getLexer("character" + LF + "NotEscaped", formatWithEscaping);
assertThat(lexer.nextToken(new Token()), hasContent("character"));
assertThat(lexer.nextToken(new Token()), hasContent("NotEscaped"));
try (final Lexer lexer = createLexer("character" + LF + "NotEscaped", formatWithEscaping)) {
assertThat(lexer.nextToken(new Token()), hasContent("character"));
assertThat(lexer.nextToken(new Token()), hasContent("NotEscaped"));
}
}
@Test // TODO is this correct? Do we expect <esc>TAB to be unescaped?
public void testEscapedTab() throws Exception {
final Lexer lexer = getLexer("character\\" + TAB + "Escaped", formatWithEscaping);
assertThat(lexer.nextToken(new Token()), hasContent("character" + TAB + "Escaped"));
try (final Lexer lexer = createLexer("character\\" + TAB + "Escaped", formatWithEscaping)) {
assertThat(lexer.nextToken(new Token()), hasContent("character" + TAB + "Escaped"));
}
}
@Test
public void testTab() throws Exception {
final Lexer lexer = getLexer("character" + TAB + "NotEscaped", formatWithEscaping);
assertThat(lexer.nextToken(new Token()), hasContent("character" + TAB + "NotEscaped"));
try (final Lexer lexer = createLexer("character" + TAB + "NotEscaped", formatWithEscaping)) {
assertThat(lexer.nextToken(new Token()), hasContent("character" + TAB + "NotEscaped"));
}
}
@Test // TODO is this correct? Do we expect <esc>BACKSPACE to be unescaped?
public void testEscapedBackspace() throws Exception {
final Lexer lexer = getLexer("character\\" + BACKSPACE + "Escaped", formatWithEscaping);
assertThat(lexer.nextToken(new Token()), hasContent("character" + BACKSPACE + "Escaped"));
try (final Lexer lexer = createLexer("character\\" + BACKSPACE + "Escaped", formatWithEscaping)) {
assertThat(lexer.nextToken(new Token()), hasContent("character" + BACKSPACE + "Escaped"));
}
}
@Test
public void testBackspace() throws Exception {
final Lexer lexer = getLexer("character" + BACKSPACE + "NotEscaped", formatWithEscaping);
assertThat(lexer.nextToken(new Token()), hasContent("character" + BACKSPACE + "NotEscaped"));
try (final Lexer lexer = createLexer("character" + BACKSPACE + "NotEscaped", formatWithEscaping)) {
assertThat(lexer.nextToken(new Token()), hasContent("character" + BACKSPACE + "NotEscaped"));
}
}
@Test // TODO is this correct? Do we expect <esc>FF to be unescaped?
public void testEscapedFF() throws Exception {
final Lexer lexer = getLexer("character\\" + FF + "Escaped", formatWithEscaping);
assertThat(lexer.nextToken(new Token()), hasContent("character" + FF + "Escaped"));
try (final Lexer lexer = createLexer("character\\" + FF + "Escaped", formatWithEscaping)) {
assertThat(lexer.nextToken(new Token()), hasContent("character" + FF + "Escaped"));
}
}
@Test
public void testFF() throws Exception {
final Lexer lexer = getLexer("character" + FF + "NotEscaped", formatWithEscaping);
assertThat(lexer.nextToken(new Token()), hasContent("character" + FF + "NotEscaped"));
try (final Lexer lexer = createLexer("character" + FF + "NotEscaped", formatWithEscaping)) {
assertThat(lexer.nextToken(new Token()), hasContent("character" + FF + "NotEscaped"));
}
}
@Test
public void testEscapedMySqlNullValue() throws Exception {
// MySQL uses \N to symbolize null values. We have to restore this
final Lexer lexer = getLexer("character\\NEscaped", formatWithEscaping);
assertThat(lexer.nextToken(new Token()), hasContent("character\\NEscaped"));
try (final Lexer lexer = createLexer("character\\NEscaped", formatWithEscaping)) {
assertThat(lexer.nextToken(new Token()), hasContent("character\\NEscaped"));
}
}
@Test
public void testEscapedCharacter() throws Exception {
final Lexer lexer = getLexer("character\\aEscaped", formatWithEscaping);
assertThat(lexer.nextToken(new Token()), hasContent("character\\aEscaped"));
try (final Lexer lexer = createLexer("character\\aEscaped", formatWithEscaping)) {
assertThat(lexer.nextToken(new Token()), hasContent("character\\aEscaped"));
}
}
@Test
public void testEscapedControlCharacter() throws Exception {
// we are explicitly using an escape different from \ here
final Lexer lexer = getLexer("character!rEscaped", CSVFormat.DEFAULT.withEscape('!'));
assertThat(lexer.nextToken(new Token()), hasContent("character" + CR + "Escaped"));
try (final Lexer lexer = createLexer("character!rEscaped", CSVFormat.DEFAULT.withEscape('!'))) {
assertThat(lexer.nextToken(new Token()), hasContent("character" + CR + "Escaped"));
}
}
@Test
public void testEscapedControlCharacter2() throws Exception {
final Lexer lexer = getLexer("character\\rEscaped", CSVFormat.DEFAULT.withEscape('\\'));
assertThat(lexer.nextToken(new Token()), hasContent("character" + CR + "Escaped"));
try (final Lexer lexer = createLexer("character\\rEscaped", CSVFormat.DEFAULT.withEscape('\\'))) {
assertThat(lexer.nextToken(new Token()), hasContent("character" + CR + "Escaped"));
}
}
@Test(expected = IOException.class)
public void testEscapingAtEOF() throws Exception {
final String code = "escaping at EOF is evil\\";
final Lexer lexer = getLexer(code, formatWithEscaping);
lexer.nextToken(new Token());
try (final Lexer lexer = createLexer(code, formatWithEscaping)) {
lexer.nextToken(new Token());
}
}
}

230
src/test/java/org/apache/commons/csv/PerformanceTest.java
View File

@ -26,6 +26,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.util.zip.GZIPInputStream;
import org.apache.commons.io.IOUtils;
@ -73,11 +74,11 @@ public class PerformanceTest {
System.out.println(String.format("Found test fixture %s: %,d bytes.", BIG_FILE, BIG_FILE.length()));
} else {
System.out.println("Decompressing test fixture " + BIG_FILE + "...");
final InputStream input = new GZIPInputStream(new FileInputStream("src/test/resources/perf/worldcitiespop.txt.gz"));
final OutputStream output = new FileOutputStream(BIG_FILE);
IOUtils.copy(input, output);
input.close();
output.close();
try (final InputStream input = new GZIPInputStream(
new FileInputStream("src/test/resources/perf/worldcitiespop.txt.gz"));
final OutputStream output = new FileOutputStream(BIG_FILE)) {
IOUtils.copy(input, output);
}
System.out.println(String.format("Decompressed test fixture %s: %,d bytes.", BIG_FILE, BIG_FILE.length()));
}
final int argc = args.length;
@ -121,7 +122,7 @@ public class PerformanceTest {
}
}
private static BufferedReader getReader() throws IOException {
private static BufferedReader createReader() throws IOException {
return new BufferedReader(new FileReader(BIG_FILE));
}
@ -155,15 +156,17 @@ public class PerformanceTest {
}
private static void testReadBigFile(final boolean split) throws Exception {
for (int i = 0; i < max; i++) {
final BufferedReader in = getReader();
final long t0 = System.currentTimeMillis();
final Stats s = readAll(in, split);
in.close();
show(split?"file+split":"file", s, t0);
}
show();
}
for (int i = 0; i < max; i++) {
final long startMillis;
final Stats stats;
try (final BufferedReader in = createReader()) {
startMillis = System.currentTimeMillis();
stats = readAll(in, split);
}
show(split ? "file+split" : "file", stats, startMillis);
}
show();
}
private static Stats readAll(final BufferedReader in, final boolean split) throws IOException {
int count = 0;
@ -176,55 +179,58 @@ public class PerformanceTest {
return new Stats(count, fields);
}
private static void testExtendedBuffer(final boolean makeString) throws Exception {
for (int i = 0; i < max; i++) {
final ExtendedBufferedReader in = new ExtendedBufferedReader(getReader());
final long t0 = System.currentTimeMillis();
int read;
int fields = 0;
int lines = 0;
if (makeString) {
StringBuilder sb = new StringBuilder();
while((read=in.read()) != -1) {
sb.append((char)read);
if (read == ',') { // count delimiters
sb.toString();
sb = new StringBuilder();
fields++;
} else if (read == '\n') {
sb.toString();
sb = new StringBuilder();
lines++;
}
}
} else {
while((read=in.read()) != -1) {
if (read == ',') { // count delimiters
fields++;
} else if (read == '\n') {
lines++;
}
}
}
fields += lines; // EOL is a delimiter too
in.close();
show("Extended"+(makeString?" toString":""), new Stats(lines, fields), t0);
}
show();
}
private static void testExtendedBuffer(final boolean makeString) throws Exception {
for (int i = 0; i < max; i++) {
int fields = 0;
int lines = 0;
final long startMillis;
try (final ExtendedBufferedReader in = new ExtendedBufferedReader(createReader())) {
startMillis = System.currentTimeMillis();
int read;
if (makeString) {
StringBuilder sb = new StringBuilder();
while ((read = in.read()) != -1) {
sb.append((char) read);
if (read == ',') { // count delimiters
sb.toString();
sb = new StringBuilder();
fields++;
} else if (read == '\n') {
sb.toString();
sb = new StringBuilder();
lines++;
}
}
} else {
while ((read = in.read()) != -1) {
if (read == ',') { // count delimiters
fields++;
} else if (read == '\n') {
lines++;
}
}
}
fields += lines; // EOL is a delimiter too
}
show("Extended" + (makeString ? " toString" : ""), new Stats(lines, fields), startMillis);
}
show();
}
private static void testParseCommonsCSV() throws Exception {
for (int i = 0; i < max; i++) {
final BufferedReader reader = getReader();
final CSVParser parser = new CSVParser(reader, format);
final long t0 = System.currentTimeMillis();
final Stats s = iterate(parser);
reader.close();
show("CSV", s, t0);
parser.close();
}
show();
}
private static void testParseCommonsCSV() throws Exception {
for (int i = 0; i < max; i++) {
final long startMillis;
final Stats stats;
try (final BufferedReader reader = createReader()) {
try (final CSVParser parser = new CSVParser(reader, format)) {
startMillis = System.currentTimeMillis();
stats = iterate(parser);
}
show("CSV", stats, startMillis);
}
}
show();
}
private static Constructor<Lexer> getLexerCtor(final String clazz) throws Exception {
@ -233,53 +239,59 @@ public class PerformanceTest {
return lexer.getConstructor(new Class<?>[]{CSVFormat.class, ExtendedBufferedReader.class});
}
private static void testCSVLexer(final boolean newToken, final String test) throws Exception {
Token token = new Token();
String dynamic = "";
for (int i = 0; i < max; i++) {
final ExtendedBufferedReader input = new ExtendedBufferedReader(getReader());
Lexer lexer = null;
if (test.startsWith("CSVLexer")) {
dynamic="!";
lexer = getLexerCtor(test).newInstance(new Object[]{format, input});
} else {
lexer = new Lexer(format, input);
}
int count = 0;
int fields = 0;
final long t0 = System.currentTimeMillis();
do {
if (newToken) {
token = new Token();
} else {
token.reset();
}
lexer.nextToken(token);
switch(token.type) {
case EOF:
break;
case EORECORD:
fields++;
count++;
break;
case INVALID:
throw new IOException("invalid parse sequence <"+token.content.toString()+">");
case TOKEN:
fields++;
break;
case COMMENT: // not really expecting these
break;
default:
throw new IllegalStateException("Unexpected Token type: " + token.type);
}
private static void testCSVLexer(final boolean newToken, final String test) throws Exception {
Token token = new Token();
String dynamic = "";
for (int i = 0; i < max; i++) {
final String simpleName;
final Stats stats;
final long startMillis;
try (final ExtendedBufferedReader input = new ExtendedBufferedReader(createReader());
Lexer lexer = createTestCSVLexer(test, input)) {
if (test.startsWith("CSVLexer")) {
dynamic = "!";
}
simpleName = lexer.getClass().getSimpleName();
int count = 0;
int fields = 0;
startMillis = System.currentTimeMillis();
do {
if (newToken) {
token = new Token();
} else {
token.reset();
}
lexer.nextToken(token);
switch (token.type) {
case EOF:
break;
case EORECORD:
fields++;
count++;
break;
case INVALID:
throw new IOException("invalid parse sequence <" + token.content.toString() + ">");
case TOKEN:
fields++;
break;
case COMMENT: // not really expecting these
break;
default:
throw new IllegalStateException("Unexpected Token type: " + token.type);
}
} while (!token.type.equals(Token.Type.EOF));
stats = new Stats(count, fields);
}
show(simpleName + dynamic + " " + (newToken ? "new" : "reset"), stats, startMillis);
}
show();
}
} while (!token.type.equals(Token.Type.EOF));
final Stats s = new Stats(count, fields);
input.close();
show(lexer.getClass().getSimpleName()+dynamic+" "+(newToken ? "new" : "reset"), s, t0);
}
show();
}
private static Lexer createTestCSVLexer(final String test, final ExtendedBufferedReader input)
throws InstantiationException, IllegalAccessException, InvocationTargetException, Exception {
return test.startsWith("CSVLexer") ? getLexerCtor(test)
.newInstance(new Object[] { format, input }) : new Lexer(format, input);
}
private static Stats iterate(final Iterable<CSVRecord> it) {
int count = 0;

22
src/test/java/org/apache/commons/csv/bugs/JiraCsv164Test.java
View File

@ -29,12 +29,13 @@ public class JiraCsv164Test {
@Test
public void testJiraCsv154_withCommentMarker() throws IOException {
final String comment = "This is a header comment";
final CSVFormat format = CSVFormat.EXCEL.withHeader("H1", "H2").withCommentMarker('#').withHeaderComments(comment);
final CSVFormat format = CSVFormat.EXCEL.withHeader("H1", "H2").withCommentMarker('#')
.withHeaderComments(comment);
final StringBuilder out = new StringBuilder();
final CSVPrinter printer = format.print(out);
printer.print("A");
printer.print("B");
printer.close();
try (final CSVPrinter printer = format.print(out)) {
printer.print("A");
printer.print("B");
}
final String s = out.toString();
assertTrue(s, s.contains(comment));
}
@ -42,12 +43,13 @@ public class JiraCsv164Test {
@Test
public void testJiraCsv154_withHeaderComments() throws IOException {
final String comment = "This is a header comment";
final CSVFormat format = CSVFormat.EXCEL.withHeader("H1", "H2").withHeaderComments(comment).withCommentMarker('#');
final CSVFormat format = CSVFormat.EXCEL.withHeader("H1", "H2").withHeaderComments(comment)
.withCommentMarker('#');
final StringBuilder out = new StringBuilder();
final CSVPrinter printer = format.print(out);
printer.print("A");
printer.print("B");
printer.close();
try (final CSVPrinter printer = format.print(out)) {
printer.print("A");
printer.print("B");
}
final String s = out.toString();
assertTrue(s, s.contains(comment));
}

35
src/test/java/org/apache/commons/csv/bugs/JiraCsv167Test.java
View File

@ -33,23 +33,23 @@ public class JiraCsv167Test {
@Test
public void parse() throws IOException {
final BufferedReader br = new BufferedReader(getTestInput());
String s = null;
int totcomment = 0;
int totrecs = 0;
boolean lastWasComment = false;
while((s=br.readLine()) != null) {
if (s.startsWith("#")) {
if (!lastWasComment) { // comments are merged
totcomment++;
try (final BufferedReader br = new BufferedReader(getTestInput())) {
String s = null;
boolean lastWasComment = false;
while ((s = br.readLine()) != null) {
if (s.startsWith("#")) {
if (!lastWasComment) { // comments are merged
totcomment++;
}
lastWasComment = true;
} else {
totrecs++;
lastWasComment = false;
}
lastWasComment = true;
} else {
totrecs++;
lastWasComment = false;
}
}
br.close();
CSVFormat format = CSVFormat.DEFAULT;
//
format = format.withAllowMissingColumnNames(false);
@ -66,13 +66,14 @@ public class JiraCsv167Test {
format = format.withRecordSeparator('\n');
format = format.withSkipHeaderRecord(false);
//
final CSVParser parser = format.parse(getTestInput());
int comments = 0;
int records = 0;
for (final CSVRecord csvRecord : parser) {
records++;
if (csvRecord.hasComment()) {
comments++;
try (final CSVParser parser = format.parse(getTestInput())) {
for (final CSVRecord csvRecord : parser) {
records++;
if (csvRecord.hasComment()) {
comments++;
}
}
}
// Comment lines are concatenated, in this example 4 lines become 2 comments.

27
src/test/java/org/apache/commons/csv/perf/PerformanceTest.java
View File

@ -56,15 +56,15 @@ public class PerformanceTest {
return;
}
System.out.println("Decompressing test fixture " + BIG_FILE + "...");
final InputStream input = new GZIPInputStream(new FileInputStream("src/test/resources/perf/worldcitiespop.txt.gz"));
final OutputStream output = new FileOutputStream(BIG_FILE);
IOUtils.copy(input, output);
System.out.println(String.format("Decompressed test fixture %s: %,d bytes.", BIG_FILE, BIG_FILE.length()));
input.close();
output.close();
try (final InputStream input = new GZIPInputStream(
new FileInputStream("src/test/resources/perf/worldcitiespop.txt.gz"));
final OutputStream output = new FileOutputStream(BIG_FILE)) {
IOUtils.copy(input, output);
System.out.println(String.format("Decompressed test fixture %s: %,d bytes.", BIG_FILE, BIG_FILE.length()));
}
}
private BufferedReader getBufferedReader() throws IOException {
private BufferedReader createBufferedReader() throws IOException {
return new BufferedReader(new FileReader(BIG_FILE));
}
@ -96,7 +96,7 @@ public class PerformanceTest {
public long testParseBigFile(final boolean traverseColumns) throws Exception {
final long startMillis = System.currentTimeMillis();
final long count = this.parse(this.getBufferedReader(), traverseColumns);
final long count = this.parse(this.createBufferedReader(), traverseColumns);
final long totalMillis = System.currentTimeMillis() - startMillis;
this.println(String.format("File parsed in %,d milliseconds with Commons CSV: %,d lines.", totalMillis, count));
return totalMillis;
@ -115,13 +115,12 @@ public class PerformanceTest {
public void testReadBigFile() throws Exception {
long bestTime = Long.MAX_VALUE;
for (int i = 0; i < this.max; i++) {
final BufferedReader in = this.getBufferedReader();
final long startMillis = System.currentTimeMillis();
long count = 0;
try {
final long startMillis;
long count;
try (final BufferedReader in = this.createBufferedReader()) {
startMillis = System.currentTimeMillis();
count = 0;
count = this.readAll(in);
} finally {
in.close();
}
final long totalMillis = System.currentTimeMillis() - startMillis;
bestTime = Math.min(totalMillis, bestTime);