Apache
/
commons-csv
mirror of https://github.com/apache/commons-csv.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Move Token into separate file for more flexibility 

git-svn-id: https://svn.apache.org/repos/asf/commons/proper/csv/trunk@1303505 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Sebastian Bazley 2012-03-21 19:04:32 +00:00
parent 0519364a83
commit 3ea8118ff3
3 changed files with 3 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
src/main/java/org/apache/commons/csv/CSVLexer.java
View File

@ -19,13 +19,10 @@ package org.apache.commons.csv;
import java.io.IOException; import java.io.IOException;
import static org.apache.commons.csv.CSVLexer.Token.Type.*; import static org.apache.commons.csv.Token.Type.*;
class CSVLexer { class CSVLexer {
/** length of the initial token (content-)buffer */
private static final int INITIAL_TOKEN_LENGTH = 50;
private final StringBuilder wsBuf = new StringBuilder(); private final StringBuilder wsBuf = new StringBuilder();
private final CSVFormat format; private final CSVFormat format;
@ -33,44 +30,6 @@ class CSVLexer {
/** The input stream */ /** The input stream */
private final ExtendedBufferedReader in; private final ExtendedBufferedReader in;
/**
* Token is an internal token representation.
* <p/>
* It is used as contract between the lexer and the parser.
*/
static class Token {
enum Type {
/** Token has no valid content, i.e. is in its initialized state. */
INVALID,
/** Token with content, at beginning or in the middle of a line. */
TOKEN,
/** Token (which can have content) when end of file is reached. */
EOF,
/** Token with content when end of a line is reached. */
EORECORD
}
/** Token type */
Type type = INVALID;
/** The content buffer. */
StringBuilder content = new StringBuilder(INITIAL_TOKEN_LENGTH);
/** Token ready flag: indicates a valid token with content (ready for the parser). */
boolean isReady;
Token reset() {
content.setLength(0);
type = INVALID;
isReady = false;
return this;
}
}
CSVLexer(CSVFormat format, ExtendedBufferedReader in) { CSVLexer(CSVFormat format, ExtendedBufferedReader in) {
this.format = format; this.format = format;
this.in = in; this.in = in;

3
src/main/java/org/apache/commons/csv/CSVParser.java
View File

@ -27,9 +27,8 @@ import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.NoSuchElementException; import java.util.NoSuchElementException;
import org.apache.commons.csv.CSVLexer.Token;
import static org.apache.commons.csv.CSVLexer.Token.Type.*; import static org.apache.commons.csv.Token.Type.*;
/** /**
* Parses CSV files according to the specified configuration. * Parses CSV files according to the specified configuration.

3
src/test/java/org/apache/commons/csv/CSVLexerTest.java
View File

@ -20,10 +20,9 @@ package org.apache.commons.csv;
import java.io.IOException; import java.io.IOException;
import java.io.StringReader; import java.io.StringReader;
import org.apache.commons.csv.CSVLexer.Token;
import org.junit.Test; import org.junit.Test;
import static org.apache.commons.csv.CSVLexer.Token.Type.*; import static org.apache.commons.csv.Token.Type.*;
import static org.junit.Assert.*; import static org.junit.Assert.*;
public class CSVLexerTest { public class CSVLexerTest {