Move Token into separate file for more flexibility
git-svn-id: https://svn.apache.org/repos/asf/commons/proper/csv/trunk@1303505 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
0519364a83
commit
3ea8118ff3
|
@ -19,13 +19,10 @@ package org.apache.commons.csv;
|
|||
|
||||
import java.io.IOException;
|
||||
|
||||
import static org.apache.commons.csv.CSVLexer.Token.Type.*;
|
||||
import static org.apache.commons.csv.Token.Type.*;
|
||||
|
||||
class CSVLexer {
|
||||
|
||||
/** length of the initial token (content-)buffer */
|
||||
private static final int INITIAL_TOKEN_LENGTH = 50;
|
||||
|
||||
private final StringBuilder wsBuf = new StringBuilder();
|
||||
|
||||
private final CSVFormat format;
|
||||
|
@ -33,44 +30,6 @@ class CSVLexer {
|
|||
/** The input stream */
|
||||
private final ExtendedBufferedReader in;
|
||||
|
||||
/**
|
||||
* Token is an internal token representation.
|
||||
* <p/>
|
||||
* It is used as contract between the lexer and the parser.
|
||||
*/
|
||||
static class Token {
|
||||
|
||||
enum Type {
|
||||
/** Token has no valid content, i.e. is in its initialized state. */
|
||||
INVALID,
|
||||
|
||||
/** Token with content, at beginning or in the middle of a line. */
|
||||
TOKEN,
|
||||
|
||||
/** Token (which can have content) when end of file is reached. */
|
||||
EOF,
|
||||
|
||||
/** Token with content when end of a line is reached. */
|
||||
EORECORD
|
||||
}
|
||||
|
||||
/** Token type */
|
||||
Type type = INVALID;
|
||||
|
||||
/** The content buffer. */
|
||||
StringBuilder content = new StringBuilder(INITIAL_TOKEN_LENGTH);
|
||||
|
||||
/** Token ready flag: indicates a valid token with content (ready for the parser). */
|
||||
boolean isReady;
|
||||
|
||||
Token reset() {
|
||||
content.setLength(0);
|
||||
type = INVALID;
|
||||
isReady = false;
|
||||
return this;
|
||||
}
|
||||
}
|
||||
|
||||
CSVLexer(CSVFormat format, ExtendedBufferedReader in) {
|
||||
this.format = format;
|
||||
this.in = in;
|
||||
|
|
|
@ -27,9 +27,8 @@ import java.util.List;
|
|||
import java.util.Map;
|
||||
import java.util.NoSuchElementException;
|
||||
|
||||
import org.apache.commons.csv.CSVLexer.Token;
|
||||
|
||||
import static org.apache.commons.csv.CSVLexer.Token.Type.*;
|
||||
import static org.apache.commons.csv.Token.Type.*;
|
||||
|
||||
/**
|
||||
* Parses CSV files according to the specified configuration.
|
||||
|
|
|
@ -20,10 +20,9 @@ package org.apache.commons.csv;
|
|||
import java.io.IOException;
|
||||
import java.io.StringReader;
|
||||
|
||||
import org.apache.commons.csv.CSVLexer.Token;
|
||||
import org.junit.Test;
|
||||
|
||||
import static org.apache.commons.csv.CSVLexer.Token.Type.*;
|
||||
import static org.apache.commons.csv.Token.Type.*;
|
||||
import static org.junit.Assert.*;
|
||||
|
||||
public class CSVLexerTest {
|
||||
|
|
Loading…
Reference in New Issue