LUCENE-3919: add convenience ctors to MockTokenizer,MockCharFilter, fix MockCharFilter to chain its correctOffset

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1310811 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Robert Muir 2012-04-07 18:07:56 +00:00
parent ed485b29ec
commit 4b638a2903
2 changed files with 16 additions and 6 deletions

View File

@ -22,19 +22,24 @@ import java.io.Reader;
import java.util.SortedMap;
import java.util.TreeMap;
// the purpose of this charfilter is to send offsets out of bounds
// if the analyzer doesn't use correctOffset or does incorrect offset math.
class MockCharFilter extends CharStream {
final Reader in;
/** the purpose of this charfilter is to send offsets out of bounds
if the analyzer doesn't use correctOffset or does incorrect offset math. */
public class MockCharFilter extends CharStream {
final CharStream in;
final int remainder;
// for testing only
public MockCharFilter(Reader in, int remainder) {
this.in = in;
this.in = CharReader.get(in);
this.remainder = remainder;
assert remainder >= 0 && remainder < 10 : "invalid parameter";
}
// for testing only, uses a remainder of 0
public MockCharFilter(Reader in) {
this(in, 0);
}
@Override
public void close() throws IOException {
in.close();
@ -89,7 +94,7 @@ class MockCharFilter extends CharStream {
SortedMap<Integer,Integer> subMap = corrections.subMap(0, currentOff+1);
int ret = subMap.isEmpty() ? currentOff : currentOff + subMap.get(subMap.lastKey());
assert ret >= 0 : "currentOff=" + currentOff + ",diff=" + (ret-currentOff);
return ret;
return in.correctOffset(ret); // chain the call
}
protected void addOffCorrectMap(int off, int cumulativeDiff) {

View File

@ -95,6 +95,11 @@ public class MockTokenizer extends Tokenizer {
this(input, runAutomaton, lowerCase, DEFAULT_MAX_TOKEN_LENGTH);
}
/** Calls {@link #MockTokenizer(Reader, CharacterRunAutomaton, boolean) MockTokenizer(Reader, WHITESPACE, true)} */
public MockTokenizer(Reader input) {
this(input, WHITESPACE, true);
}
@Override
public final boolean incrementToken() throws IOException {
assert !enableChecks || (streamState == State.RESET || streamState == State.INCREMENT)