LUCENE-1759: Set final offset correctly in contrib TokenStreams.

git-svn-id: https://svn.apache.org/repos/asf/lucene/java/trunk@799968 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael Busch 2009-08-02 02:10:46 +00:00
parent 1743081b07
commit 537aeb24e0
5 changed files with 35 additions and 1 deletions

View File

@ -272,4 +272,10 @@ public final class CJKTokenizer extends Tokenizer {
// return an empty string)
}
}
public final void end() {
// set final offset
final int finalOffset = offset;
this.offsetAtt.setOffset(finalOffset, finalOffset);
}
}

View File

@ -139,6 +139,11 @@ public final class ChineseTokenizer extends Tokenizer {
break;
}
}
}
public final void end() {
// set final offset
final int finalOffset = offset;
this.offsetAtt.setOffset(finalOffset, finalOffset);
}
}

View File

@ -152,6 +152,12 @@ public class EdgeNGramTokenizer extends Tokenizer {
return true;
}
public final void end() {
// set final offset
final int finalOffset = inLen;
this.offsetAtt.setOffset(finalOffset, finalOffset);
}
/** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
* not be overridden. Delegates to the backwards compatibility layer. */
public final Token next(final Token reusableToken) throws java.io.IOException {

View File

@ -97,6 +97,12 @@ public class NGramTokenizer extends Tokenizer {
return true;
}
public final void end() {
// set final offset
final int finalOffset = inLen;
this.offsetAtt.setOffset(finalOffset, finalOffset);
}
/** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
* not be overridden. Delegates to the backwards compatibility layer. */
public final Token next(final Token reusableToken) throws java.io.IOException {

View File

@ -367,6 +367,11 @@ public class PatternAnalyzer extends Analyzer {
}
}
public final void end() {
// set final offset
final int finalOffset = str.length();
this.offsetAtt.setOffset(finalOffset, finalOffset);
}
}
@ -442,6 +447,12 @@ public class PatternAnalyzer extends Analyzer {
return true;
}
public final void end() {
// set final offset
final int finalOffset = str.length();
this.offsetAtt.setOffset(finalOffset, finalOffset);
}
private boolean isTokenChar(char c, boolean isLetter) {
return isLetter ? Character.isLetter(c) : !Character.isWhitespace(c);
}