Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-1759: Set final offset correctly in contrib TokenStreams. 

git-svn-id: https://svn.apache.org/repos/asf/lucene/java/trunk@799968 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael Busch 2009-08-02 02:10:46 +00:00
parent 1743081b07
commit 537aeb24e0
5 changed files with 35 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
contrib/analyzers/common/src/java/org/apache/lucene/analysis/cjk/CJKTokenizer.java
View File

@ -272,4 +272,10 @@ public final class CJKTokenizer extends Tokenizer {
// return an empty string)
}
}
public final void end() {
// set final offset
final int finalOffset = offset;
this.offsetAtt.setOffset(finalOffset, finalOffset);
}
}

7
contrib/analyzers/common/src/java/org/apache/lucene/analysis/cn/ChineseTokenizer.java
View File

@ -139,6 +139,11 @@ public final class ChineseTokenizer extends Tokenizer {
break;
}
}
}
public final void end() {
// set final offset
final int finalOffset = offset;
this.offsetAtt.setOffset(finalOffset, finalOffset);
}
}

6
contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizer.java
View File

@ -152,6 +152,12 @@ public class EdgeNGramTokenizer extends Tokenizer {
return true;
}
public final void end() {
// set final offset
final int finalOffset = inLen;
this.offsetAtt.setOffset(finalOffset, finalOffset);
}
/** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
* not be overridden. Delegates to the backwards compatibility layer. */
public final Token next(final Token reusableToken) throws java.io.IOException {

6
contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizer.java
View File

@ -97,6 +97,12 @@ public class NGramTokenizer extends Tokenizer {
return true;
}
public final void end() {
// set final offset
final int finalOffset = inLen;
this.offsetAtt.setOffset(finalOffset, finalOffset);
}
/** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
* not be overridden. Delegates to the backwards compatibility layer. */
public final Token next(final Token reusableToken) throws java.io.IOException {

11
contrib/memory/src/java/org/apache/lucene/index/memory/PatternAnalyzer.java
View File

@ -367,6 +367,11 @@ public class PatternAnalyzer extends Analyzer {
}
}
public final void end() {
// set final offset
final int finalOffset = str.length();
this.offsetAtt.setOffset(finalOffset, finalOffset);
}
}
@ -442,6 +447,12 @@ public class PatternAnalyzer extends Analyzer {
return true;
}
public final void end() {
// set final offset
final int finalOffset = str.length();
this.offsetAtt.setOffset(finalOffset, finalOffset);
}
private boolean isTokenChar(char c, boolean isLetter) {
return isLetter ? Character.isLetter(c) : !Character.isWhitespace(c);
}