Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-437 - Add position increment pass through on SnowballFilter tokens 

git-svn-id: https://svn.apache.org/repos/asf/lucene/java/trunk@290943 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Erik Hatcher 2005-09-22 13:38:58 +00:00
parent 1f5925c639
commit 32fb624ebc
3 changed files with 35 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
CHANGES.txt
View File

@ -213,6 +213,10 @@ Bug fixes
inside a BooleanQuery.
(Hans Hjelm and Scotty Allen via Daniel Naber, Bug #35626)
11. Fixed SnowballFilter to pass through the position increment from
the original token.
(Yonik Seeley via Erik Hatcher, LUCENE-437)
Optimizations
1. Disk usage (peak requirements during indexing and optimization)

7
contrib/snowball/src/java/org/apache/lucene/analysis/snowball/SnowballFilter.java
View File

@ -69,7 +69,10 @@ public class SnowballFilter extends TokenFilter {
} catch (Exception e) {
throw new RuntimeException(e.toString());
}
return new Token(stemmer.getCurrent(),
token.startOffset(), token.endOffset(), token.type());
Token newToken = new Token(stemmer.getCurrent(),
token.startOffset(), token.endOffset(), token.type());
newToken.setPositionIncrement(token.getPositionIncrement());
return newToken;
}
}

28
contrib/snowball/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java
View File

@ -55,6 +55,7 @@ package org.apache.lucene.analysis.snowball;
*/
import java.io.*;
import junit.framework.*;
import org.apache.lucene.analysis.*;
@ -65,7 +66,7 @@ public class TestSnowball extends TestCase {
String input,
String[] output) throws Exception {
TokenStream ts = a.tokenStream("dummy", new StringReader(input));
for (int i=0; i<output.length; i++) {
for (int i = 0; i < output.length; i++) {
Token t = ts.next();
assertNotNull(t);
assertEquals(output[i], t.termText());
@ -77,7 +78,30 @@ public class TestSnowball extends TestCase {
public void testEnglish() throws Exception {
Analyzer a = new SnowballAnalyzer("English");
assertAnalyzesTo(a, "he abhorred accents",
new String[] { "he", "abhor", "accent" });
new String[]{"he", "abhor", "accent"});
}
public void testFilterTokens() throws Exception {
final Token tok = new Token("accents", 2, 7, "wrd");
tok.setPositionIncrement(3);
SnowballFilter filter = new SnowballFilter(
new TokenStream() {
public Token next() {
return tok;
}
},
"English"
);
Token newtok = filter.next();
assertEquals("accent", newtok.termText());
assertEquals(2, newtok.startOffset());
assertEquals(7, newtok.endOffset());
assertEquals("wrd", newtok.type());
assertEquals(3, newtok.getPositionIncrement());
}
}