LUCENE-437 - Add position increment pass through on SnowballFilter tokens

git-svn-id: https://svn.apache.org/repos/asf/lucene/java/trunk@290943 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -213,6 +213,10 @@ Bug fixes
     inside a BooleanQuery.
     (Hans Hjelm and Scotty Allen via Daniel Naber, Bug #35626)
 
+11. Fixed SnowballFilter to pass through the position increment from
+    the original token.
+    (Yonik Seeley via Erik Hatcher, LUCENE-437)
+
 Optimizations
      
  1. Disk usage (peak requirements during indexing and optimization)

contrib/snowball/src/java/org/apache/lucene/analysis/snowball/SnowballFilter.java

@@ -69,7 +69,10 @@ public class SnowballFilter extends TokenFilter {
     } catch (Exception e) {
       throw new RuntimeException(e.toString());
     }
-    return new Token(stemmer.getCurrent(),
+    
+    Token newToken = new Token(stemmer.getCurrent(),
                       token.startOffset(), token.endOffset(), token.type());
+    newToken.setPositionIncrement(token.getPositionIncrement());
+    return newToken;
   }
 }

contrib/snowball/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java

@@ -55,6 +55,7 @@ package org.apache.lucene.analysis.snowball;
  */
 
 import java.io.*;
+
 import junit.framework.*;
 
 import org.apache.lucene.analysis.*;
@@ -79,5 +80,28 @@ public class TestSnowball extends TestCase {
     assertAnalyzesTo(a, "he abhorred accents",
         new String[]{"he", "abhor", "accent"});
   }
+
+
+  public void testFilterTokens() throws Exception {
+    final Token tok = new Token("accents", 2, 7, "wrd");
+    tok.setPositionIncrement(3);
+
+    SnowballFilter filter = new SnowballFilter(
+        new TokenStream() {
+          public Token next() {
+            return tok;
+          }
+        },
+        "English"
+    );
+
+    Token newtok = filter.next();
+
+    assertEquals("accent", newtok.termText());
+    assertEquals(2, newtok.startOffset());
+    assertEquals(7, newtok.endOffset());
+    assertEquals("wrd", newtok.type());
+    assertEquals(3, newtok.getPositionIncrement());
+  }
 }