SOLR-751: WordDelimiterFilter didn't adjust the start offset of single tokens

git-svn-id: https://svn.apache.org/repos/asf/lucene/solr/trunk@711450 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -95,6 +95,10 @@ Bug Fixes
 
  4. SOLR-805: DisMax queries are not being cached in QueryResultCache (Todd Feak via koji)
 
+ 5. SOLR-751: WordDelimiterFilter didn't adjust the start offset of single
+    tokens that started with delimiters, leading to incorrect highlighting.
+    (Stefan Oestreicher via yonik)
+
 
 Other Changes
 ----------------------

src/java/org/apache/solr/analysis/WordDelimiterFilter.java

@@ -373,6 +373,7 @@ final class WordDelimiterFilter extends TokenFilter {
               // just adjust the text w/o changing the rest
               // of the original token
               t.setTermBuffer(termBuffer, start, len-start);
+              t.setStartOffset(t.startOffset() + start);
               return t;
             }
 

src/test/org/apache/solr/analysis/TestWordDelimiterFilter.java

@@ -183,4 +183,99 @@ public class TestWordDelimiterFilter extends AbstractSolrTestCase {
       assertEquals(6, t.endOffset());
     }
   }
+  
+  public void testOffsetChange() throws Exception
+  {
+    WordDelimiterFilter wdf = new WordDelimiterFilter(
+      new TokenStream() {
+        Token t;
+        public Token next() {
+         if (t != null) return null;
+         t = new Token("übelkeit)", 7, 16);
+         return t;
+        }
+      },
+      1,1,0,0,1,1,0
+    );
+    
+    Token t = wdf.next();
+    
+    assertNotNull(t);
+    assertEquals("übelkeit", t.term());
+    assertEquals(7, t.startOffset());
+    assertEquals(15, t.endOffset());
+  }
+  
+  public void testOffsetChange2() throws Exception
+  {
+    WordDelimiterFilter wdf = new WordDelimiterFilter(
+      new TokenStream() {
+        Token t;
+        public Token next() {
+         if (t != null) return null;
+         t = new Token("(übelkeit", 7, 17);
+         return t;
+        }
+      },
+      1,1,0,0,1,1,0
+    );
+    
+    Token t = wdf.next();
+    
+    assertNotNull(t);
+    assertEquals("übelkeit", t.term());
+    assertEquals(8, t.startOffset());
+    assertEquals(17, t.endOffset());
+  }
+  
+  public void testOffsetChange3() throws Exception
+  {
+    WordDelimiterFilter wdf = new WordDelimiterFilter(
+      new TokenStream() {
+        Token t;
+        public Token next() {
+         if (t != null) return null;
+         t = new Token("(übelkeit", 7, 16);
+         return t;
+        }
+      },
+      1,1,0,0,1,1,0
+    );
+    
+    Token t = wdf.next();
+    
+    assertNotNull(t);
+    assertEquals("übelkeit", t.term());
+    assertEquals(8, t.startOffset());
+    assertEquals(16, t.endOffset());
+  }
+  
+  public void testOffsetChange4() throws Exception
+  {
+    WordDelimiterFilter wdf = new WordDelimiterFilter(
+      new TokenStream() {
+        private Token t;
+        public Token next() {
+         if (t != null) return null;
+         t = new Token("(foo,bar)", 7, 16);
+         return t;
+        }
+      },
+      1,1,0,0,1,1,0
+    );
+    
+    Token t = wdf.next();
+    
+    assertNotNull(t);
+    assertEquals("foo", t.term());
+    assertEquals(8, t.startOffset());
+    assertEquals(11, t.endOffset());
+    
+    t = wdf.next();
+    
+    assertNotNull(t);
+    assertEquals("bar", t.term());
+    assertEquals(12, t.startOffset());
+    assertEquals(15, t.endOffset());
+  }
 }