LUCENE-3698: FVH adds a multi value separator to the end of the highlighted text

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1232470 13f79535-47bb-0310-9956-ffa450edef68

lucene/contrib/CHANGES.txt

@@ -105,6 +105,11 @@ Changes in backwards compatibility policy
  * LUCENE-3626: The internal implementation classes in PKIndexSplitter
    and MultiPassIndexSplitter were made private as they now work
    per segment.  (Uwe Schindler)
+  
+Changes in Runtime Behavior
+
+ * LUCENE-3698: FastVectorHighlighter no longer adds a multi value separator
+   to the end of the highlighted text. (Shay Banon via Koji Sekiguchi)
 
 New Features
 

lucene/contrib/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragmentsBuilder.java

@@ -166,7 +166,12 @@ public abstract class BaseFragmentsBuilder implements FragmentsBuilder {
       buffer.append( values[index[0]++].stringValue() );
       buffer.append( getMultiValuedSeparator() );
     }
-    int eo = buffer.length() < endOffset ? buffer.length() : boundaryScanner.findEndOffset( buffer, endOffset );
+    int bufferLength = buffer.length();
+    // we added the multi value char to the last buffer, ignore it
+    if (values[index[0] - 1].fieldType().tokenized()) {
+      bufferLength--;
+    }
+    int eo = bufferLength < endOffset ? bufferLength : boundaryScanner.findEndOffset( buffer, endOffset );
     modifiedStartOffset[0] = boundaryScanner.findStartOffset( buffer, startOffset );
     return buffer.substring( modifiedStartOffset[0], eo );
   }

lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/ScoreOrderFragmentsBuilderTest.java

@@ -35,7 +35,7 @@ public class ScoreOrderFragmentsBuilderTest extends AbstractTestCase {
     String[] f = sofb.createFragments( reader, 0, F, ffl, 3 );
     assertEquals( 3, f.length );
     // check score order
-    assertEquals( "<b>c</b> <b>a</b> <b>a</b> b b ", f[0] );
+    assertEquals( "<b>c</b> <b>a</b> <b>a</b> b b", f[0] );
     assertEquals( "b b <b>a</b> b <b>a</b> b b b b b c", f[1] );
     assertEquals( "<b>a</b> b b b b b b b b b b", f[2] );
   }

lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilderTest.java

@@ -37,11 +37,11 @@ public class SimpleFragmentsBuilderTest extends AbstractTestCase {
   public void test1TermIndex() throws Exception {
     FieldFragList ffl = ffl(new TermQuery(new Term(F, "a")), "a" );
     SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();
-    assertEquals( "<b>a</b> ", sfb.createFragment( reader, 0, F, ffl ) );
+    assertEquals( "<b>a</b>", sfb.createFragment( reader, 0, F, ffl ) );
 
     // change tags
     sfb = new SimpleFragmentsBuilder( new String[]{ "[" }, new String[]{ "]" } );
-    assertEquals( "[a] ", sfb.createFragment( reader, 0, F, ffl ) );
+    assertEquals( "[a]", sfb.createFragment( reader, 0, F, ffl ) );
   }
   
   public void test2Frags() throws Exception {
@@ -51,7 +51,7 @@ public class SimpleFragmentsBuilderTest extends AbstractTestCase {
     // 3 snippets requested, but should be 2
     assertEquals( 2, f.length );
     assertEquals( "<b>a</b> b b b b b b b b b b", f[0] );
-    assertEquals( "b b <b>a</b> b <b>a</b> b ", f[1] );
+    assertEquals( "b b <b>a</b> b <b>a</b> b", f[1] );
   }
   
   public void test3Frags() throws Exception {
@@ -65,7 +65,7 @@ public class SimpleFragmentsBuilderTest extends AbstractTestCase {
     assertEquals( 3, f.length );
     assertEquals( "<b>a</b> b b b b b b b b b b", f[0] );
     assertEquals( "b b <b>a</b> b <b>a</b> b b b b b c", f[1] );
-    assertEquals( "<b>c</b> <b>a</b> <b>a</b> b b ", f[2] );
+    assertEquals( "<b>c</b> <b>a</b> <b>a</b> b b", f[2] );
   }
   
   public void testTagsAndEncoder() throws Exception {
@@ -73,7 +73,7 @@ public class SimpleFragmentsBuilderTest extends AbstractTestCase {
     SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();
     String[] preTags = { "[" };
     String[] postTags = { "]" };
-    assertEquals( "&lt;h1&gt; [a] &lt;/h1&gt; ",
+    assertEquals( "&lt;h1&gt; [a] &lt;/h1&gt;",
         sfb.createFragment( reader, 0, F, ffl, preTags, postTags, new SimpleHTMLEncoder() ) );
   }
 
@@ -94,7 +94,7 @@ public class SimpleFragmentsBuilderTest extends AbstractTestCase {
     SimpleFragListBuilder sflb = new SimpleFragListBuilder();
     FieldFragList ffl = sflb.createFieldFragList( fpl, 100 );
     SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();
-    assertEquals( "a b c  <b>d</b> e ", sfb.createFragment( reader, 0, F, ffl ) );
+    assertEquals( "a b c  <b>d</b> e", sfb.createFragment( reader, 0, F, ffl ) );
   }
   
   public void test1PhraseLongMV() throws Exception {
@@ -119,7 +119,7 @@ public class SimpleFragmentsBuilderTest extends AbstractTestCase {
     SimpleFragListBuilder sflb = new SimpleFragListBuilder();
     FieldFragList ffl = sflb.createFieldFragList( fpl, 100 );
     SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();
-    assertEquals( "processing <b>speed</b>, the ", sfb.createFragment( reader, 0, F, ffl ) );
+    assertEquals( "processing <b>speed</b>, the", sfb.createFragment( reader, 0, F, ffl ) );
   }
   
   public void testUnstoredField() throws Exception {
@@ -160,7 +160,7 @@ public class SimpleFragmentsBuilderTest extends AbstractTestCase {
     FieldFragList ffl = sflb.createFieldFragList( fpl, 100 );
     SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();
     sfb.setMultiValuedSeparator( '/' );
-    assertEquals( "abc/<b>defg</b>/hijkl/", sfb.createFragment( reader, 0, F, ffl ) );
+    assertEquals( "abc/<b>defg</b>/hijkl", sfb.createFragment( reader, 0, F, ffl ) );
   }
   
   public void testMVSeparator() throws Exception {
@@ -173,6 +173,6 @@ public class SimpleFragmentsBuilderTest extends AbstractTestCase {
     FieldFragList ffl = sflb.createFieldFragList( fpl, 100 );
     SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();
     sfb.setMultiValuedSeparator( '/' );
-    assertEquals( " b c//<b>d</b> e/", sfb.createFragment( reader, 0, F, ffl ) );
+    assertEquals( " b c//<b>d</b> e", sfb.createFragment( reader, 0, F, ffl ) );
   }
 }