Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-3698: FVH adds a multi value separator to the end of the highlighted text 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1232470 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Koji Sekiguchi 2012-01-17 16:43:23 +00:00
parent 3e4fc88708
commit 1b82292e02
4 changed files with 21 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
lucene/contrib/CHANGES.txt
View File

@ -106,6 +106,11 @@ Changes in backwards compatibility policy
and MultiPassIndexSplitter were made private as they now work
per segment. (Uwe Schindler)
Changes in Runtime Behavior
* LUCENE-3698: FastVectorHighlighter no longer adds a multi value separator
to the end of the highlighted text. (Shay Banon via Koji Sekiguchi)
New Features
* LUCENE-3596: DirectoryTaxonomyWriter extensions can override createIndexWriterConfig()

7
lucene/contrib/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragmentsBuilder.java
View File

@ -166,7 +166,12 @@ public abstract class BaseFragmentsBuilder implements FragmentsBuilder {
buffer.append( values[index[0]++].stringValue() );
buffer.append( getMultiValuedSeparator() );
}
int eo = buffer.length() < endOffset ? buffer.length() : boundaryScanner.findEndOffset( buffer, endOffset );
int bufferLength = buffer.length();
// we added the multi value char to the last buffer, ignore it
if (values[index[0] - 1].fieldType().tokenized()) {
bufferLength--;
}
int eo = bufferLength < endOffset ? bufferLength : boundaryScanner.findEndOffset( buffer, endOffset );
modifiedStartOffset[0] = boundaryScanner.findStartOffset( buffer, startOffset );
return buffer.substring( modifiedStartOffset[0], eo );
}

2
lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/ScoreOrderFragmentsBuilderTest.java
View File

@ -35,7 +35,7 @@ public class ScoreOrderFragmentsBuilderTest extends AbstractTestCase {
String[] f = sofb.createFragments( reader, 0, F, ffl, 3 );
assertEquals( 3, f.length );
// check score order
assertEquals( "<b>c</b> <b>a</b> <b>a</b> b b ", f[0] );
assertEquals( "<b>c</b> <b>a</b> <b>a</b> b b", f[0] );
assertEquals( "b b <b>a</b> b <b>a</b> b b b b b c", f[1] );
assertEquals( "<b>a</b> b b b b b b b b b b", f[2] );
}

18
lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilderTest.java
View File

@ -37,11 +37,11 @@ public class SimpleFragmentsBuilderTest extends AbstractTestCase {
public void test1TermIndex() throws Exception {
FieldFragList ffl = ffl(new TermQuery(new Term(F, "a")), "a" );
SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();
assertEquals( "<b>a</b> ", sfb.createFragment( reader, 0, F, ffl ) );
assertEquals( "<b>a</b>", sfb.createFragment( reader, 0, F, ffl ) );
// change tags
sfb = new SimpleFragmentsBuilder( new String[]{ "[" }, new String[]{ "]" } );
assertEquals( "[a] ", sfb.createFragment( reader, 0, F, ffl ) );
assertEquals( "[a]", sfb.createFragment( reader, 0, F, ffl ) );
}
public void test2Frags() throws Exception {
@ -51,7 +51,7 @@ public class SimpleFragmentsBuilderTest extends AbstractTestCase {
// 3 snippets requested, but should be 2
assertEquals( 2, f.length );
assertEquals( "<b>a</b> b b b b b b b b b b", f[0] );
assertEquals( "b b <b>a</b> b <b>a</b> b ", f[1] );
assertEquals( "b b <b>a</b> b <b>a</b> b", f[1] );
}
public void test3Frags() throws Exception {
@ -65,7 +65,7 @@ public class SimpleFragmentsBuilderTest extends AbstractTestCase {
assertEquals( 3, f.length );
assertEquals( "<b>a</b> b b b b b b b b b b", f[0] );
assertEquals( "b b <b>a</b> b <b>a</b> b b b b b c", f[1] );
assertEquals( "<b>c</b> <b>a</b> <b>a</b> b b ", f[2] );
assertEquals( "<b>c</b> <b>a</b> <b>a</b> b b", f[2] );
}
public void testTagsAndEncoder() throws Exception {
@ -73,7 +73,7 @@ public class SimpleFragmentsBuilderTest extends AbstractTestCase {
SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();
String[] preTags = { "[" };
String[] postTags = { "]" };
assertEquals( "&lt;h1&gt; [a] &lt;/h1&gt; ",
assertEquals( "&lt;h1&gt; [a] &lt;/h1&gt;",
sfb.createFragment( reader, 0, F, ffl, preTags, postTags, new SimpleHTMLEncoder() ) );
}
@ -94,7 +94,7 @@ public class SimpleFragmentsBuilderTest extends AbstractTestCase {
SimpleFragListBuilder sflb = new SimpleFragListBuilder();
FieldFragList ffl = sflb.createFieldFragList( fpl, 100 );
SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();
assertEquals( "a b c <b>d</b> e ", sfb.createFragment( reader, 0, F, ffl ) );
assertEquals( "a b c <b>d</b> e", sfb.createFragment( reader, 0, F, ffl ) );
}
public void test1PhraseLongMV() throws Exception {
@ -119,7 +119,7 @@ public class SimpleFragmentsBuilderTest extends AbstractTestCase {
SimpleFragListBuilder sflb = new SimpleFragListBuilder();
FieldFragList ffl = sflb.createFieldFragList( fpl, 100 );
SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();
assertEquals( "processing <b>speed</b>, the ", sfb.createFragment( reader, 0, F, ffl ) );
assertEquals( "processing <b>speed</b>, the", sfb.createFragment( reader, 0, F, ffl ) );
}
public void testUnstoredField() throws Exception {
@ -160,7 +160,7 @@ public class SimpleFragmentsBuilderTest extends AbstractTestCase {
FieldFragList ffl = sflb.createFieldFragList( fpl, 100 );
SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();
sfb.setMultiValuedSeparator( '/' );
assertEquals( "abc/<b>defg</b>/hijkl/", sfb.createFragment( reader, 0, F, ffl ) );
assertEquals( "abc/<b>defg</b>/hijkl", sfb.createFragment( reader, 0, F, ffl ) );
}
public void testMVSeparator() throws Exception {
@ -173,6 +173,6 @@ public class SimpleFragmentsBuilderTest extends AbstractTestCase {
FieldFragList ffl = sflb.createFieldFragList( fpl, 100 );
SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();
sfb.setMultiValuedSeparator( '/' );
assertEquals( " b c//<b>d</b> e/", sfb.createFragment( reader, 0, F, ffl ) );
assertEquals( " b c//<b>d</b> e", sfb.createFragment( reader, 0, F, ffl ) );
}
}