Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-3698: FVH adds a multi value separator to the end of the highlighted text 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1232470 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Koji Sekiguchi 2012-01-17 16:43:23 +00:00
parent 3e4fc88708
commit 1b82292e02
4 changed files with 21 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
lucene/contrib/CHANGES.txt
View File

@ -106,6 +106,11 @@ Changes in backwards compatibility policy
and MultiPassIndexSplitter were made private as they now work and MultiPassIndexSplitter were made private as they now work
per segment. (Uwe Schindler) per segment. (Uwe Schindler)
Changes in Runtime Behavior
* LUCENE-3698: FastVectorHighlighter no longer adds a multi value separator
to the end of the highlighted text. (Shay Banon via Koji Sekiguchi)
New Features New Features
* LUCENE-3596: DirectoryTaxonomyWriter extensions can override createIndexWriterConfig() * LUCENE-3596: DirectoryTaxonomyWriter extensions can override createIndexWriterConfig()

7
lucene/contrib/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragmentsBuilder.java
View File

@ -166,7 +166,12 @@ public abstract class BaseFragmentsBuilder implements FragmentsBuilder {
buffer.append( values[index[0]++].stringValue() ); buffer.append( values[index[0]++].stringValue() );
buffer.append( getMultiValuedSeparator() ); buffer.append( getMultiValuedSeparator() );
} }
int eo = buffer.length() < endOffset ? buffer.length() : boundaryScanner.findEndOffset( buffer, endOffset ); int bufferLength = buffer.length();
// we added the multi value char to the last buffer, ignore it
if (values[index[0] - 1].fieldType().tokenized()) {
bufferLength--;
}
int eo = bufferLength < endOffset ? bufferLength : boundaryScanner.findEndOffset( buffer, endOffset );
modifiedStartOffset[0] = boundaryScanner.findStartOffset( buffer, startOffset ); modifiedStartOffset[0] = boundaryScanner.findStartOffset( buffer, startOffset );
return buffer.substring( modifiedStartOffset[0], eo ); return buffer.substring( modifiedStartOffset[0], eo );
} }

2
lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/ScoreOrderFragmentsBuilderTest.java
View File

@ -35,7 +35,7 @@ public class ScoreOrderFragmentsBuilderTest extends AbstractTestCase {
String[] f = sofb.createFragments( reader, 0, F, ffl, 3 ); String[] f = sofb.createFragments( reader, 0, F, ffl, 3 );
assertEquals( 3, f.length ); assertEquals( 3, f.length );
// check score order // check score order
assertEquals( "<b>c</b> <b>a</b> <b>a</b> b b ", f[0] ); assertEquals( "<b>c</b> <b>a</b> <b>a</b> b b", f[0] );
assertEquals( "b b <b>a</b> b <b>a</b> b b b b b c", f[1] ); assertEquals( "b b <b>a</b> b <b>a</b> b b b b b c", f[1] );
assertEquals( "<b>a</b> b b b b b b b b b b", f[2] ); assertEquals( "<b>a</b> b b b b b b b b b b", f[2] );
} }

18
lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilderTest.java
View File

@ -37,11 +37,11 @@ public class SimpleFragmentsBuilderTest extends AbstractTestCase {
public void test1TermIndex() throws Exception { public void test1TermIndex() throws Exception {
FieldFragList ffl = ffl(new TermQuery(new Term(F, "a")), "a" ); FieldFragList ffl = ffl(new TermQuery(new Term(F, "a")), "a" );
SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder(); SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();
assertEquals( "<b>a</b> ", sfb.createFragment( reader, 0, F, ffl ) ); assertEquals( "<b>a</b>", sfb.createFragment( reader, 0, F, ffl ) );
// change tags // change tags
sfb = new SimpleFragmentsBuilder( new String[]{ "[" }, new String[]{ "]" } ); sfb = new SimpleFragmentsBuilder( new String[]{ "[" }, new String[]{ "]" } );
assertEquals( "[a] ", sfb.createFragment( reader, 0, F, ffl ) ); assertEquals( "[a]", sfb.createFragment( reader, 0, F, ffl ) );
} }
public void test2Frags() throws Exception { public void test2Frags() throws Exception {
@ -51,7 +51,7 @@ public class SimpleFragmentsBuilderTest extends AbstractTestCase {
// 3 snippets requested, but should be 2 // 3 snippets requested, but should be 2
assertEquals( 2, f.length ); assertEquals( 2, f.length );
assertEquals( "<b>a</b> b b b b b b b b b b", f[0] ); assertEquals( "<b>a</b> b b b b b b b b b b", f[0] );
assertEquals( "b b <b>a</b> b <b>a</b> b ", f[1] ); assertEquals( "b b <b>a</b> b <b>a</b> b", f[1] );
} }
public void test3Frags() throws Exception { public void test3Frags() throws Exception {
@ -65,7 +65,7 @@ public class SimpleFragmentsBuilderTest extends AbstractTestCase {
assertEquals( 3, f.length ); assertEquals( 3, f.length );
assertEquals( "<b>a</b> b b b b b b b b b b", f[0] ); assertEquals( "<b>a</b> b b b b b b b b b b", f[0] );
assertEquals( "b b <b>a</b> b <b>a</b> b b b b b c", f[1] ); assertEquals( "b b <b>a</b> b <b>a</b> b b b b b c", f[1] );
assertEquals( "<b>c</b> <b>a</b> <b>a</b> b b ", f[2] ); assertEquals( "<b>c</b> <b>a</b> <b>a</b> b b", f[2] );
} }
public void testTagsAndEncoder() throws Exception { public void testTagsAndEncoder() throws Exception {
@ -73,7 +73,7 @@ public class SimpleFragmentsBuilderTest extends AbstractTestCase {
SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder(); SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();
String[] preTags = { "[" }; String[] preTags = { "[" };
String[] postTags = { "]" }; String[] postTags = { "]" };
assertEquals( "&lt;h1&gt; [a] &lt;/h1&gt; ", assertEquals( "&lt;h1&gt; [a] &lt;/h1&gt;",
sfb.createFragment( reader, 0, F, ffl, preTags, postTags, new SimpleHTMLEncoder() ) ); sfb.createFragment( reader, 0, F, ffl, preTags, postTags, new SimpleHTMLEncoder() ) );
} }
@ -94,7 +94,7 @@ public class SimpleFragmentsBuilderTest extends AbstractTestCase {
SimpleFragListBuilder sflb = new SimpleFragListBuilder(); SimpleFragListBuilder sflb = new SimpleFragListBuilder();
FieldFragList ffl = sflb.createFieldFragList( fpl, 100 ); FieldFragList ffl = sflb.createFieldFragList( fpl, 100 );
SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder(); SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();
assertEquals( "a b c <b>d</b> e ", sfb.createFragment( reader, 0, F, ffl ) ); assertEquals( "a b c <b>d</b> e", sfb.createFragment( reader, 0, F, ffl ) );
} }
public void test1PhraseLongMV() throws Exception { public void test1PhraseLongMV() throws Exception {
@ -119,7 +119,7 @@ public class SimpleFragmentsBuilderTest extends AbstractTestCase {
SimpleFragListBuilder sflb = new SimpleFragListBuilder(); SimpleFragListBuilder sflb = new SimpleFragListBuilder();
FieldFragList ffl = sflb.createFieldFragList( fpl, 100 ); FieldFragList ffl = sflb.createFieldFragList( fpl, 100 );
SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder(); SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();
assertEquals( "processing <b>speed</b>, the ", sfb.createFragment( reader, 0, F, ffl ) ); assertEquals( "processing <b>speed</b>, the", sfb.createFragment( reader, 0, F, ffl ) );
} }
public void testUnstoredField() throws Exception { public void testUnstoredField() throws Exception {
@ -160,7 +160,7 @@ public class SimpleFragmentsBuilderTest extends AbstractTestCase {
FieldFragList ffl = sflb.createFieldFragList( fpl, 100 ); FieldFragList ffl = sflb.createFieldFragList( fpl, 100 );
SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder(); SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();
sfb.setMultiValuedSeparator( '/' ); sfb.setMultiValuedSeparator( '/' );
assertEquals( "abc/<b>defg</b>/hijkl/", sfb.createFragment( reader, 0, F, ffl ) ); assertEquals( "abc/<b>defg</b>/hijkl", sfb.createFragment( reader, 0, F, ffl ) );
} }
public void testMVSeparator() throws Exception { public void testMVSeparator() throws Exception {
@ -173,6 +173,6 @@ public class SimpleFragmentsBuilderTest extends AbstractTestCase {
FieldFragList ffl = sflb.createFieldFragList( fpl, 100 ); FieldFragList ffl = sflb.createFieldFragList( fpl, 100 );
SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder(); SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();
sfb.setMultiValuedSeparator( '/' ); sfb.setMultiValuedSeparator( '/' );
assertEquals( " b c//<b>d</b> e/", sfb.createFragment( reader, 0, F, ffl ) ); assertEquals( " b c//<b>d</b> e", sfb.createFragment( reader, 0, F, ffl ) );
} }
} }