Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-4107: FVH: make FieldFragList plugable, part of LUCENE-3440 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1346001 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Koji Sekiguchi 2012-06-04 15:05:42 +00:00
parent 8d3f45da8a
commit 6002a37c4d
6 changed files with 175 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

96
lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragListBuilder.java Normal file
View File

@ -0,0 +1,96 @@
package org.apache.lucene.search.vectorhighlight;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import org.apache.lucene.search.vectorhighlight.FieldPhraseList.WeightedPhraseInfo;
/**
* A abstract implementation of {@link FragListBuilder}.
*/
public abstract class BaseFragListBuilder implements FragListBuilder {
public static final int MARGIN_DEFAULT = 6;
public static final int MIN_FRAG_CHAR_SIZE_FACTOR = 3;
final int margin;
final int minFragCharSize;
public BaseFragListBuilder( int margin ){
if( margin < 0 )
throw new IllegalArgumentException( "margin(" + margin + ") is too small. It must be 0 or higher." );
this.margin = margin;
this.minFragCharSize = Math.max( 1, margin * MIN_FRAG_CHAR_SIZE_FACTOR );
}
public BaseFragListBuilder(){
this( MARGIN_DEFAULT );
}
protected FieldFragList createFieldFragList( FieldPhraseList fieldPhraseList, FieldFragList fieldFragList, int fragCharSize ){
if( fragCharSize < minFragCharSize )
throw new IllegalArgumentException( "fragCharSize(" + fragCharSize + ") is too small. It must be " + minFragCharSize + " or higher." );
List<WeightedPhraseInfo> wpil = new ArrayList<WeightedPhraseInfo>();
Iterator<WeightedPhraseInfo> ite = fieldPhraseList.getPhraseList().iterator();
WeightedPhraseInfo phraseInfo = null;
int startOffset = 0;
boolean taken = false;
while( true ){
if( !taken ){
if( !ite.hasNext() ) break;
phraseInfo = ite.next();
}
taken = false;
if( phraseInfo == null ) break;
// if the phrase violates the border of previous fragment, discard it and try next phrase
if( phraseInfo.getStartOffset() < startOffset ) continue;
wpil.clear();
wpil.add( phraseInfo );
int st = phraseInfo.getStartOffset() - margin < startOffset ?
startOffset : phraseInfo.getStartOffset() - margin;
int en = st + fragCharSize;
if( phraseInfo.getEndOffset() > en )
en = phraseInfo.getEndOffset();
startOffset = en;
while( true ){
if( ite.hasNext() ){
phraseInfo = ite.next();
taken = true;
if( phraseInfo == null ) break;
}
else
break;
if( phraseInfo.getEndOffset() <= en )
wpil.add( phraseInfo );
else
break;
}
fieldFragList.add( st, en, wpil );
}
return fieldFragList;
}
}

16
lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldFragList.java
View File

@ -27,7 +27,7 @@ import org.apache.lucene.search.vectorhighlight.FieldPhraseList.WeightedPhraseIn
* FieldFragList has a list of "frag info" that is used by FragmentsBuilder class
* to create fragments (snippets).
*/
public class FieldFragList {
public abstract class FieldFragList {
private List<WeightedFragInfo> fragInfos = new ArrayList<WeightedFragInfo>();
@ -46,9 +46,7 @@ public class FieldFragList {
* @param endOffset end offset of the fragment
* @param phraseInfoList list of WeightedPhraseInfo objects
*/
public void add( int startOffset, int endOffset, List<WeightedPhraseInfo> phraseInfoList ){
fragInfos.add( new WeightedFragInfo( startOffset, endOffset, phraseInfoList ) );
}
public abstract void add( int startOffset, int endOffset, List<WeightedPhraseInfo> phraseInfoList );
/**
* return the list of WeightedFragInfos.
@ -66,15 +64,11 @@ public class FieldFragList {
private int startOffset;
private int endOffset;
public WeightedFragInfo( int startOffset, int endOffset, List<WeightedPhraseInfo> phraseInfoList ){
public WeightedFragInfo( int startOffset, int endOffset, List<SubInfo> subInfos, float totalBoost ){
this.startOffset = startOffset;
this.endOffset = endOffset;
subInfos = new ArrayList<SubInfo>();
for( WeightedPhraseInfo phraseInfo : phraseInfoList ){
SubInfo subInfo = new SubInfo( phraseInfo.getText(), phraseInfo.getTermsOffsets(), phraseInfo.getSeqnum() );
subInfos.add( subInfo );
totalBoost += phraseInfo.getBoost();
}
this.totalBoost = totalBoost;
this.subInfos = subInfos;
}
public List<SubInfo> getSubInfos(){

11
lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldPhraseList.java
View File

@ -41,6 +41,15 @@ public class FieldPhraseList {
this (fieldTermStack, fieldQuery, Integer.MAX_VALUE);
}
/**
* return the list of WeightedPhraseInfo.
*
* @return phraseList.
*/
public List<WeightedPhraseInfo> getPhraseList() {
return phraseList;
}
/**
* a constructor.
*
@ -48,7 +57,7 @@ public class FieldPhraseList {
* @param fieldQuery FieldQuery object
* @param phraseLimit maximum size of phraseList
*/
public FieldPhraseList( FieldTermStack fieldTermStack, FieldQuery fieldQuery, int phraseLimit){
public FieldPhraseList( FieldTermStack fieldTermStack, FieldQuery fieldQuery, int phraseLimit ){
final String field = fieldTermStack.getFieldName();
LinkedList<TermInfo> phraseCandidate = new LinkedList<TermInfo>();

53
lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/SimpleFieldFragList.java Normal file
View File

@ -0,0 +1,53 @@
package org.apache.lucene.search.vectorhighlight;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.util.ArrayList;
import java.util.List;
import org.apache.lucene.search.vectorhighlight.FieldFragList.WeightedFragInfo.SubInfo;
import org.apache.lucene.search.vectorhighlight.FieldPhraseList.WeightedPhraseInfo;
/**
* A simple implementation of {@link FielFragList}.
*/
public class SimpleFieldFragList extends FieldFragList {
/**
* a constructor.
*
* @param fragCharSize the length (number of chars) of a fragment
*/
public SimpleFieldFragList( int fragCharSize ) {
super( fragCharSize );
}
/* (non-Javadoc)
* @see org.apache.lucene.search.vectorhighlight.FieldFragList#add( int startOffset, int endOffset, List<WeightedPhraseInfo> phraseInfoList )
*/
@Override
public void add( int startOffset, int endOffset, List<WeightedPhraseInfo> phraseInfoList ) {
float score = 0;
List<SubInfo> subInfos = new ArrayList<SubInfo>();
for( WeightedPhraseInfo phraseInfo : phraseInfoList ){
subInfos.add( new SubInfo( phraseInfo.getText(), phraseInfo.getTermsOffsets(), phraseInfo.getSeqnum() ) );
score += phraseInfo.getBoost();
}
getFragInfos().add( new WeightedFragInfo( startOffset, endOffset, subInfos, score ) );
}
}

78
lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/SimpleFragListBuilder.java
View File

@ -17,83 +17,25 @@ package org.apache.lucene.search.vectorhighlight;
* limitations under the License.
*/
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import org.apache.lucene.search.vectorhighlight.FieldPhraseList.WeightedPhraseInfo;
/**
* A simple implementation of {@link FragListBuilder}.
*/
public class SimpleFragListBuilder implements FragListBuilder {
public class SimpleFragListBuilder extends BaseFragListBuilder {
public static final int MARGIN_DEFAULT = 6;
public static final int MIN_FRAG_CHAR_SIZE_FACTOR = 3;
final int margin;
final int minFragCharSize;
public SimpleFragListBuilder( int margin ){
if( margin < 0 )
throw new IllegalArgumentException( "margin(" + margin + ") is too small. It must be 0 or higher." );
this.margin = margin;
this.minFragCharSize = Math.max( 1, margin * MIN_FRAG_CHAR_SIZE_FACTOR );
public SimpleFragListBuilder() {
super();
}
public SimpleFragListBuilder(){
this( MARGIN_DEFAULT );
public SimpleFragListBuilder(int margin) {
super(margin);
}
public FieldFragList createFieldFragList(FieldPhraseList fieldPhraseList, int fragCharSize) {
if( fragCharSize < minFragCharSize )
throw new IllegalArgumentException( "fragCharSize(" + fragCharSize + ") is too small. It must be " +
minFragCharSize + " or higher." );
FieldFragList ffl = new FieldFragList( fragCharSize );
List<WeightedPhraseInfo> wpil = new ArrayList<WeightedPhraseInfo>();
Iterator<WeightedPhraseInfo> ite = fieldPhraseList.phraseList.iterator();
WeightedPhraseInfo phraseInfo = null;
int startOffset = 0;
boolean taken = false;
while( true ){
if( !taken ){
if( !ite.hasNext() ) break;
phraseInfo = ite.next();
}
taken = false;
if( phraseInfo == null ) break;
// if the phrase violates the border of previous fragment, discard it and try next phrase
if( phraseInfo.getStartOffset() < startOffset ) continue;
wpil.clear();
wpil.add( phraseInfo );
int st = phraseInfo.getStartOffset() - margin < startOffset ?
startOffset : phraseInfo.getStartOffset() - margin;
int en = st + fragCharSize;
if( phraseInfo.getEndOffset() > en )
en = phraseInfo.getEndOffset();
startOffset = en;
while( true ){
if( ite.hasNext() ){
phraseInfo = ite.next();
taken = true;
if( phraseInfo == null ) break;
}
else
break;
if( phraseInfo.getEndOffset() <= en )
wpil.add( phraseInfo );
else
break;
}
ffl.add( st, en, wpil );
}
return ffl;
/* (non-Javadoc)
* @see org.apache.lucene.search.vectorhighlight.FragListBuilder#createFieldFragList(FieldPhraseList fieldPhraseList, int fragCharSize)
*/
public FieldFragList createFieldFragList( FieldPhraseList fieldPhraseList, int fragCharSize ){
return createFieldFragList( fieldPhraseList, new SimpleFieldFragList( fragCharSize ), fragCharSize );
}
}

2
lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/SingleFragListBuilder.java
View File

@ -38,7 +38,7 @@ public class SingleFragListBuilder implements FragListBuilder {
public FieldFragList createFieldFragList(FieldPhraseList fieldPhraseList,
int fragCharSize) {
FieldFragList ffl = new FieldFragList( fragCharSize );
FieldFragList ffl = new SimpleFieldFragList( fragCharSize );
List<WeightedPhraseInfo> wpil = new ArrayList<WeightedPhraseInfo>();
Iterator<WeightedPhraseInfo> ite = fieldPhraseList.phraseList.iterator();