Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-1522: adding new Fast Vector Highlighter contrib 

git-svn-id: https://svn.apache.org/repos/asf/lucene/java/trunk@792542 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael McCandless 2009-07-09 13:06:51 +00:00
parent e49af570d1
commit 9cbe5f4ff4
41 changed files with 3982 additions and 188 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
build.xml
View File

@ -312,6 +312,7 @@
<packageset dir="contrib/collation/src/java"/>
<packageset dir="contrib/db/bdb-je/src/java"/>
<packageset dir="contrib/db/bdb/src/java"/>
<packageset dir="contrib/fast-vector-highlighter/src/java"/>
<packageset dir="contrib/highlighter/src/java"/>
<packageset dir="contrib/instantiated/src/java"/>
<packageset dir="contrib/lucli/src/java"/>
@ -343,6 +344,7 @@
<group title="contrib: Benchmark" packages="org.apache.lucene.benchmark*"/>
<group title="contrib: Collation" packages="org.apache.lucene.collation*"/>
<group title="contrib: DB" packages="org.apache.lucene.store.db*:org.apache.lucene.store.je*:com.sleepycat*"/>
<group title="contrib: Fast Vector Highlighter" packages="org.apache.lucene.search.vectorhighlight*"/>
<group title="contrib: Highlighter" packages="org.apache.lucene.search.highlight*"/>
<group title="contrib: Instantiated" packages="org.apache.lucene.store.instantiated*"/>
<group title="contrib: Lucli" packages="lucli*"/>

3
contrib/CHANGES.txt
View File

@ -65,6 +65,9 @@ New features
7. LUCENE-1704: Allow specifying the Tidy configuration file when
parsing HTML docs with contrib/ant. (Keith Sprochi via Mike
McCandless)
8. LUCENE-1522: Added contrib/fast-vector-highlighter, a new alternative
highlighter. (Koji Sekiguchi via Mike McCandless)
Optimizations

47
contrib/fast-vector-highlighter/build.xml Normal file
View File

@ -0,0 +1,47 @@
<?xml version="1.0"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project name="fast-vector-highlighter" default="default">
<description>
Hits highlighter using TermVectors
</description>
<property name="javac.source" value="1.5" />
<property name="javac.target" value="1.5" />
<import file="../contrib-build.xml"/>
<property name="analyzers.jar" location="${common.dir}/build/contrib/analyzers/lucene-analyzers-${version}.jar"/>
<available property="analyzers.jar.present" type="file" file="${analyzers.jar}"/>
<path id="classpath">
<pathelement path="${lucene.jar}"/>
<pathelement path="${analyzers.jar}"/>
<pathelement path="${project.classpath}"/>
</path>
<target name="compile-core" depends="build-analyzers, common.compile-core" />
<target name="build-analyzers" unless="analyzers.jar.present">
<echo>Fast Vector Highlighter building dependency ${analyzers.jar}</echo>
<ant antfile="../analyzers/build.xml" target="default" inheritall="false" dir="../analyzers" />
</target>
</project>

124
contrib/fast-vector-highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragmentsBuilder.java Normal file
View File

@ -0,0 +1,124 @@
package org.apache.lucene.search.vectorhighlight;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.MapFieldSelector;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.vectorhighlight.FieldFragList.WeightedFragInfo;
import org.apache.lucene.search.vectorhighlight.FieldFragList.WeightedFragInfo.SubInfo;
import org.apache.lucene.search.vectorhighlight.FieldPhraseList.WeightedPhraseInfo.Toffs;
public abstract class BaseFragmentsBuilder implements FragmentsBuilder {
protected String[] preTags, postTags;
public static final String[] COLORED_PRE_TAGS = {
"<b style=\"background:yellow\">", "<b style=\"background:lawngreen\">", "<b style=\"background:aquamarine\">",
"<b style=\"background:magenta\">", "<b style=\"background:palegreen\">", "<b style=\"background:coral\">",
"<b style=\"background:wheat\">", "<b style=\"background:khaki\">", "<b style=\"background:lime\">",
"<b style=\"background:deepskyblue\">"
};
public static final String[] COLORED_POST_TAGS = { "</b>" };
protected BaseFragmentsBuilder(){
this( new String[]{ "<b>" }, new String[]{ "</b>" } );
}
protected BaseFragmentsBuilder( String[] preTags, String[] postTags ){
this.preTags = preTags;
this.postTags = postTags;
}
static Object checkTagsArgument( Object tags ){
if( tags instanceof String ) return tags;
else if( tags instanceof String[] ) return tags;
throw new IllegalArgumentException( "type of preTags/postTags must be a String or String[]" );
}
public abstract List<WeightedFragInfo> getWeightedFragInfoList( List<WeightedFragInfo> src );
public String createFragment( IndexReader reader, int docId,
String fieldName, FieldFragList fieldFragList ) throws IOException {
String[] fragments = createFragments( reader, docId, fieldName, fieldFragList, 1 );
if( fragments == null || fragments.length == 0 ) return null;
return fragments[0];
}
public String[] createFragments( IndexReader reader, int docId,
String fieldName, FieldFragList fieldFragList, int maxNumFragments )
throws IOException {
if( maxNumFragments < 0 )
throw new IllegalArgumentException( "maxNumFragments(" + maxNumFragments + ") must be positive number." );
List<WeightedFragInfo> fragInfos = getWeightedFragInfoList( fieldFragList.fragInfos );
List<String> fragments = new ArrayList<String>( maxNumFragments );
String[] values = getFieldValues( reader, docId, fieldName );
StringBuilder buffer = new StringBuilder();
int[] nextValueIndex = { 0 };
for( int n = 0; n < maxNumFragments && n < fragInfos.size(); n++ ){
WeightedFragInfo fragInfo = fragInfos.get( n );
fragments.add( makeFragment( buffer, nextValueIndex, values, fragInfo ) );
}
return fragments.toArray( new String[fragments.size()] );
}
protected String[] getFieldValues( IndexReader reader, int docId, String fieldName) throws IOException {
Document doc = reader.document( docId, new MapFieldSelector( new String[]{ fieldName } ) );
return doc.getValues( fieldName );
}
protected String makeFragment( StringBuilder buffer, int[] index, String[] values, WeightedFragInfo fragInfo ){
StringBuilder fragment = new StringBuilder();
final int s = fragInfo.startOffset;
String src = getFragmentSource( buffer, index, values, s, fragInfo.endOffset );
int srcIndex = 0;
for( SubInfo subInfo : fragInfo.subInfos ){
for( Toffs to : subInfo.termsOffsets ){
fragment.append( src.substring( srcIndex, to.startOffset - s ) ).append( getPreTag( subInfo.seqnum ) )
.append( src.substring( to.startOffset - s, to.endOffset - s ) ).append( getPostTag( subInfo.seqnum ) );
srcIndex = to.endOffset - s;
}
}
fragment.append( src.substring( srcIndex ) );
return fragment.toString();
}
protected String getFragmentSource( StringBuilder buffer, int[] index, String[] values,
int startOffset, int endOffset ){
while( buffer.length() < endOffset && index[0] < values.length ){
if( index[0] > 0 && values[index[0]].length() > 0 )
buffer.append( ' ' );
buffer.append( values[index[0]++] );
}
int eo = buffer.length() < endOffset ? buffer.length() : endOffset;
return buffer.substring( startOffset, eo );
}
protected String getPreTag( int num ){
return preTags.length > num ? preTags[num] : preTags[0];
}
protected String getPostTag( int num ){
return postTags.length > num ? postTags[num] : postTags[0];
}
}

137
contrib/fast-vector-highlighter/src/java/org/apache/lucene/search/vectorhighlight/FastVectorHighlighter.java Normal file
View File

@ -0,0 +1,137 @@
package org.apache.lucene.search.vectorhighlight;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.Query;
/**
* Another highlighter implementation.
*
*/
public class FastVectorHighlighter {
public static final boolean DEFAULT_PHRASE_HIGHLIGHT = true;
public static final boolean DEFAULT_FIELD_MATCH = true;
private final boolean phraseHighlight;
private final boolean fieldMatch;
private final FragListBuilder fragListBuilder;
private final FragmentsBuilder fragmentsBuilder;
/**
* the default constructor.
*/
public FastVectorHighlighter(){
this( DEFAULT_PHRASE_HIGHLIGHT, DEFAULT_FIELD_MATCH );
}
/**
* a constructor. Using SimpleFragListBuilder and ScoreOrderFragmentsBuilder.
*
* @param phraseHighlight true or false for phrase highlighting
* @param fieldMatch true of false for field matching
*/
public FastVectorHighlighter( boolean phraseHighlight, boolean fieldMatch ){
this( phraseHighlight, fieldMatch, new SimpleFragListBuilder(), new ScoreOrderFragmentsBuilder() );
}
/**
* a constructor. A FragListBuilder and a FragmentsBuilder can be specified (plugins).
*
* @param phraseHighlight true of false for phrase highlighting
* @param fieldMatch true of false for field matching
* @param fragListBuilder an instance of FragListBuilder
* @param fragmentsBuilder an instance of FragmentsBuilder
*/
public FastVectorHighlighter( boolean phraseHighlight, boolean fieldMatch,
FragListBuilder fragListBuilder, FragmentsBuilder fragmentsBuilder ){
this.phraseHighlight = phraseHighlight;
this.fieldMatch = fieldMatch;
this.fragListBuilder = fragListBuilder;
this.fragmentsBuilder = fragmentsBuilder;
}
/**
* create a FieldQuery object.
*
* @param query a query
* @return the created FieldQuery object
*/
public FieldQuery getFieldQuery( Query query ){
return new FieldQuery( query, phraseHighlight, fieldMatch );
}
/**
* return the best fragment.
*
* @param fieldQuery FieldQuery object
* @param reader IndexReader of the index
* @param docId document id to be highlighted
* @param fieldName field of the document to be highlighted
* @param fragCharSize the length (number of chars) of a fragment
* @return the best fragment (snippet) string
* @throws IOException
*/
public final String getBestFragment( final FieldQuery fieldQuery, IndexReader reader, int docId,
String fieldName, int fragCharSize ) throws IOException {
FieldFragList fieldFragList = getFieldFragList( fieldQuery, reader, docId, fieldName, fragCharSize );
return fragmentsBuilder.createFragment( reader, docId, fieldName, fieldFragList );
}
/**
* return the best fragments.
*
* @param fieldQuery FieldQuery object
* @param reader IndexReader of the index
* @param docId document id to be highlighted
* @param fieldName field of the document to be highlighted
* @param fragCharSize the length (number of chars) of a fragment
* @param maxNumFragments maximum number of fragments
* @return created fragments or null when no fragments created.
* size of the array can be less than maxNumFragments
* @throws IOException
*/
public final String[] getBestFragments( final FieldQuery fieldQuery, IndexReader reader, int docId,
String fieldName, int fragCharSize, int maxNumFragments ) throws IOException {
FieldFragList fieldFragList = getFieldFragList( fieldQuery, reader, docId, fieldName, fragCharSize );
return fragmentsBuilder.createFragments( reader, docId, fieldName, fieldFragList, maxNumFragments );
}
private FieldFragList getFieldFragList( final FieldQuery fieldQuery, IndexReader reader, int docId,
String fieldName, int fragCharSize ) throws IOException {
FieldTermStack fieldTermStack = new FieldTermStack( reader, docId, fieldName, fieldQuery );
FieldPhraseList fieldPhraseList = new FieldPhraseList( fieldTermStack, fieldQuery );
return fragListBuilder.createFieldFragList( fieldPhraseList, fragCharSize );
}
/**
* return whether phraseHighlight or not.
*
* @return
*/
public boolean isPhraseHighlight(){ return phraseHighlight; }
/**
* return whether fieldMatch or not.
*
* @return
*/
public boolean isFieldMatch(){ return fieldMatch; }
}

103
contrib/fast-vector-highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldFragList.java Normal file
View File

@ -0,0 +1,103 @@
package org.apache.lucene.search.vectorhighlight;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.util.ArrayList;
import java.util.List;
import org.apache.lucene.search.vectorhighlight.FieldPhraseList.WeightedPhraseInfo;
import org.apache.lucene.search.vectorhighlight.FieldPhraseList.WeightedPhraseInfo.Toffs;
/**
* FieldFragList has a list of "frag info" that is used by FragmentsBuilder class
* to create fragments (snippets).
*/
public class FieldFragList {
private final int fragCharSize;
List<WeightedFragInfo> fragInfos = new ArrayList<WeightedFragInfo>();
/**
* a constructor.
*
* @param fragCharSize the length (number of chars) of a fragment
*/
public FieldFragList( int fragCharSize ){
this.fragCharSize = fragCharSize;
}
/**
* convert the list of WeightedPhraseInfo to WeightedFragInfo, then add it to the fragInfos
*
* @param startOffset start offset of the fragment
* @param endOffset end offset of the fragment
* @param phraseInfoList list of WeightedPhraseInfo objects
*/
public void add( int startOffset, int endOffset, List<WeightedPhraseInfo> phraseInfoList ){
fragInfos.add( new WeightedFragInfo( startOffset, endOffset, phraseInfoList ) );
}
public static class WeightedFragInfo {
List<SubInfo> subInfos;
float totalBoost;
int startOffset;
int endOffset;
public WeightedFragInfo( int startOffset, int endOffset, List<WeightedPhraseInfo> phraseInfoList ){
this.startOffset = startOffset;
this.endOffset = endOffset;
subInfos = new ArrayList<SubInfo>();
for( WeightedPhraseInfo phraseInfo : phraseInfoList ){
SubInfo subInfo = new SubInfo( phraseInfo.text, phraseInfo.termsOffsets, phraseInfo.seqnum );
subInfos.add( subInfo );
totalBoost += phraseInfo.boost;
}
}
public String toString(){
StringBuilder sb = new StringBuilder();
sb.append( "subInfos=(" );
for( SubInfo si : subInfos )
sb.append( si.toString() );
sb.append( ")/" ).append( totalBoost ).append( '(' ).append( startOffset ).append( ',' ).append( endOffset ).append( ')' );
return sb.toString();
}
static class SubInfo {
final String text; // unnecessary member, just exists for debugging purpose
final List<Toffs> termsOffsets; // usually termsOffsets.size() == 1,
// but if position-gap > 1 and slop > 0 then size() could be greater than 1
int seqnum;
SubInfo( String text, List<Toffs> termsOffsets, int seqnum ){
this.text = text;
this.termsOffsets = termsOffsets;
this.seqnum = seqnum;
}
public String toString(){
StringBuilder sb = new StringBuilder();
sb.append( text ).append( '(' );
for( Toffs to : termsOffsets )
sb.append( to.toString() );
sb.append( ')' );
return sb.toString();
}
}
}
}

183
contrib/fast-vector-highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldPhraseList.java Normal file
View File

@ -0,0 +1,183 @@
package org.apache.lucene.search.vectorhighlight;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import org.apache.lucene.search.vectorhighlight.FieldQuery.QueryPhraseMap;
import org.apache.lucene.search.vectorhighlight.FieldTermStack.TermInfo;
/**
* FieldPhraseList has a list of WeightedPhraseInfo that is used by FragListBuilder
* to create a FieldFragList object.
*/
public class FieldPhraseList {
LinkedList<WeightedPhraseInfo> phraseList = new LinkedList<WeightedPhraseInfo>();
/**
* a constructor.
*
* @param fieldTermStack FieldTermStack object
* @param fieldQuery FieldQuery object
*/
public FieldPhraseList( FieldTermStack fieldTermStack, FieldQuery fieldQuery ){
final String field = fieldTermStack.getFieldName();
LinkedList<TermInfo> phraseCandidate = new LinkedList<TermInfo>();
QueryPhraseMap currMap = null;
QueryPhraseMap nextMap = null;
while( !fieldTermStack.isEmpty() ){
phraseCandidate.clear();
TermInfo ti = fieldTermStack.pop();
currMap = fieldQuery.getFieldTermMap( field, ti.getText() );
// if not found, discard top TermInfo from stack, then try next element
if( currMap == null ) continue;
// if found, search the longest phrase
phraseCandidate.add( ti );
while( true ){
ti = fieldTermStack.pop();
nextMap = null;
if( ti != null )
nextMap = currMap.getTermMap( ti.getText() );
if( ti == null || nextMap == null ){
if( ti != null )
fieldTermStack.push( ti );
if( currMap.isValidTermOrPhrase( phraseCandidate ) ){
addIfNoOverlap( new WeightedPhraseInfo( phraseCandidate, currMap.getBoost(), currMap.getTermOrPhraseNumber() ) );
}
else{
while( phraseCandidate.size() > 1 ){
fieldTermStack.push( phraseCandidate.removeLast() );
currMap = fieldQuery.searchPhrase( field, phraseCandidate );
if( currMap != null ){
addIfNoOverlap( new WeightedPhraseInfo( phraseCandidate, currMap.getBoost(), currMap.getTermOrPhraseNumber() ) );
break;
}
}
}
break;
}
else{
phraseCandidate.add( ti );
currMap = nextMap;
}
}
}
}
void addIfNoOverlap( WeightedPhraseInfo wpi ){
for( WeightedPhraseInfo existWpi : phraseList ){
if( existWpi.isOffsetOverlap( wpi ) ) return;
}
phraseList.add( wpi );
}
public static class WeightedPhraseInfo {
String text; // unnecessary member, just exists for debugging purpose
List<Toffs> termsOffsets; // usually termsOffsets.size() == 1,
// but if position-gap > 1 and slop > 0 then size() could be greater than 1
float boost; // query boost
int seqnum;
public WeightedPhraseInfo( LinkedList<TermInfo> terms, float boost ){
this( terms, boost, 0 );
}
public WeightedPhraseInfo( LinkedList<TermInfo> terms, float boost, int number ){
this.boost = boost;
this.seqnum = number;
termsOffsets = new ArrayList<Toffs>( terms.size() );
TermInfo ti = terms.get( 0 );
termsOffsets.add( new Toffs( ti.getStartOffset(), ti.getEndOffset() ) );
if( terms.size() == 1 ){
text = ti.getText();
return;
}
StringBuilder sb = new StringBuilder();
sb.append( ti.getText() );
int pos = ti.getPosition();
for( int i = 1; i < terms.size(); i++ ){
ti = terms.get( i );
sb.append( ti.getText() );
if( ti.getPosition() - pos == 1 ){
Toffs to = termsOffsets.get( termsOffsets.size() - 1 );
to.setEndOffset( ti.getEndOffset() );
}
else{
termsOffsets.add( new Toffs( ti.getStartOffset(), ti.getEndOffset() ) );
}
pos = ti.getPosition();
}
text = sb.toString();
}
public int getStartOffset(){
return termsOffsets.get( 0 ).startOffset;
}
public int getEndOffset(){
return termsOffsets.get( termsOffsets.size() - 1 ).endOffset;
}
public boolean isOffsetOverlap( WeightedPhraseInfo other ){
int so = getStartOffset();
int eo = getEndOffset();
int oso = other.getStartOffset();
int oeo = other.getEndOffset();
if( so <= oso && oso <= eo ) return true;
if( so <= oeo && oeo <= eo ) return true;
if( oso <= so && so <= oeo ) return true;
if( oso <= eo && eo <= oeo ) return true;
return false;
}
public String toString(){
StringBuilder sb = new StringBuilder();
sb.append( text ).append( '(' ).append( boost ).append( ")(" );
for( Toffs to : termsOffsets ){
sb.append( to );
}
sb.append( ')' );
return sb.toString();
}
public static class Toffs {
int startOffset;
int endOffset;
public Toffs( int startOffset, int endOffset ){
this.startOffset = startOffset;
this.endOffset = endOffset;
}
void setEndOffset( int endOffset ){
this.endOffset = endOffset;
}
public String toString(){
StringBuilder sb = new StringBuilder();
sb.append( '(' ).append( startOffset ).append( ',' ).append( endOffset ).append( ')' );
return sb.toString();
}
}
}
}

391
contrib/fast-vector-highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldQuery.java Normal file
View File

@ -0,0 +1,391 @@
package org.apache.lucene.search.vectorhighlight;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.PhraseQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.vectorhighlight.FieldTermStack.TermInfo;
/**
* FieldQuery breaks down query object into terms/phrases and keep
* them in QueryPhraseMap structure.
*/
public class FieldQuery {
final boolean fieldMatch;
// fieldMatch==true, Map<fieldName,QueryPhraseMap>
// fieldMatch==false, Map<null,QueryPhraseMap>
Map<String, QueryPhraseMap> rootMaps = new HashMap<String, QueryPhraseMap>();
// fieldMatch==true, Map<fieldName,setOfTermsInQueries>
// fieldMatch==false, Map<null,setOfTermsInQueries>
Map<String, Set<String>> termSetMap = new HashMap<String, Set<String>>();
int termOrPhraseNumber; // used for colored tag support
FieldQuery( Query query, boolean phraseHighlight, boolean fieldMatch ){
this.fieldMatch = fieldMatch;
Set<Query> flatQueries = new HashSet<Query>();
flatten( query, flatQueries );
saveTerms( flatQueries );
Collection<Query> expandQueries = expand( flatQueries );
for( Query flatQuery : expandQueries ){
QueryPhraseMap rootMap = getRootMap( flatQuery );
rootMap.add( flatQuery );
if( !phraseHighlight && flatQuery instanceof PhraseQuery ){
PhraseQuery pq = (PhraseQuery)flatQuery;
if( pq.getTerms().length > 1 ){
for( Term term : pq.getTerms() )
rootMap.addTerm( term, flatQuery.getBoost() );
}
}
}
}
void flatten( Query sourceQuery, Collection<Query> flatQueries ){
if( sourceQuery instanceof BooleanQuery ){
BooleanQuery bq = (BooleanQuery)sourceQuery;
for( BooleanClause clause : bq.getClauses() ){
if( !clause.isProhibited() )
flatten( clause.getQuery(), flatQueries );
}
}
else if( sourceQuery instanceof TermQuery ){
if( !flatQueries.contains( sourceQuery ) )
flatQueries.add( sourceQuery );
}
else if( sourceQuery instanceof PhraseQuery ){
if( !flatQueries.contains( sourceQuery ) ){
PhraseQuery pq = (PhraseQuery)sourceQuery;
if( pq.getTerms().length > 1 )
flatQueries.add( pq );
else if( pq.getTerms().length == 1 ){
flatQueries.add( new TermQuery( pq.getTerms()[0] ) );
}
}
}
// else discard queries
}
/*
* Create expandQueries from flatQueries.
*
* expandQueries := flatQueries + overlapped phrase queries
*
* ex1) flatQueries={a,b,c}
* => expandQueries={a,b,c}
* ex2) flatQueries={a,"b c","c d"}
* => expandQueries={a,"b c","c d","b c d"}
*/
Collection<Query> expand( Collection<Query> flatQueries ){
Set<Query> expandQueries = new HashSet<Query>();
for( Iterator<Query> i = flatQueries.iterator(); i.hasNext(); ){
Query query = i.next();
i.remove();
expandQueries.add( query );
if( !( query instanceof PhraseQuery ) ) continue;
for( Iterator<Query> j = flatQueries.iterator(); j.hasNext(); ){
Query qj = j.next();
if( !( qj instanceof PhraseQuery ) ) continue;
checkOverlap( expandQueries, (PhraseQuery)query, (PhraseQuery)qj );
}
}
return expandQueries;
}
/*
* Check if PhraseQuery A and B have overlapped part.
*
* ex1) A="a b", B="b c" => overlap; expandQueries={"a b c"}
* ex2) A="b c", B="a b" => overlap; expandQueries={"a b c"}
* ex3) A="a b", B="c d" => no overlap; expandQueries={}
*/
private void checkOverlap( Collection<Query> expandQueries, PhraseQuery a, PhraseQuery b ){
if( a.getSlop() != b.getSlop() ) return;
Term[] ats = a.getTerms();
Term[] bts = b.getTerms();
if( fieldMatch && !ats[0].field().equals( bts[0].field() ) ) return;
checkOverlap( expandQueries, ats, bts, a.getSlop(), a.getBoost() );
checkOverlap( expandQueries, bts, ats, b.getSlop(), b.getBoost() );
}
/*
* Check if src and dest have overlapped part and if it is, create PhraseQueries and add expandQueries.
*
* ex1) src="a b", dest="c d" => no overlap
* ex2) src="a b", dest="a b c" => no overlap
* ex3) src="a b", dest="b c" => overlap; expandQueries={"a b c"}
* ex4) src="a b c", dest="b c d" => overlap; expandQueries={"a b c d"}
* ex5) src="a b c", dest="b c" => no overlap
* ex6) src="a b c", dest="b" => no overlap
* ex7) src="a a a a", dest="a a a" => overlap;
* expandQueries={"a a a a a","a a a a a a"}
*/
private void checkOverlap( Collection<Query> expandQueries, Term[] src, Term[] dest, int slop, float boost ){
// beginning from 1 (not 0) is safe because that the PhraseQuery has multiple terms
// is guaranteed in flatten() method (if PhraseQuery has only one term, flatten()
// converts PhraseQuery to TermQuery)
for( int i = 1; i < src.length; i++ ){
boolean overlap = true;
for( int j = i; j < src.length; j++ ){
if( !src[j].text().equals( dest[j-i].text() ) ){
overlap = false;
break;
}
}
if( overlap && src.length - i < dest.length ){
PhraseQuery pq = new PhraseQuery();
for( Term srcTerm : src )
pq.add( srcTerm );
for( int k = src.length - i; k < dest.length; k++ ){
pq.add( new Term( src[0].field(), dest[k].text() ) );
}
pq.setSlop( slop );
pq.setBoost( boost );
if(!expandQueries.contains( pq ) )
expandQueries.add( pq );
}
}
}
QueryPhraseMap getRootMap( Query query ){
String key = getKey( query );
QueryPhraseMap map = rootMaps.get( key );
if( map == null ){
map = new QueryPhraseMap( this );
rootMaps.put( key, map );
}
return map;
}
/*
* Return 'key' string. 'key' is the field name of the Query.
* If not fieldMatch, 'key' will be null.
*/
private String getKey( Query query ){
if( !fieldMatch ) return null;
if( query instanceof TermQuery )
return ((TermQuery)query).getTerm().field();
else if ( query instanceof PhraseQuery ){
PhraseQuery pq = (PhraseQuery)query;
Term[] terms = pq.getTerms();
return terms[0].field();
}
else
throw new RuntimeException( "query \"" + query.toString() + "\" must be flatten first." );
}
/*
* Save the set of terms in the queries to termSetMap.
*
* ex1) q=name:john
* - fieldMatch==true
* termSetMap=Map<"name",Set<"john">>
* - fieldMatch==false
* termSetMap=Map<null,Set<"john">>
*
* ex2) q=name:john title:manager
* - fieldMatch==true
* termSetMap=Map<"name",Set<"john">,
* "title",Set<"manager">>
* - fieldMatch==false
* termSetMap=Map<null,Set<"john","manager">>
*
* ex3) q=name:"john lennon"
* - fieldMatch==true
* termSetMap=Map<"name",Set<"john","lennon">>
* - fieldMatch==false
* termSetMap=Map<null,Set<"john","lennon">>
*/
void saveTerms( Collection<Query> flatQueries ){
for( Query query : flatQueries ){
Set<String> termSet = getTermSet( query );
if( query instanceof TermQuery )
termSet.add( ((TermQuery)query).getTerm().text() );
else if( query instanceof PhraseQuery ){
for( Term term : ((PhraseQuery)query).getTerms() )
termSet.add( term.text() );
}
else
throw new RuntimeException( "query \"" + query.toString() + "\" must be flatten first." );
}
}
private Set<String> getTermSet( Query query ){
String key = getKey( query );
Set<String> set = termSetMap.get( key );
if( set == null ){
set = new HashSet<String>();
termSetMap.put( key, set );
}
return set;
}
Set<String> getTermSet( String field ){
return termSetMap.get( fieldMatch ? field : null );
}
/**
*
* @param fieldName
* @param term
* @return
*/
public QueryPhraseMap getFieldTermMap( String fieldName, String term ){
QueryPhraseMap rootMap = getRootMap( fieldName );
return rootMap == null ? null : rootMap.subMap.get( term );
}
/**
*
* @param fieldName
* @param phraseCandidate
* @return
*/
public QueryPhraseMap searchPhrase( String fieldName, final List<TermInfo> phraseCandidate ){
QueryPhraseMap root = getRootMap( fieldName );
if( root == null ) return null;
return root.searchPhrase( phraseCandidate );
}
private QueryPhraseMap getRootMap( String fieldName ){
return rootMaps.get( fieldMatch ? fieldName : null );
}
int nextTermOrPhraseNumber(){
return termOrPhraseNumber++;
}
public static class QueryPhraseMap {
boolean terminal;
int slop; // valid if terminal == true and phraseHighlight == true
float boost; // valid if terminal == true
int termOrPhraseNumber; // valid if terminal == true
FieldQuery fieldQuery;
Map<String, QueryPhraseMap> subMap = new HashMap<String, QueryPhraseMap>();
public QueryPhraseMap( FieldQuery fieldQuery ){
this.fieldQuery = fieldQuery;
}
void addTerm( Term term, float boost ){
QueryPhraseMap map = getOrNewMap( subMap, term.text() );
map.markTerminal( boost );
}
private QueryPhraseMap getOrNewMap( Map<String, QueryPhraseMap> subMap, String term ){
QueryPhraseMap map = subMap.get( term );
if( map == null ){
map = new QueryPhraseMap( fieldQuery );
subMap.put( term, map );
}
return map;
}
void add( Query query ){
if( query instanceof TermQuery ){
addTerm( ((TermQuery)query).getTerm(), query.getBoost() );
}
else if( query instanceof PhraseQuery ){
PhraseQuery pq = (PhraseQuery)query;
Term[] terms = pq.getTerms();
Map<String, QueryPhraseMap> map = subMap;
QueryPhraseMap qpm = null;
for( Term term : terms ){
qpm = getOrNewMap( map, term.text() );
map = qpm.subMap;
}
qpm.markTerminal( pq.getSlop(), pq.getBoost() );
}
else
throw new RuntimeException( "query \"" + query.toString() + "\" must be flatten first." );
}
public QueryPhraseMap getTermMap( String term ){
return subMap.get( term );
}
private void markTerminal( float boost ){
markTerminal( 0, boost );
}
private void markTerminal( int slop, float boost ){
this.terminal = true;
this.slop = slop;
this.boost = boost;
this.termOrPhraseNumber = fieldQuery.nextTermOrPhraseNumber();
}
public boolean isTerminal(){
return terminal;
}
public int getSlop(){
return slop;
}
public float getBoost(){
return boost;
}
public int getTermOrPhraseNumber(){
return termOrPhraseNumber;
}
public QueryPhraseMap searchPhrase( final List<TermInfo> phraseCandidate ){
QueryPhraseMap currMap = this;
for( TermInfo ti : phraseCandidate ){
currMap = currMap.subMap.get( ti.getText() );
if( currMap == null ) return null;
}
return currMap.isValidTermOrPhrase( phraseCandidate ) ? currMap : null;
}
public boolean isValidTermOrPhrase( final List<TermInfo> phraseCandidate ){
// check terminal
if( !terminal ) return false;
// if the candidate is a term, it is valid
if( phraseCandidate.size() == 1 ) return true;
// else check whether the candidate is valid phrase
// compare position-gaps between terms to slop
int pos = phraseCandidate.get( 0 ).getPosition();
for( int i = 1; i < phraseCandidate.size(); i++ ){
int nextPos = phraseCandidate.get( i ).getPosition();
if( Math.abs( nextPos - pos - 1 ) > slop ) return false;
pos = nextPos;
}
return true;
}
}
}

171
contrib/fast-vector-highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldTermStack.java Normal file
View File

@ -0,0 +1,171 @@
package org.apache.lucene.search.vectorhighlight;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import java.util.Collections;
import java.util.LinkedList;
import java.util.Set;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.WhitespaceAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.Index;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.Field.TermVector;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.TermFreqVector;
import org.apache.lucene.index.TermPositionVector;
import org.apache.lucene.index.TermVectorOffsetInfo;
import org.apache.lucene.index.IndexWriter.MaxFieldLength;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.search.Query;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.RAMDirectory;
/**
* <code>FieldTermStack</code> is a stack that keeps query terms in the specified field
* of the document to be highlighted.
*/
public class FieldTermStack {
private final String fieldName;
LinkedList<TermInfo> termList = new LinkedList<TermInfo>();
public static void main( String[] args ) throws Exception {
Analyzer analyzer = new WhitespaceAnalyzer();
QueryParser parser = new QueryParser( "f", analyzer );
Query query = parser.parse( "a x:b" );
FieldQuery fieldQuery = new FieldQuery( query, true, false );
Directory dir = new RAMDirectory();
IndexWriter writer = new IndexWriter( dir, analyzer, MaxFieldLength.LIMITED );
Document doc = new Document();
doc.add( new Field( "f", "a a a b b c a b b c d e f", Store.YES, Index.ANALYZED, TermVector.WITH_POSITIONS_OFFSETS ) );
doc.add( new Field( "f", "b a b a f", Store.YES, Index.ANALYZED, TermVector.WITH_POSITIONS_OFFSETS ) );
writer.addDocument( doc );
writer.close();
IndexReader reader = IndexReader.open( dir );
FieldTermStack ftl = new FieldTermStack( reader, 0, "f", fieldQuery );
reader.close();
}
/**
* a constructor.
*
* @param reader IndexReader of the index
* @param docId document id to be highlighted
* @param fieldName field of the document to be highlighted
* @param fieldQuery FieldQuery object
* @throws IOException
*/
public FieldTermStack( IndexReader reader, int docId, String fieldName, final FieldQuery fieldQuery ) throws IOException {
this.fieldName = fieldName;
TermFreqVector tfv = reader.getTermFreqVector( docId, fieldName );
if( tfv == null ) return; // just return to make null snippets
TermPositionVector tpv = null;
try{
tpv = (TermPositionVector)tfv;
}
catch( ClassCastException e ){
return; // just return to make null snippets
}
Set<String> termSet = fieldQuery.getTermSet( fieldName );
// just return to make null snippet if un-matched fieldName specified when fieldMatch == true
if( termSet == null ) return;
for( String term : tpv.getTerms() ){
if( !termSet.contains( term ) ) continue;
int index = tpv.indexOf( term );
TermVectorOffsetInfo[] tvois = tpv.getOffsets( index );
if( tvois == null ) return; // just return to make null snippets
int[] poss = tpv.getTermPositions( index );
if( poss == null ) return; // just return to make null snippets
for( int i = 0; i < tvois.length; i++ )
termList.add( new TermInfo( term, tvois[i].getStartOffset(), tvois[i].getEndOffset(), poss[i] ) );
}
// sort by position
Collections.sort( termList );
}
/**
* @return field name
*/
public String getFieldName(){
return fieldName;
}
/**
* @return the top TermInfo object of the stack
*/
public TermInfo pop(){
return termList.poll();
}
/**
* @param termInfo the TermInfo object to be put on the top of the stack
*/
public void push( TermInfo termInfo ){
// termList.push( termInfo ); // avoid Java 1.6 feature
termList.addFirst( termInfo );
}
/**
* to know whether the stack is empty
*
* @return true if the stack is empty, false if not
*/
public boolean isEmpty(){
return termList == null || termList.size() == 0;
}
public static class TermInfo implements Comparable<TermInfo>{
final String text;
final int startOffset;
final int endOffset;
final int position;
TermInfo( String text, int startOffset, int endOffset, int position ){
this.text = text;
this.startOffset = startOffset;
this.endOffset = endOffset;
this.position = position;
}
public String getText(){ return text; }
public int getStartOffset(){ return startOffset; }
public int getEndOffset(){ return endOffset; }
public int getPosition(){ return position; }
public String toString(){
StringBuilder sb = new StringBuilder();
sb.append( text ).append( '(' ).append(startOffset).append( ',' ).append( endOffset ).append( ',' ).append( position ).append( ')' );
return sb.toString();
}
public int compareTo( TermInfo o ) {
return ( this.position - o.position );
}
}
}

34
contrib/fast-vector-highlighter/src/java/org/apache/lucene/search/vectorhighlight/FragListBuilder.java Normal file
View File

@ -0,0 +1,34 @@
package org.apache.lucene.search.vectorhighlight;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* FragListBuilder is an interface for FieldFragList builder classes.
* A FragListBuilder class can be plugged in to Highlighter.
*/
public interface FragListBuilder {
/**
* create a FieldFragList.
*
* @param fieldPhraseList FieldPhraseList object
* @param fragCharSize the length (number of chars) of a fragment
* @return the created FieldFragList object
*/
public FieldFragList createFieldFragList( FieldPhraseList fieldPhraseList, int fragCharSize );
}

57
contrib/fast-vector-highlighter/src/java/org/apache/lucene/search/vectorhighlight/FragmentsBuilder.java Normal file
View File

@ -0,0 +1,57 @@
package org.apache.lucene.search.vectorhighlight;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import org.apache.lucene.index.IndexReader;
/**
* FragmentsBuilder is an interface for fragments (snippets) builder classes.
* A FragmentsBuilder class can be plugged in to Highlighter.
*/
public interface FragmentsBuilder {
/**
* create a fragment.
*
* @param reader IndexReader of the index
* @param docId document id to be highlighted
* @param fieldName field of the document to be highlighted
* @param fieldFragList FieldFragList object
* @return a created fragment or null when no fragment created
* @throws IOException
*/
public String createFragment( IndexReader reader, int docId, String fieldName,
FieldFragList fieldFragList ) throws IOException;
/**
* create multiple fragments.
*
* @param reader IndexReader of the index
* @param docId document id to be highlighter
* @param fieldName field of the document to be highlighted
* @param fieldFragList FieldFragList object
* @param maxNumFragments maximum number of fragments
* @return created fragments or null when no fragments created.
* size of the array can be less than maxNumFragments
* @throws IOException
*/
public String[] createFragments( IndexReader reader, int docId, String fieldName,
FieldFragList fieldFragList, int maxNumFragments ) throws IOException;
}

69
contrib/fast-vector-highlighter/src/java/org/apache/lucene/search/vectorhighlight/ScoreOrderFragmentsBuilder.java Normal file
View File

@ -0,0 +1,69 @@
package org.apache.lucene.search.vectorhighlight;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import org.apache.lucene.search.vectorhighlight.FieldFragList.WeightedFragInfo;
/**
* An implementation of FragmentsBuilder that outputs score-order fragments.
*/
public class ScoreOrderFragmentsBuilder extends BaseFragmentsBuilder {
/**
* a constructor.
*/
public ScoreOrderFragmentsBuilder(){
super();
}
/**
* a constructor.
*
* @param preTags aray of pre-tags for markup terms.
* @param postTags array of post-tags for markup terms.
*/
public ScoreOrderFragmentsBuilder( String[] preTags, String[] postTags ){
super( preTags, postTags );
}
/**
* Sort by score the list of WeightedFragInfo
*/
public List<WeightedFragInfo> getWeightedFragInfoList( List<WeightedFragInfo> src ) {
Collections.sort( src, new ScoreComparator() );
return src;
}
public static class ScoreComparator implements Comparator<WeightedFragInfo> {
public int compare( WeightedFragInfo o1, WeightedFragInfo o2 ) {
if( o1.totalBoost > o2.totalBoost ) return -1;
else if( o1.totalBoost < o2.totalBoost ) return 1;
// if same score then check startOffset
else{
if( o1.startOffset < o2.startOffset ) return -1;
else if( o1.startOffset > o2.startOffset ) return 1;
}
return 0;
}
}
}

82
contrib/fast-vector-highlighter/src/java/org/apache/lucene/search/vectorhighlight/SimpleFragListBuilder.java Normal file
View File

@ -0,0 +1,82 @@
package org.apache.lucene.search.vectorhighlight;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import org.apache.lucene.search.vectorhighlight.FieldPhraseList.WeightedPhraseInfo;
/**
* A simple implementation of FragListBuilder.
*/
public class SimpleFragListBuilder implements FragListBuilder {
public static final int MARGIN = 6;
public static final int MIN_FRAG_CHAR_SIZE = MARGIN * 3;
public FieldFragList createFieldFragList(FieldPhraseList fieldPhraseList, int fragCharSize) {
if( fragCharSize < MIN_FRAG_CHAR_SIZE )
throw new IllegalArgumentException( "fragCharSize(" + fragCharSize + ") is too small. It must be " +
MIN_FRAG_CHAR_SIZE + " or higher." );
FieldFragList ffl = new FieldFragList( fragCharSize );
List<WeightedPhraseInfo> wpil = new ArrayList<WeightedPhraseInfo>();
Iterator<WeightedPhraseInfo> ite = fieldPhraseList.phraseList.iterator();
WeightedPhraseInfo phraseInfo = null;
int startOffset = 0;
boolean taken = false;
while( true ){
if( !taken ){
if( !ite.hasNext() ) break;
phraseInfo = ite.next();
}
taken = false;
if( phraseInfo == null ) break;
// if the phrase violates the border of previous fragment, discard it and try next phrase
if( phraseInfo.getStartOffset() < startOffset ) continue;
wpil.clear();
wpil.add( phraseInfo );
int st = phraseInfo.getStartOffset() - MARGIN < startOffset ?
startOffset : phraseInfo.getStartOffset() - MARGIN;
int en = st + fragCharSize;
startOffset = en;
while( true ){
if( ite.hasNext() ){
phraseInfo = ite.next();
taken = true;
if( phraseInfo == null ) break;
}
else
break;
if( phraseInfo.getEndOffset() <= en )
wpil.add( phraseInfo );
else
break;
}
ffl.add( st, en, wpil );
}
return ffl;
}
}

53
contrib/fast-vector-highlighter/src/java/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilder.java Normal file
View File

@ -0,0 +1,53 @@
package org.apache.lucene.search.vectorhighlight;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.util.List;
import org.apache.lucene.search.vectorhighlight.FieldFragList.WeightedFragInfo;
/**
* A simple implementation of FragmentsBuilder.
*
*/
public class SimpleFragmentsBuilder extends BaseFragmentsBuilder {
/**
* a constructor.
*/
public SimpleFragmentsBuilder() {
super();
}
/**
* a constructor.
*
* @param preTags array of pre-tags for markup terms.
* @param postTags array of post-tags for markup terms.
*/
public SimpleFragmentsBuilder( String[] preTags, String[] postTags ) {
super( preTags, postTags );
}
/**
* do nothing. return the source list.
*/
public List<WeightedFragInfo> getWeightedFragInfoList( List<WeightedFragInfo> src ) {
return src;
}
}

126
contrib/fast-vector-highlighter/src/java/org/apache/lucene/search/vectorhighlight/package.html Normal file
View File

@ -0,0 +1,126 @@
<html>
<body>
This is an another highlighter implementation.
<h2>Features</h2>
<ul>
<li>fast for large docs</li>
<li>support N-gram fields</li>
<li>support phrase-unit highlighting with slops</li>
<li>need Java 1.5</li>
<li>highlight fields need to be TermVector.WITH_POSITIONS_OFFSETS</li>
<li>take into account query boost to score fragments</li>
<li>support colored highlight tags</li>
<li>pluggable FragListBuilder</li>
<li>pluggable FragmentsBuilder</li>
</ul>
<h2>Algorithm</h2>
<p>To explain the algorithm, let's use the following sample text
(to be highlighted) and user query:</p>
<table border=1>
<tr>
<td><b>Sample Text</b></td>
<td>Lucene is a search engine library.</td>
</tr>
<tr>
<td><b>User Query</b></td>
<td>Lucene^2 OR "search library"~1</td>
</tr>
</table>
<p>The user query is a BooleanQuery that consists of TermQuery("Lucene")
with boost of 2 and PhraseQuery("search library") with slop of 1.</p>
<p>For your convenience, here is the offsets and positions info of the
sample text.</p>
<pre>
+--------+-----------------------------------+
| | 1111111111222222222233333|
| offset|01234567890123456789012345678901234|
+--------+-----------------------------------+
|document|Lucene is a search engine library. |
+--------*-----------------------------------+
|position|0 1 2 3 4 5 |
+--------*-----------------------------------+
</pre>
<h3>Step 1.</h3>
<p>In Step 1, Fast Vector Highlighter generates {@link org.apache.lucene.search.vectorhighlight.FieldQuery.QueryPhraseMap} from the user query.
<code>QueryPhraseMap</code> consists of the following members:</p>
<pre>
public class QueryPhraseMap {
boolean terminal;
int slop; // valid if terminal == true and phraseHighlight == true
float boost; // valid if terminal == true
Map&lt;String, QueryPhraseMap&gt; subMap;
}
</pre>
<p><code>QueryPhraseMap</code> has subMap. The key of the subMap is a term
text in the user query and the value is a subsequent <code>QueryPhraseMap</code>.
If the query is a term (not phrase), then the subsequent <code>QueryPhraseMap</code>
is marked as terminal. If the query is a phrase, then the subsequent <code>QueryPhraseMap</code>
is not a terminal and it has the next term text in the phrase.</p>
<p>From the sample user query, the following <code>QueryPhraseMap</code>
will be generated:</p>
<pre>
QueryPhraseMap
+--------+-+ +-------+-+
|"Lucene"|o+->|boost=2|*| * : terminal
+--------+-+ +-------+-+
+--------+-+ +---------+-+ +-------+------+-+
|"search"|o+->|"library"|o+->|boost=1|slop=1|*|
+--------+-+ +---------+-+ +-------+------+-+
</pre>
<h3>Step 2.</h3>
<p>In Step 2, Fast Vector Highlighter generates {@link org.apache.lucene.search.vectorhighlight.FieldTermStack}. Fast Vector Highlighter uses {@link org.apache.lucene.index.TermFreqVector} data
(must be stored {@link org.apache.lucene.document.Field.TermVector#WITH_POSITIONS_OFFSETS})
to generate it. <code>FieldTermStack</code> keeps the terms in the user query.
Therefore, in this sample case, Fast Vector Highlighter generates the following <code>FieldTermStack</code>:</p>
<pre>
FieldTermStack
+------------------+
|"Lucene"(0,6,0) |
+------------------+
|"search"(12,18,3) |
+------------------+
|"library"(26,33,5)|
+------------------+
where : "termText"(startOffset,endOffset,position)
</pre>
<h3>Step 3.</h3>
<p>In Step 3, Fast Vector Highlighter generates {@link org.apache.lucene.search.vectorhighlight.FieldPhraseList}
by reference to <code>QueryPhraseMap</code> and <code>FieldTermStack</code>.</p>
<pre>
FieldPhraseList
+----------------+-----------------+---+
|"Lucene" |[(0,6)] |w=2|
+----------------+-----------------+---+
|"search library"|[(12,18),(26,33)]|w=1|
+----------------+-----------------+---+
</pre>
<p>The type of each entry is <code>WeightedPhraseInfo</code> that consists of
an array of terms offsets and weight. The weight (Fast Vector Highlighter uses query boost to
calculate the weight) will be taken into account when Fast Vector Highlighter creates
{@link org.apache.lucene.search.vectorhighlight.FieldFragList} in the next step.</p>
<h3>Step 4.</h3>
<p>In Step 4, Fast Vector Highlighter creates <code>FieldFragList</code> by reference to
<code>FieldPhraseList</code>. In this sample case, the following
<code>FieldFragList</code> will be generated:</p>
<pre>
FieldFragList
+---------------------------------+
|"Lucene"[(0,6)] |
|"search library"[(12,18),(26,33)]|
|totalBoost=3 |
+---------------------------------+
</pre>
<h3>Step 5.</h3>
<p>In Step 5, by using <code>FieldFragList</code> and the field stored data,
Fast Vector Highlighter creates highlighted snippets!</p>
</body>
</html>

345
contrib/fast-vector-highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java Normal file
View File

@ -0,0 +1,345 @@
package org.apache.lucene.search.vectorhighlight;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import java.io.Reader;
import java.util.Collection;
import junit.framework.TestCase;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.WhitespaceAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.Index;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.Field.TermVector;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.IndexWriter.MaxFieldLength;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.search.PhraseQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.RAMDirectory;
public abstract class AbstractTestCase extends TestCase {
protected final String F = "f";
protected final String F1 = "f1";
protected final String F2 = "f2";
protected Directory dir;
protected Analyzer analyzerW;
protected Analyzer analyzerB;
protected IndexReader reader;
protected QueryParser paW;
protected QueryParser paB;
protected static final String[] shortMVValues = {
"a b c",
"", // empty data in multi valued field
"d e"
};
protected static final String[] longMVValues = {
"Followings are the examples of customizable parameters and actual examples of customization:",
"The most search engines use only one of these methods. Even the search engines that says they can use the both methods basically"
};
// test data for LUCENE-1448 bug
protected static final String[] biMVValues = {
"\nLucene/Solr does not require such additional hardware.",
"\nWhen you talk about processing speed, the"
};
protected void setUp() throws Exception {
analyzerW = new WhitespaceAnalyzer();
analyzerB = new BigramAnalyzer();
paW = new QueryParser( F, analyzerW );
paB = new QueryParser( F, analyzerB );
dir = new RAMDirectory();
}
protected void tearDown() throws Exception {
if( reader != null ){
reader.close();
reader = null;
}
}
protected Query tq( String text ){
return tq( 1F, text );
}
protected Query tq( float boost, String text ){
return tq( boost, F, text );
}
protected Query tq( String field, String text ){
return tq( 1F, field, text );
}
protected Query tq( float boost, String field, String text ){
Query query = new TermQuery( new Term( field, text ) );
query.setBoost( boost );
return query;
}
protected Query pqF( String... texts ){
return pqF( 1F, texts );
}
protected Query pqF( float boost, String... texts ){
return pqF( boost, 0, texts );
}
protected Query pqF( float boost, int slop, String... texts ){
return pq( boost, slop, F, texts );
}
protected Query pq( String field, String... texts ){
return pq( 1F, 0, field, texts );
}
protected Query pq( float boost, String field, String... texts ){
return pq( boost, 0, field, texts );
}
protected Query pq( float boost, int slop, String field, String... texts ){
PhraseQuery query = new PhraseQuery();
for( String text : texts ){
query.add( new Term( field, text ) );
}
query.setBoost( boost );
query.setSlop( slop );
return query;
}
protected void assertCollectionQueries( Collection<Query> actual, Query... expected ){
assertEquals( expected.length, actual.size() );
for( Query query : expected ){
assertTrue( actual.contains( query ) );
}
}
static class BigramAnalyzer extends Analyzer {
public TokenStream tokenStream(String fieldName, Reader reader) {
return new BasicNGramTokenizer( reader );
}
}
static class BasicNGramTokenizer extends Tokenizer {
public static final int DEFAULT_N_SIZE = 2;
public static final String DEFAULT_DELIMITERS = " \t\n.,";
private final int n;
private final String delimiters;
private int startTerm;
private int lenTerm;
private int startOffset;
private int nextStartOffset;
private int ch;
private String snippet;
private StringBuilder snippetBuffer;
private static final int BUFFER_SIZE = 4096;
private char[] charBuffer;
private int charBufferIndex;
private int charBufferLen;
public BasicNGramTokenizer( Reader in ){
this( in, DEFAULT_N_SIZE );
}
public BasicNGramTokenizer( Reader in, int n ){
this( in, n, DEFAULT_DELIMITERS );
}
public BasicNGramTokenizer( Reader in, String delimiters ){
this( in, DEFAULT_N_SIZE, delimiters );
}
public BasicNGramTokenizer( Reader in, int n, String delimiters ){
super(in);
this.n = n;
this.delimiters = delimiters;
startTerm = 0;
nextStartOffset = 0;
snippet = null;
snippetBuffer = new StringBuilder();
charBuffer = new char[BUFFER_SIZE];
charBufferIndex = BUFFER_SIZE;
charBufferLen = 0;
ch = 0;
}
public Token next( Token reusableToken ) throws IOException {
if( !getNextPartialSnippet() )
return null;
reusableToken.reinit( snippet, startTerm, lenTerm, startOffset, startOffset + lenTerm );
return reusableToken;
}
public int getFinalOffset() {
return nextStartOffset;
}
protected boolean getNextPartialSnippet() throws IOException {
if( snippet != null && snippet.length() >= startTerm + 1 + n ){
startTerm++;
startOffset++;
lenTerm = n;
return true;
}
return getNextSnippet();
}
protected boolean getNextSnippet() throws IOException {
startTerm = 0;
startOffset = nextStartOffset;
snippetBuffer.delete( 0, snippetBuffer.length() );
while( true ){
if( ch != -1 )
ch = readCharFromBuffer();
if( ch == -1 ) break;
else if( !isDelimiter( ch ) )
snippetBuffer.append( (char)ch );
else if( snippetBuffer.length() > 0 )
break;
else
startOffset++;
}
if( snippetBuffer.length() == 0 )
return false;
snippet = snippetBuffer.toString();
lenTerm = snippet.length() >= n ? n : snippet.length();
return true;
}
protected int readCharFromBuffer() throws IOException {
if( charBufferIndex >= charBufferLen ){
charBufferLen = input.read( charBuffer );
if( charBufferLen == -1 ){
return -1;
}
charBufferIndex = 0;
}
int c = (int)charBuffer[charBufferIndex++];
nextStartOffset++;
return c;
}
protected boolean isDelimiter( int c ){
return delimiters.indexOf( c ) >= 0;
}
}
protected void make1d1fIndex( String value ) throws Exception {
make1dmfIndex( value );
}
protected void make1d1fIndexB( String value ) throws Exception {
make1dmfIndexB( value );
}
protected void make1dmfIndex( String... values ) throws Exception {
make1dmfIndex( analyzerW, values );
}
protected void make1dmfIndexB( String... values ) throws Exception {
make1dmfIndex( analyzerB, values );
}
protected void make1dmfIndex( Analyzer analyzer, String... values ) throws Exception {
IndexWriter writer = new IndexWriter( dir, analyzer, true, MaxFieldLength.LIMITED );
Document doc = new Document();
for( String value: values )
doc.add( new Field( F, value, Store.YES, Index.ANALYZED, TermVector.WITH_POSITIONS_OFFSETS ) );
writer.addDocument( doc );
writer.close();
reader = IndexReader.open( dir );
}
protected void makeIndexShortMV() throws Exception {
// 012345
// "a b c"
// 0 1 2
// ""
// 6789
// "d e"
// 3 4
make1dmfIndex( shortMVValues );
}
protected void makeIndexLongMV() throws Exception {
// 11111111112222222222333333333344444444445555555555666666666677777777778888888888999
// 012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012
// Followings are the examples of customizable parameters and actual examples of customization:
// 0 1 2 3 4 5 6 7 8 9 10 11
// 1 2
// 999999900000000001111111111222222222233333333334444444444555555555566666666667777777777888888888899999999990000000000111111111122
// 345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901
// The most search engines use only one of these methods. Even the search engines that says they can use the both methods basically
// 12 13 (14) (15) 16 17 18 19 20 21 22 23 (24) (25) 26 27 28 29 30 31 32 33 34
make1dmfIndex( longMVValues );
}
protected void makeIndexLongMVB() throws Exception {
// "*" ... LF
// 1111111111222222222233333333334444444444555555
// 01234567890123456789012345678901234567890123456789012345
// *Lucene/Solr does not require such additional hardware.
// Lu 0 do 10 re 15 su 21 na 31
// uc 1 oe 11 eq 16 uc 22 al 32
// ce 2 es 12 qu 17 ch 23 ha 33
// en 3 no 13 ui 18 ad 24 ar 34
// ne 4 ot 14 ir 19 dd 25 rd 35
// e/ 5 re 20 di 26 dw 36
// /S 6 it 27 wa 37
// So 7 ti 28 ar 38
// ol 8 io 29 re 39
// lr 9 on 30
// 5555666666666677777777778888888888999999999
// 6789012345678901234567890123456789012345678
// *When you talk about processing speed, the
// Wh 40 ab 48 es 56 th 65
// he 41 bo 49 ss 57 he 66
// en 42 ou 50 si 58
// yo 43 ut 51 in 59
// ou 44 pr 52 ng 60
// ta 45 ro 53 sp 61
// al 46 oc 54 pe 62
// lk 47 ce 55 ee 63
// ed 64
make1dmfIndexB( biMVValues );
}
}

182
contrib/fast-vector-highlighter/src/test/org/apache/lucene/search/vectorhighlight/FieldPhraseListTest.java Normal file
View File

@ -0,0 +1,182 @@
package org.apache.lucene.search.vectorhighlight;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.BooleanClause.Occur;
public class FieldPhraseListTest extends AbstractTestCase {
public void test1TermIndex() throws Exception {
make1d1fIndex( "a" );
FieldQuery fq = new FieldQuery( tq( "a" ), true, true );
FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
FieldPhraseList fpl = new FieldPhraseList( stack, fq );
assertEquals( 1, fpl.phraseList.size() );
assertEquals( "a(1.0)((0,1))", fpl.phraseList.get( 0 ).toString() );
fq = new FieldQuery( tq( "b" ), true, true );
stack = new FieldTermStack( reader, 0, F, fq );
fpl = new FieldPhraseList( stack, fq );
assertEquals( 0, fpl.phraseList.size() );
}
public void test2TermsIndex() throws Exception {
make1d1fIndex( "a a" );
FieldQuery fq = new FieldQuery( tq( "a" ), true, true );
FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
FieldPhraseList fpl = new FieldPhraseList( stack, fq );
assertEquals( 2, fpl.phraseList.size() );
assertEquals( "a(1.0)((0,1))", fpl.phraseList.get( 0 ).toString() );
assertEquals( "a(1.0)((2,3))", fpl.phraseList.get( 1 ).toString() );
}
public void test1PhraseIndex() throws Exception {
make1d1fIndex( "a b" );
FieldQuery fq = new FieldQuery( pqF( "a", "b" ), true, true );
FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
FieldPhraseList fpl = new FieldPhraseList( stack, fq );
assertEquals( 1, fpl.phraseList.size() );
assertEquals( "ab(1.0)((0,3))", fpl.phraseList.get( 0 ).toString() );
fq = new FieldQuery( tq( "b" ), true, true );
stack = new FieldTermStack( reader, 0, F, fq );
fpl = new FieldPhraseList( stack, fq );
assertEquals( 1, fpl.phraseList.size() );
assertEquals( "b(1.0)((2,3))", fpl.phraseList.get( 0 ).toString() );
}
public void test1PhraseIndexB() throws Exception {
// 01 12 23 34 45 56 67 78 (offsets)
// bb|bb|ba|ac|cb|ba|ab|bc
// 0 1 2 3 4 5 6 7 (positions)
make1d1fIndexB( "bbbacbabc" );
FieldQuery fq = new FieldQuery( pqF( "ba", "ac" ), true, true );
FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
FieldPhraseList fpl = new FieldPhraseList( stack, fq );
assertEquals( 1, fpl.phraseList.size() );
assertEquals( "baac(1.0)((2,5))", fpl.phraseList.get( 0 ).toString() );
}
public void test2Terms1PhraseIndex() throws Exception {
make1d1fIndex( "c a a b" );
// phraseHighlight = true
FieldQuery fq = new FieldQuery( pqF( "a", "b" ), true, true );
FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
FieldPhraseList fpl = new FieldPhraseList( stack, fq );
assertEquals( 1, fpl.phraseList.size() );
assertEquals( "ab(1.0)((4,7))", fpl.phraseList.get( 0 ).toString() );
// phraseHighlight = false
fq = new FieldQuery( pqF( "a", "b" ), false, true );
stack = new FieldTermStack( reader, 0, F, fq );
fpl = new FieldPhraseList( stack, fq );
assertEquals( 2, fpl.phraseList.size() );
assertEquals( "a(1.0)((2,3))", fpl.phraseList.get( 0 ).toString() );
assertEquals( "ab(1.0)((4,7))", fpl.phraseList.get( 1 ).toString() );
}
public void testPhraseSlop() throws Exception {
make1d1fIndex( "c a a b c" );
FieldQuery fq = new FieldQuery( pqF( 2F, 1, "a", "c" ), true, true );
FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
FieldPhraseList fpl = new FieldPhraseList( stack, fq );
assertEquals( 1, fpl.phraseList.size() );
assertEquals( "ac(2.0)((4,5)(8,9))", fpl.phraseList.get( 0 ).toString() );
assertEquals( 4, fpl.phraseList.get( 0 ).getStartOffset() );
assertEquals( 9, fpl.phraseList.get( 0 ).getEndOffset() );
}
public void test2PhrasesOverlap() throws Exception {
make1d1fIndex( "d a b c d" );
BooleanQuery query = new BooleanQuery();
query.add( pqF( "a", "b" ), Occur.SHOULD );
query.add( pqF( "b", "c" ), Occur.SHOULD );
FieldQuery fq = new FieldQuery( query, true, true );
FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
FieldPhraseList fpl = new FieldPhraseList( stack, fq );
assertEquals( 1, fpl.phraseList.size() );
assertEquals( "abc(1.0)((2,7))", fpl.phraseList.get( 0 ).toString() );
}
public void test3TermsPhrase() throws Exception {
make1d1fIndex( "d a b a b c d" );
FieldQuery fq = new FieldQuery( pqF( "a", "b", "c" ), true, true );
FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
FieldPhraseList fpl = new FieldPhraseList( stack, fq );
assertEquals( 1, fpl.phraseList.size() );
assertEquals( "abc(1.0)((6,11))", fpl.phraseList.get( 0 ).toString() );
}
public void testSearchLongestPhrase() throws Exception {
make1d1fIndex( "d a b d c a b c" );
BooleanQuery query = new BooleanQuery();
query.add( pqF( "a", "b" ), Occur.SHOULD );
query.add( pqF( "a", "b", "c" ), Occur.SHOULD );
FieldQuery fq = new FieldQuery( query, true, true );
FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
FieldPhraseList fpl = new FieldPhraseList( stack, fq );
assertEquals( 2, fpl.phraseList.size() );
assertEquals( "ab(1.0)((2,5))", fpl.phraseList.get( 0 ).toString() );
assertEquals( "abc(1.0)((10,15))", fpl.phraseList.get( 1 ).toString() );
}
public void test1PhraseShortMV() throws Exception {
makeIndexShortMV();
FieldQuery fq = new FieldQuery( tq( "d" ), true, true );
FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
FieldPhraseList fpl = new FieldPhraseList( stack, fq );
assertEquals( 1, fpl.phraseList.size() );
assertEquals( "d(1.0)((6,7))", fpl.phraseList.get( 0 ).toString() );
}
public void test1PhraseLongMV() throws Exception {
makeIndexLongMV();
FieldQuery fq = new FieldQuery( pqF( "search", "engines" ), true, true );
FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
FieldPhraseList fpl = new FieldPhraseList( stack, fq );
assertEquals( 2, fpl.phraseList.size() );
assertEquals( "searchengines(1.0)((102,116))", fpl.phraseList.get( 0 ).toString() );
assertEquals( "searchengines(1.0)((157,171))", fpl.phraseList.get( 1 ).toString() );
}
/*
* ----------------------------------
* THIS TEST DEPENDS ON LUCENE-1448
* UNCOMMENT WHEN IT IS COMMITTED.
* ----------------------------------
public void test1PhraseLongMVB() throws Exception {
makeIndexLongMVB();
FieldQuery fq = new FieldQuery( pqF( "sp", "pe", "ee", "ed" ), true, true ); // "speed" -(2gram)-> "sp","pe","ee","ed"
FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
FieldPhraseList fpl = new FieldPhraseList( stack, fq );
assertEquals( 1, fpl.phraseList.size() );
assertEquals( "sppeeeed(1.0)((88,93))", fpl.phraseList.get( 0 ).toString() );
}
*/
}

822
contrib/fast-vector-highlighter/src/test/org/apache/lucene/search/vectorhighlight/FieldQueryTest.java Normal file
View File

@ -0,0 +1,822 @@
package org.apache.lucene.search.vectorhighlight;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.vectorhighlight.FieldQuery.QueryPhraseMap;
import org.apache.lucene.search.vectorhighlight.FieldTermStack.TermInfo;
public class FieldQueryTest extends AbstractTestCase {
public void testFlattenBoolean() throws Exception {
Query query = paW.parse( "A AND B OR C NOT (D AND E)" );
FieldQuery fq = new FieldQuery( query, true, true );
Set<Query> flatQueries = new HashSet<Query>();
fq.flatten( query, flatQueries );
assertCollectionQueries( flatQueries, tq( "A" ), tq( "B" ), tq( "C" ) );
}
public void testFlattenTermAndPhrase() throws Exception {
Query query = paW.parse( "A AND \"B C\"" );
FieldQuery fq = new FieldQuery( query, true, true );
Set<Query> flatQueries = new HashSet<Query>();
fq.flatten( query, flatQueries );
assertCollectionQueries( flatQueries, tq( "A" ), pqF( "B", "C" ) );
}
public void testFlattenTermAndPhrase2gram() throws Exception {
Query query = paB.parse( "AA AND BCD OR EFGH" );
FieldQuery fq = new FieldQuery( query, true, true );
Set<Query> flatQueries = new HashSet<Query>();
fq.flatten( query, flatQueries );
assertCollectionQueries( flatQueries, tq( "AA" ), pqF( "BC", "CD" ), pqF( "EF", "FG", "GH" ) );
}
public void testFlatten1TermPhrase() throws Exception {
Query query = pqF( "A" );
FieldQuery fq = new FieldQuery( query, true, true );
Set<Query> flatQueries = new HashSet<Query>();
fq.flatten( query, flatQueries );
assertCollectionQueries( flatQueries, tq( "A" ) );
}
public void testExpand() throws Exception {
Query dummy = pqF( "DUMMY" );
FieldQuery fq = new FieldQuery( dummy, true, true );
// "a b","b c" => "a b","b c","a b c"
Set<Query> flatQueries = new HashSet<Query>();
flatQueries.add( pqF( "a", "b" ) );
flatQueries.add( pqF( "b", "c" ) );
assertCollectionQueries( fq.expand( flatQueries ),
pqF( "a", "b" ), pqF( "b", "c" ), pqF( "a", "b", "c" ) );
// "a b","b c d" => "a b","b c d","a b c d"
flatQueries = new HashSet<Query>();
flatQueries.add( pqF( "a", "b" ) );
flatQueries.add( pqF( "b", "c", "d" ) );
assertCollectionQueries( fq.expand( flatQueries ),
pqF( "a", "b" ), pqF( "b", "c", "d" ), pqF( "a", "b", "c", "d" ) );
// "a b c","b c d" => "a b c","b c d","a b c d"
flatQueries = new HashSet<Query>();
flatQueries.add( pqF( "a", "b", "c" ) );
flatQueries.add( pqF( "b", "c", "d" ) );
assertCollectionQueries( fq.expand( flatQueries ),
pqF( "a", "b", "c" ), pqF( "b", "c", "d" ), pqF( "a", "b", "c", "d" ) );
// "a b c","c d e" => "a b c","c d e","a b c d e"
flatQueries = new HashSet<Query>();
flatQueries.add( pqF( "a", "b", "c" ) );
flatQueries.add( pqF( "c", "d", "e" ) );
assertCollectionQueries( fq.expand( flatQueries ),
pqF( "a", "b", "c" ), pqF( "c", "d", "e" ), pqF( "a", "b", "c", "d", "e" ) );
// "a b b","b c" => "a b b","b c","a b b c"
flatQueries = new HashSet<Query>();
flatQueries.add( pqF( "a", "b", "b" ) );
flatQueries.add( pqF( "b", "c" ) );
assertCollectionQueries( fq.expand( flatQueries ),
pqF( "a", "b", "b" ), pqF( "b", "c" ), pqF( "a", "b", "b", "c" ) );
// "a b","b a" => "a b","b a","a b a", "b a b"
flatQueries = new HashSet<Query>();
flatQueries.add( pqF( "a", "b" ) );
flatQueries.add( pqF( "b", "a" ) );
assertCollectionQueries( fq.expand( flatQueries ),
pqF( "a", "b" ), pqF( "b", "a" ), pqF( "a", "b", "a" ), pqF( "b", "a", "b" ) );
// "a b","a b c" => "a b","a b c"
flatQueries = new HashSet<Query>();
flatQueries.add( pqF( "a", "b" ) );
flatQueries.add( pqF( "a", "b", "c" ) );
assertCollectionQueries( fq.expand( flatQueries ),
pqF( "a", "b" ), pqF( "a", "b", "c" ) );
}
public void testNoExpand() throws Exception {
Query dummy = pqF( "DUMMY" );
FieldQuery fq = new FieldQuery( dummy, true, true );
// "a b","c d" => "a b","c d"
Set<Query> flatQueries = new HashSet<Query>();
flatQueries.add( pqF( "a", "b" ) );
flatQueries.add( pqF( "c", "d" ) );
assertCollectionQueries( fq.expand( flatQueries ),
pqF( "a", "b" ), pqF( "c", "d" ) );
// "a","a b" => "a", "a b"
flatQueries = new HashSet<Query>();
flatQueries.add( tq( "a" ) );
flatQueries.add( pqF( "a", "b" ) );
assertCollectionQueries( fq.expand( flatQueries ),
tq( "a" ), pqF( "a", "b" ) );
// "a b","b" => "a b", "b"
flatQueries = new HashSet<Query>();
flatQueries.add( pqF( "a", "b" ) );
flatQueries.add( tq( "b" ) );
assertCollectionQueries( fq.expand( flatQueries ),
pqF( "a", "b" ), tq( "b" ) );
// "a b c","b c" => "a b c","b c"
flatQueries = new HashSet<Query>();
flatQueries.add( pqF( "a", "b", "c" ) );
flatQueries.add( pqF( "b", "c" ) );
assertCollectionQueries( fq.expand( flatQueries ),
pqF( "a", "b", "c" ), pqF( "b", "c" ) );
// "a b","a b c" => "a b","a b c"
flatQueries = new HashSet<Query>();
flatQueries.add( pqF( "a", "b" ) );
flatQueries.add( pqF( "a", "b", "c" ) );
assertCollectionQueries( fq.expand( flatQueries ),
pqF( "a", "b" ), pqF( "a", "b", "c" ) );
// "a b c","b d e" => "a b c","b d e"
flatQueries = new HashSet<Query>();
flatQueries.add( pqF( "a", "b", "c" ) );
flatQueries.add( pqF( "b", "d", "e" ) );
assertCollectionQueries( fq.expand( flatQueries ),
pqF( "a", "b", "c" ), pqF( "b", "d", "e" ) );
}
public void testExpandNotFieldMatch() throws Exception {
Query dummy = pqF( "DUMMY" );
FieldQuery fq = new FieldQuery( dummy, true, false );
// f1:"a b",f2:"b c" => f1:"a b",f2:"b c",f1:"a b c"
Set<Query> flatQueries = new HashSet<Query>();
flatQueries.add( pq( F1, "a", "b" ) );
flatQueries.add( pq( F2, "b", "c" ) );
assertCollectionQueries( fq.expand( flatQueries ),
pq( F1, "a", "b" ), pq( F2, "b", "c" ), pq( F1, "a", "b", "c" ) );
}
public void testGetFieldTermMap() throws Exception {
Query query = tq( "a" );
FieldQuery fq = new FieldQuery( query, true, true );
QueryPhraseMap pqm = fq.getFieldTermMap( F, "a" );
assertNotNull( pqm );
assertTrue( pqm.isTerminal() );
pqm = fq.getFieldTermMap( F, "b" );
assertNull( pqm );
pqm = fq.getFieldTermMap( F1, "a" );
assertNull( pqm );
}
public void testGetRootMap() throws Exception {
Query dummy = pqF( "DUMMY" );
FieldQuery fq = new FieldQuery( dummy, true, true );
QueryPhraseMap rootMap1 = fq.getRootMap( tq( "a" ) );
QueryPhraseMap rootMap2 = fq.getRootMap( tq( "a" ) );
assertTrue( rootMap1 == rootMap2 );
QueryPhraseMap rootMap3 = fq.getRootMap( tq( "b" ) );
assertTrue( rootMap1 == rootMap3 );
QueryPhraseMap rootMap4 = fq.getRootMap( tq( F1, "b" ) );
assertFalse( rootMap4 == rootMap3 );
}
public void testGetRootMapNotFieldMatch() throws Exception {
Query dummy = pqF( "DUMMY" );
FieldQuery fq = new FieldQuery( dummy, true, false );
QueryPhraseMap rootMap1 = fq.getRootMap( tq( "a" ) );
QueryPhraseMap rootMap2 = fq.getRootMap( tq( "a" ) );
assertTrue( rootMap1 == rootMap2 );
QueryPhraseMap rootMap3 = fq.getRootMap( tq( "b" ) );
assertTrue( rootMap1 == rootMap3 );
QueryPhraseMap rootMap4 = fq.getRootMap( tq( F1, "b" ) );
assertTrue( rootMap4 == rootMap3 );
}
public void testGetTermSet() throws Exception {
Query query = paW.parse( "A AND B OR x:C NOT (D AND E)" );
FieldQuery fq = new FieldQuery( query, true, true );
assertEquals( 2, fq.termSetMap.size() );
Set<String> termSet = fq.getTermSet( F );
assertEquals( 2, termSet.size() );
assertTrue( termSet.contains( "A" ) );
assertTrue( termSet.contains( "B" ) );
termSet = fq.getTermSet( "x" );
assertEquals( 1, termSet.size() );
assertTrue( termSet.contains( "C" ) );
termSet = fq.getTermSet( "y" );
assertNull( termSet );
}
public void testQueryPhraseMap1Term() throws Exception {
Query query = tq( "a" );
// phraseHighlight = true, fieldMatch = true
FieldQuery fq = new FieldQuery( query, true, true );
Map<String, QueryPhraseMap> map = fq.rootMaps;
assertEquals( 1, map.size() );
assertNull( map.get( null ) );
assertNotNull( map.get( F ) );
QueryPhraseMap qpm = map.get( F );
assertEquals( 1, qpm.subMap.size() );
assertTrue( qpm.subMap.get( "a" ) != null );
assertTrue( qpm.subMap.get( "a" ).terminal );
assertEquals( 1F, qpm.subMap.get( "a" ).boost );
// phraseHighlight = true, fieldMatch = false
fq = new FieldQuery( query, true, false );
map = fq.rootMaps;
assertEquals( 1, map.size() );
assertNull( map.get( F ) );
assertNotNull( map.get( null ) );
qpm = map.get( null );
assertEquals( 1, qpm.subMap.size() );
assertTrue( qpm.subMap.get( "a" ) != null );
assertTrue( qpm.subMap.get( "a" ).terminal );
assertEquals( 1F, qpm.subMap.get( "a" ).boost );
// phraseHighlight = false, fieldMatch = true
fq = new FieldQuery( query, false, true );
map = fq.rootMaps;
assertEquals( 1, map.size() );
assertNull( map.get( null ) );
assertNotNull( map.get( F ) );
qpm = map.get( F );
assertEquals( 1, qpm.subMap.size() );
assertTrue( qpm.subMap.get( "a" ) != null );
assertTrue( qpm.subMap.get( "a" ).terminal );
assertEquals( 1F, qpm.subMap.get( "a" ).boost );
// phraseHighlight = false, fieldMatch = false
fq = new FieldQuery( query, false, false );
map = fq.rootMaps;
assertEquals( 1, map.size() );
assertNull( map.get( F ) );
assertNotNull( map.get( null ) );
qpm = map.get( null );
assertEquals( 1, qpm.subMap.size() );
assertTrue( qpm.subMap.get( "a" ) != null );
assertTrue( qpm.subMap.get( "a" ).terminal );
assertEquals( 1F, qpm.subMap.get( "a" ).boost );
// boost != 1
query = tq( 2, "a" );
fq = new FieldQuery( query, true, true );
map = fq.rootMaps;
qpm = map.get( F );
assertEquals( 2F, qpm.subMap.get( "a" ).boost );
}
public void testQueryPhraseMap1Phrase() throws Exception {
Query query = pqF( "a", "b" );
// phraseHighlight = true, fieldMatch = true
FieldQuery fq = new FieldQuery( query, true, true );
Map<String, QueryPhraseMap> map = fq.rootMaps;
assertEquals( 1, map.size() );
assertNull( map.get( null ) );
assertNotNull( map.get( F ) );
QueryPhraseMap qpm = map.get( F );
assertEquals( 1, qpm.subMap.size() );
assertNotNull( qpm.subMap.get( "a" ) );
QueryPhraseMap qpm2 = qpm.subMap.get( "a" );
assertFalse( qpm2.terminal );
assertEquals( 1, qpm2.subMap.size() );
assertNotNull( qpm2.subMap.get( "b" ) );
QueryPhraseMap qpm3 = qpm2.subMap.get( "b" );
assertTrue( qpm3.terminal );
assertEquals( 1F, qpm3.boost );
// phraseHighlight = true, fieldMatch = false
fq = new FieldQuery( query, true, false );
map = fq.rootMaps;
assertEquals( 1, map.size() );
assertNull( map.get( F ) );
assertNotNull( map.get( null ) );
qpm = map.get( null );
assertEquals( 1, qpm.subMap.size() );
assertNotNull( qpm.subMap.get( "a" ) );
qpm2 = qpm.subMap.get( "a" );
assertFalse( qpm2.terminal );
assertEquals( 1, qpm2.subMap.size() );
assertNotNull( qpm2.subMap.get( "b" ) );
qpm3 = qpm2.subMap.get( "b" );
assertTrue( qpm3.terminal );
assertEquals( 1F, qpm3.boost );
// phraseHighlight = false, fieldMatch = true
fq = new FieldQuery( query, false, true );
map = fq.rootMaps;
assertEquals( 1, map.size() );
assertNull( map.get( null ) );
assertNotNull( map.get( F ) );
qpm = map.get( F );
assertEquals( 2, qpm.subMap.size() );
assertNotNull( qpm.subMap.get( "a" ) );
qpm2 = qpm.subMap.get( "a" );
assertTrue( qpm2.terminal );
assertEquals( 1F, qpm2.boost );
assertEquals( 1, qpm2.subMap.size() );
assertNotNull( qpm2.subMap.get( "b" ) );
qpm3 = qpm2.subMap.get( "b" );
assertTrue( qpm3.terminal );
assertEquals( 1F, qpm3.boost );
assertNotNull( qpm.subMap.get( "b" ) );
qpm2 = qpm.subMap.get( "b" );
assertTrue( qpm2.terminal );
assertEquals( 1F, qpm2.boost );
// phraseHighlight = false, fieldMatch = false
fq = new FieldQuery( query, false, false );
map = fq.rootMaps;
assertEquals( 1, map.size() );
assertNull( map.get( F ) );
assertNotNull( map.get( null ) );
qpm = map.get( null );
assertEquals( 2, qpm.subMap.size() );
assertNotNull( qpm.subMap.get( "a" ) );
qpm2 = qpm.subMap.get( "a" );
assertTrue( qpm2.terminal );
assertEquals( 1F, qpm2.boost );
assertEquals( 1, qpm2.subMap.size() );
assertNotNull( qpm2.subMap.get( "b" ) );
qpm3 = qpm2.subMap.get( "b" );
assertTrue( qpm3.terminal );
assertEquals( 1F, qpm3.boost );
assertNotNull( qpm.subMap.get( "b" ) );
qpm2 = qpm.subMap.get( "b" );
assertTrue( qpm2.terminal );
assertEquals( 1F, qpm2.boost );
// boost != 1
query = pqF( 2, "a", "b" );
// phraseHighlight = false, fieldMatch = false
fq = new FieldQuery( query, false, false );
map = fq.rootMaps;
qpm = map.get( null );
qpm2 = qpm.subMap.get( "a" );
assertEquals( 2F, qpm2.boost );
qpm3 = qpm2.subMap.get( "b" );
assertEquals( 2F, qpm3.boost );
qpm2 = qpm.subMap.get( "b" );
assertEquals( 2F, qpm2.boost );
}
public void testQueryPhraseMap1PhraseAnother() throws Exception {
Query query = pqF( "search", "engines" );
// phraseHighlight = true, fieldMatch = true
FieldQuery fq = new FieldQuery( query, true, true );
Map<String, QueryPhraseMap> map = fq.rootMaps;
assertEquals( 1, map.size() );
assertNull( map.get( null ) );
assertNotNull( map.get( F ) );
QueryPhraseMap qpm = map.get( F );
assertEquals( 1, qpm.subMap.size() );
assertNotNull( qpm.subMap.get( "search" ) );
QueryPhraseMap qpm2 = qpm.subMap.get( "search" );
assertFalse( qpm2.terminal );
assertEquals( 1, qpm2.subMap.size() );
assertNotNull( qpm2.subMap.get( "engines" ) );
QueryPhraseMap qpm3 = qpm2.subMap.get( "engines" );
assertTrue( qpm3.terminal );
assertEquals( 1F, qpm3.boost );
}
public void testQueryPhraseMap2Phrases() throws Exception {
BooleanQuery query = new BooleanQuery();
query.add( pqF( "a", "b" ), Occur.SHOULD );
query.add( pqF( 2, "c", "d" ), Occur.SHOULD );
// phraseHighlight = true, fieldMatch = true
FieldQuery fq = new FieldQuery( query, true, true );
Map<String, QueryPhraseMap> map = fq.rootMaps;
assertEquals( 1, map.size() );
assertNull( map.get( null ) );
assertNotNull( map.get( F ) );
QueryPhraseMap qpm = map.get( F );
assertEquals( 2, qpm.subMap.size() );
// "a b"
assertNotNull( qpm.subMap.get( "a" ) );
QueryPhraseMap qpm2 = qpm.subMap.get( "a" );
assertFalse( qpm2.terminal );
assertEquals( 1, qpm2.subMap.size() );
assertNotNull( qpm2.subMap.get( "b" ) );
QueryPhraseMap qpm3 = qpm2.subMap.get( "b" );
assertTrue( qpm3.terminal );
assertEquals( 1F, qpm3.boost );
// "c d"^2
assertNotNull( qpm.subMap.get( "c" ) );
qpm2 = qpm.subMap.get( "c" );
assertFalse( qpm2.terminal );
assertEquals( 1, qpm2.subMap.size() );
assertNotNull( qpm2.subMap.get( "d" ) );
qpm3 = qpm2.subMap.get( "d" );
assertTrue( qpm3.terminal );
assertEquals( 2F, qpm3.boost );
}
public void testQueryPhraseMap2PhrasesFields() throws Exception {
BooleanQuery query = new BooleanQuery();
query.add( pq( F1, "a", "b" ), Occur.SHOULD );
query.add( pq( 2F, F2, "c", "d" ), Occur.SHOULD );
// phraseHighlight = true, fieldMatch = true
FieldQuery fq = new FieldQuery( query, true, true );
Map<String, QueryPhraseMap> map = fq.rootMaps;
assertEquals( 2, map.size() );
assertNull( map.get( null ) );
// "a b"
assertNotNull( map.get( F1 ) );
QueryPhraseMap qpm = map.get( F1 );
assertEquals( 1, qpm.subMap.size() );
assertNotNull( qpm.subMap.get( "a" ) );
QueryPhraseMap qpm2 = qpm.subMap.get( "a" );
assertFalse( qpm2.terminal );
assertEquals( 1, qpm2.subMap.size() );
assertNotNull( qpm2.subMap.get( "b" ) );
QueryPhraseMap qpm3 = qpm2.subMap.get( "b" );
assertTrue( qpm3.terminal );
assertEquals( 1F, qpm3.boost );
// "c d"^2
assertNotNull( map.get( F2 ) );
qpm = map.get( F2 );
assertEquals( 1, qpm.subMap.size() );
assertNotNull( qpm.subMap.get( "c" ) );
qpm2 = qpm.subMap.get( "c" );
assertFalse( qpm2.terminal );
assertEquals( 1, qpm2.subMap.size() );
assertNotNull( qpm2.subMap.get( "d" ) );
qpm3 = qpm2.subMap.get( "d" );
assertTrue( qpm3.terminal );
assertEquals( 2F, qpm3.boost );
// phraseHighlight = true, fieldMatch = false
fq = new FieldQuery( query, true, false );
map = fq.rootMaps;
assertEquals( 1, map.size() );
assertNull( map.get( F1 ) );
assertNull( map.get( F2 ) );
assertNotNull( map.get( null ) );
qpm = map.get( null );
assertEquals( 2, qpm.subMap.size() );
// "a b"
assertNotNull( qpm.subMap.get( "a" ) );
qpm2 = qpm.subMap.get( "a" );
assertFalse( qpm2.terminal );
assertEquals( 1, qpm2.subMap.size() );
assertNotNull( qpm2.subMap.get( "b" ) );
qpm3 = qpm2.subMap.get( "b" );
assertTrue( qpm3.terminal );
assertEquals( 1F, qpm3.boost );
// "c d"^2
assertNotNull( qpm.subMap.get( "c" ) );
qpm2 = qpm.subMap.get( "c" );
assertFalse( qpm2.terminal );
assertEquals( 1, qpm2.subMap.size() );
assertNotNull( qpm2.subMap.get( "d" ) );
qpm3 = qpm2.subMap.get( "d" );
assertTrue( qpm3.terminal );
assertEquals( 2F, qpm3.boost );
}
/*
* <t>...terminal
*
* a-b-c-<t>
* +-d-<t>
* b-c-d-<t>
* +-d-<t>
*/
public void testQueryPhraseMapOverlapPhrases() throws Exception {
BooleanQuery query = new BooleanQuery();
query.add( pqF( "a", "b", "c" ), Occur.SHOULD );
query.add( pqF( 2, "b", "c", "d" ), Occur.SHOULD );
query.add( pqF( 3, "b", "d" ), Occur.SHOULD );
// phraseHighlight = true, fieldMatch = true
FieldQuery fq = new FieldQuery( query, true, true );
Map<String, QueryPhraseMap> map = fq.rootMaps;
assertEquals( 1, map.size() );
assertNull( map.get( null ) );
assertNotNull( map.get( F ) );
QueryPhraseMap qpm = map.get( F );
assertEquals( 2, qpm.subMap.size() );
// "a b c"
assertNotNull( qpm.subMap.get( "a" ) );
QueryPhraseMap qpm2 = qpm.subMap.get( "a" );
assertFalse( qpm2.terminal );
assertEquals( 1, qpm2.subMap.size() );
assertNotNull( qpm2.subMap.get( "b" ) );
QueryPhraseMap qpm3 = qpm2.subMap.get( "b" );
assertFalse( qpm3.terminal );
assertEquals( 1, qpm3.subMap.size() );
assertNotNull( qpm3.subMap.get( "c" ) );
QueryPhraseMap qpm4 = qpm3.subMap.get( "c" );
assertTrue( qpm4.terminal );
assertEquals( 1F, qpm4.boost );
assertNotNull( qpm4.subMap.get( "d" ) );
QueryPhraseMap qpm5 = qpm4.subMap.get( "d" );
assertTrue( qpm5.terminal );
assertEquals( 1F, qpm5.boost );
// "b c d"^2, "b d"^3
assertNotNull( qpm.subMap.get( "b" ) );
qpm2 = qpm.subMap.get( "b" );
assertFalse( qpm2.terminal );
assertEquals( 2, qpm2.subMap.size() );
assertNotNull( qpm2.subMap.get( "c" ) );
qpm3 = qpm2.subMap.get( "c" );
assertFalse( qpm3.terminal );
assertEquals( 1, qpm3.subMap.size() );
assertNotNull( qpm3.subMap.get( "d" ) );
qpm4 = qpm3.subMap.get( "d" );
assertTrue( qpm4.terminal );
assertEquals( 2F, qpm4.boost );
assertNotNull( qpm2.subMap.get( "d" ) );
qpm3 = qpm2.subMap.get( "d" );
assertTrue( qpm3.terminal );
assertEquals( 3F, qpm3.boost );
}
/*
* <t>...terminal
*
* a-b-<t>
* +-c-<t>
*/
public void testQueryPhraseMapOverlapPhrases2() throws Exception {
BooleanQuery query = new BooleanQuery();
query.add( pqF( "a", "b" ), Occur.SHOULD );
query.add( pqF( 2, "a", "b", "c" ), Occur.SHOULD );
// phraseHighlight = true, fieldMatch = true
FieldQuery fq = new FieldQuery( query, true, true );
Map<String, QueryPhraseMap> map = fq.rootMaps;
assertEquals( 1, map.size() );
assertNull( map.get( null ) );
assertNotNull( map.get( F ) );
QueryPhraseMap qpm = map.get( F );
assertEquals( 1, qpm.subMap.size() );
// "a b"
assertNotNull( qpm.subMap.get( "a" ) );
QueryPhraseMap qpm2 = qpm.subMap.get( "a" );
assertFalse( qpm2.terminal );
assertEquals( 1, qpm2.subMap.size() );
assertNotNull( qpm2.subMap.get( "b" ) );
QueryPhraseMap qpm3 = qpm2.subMap.get( "b" );
assertTrue( qpm3.terminal );
assertEquals( 1F, qpm3.boost );
// "a b c"^2
assertEquals( 1, qpm3.subMap.size() );
assertNotNull( qpm3.subMap.get( "c" ) );
QueryPhraseMap qpm4 = qpm3.subMap.get( "c" );
assertTrue( qpm4.terminal );
assertEquals( 2F, qpm4.boost );
}
/*
* <t>...terminal
*
* a-a-a-<t>
* +-a-<t>
* +-a-<t>
* +-a-<t>
*/
public void testQueryPhraseMapOverlapPhrases3() throws Exception {
BooleanQuery query = new BooleanQuery();
query.add( pqF( "a", "a", "a", "a" ), Occur.SHOULD );
query.add( pqF( 2, "a", "a", "a" ), Occur.SHOULD );
// phraseHighlight = true, fieldMatch = true
FieldQuery fq = new FieldQuery( query, true, true );
Map<String, QueryPhraseMap> map = fq.rootMaps;
assertEquals( 1, map.size() );
assertNull( map.get( null ) );
assertNotNull( map.get( F ) );
QueryPhraseMap qpm = map.get( F );
assertEquals( 1, qpm.subMap.size() );
// "a a a"
assertNotNull( qpm.subMap.get( "a" ) );
QueryPhraseMap qpm2 = qpm.subMap.get( "a" );
assertFalse( qpm2.terminal );
assertEquals( 1, qpm2.subMap.size() );
assertNotNull( qpm2.subMap.get( "a" ) );
QueryPhraseMap qpm3 = qpm2.subMap.get( "a" );
assertFalse( qpm3.terminal );
assertEquals( 1, qpm3.subMap.size() );
assertNotNull( qpm3.subMap.get( "a" ) );
QueryPhraseMap qpm4 = qpm3.subMap.get( "a" );
assertTrue( qpm4.terminal );
// "a a a a"
assertEquals( 1, qpm4.subMap.size() );
assertNotNull( qpm4.subMap.get( "a" ) );
QueryPhraseMap qpm5 = qpm4.subMap.get( "a" );
assertTrue( qpm5.terminal );
// "a a a a a"
assertEquals( 1, qpm5.subMap.size() );
assertNotNull( qpm5.subMap.get( "a" ) );
QueryPhraseMap qpm6 = qpm5.subMap.get( "a" );
assertTrue( qpm6.terminal );
// "a a a a a a"
assertEquals( 1, qpm6.subMap.size() );
assertNotNull( qpm6.subMap.get( "a" ) );
QueryPhraseMap qpm7 = qpm6.subMap.get( "a" );
assertTrue( qpm7.terminal );
}
public void testQueryPhraseMapOverlap2gram() throws Exception {
Query query = paB.parse( "abc AND bcd" );
// phraseHighlight = true, fieldMatch = true
FieldQuery fq = new FieldQuery( query, true, true );
Map<String, QueryPhraseMap> map = fq.rootMaps;
assertEquals( 1, map.size() );
assertNull( map.get( null ) );
assertNotNull( map.get( F ) );
QueryPhraseMap qpm = map.get( F );
assertEquals( 2, qpm.subMap.size() );
// "ab bc"
assertNotNull( qpm.subMap.get( "ab" ) );
QueryPhraseMap qpm2 = qpm.subMap.get( "ab" );
assertFalse( qpm2.terminal );
assertEquals( 1, qpm2.subMap.size() );
assertNotNull( qpm2.subMap.get( "bc" ) );
QueryPhraseMap qpm3 = qpm2.subMap.get( "bc" );
assertTrue( qpm3.terminal );
assertEquals( 1F, qpm3.boost );
// "ab bc cd"
assertEquals( 1, qpm3.subMap.size() );
assertNotNull( qpm3.subMap.get( "cd" ) );
QueryPhraseMap qpm4 = qpm3.subMap.get( "cd" );
assertTrue( qpm4.terminal );
assertEquals( 1F, qpm4.boost );
// "bc cd"
assertNotNull( qpm.subMap.get( "bc" ) );
qpm2 = qpm.subMap.get( "bc" );
assertFalse( qpm2.terminal );
assertEquals( 1, qpm2.subMap.size() );
assertNotNull( qpm2.subMap.get( "cd" ) );
qpm3 = qpm2.subMap.get( "cd" );
assertTrue( qpm3.terminal );
assertEquals( 1F, qpm3.boost );
// phraseHighlight = false, fieldMatch = true
fq = new FieldQuery( query, false, true );
map = fq.rootMaps;
assertEquals( 1, map.size() );
assertNull( map.get( null ) );
assertNotNull( map.get( F ) );
qpm = map.get( F );
assertEquals( 3, qpm.subMap.size() );
// "ab bc"
assertNotNull( qpm.subMap.get( "ab" ) );
qpm2 = qpm.subMap.get( "ab" );
assertTrue( qpm2.terminal );
assertEquals( 1F, qpm2.boost );
assertEquals( 1, qpm2.subMap.size() );
assertNotNull( qpm2.subMap.get( "bc" ) );
qpm3 = qpm2.subMap.get( "bc" );
assertTrue( qpm3.terminal );
assertEquals( 1F, qpm3.boost );
// "ab bc cd"
assertEquals( 1, qpm3.subMap.size() );
assertNotNull( qpm3.subMap.get( "cd" ) );
qpm4 = qpm3.subMap.get( "cd" );
assertTrue( qpm4.terminal );
assertEquals( 1F, qpm4.boost );
// "bc cd"
assertNotNull( qpm.subMap.get( "bc" ) );
qpm2 = qpm.subMap.get( "bc" );
assertTrue( qpm2.terminal );
assertEquals( 1F, qpm2.boost );
assertEquals( 1, qpm2.subMap.size() );
assertNotNull( qpm2.subMap.get( "cd" ) );
qpm3 = qpm2.subMap.get( "cd" );
assertTrue( qpm3.terminal );
assertEquals( 1F, qpm3.boost );
// "cd"
assertNotNull( qpm.subMap.get( "cd" ) );
qpm2 = qpm.subMap.get( "cd" );
assertTrue( qpm2.terminal );
assertEquals( 1F, qpm2.boost );
assertEquals( 0, qpm2.subMap.size() );
}
public void testSearchPhrase() throws Exception {
Query query = pqF( "a", "b", "c" );
// phraseHighlight = true, fieldMatch = true
FieldQuery fq = new FieldQuery( query, true, true );
// "a"
List<TermInfo> phraseCandidate = new ArrayList<TermInfo>();
phraseCandidate.add( new TermInfo( "a", 0, 1, 0 ) );
assertNull( fq.searchPhrase( F, phraseCandidate ) );
// "a b"
phraseCandidate.add( new TermInfo( "b", 2, 3, 1 ) );
assertNull( fq.searchPhrase( F, phraseCandidate ) );
// "a b c"
phraseCandidate.add( new TermInfo( "c", 4, 5, 2 ) );
assertNotNull( fq.searchPhrase( F, phraseCandidate ) );
assertNull( fq.searchPhrase( "x", phraseCandidate ) );
// phraseHighlight = true, fieldMatch = false
fq = new FieldQuery( query, true, false );
// "a b c"
assertNotNull( fq.searchPhrase( F, phraseCandidate ) );
assertNotNull( fq.searchPhrase( "x", phraseCandidate ) );
// phraseHighlight = false, fieldMatch = true
fq = new FieldQuery( query, false, true );
// "a"
phraseCandidate.clear();
phraseCandidate.add( new TermInfo( "a", 0, 1, 0 ) );
assertNotNull( fq.searchPhrase( F, phraseCandidate ) );
// "a b"
phraseCandidate.add( new TermInfo( "b", 2, 3, 1 ) );
assertNull( fq.searchPhrase( F, phraseCandidate ) );
// "a b c"
phraseCandidate.add( new TermInfo( "c", 4, 5, 2 ) );
assertNotNull( fq.searchPhrase( F, phraseCandidate ) );
assertNull( fq.searchPhrase( "x", phraseCandidate ) );
}
public void testSearchPhraseSlop() throws Exception {
// "a b c"~0
Query query = pqF( "a", "b", "c" );
// phraseHighlight = true, fieldMatch = true
FieldQuery fq = new FieldQuery( query, true, true );
// "a b c" w/ position-gap = 2
List<TermInfo> phraseCandidate = new ArrayList<TermInfo>();
phraseCandidate.add( new TermInfo( "a", 0, 1, 0 ) );
phraseCandidate.add( new TermInfo( "b", 2, 3, 2 ) );
phraseCandidate.add( new TermInfo( "c", 4, 5, 4 ) );
assertNull( fq.searchPhrase( F, phraseCandidate ) );
// "a b c"~1
query = pqF( 1F, 1, "a", "b", "c" );
// phraseHighlight = true, fieldMatch = true
fq = new FieldQuery( query, true, true );
// "a b c" w/ position-gap = 2
assertNotNull( fq.searchPhrase( F, phraseCandidate ) );
// "a b c" w/ position-gap = 3
phraseCandidate.clear();
phraseCandidate.add( new TermInfo( "a", 0, 1, 0 ) );
phraseCandidate.add( new TermInfo( "b", 2, 3, 3 ) );
phraseCandidate.add( new TermInfo( "c", 4, 5, 6 ) );
assertNull( fq.searchPhrase( F, phraseCandidate ) );
}
}

166
contrib/fast-vector-highlighter/src/test/org/apache/lucene/search/vectorhighlight/FieldTermStackTest.java Normal file
View File

@ -0,0 +1,166 @@
package org.apache.lucene.search.vectorhighlight;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.BooleanClause.Occur;
public class FieldTermStackTest extends AbstractTestCase {
public void test1Term() throws Exception {
makeIndex();
FieldQuery fq = new FieldQuery( tq( "a" ), true, true );
FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
assertEquals( 6, stack.termList.size() );
assertEquals( "a(0,1,0)", stack.pop().toString() );
assertEquals( "a(2,3,1)", stack.pop().toString() );
assertEquals( "a(4,5,2)", stack.pop().toString() );
assertEquals( "a(12,13,6)", stack.pop().toString() );
assertEquals( "a(28,29,14)", stack.pop().toString() );
assertEquals( "a(32,33,16)", stack.pop().toString() );
}
public void test2Terms() throws Exception {
makeIndex();
BooleanQuery query = new BooleanQuery();
query.add( tq( "b" ), Occur.SHOULD );
query.add( tq( "c" ), Occur.SHOULD );
FieldQuery fq = new FieldQuery( query, true, true );
FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
assertEquals( 8, stack.termList.size() );
assertEquals( "b(6,7,3)", stack.pop().toString() );
assertEquals( "b(8,9,4)", stack.pop().toString() );
assertEquals( "c(10,11,5)", stack.pop().toString() );
assertEquals( "b(14,15,7)", stack.pop().toString() );
assertEquals( "b(16,17,8)", stack.pop().toString() );
assertEquals( "c(18,19,9)", stack.pop().toString() );
assertEquals( "b(26,27,13)", stack.pop().toString() );
assertEquals( "b(30,31,15)", stack.pop().toString() );
}
public void test1Phrase() throws Exception {
makeIndex();
FieldQuery fq = new FieldQuery( pqF( "c", "d" ), true, true );
FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
assertEquals( 3, stack.termList.size() );
assertEquals( "c(10,11,5)", stack.pop().toString() );
assertEquals( "c(18,19,9)", stack.pop().toString() );
assertEquals( "d(20,21,10)", stack.pop().toString() );
}
private void makeIndex() throws Exception {
// 111111111122222
// 0123456789012345678901234 (offsets)
// a a a b b c a b b c d e f
// 0 1 2 3 4 5 6 7 8 9101112 (position)
String value1 = "a a a b b c a b b c d e f";
// 222233333
// 678901234 (offsets)
// b a b a f
//1314151617 (position)
String value2 = "b a b a f";
make1dmfIndex( value1, value2 );
}
public void test1TermB() throws Exception {
makeIndexB();
FieldQuery fq = new FieldQuery( tq( "ab" ), true, true );
FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
assertEquals( 2, stack.termList.size() );
assertEquals( "ab(2,4,2)", stack.pop().toString() );
assertEquals( "ab(6,8,6)", stack.pop().toString() );
}
public void test2TermsB() throws Exception {
makeIndexB();
BooleanQuery query = new BooleanQuery();
query.add( tq( "bc" ), Occur.SHOULD );
query.add( tq( "ef" ), Occur.SHOULD );
FieldQuery fq = new FieldQuery( query, true, true );
FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
assertEquals( 3, stack.termList.size() );
assertEquals( "bc(4,6,4)", stack.pop().toString() );
assertEquals( "bc(8,10,8)", stack.pop().toString() );
assertEquals( "ef(11,13,11)", stack.pop().toString() );
}
public void test1PhraseB() throws Exception {
makeIndexB();
FieldQuery fq = new FieldQuery( pqF( "ab", "bb" ), true, true );
FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
assertEquals( 4, stack.termList.size() );
assertEquals( "ab(2,4,2)", stack.pop().toString() );
assertEquals( "bb(3,5,3)", stack.pop().toString() );
assertEquals( "ab(6,8,6)", stack.pop().toString() );
assertEquals( "bb(7,9,7)", stack.pop().toString() );
}
private void makeIndexB() throws Exception {
// 1 11 11
// 01 12 23 34 45 56 67 78 89 90 01 12 (offsets)
// aa|aa|ab|bb|bc|ca|ab|bb|bc|cd|de|ef
// 0 1 2 3 4 5 6 7 8 9 10 11 (position)
String value = "aaabbcabbcdef";
make1dmfIndexB( value );
}
public void test1PhraseShortMV() throws Exception {
makeIndexShortMV();
FieldQuery fq = new FieldQuery( tq( "d" ), true, true );
FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
assertEquals( 1, stack.termList.size() );
assertEquals( "d(6,7,3)", stack.pop().toString() );
}
public void test1PhraseLongMV() throws Exception {
makeIndexLongMV();
FieldQuery fq = new FieldQuery( pqF( "search", "engines" ), true, true );
FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
assertEquals( 4, stack.termList.size() );
assertEquals( "search(102,108,14)", stack.pop().toString() );
assertEquals( "engines(109,116,15)", stack.pop().toString() );
assertEquals( "search(157,163,24)", stack.pop().toString() );
assertEquals( "engines(164,171,25)", stack.pop().toString() );
}
/*
* ----------------------------------
* THIS TEST DEPENDS ON LUCENE-1448
* UNCOMMENT WHEN IT IS COMMITTED.
* ----------------------------------
public void test1PhraseMVB() throws Exception {
makeIndexLongMVB();
FieldQuery fq = new FieldQuery( pqF( "sp", "pe", "ee", "ed" ), true, true ); // "speed" -(2gram)-> "sp","pe","ee","ed"
FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
assertEquals( 4, stack.termList.size() );
assertEquals( "sp(88,90,61)", stack.pop().toString() );
assertEquals( "pe(89,91,62)", stack.pop().toString() );
assertEquals( "ee(90,92,63)", stack.pop().toString() );
assertEquals( "ed(91,93,64)", stack.pop().toString() );
}
*/
}

308
contrib/fast-vector-highlighter/src/test/org/apache/lucene/search/vectorhighlight/IndexTimeSynonymTest.java Normal file
View File

@ -0,0 +1,308 @@
package org.apache.lucene.search.vectorhighlight;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import java.io.Reader;
import java.util.HashSet;
import java.util.Set;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.BooleanClause.Occur;
public class IndexTimeSynonymTest extends AbstractTestCase {
public void testFieldTermStackIndex1wSearch1term() throws Exception {
makeIndex1w();
FieldQuery fq = new FieldQuery( tq( "Mac" ), true, true );
FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
assertEquals( 1, stack.termList.size() );
assertEquals( "Mac(11,20,3)", stack.pop().toString() );
}
public void testFieldTermStackIndex1wSearch2terms() throws Exception {
makeIndex1w();
BooleanQuery bq = new BooleanQuery();
bq.add( tq( "Mac" ), Occur.SHOULD );
bq.add( tq( "MacBook" ), Occur.SHOULD );
FieldQuery fq = new FieldQuery( bq, true, true );
FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
assertEquals( 2, stack.termList.size() );
Set<String> expectedSet = new HashSet<String>();
expectedSet.add( "Mac(11,20,3)" );
expectedSet.add( "MacBook(11,20,3)" );
assertTrue( expectedSet.contains( stack.pop().toString() ) );
assertTrue( expectedSet.contains( stack.pop().toString() ) );
}
public void testFieldTermStackIndex1w2wSearch1term() throws Exception {
makeIndex1w2w();
FieldQuery fq = new FieldQuery( tq( "pc" ), true, true );
FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
assertEquals( 1, stack.termList.size() );
assertEquals( "pc(3,5,1)", stack.pop().toString() );
}
public void testFieldTermStackIndex1w2wSearch1phrase() throws Exception {
makeIndex1w2w();
FieldQuery fq = new FieldQuery( pqF( "personal", "computer" ), true, true );
FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
assertEquals( 2, stack.termList.size() );
assertEquals( "personal(3,5,1)", stack.pop().toString() );
assertEquals( "computer(3,5,2)", stack.pop().toString() );
}
public void testFieldTermStackIndex1w2wSearch1partial() throws Exception {
makeIndex1w2w();
FieldQuery fq = new FieldQuery( tq( "computer" ), true, true );
FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
assertEquals( 1, stack.termList.size() );
assertEquals( "computer(3,5,2)", stack.pop().toString() );
}
public void testFieldTermStackIndex1w2wSearch1term1phrase() throws Exception {
makeIndex1w2w();
BooleanQuery bq = new BooleanQuery();
bq.add( tq( "pc" ), Occur.SHOULD );
bq.add( pqF( "personal", "computer" ), Occur.SHOULD );
FieldQuery fq = new FieldQuery( bq, true, true );
FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
assertEquals( 3, stack.termList.size() );
Set<String> expectedSet = new HashSet<String>();
expectedSet.add( "pc(3,5,1)" );
expectedSet.add( "personal(3,5,1)" );
assertTrue( expectedSet.contains( stack.pop().toString() ) );
assertTrue( expectedSet.contains( stack.pop().toString() ) );
assertEquals( "computer(3,5,2)", stack.pop().toString() );
}
public void testFieldTermStackIndex2w1wSearch1term() throws Exception {
makeIndex2w1w();
FieldQuery fq = new FieldQuery( tq( "pc" ), true, true );
FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
assertEquals( 1, stack.termList.size() );
assertEquals( "pc(3,20,1)", stack.pop().toString() );
}
public void testFieldTermStackIndex2w1wSearch1phrase() throws Exception {
makeIndex2w1w();
FieldQuery fq = new FieldQuery( pqF( "personal", "computer" ), true, true );
FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
assertEquals( 2, stack.termList.size() );
assertEquals( "personal(3,20,1)", stack.pop().toString() );
assertEquals( "computer(3,20,2)", stack.pop().toString() );
}
public void testFieldTermStackIndex2w1wSearch1partial() throws Exception {
makeIndex2w1w();
FieldQuery fq = new FieldQuery( tq( "computer" ), true, true );
FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
assertEquals( 1, stack.termList.size() );
assertEquals( "computer(3,20,2)", stack.pop().toString() );
}
public void testFieldTermStackIndex2w1wSearch1term1phrase() throws Exception {
makeIndex2w1w();
BooleanQuery bq = new BooleanQuery();
bq.add( tq( "pc" ), Occur.SHOULD );
bq.add( pqF( "personal", "computer" ), Occur.SHOULD );
FieldQuery fq = new FieldQuery( bq, true, true );
FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
assertEquals( 3, stack.termList.size() );
Set<String> expectedSet = new HashSet<String>();
expectedSet.add( "pc(3,20,1)" );
expectedSet.add( "personal(3,20,1)" );
assertTrue( expectedSet.contains( stack.pop().toString() ) );
assertTrue( expectedSet.contains( stack.pop().toString() ) );
assertEquals( "computer(3,20,2)", stack.pop().toString() );
}
public void testFieldPhraseListIndex1w2wSearch1phrase() throws Exception {
makeIndex1w2w();
FieldQuery fq = new FieldQuery( pqF( "personal", "computer" ), true, true );
FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
FieldPhraseList fpl = new FieldPhraseList( stack, fq );
assertEquals( 1, fpl.phraseList.size() );
assertEquals( "personalcomputer(1.0)((3,5))", fpl.phraseList.get( 0 ).toString() );
assertEquals( 3, fpl.phraseList.get( 0 ).getStartOffset() );
assertEquals( 5, fpl.phraseList.get( 0 ).getEndOffset() );
}
public void testFieldPhraseListIndex1w2wSearch1partial() throws Exception {
makeIndex1w2w();
FieldQuery fq = new FieldQuery( tq( "computer" ), true, true );
FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
FieldPhraseList fpl = new FieldPhraseList( stack, fq );
assertEquals( 1, fpl.phraseList.size() );
assertEquals( "computer(1.0)((3,5))", fpl.phraseList.get( 0 ).toString() );
assertEquals( 3, fpl.phraseList.get( 0 ).getStartOffset() );
assertEquals( 5, fpl.phraseList.get( 0 ).getEndOffset() );
}
public void testFieldPhraseListIndex1w2wSearch1term1phrase() throws Exception {
makeIndex1w2w();
BooleanQuery bq = new BooleanQuery();
bq.add( tq( "pc" ), Occur.SHOULD );
bq.add( pqF( "personal", "computer" ), Occur.SHOULD );
FieldQuery fq = new FieldQuery( bq, true, true );
FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
FieldPhraseList fpl = new FieldPhraseList( stack, fq );
assertEquals( 1, fpl.phraseList.size() );
assertTrue( fpl.phraseList.get( 0 ).toString().indexOf( "(1.0)((3,5))" ) > 0 );
assertEquals( 3, fpl.phraseList.get( 0 ).getStartOffset() );
assertEquals( 5, fpl.phraseList.get( 0 ).getEndOffset() );
}
public void testFieldPhraseListIndex2w1wSearch1term() throws Exception {
makeIndex2w1w();
FieldQuery fq = new FieldQuery( tq( "pc" ), true, true );
FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
FieldPhraseList fpl = new FieldPhraseList( stack, fq );
assertEquals( 1, fpl.phraseList.size() );
assertEquals( "pc(1.0)((3,20))", fpl.phraseList.get( 0 ).toString() );
assertEquals( 3, fpl.phraseList.get( 0 ).getStartOffset() );
assertEquals( 20, fpl.phraseList.get( 0 ).getEndOffset() );
}
public void testFieldPhraseListIndex2w1wSearch1phrase() throws Exception {
makeIndex2w1w();
FieldQuery fq = new FieldQuery( pqF( "personal", "computer" ), true, true );
FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
FieldPhraseList fpl = new FieldPhraseList( stack, fq );
assertEquals( 1, fpl.phraseList.size() );
assertEquals( "personalcomputer(1.0)((3,20))", fpl.phraseList.get( 0 ).toString() );
assertEquals( 3, fpl.phraseList.get( 0 ).getStartOffset() );
assertEquals( 20, fpl.phraseList.get( 0 ).getEndOffset() );
}
public void testFieldPhraseListIndex2w1wSearch1partial() throws Exception {
makeIndex2w1w();
FieldQuery fq = new FieldQuery( tq( "computer" ), true, true );
FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
FieldPhraseList fpl = new FieldPhraseList( stack, fq );
assertEquals( 1, fpl.phraseList.size() );
assertEquals( "computer(1.0)((3,20))", fpl.phraseList.get( 0 ).toString() );
assertEquals( 3, fpl.phraseList.get( 0 ).getStartOffset() );
assertEquals( 20, fpl.phraseList.get( 0 ).getEndOffset() );
}
public void testFieldPhraseListIndex2w1wSearch1term1phrase() throws Exception {
makeIndex2w1w();
BooleanQuery bq = new BooleanQuery();
bq.add( tq( "pc" ), Occur.SHOULD );
bq.add( pqF( "personal", "computer" ), Occur.SHOULD );
FieldQuery fq = new FieldQuery( bq, true, true );
FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
FieldPhraseList fpl = new FieldPhraseList( stack, fq );
assertEquals( 1, fpl.phraseList.size() );
assertTrue( fpl.phraseList.get( 0 ).toString().indexOf( "(1.0)((3,20))" ) > 0 );
assertEquals( 3, fpl.phraseList.get( 0 ).getStartOffset() );
assertEquals( 20, fpl.phraseList.get( 0 ).getEndOffset() );
}
private void makeIndex1w() throws Exception {
// 11111111112
// 012345678901234567890
// I'll buy a Macintosh
// Mac
// MacBook
// 0 1 2 3
makeSynonymIndex( "I'll buy a Macintosh",
t("I'll",0,4),
t("buy",5,8),
t("a",9,10),
t("Macintosh",11,20),t("Mac",11,20,0),t("MacBook",11,20,0));
}
private void makeIndex1w2w() throws Exception {
// 1111111
// 01234567890123456
// My pc was broken
// personal computer
// 0 1 2 3
makeSynonymIndex( "My pc was broken",
t("My",0,2),
t("pc",3,5),t("personal",3,5,0),t("computer",3,5),
t("was",6,9),
t("broken",10,16));
}
private void makeIndex2w1w() throws Exception {
// 1111111111222222222233
// 01234567890123456789012345678901
// My personal computer was broken
// pc
// 0 1 2 3 4
makeSynonymIndex( "My personal computer was broken",
t("My",0,2),
t("personal",3,20),t("pc",3,20,0),t("computer",3,20),
t("was",21,24),
t("broken",25,31));
}
void makeSynonymIndex( String value, Token... tokens ) throws Exception {
Analyzer analyzer = new TokenArrayAnalyzer( tokens );
make1dmfIndex( analyzer, value );
}
public static Token t( String text, int startOffset, int endOffset ){
return t( text, startOffset, endOffset, 1 );
}
public static Token t( String text, int startOffset, int endOffset, int positionIncrement ){
Token token = new Token( text, startOffset, endOffset );
token.setPositionIncrement( positionIncrement );
return token;
}
public static class TokenArrayAnalyzer extends Analyzer {
Token[] tokens;
public TokenArrayAnalyzer( Token... tokens ){
this.tokens = tokens;
}
public TokenStream tokenStream(String fieldName, Reader reader) {
return new TokenStream(){
int p = 0;
public Token next( Token reusableToken ) throws IOException {
if( p >= tokens.length ) return null;
return tokens[p++];
}
};
}
}
}

43
contrib/fast-vector-highlighter/src/test/org/apache/lucene/search/vectorhighlight/ScoreOrderFragmentsBuilderTest.java Normal file
View File

@ -0,0 +1,43 @@
package org.apache.lucene.search.vectorhighlight;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import org.apache.lucene.search.Query;
public class ScoreOrderFragmentsBuilderTest extends AbstractTestCase {
public void test3Frags() throws Exception {
FieldFragList ffl = ffl( "a c", "a b b b b b b b b b b b a b a b b b b b c a a b b" );
ScoreOrderFragmentsBuilder sofb = new ScoreOrderFragmentsBuilder();
String[] f = sofb.createFragments( reader, 0, F, ffl, 3 );
assertEquals( 3, f.length );
// check score order
assertEquals( "<b>c</b> <b>a</b> <b>a</b> b b", f[0] );
assertEquals( "b b <b>a</b> b <b>a</b> b b b b b ", f[1] );
assertEquals( "<b>a</b> b b b b b b b b b ", f[2] );
}
private FieldFragList ffl( String queryValue, String indexValue ) throws Exception {
make1d1fIndex( indexValue );
Query query = paW.parse( queryValue );
FieldQuery fq = new FieldQuery( query, true, true );
FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
FieldPhraseList fpl = new FieldPhraseList( stack, fq );
return new SimpleFragListBuilder().createFieldFragList( fpl, 20 );
}
}

162
contrib/fast-vector-highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleFragListBuilderTest.java Normal file
View File

@ -0,0 +1,162 @@
package org.apache.lucene.search.vectorhighlight;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import org.apache.lucene.search.Query;
public class SimpleFragListBuilderTest extends AbstractTestCase {
public void testNullFieldFragList() throws Exception {
SimpleFragListBuilder sflb = new SimpleFragListBuilder();
FieldFragList ffl = sflb.createFieldFragList( fpl( "a", "b c d" ), 100 );
assertEquals( 0, ffl.fragInfos.size() );
}
public void testTooSmallFragSize() throws Exception {
try{
SimpleFragListBuilder sflb = new SimpleFragListBuilder();
sflb.createFieldFragList( fpl( "a", "b c d" ), SimpleFragListBuilder.MIN_FRAG_CHAR_SIZE - 1 );
fail( "IllegalArgumentException must be thrown" );
}
catch ( IllegalArgumentException expected ) {
}
}
public void test1TermIndex() throws Exception {
SimpleFragListBuilder sflb = new SimpleFragListBuilder();
FieldFragList ffl = sflb.createFieldFragList( fpl( "a", "a" ), 100 );
assertEquals( 1, ffl.fragInfos.size() );
assertEquals( "subInfos=(a((0,1)))/1.0(0,100)", ffl.fragInfos.get( 0 ).toString() );
}
public void test2TermsIndex1Frag() throws Exception {
SimpleFragListBuilder sflb = new SimpleFragListBuilder();
FieldFragList ffl = sflb.createFieldFragList( fpl( "a", "a a" ), 100 );
assertEquals( 1, ffl.fragInfos.size() );
assertEquals( "subInfos=(a((0,1))a((2,3)))/2.0(0,100)", ffl.fragInfos.get( 0 ).toString() );
ffl = sflb.createFieldFragList( fpl( "a", "a b b b b b b b b a" ), 20 );
assertEquals( 1, ffl.fragInfos.size() );
assertEquals( "subInfos=(a((0,1))a((18,19)))/2.0(0,20)", ffl.fragInfos.get( 0 ).toString() );
ffl = sflb.createFieldFragList( fpl( "a", "b b b b a b b b b a" ), 20 );
assertEquals( 1, ffl.fragInfos.size() );
assertEquals( "subInfos=(a((8,9))a((18,19)))/2.0(2,22)", ffl.fragInfos.get( 0 ).toString() );
}
public void test2TermsIndex2Frags() throws Exception {
SimpleFragListBuilder sflb = new SimpleFragListBuilder();
FieldFragList ffl = sflb.createFieldFragList( fpl( "a", "a b b b b b b b b b b b b b a" ), 20 );
assertEquals( 2, ffl.fragInfos.size() );
assertEquals( "subInfos=(a((0,1)))/1.0(0,20)", ffl.fragInfos.get( 0 ).toString() );
assertEquals( "subInfos=(a((28,29)))/1.0(22,42)", ffl.fragInfos.get( 1 ).toString() );
ffl = sflb.createFieldFragList( fpl( "a", "a b b b b b b b b b b b b a" ), 20 );
assertEquals( 2, ffl.fragInfos.size() );
assertEquals( "subInfos=(a((0,1)))/1.0(0,20)", ffl.fragInfos.get( 0 ).toString() );
assertEquals( "subInfos=(a((26,27)))/1.0(20,40)", ffl.fragInfos.get( 1 ).toString() );
ffl = sflb.createFieldFragList( fpl( "a", "a b b b b b b b b b a" ), 20 );
assertEquals( 2, ffl.fragInfos.size() );
assertEquals( "subInfos=(a((0,1)))/1.0(0,20)", ffl.fragInfos.get( 0 ).toString() );
assertEquals( "subInfos=(a((20,21)))/1.0(20,40)", ffl.fragInfos.get( 1 ).toString() );
}
public void test2TermsQuery() throws Exception {
SimpleFragListBuilder sflb = new SimpleFragListBuilder();
FieldFragList ffl = sflb.createFieldFragList( fpl( "a b", "c d e" ), 20 );
assertEquals( 0, ffl.fragInfos.size() );
ffl = sflb.createFieldFragList( fpl( "a b", "d b c" ), 20 );
assertEquals( 1, ffl.fragInfos.size() );
assertEquals( "subInfos=(b((2,3)))/1.0(0,20)", ffl.fragInfos.get( 0 ).toString() );
ffl = sflb.createFieldFragList( fpl( "a b", "a b c" ), 20 );
assertEquals( 1, ffl.fragInfos.size() );
assertEquals( "subInfos=(a((0,1))b((2,3)))/2.0(0,20)", ffl.fragInfos.get( 0 ).toString() );
}
public void testPhraseQuery() throws Exception {
SimpleFragListBuilder sflb = new SimpleFragListBuilder();
FieldFragList ffl = sflb.createFieldFragList( fpl( "\"a b\"", "c d e" ), 20 );
assertEquals( 0, ffl.fragInfos.size() );
ffl = sflb.createFieldFragList( fpl( "\"a b\"", "a c b" ), 20 );
assertEquals( 0, ffl.fragInfos.size() );
ffl = sflb.createFieldFragList( fpl( "\"a b\"", "a b c" ), 20 );
assertEquals( 1, ffl.fragInfos.size() );
assertEquals( "subInfos=(ab((0,3)))/1.0(0,20)", ffl.fragInfos.get( 0 ).toString() );
}
public void testPhraseQuerySlop() throws Exception {
SimpleFragListBuilder sflb = new SimpleFragListBuilder();
FieldFragList ffl = sflb.createFieldFragList( fpl( "\"a b\"~1", "a c b" ), 20 );
assertEquals( 1, ffl.fragInfos.size() );
assertEquals( "subInfos=(ab((0,1)(4,5)))/1.0(0,20)", ffl.fragInfos.get( 0 ).toString() );
}
private FieldPhraseList fpl( String queryValue, String indexValue ) throws Exception {
make1d1fIndex( indexValue );
Query query = paW.parse( queryValue );
FieldQuery fq = new FieldQuery( query, true, true );
FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
return new FieldPhraseList( stack, fq );
}
public void test1PhraseShortMV() throws Exception {
makeIndexShortMV();
FieldQuery fq = new FieldQuery( tq( "d" ), true, true );
FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
FieldPhraseList fpl = new FieldPhraseList( stack, fq );
SimpleFragListBuilder sflb = new SimpleFragListBuilder();
FieldFragList ffl = sflb.createFieldFragList( fpl, 100 );
assertEquals( 1, ffl.fragInfos.size() );
assertEquals( "subInfos=(d((6,7)))/1.0(0,100)", ffl.fragInfos.get( 0 ).toString() );
}
public void test1PhraseLongMV() throws Exception {
makeIndexLongMV();
FieldQuery fq = new FieldQuery( pqF( "search", "engines" ), true, true );
FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
FieldPhraseList fpl = new FieldPhraseList( stack, fq );
SimpleFragListBuilder sflb = new SimpleFragListBuilder();
FieldFragList ffl = sflb.createFieldFragList( fpl, 100 );
assertEquals( 1, ffl.fragInfos.size() );
assertEquals( "subInfos=(searchengines((102,116))searchengines((157,171)))/2.0(96,196)", ffl.fragInfos.get( 0 ).toString() );
}
/*
* ----------------------------------
* THIS TEST DEPENDS ON LUCENE-1448
* UNCOMMENT WHEN IT IS COMMITTED.
* ----------------------------------
public void test1PhraseLongMVB() throws Exception {
makeIndexLongMVB();
FieldQuery fq = new FieldQuery( pqF( "sp", "pe", "ee", "ed" ), true, true ); // "speed" -(2gram)-> "sp","pe","ee","ed"
FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
FieldPhraseList fpl = new FieldPhraseList( stack, fq );
SimpleFragListBuilder sflb = new SimpleFragListBuilder();
FieldFragList ffl = sflb.createFieldFragList( fpl, 100 );
assertEquals( 1, ffl.fragInfos.size() );
assertEquals( "subInfos=(sppeeeed((88,93)))/1.0(82,182)", ffl.fragInfos.get( 0 ).toString() );
}
*/
}

104
contrib/fast-vector-highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilderTest.java Normal file
View File

@ -0,0 +1,104 @@
package org.apache.lucene.search.vectorhighlight;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import org.apache.lucene.search.Query;
public class SimpleFragmentsBuilderTest extends AbstractTestCase {
public void test1TermIndex() throws Exception {
FieldFragList ffl = ffl( "a", "a" );
SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();
assertEquals( "<b>a</b>", sfb.createFragment( reader, 0, F, ffl ) );
// change tags
sfb = new SimpleFragmentsBuilder( new String[]{ "[" }, new String[]{ "]" } );
assertEquals( "[a]", sfb.createFragment( reader, 0, F, ffl ) );
}
public void test2Frags() throws Exception {
FieldFragList ffl = ffl( "a", "a b b b b b b b b b b b a b a b" );
SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();
String[] f = sfb.createFragments( reader, 0, F, ffl, 3 );
// 3 snippets requested, but should be 2
assertEquals( 2, f.length );
assertEquals( "<b>a</b> b b b b b b b b b ", f[0] );
assertEquals( "b b <b>a</b> b <b>a</b> b", f[1] );
}
public void test3Frags() throws Exception {
FieldFragList ffl = ffl( "a c", "a b b b b b b b b b b b a b a b b b b b c a a b b" );
SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();
String[] f = sfb.createFragments( reader, 0, F, ffl, 3 );
assertEquals( 3, f.length );
assertEquals( "<b>a</b> b b b b b b b b b ", f[0] );
assertEquals( "b b <b>a</b> b <b>a</b> b b b b b ", f[1] );
assertEquals( "<b>c</b> <b>a</b> <b>a</b> b b", f[2] );
}
private FieldFragList ffl( String queryValue, String indexValue ) throws Exception {
make1d1fIndex( indexValue );
Query query = paW.parse( queryValue );
FieldQuery fq = new FieldQuery( query, true, true );
FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
FieldPhraseList fpl = new FieldPhraseList( stack, fq );
return new SimpleFragListBuilder().createFieldFragList( fpl, 20 );
}
public void test1PhraseShortMV() throws Exception {
makeIndexShortMV();
FieldQuery fq = new FieldQuery( tq( "d" ), true, true );
FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
FieldPhraseList fpl = new FieldPhraseList( stack, fq );
SimpleFragListBuilder sflb = new SimpleFragListBuilder();
FieldFragList ffl = sflb.createFieldFragList( fpl, 100 );
SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();
assertEquals( "a b c <b>d</b> e", sfb.createFragment( reader, 0, F, ffl ) );
}
public void test1PhraseLongMV() throws Exception {
makeIndexLongMV();
FieldQuery fq = new FieldQuery( pqF( "search", "engines" ), true, true );
FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
FieldPhraseList fpl = new FieldPhraseList( stack, fq );
SimpleFragListBuilder sflb = new SimpleFragListBuilder();
FieldFragList ffl = sflb.createFieldFragList( fpl, 100 );
SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();
assertEquals( " most <b>search engines</b> use only one of these methods. Even the <b>search engines</b> that says they can use t",
sfb.createFragment( reader, 0, F, ffl ) );
}
/*
* ----------------------------------
* THIS TEST DEPENDS ON LUCENE-1448
* UNCOMMENT WHEN IT IS COMMITTED.
* ----------------------------------
public void test1PhraseLongMVB() throws Exception {
makeIndexLongMVB();
FieldQuery fq = new FieldQuery( pqF( "sp", "pe", "ee", "ed" ), true, true ); // "speed" -(2gram)-> "sp","pe","ee","ed"
FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
FieldPhraseList fpl = new FieldPhraseList( stack, fq );
SimpleFragListBuilder sflb = new SimpleFragListBuilder();
FieldFragList ffl = sflb.createFieldFragList( fpl, 100 );
SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();
assertEquals( "ssing <b>speed</b>, the", sfb.createFragment( reader, 0, F, ffl ) );
}
*/
}

5
docs/benchmarks.html
View File

@ -50,7 +50,7 @@
+-->
<div class="searchbox">
<form action="http://search.lucidimagination.com/p:lucene" method="get" class="roundtopsmall">
<input onFocus="getBlank (this, 'Search the site with google');" size="25" name="q" id="query" type="text" value="Search the site with google">&nbsp;
<input onFocus="getBlank (this, 'Search the site with Lucene');" size="25" name="q" id="query" type="text" value="Search the site with Lucene">&nbsp;
<input name="Search" value="Search" type="submit">
</form>
<div style="position: relative; top: -5px; left: -10px">Powered by <a href="http://www.lucidimagination.com" style="color: #033268">Lucid Imagination</a>
@ -142,6 +142,9 @@ document.write("Last Published: " + document.lastModified);
<a href="api/contrib-collation/index.html">Collation</a>
</div>
<div class="menuitem">
<a href="api/contrib-fast-vector-highlighter/index.html">Fast Vector Highlighter</a>
</div>
<div class="menuitem">
<a href="api/contrib-highlighter/index.html">Highlighter</a>
</div>
<div class="menuitem">

12
docs/broken-links.xml
View File

@ -1,14 +1,2 @@
<broken-links>
<link message="/usr/local/apache-forrest-0.8/main/webapp/. (No such file or directory)" uri="skin/images/chapter.gif">
<referrer uri="skin/screen.css"/>
</link>
<link message="/usr/local/apache-forrest-0.8/main/webapp/. (No such file or directory)" uri="skin/images/page.gif">
<referrer uri="skin/screen.css"/>
</link>
<link message="/usr/local/apache-forrest-0.8/main/webapp/. (No such file or directory)" uri="skin/images/current.gif">
<referrer uri="skin/screen.css"/>
</link>
<link message="/Users/grantingersoll/projects/lucene/java/clean/src/site/src/documentation/content/xdocs/images.instruction_arrow.png (No such file or directory)" uri="images/instruction_arrow.png">
<referrer uri="skin/screen.css"/>
</link>
</broken-links>

5
docs/contributions.html
View File

@ -52,7 +52,7 @@
+-->
<div class="searchbox">
<form action="http://search.lucidimagination.com/p:lucene" method="get" class="roundtopsmall">
<input onFocus="getBlank (this, 'Search the site with google');" size="25" name="q" id="query" type="text" value="Search the site with google">&nbsp;
<input onFocus="getBlank (this, 'Search the site with Lucene');" size="25" name="q" id="query" type="text" value="Search the site with Lucene">&nbsp;
<input name="Search" value="Search" type="submit">
</form>
<div style="position: relative; top: -5px; left: -10px">Powered by <a href="http://www.lucidimagination.com" style="color: #033268">Lucid Imagination</a>
@ -144,6 +144,9 @@ document.write("Last Published: " + document.lastModified);
<a href="api/contrib-collation/index.html">Collation</a>
</div>
<div class="menuitem">
<a href="api/contrib-fast-vector-highlighter/index.html">Fast Vector Highlighter</a>
</div>
<div class="menuitem">
<a href="api/contrib-highlighter/index.html">Highlighter</a>
</div>
<div class="menuitem">

5
docs/demo.html
View File

@ -52,7 +52,7 @@
+-->
<div class="searchbox">
<form action="http://search.lucidimagination.com/p:lucene" method="get" class="roundtopsmall">
<input onFocus="getBlank (this, 'Search the site with google');" size="25" name="q" id="query" type="text" value="Search the site with google">&nbsp;
<input onFocus="getBlank (this, 'Search the site with Lucene');" size="25" name="q" id="query" type="text" value="Search the site with Lucene">&nbsp;
<input name="Search" value="Search" type="submit">
</form>
<div style="position: relative; top: -5px; left: -10px">Powered by <a href="http://www.lucidimagination.com" style="color: #033268">Lucid Imagination</a>
@ -144,6 +144,9 @@ document.write("Last Published: " + document.lastModified);
<a href="api/contrib-collation/index.html">Collation</a>
</div>
<div class="menuitem">
<a href="api/contrib-fast-vector-highlighter/index.html">Fast Vector Highlighter</a>
</div>
<div class="menuitem">
<a href="api/contrib-highlighter/index.html">Highlighter</a>
</div>
<div class="menuitem">

5
docs/demo2.html
View File

@ -52,7 +52,7 @@
+-->
<div class="searchbox">
<form action="http://search.lucidimagination.com/p:lucene" method="get" class="roundtopsmall">
<input onFocus="getBlank (this, 'Search the site with google');" size="25" name="q" id="query" type="text" value="Search the site with google">&nbsp;
<input onFocus="getBlank (this, 'Search the site with Lucene');" size="25" name="q" id="query" type="text" value="Search the site with Lucene">&nbsp;
<input name="Search" value="Search" type="submit">
</form>
<div style="position: relative; top: -5px; left: -10px">Powered by <a href="http://www.lucidimagination.com" style="color: #033268">Lucid Imagination</a>
@ -144,6 +144,9 @@ document.write("Last Published: " + document.lastModified);
<a href="api/contrib-collation/index.html">Collation</a>
</div>
<div class="menuitem">
<a href="api/contrib-fast-vector-highlighter/index.html">Fast Vector Highlighter</a>
</div>
<div class="menuitem">
<a href="api/contrib-highlighter/index.html">Highlighter</a>
</div>
<div class="menuitem">

5
docs/demo3.html
View File

@ -52,7 +52,7 @@
+-->
<div class="searchbox">
<form action="http://search.lucidimagination.com/p:lucene" method="get" class="roundtopsmall">
<input onFocus="getBlank (this, 'Search the site with google');" size="25" name="q" id="query" type="text" value="Search the site with google">&nbsp;
<input onFocus="getBlank (this, 'Search the site with Lucene');" size="25" name="q" id="query" type="text" value="Search the site with Lucene">&nbsp;
<input name="Search" value="Search" type="submit">
</form>
<div style="position: relative; top: -5px; left: -10px">Powered by <a href="http://www.lucidimagination.com" style="color: #033268">Lucid Imagination</a>
@ -144,6 +144,9 @@ document.write("Last Published: " + document.lastModified);
<a href="api/contrib-collation/index.html">Collation</a>
</div>
<div class="menuitem">
<a href="api/contrib-fast-vector-highlighter/index.html">Fast Vector Highlighter</a>
</div>
<div class="menuitem">
<a href="api/contrib-highlighter/index.html">Highlighter</a>
</div>
<div class="menuitem">

5
docs/demo4.html
View File

@ -52,7 +52,7 @@
+-->
<div class="searchbox">
<form action="http://search.lucidimagination.com/p:lucene" method="get" class="roundtopsmall">
<input onFocus="getBlank (this, 'Search the site with google');" size="25" name="q" id="query" type="text" value="Search the site with google">&nbsp;
<input onFocus="getBlank (this, 'Search the site with Lucene');" size="25" name="q" id="query" type="text" value="Search the site with Lucene">&nbsp;
<input name="Search" value="Search" type="submit">
</form>
<div style="position: relative; top: -5px; left: -10px">Powered by <a href="http://www.lucidimagination.com" style="color: #033268">Lucid Imagination</a>
@ -144,6 +144,9 @@ document.write("Last Published: " + document.lastModified);
<a href="api/contrib-collation/index.html">Collation</a>
</div>
<div class="menuitem">
<a href="api/contrib-fast-vector-highlighter/index.html">Fast Vector Highlighter</a>
</div>
<div class="menuitem">
<a href="api/contrib-highlighter/index.html">Highlighter</a>
</div>
<div class="menuitem">

5
docs/fileformats.html
View File

@ -52,7 +52,7 @@
+-->
<div class="searchbox">
<form action="http://search.lucidimagination.com/p:lucene" method="get" class="roundtopsmall">
<input onFocus="getBlank (this, 'Search the site with google');" size="25" name="q" id="query" type="text" value="Search the site with google">&nbsp;
<input onFocus="getBlank (this, 'Search the site with Lucene');" size="25" name="q" id="query" type="text" value="Search the site with Lucene">&nbsp;
<input name="Search" value="Search" type="submit">
</form>
<div style="position: relative; top: -5px; left: -10px">Powered by <a href="http://www.lucidimagination.com" style="color: #033268">Lucid Imagination</a>
@ -144,6 +144,9 @@ document.write("Last Published: " + document.lastModified);
<a href="api/contrib-collation/index.html">Collation</a>
</div>
<div class="menuitem">
<a href="api/contrib-fast-vector-highlighter/index.html">Fast Vector Highlighter</a>
</div>
<div class="menuitem">
<a href="api/contrib-highlighter/index.html">Highlighter</a>
</div>
<div class="menuitem">

5
docs/gettingstarted.html
View File

@ -52,7 +52,7 @@
+-->
<div class="searchbox">
<form action="http://search.lucidimagination.com/p:lucene" method="get" class="roundtopsmall">
<input onFocus="getBlank (this, 'Search the site with google');" size="25" name="q" id="query" type="text" value="Search the site with google">&nbsp;
<input onFocus="getBlank (this, 'Search the site with Lucene');" size="25" name="q" id="query" type="text" value="Search the site with Lucene">&nbsp;
<input name="Search" value="Search" type="submit">
</form>
<div style="position: relative; top: -5px; left: -10px">Powered by <a href="http://www.lucidimagination.com" style="color: #033268">Lucid Imagination</a>
@ -144,6 +144,9 @@ document.write("Last Published: " + document.lastModified);
<a href="api/contrib-collation/index.html">Collation</a>
</div>
<div class="menuitem">
<a href="api/contrib-fast-vector-highlighter/index.html">Fast Vector Highlighter</a>
</div>
<div class="menuitem">
<a href="api/contrib-highlighter/index.html">Highlighter</a>
</div>
<div class="menuitem">

5
docs/index.html
View File

@ -50,7 +50,7 @@
+-->
<div class="searchbox">
<form action="http://search.lucidimagination.com/p:lucene" method="get" class="roundtopsmall">
<input onFocus="getBlank (this, 'Search the site with google');" size="25" name="q" id="query" type="text" value="Search the site with google">&nbsp;
<input onFocus="getBlank (this, 'Search the site with Lucene');" size="25" name="q" id="query" type="text" value="Search the site with Lucene">&nbsp;
<input name="Search" value="Search" type="submit">
</form>
<div style="position: relative; top: -5px; left: -10px">Powered by <a href="http://www.lucidimagination.com" style="color: #033268">Lucid Imagination</a>
@ -142,6 +142,9 @@ document.write("Last Published: " + document.lastModified);
<a href="api/contrib-collation/index.html">Collation</a>
</div>
<div class="menuitem">
<a href="api/contrib-fast-vector-highlighter/index.html">Fast Vector Highlighter</a>
</div>
<div class="menuitem">
<a href="api/contrib-highlighter/index.html">Highlighter</a>
</div>
<div class="menuitem">

11
docs/linkmap.html
View File

@ -50,7 +50,7 @@
+-->
<div class="searchbox">
<form action="http://search.lucidimagination.com/p:lucene" method="get" class="roundtopsmall">
<input onFocus="getBlank (this, 'Search the site with google');" size="25" name="q" id="query" type="text" value="Search the site with google">&nbsp;
<input onFocus="getBlank (this, 'Search the site with Lucene');" size="25" name="q" id="query" type="text" value="Search the site with Lucene">&nbsp;
<input name="Search" value="Search" type="submit">
</form>
<div style="position: relative; top: -5px; left: -10px">Powered by <a href="http://www.lucidimagination.com" style="color: #033268">Lucid Imagination</a>
@ -142,6 +142,9 @@ document.write("Last Published: " + document.lastModified);
<a href="api/contrib-collation/index.html">Collation</a>
</div>
<div class="menuitem">
<a href="api/contrib-fast-vector-highlighter/index.html">Fast Vector Highlighter</a>
</div>
<div class="menuitem">
<a href="api/contrib-highlighter/index.html">Highlighter</a>
</div>
<div class="menuitem">
@ -329,6 +332,12 @@ document.write("Last Published: " + document.lastModified);
</li>
</ul>
<ul>
<li>
<a href="api/contrib-fast-vector-highlighter/index.html">Fast Vector Highlighter</a>&nbsp;&nbsp;___________________&nbsp;&nbsp;<em>javadoc-contrib-fast-vector-highlighter</em>
</li>
</ul>
<ul>
<li>
<a href="api/contrib-highlighter/index.html">Highlighter</a>&nbsp;&nbsp;___________________&nbsp;&nbsp;<em>javadoc-contrib-highlighter</em>

32
docs/linkmap.pdf
View File

@ -5,10 +5,10 @@
/Producer (FOP 0.20.5) >>
endobj
5 0 obj
<< /Length 1110 /Filter [ /ASCII85Decode /FlateDecode ]
<< /Length 1101 /Filter [ /ASCII85Decode /FlateDecode ]
>>
stream
GatUt>u03/'L:RQ/+NrVp&(BAOg:YiJL3kD`\`D!cp4$bikj?OT:Md;S^(=m+A2"!pDi)c`_I;mfr$V=@1$DL]UM4c0;$)EIYK&FISl?p5#*7D4GfAnki^?9+iSFXa)Rj!**j9/j!K*k4-nQ:Bq2Bq`oGOZP+n-,EV(bbSguRS&FRXUNl<a2OS63G\nH3lVU$6Ma4Y\s)"`e/fcSDF(`Rb6Gq+M)HJ65PA_"pG8H[McAt-t\,J`.V3S[aEdU[=3_3hnl=9NbM(l*Y07XMDb7AZ7dm(g@HJ+`p=5sWXm5S1Clp1&2;@'1e(!Z5+pF+jD+J$/D17U<Js(3L@l7OS:OMJ^_?cJP*"M!GP*eYbe%qho,o$Lo$N`CQUTP>Ue$qTL&dM8S[BHcc-CL#u$68o,^nqO$_sV0YVYKT,R[nN&,Bn79Lg&8m7FSE?6"lgs+XS<ahle%,jWp4P5eX2L%5'?e#'4u:@:dpHqV@'^^CAnLt3fr>GH1XkIMbZQ6]i\k$o!Be@[@SGMfgZaKM?8X.lc5J[AZL[CdBS).-RrOnLOGVpY)^uh0P/Qe"W+9ig;a(Y>kU?CQ4E4EB@MO?Woe]QTXmf9(gQhe2lT:917[2g.+h2tMJq.oW'E4cZ+afRY3pU?#WdMM%R`Af56M+"*@N9WQ&>,s@[]=LFla$H8H:ueG`a`mRVQT_cn[RQm3D8ONEPF-VT+oP*'!!d'@A^JKPN27oZ*Th3<NT4(=_(F34sI);1DXUQ>Z`,@Z.b)Fb"M+/b9C%'%_4Ft;q80$p`/t)_spX,B$FQT/KjEr8rGBgg^FI\-)/\r\h)Q?W(C?QH5jRL;7."Qe-6r7.[!jJV9E(C='O&d,$YcF9p&/;3^3!D50]HVX6GDL(Os<bVV6W@48r3bklEt:[5iApcOVmOj9J^5f%#gM30`bA5"h%u$g>1%?Z3:?m5`.]gR_W.WFrhuFE,#GB@#DoIeW2d?jpK[dEVsubb@QpGI+@'6GFjD)1Z+>9p";#1?0qe@LT'&74A6gm`DC"dp&U9f&QB%%;'9i@0jF6g0ed11cTd'EjL1"hoXKO0m_.1D7/Yp/ms(/9A\%ZfWEO:Xa7&^.khJ3+&,?U3<~>
Gatn&?#Q2d'Sc)J/%BCKiqXP'CpI&FeS<eVm:Mn9$;Nr]+9a,upZlGo$(/q&F4[?"2X:0'[a24,ArFdC9B'@eTXS#uT`0frd!M-&&#p026J][2"G@$:@dSS.]1o+]pSQRRe=3f11k/\!S$$65h.OFq`Nji^(V(6jXNnL$hVq-f1CrHafs"u#%lgAEh-uXM>(YutBDq"lHYa"W7Ut+6bE_90EO"A26M>`*[,6c3Uus"M<g>5FZ.tWH*8$C-SV80p>4t%:fnmNhW2G>h#j?`Ef@sEg)Mu;apuh1n6'*/p@NGT&\1-*id11`<L7ef[BOFUJC'O^PLpq0-DNRVNg_GfF!l>M<7S+2a$3nFk#K,i1"mtnVPe0o-3)_Q[9N^2I3/ETW5n!f&fi>D;Rc1g&QWd!t>!+^3Vmo&6;)BD$rX?:hZHRa&Hp@rrj]eTrUij_f3:4S'^bmc;kHH>(WOW@0rB47f$]gEIAe0Br/tC%2e<'@rpLHiI4@Z3sdlM-P/CSri\,^IiRhjhe[G"YF9_>;M+h-UL736$D^I@CW"Ib:u^bco?:8?*GKTu\S4YLj?4HCX-MB=XsEB5.CZX]<2YHeW_-#:R4q<jI:Q.A$[/OpN4o[,[K.>!(kC-)7rZ*kLE,g_u51Qf_YF]:ObK+an;A]*PaZSNId<Pu8AcFh6(2k@EWSS6gAVTFoWmT`4`EJU]qEu@u`]./4BYq4$un0#TTWkJZY6L\L]YfbJ!']kgM6`l-;3um9Yd7ngBe9'78*#gW=L\q*^B.%0fT6R=un3?$"<*N0;8WgXh=C9tpT-8/g(l],;Hi0pkE0`P[_KKheM:g,mLG;83O8Bl=I[@8+RpK3(%1J"8jLV`9TgSpi*Z[1R'ap;h\L=oS#;K4[5>4eH,?X%aNlm29XhJU?R0n./Up#?H_iXD6D_Zu<WAU0ZM?i[p4Z:ieMjsQ1aKdP!2*NQ<cZMN[=-j3h^h4^j5p3te.ttI&I$r3#+g8lq6!MD8fVbfPa!8ShaI*hW_#22q?>Nq@]t8A$qAj1HCLIKoYHVC,b5.#j'_g@k`DXE<AZt:&f]GbR\:EGNe^[1EW4f6<>?%Wl&ddcVrWf_4&fC~>
endstream
endobj
6 0 obj
@ -20,10 +20,10 @@ endobj
>>
endobj
7 0 obj
<< /Length 913 /Filter [ /ASCII85Decode /FlateDecode ]
<< /Length 971 /Filter [ /ASCII85Decode /FlateDecode ]
>>
stream
GatUr?#SFN'Sc)P'u"K;iqXhuaCV?jUoBPYJ^d9W1n!8e7S%2HV>^.05(FL27S;Y))"PD%STB2(GW,CR@/pss^ap?HYT.)%lk]aj6bCDWLE)"U[K4-]SdAi7+Ma?fqpWf3X%]NK]>@CXnFud#$iY\0GAG&k-:`\"@n8%m/cM_N[4(#W/ojmfo3ce7BP#h]9Zp!2%)&/lbMneIf=Dq19-Y9FfB/O0ldH)t,pY\4I9@?_HncJ(&BK.\H<ejh9R964'LCe)4f%mo0IDsE$fW(0@5\UDN$<l%_t]J!e#crPCA;cU(e)A#\`>lC?2@Y`ns7d(;(cm+L6kl)V:)c],!C8EEJE+M;SRZCfSDB#j'J)@XkZL"<LSkg'(DeKGFp9K,atit%s%M`P6ETS9sAKZ]R.T0#"8IG*pYppD,e(&%:uE,29]?mGt5;u/sQ@:^2LHm2eeBb#8D=^'A"XUQ2;DP5FX$EO+YP&\eIri90WmWkNjJ=oJpV57Cc:X'<bHh="7Ln`&\%C;7I.%&oRmIVN#:Bes:Jo<,)!_L-UI"2]-a%c/'/ra1\l,`RJ*;&,TJlSR\4jHnd;2#(oMX.H[dE<dOt]UN@#!NMs=$g8jL,qjn`a*7e.$-)#,,>X,i$\.;Smroko@DeF<9V'37q'lbgK8'O4M`k5k^[F<H%[A/2Ljk;'fO>+>;A0dj6fR1I[C*&e?VTq/04+cD/-=N@H2eqZ[c."jss&C=>=C%R\58EJ8g$#9(J"m>_gMS7'XmRZ(O>'LGOGk5Nj(_uAQ^H"V;+X%f67JX=]3DlGUpEdSg4A#H:5#])G%5kY&SF@`G/LhP\hCM\A\)-7>#/2^r_+.m7g'uaNb0ei%dpbTm`*^"0aC^%<[Q?2W2eSnFrni4F&75P3jH-,oi3['q<\(__`%Jl~>
GatUs?#SFN'Sc)P'u"K;nGElJjK_`pdaUkS5jG.fRS5Ds&JAhPC&e.%%j2(%O]&\ML:ltbk8YRj1`^r^1Ood"@(I0tYT.&$g_U&Z,<*[]LE&`j\cKQ`SdAi7@#-O)m\cNrH;dr5"rOB"5M)N3n/G'93Y-]139j#$5t["&n5m<'H'&N!,.Ani%8C?[W3@e;L6;l\"XkaDEL<^Vlci#"j5dqp-0O],dRB$>,U(U<ZDm+"MK'*Z[^ZB%H?=,33//j&WO"-Yl"$njDcdpq=Sar-XDOR^?kGaD!o=kpm.5N&'HiM/aS33=4dgDSEYL((aW\oq]ePY&>S,fN$F=p>@Z591TUsSS]%-COoj:Gpj-]TgQ^ii;9,FXU4/%g`.a1$,[XUl6=XniX/(11_(FPrnP5>"A_*t.0*r"C$2f^8P<bb+0Q%7bWfmTLnOf5/>8R>0ZD3GE98t+_I0al8=o*S<IF*j_SA@59H^$Ej83NJS&0!gj[(EElromK!b]hOEt5G2rtR.s.-YM@)f_q@/dHgFHrdA?V)]l_`]E6aKm9V7Ml>aUMV+*b31)+i_)i>dT1UMpGfNG5TtI*pNr)f%f&Z-$2)]-0uoh%%GQ:V4)iq1!="84=4DP)"I[o@C3B<'HohDnSF_LD4ge193Nl];'Ha/n;_tBst#Tm<`_NIooc8PXN5o^&@:9i@\*P_l'QR]q,/IPt&,AY'LkqULk;Y(;VaUgZcbYg/C48Em57#8/PX(A0dj6\2(rIXrj(-eL@'qf+A-5/d^!S]hr!PCoV*8s0#njL#iN=o2D?3Sdo),qa=[5@b5H"F8KDa+@m>')Cu)RK2Hmp_3^P5iDE04*'l%5A%PtArhMDGN1(-5S"5@;lPbC]1<#aa)7^(6>3`iS^705TZB_"4fDV*kI#\J!r#AF4qAe:-<RQmT`PN[\BoS(p;kPUaHe<YG\puV'jh5HKg@a#u4br4.qZgfZ;F(~>
endstream
endobj
8 0 obj
@ -87,19 +87,19 @@ endobj
xref
0 14
0000000000 65535 f
0000003046 00000 n
0000003110 00000 n
0000003160 00000 n
0000003095 00000 n
0000003159 00000 n
0000003209 00000 n
0000000015 00000 n
0000000071 00000 n
0000001273 00000 n
0000001379 00000 n
0000002383 00000 n
0000002489 00000 n
0000002601 00000 n
0000002711 00000 n
0000002822 00000 n
0000002930 00000 n
0000001264 00000 n
0000001370 00000 n
0000002432 00000 n
0000002538 00000 n
0000002650 00000 n
0000002760 00000 n
0000002871 00000 n
0000002979 00000 n
trailer
<<
/Size 14
@ -107,5 +107,5 @@ trailer
/Info 4 0 R
>>
startxref
3282
3331
%%EOF

22
docs/lucene-sandbox/index.html
View File

@ -52,7 +52,7 @@
+-->
<div class="searchbox">
<form action="http://search.lucidimagination.com/p:lucene" method="get" class="roundtopsmall">
<input onFocus="getBlank (this, 'Search the site with google');" size="25" name="q" id="query" type="text" value="Search the site with google">&nbsp;
<input onFocus="getBlank (this, 'Search the site with Lucene');" size="25" name="q" id="query" type="text" value="Search the site with Lucene">&nbsp;
<input name="Search" value="Search" type="submit">
</form>
<div style="position: relative; top: -5px; left: -10px">Powered by <a href="http://www.lucidimagination.com" style="color: #033268">Lucid Imagination</a>
@ -144,6 +144,9 @@ document.write("Last Published: " + document.lastModified);
<a href="../api/contrib-collation/index.html">Collation</a>
</div>
<div class="menuitem">
<a href="../api/contrib-fast-vector-highlighter/index.html">Fast Vector Highlighter</a>
</div>
<div class="menuitem">
<a href="../api/contrib-highlighter/index.html">Highlighter</a>
</div>
<div class="menuitem">
@ -270,6 +273,9 @@ document.write("Last Published: " + document.lastModified);
<a href="#Term Highlighter">Term Highlighter</a>
</li>
<li>
<a href="#Fast Vector Highlighter">Fast Vector Highlighter</a>
</li>
<li>
<a href="#Javascript Query Constructor">Javascript Query Constructor</a>
</li>
<li>
@ -381,7 +387,13 @@ document.write("Last Published: " + document.lastModified);
search results.
</p>
<a href="http://svn.apache.org/repos/asf/lucene/java/trunk/contrib/highlighter/">The
repository for the Highlighter contribution.</a><a name="N10092"></a><a name="Javascript Query Constructor"></a>
repository for the Highlighter contribution.</a><a name="N10092"></a><a name="Fast Vector Highlighter"></a>
<h3 class="boxed">Fast Vector Highlighter</h3>
<p>
An alternative set of classes for highlighting matching terms in search results.
</p>
<a href="http://svn.apache.org/repos/asf/lucene/java/trunk/contrib/fast-vector-highlighter/">The
repository for the Fast Vector Highlighter contribution.</a><a name="N1009F"></a><a name="Javascript Query Constructor"></a>
<h3 class="boxed">Javascript Query Constructor</h3>
<p>
Javascript library to support client-side query-building. Provides support for a user interface similar to
@ -394,7 +406,7 @@ document.write("Last Published: " + document.lastModified);
repository for the Javascript Query Constructor files.</a>
</p>
<a name="N100A6"></a><a name="Javascript Query Validator"></a>
<a name="N100B3"></a><a name="Javascript Query Validator"></a>
<h3 class="boxed">Javascript Query Validator</h3>
<p>
Javascript library to support client-side query validation. Lucene doesn't like malformed queries and tends to
@ -408,7 +420,7 @@ document.write("Last Published: " + document.lastModified);
repository for the Javascript Query Validator files.</a>
</p>
<a name="N100B6"></a><a name="High Frequency Terms"></a>
<a name="N100C3"></a><a name="High Frequency Terms"></a>
<h3 class="boxed">High Frequency Terms</h3>
<p>
The miscellaneous package is for classes that don't fit anywhere else. The only class in it right now determines
@ -422,7 +434,7 @@ document.write("Last Published: " + document.lastModified);
repository for miscellaneous classes.</a>
</p>
<a name="N100C6"></a><a name="InstantiatedIndex"></a>
<a name="N100D3"></a><a name="InstantiatedIndex"></a>
<h3 class="boxed">InstantiatedIndex</h3>
<p>
RAM-based index that enables much faster searching than RAMDirectory.

312
docs/lucene-sandbox/index.pdf
View File

@ -5,10 +5,10 @@
/Producer (FOP 0.20.5) >>
endobj
5 0 obj
<< /Length 857 /Filter [ /ASCII85Decode /FlateDecode ]
<< /Length 906 /Filter [ /ASCII85Decode /FlateDecode ]
>>
stream
Gaua>;/_pX&:i[6'n/iQ`O-.5n<d8BenWo<"`4e)*aEYf1pBnGg1A%52D!MJZB@aO7MLt@`uX(5mW+h/_<FAo8-BOG('1_=4@%g_^]9c5#,tm<&V;0!5ZWN@PM),^GN&a!8Xu/`P7O^!85GiVrR.4Zi^Wl/!T-1dANXIQ8alJC4.9Q5PGi^"W@]'%iLO_LWXuHfo>+$-E4!js_iTSR\!$9NWd>:l029s5rQ^[ENPT527fc**m9sNIPD1ecj5(jN[9kWGmWMsl`M%K9`1`CQYn-FK8H_D-J-gLdgBO5.'ZM)0paS91+ap4PQg4*q=-^h_MRjS7E@VEnSZiFNi?DcZ@G93!L6"$a"$RUCp[i*]=6j4Jp5L<@e.3_G(Q0)@eTO5=A4@D[BtF2U7nF!(!B_>*p[r1)rEn6kKc3mq#<&=g&(o8h[\s-ol/JIO:ZlMmjLRPEIVo0\clShlF8ZNqEDTHgP\F./oUYta^`3)Se[9+8e#N%A029dC$_ICY9",:L^Xj'bK]=3uM:^J6GR-G19^(12]e=<m.e>N"a%q)poAk*k@ln\"A_5DjEGeSPMEgSWfB';\o[bUefD_AbVU_`[<JM&RQh*RaO+W3:nTn@o$L[6K!</BfA\en<]g4%9*p@V)osJl]h:&jdmJU1%EPE,ApV:W#*3\+>k\LcO9"duTdAuVjAbU)P1qCiT^r]eX>foAQIr$pN&V8.IM8Ob2"o$iOhG(U!`@s3(g2.(HKE<a@,.J-.m"eGl^`]#+3258_O"DlW+:X)=5?H%9[a!#*hVSUX:*Kru-Y<;C^rt>^\QZA$>^It;C*cT:PY+:2[XL^4L!I.dokVq\r?)I2d%:<C!N)L~>
Gaua>hbVu\&BE]*=89$kX0J3T599j-)edrY)R5'7%j>p9>%/9qKlql.G-]*m@ni'9P,`9aSF?#/gY_d$PKE@M6d3RR--HV"5M:^_RgDO%Hmjkh?f2Ua"VF?VoPJo[9!_`AKOucobe<8=FXAAIci/?.OimT*rAm80:i]tg;aab)q+3Oe;+r4L$hpJHC?OQ.W`LYdZ73*998KPHP*^^Y^^E1V8U=[N&&R_9WEQn5R?EPL=SMeSU?&:<9'g&.0>qk_q4l=5)$=`26Yi$D/nCT0[Y56?R0SRrm+Db@fhQFSFA?<cDRo:4Eopg0`-rl[_f].KJ@uOp-$8O#Q:ed/2HBXtN/9uK7m@KU:s=962N/15:LmXHIN.!bN\F<pKe/g<cASuBPHrLL@D^"Rpj)TY%spIL2[Bo\:Ls&tX8<'8b1S;L"-.)Vm13\/ql`WTT1*h6fLR;IeXksq"ji(<!n7aNSmaek$LH"q/IT%8B7FjXD>R",Q'&[]=hrm8UUb]=B)d.SgAnUMCEH9ai=rL%86[d<!aqW+>reR=`Q`76^'<om$fB5lqkIlA-uRN$1bB^0#57YqDB/D9a;V#7Dif`tK6uW+h2</t[:g*mg0^ZMBq=Kt"]B<M(4$'!]W<'cmaADm%($tf=1/_!Tb!A>*X2lC,J^M2ZH1Mn@rKatp^2,gJ_!](P<H5<@";P3i)Qnuh-.l.OBX.nPh+Tf,fYDG_GTCV*"#GZ]8pk^3>c^QQ?$^MgM%o'F.%s(+'B&UcWXLNIn[=ij7g[VXt<T+P<L;TM2b'6P<gn_8.2ep>Y\%2H,KKs&0.t%kFfD[G*W5j@]u--1UYqM.q#ODps(jA/Oi:h=pZa6SsMnS[;6m^%=7/O/^5=cPf-s3G<YF5D/<&=BMf.c066F+*s'7$g)^~>
endstream
endobj
6 0 obj
@ -33,6 +33,7 @@ endobj
24 0 R
26 0 R
28 0 R
30 0 R
]
endobj
8 0 obj
@ -108,7 +109,7 @@ endobj
22 0 obj
<< /Type /Annot
/Subtype /Link
/Rect [ 108.0 418.766 264.14 406.766 ]
/Rect [ 108.0 418.766 237.488 406.766 ]
/C [ 0 0 0 ]
/Border [ 0 0 0 ]
/A 23 0 R
@ -118,7 +119,7 @@ endobj
24 0 obj
<< /Type /Annot
/Subtype /Link
/Rect [ 108.0 400.566 252.8 388.566 ]
/Rect [ 108.0 400.566 264.14 388.566 ]
/C [ 0 0 0 ]
/Border [ 0 0 0 ]
/A 25 0 R
@ -128,7 +129,7 @@ endobj
26 0 obj
<< /Type /Annot
/Subtype /Link
/Rect [ 108.0 382.366 234.812 370.366 ]
/Rect [ 108.0 382.366 252.8 370.366 ]
/C [ 0 0 0 ]
/Border [ 0 0 0 ]
/A 27 0 R
@ -138,7 +139,7 @@ endobj
28 0 obj
<< /Type /Annot
/Subtype /Link
/Rect [ 108.0 364.166 211.816 352.166 ]
/Rect [ 108.0 364.166 239.812 352.166 ]
/C [ 0 0 0 ]
/Border [ 0 0 0 ]
/A 29 0 R
@ -146,27 +147,22 @@ endobj
>>
endobj
30 0 obj
<< /Type /Annot
/Subtype /Link
/Rect [ 108.0 345.966 211.816 333.966 ]
/C [ 0 0 0 ]
/Border [ 0 0 0 ]
/A 31 0 R
/H /I
>>
endobj
32 0 obj
<< /Length 1870 /Filter [ /ASCII85Decode /FlateDecode ]
>>
stream
Gau0D9on$e&A@sBka2?U>,8lZ98[Q>TIJ<lMkXm3TI&<(<Yc]RZK(r\Nb$6Qb1*TKUf_$Vd`S(=O)Nk`om/?>Z^?00q'=3_5OdCA+f4S`aNmc1dq`muT>/&3AslGB09/Bo*CfGZ3uS\LKUrt9Eq]+kojTho:H[>pYZT.tD`r^fi.lWVYD*'?*d`gIpW!Y6?@$gs>NWnc9eIo8\UNI:YuI?&H@L3uG:1nI>O63f+UCXX`rh2Q$V=$["XERd42J-E0M94CKO>dBVq";"f9[Gh2!T+HX/b<f1?eb)NsDL'c&UuHRqK&o:Xg>D_F_,Ag5-/"V_G(?3^h#PSZ&.negHoem8hfGcj%JO.,$uF7CbM!PrKn6og\/gAKgcA"O,Sm?!V8#\W,GpK5RV&k/"J$/m^4F)eBS-H`j2[N/NTBMhp83;/`6_&1k!MUV4]O/t6%Eas*l9g_c!sHgrY0]+U@e?D,jt]P:80($8^9_HIo=fkCJ$#`Ht[g,Gk!Unj6N4Xm\ZbJlfm-t$V[`#Fpk*,M_I6CK]a`#1O08_aRM(&5[\iIt?:9Q2=BK@8g]</@Z/X>c>f[l[U+djWWefE,^`ZlkJNQ?]$<Za*MhU9iUa:`La`]1Q5<T*o-odj<G_&u"$2(U\];$3[GqZcno%C](S;AeGhrhIpcrBPLi]pV)4C%Z^+lC=!k=Xj?SSpB6\D5!Rg&,qC3UKo;\58-^%entDp>FhRZq4u`@rE:DeF+A/j#h-+l;<qEU)49*CrHsZ-9HCX_P4t!kZHspNDHqN%f.CEG8$HB2&E:fq[UP'uYck^V2I&PP'BfZ/[/[elaBY+%`2MCD9)5[FC1`.nn0hH<JRUM_KPkJoH2V=+pRO6hBK3MK>Lq?^dZGbk5LEp&i]qlrYZ!_HX_5ZEJ"S]I&,+dAW[H]Q8E:7b$AdJd!d'%`o\CR[Z/-gcO>@(N'N'@h=;8jV81.T>3h$p=De[j^T0p1-HSIU-S*h>lRcIn</qpp3-h@E5reF>DS/:^SN]En<++q)h\"=tIKN1hGB<UoYT[lk``e>&u'kiTQ0l].ful3a2UK$.-989ddI"Yh6)*1i7?2i%_qIks?*/3'DIDY\Q_)GA".[2Xd`,Ql+@.%>9-:Y3imOU?2dlp'-B5hj,2%8$pXV:<t`mft+jNmu+%c6(rLq[@pDJH)(j=-`0Q[`qP]H=Omn.45n1\Ts]+I,nHrj**.;DVMHH"DZZAe;?s?cNP)G'sn#eC;fHL=3%IZ%C.6&_*F:G1kq`]c[<>T&`iBsmsh^a,U;pq'u1trp>&%N7qYQ#-mmHSFoSlslMnU;=9.3"Olu,n^fV!LT+0lnU.tMZl/F'fcQA14WfGhJ@q68`UJZ7W4&Yj%95XHCb`>X,_.,q-82-$WFTSKM*(qVj%A\(jX#62;??/0PNgaf4UPTYV<i'!SMT84+[2k+_>$_o__Ka0$.RB/_QkJnPD^n@n-Hbujj=Unamf5YO%\9KSo;V?\Lq.(AbMm5[EA.AZ#mUZ(VW4=g%*`B+!JSu1Wl7bncpua_C%4#+8phkgUKY"n#U2I<G53A@ft$2l7U+p2,`\O^H;5HJ::^kX!a?&u'RPFI/FnfBVuP/K23As&Hg@NDZ]ja>a20@3_kHK17[%WRRHTE,cr+&N-[<n&I/G1g95]4)4:iL@U!QS,P(d3Dejrq^K!2aAB"J%"Ub%??cJX*9m^WQB00.cGVC--WKHsF`QTf51CR;*<0'O3b"Rq""n6e4NP9/h@Ib[0.@jk(:TBm?i"A?T;]r7b]\7BU?_73/V*JW"u;9SX0UUr>N(Or`H?ET[aa-Wj`#>Q(kXR>pNXm/:RT'(4Y^-)Vh^>GKrFuics0"T[>?&S-Y>ClY25NUXA<W~>
endstream
endobj
31 0 obj
<< /Type /Page
/Parent 1 0 R
/MediaBox [ 0 0 612 792 ]
/Resources 3 0 R
/Contents 30 0 R
>>
endobj
32 0 obj
<< /Length 1798 /Filter [ /ASCII85Decode /FlateDecode ]
>>
stream
GauHL>EbO7'Roe[i%=1eXWTl6*]Idk9r?1rib9Lg50+l$RL"`m=NCC54W%:]*([_Zdb40E&g3agG<UG1Vjl6.c0F>HH@,G"=Cj\d]f;&2J&JqUCDQ^kVQ!W)DnC>-HM[:>(l?Yr:MuRj"1YSgXo.cX55\9]IujgZp*4KSH@J&+Y4kQB_[kTb33VsVEP6;d?;*.F;\sTq2@&)4UYP'f99$+/P;`8:g)m5l6L:M6NAMR"Ea;@k4'kS/U1J)5q"]c!XC1lXDtE82NA)U)kf-X[7j7M;<hKcH=>IJu9h@%AZT/;-6=m679RWAlG,E9Q5^35<@<sM%BM)418rS+g'BkW:9E!6'^*?7^Drt]3kLFotq;T"&?Cu8(V'HsX:*\c)cq/QCdBHEXiuq&o'%3HIO:k#0)h2RC-)]_U\KWkNn!PUtaOjZf4Sr#8U>GEF?nWOfZIC71P`/X5!I-:/<7b_IGK/i?<\@).<Ma5TL'jAKarD"\Ub9S5PSpW8eZ+O_G-rb8Egn<:B0jH[7Zpnq*sSGIkITrYjgOObrMDge(s/YN49/!<_.)('V&2L4,AIq,TB6'FhuiINL5)=B(62FK&P9-V7[-=e!dDpJgn5!-+mi%@:BRpj5ul]D@2I@3(_)gXNtKk%#A45\.ij<%SU$bJ0Ec3m-MD_*9,AR@@Zi(;"/7(>*(9b\@!+\InD-k+a3!cRV'r3oB)dIJ+pJSSnHbro60T-!j#4)PKeiG3'tApg_9('RD;LXkHppn'BLK$`,@/3=N$K#$%@].Ba_f;-0o\K@Dl_%1pk2UEk[n[CJa3/CkKS>-8lYkkOR$h"EsHpC0.c>&PtsVK.r#J`Q^M+qBh[1CU6VQS]Uj?1Ke[p#*[,lJME@t^J^3]XB=uR]@[JlXJ+fL[nY(cK?Vr:]^Eq[QKL+\__r2ZVN'qXN!;H+oE;tnJ=n"(q4_+kUdW@;s:.]a@fY[tWaF`jkFtE=3^@XOp,W?&TXFB6_nGuo"WX#bm*=?FE(O#TnI-NVV\=O\nfa-tY>D)HIRj0CV?<cst;_G)qE[5.U#DpQof>pXlCTo3=jDH>d3"+km4I9kaj)`Bu[XrQ.XfZ5aV[82Mhd(Gf>ocr'U\c2$(G^FnPun;)S@IQRnj_PsE.:^7Aue<[9+F6'_a,@A+67t;kIXSY!69:G<T*Q"&:i?F56j!/QgPug7^oc*0^B^h[HU-$![Rt#-#q!qK@6q%YFpOKZtp:c1MJ;7c%U`Z;)k[]`UaU8a!d:meS!=R"\W9oP2Aj/&l-K"^r7Ua'b6%>iG!GGfQLT%F^*'41PEQ&+fIIO2*IWf.hn`,J^>.)jc>Et(HTt)H+Saidb"b.l*;]ukqC0LgtjU5F)G08=Cgag^og\D&V.#D*@'Jl%aEPgDat1B:uI&N,tFr]!\4LL-$tsbc)g)RCE)'*_5K\D5RWTtbV#llX_8V(Q:AhW'*fUYj#n#[+n+2W;s6hcX6-Hn[SU!lZ$]3%U2bi(iHgBU/-=LgN$Li?KXR5?]=;ai;kSBg2qeXm6mIeaBeXEH:_I##82LGiT+2;*mcN-oQ<T<BMsfH`'VRN,5T#M5`p/T=)'l__hLQogJGjt\[HM/`C$Om`YBr'HBm9]aUqqS<)kb<H91qgA_u[%$ql08?DKl\T?u)8LGAm9Q5X@c.N7mkq^q/at`otiR)kiZpPJP0r",l&T(*X!_hkpZT'm/ijoN%)@B9gH[K*n9fn.+NIWD<3+fls*=#)l<P_"Zl'jE/!^;WmRsOAX%a=7nAYVU\4(%Zg;=s3NKEHWr<B+F3BCJZ@rVF(QQ,~>
endstream
endobj
33 0 obj
<< /Type /Page
/Parent 1 0 R
@ -176,10 +172,10 @@ endobj
>>
endobj
34 0 obj
<< /Length 398 /Filter [ /ASCII85Decode /FlateDecode ]
<< /Length 1750 /Filter [ /ASCII85Decode /FlateDecode ]
>>
stream
Gat%]9i&Y\%#46H'g?MjR3]sGHXqpU+V(1<"rsp+h("CrU`F:OT(HN-`W`'7<F=AOCHks4j?=f\KY]Q/%j[uB5q*Q*6FPc-cB*[fYU2#6n_`]&V$+fUO/sJ2/#(uVkR/\,70R3Dn2e[7FrJN8/e`.6XtC[VcXX%qIM??n2l:=^e(?dCk4XI,G_bW?=E@#0[*BIh5nhbI!6sZIGhO+o*mCMUef%Jh3)#p()d?U'Aq),.9U2g;Be]!([0q2:19+3p!>.aQ>=-;_&GnVf%-,Tr3MZ]i4\YZf[b#sP>*I?b0IC$4)0'7d@]C\!+tUt7<d3tNp>pJLFT%@<g`)Sr<1HiDWo7E5&]*c)aq`11nPNi]N`BtcdT%-LmJ!C>]^&,CgA3Ilj2gWnXa6='~>
Gb!;d9on$e&A@sBk`up]),]9P-:+qS:_ZLQ8.TPJZp`Y:-rQ`a479<@aHEWu=5);$(.GLtg8%,W*p:f.$t$+PDcVeB$gMQeF14\Gi]9pN_>bWoIkO1H,:Q>,_k$J[_rB4u#D=?hDngN93n/ceX3JLE%<CEg3o.Tu,EWp)X2=\mq&?`,U>#S(/A9M+cQuNEL2Hb=F3l8;`b)nop"7EoWgQ]IH$*(i*K04'Yq+uVTgUK>_S&Y)@q3iDcjcG`E61j"G+sZX^RWid$Q+=bJcWl[f*BUO""R1QY:D@g%o%j=bd6#CCdP*O;/:4mq_m0dX[$h)%bpV+7pApU]TISDoAi0',\c97:AZ&.g4"fpRQ->\3qEl3]5p$"LM9),_<!KrDWk;"NudM0::Cdh8O@giFU0E&a_9]f9d[j3>Zsq56L\=IU&^6+V80c/X.#Leb;_t*d54O%iWo!]Fl3:PBqj<>"2!$D:3bm;A6,!oA1+CM:+>i:O\)LAImKZ78sG=J(pl$/R.5$J^e^+A$pkS&RX6!10#4mBA]2/o%K3Ftnj8M"::YpJ1ka!!JP$NKdY]!NI6WSU:;R8"0i&6;DClj0n,j;16]E2t8-!U(2S;Ei?@^e4J0,_VK/Q`8WoDQf4*H886>\Gel>1,+!3j=L(g`VuCkjH920)Zg(N8OHYW*guS6.%oPD:FKqKf3S7ZH<6p3u4Q.OkT6,rGQ:WYC+?d#A5Z`J@-)=bV7tO_6QFco';[!h[hihFoLS-#jLie?T2\<`:+XHiCcl)"^]Zr+:Zm%dI5mKDkuPq.W6KdF6_=cVGbE_lGs#'#_P/CsT-*L_BOX6c60^^P9t.2GFF(L4h`5pQOZq*6gAm$:pWZa7YZG;QL>F`mbo(UFAiQE/#\C=7rj33MPs)(<pmU#akeX&G5]dn);OVOU3TpZgh[,EJ*LjB*L7^M#DRK=G]@8Au=PBBt]f0)7&F:^1SqU.!N<g7Dn2sm/&CALP<J$>eZI8k8B!7eT[iOF\+]9l3=i30Q8.]H5kO%1+@8;(tH=L,I&=#Ajm"tY+`ECVo8>YVl)R.1eq.6'q.d=F3Q6.bd-7BH45W7:o.hcplKM;A?L$ie1N@m1g*7Cc';:3C!8c9%k&usWVDr9))#4;;%[s'$%*:<K?Vi7)OEk&fPEq[kJ["MVgp?.51;&m9/tDIghIH`Pa:Qah1gRKC4u^:N]U*$ipiQlq910O^;ou"eJp5qX4,M\[H,<&1jD2tcsZ1g8cssC-PH]F6<K9\GeGeU/f]>V=W0P4lr4@hAVL;Z<JHU;\f>9N'kllr,k90[i*N;g'M?@fM'b>`LaqF_1<$B-Js'];X=FZ5Q[:m:b\Y#Sg5>`dY_(hYfPqm<>ZTY>0oeCCNu,u7VfM/;&'7D<R!?LaQo:9L7jons3H*A7-qht[#7K!L)QC'ERA4erES`:3/YqU/^XUG*I6AOtR;O8_P-lkFRmp0@5R7c>G/-j5'U-SPX]5*_"YUu7%(S]ES2t%8g"Ko6-]r%9<T@irBU-XKD:%?A<-)d&Rb'FBQA1'A@NPZ-RPPU0o6ekuDhK_F.#(!b?8$.K+ZfJT4XOk_H.[Q.Y3"8A\;.t0XhRtq"gQq;#_dXg8<6E:^jmij$rKbID1j)sM0fonpVD2_bXVZ4gIVNaK@e[dj)$,sN&lqb-OBciJ+rM;QcF0[FHsn=?:`,&kIT'nZ>(Mh;lB=P*2&)d7.':]h'eLMoY\7ELVAd`f<.r>_dU%;3Kj5g!$7gsU]~>
endstream
endobj
35 0 obj
@ -190,122 +186,145 @@ endobj
/Contents 34 0 R
>>
endobj
36 0 obj
<< /Length 608 /Filter [ /ASCII85Decode /FlateDecode ]
>>
stream
Gat$t9lHLT(r#SlHqY&rj4Dr9*8>>03c3<rEpImf^`c7eA=.3!^Rt<7VG^DgLl]U/4C@k9M`t_"Y6[_achf,e5Y7+arlR\0]M2m,6qgcO%ak%7mbR[IJH`B(HU3bh(YLO;_]n@V,i_5-+JG<9BC`^W:?KPL+25$pc)#D[f<d3SfWf-,E3<G`l<7E@>hq0*f@i7Fi.h.Tg#l3_"dLPUPQ&i>qn&N#,,?g4gFjZ;alG'dTOrDhJZdGRm3TSnoXFP6(eA242ZuNN<'FF9C"lY!1W=tubhl7b3a,<-\X_nE"9(d"CN>:D$QR81Dng0#g6lC^:lV8,lqY,rOKnXYRip5P2*V7%B\/ZF$"n@LT.4'J^1S8Cq*.o^ann!Z@TI5tU4BT(s8*8*+N[^YAua[Y&`mL11?)8a9KA6L3FqAFI!a9:a=S^611=_bjdI.#*nn=eiCVe`d&\/Qs80V\;@-5pAPup1n+&G^M6CIt0/$?\8NA1ZWViX:2'3?F,+UV1-mlJ1[MoPSBd/AmhXHd%LZ\0U<on%o?E-3!+&(UE(o>?+VIE"+KJh\ck1KX[D(r86Y-u\_eQ<*Ceb&tJDAQ^@9`*QY_<1\$^ul=]~>
endstream
endobj
37 0 obj
<<
/Title (\376\377\0\61\0\40\0\114\0\165\0\143\0\145\0\156\0\145\0\40\0\123\0\141\0\156\0\144\0\142\0\157\0\170)
/Parent 36 0 R
/First 38 0 R
/Last 47 0 R
/Count -10
/A 9 0 R
>> endobj
38 0 obj
<<
/Title (\376\377\0\61\0\56\0\61\0\40\0\123\0\156\0\157\0\167\0\142\0\141\0\154\0\154\0\40\0\123\0\164\0\145\0\155\0\155\0\145\0\162\0\163\0\40\0\146\0\157\0\162\0\40\0\114\0\165\0\143\0\145\0\156\0\145)
/Parent 37 0 R
/Next 39 0 R
/A 11 0 R
>> endobj
<< /Type /Page
/Parent 1 0 R
/MediaBox [ 0 0 612 792 ]
/Resources 3 0 R
/Contents 36 0 R
>>
endobj
39 0 obj
<<
/Title (\376\377\0\61\0\56\0\62\0\40\0\101\0\156\0\141\0\154\0\171\0\172\0\145\0\162\0\163\0\54\0\40\0\124\0\157\0\153\0\145\0\156\0\151\0\172\0\145\0\162\0\163\0\54\0\40\0\106\0\151\0\154\0\164\0\145\0\162\0\163)
/Parent 37 0 R
/Prev 38 0 R
/Next 40 0 R
/A 13 0 R
/Title (\376\377\0\61\0\40\0\114\0\165\0\143\0\145\0\156\0\145\0\40\0\123\0\141\0\156\0\144\0\142\0\157\0\170)
/Parent 38 0 R
/First 40 0 R
/Last 50 0 R
/Count -11
/A 9 0 R
>> endobj
40 0 obj
<<
/Title (\376\377\0\61\0\56\0\63\0\40\0\101\0\156\0\164)
/Parent 37 0 R
/Prev 39 0 R
/Title (\376\377\0\61\0\56\0\61\0\40\0\123\0\156\0\157\0\167\0\142\0\141\0\154\0\154\0\40\0\123\0\164\0\145\0\155\0\155\0\145\0\162\0\163\0\40\0\146\0\157\0\162\0\40\0\114\0\165\0\143\0\145\0\156\0\145)
/Parent 39 0 R
/Next 41 0 R
/A 15 0 R
/A 11 0 R
>> endobj
41 0 obj
<<
/Title (\376\377\0\61\0\56\0\64\0\40\0\127\0\157\0\162\0\144\0\116\0\145\0\164\0\57\0\123\0\171\0\156\0\157\0\156\0\171\0\155\0\163)
/Parent 37 0 R
/Title (\376\377\0\61\0\56\0\62\0\40\0\101\0\156\0\141\0\154\0\171\0\172\0\145\0\162\0\163\0\54\0\40\0\124\0\157\0\153\0\145\0\156\0\151\0\172\0\145\0\162\0\163\0\54\0\40\0\106\0\151\0\154\0\164\0\145\0\162\0\163)
/Parent 39 0 R
/Prev 40 0 R
/Next 42 0 R
/A 17 0 R
/A 13 0 R
>> endobj
42 0 obj
<<
/Title (\376\377\0\61\0\56\0\65\0\40\0\114\0\165\0\143\0\154\0\151\0\40\0\55\0\40\0\114\0\165\0\143\0\145\0\156\0\145\0\40\0\103\0\157\0\155\0\155\0\141\0\156\0\144\0\55\0\154\0\151\0\156\0\145\0\40\0\111\0\156\0\164\0\145\0\162\0\146\0\141\0\143\0\145)
/Parent 37 0 R
/Title (\376\377\0\61\0\56\0\63\0\40\0\101\0\156\0\164)
/Parent 39 0 R
/Prev 41 0 R
/Next 43 0 R
/A 19 0 R
/A 15 0 R
>> endobj
43 0 obj
<<
/Title (\376\377\0\61\0\56\0\66\0\40\0\124\0\145\0\162\0\155\0\40\0\110\0\151\0\147\0\150\0\154\0\151\0\147\0\150\0\164\0\145\0\162)
/Parent 37 0 R
/Title (\376\377\0\61\0\56\0\64\0\40\0\127\0\157\0\162\0\144\0\116\0\145\0\164\0\57\0\123\0\171\0\156\0\157\0\156\0\171\0\155\0\163)
/Parent 39 0 R
/Prev 42 0 R
/Next 44 0 R
/A 21 0 R
/A 17 0 R
>> endobj
44 0 obj
<<
/Title (\376\377\0\61\0\56\0\67\0\40\0\112\0\141\0\166\0\141\0\163\0\143\0\162\0\151\0\160\0\164\0\40\0\121\0\165\0\145\0\162\0\171\0\40\0\103\0\157\0\156\0\163\0\164\0\162\0\165\0\143\0\164\0\157\0\162)
/Parent 37 0 R
/Title (\376\377\0\61\0\56\0\65\0\40\0\114\0\165\0\143\0\154\0\151\0\40\0\55\0\40\0\114\0\165\0\143\0\145\0\156\0\145\0\40\0\103\0\157\0\155\0\155\0\141\0\156\0\144\0\55\0\154\0\151\0\156\0\145\0\40\0\111\0\156\0\164\0\145\0\162\0\146\0\141\0\143\0\145)
/Parent 39 0 R
/Prev 43 0 R
/Next 45 0 R
/A 23 0 R
/A 19 0 R
>> endobj
45 0 obj
<<
/Title (\376\377\0\61\0\56\0\70\0\40\0\112\0\141\0\166\0\141\0\163\0\143\0\162\0\151\0\160\0\164\0\40\0\121\0\165\0\145\0\162\0\171\0\40\0\126\0\141\0\154\0\151\0\144\0\141\0\164\0\157\0\162)
/Parent 37 0 R
/Title (\376\377\0\61\0\56\0\66\0\40\0\124\0\145\0\162\0\155\0\40\0\110\0\151\0\147\0\150\0\154\0\151\0\147\0\150\0\164\0\145\0\162)
/Parent 39 0 R
/Prev 44 0 R
/Next 46 0 R
/A 25 0 R
/A 21 0 R
>> endobj
46 0 obj
<<
/Title (\376\377\0\61\0\56\0\71\0\40\0\110\0\151\0\147\0\150\0\40\0\106\0\162\0\145\0\161\0\165\0\145\0\156\0\143\0\171\0\40\0\124\0\145\0\162\0\155\0\163)
/Parent 37 0 R
/Title (\376\377\0\61\0\56\0\67\0\40\0\106\0\141\0\163\0\164\0\40\0\126\0\145\0\143\0\164\0\157\0\162\0\40\0\110\0\151\0\147\0\150\0\154\0\151\0\147\0\150\0\164\0\145\0\162)
/Parent 39 0 R
/Prev 45 0 R
/Next 47 0 R
/A 27 0 R
/A 23 0 R
>> endobj
47 0 obj
<<
/Title (\376\377\0\61\0\56\0\61\0\60\0\40\0\111\0\156\0\163\0\164\0\141\0\156\0\164\0\151\0\141\0\164\0\145\0\144\0\111\0\156\0\144\0\145\0\170)
/Parent 37 0 R
/Title (\376\377\0\61\0\56\0\70\0\40\0\112\0\141\0\166\0\141\0\163\0\143\0\162\0\151\0\160\0\164\0\40\0\121\0\165\0\145\0\162\0\171\0\40\0\103\0\157\0\156\0\163\0\164\0\162\0\165\0\143\0\164\0\157\0\162)
/Parent 39 0 R
/Prev 46 0 R
/A 29 0 R
/Next 48 0 R
/A 25 0 R
>> endobj
48 0 obj
<<
/Title (\376\377\0\61\0\56\0\71\0\40\0\112\0\141\0\166\0\141\0\163\0\143\0\162\0\151\0\160\0\164\0\40\0\121\0\165\0\145\0\162\0\171\0\40\0\126\0\141\0\154\0\151\0\144\0\141\0\164\0\157\0\162)
/Parent 39 0 R
/Prev 47 0 R
/Next 49 0 R
/A 27 0 R
>> endobj
49 0 obj
<<
/Title (\376\377\0\61\0\56\0\61\0\60\0\40\0\110\0\151\0\147\0\150\0\40\0\106\0\162\0\145\0\161\0\165\0\145\0\156\0\143\0\171\0\40\0\124\0\145\0\162\0\155\0\163)
/Parent 39 0 R
/Prev 48 0 R
/Next 50 0 R
/A 29 0 R
>> endobj
50 0 obj
<<
/Title (\376\377\0\61\0\56\0\61\0\61\0\40\0\111\0\156\0\163\0\164\0\141\0\156\0\164\0\151\0\141\0\164\0\145\0\144\0\111\0\156\0\144\0\145\0\170)
/Parent 39 0 R
/Prev 49 0 R
/A 31 0 R
>> endobj
51 0 obj
<< /Type /Font
/Subtype /Type1
/Name /F3
/BaseFont /Helvetica-Bold
/Encoding /WinAnsiEncoding >>
endobj
49 0 obj
52 0 obj
<< /Type /Font
/Subtype /Type1
/Name /F5
/BaseFont /Times-Roman
/Encoding /WinAnsiEncoding >>
endobj
50 0 obj
53 0 obj
<< /Type /Font
/Subtype /Type1
/Name /F1
/BaseFont /Helvetica
/Encoding /WinAnsiEncoding >>
endobj
51 0 obj
54 0 obj
<< /Type /Font
/Subtype /Type1
/Name /F2
/BaseFont /Helvetica-Oblique
/Encoding /WinAnsiEncoding >>
endobj
52 0 obj
55 0 obj
<< /Type /Font
/Subtype /Type1
/Name /F7
@ -315,152 +334,161 @@ endobj
1 0 obj
<< /Type /Pages
/Count 4
/Kids [6 0 R 31 0 R 33 0 R 35 0 R ] >>
/Kids [6 0 R 33 0 R 35 0 R 37 0 R ] >>
endobj
2 0 obj
<< /Type /Catalog
/Pages 1 0 R
/Outlines 36 0 R
/Outlines 38 0 R
/PageMode /UseOutlines
>>
endobj
3 0 obj
<<
/Font << /F3 48 0 R /F5 49 0 R /F1 50 0 R /F2 51 0 R /F7 52 0 R >>
/Font << /F3 51 0 R /F5 52 0 R /F1 53 0 R /F2 54 0 R /F7 55 0 R >>
/ProcSet [ /PDF /ImageC /Text ] >>
endobj
9 0 obj
<<
/S /GoTo
/D [31 0 R /XYZ 85.0 659.0 null]
/D [33 0 R /XYZ 85.0 659.0 null]
>>
endobj
11 0 obj
<<
/S /GoTo
/D [31 0 R /XYZ 85.0 506.266 null]
/D [33 0 R /XYZ 85.0 506.266 null]
>>
endobj
13 0 obj
<<
/S /GoTo
/D [31 0 R /XYZ 85.0 399.413 null]
/D [33 0 R /XYZ 85.0 399.413 null]
>>
endobj
15 0 obj
<<
/S /GoTo
/D [31 0 R /XYZ 85.0 326.96 null]
/D [33 0 R /XYZ 85.0 326.96 null]
>>
endobj
17 0 obj
<<
/S /GoTo
/D [31 0 R /XYZ 85.0 241.307 null]
/D [33 0 R /XYZ 85.0 241.307 null]
>>
endobj
19 0 obj
<<
/S /GoTo
/D [33 0 R /XYZ 85.0 603.4 null]
/D [35 0 R /XYZ 85.0 603.4 null]
>>
endobj
21 0 obj
<<
/S /GoTo
/D [33 0 R /XYZ 85.0 530.947 null]
/D [35 0 R /XYZ 85.0 530.947 null]
>>
endobj
23 0 obj
<<
/S /GoTo
/D [33 0 R /XYZ 85.0 466.194 null]
/D [35 0 R /XYZ 85.0 466.194 null]
>>
endobj
25 0 obj
<<
/S /GoTo
/D [33 0 R /XYZ 85.0 380.541 null]
/D [35 0 R /XYZ 85.0 401.441 null]
>>
endobj
27 0 obj
<<
/S /GoTo
/D [33 0 R /XYZ 85.0 281.688 null]
/D [35 0 R /XYZ 85.0 315.788 null]
>>
endobj
29 0 obj
<<
/S /GoTo
/D [33 0 R /XYZ 85.0 169.635 null]
/D [35 0 R /XYZ 85.0 216.935 null]
>>
endobj
36 0 obj
31 0 obj
<<
/First 37 0 R
/Last 37 0 R
/S /GoTo
/D [37 0 R /XYZ 85.0 637.8 null]
>>
endobj
38 0 obj
<<
/First 39 0 R
/Last 39 0 R
>> endobj
xref
0 53
0 56
0000000000 65535 f
0000010600 00000 n
0000010679 00000 n
0000010771 00000 n
0000011212 00000 n
0000011291 00000 n
0000011383 00000 n
0000000015 00000 n
0000000071 00000 n
0000001019 00000 n
0000001139 00000 n
0000001234 00000 n
0000010894 00000 n
0000001369 00000 n
0000010957 00000 n
0000001506 00000 n
0000011023 00000 n
0000001643 00000 n
0000011089 00000 n
0000001778 00000 n
0000011154 00000 n
0000001915 00000 n
0000011220 00000 n
0000002051 00000 n
0000011284 00000 n
0000002188 00000 n
0000011350 00000 n
0000002324 00000 n
0000011416 00000 n
0000002459 00000 n
0000011482 00000 n
0000002596 00000 n
0000011548 00000 n
0000002733 00000 n
0000004696 00000 n
0000004804 00000 n
0000006695 00000 n
0000006803 00000 n
0000007293 00000 n
0000011614 00000 n
0000007401 00000 n
0000007602 00000 n
0000007869 00000 n
0000008161 00000 n
0000008295 00000 n
0000008506 00000 n
0000008838 00000 n
0000009049 00000 n
0000009331 00000 n
0000009601 00000 n
0000009835 00000 n
0000010044 00000 n
0000010157 00000 n
0000010267 00000 n
0000010375 00000 n
0000010491 00000 n
0000001068 00000 n
0000001188 00000 n
0000001290 00000 n
0000011506 00000 n
0000001425 00000 n
0000011569 00000 n
0000001562 00000 n
0000011635 00000 n
0000001699 00000 n
0000011701 00000 n
0000001834 00000 n
0000011766 00000 n
0000001971 00000 n
0000011832 00000 n
0000002107 00000 n
0000011896 00000 n
0000002244 00000 n
0000011962 00000 n
0000002381 00000 n
0000012028 00000 n
0000002517 00000 n
0000012094 00000 n
0000002652 00000 n
0000012160 00000 n
0000002789 00000 n
0000012226 00000 n
0000002926 00000 n
0000004889 00000 n
0000004997 00000 n
0000006840 00000 n
0000006948 00000 n
0000007648 00000 n
0000012290 00000 n
0000007756 00000 n
0000007957 00000 n
0000008224 00000 n
0000008516 00000 n
0000008650 00000 n
0000008861 00000 n
0000009193 00000 n
0000009404 00000 n
0000009656 00000 n
0000009938 00000 n
0000010208 00000 n
0000010447 00000 n
0000010656 00000 n
0000010769 00000 n
0000010879 00000 n
0000010987 00000 n
0000011103 00000 n
trailer
<<
/Size 53
/Size 56
/Root 2 0 R
/Info 4 0 R
>>
startxref
11665
12341
%%EOF

5
docs/queryparsersyntax.html
View File

@ -52,7 +52,7 @@
+-->
<div class="searchbox">
<form action="http://search.lucidimagination.com/p:lucene" method="get" class="roundtopsmall">
<input onFocus="getBlank (this, 'Search the site with google');" size="25" name="q" id="query" type="text" value="Search the site with google">&nbsp;
<input onFocus="getBlank (this, 'Search the site with Lucene');" size="25" name="q" id="query" type="text" value="Search the site with Lucene">&nbsp;
<input name="Search" value="Search" type="submit">
</form>
<div style="position: relative; top: -5px; left: -10px">Powered by <a href="http://www.lucidimagination.com" style="color: #033268">Lucid Imagination</a>
@ -144,6 +144,9 @@ document.write("Last Published: " + document.lastModified);
<a href="api/contrib-collation/index.html">Collation</a>
</div>
<div class="menuitem">
<a href="api/contrib-fast-vector-highlighter/index.html">Fast Vector Highlighter</a>
</div>
<div class="menuitem">
<a href="api/contrib-highlighter/index.html">Highlighter</a>
</div>
<div class="menuitem">

5
docs/scoring.html
View File

@ -52,7 +52,7 @@
+-->
<div class="searchbox">
<form action="http://search.lucidimagination.com/p:lucene" method="get" class="roundtopsmall">
<input onFocus="getBlank (this, 'Search the site with google');" size="25" name="q" id="query" type="text" value="Search the site with google">&nbsp;
<input onFocus="getBlank (this, 'Search the site with Lucene');" size="25" name="q" id="query" type="text" value="Search the site with Lucene">&nbsp;
<input name="Search" value="Search" type="submit">
</form>
<div style="position: relative; top: -5px; left: -10px">Powered by <a href="http://www.lucidimagination.com" style="color: #033268">Lucid Imagination</a>
@ -144,6 +144,9 @@ document.write("Last Published: " + document.lastModified);
<a href="api/contrib-collation/index.html">Collation</a>
</div>
<div class="menuitem">
<a href="api/contrib-fast-vector-highlighter/index.html">Fast Vector Highlighter</a>
</div>
<div class="menuitem">
<a href="api/contrib-highlighter/index.html">Highlighter</a>
</div>
<div class="menuitem">

8
src/site/src/documentation/content/xdocs/lucene-sandbox/index.xml
View File

@ -104,6 +104,14 @@
repository for the Highlighter contribution.</a>
</section>
<section id="Fast Vector Highlighter"><title>Fast Vector Highlighter</title>
<p>
An alternative set of classes for highlighting matching terms in search results.
</p>
<a href="http://svn.apache.org/repos/asf/lucene/java/trunk/contrib/fast-vector-highlighter/">The
repository for the Fast Vector Highlighter contribution.</a>
</section>
<section id="Javascript Query Constructor"><title>Javascript Query Constructor</title>
<p>
Javascript library to support client-side query-building. Provides support for a user interface similar to

4
src/site/src/documentation/content/xdocs/site.xml
View File

@ -54,6 +54,7 @@ See http://forrest.apache.org/docs/linking.html for more info
<javadoc-contrib-bdb-je label="Bdb-je" href="ext:javadocs-contrib-bdb-je"/>
<javadoc-contrib-benchmark label="Benchmark" href="ext:javadocs-contrib-benchmark"/>
<javadoc-contrib-collation label="Collation" href="ext:javadocs-contrib-collation"/>
<javadoc-contrib-fast-vector-highlighter label="Fast Vector Highlighter" href="ext:javadocs-contrib-fast-vector-highlighter"/>
<javadoc-contrib-highlighter label="Highlighter" href="ext:javadocs-contrib-highlighter"/>
<javadoc-contrib-instantiated label="Instantiated" href="ext:javadocs-contrib-instantiated"/>
<javadoc-contrib-lucli label="Lucli" href="ext:javadocs-contrib-lucli"/>
@ -103,7 +104,8 @@ See http://forrest.apache.org/docs/linking.html for more info
<javadocs-contrib-bdb href="api/contrib-bdb/index.html"/>
<javadocs-contrib-bdb-je href="api/contrib-bdb-je/index.html"/>
<javadocs-contrib-benchmark href="api/contrib-benchmark/index.html"/>
<javadocs-contrib-collation href="api/contrib-collation/index.html"/>
<javadocs-contrib-collation href="api/contrib-collation/index.html"/>
<javadocs-contrib-fast-vector-highlighter href="api/contrib-fast-vector-highlighter/index.html"/>
<javadocs-contrib-highlighter href="api/contrib-highlighter/index.html"/>
<javadocs-contrib-instantiated href="api/contrib-instantiated/index.html"/>
<javadocs-contrib-lucli href="api/contrib-lucli/index.html"/>