mirror of https://github.com/apache/lucene.git
renaming - sorry, wasn't paying attention
git-svn-id: https://svn.apache.org/repos/asf/lucene/java/trunk@150971 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Erik Hatcher
parent
268aa9e55f
commit
45898ec436
|
|
@ -1,10 +0,0 @@
|
|||
<?xml version="1.0"?>
|
||||
|
||||
<project name="hilighter" default="default">
|
||||
|
||||
<description>
|
||||
Hits hilighter
|
||||
</description>
|
||||
|
||||
<import file="../common.xml"/>
|
||||
</project>
|
||||
|
|
@ -1,38 +0,0 @@
|
|||
package org.apache.lucene.search.highlight;
|
||||
/**
|
||||
* Copyright 2002-2004 The Apache Software Foundation
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
|
||||
/**
|
||||
* Processes terms found in the original text, typically by applying some form
|
||||
* of mark-up to highlight terms in HTML search results pages.
|
||||
*
|
||||
*/
|
||||
public interface Formatter
|
||||
{
|
||||
/**
|
||||
* Highlights a search term. For example, an HTML Formatter could simply do:
|
||||
*
|
||||
* <p><dl><dt></dt><dd><code>return "<b>" + term + "</b>";</code></dd></dl>
|
||||
*
|
||||
* @param originalTermText (unstemmed) term text to highlight
|
||||
* @param stemmedTerm the stemmed form of the originalTermText
|
||||
* @param startOffset the position of the originalTermText in the text being highlighted
|
||||
*
|
||||
* @return highlighted term text
|
||||
*/
|
||||
String highlightTerm(String originalTermText, String stemmedTerm, float score, int startOffset);
|
||||
}
|
||||
|
|
@ -1,40 +0,0 @@
|
|||
package org.apache.lucene.search.highlight;
|
||||
/**
|
||||
* Copyright 2002-2004 The Apache Software Foundation
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import org.apache.lucene.analysis.Token;
|
||||
|
||||
/**
|
||||
* Implements the policy for breaking text into multiple fragments for consideration
|
||||
* by the {@link Highlighter} class. A sophisticated implementation may do this on the basis
|
||||
* of detecting end of sentences in the text.
|
||||
* @author mark@searcharea.co.uk
|
||||
*/
|
||||
public interface Fragmenter
|
||||
{
|
||||
/**
|
||||
* Initializes the Fragmenter
|
||||
* @param originalText
|
||||
*/
|
||||
public void start(String originalText);
|
||||
|
||||
/**
|
||||
* Test to see if this token from the stream should be held in a new TextFragment
|
||||
* @param token
|
||||
* @return
|
||||
*/
|
||||
public boolean isNewFragment(Token nextToken);
|
||||
}
|
||||
|
|
@ -1,430 +0,0 @@
|
|||
package org.apache.lucene.search.highlight;
|
||||
/**
|
||||
* Copyright 2002-2004 The Apache Software Foundation
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Iterator;
|
||||
|
||||
import org.apache.lucene.analysis.TokenStream;
|
||||
import org.apache.lucene.util.PriorityQueue;
|
||||
|
||||
/**
|
||||
* Class used to markup highlighted terms found in the best sections of a
|
||||
* text, using configurable {@link Fragmenter}, {@link Scorer}, {@link Formatter}
|
||||
* and tokenizers.
|
||||
* @author mark@searcharea.co.uk
|
||||
*/
|
||||
public class Highlighter
|
||||
{
|
||||
|
||||
public static final int DEFAULT_MAX_DOC_BYTES_TO_ANALYZE=50*1024;
|
||||
private int maxDocBytesToAnalyze=DEFAULT_MAX_DOC_BYTES_TO_ANALYZE;
|
||||
private Formatter formatter;
|
||||
private Fragmenter textFragmenter=new SimpleFragmenter();
|
||||
private Scorer fragmentScorer=null;
|
||||
|
||||
public Highlighter(Scorer fragmentScorer)
|
||||
{
|
||||
this(new SimpleHTMLFormatter(),fragmentScorer);
|
||||
}
|
||||
|
||||
|
||||
public Highlighter(Formatter formatter, Scorer fragmentScorer)
|
||||
{
|
||||
this.formatter = formatter;
|
||||
this.fragmentScorer = fragmentScorer;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* Highlights chosen terms in a text, extracting the most relevant section.
|
||||
* The document text is analysed in chunks to record hit statistics
|
||||
* across the document. After accumulating stats, the fragment with the highest score
|
||||
* is returned
|
||||
*
|
||||
* @param tokenStream a stream of tokens identified in the text parameter, including offset information.
|
||||
* This is typically produced by an analyzer re-parsing a document's
|
||||
* text. Some work may be done on retrieving TokenStreams more efficently
|
||||
* by adding support for storing original text position data in the Lucene
|
||||
* index but this support is not currently available (as of Lucene 1.4 rc2).
|
||||
* @param text text to highlight terms in
|
||||
*
|
||||
* @return highlighted text fragment or null if no terms found
|
||||
*/
|
||||
public final String getBestFragment(TokenStream tokenStream, String text)
|
||||
throws IOException
|
||||
{
|
||||
String[] results = getBestFragments(tokenStream,text, 1);
|
||||
if (results.length > 0)
|
||||
{
|
||||
return results[0];
|
||||
}
|
||||
return null;
|
||||
}
|
||||
/**
|
||||
* Highlights chosen terms in a text, extracting the most relevant sections.
|
||||
* The document text is analysed in chunks to record hit statistics
|
||||
* across the document. After accumulating stats, the fragments with the highest scores
|
||||
* are returned as an array of strings in order of score (contiguous fragments are merged into
|
||||
* one in their original order to improve readability)
|
||||
*
|
||||
* @param text text to highlight terms in
|
||||
* @param maxNumFragments the maximum number of fragments.
|
||||
*
|
||||
* @return highlighted text fragments (between 0 and maxNumFragments number of fragments)
|
||||
*/
|
||||
public final String[] getBestFragments(
|
||||
TokenStream tokenStream,
|
||||
String text,
|
||||
int maxNumFragments)
|
||||
throws IOException
|
||||
{
|
||||
maxNumFragments = Math.max(1, maxNumFragments); //sanity check
|
||||
StringBuffer newText = new StringBuffer();
|
||||
|
||||
TextFragment[] frag =getBestDocFragments(tokenStream,text, newText, maxNumFragments);
|
||||
|
||||
mergeContiguousFragments(frag);
|
||||
|
||||
//Get text
|
||||
ArrayList fragTexts = new ArrayList();
|
||||
int n = 0;
|
||||
for (int i = 0; i < frag.length; i++)
|
||||
{
|
||||
if ((frag[i] != null) && (frag[i].getScore() > 0))
|
||||
{
|
||||
fragTexts.add(
|
||||
newText.substring(
|
||||
frag[i].textStartPos,
|
||||
frag[i].textEndPos));
|
||||
}
|
||||
}
|
||||
return (String[]) fragTexts.toArray(new String[0]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Low level api to get the most relevant sections of the document
|
||||
* @param tokenStream
|
||||
* @param text
|
||||
* @param maxNumFragments
|
||||
* @return
|
||||
* @throws IOException
|
||||
*/
|
||||
private final TextFragment[] getBestDocFragments(
|
||||
TokenStream tokenStream,
|
||||
String text,
|
||||
StringBuffer newText,
|
||||
int maxNumFragments)
|
||||
throws IOException
|
||||
{
|
||||
ArrayList docFrags = new ArrayList();
|
||||
|
||||
TextFragment currentFrag = new TextFragment(newText.length(), docFrags.size());
|
||||
fragmentScorer.startFragment(currentFrag);
|
||||
docFrags.add(currentFrag);
|
||||
|
||||
FragmentQueue fragQueue = new FragmentQueue(maxNumFragments);
|
||||
|
||||
try
|
||||
{
|
||||
org.apache.lucene.analysis.Token token;
|
||||
String tokenText;
|
||||
int startOffset;
|
||||
int endOffset;
|
||||
int lastEndOffset = 0;
|
||||
textFragmenter.start(text);
|
||||
|
||||
while ((token = tokenStream.next()) != null)
|
||||
{
|
||||
|
||||
startOffset = token.startOffset();
|
||||
endOffset = token.endOffset();
|
||||
//FIXME an issue was reported with CJKTokenizer that I couldnt reproduce
|
||||
// where the analyzer was producing overlapping tokens.
|
||||
// I suspect the fix is to make startOffset=Math.max(startOffset,lastEndOffset+1)
|
||||
// but cant be sure so I'll just leave this comment in for now
|
||||
tokenText = text.substring(startOffset, endOffset);
|
||||
|
||||
|
||||
// append text between end of last token (or beginning of text) and start of current token
|
||||
if (startOffset > lastEndOffset)
|
||||
newText.append(text.substring(lastEndOffset, startOffset));
|
||||
|
||||
// does query contain current token?
|
||||
float score=fragmentScorer.getTokenScore(token);
|
||||
newText.append(formatter.highlightTerm(tokenText, token.termText(), score, startOffset));
|
||||
|
||||
|
||||
if(textFragmenter.isNewFragment(token))
|
||||
{
|
||||
currentFrag.setScore(fragmentScorer.getFragmentScore());
|
||||
//record stats for a new fragment
|
||||
currentFrag.textEndPos = newText.length();
|
||||
currentFrag =new TextFragment(newText.length(), docFrags.size());
|
||||
fragmentScorer.startFragment(currentFrag);
|
||||
docFrags.add(currentFrag);
|
||||
}
|
||||
|
||||
lastEndOffset = endOffset;
|
||||
if(lastEndOffset>maxDocBytesToAnalyze)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
currentFrag.setScore(fragmentScorer.getFragmentScore());
|
||||
|
||||
|
||||
// append text after end of last token
|
||||
if (lastEndOffset < text.length())
|
||||
newText.append(text.substring(lastEndOffset));
|
||||
|
||||
currentFrag.textEndPos = newText.length();
|
||||
|
||||
//sort the most relevant sections of the text
|
||||
int minScore = 0;
|
||||
for (Iterator i = docFrags.iterator(); i.hasNext();)
|
||||
{
|
||||
currentFrag = (TextFragment) i.next();
|
||||
|
||||
//If you are running with a version of Lucene before 11th Sept 03
|
||||
// you do not have PriorityQueue.insert() - so uncomment the code below
|
||||
/*
|
||||
if (currentFrag.getScore() >= minScore)
|
||||
{
|
||||
fragQueue.put(currentFrag);
|
||||
if (fragQueue.size() > maxNumFragments)
|
||||
{ // if hit queue overfull
|
||||
fragQueue.pop(); // remove lowest in hit queue
|
||||
minScore = ((TextFragment) fragQueue.top()).getScore(); // reset minScore
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
*/
|
||||
//The above code caused a problem as a result of Christoph Goller's 11th Sept 03
|
||||
//fix to PriorityQueue. The correct method to use here is the new "insert" method
|
||||
// USE ABOVE CODE IF THIS DOES NOT COMPILE!
|
||||
fragQueue.insert(currentFrag);
|
||||
}
|
||||
|
||||
//return the most relevant fragments
|
||||
TextFragment frag[] = new TextFragment[fragQueue.size()];
|
||||
for (int i = frag.length - 1; i >= 0; i--)
|
||||
{
|
||||
frag[i] = (TextFragment) fragQueue.pop();
|
||||
}
|
||||
return frag;
|
||||
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (tokenStream != null)
|
||||
{
|
||||
try
|
||||
{
|
||||
tokenStream.close();
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/** Improves readability of a score-sorted list of TextFragments by merging any fragments
|
||||
* that were contiguous in the original text into one larger fragment with the correct order.
|
||||
* This will leave a "null" in the array entry for the lesser scored fragment.
|
||||
*
|
||||
* @param frag An array of document fragments in descending score
|
||||
*/
|
||||
private void mergeContiguousFragments(TextFragment[] frag)
|
||||
{
|
||||
boolean mergingStillBeingDone;
|
||||
if (frag.length > 1)
|
||||
do
|
||||
{
|
||||
mergingStillBeingDone = false; //initialise loop control flag
|
||||
//for each fragment, scan other frags looking for contiguous blocks
|
||||
for (int i = 0; i < frag.length; i++)
|
||||
{
|
||||
if (frag[i] == null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
//merge any contiguous blocks
|
||||
for (int x = 0; x < frag.length; x++)
|
||||
{
|
||||
if (frag[x] == null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
if (frag[i] == null)
|
||||
{
|
||||
break;
|
||||
}
|
||||
TextFragment frag1 = null;
|
||||
TextFragment frag2 = null;
|
||||
int frag1Num = 0;
|
||||
int frag2Num = 0;
|
||||
int bestScoringFragNum;
|
||||
int worstScoringFragNum;
|
||||
//if blocks are contiguous....
|
||||
if (frag[i].follows(frag[x]))
|
||||
{
|
||||
frag1 = frag[x];
|
||||
frag1Num = x;
|
||||
frag2 = frag[i];
|
||||
frag2Num = i;
|
||||
}
|
||||
else
|
||||
if (frag[x].follows(frag[i]))
|
||||
{
|
||||
frag1 = frag[i];
|
||||
frag1Num = i;
|
||||
frag2 = frag[x];
|
||||
frag2Num = x;
|
||||
}
|
||||
//merging required..
|
||||
if (frag1 != null)
|
||||
{
|
||||
if (frag1.getScore() > frag2.getScore())
|
||||
{
|
||||
bestScoringFragNum = frag1Num;
|
||||
worstScoringFragNum = frag2Num;
|
||||
}
|
||||
else
|
||||
{
|
||||
bestScoringFragNum = frag2Num;
|
||||
worstScoringFragNum = frag1Num;
|
||||
}
|
||||
frag1.merge(frag2);
|
||||
frag[worstScoringFragNum] = null;
|
||||
mergingStillBeingDone = true;
|
||||
frag[bestScoringFragNum] = frag1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
while (mergingStillBeingDone);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Highlights terms in the text , extracting the most relevant sections
|
||||
* and concatenating the chosen fragments with a separator (typically "...").
|
||||
* The document text is analysed in chunks to record hit statistics
|
||||
* across the document. After accumulating stats, the fragments with the highest scores
|
||||
* are returned in order as "separator" delimited strings.
|
||||
*
|
||||
* @param text text to highlight terms in
|
||||
* @param maxNumFragments the maximum number of fragments.
|
||||
* @param separator the separator used to intersperse the document fragments (typically "...")
|
||||
*
|
||||
* @return highlighted text
|
||||
*/
|
||||
public final String getBestFragments(
|
||||
TokenStream tokenStream,
|
||||
String text,
|
||||
int maxNumFragments,
|
||||
String separator)
|
||||
throws IOException
|
||||
{
|
||||
String sections[] = getBestFragments(tokenStream,text, maxNumFragments);
|
||||
StringBuffer result = new StringBuffer();
|
||||
for (int i = 0; i < sections.length; i++)
|
||||
{
|
||||
if (i > 0)
|
||||
{
|
||||
result.append(separator);
|
||||
}
|
||||
result.append(sections[i]);
|
||||
}
|
||||
return result.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the maximum number of bytes to be tokenized per doc
|
||||
*/
|
||||
public int getMaxDocBytesToAnalyze()
|
||||
{
|
||||
return maxDocBytesToAnalyze;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param byteCount the maximum number of bytes to be tokenized per doc
|
||||
* (This can improve performance with large documents)
|
||||
*/
|
||||
public void setMaxDocBytesToAnalyze(int byteCount)
|
||||
{
|
||||
maxDocBytesToAnalyze = byteCount;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return
|
||||
*/
|
||||
public Fragmenter getTextFragmenter()
|
||||
{
|
||||
return textFragmenter;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param fragmenter
|
||||
*/
|
||||
public void setTextFragmenter(Fragmenter fragmenter)
|
||||
{
|
||||
textFragmenter = fragmenter;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return Object used to score each text fragment
|
||||
*/
|
||||
public Scorer getFragmentScorer()
|
||||
{
|
||||
return fragmentScorer;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* @param scorer
|
||||
*/
|
||||
public void setFragmentScorer(Scorer scorer)
|
||||
{
|
||||
fragmentScorer = scorer;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
class FragmentQueue extends PriorityQueue
|
||||
{
|
||||
public FragmentQueue(int size)
|
||||
{
|
||||
initialize(size);
|
||||
}
|
||||
|
||||
public final boolean lessThan(Object a, Object b)
|
||||
{
|
||||
TextFragment fragA = (TextFragment) a;
|
||||
TextFragment fragB = (TextFragment) b;
|
||||
if (fragA.getScore() == fragB.getScore())
|
||||
return fragA.fragNum > fragB.fragNum;
|
||||
else
|
||||
return fragA.getScore() < fragB.getScore();
|
||||
}
|
||||
}
|
||||
|
|
@ -1,442 +0,0 @@
|
|||
package org.apache.lucene.search.highlight;
|
||||
/**
|
||||
* Copyright 2002-2004 The Apache Software Foundation
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.StringReader;
|
||||
|
||||
import junit.framework.TestCase;
|
||||
|
||||
import org.apache.lucene.analysis.Analyzer;
|
||||
import org.apache.lucene.analysis.TokenStream;
|
||||
//import org.apache.lucene.analysis.cjk.CJKAnalyzer;
|
||||
import org.apache.lucene.analysis.standard.StandardAnalyzer;
|
||||
import org.apache.lucene.document.Document;
|
||||
import org.apache.lucene.document.Field;
|
||||
import org.apache.lucene.index.IndexReader;
|
||||
import org.apache.lucene.index.IndexWriter;
|
||||
import org.apache.lucene.queryParser.ParseException;
|
||||
import org.apache.lucene.queryParser.QueryParser;
|
||||
import org.apache.lucene.search.Hits;
|
||||
import org.apache.lucene.search.IndexSearcher;
|
||||
import org.apache.lucene.search.MultiSearcher;
|
||||
import org.apache.lucene.search.Query;
|
||||
import org.apache.lucene.search.Searcher;
|
||||
import org.apache.lucene.store.RAMDirectory;
|
||||
|
||||
/**
|
||||
* JUnit Test for Highlighter class.
|
||||
* @author mark@searcharea.co.uk
|
||||
*/
|
||||
public class HighlighterTest extends TestCase implements Formatter
|
||||
{
|
||||
private IndexReader reader;
|
||||
private static final String FIELD_NAME = "contents";
|
||||
private Query query;
|
||||
RAMDirectory ramDir;
|
||||
public Searcher searcher = null;
|
||||
public Hits hits = null;
|
||||
int numHighlights = 0;
|
||||
Analyzer analyzer=new StandardAnalyzer();
|
||||
|
||||
String texts[] =
|
||||
{
|
||||
"Hello this is a piece of text that is very long and contains too much preamble and the meat is really here which says kennedy has been shot",
|
||||
"This piece of text refers to Kennedy at the beginning then has a longer piece of text that is very long in the middle and finally ends with another reference to Kennedy",
|
||||
"JFK has been shot",
|
||||
"John Kennedy has been shot",
|
||||
"This text has a typo in referring to Keneddy" };
|
||||
|
||||
/**
|
||||
* Constructor for HighlightExtractorTest.
|
||||
* @param arg0
|
||||
*/
|
||||
public HighlighterTest(String arg0)
|
||||
{
|
||||
super(arg0);
|
||||
}
|
||||
|
||||
public void testSimpleHighlighter() throws Exception
|
||||
{
|
||||
doSearching("Kennedy");
|
||||
Highlighter highlighter = new Highlighter(new QueryScorer(query));
|
||||
highlighter.setTextFragmenter(new SimpleFragmenter(40));
|
||||
int maxNumFragmentsRequired = 2;
|
||||
for (int i = 0; i < hits.length(); i++)
|
||||
{
|
||||
String text = hits.doc(i).get(FIELD_NAME);
|
||||
TokenStream tokenStream=analyzer.tokenStream(FIELD_NAME,new StringReader(text));
|
||||
|
||||
String result =
|
||||
highlighter.getBestFragments(tokenStream,text,maxNumFragmentsRequired, "...");
|
||||
System.out.println("\t" + result);
|
||||
}
|
||||
//Not sure we can assert anything here - just running to check we dont throw any exceptions
|
||||
}
|
||||
|
||||
|
||||
|
||||
public void testGetBestFragmentsSimpleQuery() throws Exception
|
||||
{
|
||||
doSearching("Kennedy");
|
||||
doStandardHighlights();
|
||||
assertTrue("Failed to find correct number of highlights " + numHighlights + " found", numHighlights == 4);
|
||||
}
|
||||
public void testGetFuzzyFragments() throws Exception
|
||||
{
|
||||
doSearching("Kinnedy~");
|
||||
doStandardHighlights();
|
||||
assertTrue("Failed to find correct number of highlights " + numHighlights + " found", numHighlights == 4);
|
||||
}
|
||||
|
||||
public void testGetWildCardFragments() throws Exception
|
||||
{
|
||||
doSearching("K?nnedy");
|
||||
doStandardHighlights();
|
||||
assertTrue("Failed to find correct number of highlights " + numHighlights + " found", numHighlights == 4);
|
||||
}
|
||||
public void testGetMidWildCardFragments() throws Exception
|
||||
{
|
||||
doSearching("K*dy");
|
||||
doStandardHighlights();
|
||||
assertTrue("Failed to find correct number of highlights " + numHighlights + " found", numHighlights == 5);
|
||||
}
|
||||
public void testGetRangeFragments() throws Exception
|
||||
{
|
||||
doSearching(FIELD_NAME + ":[kannedy TO kznnedy]"); //bug?needs lower case
|
||||
doStandardHighlights();
|
||||
assertTrue("Failed to find correct number of highlights " + numHighlights + " found", numHighlights == 5);
|
||||
}
|
||||
|
||||
public void testGetBestFragmentsPhrase() throws Exception
|
||||
{
|
||||
doSearching("\"John Kennedy\"");
|
||||
doStandardHighlights();
|
||||
//Currently highlights "John" and "Kennedy" separately
|
||||
assertTrue("Failed to find correct number of highlights " + numHighlights + " found", numHighlights == 2);
|
||||
}
|
||||
|
||||
public void testGetBestFragmentsMultiTerm() throws Exception
|
||||
{
|
||||
doSearching("John Kenn*");
|
||||
doStandardHighlights();
|
||||
assertTrue("Failed to find correct number of highlights " + numHighlights + " found", numHighlights == 5);
|
||||
}
|
||||
public void testGetBestFragmentsWithOr() throws Exception
|
||||
{
|
||||
doSearching("JFK OR Kennedy");
|
||||
doStandardHighlights();
|
||||
assertTrue("Failed to find correct number of highlights " + numHighlights + " found", numHighlights == 5);
|
||||
}
|
||||
|
||||
|
||||
public void testGetBestSingleFragment() throws Exception
|
||||
{
|
||||
doSearching("Kennedy");
|
||||
// QueryHighlightExtractor highlighter = new QueryHighlightExtractor(this, query, new StandardAnalyzer());
|
||||
Highlighter highlighter =new Highlighter(this,new QueryScorer(query));
|
||||
highlighter.setTextFragmenter(new SimpleFragmenter(40));
|
||||
|
||||
for (int i = 0; i < hits.length(); i++)
|
||||
{
|
||||
String text = hits.doc(i).get(FIELD_NAME);
|
||||
TokenStream tokenStream=analyzer.tokenStream(FIELD_NAME,new StringReader(text));
|
||||
String result = highlighter.getBestFragment(tokenStream,text);
|
||||
System.out.println("\t" + result);
|
||||
}
|
||||
assertTrue("Failed to find correct number of highlights " + numHighlights + " found", numHighlights == 4);
|
||||
}
|
||||
|
||||
public void testGetBestSingleFragmentWithWeights() throws Exception
|
||||
{
|
||||
WeightedTerm[]wTerms=new WeightedTerm[2];
|
||||
wTerms[0]=new WeightedTerm(10f,"hello");
|
||||
wTerms[1]=new WeightedTerm(1f,"kennedy");
|
||||
Highlighter highlighter =new Highlighter(new QueryScorer(wTerms));
|
||||
TokenStream tokenStream=analyzer.tokenStream(FIELD_NAME,new StringReader(texts[0]));
|
||||
highlighter.setTextFragmenter(new SimpleFragmenter(2));
|
||||
|
||||
String result = highlighter.getBestFragment(tokenStream,texts[0]).trim();
|
||||
assertTrue("Failed to find best section using weighted terms. Found: "+result
|
||||
, "<B>Hello</B>".equals(result));
|
||||
|
||||
//readjust weights
|
||||
wTerms[1].setWeight(50f);
|
||||
tokenStream=analyzer.tokenStream(FIELD_NAME,new StringReader(texts[0]));
|
||||
highlighter =new Highlighter(new QueryScorer(wTerms));
|
||||
highlighter.setTextFragmenter(new SimpleFragmenter(2));
|
||||
|
||||
result = highlighter.getBestFragment(tokenStream,texts[0]).trim();
|
||||
assertTrue("Failed to find best section using weighted terms. Found: "+result
|
||||
, "<B>kennedy</B>".equals(result));
|
||||
}
|
||||
|
||||
|
||||
|
||||
public void testGetSimpleHighlight() throws Exception
|
||||
{
|
||||
doSearching("Kennedy");
|
||||
Highlighter highlighter =
|
||||
new Highlighter(this,new QueryScorer(query));
|
||||
|
||||
for (int i = 0; i < hits.length(); i++)
|
||||
{
|
||||
String text = hits.doc(i).get(FIELD_NAME);
|
||||
TokenStream tokenStream=analyzer.tokenStream(FIELD_NAME,new StringReader(text));
|
||||
|
||||
String result = highlighter.getBestFragment(tokenStream,text);
|
||||
System.out.println("\t" + result);
|
||||
}
|
||||
assertTrue("Failed to find correct number of highlights " + numHighlights + " found", numHighlights == 4);
|
||||
}
|
||||
|
||||
public void testMaxSizeHighlight() throws Exception
|
||||
{
|
||||
doSearching("meat");
|
||||
Highlighter highlighter =
|
||||
new Highlighter(this,new QueryScorer(query));
|
||||
highlighter.setMaxDocBytesToAnalyze(30);
|
||||
TokenStream tokenStream=analyzer.tokenStream(FIELD_NAME,new StringReader(texts[0]));
|
||||
String result = highlighter.getBestFragment(tokenStream,texts[0]);
|
||||
assertTrue("Setting MaxDocBytesToAnalyze should have prevented " +
|
||||
"us from finding matches for this record" + numHighlights +
|
||||
" found", numHighlights == 0);
|
||||
}
|
||||
|
||||
|
||||
|
||||
public void testUnRewrittenQuery() throws IOException, ParseException
|
||||
{
|
||||
//test to show how rewritten query can still be used
|
||||
searcher = new IndexSearcher(ramDir);
|
||||
Analyzer analyzer=new StandardAnalyzer();
|
||||
Query query = QueryParser.parse("JF? or Kenned*", FIELD_NAME, analyzer);
|
||||
System.out.println("Searching with primitive query");
|
||||
//forget to set this and...
|
||||
//query=query.rewrite(reader);
|
||||
Hits hits = searcher.search(query);
|
||||
|
||||
//create an instance of the highlighter with the tags used to surround highlighted text
|
||||
// QueryHighlightExtractor highlighter = new QueryHighlightExtractor(this, query, new StandardAnalyzer());
|
||||
Highlighter highlighter =
|
||||
new Highlighter(this,new QueryScorer(query));
|
||||
|
||||
highlighter.setTextFragmenter(new SimpleFragmenter(40));
|
||||
|
||||
int maxNumFragmentsRequired = 3;
|
||||
|
||||
for (int i = 0; i < hits.length(); i++)
|
||||
{
|
||||
String text = hits.doc(i).get(FIELD_NAME);
|
||||
TokenStream tokenStream=analyzer.tokenStream(FIELD_NAME,new StringReader(text));
|
||||
|
||||
String highlightedText = highlighter.getBestFragments(tokenStream,text,maxNumFragmentsRequired,"...");
|
||||
System.out.println(highlightedText);
|
||||
}
|
||||
//We expect to have zero highlights if the query is multi-terms and is not rewritten!
|
||||
assertTrue("Failed to find correct number of highlights " + numHighlights + " found", numHighlights == 0);
|
||||
}
|
||||
|
||||
public void testNoFragments() throws Exception
|
||||
{
|
||||
doSearching("AnInvalidQueryWhichShouldYieldNoResults");
|
||||
Highlighter highlighter =
|
||||
new Highlighter(this,new QueryScorer(query));
|
||||
|
||||
int highlightFragmentSizeInBytes = 40;
|
||||
for (int i = 0; i < texts.length; i++)
|
||||
{
|
||||
String text = texts[i];
|
||||
TokenStream tokenStream=analyzer.tokenStream(FIELD_NAME,new StringReader(text));
|
||||
|
||||
String result = highlighter.getBestFragment(tokenStream,text);
|
||||
assertNull("The highlight result should be null for text with no query terms", result);
|
||||
}
|
||||
}
|
||||
|
||||
public void testMultiSearcher() throws Exception
|
||||
{
|
||||
//setup index 1
|
||||
RAMDirectory ramDir1 = new RAMDirectory();
|
||||
IndexWriter writer1 = new IndexWriter(ramDir1, new StandardAnalyzer(), true);
|
||||
Document d = new Document();
|
||||
Field f = new Field(FIELD_NAME, "multiOne", true, true, true);
|
||||
d.add(f);
|
||||
writer1.addDocument(d);
|
||||
writer1.optimize();
|
||||
writer1.close();
|
||||
IndexReader reader1 = IndexReader.open(ramDir1);
|
||||
|
||||
//setup index 2
|
||||
RAMDirectory ramDir2 = new RAMDirectory();
|
||||
IndexWriter writer2 = new IndexWriter(ramDir2, new StandardAnalyzer(), true);
|
||||
d = new Document();
|
||||
f = new Field(FIELD_NAME, "multiTwo", true, true, true);
|
||||
d.add(f);
|
||||
writer2.addDocument(d);
|
||||
writer2.optimize();
|
||||
writer2.close();
|
||||
IndexReader reader2 = IndexReader.open(ramDir2);
|
||||
|
||||
|
||||
|
||||
IndexSearcher searchers[]=new IndexSearcher[2];
|
||||
searchers[0] = new IndexSearcher(ramDir1);
|
||||
searchers[1] = new IndexSearcher(ramDir2);
|
||||
MultiSearcher multiSearcher=new MultiSearcher(searchers);
|
||||
query = QueryParser.parse("multi*", FIELD_NAME, new StandardAnalyzer());
|
||||
System.out.println("Searching for: " + query.toString(FIELD_NAME));
|
||||
//at this point the multisearcher calls combine(query[])
|
||||
hits = multiSearcher.search(query);
|
||||
|
||||
//query = QueryParser.parse("multi*", FIELD_NAME, new StandardAnalyzer());
|
||||
Query expandedQueries[]=new Query[2];
|
||||
expandedQueries[0]=query.rewrite(reader1);
|
||||
expandedQueries[1]=query.rewrite(reader2);
|
||||
query=query.combine(expandedQueries);
|
||||
|
||||
|
||||
//create an instance of the highlighter with the tags used to surround highlighted text
|
||||
Highlighter highlighter =
|
||||
new Highlighter(this,new QueryScorer(query));
|
||||
|
||||
for (int i = 0; i < hits.length(); i++)
|
||||
{
|
||||
String text = hits.doc(i).get(FIELD_NAME);
|
||||
TokenStream tokenStream=analyzer.tokenStream(FIELD_NAME,new StringReader(text));
|
||||
String highlightedText = highlighter.getBestFragment(tokenStream,text);
|
||||
System.out.println(highlightedText);
|
||||
}
|
||||
assertTrue("Failed to find correct number of highlights " + numHighlights + " found", numHighlights == 2);
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
public void testBigramAnalyzer() throws IOException, ParseException
|
||||
{
|
||||
//test to ensure analyzers with none-consecutive start/end offsets
|
||||
//dont double-highlight text
|
||||
//setup index 1
|
||||
RAMDirectory ramDir = new RAMDirectory();
|
||||
Analyzer bigramAnalyzer=new CJKAnalyzer();
|
||||
IndexWriter writer = new IndexWriter(ramDir,bigramAnalyzer , true);
|
||||
Document d = new Document();
|
||||
Field f = new Field(FIELD_NAME, "java abc def", true, true, true);
|
||||
d.add(f);
|
||||
writer.addDocument(d);
|
||||
writer.close();
|
||||
IndexReader reader = IndexReader.open(ramDir);
|
||||
|
||||
IndexSearcher searcher=new IndexSearcher(reader);
|
||||
query = QueryParser.parse("abc", FIELD_NAME, bigramAnalyzer);
|
||||
System.out.println("Searching for: " + query.toString(FIELD_NAME));
|
||||
hits = searcher.search(query);
|
||||
|
||||
Highlighter highlighter =
|
||||
new Highlighter(this,new QueryFragmentScorer(query));
|
||||
|
||||
for (int i = 0; i < hits.length(); i++)
|
||||
{
|
||||
String text = hits.doc(i).get(FIELD_NAME);
|
||||
TokenStream tokenStream=bigramAnalyzer.tokenStream(FIELD_NAME,new StringReader(text));
|
||||
String highlightedText = highlighter.getBestFragment(tokenStream,text);
|
||||
System.out.println(highlightedText);
|
||||
}
|
||||
|
||||
}
|
||||
*/
|
||||
|
||||
|
||||
public String highlightTerm(String originalText , String weightedTerm, float score, int startOffset)
|
||||
{
|
||||
if(score<=0)
|
||||
{
|
||||
return originalText;
|
||||
}
|
||||
numHighlights++; //update stats used in assertions
|
||||
return "<b>" + originalText + "</b>";
|
||||
}
|
||||
|
||||
public void doSearching(String queryString) throws Exception
|
||||
{
|
||||
searcher = new IndexSearcher(ramDir);
|
||||
query = QueryParser.parse(queryString, FIELD_NAME, new StandardAnalyzer());
|
||||
//for any multi-term queries to work (prefix, wildcard, range,fuzzy etc) you must use a rewritten query!
|
||||
query=query.rewrite(reader);
|
||||
System.out.println("Searching for: " + query.toString(FIELD_NAME));
|
||||
hits = searcher.search(query);
|
||||
}
|
||||
|
||||
void doStandardHighlights() throws Exception
|
||||
{
|
||||
Highlighter highlighter =new Highlighter(this,new QueryScorer(query));
|
||||
highlighter.setTextFragmenter(new SimpleFragmenter(20));
|
||||
for (int i = 0; i < hits.length(); i++)
|
||||
{
|
||||
String text = hits.doc(i).get(FIELD_NAME);
|
||||
int maxNumFragmentsRequired = 2;
|
||||
String fragmentSeparator = "...";
|
||||
TokenStream tokenStream=analyzer.tokenStream(FIELD_NAME,new StringReader(text));
|
||||
|
||||
String result =
|
||||
highlighter.getBestFragments(
|
||||
tokenStream,
|
||||
text,
|
||||
maxNumFragmentsRequired,
|
||||
fragmentSeparator);
|
||||
System.out.println("\t" + result);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* @see TestCase#setUp()
|
||||
*/
|
||||
protected void setUp() throws Exception
|
||||
{
|
||||
ramDir = new RAMDirectory();
|
||||
IndexWriter writer = new IndexWriter(ramDir, new StandardAnalyzer(), true);
|
||||
for (int i = 0; i < texts.length; i++)
|
||||
{
|
||||
addDoc(writer, texts[i]);
|
||||
}
|
||||
|
||||
writer.optimize();
|
||||
writer.close();
|
||||
reader = IndexReader.open(ramDir);
|
||||
numHighlights = 0;
|
||||
}
|
||||
|
||||
private void addDoc(IndexWriter writer, String text) throws IOException
|
||||
{
|
||||
Document d = new Document();
|
||||
Field f = new Field(FIELD_NAME, text, true, true, true);
|
||||
d.add(f);
|
||||
writer.addDocument(d);
|
||||
|
||||
}
|
||||
|
||||
/*
|
||||
* @see TestCase#tearDown()
|
||||
*/
|
||||
protected void tearDown() throws Exception
|
||||
{
|
||||
super.tearDown();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,113 +0,0 @@
|
|||
package org.apache.lucene.search.highlight;
|
||||
/**
|
||||
* Copyright 2002-2004 The Apache Software Foundation
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
|
||||
import org.apache.lucene.analysis.Token;
|
||||
import org.apache.lucene.search.Query;
|
||||
|
||||
/**
|
||||
* {@link Scorer} implementation which scores text fragments by the number of unique query terms found.
|
||||
* This class uses the {@link QueryTermExtractor} class to process determine the query terms and
|
||||
* their boosts to be used.
|
||||
* @author mark@searcharea.co.uk
|
||||
*/
|
||||
//TODO: provide option to roll idf into the scoring equation by passing a IndexReader.
|
||||
//TODO: provide option to boost score of fragments near beginning of document
|
||||
// based on fragment.getFragNum()
|
||||
public class QueryScorer implements Scorer
|
||||
{
|
||||
TextFragment currentTextFragment=null;
|
||||
HashSet uniqueTermsInFragment;
|
||||
float totalScore=0;
|
||||
private HashMap termsToFind;
|
||||
|
||||
|
||||
/**
|
||||
*
|
||||
* @param query a Lucene query (ideally rewritten using query.rewrite
|
||||
* before being passed to this class and the searcher)
|
||||
*/
|
||||
public QueryScorer(Query query)
|
||||
{
|
||||
this(QueryTermExtractor.getTerms(query));
|
||||
}
|
||||
|
||||
|
||||
public QueryScorer(WeightedTerm []weightedTerms )
|
||||
{
|
||||
termsToFind = new HashMap();
|
||||
for (int i = 0; i < weightedTerms.length; i++)
|
||||
{
|
||||
termsToFind.put(weightedTerms[i].term,weightedTerms[i]);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/* (non-Javadoc)
|
||||
* @see org.apache.lucene.search.highlight.FragmentScorer#startFragment(org.apache.lucene.search.highlight.TextFragment)
|
||||
*/
|
||||
public void startFragment(TextFragment newFragment)
|
||||
{
|
||||
uniqueTermsInFragment = new HashSet();
|
||||
currentTextFragment=newFragment;
|
||||
totalScore=0;
|
||||
|
||||
}
|
||||
|
||||
/* (non-Javadoc)
|
||||
* @see org.apache.lucene.search.highlight.FragmentScorer#scoreToken(org.apache.lucene.analysis.Token)
|
||||
*/
|
||||
public float getTokenScore(Token token)
|
||||
{
|
||||
String termText=token.termText();
|
||||
|
||||
WeightedTerm queryTerm=(WeightedTerm) termsToFind.get(termText);
|
||||
if(queryTerm==null)
|
||||
{
|
||||
//not a query term - return
|
||||
return 0;
|
||||
}
|
||||
//found a query term - is it unique in this doc?
|
||||
if(!uniqueTermsInFragment.contains(termText))
|
||||
{
|
||||
totalScore+=queryTerm.getWeight();
|
||||
uniqueTermsInFragment.add(termText);
|
||||
}
|
||||
return queryTerm.getWeight();
|
||||
}
|
||||
|
||||
|
||||
/* (non-Javadoc)
|
||||
* @see org.apache.lucene.search.highlight.FragmentScorer#endFragment(org.apache.lucene.search.highlight.TextFragment)
|
||||
*/
|
||||
public float getFragmentScore()
|
||||
{
|
||||
return totalScore;
|
||||
}
|
||||
|
||||
|
||||
/* (non-Javadoc)
|
||||
* @see org.apache.lucene.search.highlight.FragmentScorer#allFragmentsProcessed()
|
||||
*/
|
||||
public void allFragmentsProcessed()
|
||||
{
|
||||
//this class has no special operations to perform at end of processing
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,115 +0,0 @@
|
|||
package org.apache.lucene.search.highlight;
|
||||
/**
|
||||
* Copyright 2002-2004 The Apache Software Foundation
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import java.util.HashSet;
|
||||
|
||||
import org.apache.lucene.index.Term;
|
||||
import org.apache.lucene.search.BooleanClause;
|
||||
import org.apache.lucene.search.BooleanQuery;
|
||||
import org.apache.lucene.search.PhraseQuery;
|
||||
import org.apache.lucene.search.Query;
|
||||
import org.apache.lucene.search.TermQuery;
|
||||
|
||||
/**
|
||||
* Utility class used to extract the terms used in a query, plus any weights.
|
||||
* This class will not find terms for MultiTermQuery, RangeQuery and PrefixQuery classes
|
||||
* so the caller must pass a rewritten query (see query.rewrite) to obtain a list of
|
||||
* expanded terms.
|
||||
*
|
||||
*/
|
||||
public final class QueryTermExtractor
|
||||
{
|
||||
|
||||
/**
|
||||
* Extracts all terms texts of a given Query into an array of WeightedTerms
|
||||
*
|
||||
* @param query Query to extract term texts from
|
||||
* @return an array of the terms used in a query, plus their weights.
|
||||
* @throws IOException
|
||||
*/
|
||||
public static final WeightedTerm[] getTerms(Query query)
|
||||
{
|
||||
return getTerms(query,false);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Extracts all terms texts of a given Query into an array of WeightedTerms
|
||||
*
|
||||
* @param query Query to extract term texts from
|
||||
* @param prohibited <code>true</code> to extract "prohibited" terms, too
|
||||
* @return an array of the terms used in a query, plus their weights.
|
||||
* @throws IOException
|
||||
*/
|
||||
public static final WeightedTerm[] getTerms(Query query, boolean prohibited)
|
||||
{
|
||||
HashSet terms=new HashSet();
|
||||
getTerms(query,terms,prohibited);
|
||||
return (WeightedTerm[]) terms.toArray(new WeightedTerm[0]);
|
||||
}
|
||||
|
||||
private static final void getTerms(Query query, HashSet terms,boolean prohibited)
|
||||
{
|
||||
if (query instanceof BooleanQuery)
|
||||
getTermsFromBooleanQuery((BooleanQuery) query, terms, prohibited);
|
||||
else
|
||||
if (query instanceof PhraseQuery)
|
||||
getTermsFromPhraseQuery((PhraseQuery) query, terms);
|
||||
else
|
||||
if (query instanceof TermQuery)
|
||||
getTermsFromTermQuery((TermQuery) query, terms);
|
||||
// else
|
||||
// if ((query instanceof PrefixQuery)
|
||||
// || (query instanceof RangeQuery)
|
||||
// || (query instanceof MultiTermQuery))
|
||||
// {
|
||||
// //client should call rewrite BEFORE calling highlighter
|
||||
// // Query expandedQuery = rewrite(reader, query);
|
||||
// // getTerms(reader, expandedQuery, terms, prohibited);
|
||||
// }
|
||||
}
|
||||
|
||||
private static final void getTermsFromBooleanQuery(BooleanQuery query, HashSet terms, boolean prohibited)
|
||||
{
|
||||
BooleanClause[] queryClauses = query.getClauses();
|
||||
int i;
|
||||
|
||||
for (i = 0; i < queryClauses.length; i++)
|
||||
{
|
||||
if (prohibited || !queryClauses[i].prohibited)
|
||||
getTerms(queryClauses[i].query, terms, prohibited);
|
||||
}
|
||||
}
|
||||
|
||||
private static final void getTermsFromPhraseQuery(PhraseQuery query, HashSet terms)
|
||||
{
|
||||
Term[] queryTerms = query.getTerms();
|
||||
int i;
|
||||
|
||||
for (i = 0; i < queryTerms.length; i++)
|
||||
{
|
||||
terms.add(new WeightedTerm(query.getBoost(),queryTerms[i].text()));
|
||||
}
|
||||
}
|
||||
|
||||
private static final void getTermsFromTermQuery(TermQuery query, HashSet terms)
|
||||
{
|
||||
terms.add(new WeightedTerm(query.getBoost(),query.getTerm().text()));
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -1,48 +0,0 @@
|
|||
package org.apache.lucene.search.highlight;
|
||||
/**
|
||||
* Copyright 2002-2004 The Apache Software Foundation
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import org.apache.lucene.analysis.Token;
|
||||
|
||||
/**
|
||||
* Adds to the score for a fragment based on its tokens
|
||||
* @author mark@searcharea.co.uk
|
||||
*/
|
||||
public interface Scorer
|
||||
{
|
||||
/**
|
||||
* called when a new fragment is started for consideration
|
||||
* @param newFragment
|
||||
*/
|
||||
public void startFragment(TextFragment newFragment);
|
||||
|
||||
/**
|
||||
* Called for each token in the current fragment
|
||||
* @param token The token to be scored
|
||||
* @return a score which is passed to the TermHighlighter class to influence the mark-up of the text
|
||||
* (this return value is NOT used to score the fragment)
|
||||
*/
|
||||
public float getTokenScore(Token token);
|
||||
|
||||
|
||||
/**
|
||||
* Called when the highlighter has no more tokens for the current fragment - the scorer will typically
|
||||
* call setScore() on the fragment passed in startFragment to record total info
|
||||
*
|
||||
*/
|
||||
public float getFragmentScore();
|
||||
|
||||
}
|
||||
|
|
@ -1,84 +0,0 @@
|
|||
package org.apache.lucene.search.highlight;
|
||||
/**
|
||||
* Copyright 2002-2004 The Apache Software Foundation
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import org.apache.lucene.analysis.Token;
|
||||
|
||||
/**
|
||||
* {@link Fragmenter} implementation which breaks text up into same-size
|
||||
* fragments with no concerns over spotting sentence boundaries.
|
||||
* @author mark@searcharea.co.uk
|
||||
*/
|
||||
public class SimpleFragmenter implements Fragmenter
|
||||
{
|
||||
private static final int DEFAULT_FRAGMENT_SIZE =100;
|
||||
private int currentNumFrags;
|
||||
private int fragmentSize;
|
||||
|
||||
|
||||
public SimpleFragmenter()
|
||||
{
|
||||
this(DEFAULT_FRAGMENT_SIZE);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
*
|
||||
* @param fragmentSize size in bytes of each fragment
|
||||
*/
|
||||
public SimpleFragmenter(int fragmentSize)
|
||||
{
|
||||
this.fragmentSize=fragmentSize;
|
||||
}
|
||||
|
||||
/* (non-Javadoc)
|
||||
* @see org.apache.lucene.search.highlight.TextFragmenter#start(java.lang.String)
|
||||
*/
|
||||
public void start(String originalText)
|
||||
{
|
||||
currentNumFrags=1;
|
||||
}
|
||||
|
||||
/* (non-Javadoc)
|
||||
* @see org.apache.lucene.search.highlight.TextFragmenter#isNewFragment(org.apache.lucene.analysis.Token)
|
||||
*/
|
||||
public boolean isNewFragment(Token token)
|
||||
{
|
||||
boolean isNewFrag= token.endOffset()>=(fragmentSize*currentNumFrags);
|
||||
if(isNewFrag)
|
||||
{
|
||||
currentNumFrags++;
|
||||
}
|
||||
return isNewFrag;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return size in bytes of each fragment
|
||||
*/
|
||||
public int getFragmentSize()
|
||||
{
|
||||
return fragmentSize;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param size size in bytes of each fragment
|
||||
*/
|
||||
public void setFragmentSize(int size)
|
||||
{
|
||||
fragmentSize = size;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,57 +0,0 @@
|
|||
package org.apache.lucene.search.highlight;
|
||||
/**
|
||||
* Copyright 2002-2004 The Apache Software Foundation
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Simple {@link Formatter} implementation to highlight terms with a pre and post tag
|
||||
* @author MAHarwood
|
||||
*
|
||||
*/
|
||||
public class SimpleHTMLFormatter implements Formatter
|
||||
{
|
||||
String preTag;
|
||||
String postTag;
|
||||
|
||||
public SimpleHTMLFormatter(String preTag, String postTag)
|
||||
{
|
||||
this.preTag = preTag;
|
||||
this.postTag = postTag;
|
||||
}
|
||||
|
||||
/**
|
||||
* Default constructor uses HTML: <B> tags to markup terms
|
||||
*
|
||||
**/
|
||||
public SimpleHTMLFormatter()
|
||||
{
|
||||
this.preTag = "<B>";
|
||||
this.postTag = "</B>";
|
||||
}
|
||||
|
||||
public String highlightTerm(String originalText, String term, float score, int startOffset)
|
||||
{
|
||||
if(score<=0)
|
||||
{
|
||||
return originalText;
|
||||
}
|
||||
StringBuffer sb = new StringBuffer();
|
||||
sb.append(preTag);
|
||||
sb.append(originalText);
|
||||
sb.append(postTag);
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,70 +0,0 @@
|
|||
package org.apache.lucene.search.highlight;
|
||||
/**
|
||||
* Copyright 2002-2004 The Apache Software Foundation
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
|
||||
/**
|
||||
* Low-level class used to record information about a section of a document
|
||||
* with a score.
|
||||
* @author MAHarwood
|
||||
*
|
||||
*
|
||||
*/
|
||||
public class TextFragment
|
||||
{
|
||||
int fragNum;
|
||||
int textStartPos;
|
||||
int textEndPos;
|
||||
float score;
|
||||
|
||||
public TextFragment(int textStartPos, int fragNum)
|
||||
{
|
||||
this.textStartPos = textStartPos;
|
||||
this.fragNum = fragNum;
|
||||
}
|
||||
void setScore(float score)
|
||||
{
|
||||
this.score=score;
|
||||
}
|
||||
public float getScore()
|
||||
{
|
||||
return score;
|
||||
}
|
||||
/**
|
||||
* @param frag2 Fragment to be merged into this one
|
||||
*/
|
||||
public void merge(TextFragment frag2)
|
||||
{
|
||||
textEndPos = frag2.textEndPos;
|
||||
}
|
||||
/**
|
||||
* @param fragment
|
||||
* @return true if this fragment follows the one passed
|
||||
*/
|
||||
public boolean follows(TextFragment fragment)
|
||||
{
|
||||
return textStartPos == fragment.textEndPos;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the fragment sequence number
|
||||
*/
|
||||
public int getFragNum()
|
||||
{
|
||||
return fragNum;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,64 +0,0 @@
|
|||
package org.apache.lucene.search.highlight;
|
||||
/**
|
||||
* Copyright 2002-2004 The Apache Software Foundation
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
/** Lightweight class to hold term and a weight value used for scoring this term
|
||||
* @author Mark Harwood
|
||||
*/
|
||||
public class WeightedTerm
|
||||
{
|
||||
float weight; // multiplier
|
||||
String term; //stemmed form
|
||||
public WeightedTerm (float weight,String term)
|
||||
{
|
||||
this.weight=weight;
|
||||
this.term=term;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* @return the term value (stemmed)
|
||||
*/
|
||||
public String getTerm()
|
||||
{
|
||||
return term;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the weight associated with this term
|
||||
*/
|
||||
public float getWeight()
|
||||
{
|
||||
return weight;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param term the term value (stemmed)
|
||||
*/
|
||||
public void setTerm(String term)
|
||||
{
|
||||
this.term = term;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param weight the weight associated with this term
|
||||
*/
|
||||
public void setWeight(float weight)
|
||||
{
|
||||
this.weight = weight;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,28 +0,0 @@
|
|||
<html>
|
||||
<body>
|
||||
The highlight package contains classes to provide "keyword in context" features
|
||||
typically used to highlight search terms in the text of results pages. <br>
|
||||
The Highlighter class is the central component and can be used to extract the
|
||||
most interesting sections of a piece of text and highlight them, with the help of
|
||||
Fragmenter, FragmentScorer and Formatter classes.
|
||||
<h2>Example Usage</h2>
|
||||
|
||||
<pre>
|
||||
IndexSearcher searcher = new IndexSearcher(ramDir);
|
||||
Query query = QueryParser.parse("Kenne*", FIELD_NAME, analyzer);
|
||||
query=query.rewrite(reader); //required to expand search terms
|
||||
Hits hits = searcher.search(query);
|
||||
|
||||
Highlighter highlighter =new Highlighter(this,new QueryScorer(query));
|
||||
for (int i = 0; i < hits.length(); i++)
|
||||
{
|
||||
String text = hits.doc(i).get(FIELD_NAME);
|
||||
TokenStream tokenStream=analyzer.tokenStream(FIELD_NAME,new StringReader(text));
|
||||
// Get 3 best fragments and seperate with a "..."
|
||||
String result = highlighter.getBestFragments(tokenStream,text,3,"...");
|
||||
System.out.println(result);
|
||||
}
|
||||
</pre>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
Loading…
Reference in New Issue