Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix SpanScorer next()/skipTo() incompatability: LUCENE-413 

git-svn-id: https://svn.apache.org/repos/asf/lucene/java/trunk@391337 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Yonik Seeley 2006-04-04 15:25:58 +00:00
parent 286f4f5f07
commit 040e9c6c9a
5 changed files with 76 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
CHANGES.txt
View File

@ -16,6 +16,9 @@ Bug fixes
IndexWriter. These could be set in Lucene 1.4 using a system property.
This feature had been removed without adding the corresponding
getter/setter methods. (Daniel Naber)
4. LUCENE-413: Fixed ArrayIndexOutOfBoundsException exceptions
when using SpanQueries. (Paul Elschot via Yonik Seeley)
1.9.1

2
src/java/org/apache/lucene/search/DisjunctionSumScorer.java
View File

@ -207,7 +207,7 @@ public class DisjunctionSumScorer extends Scorer {
return false;
}
if (target <= currentDoc) {
target = currentDoc + 1;
return true;
}
do {
Scorer top = (Scorer) scorerQueue.top();

45
src/java/org/apache/lucene/search/spans/SpanScorer.java
View File

@ -1,7 +1,7 @@
package org.apache.lucene.search.spans;
/**
* Copyright 2004 The Apache Software Foundation
* Copyright 2006 The Apache Software Foundation
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -43,6 +43,7 @@ class SpanScorer extends Scorer {
this.norms = norms;
this.weight = weight;
this.value = weight.getValue();
doc = -1;
}
public boolean next() throws IOException {
@ -50,19 +51,35 @@ class SpanScorer extends Scorer {
more = spans.next();
firstTime = false;
}
return setFreqCurrentDoc();
}
if (!more) return false;
public boolean skipTo(int target) throws IOException {
if (firstTime) {
more = spans.skipTo(target);
firstTime = false;
}
if (! more) {
return false;
}
if (spans.doc() < target) { // setFreqCurrentDoc() leaves spans.doc() ahead
more = spans.skipTo(target);
}
return setFreqCurrentDoc();
}
freq = 0.0f;
private boolean setFreqCurrentDoc() throws IOException {
if (! more) {
return false;
}
doc = spans.doc();
freq = 0.0f;
while (more && doc == spans.doc()) {
int matchLength = spans.end() - spans.start();
freq += getSimilarity().sloppyFreq(matchLength);
more = spans.next();
}
return more || freq != 0.0f;
return more || (freq != 0);
}
public int doc() { return doc; }
@ -72,22 +89,6 @@ class SpanScorer extends Scorer {
return raw * Similarity.decodeNorm(norms[doc]); // normalize
}
public boolean skipTo(int target) throws IOException {
more = spans.skipTo(target);
if (!more) return false;
freq = 0.0f;
doc = spans.doc();
while (more && spans.doc() == target) {
freq += getSimilarity().sloppyFreq(spans.end() - spans.start());
more = spans.next();
}
return more || freq != 0.0f;
}
public Explanation explain(final int doc) throws IOException {
Explanation tfExplanation = new Explanation();

70
src/test/org/apache/lucene/search/spans/TestSpansAdvanced.java
View File

@ -25,11 +25,7 @@ import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.Hits;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.*;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.RAMDirectory;
@ -44,6 +40,8 @@ public class TestSpansAdvanced extends TestCase {
// location to the index
protected Directory mDirectory;;
protected IndexSearcher searcher;
// field names in the index
private final static String FIELD_ID = "ID";
protected final static String FIELD_TEXT = "TEXT";
@ -52,7 +50,6 @@ public class TestSpansAdvanced extends TestCase {
* Initializes the tests by adding 4 identical documents to the index.
*/
protected void setUp() throws Exception {
super.setUp();
// create test index
@ -63,10 +60,11 @@ public class TestSpansAdvanced extends TestCase {
addDocument(writer, "3", "I think it should work.");
addDocument(writer, "4", "I think it should work.");
writer.close();
searcher = new IndexSearcher(mDirectory);
}
protected void tearDown() throws Exception {
searcher.close();
mDirectory.close();
mDirectory = null;
}
@ -94,7 +92,7 @@ public class TestSpansAdvanced extends TestCase {
*/
public void testBooleanQueryWithSpanQueries() throws IOException {
doTestBooleanQueryWithSpanQueries(0.3884282f);
doTestBooleanQueryWithSpanQueries(searcher,0.3884282f);
}
/**
@ -102,57 +100,63 @@ public class TestSpansAdvanced extends TestCase {
*
* @throws IOException
*/
protected void doTestBooleanQueryWithSpanQueries(final float expectedScore) throws IOException {
protected void doTestBooleanQueryWithSpanQueries(IndexSearcher s, final float expectedScore) throws IOException {
final Query spanQuery = new SpanTermQuery(new Term(FIELD_TEXT, "work"));
final BooleanQuery query = new BooleanQuery();
query.add(spanQuery, BooleanClause.Occur.MUST);
query.add(spanQuery, BooleanClause.Occur.MUST);
final Hits hits = executeQuery(query);
final String[] expectedIds = new String[] { "1", "2", "3", "4" };
final float[] expectedScores = new float[] { expectedScore, expectedScore, expectedScore, expectedScore };
assertHits(hits, "two span queries", expectedIds, expectedScores);
assertHits(s, query, "two span queries", expectedIds, expectedScores);
}
/**
* Executes the query and throws an assertion if the results don't match the
* expectedHits.
*
* @param query the query to execute
* @throws IOException
*/
protected Hits executeQuery(final Query query) throws IOException {
final IndexSearcher searcher = new IndexSearcher(mDirectory);
final Hits hits = searcher.search(query);
searcher.close();
return hits;
}
/**
* Checks to see if the hits are what we expected.
*
* @param hits the search results
* @param query the query to execute
* @param description the description of the search
* @param expectedIds the expected document ids of the hits
* @param expectedScores the expected scores of the hits
*
* @throws IOException
*/
protected void assertHits(final Hits hits, final String description, final String[] expectedIds,
protected static void assertHits(Searcher s, Query query, final String description, final String[] expectedIds,
final float[] expectedScores) throws IOException {
final float tolerance = 1e-5f;
// Hits hits = searcher.search(query);
// hits normalizes and throws things off if one score is greater than 1.0
TopDocs topdocs = s.search(query,null,10000);
/*****
// display the hits
/*System.out.println(hits.length() + " hits for search: \"" + description + '\"');
System.out.println(hits.length() + " hits for search: \"" + description + '\"');
for (int i = 0; i < hits.length(); i++) {
System.out.println(" " + FIELD_ID + ':' + hits.doc(i).get(FIELD_ID) + " (score:" + hits.score(i) + ')');
}*/
}
*****/
// did we get the hits we expected
assertEquals(expectedIds.length, hits.length());
for (int i = 0; i < hits.length(); i++) {
assertTrue(expectedIds[i].equals(hits.doc(i).get(FIELD_ID)));
assertEquals(expectedScores[i], hits.score(i), 0);
assertEquals(expectedIds.length, topdocs.totalHits);
for (int i = 0; i < topdocs.totalHits; i++) {
//System.out.println(i + " exp: " + expectedIds[i]);
//System.out.println(i + " field: " + hits.doc(i).get(FIELD_ID));
int id = topdocs.scoreDocs[i].doc;
float score = topdocs.scoreDocs[i].score;
Document doc = s.doc(id);
assertEquals(expectedIds[i], doc.get(FIELD_ID));
boolean scoreEq = Math.abs(expectedScores[i] - score) < tolerance;
if (!scoreEq) {
System.out.println(i + " warning, expected score: " + expectedScores[i] + ", actual " + score);
System.out.println(s.explain(query,id));
}
assertEquals(expectedScores[i], score, tolerance);
assertEquals(s.explain(query,id).getValue(), score, tolerance);
}
}
}

24
src/test/org/apache/lucene/search/spans/TestSpansAdvanced2.java
View File

@ -22,10 +22,7 @@ import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.Hits;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.*;
/*******************************************************************************
* Some expanded tests to make sure my patch doesn't break other SpanTermQuery
@ -34,7 +31,7 @@ import org.apache.lucene.search.Query;
* @author Reece Wilton
*/
public class TestSpansAdvanced2 extends TestSpansAdvanced {
IndexSearcher searcher2;
/**
* Initializes the tests by adding documents to the index.
*/
@ -48,6 +45,9 @@ public class TestSpansAdvanced2 extends TestSpansAdvanced {
addDocument(writer, "C", "It shouldn't.");
addDocument(writer, "D", "Should we, should we, should we.");
writer.close();
// re-open the searcher since we added more docs
searcher2 = new IndexSearcher(mDirectory);
}
/**
@ -69,11 +69,10 @@ public class TestSpansAdvanced2 extends TestSpansAdvanced {
public void testSingleSpanQuery() throws IOException {
final Query spanQuery = new SpanTermQuery(new Term(FIELD_TEXT, "should"));
final Hits hits = executeQuery(spanQuery);
final String[] expectedIds = new String[] { "B", "D", "1", "2", "3", "4", "A" };
final float[] expectedScores = new float[] { 0.625f, 0.45927936f, 0.35355338f, 0.35355338f, 0.35355338f,
0.35355338f, 0.26516503f, };
assertHits(hits, "single span query", expectedIds, expectedScores);
assertHits(searcher2, spanQuery, "single span query", expectedIds, expectedScores);
}
/**
@ -88,10 +87,11 @@ public class TestSpansAdvanced2 extends TestSpansAdvanced {
final BooleanQuery query = new BooleanQuery();
query.add(spanQuery1, BooleanClause.Occur.MUST);
query.add(spanQuery2, BooleanClause.Occur.MUST);
final Hits hits = executeQuery(query);
final String[] expectedIds = new String[] { "A", "D" };
final float[] expectedScores = new float[] { 0.93163157f, 0.20698164f };
assertHits(hits, "multiple different span queries", expectedIds, expectedScores);
final String[] expectedIds = new String[] { "D", "A" };
// these values were pre LUCENE-413
// final float[] expectedScores = new float[] { 0.93163157f, 0.20698164f };
final float[] expectedScores = new float[] { 1.0191123f, 0.93163157f };
assertHits(searcher2, query, "multiple different span queries", expectedIds, expectedScores);
}
/**
@ -101,6 +101,6 @@ public class TestSpansAdvanced2 extends TestSpansAdvanced {
*/
public void testBooleanQueryWithSpanQueries() throws IOException {
doTestBooleanQueryWithSpanQueries(0.73500174f);
doTestBooleanQueryWithSpanQueries(searcher2, 0.73500174f);
}
}