mirror of https://github.com/apache/lucene.git
LUCENE-7132: BooleanQuery sometimes assigned the wrong score when ranges of documents had only one clause matching while other ranges had more than one clause matchng
This commit is contained in:
Mike McCandless
parent
b64c558e3e
commit
c8570ed821
|
|
@ -148,6 +148,11 @@ Bug Fixes
|
|||
* LUCENE-7312: Fix geo3d's x/y/z double to int encoding to ensure it always
|
||||
rounds down (Karl Wright, Mike McCandless)
|
||||
|
||||
* LUCENE-7132: BooleanQuery sometimes assigned too-low scores in cases
|
||||
where ranges of documents had only a single clause matching while
|
||||
other ranges had more than one clause matching (Ahmet Arslan,
|
||||
hossman, Mike McCandless)
|
||||
|
||||
Documentation
|
||||
|
||||
* LUCENE-7223: Improve XXXPoint javadocs to make it clear that you
|
||||
|
|
|
|||
|
|
@ -275,13 +275,13 @@ final class BooleanScorer extends BulkScorer {
|
|||
}
|
||||
}
|
||||
|
||||
private void scoreWindowSingleScorer(BulkScorerAndDoc bulkScorer, LeafCollector collector,
|
||||
private void scoreWindowSingleScorer(BulkScorerAndDoc bulkScorer, LeafCollector collector, LeafCollector singleClauseCollector,
|
||||
Bits acceptDocs, int windowMin, int windowMax, int max) throws IOException {
|
||||
assert tail.size() == 0;
|
||||
final int nextWindowBase = head.top().next & ~MASK;
|
||||
final int end = Math.max(windowMax, Math.min(max, nextWindowBase));
|
||||
|
||||
bulkScorer.score(collector, acceptDocs, windowMin, end);
|
||||
bulkScorer.score(singleClauseCollector, acceptDocs, windowMin, end);
|
||||
|
||||
// reset the scorer that should be used for the general case
|
||||
collector.setScorer(fakeScorer);
|
||||
|
|
@ -304,7 +304,7 @@ final class BooleanScorer extends BulkScorer {
|
|||
// special case: only one scorer can match in the current window,
|
||||
// we can collect directly
|
||||
final BulkScorerAndDoc bulkScorer = leads[0];
|
||||
scoreWindowSingleScorer(bulkScorer, singleClauseCollector, acceptDocs, windowMin, windowMax, max);
|
||||
scoreWindowSingleScorer(bulkScorer, collector, singleClauseCollector, acceptDocs, windowMin, windowMax, max);
|
||||
return head.add(bulkScorer);
|
||||
} else {
|
||||
// general case, collect through a bit set first and then replay
|
||||
|
|
|
|||
|
|
@ -45,7 +45,12 @@ public abstract class FilterLeafCollector implements LeafCollector {
|
|||
|
||||
@Override
|
||||
public String toString() {
|
||||
return getClass().getSimpleName() + "(" + in + ")";
|
||||
String name = getClass().getSimpleName();
|
||||
if (name.length() == 0) {
|
||||
// an anonoymous subclass will have empty name?
|
||||
name = "FilterLeafCollector";
|
||||
}
|
||||
return name + "(" + in + ")";
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,15 +18,18 @@ package org.apache.lucene.search;
|
|||
|
||||
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.Random;
|
||||
|
||||
import org.apache.lucene.analysis.MockAnalyzer;
|
||||
import org.apache.lucene.document.Document;
|
||||
import org.apache.lucene.document.Field;
|
||||
import org.apache.lucene.index.DirectoryReader;
|
||||
import org.apache.lucene.index.IndexReader;
|
||||
import org.apache.lucene.index.IndexWriter;
|
||||
import org.apache.lucene.index.IndexWriterConfig;
|
||||
import org.apache.lucene.index.RandomIndexWriter;
|
||||
import org.apache.lucene.index.Term;
|
||||
import org.apache.lucene.index.IndexReader;
|
||||
import org.apache.lucene.search.similarities.ClassicSimilarity;
|
||||
import org.apache.lucene.search.similarities.Similarity;
|
||||
import org.apache.lucene.store.Directory;
|
||||
|
|
@ -42,24 +45,45 @@ import org.junit.Test;
|
|||
*/
|
||||
public class TestBoolean2 extends LuceneTestCase {
|
||||
private static IndexSearcher searcher;
|
||||
private static IndexSearcher singleSegmentSearcher;
|
||||
private static IndexSearcher bigSearcher;
|
||||
private static IndexReader reader;
|
||||
private static IndexReader littleReader;
|
||||
private static int NUM_EXTRA_DOCS = 6000;
|
||||
private static IndexReader singleSegmentReader;
|
||||
/** num of empty docs injected between every doc in the (main) index */
|
||||
private static int NUM_FILLER_DOCS;
|
||||
/** num of empty docs injected prior to the first doc in the (main) index */
|
||||
private static int PRE_FILLER_DOCS;
|
||||
/** num "extra" docs containing value in "field2" added to the "big" clone of the index */
|
||||
private static final int NUM_EXTRA_DOCS = 6000;
|
||||
|
||||
public static final String field = "field";
|
||||
private static Directory directory;
|
||||
private static Directory singleSegmentDirectory;
|
||||
private static Directory dir2;
|
||||
private static int mulFactor;
|
||||
|
||||
@BeforeClass
|
||||
public static void beforeClass() throws Exception {
|
||||
// in some runs, test immediate adjacency of matches - in others, force a full bucket gap betwen docs
|
||||
NUM_FILLER_DOCS = random().nextBoolean() ? 0 : BooleanScorer.SIZE;
|
||||
PRE_FILLER_DOCS = TestUtil.nextInt(random(), 0, (NUM_FILLER_DOCS / 2));
|
||||
|
||||
directory = newDirectory();
|
||||
RandomIndexWriter writer= new RandomIndexWriter(random(), directory, newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy()));
|
||||
for (int i = 0; i < docFields.length; i++) {
|
||||
|
||||
Document doc = new Document();
|
||||
for (int filler = 0; filler < PRE_FILLER_DOCS; filler++) {
|
||||
writer.addDocument(doc);
|
||||
}
|
||||
for (int i = 0; i < docFields.length; i++) {
|
||||
doc.add(newTextField(field, docFields[i], Field.Store.NO));
|
||||
writer.addDocument(doc);
|
||||
|
||||
doc = new Document();
|
||||
for (int filler = 0; filler < NUM_FILLER_DOCS; filler++) {
|
||||
writer.addDocument(doc);
|
||||
}
|
||||
}
|
||||
writer.close();
|
||||
littleReader = DirectoryReader.open(directory);
|
||||
|
|
@ -67,6 +91,18 @@ public class TestBoolean2 extends LuceneTestCase {
|
|||
// this is intentionally using the baseline sim, because it compares against bigSearcher (which uses a random one)
|
||||
searcher.setSimilarity(new ClassicSimilarity());
|
||||
|
||||
// make a copy of our index using a single segment
|
||||
singleSegmentDirectory = new MockDirectoryWrapper(random(), TestUtil.ramCopyOf(directory));
|
||||
IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
|
||||
// we need docID order to be preserved:
|
||||
iwc.setMergePolicy(newLogMergePolicy());
|
||||
try (IndexWriter w = new IndexWriter(singleSegmentDirectory, iwc)) {
|
||||
w.forceMerge(1, true);
|
||||
}
|
||||
singleSegmentReader = DirectoryReader.open(singleSegmentDirectory);
|
||||
singleSegmentSearcher = newSearcher(singleSegmentReader);
|
||||
singleSegmentSearcher.setSimilarity(searcher.getSimilarity(true));
|
||||
|
||||
// Make big index
|
||||
dir2 = new MockDirectoryWrapper(random(), TestUtil.ramCopyOf(directory));
|
||||
|
||||
|
|
@ -86,12 +122,12 @@ public class TestBoolean2 extends LuceneTestCase {
|
|||
docCount = w.maxDoc();
|
||||
w.close();
|
||||
mulFactor *= 2;
|
||||
} while(docCount < 3000);
|
||||
} while(docCount < 3000 * NUM_FILLER_DOCS);
|
||||
|
||||
RandomIndexWriter w = new RandomIndexWriter(random(), dir2,
|
||||
newIndexWriterConfig(new MockAnalyzer(random()))
|
||||
.setMaxBufferedDocs(TestUtil.nextInt(random(), 50, 1000)));
|
||||
Document doc = new Document();
|
||||
doc = new Document();
|
||||
doc.add(newTextField("field2", "xxx", Field.Store.NO));
|
||||
for(int i=0;i<NUM_EXTRA_DOCS/2;i++) {
|
||||
w.addDocument(doc);
|
||||
|
|
@ -110,8 +146,13 @@ public class TestBoolean2 extends LuceneTestCase {
|
|||
public static void afterClass() throws Exception {
|
||||
reader.close();
|
||||
littleReader.close();
|
||||
singleSegmentReader.close();
|
||||
dir2.close();
|
||||
directory.close();
|
||||
singleSegmentDirectory.close();
|
||||
singleSegmentSearcher = null;
|
||||
singleSegmentReader = null;
|
||||
singleSegmentDirectory = null;
|
||||
searcher = null;
|
||||
reader = null;
|
||||
littleReader = null;
|
||||
|
|
@ -128,26 +169,57 @@ public class TestBoolean2 extends LuceneTestCase {
|
|||
};
|
||||
|
||||
public void queriesTest(Query query, int[] expDocNrs) throws Exception {
|
||||
|
||||
// adjust the expected doc numbers according to our filler docs
|
||||
if (0 < NUM_FILLER_DOCS) {
|
||||
expDocNrs = Arrays.copyOf(expDocNrs, expDocNrs.length);
|
||||
for (int i=0; i < expDocNrs.length; i++) {
|
||||
expDocNrs[i] = PRE_FILLER_DOCS + ((NUM_FILLER_DOCS + 1) * expDocNrs[i]);
|
||||
}
|
||||
}
|
||||
|
||||
final int topDocsToCheck = atLeast(1000);
|
||||
// The asserting searcher will sometimes return the bulk scorer and
|
||||
// sometimes return a default impl around the scorer so that we can
|
||||
// compare BS1 and BS2
|
||||
TopScoreDocCollector collector = TopScoreDocCollector.create(1000);
|
||||
TopScoreDocCollector collector = TopScoreDocCollector.create(topDocsToCheck);
|
||||
searcher.search(query, collector);
|
||||
ScoreDoc[] hits1 = collector.topDocs().scoreDocs;
|
||||
|
||||
collector = TopScoreDocCollector.create(1000);
|
||||
collector = TopScoreDocCollector.create(topDocsToCheck);
|
||||
searcher.search(query, collector);
|
||||
ScoreDoc[] hits2 = collector.topDocs().scoreDocs;
|
||||
|
||||
CheckHits.checkHitsQuery(query, hits1, hits2, expDocNrs);
|
||||
|
||||
// Since we have no deleted docs, we should also be able to verify identical matches &
|
||||
// scores against an single segment copy of our index
|
||||
collector = TopScoreDocCollector.create(topDocsToCheck);
|
||||
singleSegmentSearcher.search(query, collector);
|
||||
hits2 = collector.topDocs().scoreDocs;
|
||||
CheckHits.checkHitsQuery(query, hits1, hits2, expDocNrs);
|
||||
|
||||
// sanity check expected num matches in bigSearcher
|
||||
assertEquals(mulFactor * collector.totalHits,
|
||||
bigSearcher.search(query, 1).totalHits);
|
||||
|
||||
CheckHits.checkHitsQuery(query, hits1, hits2, expDocNrs);
|
||||
// now check 2 diff scorers from the bigSearcher as well
|
||||
collector = TopScoreDocCollector.create(topDocsToCheck);
|
||||
bigSearcher.search(query, collector);
|
||||
hits1 = collector.topDocs().scoreDocs;
|
||||
collector = TopScoreDocCollector.create(topDocsToCheck);
|
||||
bigSearcher.search(query, collector);
|
||||
hits2 = collector.topDocs().scoreDocs;
|
||||
|
||||
// NOTE: just comparing results, not vetting against expDocNrs
|
||||
// since we have dups in bigSearcher
|
||||
CheckHits.checkEqual(query, hits1, hits2);
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testQueries01() throws Exception {
|
||||
BooleanQuery.Builder query = new BooleanQuery.Builder();
|
||||
query.setDisableCoord(random().nextBoolean());
|
||||
query.add(new TermQuery(new Term(field, "w3")), BooleanClause.Occur.MUST);
|
||||
query.add(new TermQuery(new Term(field, "xx")), BooleanClause.Occur.MUST);
|
||||
int[] expDocNrs = {2,3};
|
||||
|
|
@ -157,6 +229,7 @@ public class TestBoolean2 extends LuceneTestCase {
|
|||
@Test
|
||||
public void testQueries02() throws Exception {
|
||||
BooleanQuery.Builder query = new BooleanQuery.Builder();
|
||||
query.setDisableCoord(random().nextBoolean());
|
||||
query.add(new TermQuery(new Term(field, "w3")), BooleanClause.Occur.MUST);
|
||||
query.add(new TermQuery(new Term(field, "xx")), BooleanClause.Occur.SHOULD);
|
||||
int[] expDocNrs = {2,3,1,0};
|
||||
|
|
@ -166,6 +239,7 @@ public class TestBoolean2 extends LuceneTestCase {
|
|||
@Test
|
||||
public void testQueries03() throws Exception {
|
||||
BooleanQuery.Builder query = new BooleanQuery.Builder();
|
||||
query.setDisableCoord(random().nextBoolean());
|
||||
query.add(new TermQuery(new Term(field, "w3")), BooleanClause.Occur.SHOULD);
|
||||
query.add(new TermQuery(new Term(field, "xx")), BooleanClause.Occur.SHOULD);
|
||||
int[] expDocNrs = {2,3,1,0};
|
||||
|
|
@ -175,6 +249,7 @@ public class TestBoolean2 extends LuceneTestCase {
|
|||
@Test
|
||||
public void testQueries04() throws Exception {
|
||||
BooleanQuery.Builder query = new BooleanQuery.Builder();
|
||||
query.setDisableCoord(random().nextBoolean());
|
||||
query.add(new TermQuery(new Term(field, "w3")), BooleanClause.Occur.SHOULD);
|
||||
query.add(new TermQuery(new Term(field, "xx")), BooleanClause.Occur.MUST_NOT);
|
||||
int[] expDocNrs = {1,0};
|
||||
|
|
@ -184,6 +259,7 @@ public class TestBoolean2 extends LuceneTestCase {
|
|||
@Test
|
||||
public void testQueries05() throws Exception {
|
||||
BooleanQuery.Builder query = new BooleanQuery.Builder();
|
||||
query.setDisableCoord(random().nextBoolean());
|
||||
query.add(new TermQuery(new Term(field, "w3")), BooleanClause.Occur.MUST);
|
||||
query.add(new TermQuery(new Term(field, "xx")), BooleanClause.Occur.MUST_NOT);
|
||||
int[] expDocNrs = {1,0};
|
||||
|
|
@ -193,6 +269,7 @@ public class TestBoolean2 extends LuceneTestCase {
|
|||
@Test
|
||||
public void testQueries06() throws Exception {
|
||||
BooleanQuery.Builder query = new BooleanQuery.Builder();
|
||||
query.setDisableCoord(random().nextBoolean());
|
||||
query.add(new TermQuery(new Term(field, "w3")), BooleanClause.Occur.MUST);
|
||||
query.add(new TermQuery(new Term(field, "xx")), BooleanClause.Occur.MUST_NOT);
|
||||
query.add(new TermQuery(new Term(field, "w5")), BooleanClause.Occur.MUST_NOT);
|
||||
|
|
@ -203,6 +280,7 @@ public class TestBoolean2 extends LuceneTestCase {
|
|||
@Test
|
||||
public void testQueries07() throws Exception {
|
||||
BooleanQuery.Builder query = new BooleanQuery.Builder();
|
||||
query.setDisableCoord(random().nextBoolean());
|
||||
query.add(new TermQuery(new Term(field, "w3")), BooleanClause.Occur.MUST_NOT);
|
||||
query.add(new TermQuery(new Term(field, "xx")), BooleanClause.Occur.MUST_NOT);
|
||||
query.add(new TermQuery(new Term(field, "w5")), BooleanClause.Occur.MUST_NOT);
|
||||
|
|
@ -213,6 +291,7 @@ public class TestBoolean2 extends LuceneTestCase {
|
|||
@Test
|
||||
public void testQueries08() throws Exception {
|
||||
BooleanQuery.Builder query = new BooleanQuery.Builder();
|
||||
query.setDisableCoord(random().nextBoolean());
|
||||
query.add(new TermQuery(new Term(field, "w3")), BooleanClause.Occur.MUST);
|
||||
query.add(new TermQuery(new Term(field, "xx")), BooleanClause.Occur.SHOULD);
|
||||
query.add(new TermQuery(new Term(field, "w5")), BooleanClause.Occur.MUST_NOT);
|
||||
|
|
@ -223,6 +302,7 @@ public class TestBoolean2 extends LuceneTestCase {
|
|||
@Test
|
||||
public void testQueries09() throws Exception {
|
||||
BooleanQuery.Builder query = new BooleanQuery.Builder();
|
||||
query.setDisableCoord(random().nextBoolean());
|
||||
query.add(new TermQuery(new Term(field, "w3")), BooleanClause.Occur.MUST);
|
||||
query.add(new TermQuery(new Term(field, "xx")), BooleanClause.Occur.MUST);
|
||||
query.add(new TermQuery(new Term(field, "w2")), BooleanClause.Occur.MUST);
|
||||
|
|
@ -234,6 +314,7 @@ public class TestBoolean2 extends LuceneTestCase {
|
|||
@Test
|
||||
public void testQueries10() throws Exception {
|
||||
BooleanQuery.Builder query = new BooleanQuery.Builder();
|
||||
query.setDisableCoord(random().nextBoolean());
|
||||
query.add(new TermQuery(new Term(field, "w3")), BooleanClause.Occur.MUST);
|
||||
query.add(new TermQuery(new Term(field, "xx")), BooleanClause.Occur.MUST);
|
||||
query.add(new TermQuery(new Term(field, "w2")), BooleanClause.Occur.MUST);
|
||||
|
|
@ -241,16 +322,19 @@ public class TestBoolean2 extends LuceneTestCase {
|
|||
|
||||
int[] expDocNrs = {2, 3};
|
||||
Similarity oldSimilarity = searcher.getSimilarity(true);
|
||||
try {
|
||||
searcher.setSimilarity(new ClassicSimilarity(){
|
||||
Similarity newSimilarity = new ClassicSimilarity() {
|
||||
@Override
|
||||
public float coord(int overlap, int maxOverlap) {
|
||||
return overlap / ((float)maxOverlap - 1);
|
||||
}
|
||||
});
|
||||
};
|
||||
try {
|
||||
searcher.setSimilarity(newSimilarity);
|
||||
singleSegmentSearcher.setSimilarity(newSimilarity);
|
||||
queriesTest(query.build(), expDocNrs);
|
||||
} finally {
|
||||
searcher.setSimilarity(oldSimilarity);
|
||||
singleSegmentSearcher.setSimilarity(oldSimilarity);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -282,15 +366,11 @@ public class TestBoolean2 extends LuceneTestCase {
|
|||
searcher.setSimilarity(new ClassicSimilarity()); // restore
|
||||
}
|
||||
|
||||
TopFieldCollector collector = TopFieldCollector.create(sort, 1000,
|
||||
false, true, true);
|
||||
|
||||
// check diff (randomized) scorers (from AssertingSearcher) produce the same results
|
||||
TopFieldCollector collector = TopFieldCollector.create(sort, 1000, false, true, true);
|
||||
searcher.search(q1, collector);
|
||||
ScoreDoc[] hits1 = collector.topDocs().scoreDocs;
|
||||
|
||||
collector = TopFieldCollector.create(sort, 1000,
|
||||
false, true, true);
|
||||
|
||||
collector = TopFieldCollector.create(sort, 1000, false, true, true);
|
||||
searcher.search(q1, collector);
|
||||
ScoreDoc[] hits2 = collector.topDocs().scoreDocs;
|
||||
tot+=hits2.length;
|
||||
|
|
@ -301,6 +381,16 @@ public class TestBoolean2 extends LuceneTestCase {
|
|||
q3.add(new PrefixQuery(new Term("field2", "b")), BooleanClause.Occur.SHOULD);
|
||||
TopDocs hits4 = bigSearcher.search(q3.build(), 1);
|
||||
assertEquals(mulFactor*collector.totalHits + NUM_EXTRA_DOCS/2, hits4.totalHits);
|
||||
|
||||
// test diff (randomized) scorers produce the same results on bigSearcher as well
|
||||
collector = TopFieldCollector.create(sort, 1000 * mulFactor, false, true, true);
|
||||
bigSearcher.search(q1, collector);
|
||||
hits1 = collector.topDocs().scoreDocs;
|
||||
collector = TopFieldCollector.create(sort, 1000 * mulFactor, false, true, true);
|
||||
bigSearcher.search(q1, collector);
|
||||
hits2 = collector.topDocs().scoreDocs;
|
||||
CheckHits.checkEqual(q1, hits1, hits2);
|
||||
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
|
|
|
|||
|
|
@ -296,6 +296,7 @@ public class TestSimpleExplanations extends BaseExplanationTestCase {
|
|||
outerQuery.add(new TermQuery(new Term(FIELD, "w1")), BooleanClause.Occur.MUST);
|
||||
|
||||
BooleanQuery.Builder innerQuery = new BooleanQuery.Builder();
|
||||
innerQuery.setDisableCoord(random().nextBoolean());
|
||||
innerQuery.add(new TermQuery(new Term(FIELD, "qq")), BooleanClause.Occur.SHOULD);
|
||||
|
||||
BooleanQuery.Builder childLeft = new BooleanQuery.Builder();
|
||||
|
|
@ -317,6 +318,7 @@ public class TestSimpleExplanations extends BaseExplanationTestCase {
|
|||
outerQuery.add(new TermQuery(new Term(FIELD, "w1")), BooleanClause.Occur.MUST);
|
||||
|
||||
BooleanQuery.Builder innerQuery = new BooleanQuery.Builder();
|
||||
innerQuery.setDisableCoord(random().nextBoolean());
|
||||
innerQuery.add(new TermQuery(new Term(FIELD, "qq")), BooleanClause.Occur.SHOULD);
|
||||
|
||||
BooleanQuery.Builder childLeft = new BooleanQuery.Builder();
|
||||
|
|
@ -338,6 +340,7 @@ public class TestSimpleExplanations extends BaseExplanationTestCase {
|
|||
outerQuery.add(new TermQuery(new Term(FIELD, "w1")), BooleanClause.Occur.MUST);
|
||||
|
||||
BooleanQuery.Builder innerQuery = new BooleanQuery.Builder();
|
||||
innerQuery.setDisableCoord(random().nextBoolean());
|
||||
innerQuery.add(new TermQuery(new Term(FIELD, "qq")), BooleanClause.Occur.SHOULD);
|
||||
|
||||
BooleanQuery.Builder childLeft = new BooleanQuery.Builder();
|
||||
|
|
@ -359,6 +362,7 @@ public class TestSimpleExplanations extends BaseExplanationTestCase {
|
|||
outerQuery.add(new TermQuery(new Term(FIELD, "w1")), BooleanClause.Occur.MUST);
|
||||
|
||||
BooleanQuery.Builder innerQuery = new BooleanQuery.Builder();
|
||||
innerQuery.setDisableCoord(random().nextBoolean());
|
||||
innerQuery.add(new TermQuery(new Term(FIELD, "qq")), BooleanClause.Occur.SHOULD);
|
||||
|
||||
BooleanQuery.Builder childLeft = new BooleanQuery.Builder();
|
||||
|
|
@ -377,6 +381,7 @@ public class TestSimpleExplanations extends BaseExplanationTestCase {
|
|||
}
|
||||
public void testBQ11() throws Exception {
|
||||
BooleanQuery.Builder query = new BooleanQuery.Builder();
|
||||
query.setDisableCoord(random().nextBoolean());
|
||||
query.add(new TermQuery(new Term(FIELD, "w1")), BooleanClause.Occur.SHOULD);
|
||||
TermQuery boostedQuery = new TermQuery(new Term(FIELD, "w1"));
|
||||
query.add(new BoostQuery(boostedQuery, 1000), BooleanClause.Occur.SHOULD);
|
||||
|
|
@ -385,21 +390,21 @@ public class TestSimpleExplanations extends BaseExplanationTestCase {
|
|||
}
|
||||
public void testBQ14() throws Exception {
|
||||
BooleanQuery.Builder q = new BooleanQuery.Builder();
|
||||
q.setDisableCoord(true);
|
||||
q.setDisableCoord(random().nextBoolean());
|
||||
q.add(new TermQuery(new Term(FIELD, "QQQQQ")), BooleanClause.Occur.SHOULD);
|
||||
q.add(new TermQuery(new Term(FIELD, "w1")), BooleanClause.Occur.SHOULD);
|
||||
qtest(q.build(), new int[] { 0,1,2,3 });
|
||||
}
|
||||
public void testBQ15() throws Exception {
|
||||
BooleanQuery.Builder q = new BooleanQuery.Builder();
|
||||
q.setDisableCoord(true);
|
||||
q.setDisableCoord(random().nextBoolean());
|
||||
q.add(new TermQuery(new Term(FIELD, "QQQQQ")), BooleanClause.Occur.MUST_NOT);
|
||||
q.add(new TermQuery(new Term(FIELD, "w1")), BooleanClause.Occur.SHOULD);
|
||||
qtest(q.build(), new int[] { 0,1,2,3 });
|
||||
}
|
||||
public void testBQ16() throws Exception {
|
||||
BooleanQuery.Builder q = new BooleanQuery.Builder();
|
||||
q.setDisableCoord(true);
|
||||
q.setDisableCoord(random().nextBoolean());
|
||||
q.add(new TermQuery(new Term(FIELD, "QQQQQ")), BooleanClause.Occur.SHOULD);
|
||||
|
||||
BooleanQuery.Builder booleanQuery = new BooleanQuery.Builder();
|
||||
|
|
@ -411,7 +416,7 @@ public class TestSimpleExplanations extends BaseExplanationTestCase {
|
|||
}
|
||||
public void testBQ17() throws Exception {
|
||||
BooleanQuery.Builder q = new BooleanQuery.Builder();
|
||||
q.setDisableCoord(true);
|
||||
q.setDisableCoord(random().nextBoolean());
|
||||
q.add(new TermQuery(new Term(FIELD, "w2")), BooleanClause.Occur.SHOULD);
|
||||
|
||||
BooleanQuery.Builder booleanQuery = new BooleanQuery.Builder();
|
||||
|
|
@ -431,6 +436,7 @@ public class TestSimpleExplanations extends BaseExplanationTestCase {
|
|||
|
||||
public void testBQ20() throws Exception {
|
||||
BooleanQuery.Builder q = new BooleanQuery.Builder();
|
||||
q.setDisableCoord(random().nextBoolean());
|
||||
q.setMinimumNumberShouldMatch(2);
|
||||
q.add(new TermQuery(new Term(FIELD, "QQQQQ")), BooleanClause.Occur.SHOULD);
|
||||
q.add(new TermQuery(new Term(FIELD, "yy")), BooleanClause.Occur.SHOULD);
|
||||
|
|
@ -442,6 +448,16 @@ public class TestSimpleExplanations extends BaseExplanationTestCase {
|
|||
|
||||
}
|
||||
|
||||
public void testBQ21() throws Exception {
|
||||
BooleanQuery.Builder q = new BooleanQuery.Builder();
|
||||
q.setDisableCoord(random().nextBoolean());
|
||||
q.add(new TermQuery(new Term(FIELD, "yy")), BooleanClause.Occur.SHOULD);
|
||||
q.add(new TermQuery(new Term(FIELD, "zz")), BooleanClause.Occur.SHOULD);
|
||||
|
||||
qtest(q.build(), new int[] { 1,2,3 });
|
||||
|
||||
}
|
||||
|
||||
public void testBQ23() throws Exception {
|
||||
BooleanQuery.Builder query = new BooleanQuery.Builder();
|
||||
query.add(new TermQuery(new Term(FIELD, "w1")), BooleanClause.Occur.FILTER);
|
||||
|
|
@ -488,6 +504,7 @@ public class TestSimpleExplanations extends BaseExplanationTestCase {
|
|||
}
|
||||
public void testMultiFieldBQ3() throws Exception {
|
||||
BooleanQuery.Builder query = new BooleanQuery.Builder();
|
||||
query.setDisableCoord(random().nextBoolean());
|
||||
query.add(new TermQuery(new Term(FIELD, "yy")), BooleanClause.Occur.SHOULD);
|
||||
query.add(new TermQuery(new Term(ALTFIELD, "w3")), BooleanClause.Occur.MUST);
|
||||
|
||||
|
|
@ -495,6 +512,7 @@ public class TestSimpleExplanations extends BaseExplanationTestCase {
|
|||
}
|
||||
public void testMultiFieldBQ4() throws Exception {
|
||||
BooleanQuery.Builder outerQuery = new BooleanQuery.Builder();
|
||||
outerQuery.setDisableCoord(random().nextBoolean());
|
||||
outerQuery.add(new TermQuery(new Term(FIELD, "w1")), BooleanClause.Occur.SHOULD);
|
||||
|
||||
BooleanQuery.Builder innerQuery = new BooleanQuery.Builder();
|
||||
|
|
@ -506,6 +524,7 @@ public class TestSimpleExplanations extends BaseExplanationTestCase {
|
|||
}
|
||||
public void testMultiFieldBQ5() throws Exception {
|
||||
BooleanQuery.Builder outerQuery = new BooleanQuery.Builder();
|
||||
outerQuery.setDisableCoord(random().nextBoolean());
|
||||
outerQuery.add(new TermQuery(new Term(FIELD, "w1")), BooleanClause.Occur.SHOULD);
|
||||
|
||||
BooleanQuery.Builder innerQuery = new BooleanQuery.Builder();
|
||||
|
|
|
|||
|
|
@ -0,0 +1,126 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.lucene.search;
|
||||
|
||||
import java.util.Arrays;
|
||||
|
||||
import org.apache.lucene.document.Document;
|
||||
import org.apache.lucene.document.Field;
|
||||
import org.apache.lucene.index.Term;
|
||||
import org.apache.lucene.index.RandomIndexWriter;
|
||||
import org.apache.lucene.util.TestUtil;
|
||||
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Assume;
|
||||
|
||||
|
||||
/**
|
||||
* subclass of TestSimpleExplanations that adds a lot of filler docs which will be ignored at query time.
|
||||
* These filler docs will either all be empty in which case the queries will be unmodified, or they will
|
||||
* all use terms from same set of source data as our regular docs (to emphasis the DocFreq factor in scoring),
|
||||
* in which case the queries will be wrapped so they can be excluded.
|
||||
*/
|
||||
public class TestSimpleExplanationsWithFillerDocs extends TestSimpleExplanations {
|
||||
|
||||
/** num of empty docs injected between every doc in the index */
|
||||
private static final int NUM_FILLER_DOCS = BooleanScorer.SIZE;
|
||||
/** num of empty docs injected prior to the first doc in the (main) index */
|
||||
private static int PRE_FILLER_DOCS;
|
||||
/**
|
||||
* If non-null then the filler docs are not empty, and need to be filtered out from queries
|
||||
* using this as both field name & field value
|
||||
*/
|
||||
public static String EXTRA = null;
|
||||
|
||||
private static final Document EMPTY_DOC = new Document();
|
||||
|
||||
/**
|
||||
* Replaces the index created by our superclass with a new one that includes a lot of docs filler docs.
|
||||
* {@link #qtest} will account for these extra filler docs.
|
||||
* @see #qtest
|
||||
*/
|
||||
@BeforeClass
|
||||
public static void replaceIndex() throws Exception {
|
||||
EXTRA = random().nextBoolean() ? null : "extra";
|
||||
PRE_FILLER_DOCS = TestUtil.nextInt(random(), 0, (NUM_FILLER_DOCS / 2));
|
||||
|
||||
// free up what our super class created that we won't be using
|
||||
reader.close();
|
||||
directory.close();
|
||||
|
||||
directory = newDirectory();
|
||||
try (RandomIndexWriter writer = new RandomIndexWriter(random(), directory, newIndexWriterConfig(analyzer).setMergePolicy(newLogMergePolicy()))) {
|
||||
|
||||
for (int filler = 0; filler < PRE_FILLER_DOCS; filler++) {
|
||||
writer.addDocument(makeFillerDoc());
|
||||
}
|
||||
for (int i = 0; i < docFields.length; i++) {
|
||||
writer.addDocument(createDoc(i));
|
||||
|
||||
for (int filler = 0; filler < NUM_FILLER_DOCS; filler++) {
|
||||
writer.addDocument(makeFillerDoc());
|
||||
}
|
||||
}
|
||||
reader = writer.getReader();
|
||||
searcher = newSearcher(reader);
|
||||
}
|
||||
}
|
||||
|
||||
private static Document makeFillerDoc() {
|
||||
if (null == EXTRA) {
|
||||
return EMPTY_DOC;
|
||||
}
|
||||
Document doc = createDoc(TestUtil.nextInt(random(), 0, docFields.length-1));
|
||||
doc.add(newStringField(EXTRA, EXTRA, Field.Store.NO));
|
||||
return doc;
|
||||
}
|
||||
|
||||
/**
|
||||
* Adjusts <code>expDocNrs</code> based on the filler docs injected in the index,
|
||||
* and if neccessary wraps the <code>q</code> in a BooleanQuery that will filter out all
|
||||
* filler docs using the {@link #EXTRA} field.
|
||||
*
|
||||
* @see #replaceIndex
|
||||
*/
|
||||
@Override
|
||||
public void qtest(Query q, int[] expDocNrs) throws Exception {
|
||||
|
||||
expDocNrs = Arrays.copyOf(expDocNrs, expDocNrs.length);
|
||||
for (int i=0; i < expDocNrs.length; i++) {
|
||||
expDocNrs[i] = PRE_FILLER_DOCS + ((NUM_FILLER_DOCS + 1) * expDocNrs[i]);
|
||||
}
|
||||
|
||||
if (null != EXTRA) {
|
||||
BooleanQuery.Builder builder = new BooleanQuery.Builder();
|
||||
builder.add(new BooleanClause(q, BooleanClause.Occur.MUST));
|
||||
builder.add(new BooleanClause(new TermQuery(new Term(EXTRA, EXTRA)), BooleanClause.Occur.MUST_NOT));
|
||||
q = builder.build();
|
||||
}
|
||||
super.qtest(q, expDocNrs);
|
||||
}
|
||||
|
||||
public void testMA1() throws Exception {
|
||||
Assume.assumeNotNull("test is not viable with empty filler docs", EXTRA);
|
||||
super.testMA1();
|
||||
}
|
||||
public void testMA2() throws Exception {
|
||||
Assume.assumeNotNull("test is not viable with empty filler docs", EXTRA);
|
||||
super.testMA2();
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -71,21 +71,25 @@ public abstract class BaseExplanationTestCase extends LuceneTestCase {
|
|||
public static void beforeClassTestExplanations() throws Exception {
|
||||
directory = newDirectory();
|
||||
analyzer = new MockAnalyzer(random());
|
||||
RandomIndexWriter writer= new RandomIndexWriter(random(), directory, newIndexWriterConfig(analyzer).setMergePolicy(newLogMergePolicy()));
|
||||
try (RandomIndexWriter writer = new RandomIndexWriter(random(), directory, newIndexWriterConfig(analyzer).setMergePolicy(newLogMergePolicy()))) {
|
||||
for (int i = 0; i < docFields.length; i++) {
|
||||
Document doc = new Document();
|
||||
doc.add(newStringField(KEY, ""+i, Field.Store.NO));
|
||||
doc.add(new SortedDocValuesField(KEY, new BytesRef(""+i)));
|
||||
Field f = newTextField(FIELD, docFields[i], Field.Store.NO);
|
||||
f.setBoost(i);
|
||||
doc.add(f);
|
||||
doc.add(newTextField(ALTFIELD, docFields[i], Field.Store.NO));
|
||||
writer.addDocument(doc);
|
||||
writer.addDocument(createDoc(i));
|
||||
}
|
||||
reader = writer.getReader();
|
||||
writer.close();
|
||||
searcher = newSearcher(reader);
|
||||
}
|
||||
}
|
||||
|
||||
public static Document createDoc(int index) {
|
||||
Document doc = new Document();
|
||||
doc.add(newStringField(KEY, ""+index, Field.Store.NO));
|
||||
doc.add(new SortedDocValuesField(KEY, new BytesRef(""+index)));
|
||||
Field f = newTextField(FIELD, docFields[index], Field.Store.NO);
|
||||
f.setBoost(index);
|
||||
doc.add(f);
|
||||
doc.add(newTextField(ALTFIELD, docFields[index], Field.Store.NO));
|
||||
return doc;
|
||||
}
|
||||
|
||||
protected static final String[] docFields = {
|
||||
"w1 w2 w3 w4 w5",
|
||||
|
|
@ -94,8 +98,19 @@ public abstract class BaseExplanationTestCase extends LuceneTestCase {
|
|||
"w1 w3 xx w2 yy w3 zz"
|
||||
};
|
||||
|
||||
/** check the expDocNrs first, then check the query (and the explanations) */
|
||||
/**
|
||||
* check the expDocNrs match and have scores that match the explanations.
|
||||
* Query may be randomly wrapped in a BooleanQuery with a term that matches no documents in
|
||||
* order to trigger coord logic.
|
||||
*/
|
||||
public void qtest(Query q, int[] expDocNrs) throws Exception {
|
||||
if (random().nextBoolean()) {
|
||||
BooleanQuery.Builder bq = new BooleanQuery.Builder();
|
||||
bq.setDisableCoord(random().nextBoolean());
|
||||
bq.add(q, BooleanClause.Occur.SHOULD);
|
||||
bq.add(new TermQuery(new Term("NEVER","MATCH")), BooleanClause.Occur.SHOULD);
|
||||
q = bq.build();
|
||||
}
|
||||
CheckHits.checkHitCollector(random(), q, FIELD, searcher, expDocNrs);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,120 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.lucene.search;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.lucene.index.LeafReaderContext;
|
||||
import org.apache.lucene.index.Term;
|
||||
|
||||
|
||||
import junit.framework.AssertionFailedError;
|
||||
|
||||
/**
|
||||
* Tests that the {@link BaseExplanationTestCase} helper code, as well as
|
||||
* {@link CheckHits#checkNoMatchExplanations} are checking what they are suppose to.
|
||||
*/
|
||||
public class TestBaseExplanationTestCase extends BaseExplanationTestCase {
|
||||
|
||||
public void testQueryNoMatchWhenExpected() throws Exception {
|
||||
expectThrows(AssertionFailedError.class, () -> {
|
||||
qtest(new TermQuery(new Term(FIELD, "BOGUS")), new int[] { 3 /* none */ });
|
||||
});
|
||||
}
|
||||
public void testQueryMatchWhenNotExpected() throws Exception {
|
||||
expectThrows(AssertionFailedError.class, () -> {
|
||||
qtest(new TermQuery(new Term(FIELD, "w1")), new int[] { 0, 1 /*, 2, 3 */ });
|
||||
});
|
||||
}
|
||||
|
||||
public void testIncorrectExplainScores() throws Exception {
|
||||
// sanity check what a real TermQuery matches
|
||||
qtest(new TermQuery(new Term(FIELD, "zz")), new int[] { 1, 3 });
|
||||
|
||||
// ensure when the Explanations are broken, we get an error about those matches
|
||||
expectThrows(AssertionFailedError.class, () -> {
|
||||
qtest(new BrokenExplainTermQuery(new Term(FIELD, "zz"), false, true), new int[] { 1, 3 });
|
||||
|
||||
});
|
||||
}
|
||||
|
||||
public void testIncorrectExplainMatches() throws Exception {
|
||||
// sanity check what a real TermQuery matches
|
||||
qtest(new TermQuery(new Term(FIELD, "zz")), new int[] { 1, 3 });
|
||||
|
||||
// ensure when the Explanations are broken, we get an error about the non matches
|
||||
expectThrows(AssertionFailedError.class, () -> {
|
||||
CheckHits.checkNoMatchExplanations(new BrokenExplainTermQuery(new Term(FIELD, "zz"), true, false),
|
||||
FIELD, searcher, new int[] { 1, 3 });
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
public static final class BrokenExplainTermQuery extends TermQuery {
|
||||
public final boolean toggleExplainMatch;
|
||||
public final boolean breakExplainScores;
|
||||
public BrokenExplainTermQuery(Term t, boolean toggleExplainMatch, boolean breakExplainScores) {
|
||||
super(t);
|
||||
this.toggleExplainMatch = toggleExplainMatch;
|
||||
this.breakExplainScores = breakExplainScores;
|
||||
}
|
||||
public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException {
|
||||
return new BrokenExplainWeight(this, super.createWeight(searcher,needsScores));
|
||||
}
|
||||
}
|
||||
|
||||
public static final class BrokenExplainWeight extends Weight {
|
||||
final Weight in;
|
||||
public BrokenExplainWeight(BrokenExplainTermQuery q, Weight in) {
|
||||
super(q);
|
||||
this.in = in;
|
||||
}
|
||||
public BulkScorer bulkScorer(LeafReaderContext context) throws IOException {
|
||||
return in.bulkScorer(context);
|
||||
}
|
||||
public Explanation explain(LeafReaderContext context, int doc) throws IOException {
|
||||
BrokenExplainTermQuery q = (BrokenExplainTermQuery) this.getQuery();
|
||||
Explanation result = in.explain(context, doc);
|
||||
if (result.isMatch()) {
|
||||
if (q.breakExplainScores) {
|
||||
result = Explanation.match(-1F * result.getValue(), "Broken Explanation Score", result);
|
||||
}
|
||||
if (q.toggleExplainMatch) {
|
||||
result = Explanation.noMatch("Broken Explanation Matching", result);
|
||||
}
|
||||
} else {
|
||||
if (q.toggleExplainMatch) {
|
||||
result = Explanation.match(-42.0F, "Broken Explanation Matching", result);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
public void extractTerms(Set<Term> terms) {
|
||||
in.extractTerms(terms);
|
||||
}
|
||||
public float getValueForNormalization() throws IOException {
|
||||
return in.getValueForNormalization();
|
||||
}
|
||||
public void normalize(float norm, float boost) {
|
||||
in.normalize(norm, boost);
|
||||
}
|
||||
public Scorer scorer(LeafReaderContext context) throws IOException {
|
||||
return in.scorer(context);
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue