mirror of https://github.com/apache/lucene.git
add basic equivalence tests for core query matching (TODO: add spans)
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1292864 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Robert Muir
parent
c618516338
commit
30d2fc63c8
|
|
@ -0,0 +1,172 @@
|
||||||
|
package org.apache.lucene.search;
|
||||||
|
|
||||||
|
import org.apache.lucene.index.Term;
|
||||||
|
import org.apache.lucene.search.BooleanClause.Occur;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Basic equivalence tests for core queries
|
||||||
|
*/
|
||||||
|
public class TestSimpleSearchEquivalence extends SearchEquivalenceTestBase {
|
||||||
|
|
||||||
|
// TODO: we could go a little crazy for a lot of these,
|
||||||
|
// but these are just simple minimal cases in case something
|
||||||
|
// goes horribly wrong. Put more intense tests elsewhere.
|
||||||
|
|
||||||
|
/** A ⊆ (A B) */
|
||||||
|
public void testTermVersusBooleanOr() throws Exception {
|
||||||
|
Term t1 = randomTerm();
|
||||||
|
Term t2 = randomTerm();
|
||||||
|
TermQuery q1 = new TermQuery(t1);
|
||||||
|
BooleanQuery q2 = new BooleanQuery();
|
||||||
|
q2.add(new TermQuery(t1), Occur.SHOULD);
|
||||||
|
q2.add(new TermQuery(t2), Occur.SHOULD);
|
||||||
|
assertSubsetOf(q1, q2);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** A ⊆ (+A B) */
|
||||||
|
public void testTermVersusBooleanReqOpt() throws Exception {
|
||||||
|
Term t1 = randomTerm();
|
||||||
|
Term t2 = randomTerm();
|
||||||
|
TermQuery q1 = new TermQuery(t1);
|
||||||
|
BooleanQuery q2 = new BooleanQuery();
|
||||||
|
q2.add(new TermQuery(t1), Occur.MUST);
|
||||||
|
q2.add(new TermQuery(t2), Occur.SHOULD);
|
||||||
|
assertSubsetOf(q1, q2);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** (A -B) ⊆ A */
|
||||||
|
public void testBooleanReqExclVersusTerm() throws Exception {
|
||||||
|
Term t1 = randomTerm();
|
||||||
|
Term t2 = randomTerm();
|
||||||
|
BooleanQuery q1 = new BooleanQuery();
|
||||||
|
q1.add(new TermQuery(t1), Occur.MUST);
|
||||||
|
q1.add(new TermQuery(t2), Occur.MUST_NOT);
|
||||||
|
TermQuery q2 = new TermQuery(t1);
|
||||||
|
assertSubsetOf(q1, q2);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** (+A +B) ⊆ (A B) */
|
||||||
|
public void testBooleanAndVersusBooleanOr() throws Exception {
|
||||||
|
Term t1 = randomTerm();
|
||||||
|
Term t2 = randomTerm();
|
||||||
|
BooleanQuery q1 = new BooleanQuery();
|
||||||
|
q1.add(new TermQuery(t1), Occur.SHOULD);
|
||||||
|
q1.add(new TermQuery(t2), Occur.SHOULD);
|
||||||
|
BooleanQuery q2 = new BooleanQuery();
|
||||||
|
q2.add(new TermQuery(t1), Occur.SHOULD);
|
||||||
|
q2.add(new TermQuery(t2), Occur.SHOULD);
|
||||||
|
assertSubsetOf(q1, q2);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** (A B) = (A | B) */
|
||||||
|
public void testDisjunctionSumVersusDisjunctionMax() throws Exception {
|
||||||
|
Term t1 = randomTerm();
|
||||||
|
Term t2 = randomTerm();
|
||||||
|
BooleanQuery q1 = new BooleanQuery();
|
||||||
|
q1.add(new TermQuery(t1), Occur.SHOULD);
|
||||||
|
q1.add(new TermQuery(t2), Occur.SHOULD);
|
||||||
|
DisjunctionMaxQuery q2 = new DisjunctionMaxQuery(0.5f);
|
||||||
|
q2.add(new TermQuery(t1));
|
||||||
|
q2.add(new TermQuery(t2));
|
||||||
|
assertSameSet(q1, q2);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** "A B" ⊆ (+A +B) */
|
||||||
|
public void testExactPhraseVersusBooleanAnd() throws Exception {
|
||||||
|
Term t1 = randomTerm();
|
||||||
|
Term t2 = randomTerm();
|
||||||
|
PhraseQuery q1 = new PhraseQuery();
|
||||||
|
q1.add(t1);
|
||||||
|
q1.add(t2);
|
||||||
|
BooleanQuery q2 = new BooleanQuery();
|
||||||
|
q2.add(new TermQuery(t1), Occur.MUST);
|
||||||
|
q2.add(new TermQuery(t2), Occur.MUST);
|
||||||
|
assertSubsetOf(q1, q2);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** same as above, with posincs */
|
||||||
|
public void testExactPhraseVersusBooleanAndWithHoles() throws Exception {
|
||||||
|
Term t1 = randomTerm();
|
||||||
|
Term t2 = randomTerm();
|
||||||
|
PhraseQuery q1 = new PhraseQuery();
|
||||||
|
q1.add(t1);
|
||||||
|
q1.add(t2, 2);
|
||||||
|
BooleanQuery q2 = new BooleanQuery();
|
||||||
|
q2.add(new TermQuery(t1), Occur.MUST);
|
||||||
|
q2.add(new TermQuery(t2), Occur.MUST);
|
||||||
|
assertSubsetOf(q1, q2);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** "A B" ⊆ "A B"~1 */
|
||||||
|
public void testPhraseVersusSloppyPhrase() throws Exception {
|
||||||
|
Term t1 = randomTerm();
|
||||||
|
Term t2 = randomTerm();
|
||||||
|
PhraseQuery q1 = new PhraseQuery();
|
||||||
|
q1.add(t1);
|
||||||
|
q1.add(t2);
|
||||||
|
PhraseQuery q2 = new PhraseQuery();
|
||||||
|
q2.add(t1);
|
||||||
|
q2.add(t2);
|
||||||
|
q2.setSlop(1);
|
||||||
|
assertSubsetOf(q1, q2);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** same as above, with posincs */
|
||||||
|
public void testPhraseVersusSloppyPhraseWithHoles() throws Exception {
|
||||||
|
Term t1 = randomTerm();
|
||||||
|
Term t2 = randomTerm();
|
||||||
|
PhraseQuery q1 = new PhraseQuery();
|
||||||
|
q1.add(t1);
|
||||||
|
q1.add(t2, 2);
|
||||||
|
PhraseQuery q2 = new PhraseQuery();
|
||||||
|
q2.add(t1);
|
||||||
|
q2.add(t2, 2);
|
||||||
|
q2.setSlop(1);
|
||||||
|
assertSubsetOf(q1, q2);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** "A B" ⊆ "A (B C)" */
|
||||||
|
public void testExactPhraseVersusMultiPhrase() throws Exception {
|
||||||
|
Term t1 = randomTerm();
|
||||||
|
Term t2 = randomTerm();
|
||||||
|
PhraseQuery q1 = new PhraseQuery();
|
||||||
|
q1.add(t1);
|
||||||
|
q1.add(t2);
|
||||||
|
Term t3 = randomTerm();
|
||||||
|
MultiPhraseQuery q2 = new MultiPhraseQuery();
|
||||||
|
q2.add(t1);
|
||||||
|
q2.add(new Term[] { t2, t3 });
|
||||||
|
assertSubsetOf(q1, q2);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** same as above, with posincs */
|
||||||
|
public void testExactPhraseVersusMultiPhraseWithHoles() throws Exception {
|
||||||
|
Term t1 = randomTerm();
|
||||||
|
Term t2 = randomTerm();
|
||||||
|
PhraseQuery q1 = new PhraseQuery();
|
||||||
|
q1.add(t1);
|
||||||
|
q1.add(t2, 2);
|
||||||
|
Term t3 = randomTerm();
|
||||||
|
MultiPhraseQuery q2 = new MultiPhraseQuery();
|
||||||
|
q2.add(t1);
|
||||||
|
q2.add(new Term[] { t2, t3 }, 2);
|
||||||
|
assertSubsetOf(q1, q2);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,207 @@
|
||||||
|
package org.apache.lucene.search;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import java.util.BitSet;
|
||||||
|
|
||||||
|
import org.apache.lucene.analysis.Analyzer;
|
||||||
|
import org.apache.lucene.analysis.MockAnalyzer;
|
||||||
|
import org.apache.lucene.analysis.MockTokenizer;
|
||||||
|
import org.apache.lucene.document.Document;
|
||||||
|
import org.apache.lucene.document.Field;
|
||||||
|
import org.apache.lucene.document.StringField;
|
||||||
|
import org.apache.lucene.document.TextField;
|
||||||
|
import org.apache.lucene.index.IndexReader;
|
||||||
|
import org.apache.lucene.index.RandomIndexWriter;
|
||||||
|
import org.apache.lucene.index.Term;
|
||||||
|
import org.apache.lucene.store.Directory;
|
||||||
|
import org.apache.lucene.util.Bits;
|
||||||
|
import org.apache.lucene.util.LuceneTestCase;
|
||||||
|
import org.apache.lucene.util._TestUtil;
|
||||||
|
import org.apache.lucene.util.automaton.BasicAutomata;
|
||||||
|
import org.apache.lucene.util.automaton.CharacterRunAutomaton;
|
||||||
|
import org.junit.AfterClass;
|
||||||
|
import org.junit.BeforeClass;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Simple base class for checking search equivalence.
|
||||||
|
* Extend it, and write tests that create {@link #randomTerm()}s
|
||||||
|
* (all terms are single characters a-z), and use
|
||||||
|
* {@link #assertSameSet(Query, Query)} and
|
||||||
|
* {@link #assertSubsetOf(Query, Query)}
|
||||||
|
*/
|
||||||
|
public abstract class SearchEquivalenceTestBase extends LuceneTestCase {
|
||||||
|
protected static IndexSearcher s1, s2;
|
||||||
|
protected static Directory directory;
|
||||||
|
protected static IndexReader reader;
|
||||||
|
protected static Analyzer analyzer;
|
||||||
|
protected static String stopword; // we always pick a character as a stopword
|
||||||
|
|
||||||
|
@BeforeClass
|
||||||
|
public static void beforeClass() throws Exception {
|
||||||
|
directory = newDirectory();
|
||||||
|
stopword = "" + randomChar();
|
||||||
|
CharacterRunAutomaton stopset = new CharacterRunAutomaton(BasicAutomata.makeString(stopword));
|
||||||
|
analyzer = new MockAnalyzer(random, MockTokenizer.WHITESPACE, false, stopset, true);
|
||||||
|
RandomIndexWriter iw = new RandomIndexWriter(random, directory, analyzer);
|
||||||
|
Document doc = new Document();
|
||||||
|
Field id = new StringField("id", "");
|
||||||
|
Field field = new TextField("field", "");
|
||||||
|
doc.add(id);
|
||||||
|
doc.add(field);
|
||||||
|
|
||||||
|
// index some docs
|
||||||
|
int numDocs = atLeast(1000);
|
||||||
|
for (int i = 0; i < numDocs; i++) {
|
||||||
|
id.setStringValue(Integer.toString(i));
|
||||||
|
field.setStringValue(randomFieldContents());
|
||||||
|
iw.addDocument(doc);
|
||||||
|
}
|
||||||
|
|
||||||
|
// delete some docs
|
||||||
|
int numDeletes = numDocs/20;
|
||||||
|
for (int i = 0; i < numDeletes; i++) {
|
||||||
|
Term toDelete = new Term("id", Integer.toString(random.nextInt(numDocs)));
|
||||||
|
if (random.nextBoolean()) {
|
||||||
|
iw.deleteDocuments(toDelete);
|
||||||
|
} else {
|
||||||
|
iw.deleteDocuments(new TermQuery(toDelete));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
reader = iw.getReader();
|
||||||
|
s1 = newSearcher(reader);
|
||||||
|
s2 = newSearcher(reader);
|
||||||
|
iw.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
@AfterClass
|
||||||
|
public static void afterClass() throws Exception {
|
||||||
|
reader.close();
|
||||||
|
directory.close();
|
||||||
|
analyzer.close();
|
||||||
|
reader = null;
|
||||||
|
directory = null;
|
||||||
|
analyzer = null;
|
||||||
|
s1 = s2 = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* populate a field with random contents.
|
||||||
|
* terms should be single characters in lowercase (a-z)
|
||||||
|
* tokenization can be assumed to be on whitespace.
|
||||||
|
*/
|
||||||
|
static String randomFieldContents() {
|
||||||
|
// TODO: zipf-like distribution
|
||||||
|
StringBuilder sb = new StringBuilder();
|
||||||
|
int numTerms = random.nextInt(15);
|
||||||
|
for (int i = 0; i < numTerms; i++) {
|
||||||
|
if (sb.length() > 0) {
|
||||||
|
sb.append(' '); // whitespace
|
||||||
|
}
|
||||||
|
sb.append(randomChar());
|
||||||
|
}
|
||||||
|
return sb.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* returns random character (a-z)
|
||||||
|
*/
|
||||||
|
static char randomChar() {
|
||||||
|
return (char) _TestUtil.nextInt(random, 'a', 'z');
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* returns a term suitable for searching.
|
||||||
|
* terms are single characters in lowercase (a-z)
|
||||||
|
*/
|
||||||
|
protected Term randomTerm() {
|
||||||
|
return new Term("field", "" + randomChar());
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns a random filter over the document set
|
||||||
|
*/
|
||||||
|
protected Filter randomFilter() {
|
||||||
|
return new QueryWrapperFilter(TermRangeQuery.newStringRange("field", "a", "" + randomChar(), true, true));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Asserts that the documents returned by <code>q1</code>
|
||||||
|
* are the same as of those returned by <code>q2</code>
|
||||||
|
*/
|
||||||
|
public void assertSameSet(Query q1, Query q2) throws Exception {
|
||||||
|
assertSubsetOf(q1, q2);
|
||||||
|
assertSubsetOf(q2, q1);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Asserts that the documents returned by <code>q1</code>
|
||||||
|
* are a subset of those returned by <code>q2</code>
|
||||||
|
*/
|
||||||
|
public void assertSubsetOf(Query q1, Query q2) throws Exception {
|
||||||
|
// test without a filter
|
||||||
|
assertSubsetOf(q1, q2, null);
|
||||||
|
|
||||||
|
// test with a filter (this will sometimes cause advance'ing enough to test it)
|
||||||
|
assertSubsetOf(q1, q2, randomFilter());
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Asserts that the documents returned by <code>q1</code>
|
||||||
|
* are a subset of those returned by <code>q2</code>.
|
||||||
|
*
|
||||||
|
* Both queries will be filtered by <code>filter</code>
|
||||||
|
*/
|
||||||
|
protected void assertSubsetOf(Query q1, Query q2, Filter filter) throws Exception {
|
||||||
|
// TRUNK ONLY: test both filter code paths
|
||||||
|
if (filter != null && random.nextBoolean()) {
|
||||||
|
final boolean q1RandomAccess = random.nextBoolean();
|
||||||
|
final boolean q2RandomAccess = random.nextBoolean();
|
||||||
|
q1 = new FilteredQuery(q1, filter) {
|
||||||
|
@Override
|
||||||
|
protected boolean useRandomAccess(Bits bits, int firstFilterDoc) {
|
||||||
|
return q1RandomAccess;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
q2 = new FilteredQuery(q2, filter) {
|
||||||
|
@Override
|
||||||
|
protected boolean useRandomAccess(Bits bits, int firstFilterDoc) {
|
||||||
|
return q2RandomAccess;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
filter = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
// not efficient, but simple!
|
||||||
|
TopDocs td1 = s1.search(q1, filter, reader.maxDoc());
|
||||||
|
TopDocs td2 = s2.search(q2, filter, reader.maxDoc());
|
||||||
|
assertTrue(td1.totalHits <= td2.totalHits);
|
||||||
|
|
||||||
|
// fill the superset into a bitset
|
||||||
|
BitSet bitset = new BitSet();
|
||||||
|
for (int i = 0; i < td2.scoreDocs.length; i++) {
|
||||||
|
bitset.set(td2.scoreDocs[i].doc);
|
||||||
|
}
|
||||||
|
|
||||||
|
// check in the subset, that every bit was set by the super
|
||||||
|
for (int i = 0; i < td1.scoreDocs.length; i++) {
|
||||||
|
assertTrue(bitset.get(td1.scoreDocs[i].doc));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue