Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-6269: Remove BooleanFilter. 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1661366 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Adrien Grand 2015-02-21 14:08:09 +00:00
parent 27eed40337
commit 3dc3e57847
18 changed files with 56 additions and 1061 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
lucene/CHANGES.txt
View File

@ -118,6 +118,9 @@ API Changes
instead. This will be as efficient now that queries can opt out from
scoring. (Adrien Grand)
* LUCENE-6269: Removed BooleanFilter, use a QueryWrapperFilter(BooleanQuery)
instead. (Adrien Grand)
* LUCENE-6223: Move BooleanQuery.BooleanWeight to BooleanWeight.
(Robert Muir)

22
lucene/core/src/java/org/apache/lucene/util/BitDocIdSet.java
View File

@ -145,28 +145,6 @@ public class BitDocIdSet extends DocIdSet {
sparseSet.or(it);
}
/**
* Removes from this builder documents that are not contained in <code>it</code>.
*/
public void and(DocIdSetIterator it) throws IOException {
if (denseSet != null) {
denseSet.and(it);
} else if (sparseSet != null) {
sparseSet.and(it);
}
}
/**
* Removes from this builder documents that are contained in <code>it</code>.
*/
public void andNot(DocIdSetIterator it) throws IOException {
if (denseSet != null) {
denseSet.andNot(it);
} else if (sparseSet != null) {
sparseSet.andNot(it);
}
}
/**
* Build a {@link DocIdSet} that contains all doc ids that have been added.
* This method may return <tt>null</tt> if no documents were addded to this

65
lucene/core/src/java/org/apache/lucene/util/BitSet.java
View File

@ -79,69 +79,4 @@ public abstract class BitSet implements MutableBits, Accountable {
}
}
private static abstract class LeapFrogCallBack {
abstract void onMatch(int doc);
void finish() {}
}
/** Performs a leap frog between this and the provided iterator in order to find common documents. */
private void leapFrog(DocIdSetIterator iter, LeapFrogCallBack callback) throws IOException {
final int length = length();
int bitSetDoc = -1;
int disiDoc = iter.nextDoc();
while (true) {
// invariant: bitSetDoc <= disiDoc
assert bitSetDoc <= disiDoc;
if (disiDoc >= length) {
callback.finish();
return;
}
if (bitSetDoc < disiDoc) {
bitSetDoc = nextSetBit(disiDoc);
}
if (bitSetDoc == disiDoc) {
callback.onMatch(bitSetDoc);
disiDoc = iter.nextDoc();
} else {
disiDoc = iter.advance(bitSetDoc);
}
}
}
/** Does in-place AND of the bits provided by the iterator. The state of the
* iterator after this operation terminates is undefined. */
public void and(DocIdSetIterator iter) throws IOException {
assertUnpositioned(iter);
leapFrog(iter, new LeapFrogCallBack() {
int previous = -1;
@Override
public void onMatch(int doc) {
clear(previous + 1, doc);
previous = doc;
}
@Override
public void finish() {
if (previous + 1 < length()) {
clear(previous + 1, length());
}
}
});
}
/** this = this AND NOT other. The state of the iterator after this operation
* terminates is undefined. */
public void andNot(DocIdSetIterator iter) throws IOException {
assertUnpositioned(iter);
leapFrog(iter, new LeapFrogCallBack() {
@Override
public void onMatch(int doc) {
clear(doc);
}
});
}
}

22
lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java
View File

@ -277,17 +277,6 @@ public final class FixedBitSet extends BitSet implements MutableBits, Accountabl
}
}
@Override
public void and(DocIdSetIterator iter) throws IOException {
if (BitSetIterator.getFixedBitSetOrNull(iter) != null) {
assertUnpositioned(iter);
final FixedBitSet bits = BitSetIterator.getFixedBitSetOrNull(iter);
and(bits);
} else {
super.and(iter);
}
}
/** returns true if the sets have any elements in common */
public boolean intersects(FixedBitSet other) {
int pos = Math.min(numWords, other.numWords);
@ -313,17 +302,6 @@ public final class FixedBitSet extends BitSet implements MutableBits, Accountabl
}
}
@Override
public void andNot(DocIdSetIterator iter) throws IOException {
if (BitSetIterator.getFixedBitSetOrNull(iter) != null) {
assertUnpositioned(iter);
final FixedBitSet bits = BitSetIterator.getFixedBitSetOrNull(iter);
andNot(bits);
} else {
super.andNot(iter);
}
}
/** this = this AND NOT other */
public void andNot(FixedBitSet other) {
andNot(other.bits, other.bits.length);

25
lucene/core/src/java/org/apache/lucene/util/SparseFixedBitSet.java
View File

@ -485,31 +485,6 @@ public class SparseFixedBitSet extends BitSet implements Bits, Accountable {
}
}
// AND and AND_NOT do not need much specialization here since this sparse set
// is supposed to be used on sparse data and the default AND/AND_NOT impl
// (leap frog) is efficient when at least one of the sets contains sparse data
@Override
public void and(DocIdSetIterator it) throws IOException {
final SparseFixedBitSet other = BitSetIterator.getSparseFixedBitSetOrNull(it);
if (other != null) {
// if we are merging with another SparseFixedBitSet, a quick win is
// to clear up some blocks by only looking at their index. Then the set
// is sparser and the leap-frog approach of the parent class is more
// efficient. Since SparseFixedBitSet is supposed to be used for sparse
// sets, the intersection of two SparseFixedBitSet is likely very sparse
final int numCommonBlocks = Math.min(indices.length, other.indices.length);
for (int i = 0; i < numCommonBlocks; ++i) {
if ((indices[i] & other.indices[i]) == 0) {
this.nonZeroLongCount -= Long.bitCount(this.indices[i]);
this.indices[i] = 0;
this.bits[i] = null;
}
}
}
super.and(it);
}
@Override
public long ramBytesUsed() {
return ramBytesUsed;

58
lucene/core/src/test/org/apache/lucene/util/TestBitDocIdSetBuilder.java
View File

@ -75,61 +75,5 @@ public class TestBitDocIdSetBuilder extends LuceneTestCase {
set.or(other.iterator());
assertEquals(new BitDocIdSet(set), builder.build());
}
public void testAndDense() throws IOException {
final int maxDoc = TestUtil.nextInt(random(), 10000, 100000);
BitDocIdSet.Builder builder = new BitDocIdSet.Builder(maxDoc);
FixedBitSet set = new FixedBitSet(maxDoc);
DocIdSet other = randomSet(maxDoc, maxDoc / 2);
builder.or(other.iterator());
set.or(other.iterator());
assertTrue(builder.dense());
other = randomSet(maxDoc, maxDoc / 2);
builder.and(other.iterator());
set.and(other.iterator());
assertEquals(new BitDocIdSet(set), builder.build());
}
public void testAndSparse() throws IOException {
final int maxDoc = TestUtil.nextInt(random(), 10000, 100000);
BitDocIdSet.Builder builder = new BitDocIdSet.Builder(maxDoc);
FixedBitSet set = new FixedBitSet(maxDoc);
DocIdSet other = randomSet(maxDoc, maxDoc / 2000);
builder.or(other.iterator());
set.or(other.iterator());
assertFalse(builder.dense());
other = randomSet(maxDoc, maxDoc / 2);
builder.and(other.iterator());
set.and(other.iterator());
assertEquals(new BitDocIdSet(set), builder.build());
}
public void testAndNotDense() throws IOException {
final int maxDoc = TestUtil.nextInt(random(), 10000, 100000);
BitDocIdSet.Builder builder = new BitDocIdSet.Builder(maxDoc);
FixedBitSet set = new FixedBitSet(maxDoc);
DocIdSet other = randomSet(maxDoc, maxDoc / 2);
builder.or(other.iterator());
set.or(other.iterator());
assertTrue(builder.dense());
other = randomSet(maxDoc, maxDoc / 2);
builder.andNot(other.iterator());
set.andNot(other.iterator());
assertEquals(new BitDocIdSet(set), builder.build());
}
public void testAndNotSparse() throws IOException {
final int maxDoc = TestUtil.nextInt(random(), 10000, 100000);
BitDocIdSet.Builder builder = new BitDocIdSet.Builder(maxDoc);
FixedBitSet set = new FixedBitSet(maxDoc);
DocIdSet other = randomSet(maxDoc, maxDoc / 2000);
builder.or(other.iterator());
set.or(other.iterator());
assertFalse(builder.dense());
other = randomSet(maxDoc, maxDoc / 2);
builder.andNot(other.iterator());
set.andNot(other.iterator());
assertEquals(new BitDocIdSet(set), builder.build());
}
}

18
lucene/demo/src/java/org/apache/lucene/demo/facet/DistanceFacetsExample.java
View File

@ -42,13 +42,15 @@ import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.queries.BooleanFilter;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.Filter;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.NumericRangeFilter;
import org.apache.lucene.search.NumericRangeQuery;
import org.apache.lucene.search.QueryWrapperFilter;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
@ -179,28 +181,28 @@ public class DistanceFacetsExample implements Closeable {
maxLng = Math.toRadians(180);
}
BooleanFilter f = new BooleanFilter();
BooleanQuery f = new BooleanQuery();
// Add latitude range filter:
f.add(NumericRangeFilter.newDoubleRange("latitude", Math.toDegrees(minLat), Math.toDegrees(maxLat), true, true),
BooleanClause.Occur.MUST);
BooleanClause.Occur.FILTER);
// Add longitude range filter:
if (minLng > maxLng) {
// The bounding box crosses the international date
// line:
BooleanFilter lonF = new BooleanFilter();
lonF.add(NumericRangeFilter.newDoubleRange("longitude", Math.toDegrees(minLng), null, true, true),
BooleanQuery lonF = new BooleanQuery();
lonF.add(NumericRangeQuery.newDoubleRange("longitude", Math.toDegrees(minLng), null, true, true),
BooleanClause.Occur.SHOULD);
lonF.add(NumericRangeFilter.newDoubleRange("longitude", null, Math.toDegrees(maxLng), true, true),
lonF.add(NumericRangeQuery.newDoubleRange("longitude", null, Math.toDegrees(maxLng), true, true),
BooleanClause.Occur.SHOULD);
f.add(lonF, BooleanClause.Occur.MUST);
} else {
f.add(NumericRangeFilter.newDoubleRange("longitude", Math.toDegrees(minLng), Math.toDegrees(maxLng), true, true),
BooleanClause.Occur.MUST);
BooleanClause.Occur.FILTER);
}
return f;
return new QueryWrapperFilter(f);
}
/** User runs a query and counts facets. */

58
lucene/facet/src/java/org/apache/lucene/facet/range/DoubleRange.java
View File

@ -26,6 +26,7 @@ import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Filter;
import org.apache.lucene.search.FilteredDocIdSet;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.NumericUtils;
@ -121,52 +122,33 @@ public final class DoubleRange extends Range {
final int maxDoc = context.reader().maxDoc();
final Bits fastMatchBits;
final DocIdSet fastMatchDocs;
if (fastMatchFilter != null) {
DocIdSet dis = fastMatchFilter.getDocIdSet(context, null);
if (dis == null) {
fastMatchDocs = fastMatchFilter.getDocIdSet(context, null);
if (fastMatchDocs == null) {
// No documents match
return null;
}
fastMatchBits = dis.bits();
if (fastMatchBits == null) {
throw new IllegalArgumentException("fastMatchFilter does not implement DocIdSet.bits");
}
} else {
fastMatchBits = null;
fastMatchDocs = new DocIdSet() {
@Override
public long ramBytesUsed() {
return 0;
}
@Override
public DocIdSetIterator iterator() throws IOException {
return DocIdSetIterator.all(maxDoc);
}
};
}
return new DocIdSet() {
return new FilteredDocIdSet(fastMatchDocs) {
@Override
public Bits bits() {
return new Bits() {
@Override
public boolean get(int docID) {
if (acceptDocs != null && acceptDocs.get(docID) == false) {
return false;
}
if (fastMatchBits != null && fastMatchBits.get(docID) == false) {
return false;
}
return accept(values.doubleVal(docID));
}
@Override
public int length() {
return maxDoc;
}
};
}
@Override
public DocIdSetIterator iterator() {
throw new UnsupportedOperationException("this filter can only be accessed via bits()");
}
@Override
public long ramBytesUsed() {
return 0L;
protected boolean match(int docID) {
if (acceptDocs != null && acceptDocs.get(docID) == false) {
return false;
}
return accept(values.doubleVal(docID));
}
};
}

29
lucene/facet/src/java/org/apache/lucene/facet/range/DoubleRangeFacetCounts.java
View File

@ -96,28 +96,31 @@ public class DoubleRangeFacetCounts extends RangeFacetCounts {
FunctionValues fv = valueSource.getValues(Collections.emptyMap(), hits.context);
totCount += hits.totalHits;
Bits bits;
final DocIdSetIterator fastMatchDocs;
if (fastMatchFilter != null) {
DocIdSet dis = fastMatchFilter.getDocIdSet(hits.context, null);
if (dis == null) {
// No documents match
continue;
}
bits = dis.bits();
if (bits == null) {
throw new IllegalArgumentException("fastMatchFilter does not implement DocIdSet.bits");
}
fastMatchDocs = dis.iterator();
} else {
bits = null;
fastMatchDocs = null;
}
DocIdSetIterator docs = hits.bits.iterator();
int doc;
while ((doc = docs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
if (bits != null && bits.get(doc) == false) {
doc++;
continue;
for (int doc = docs.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; ) {
if (fastMatchDocs != null) {
int fastMatchDoc = fastMatchDocs.docID();
if (fastMatchDoc < doc) {
fastMatchDoc = fastMatchDocs.advance(doc);
}
if (doc != fastMatchDoc) {
doc = docs.advance(fastMatchDoc);
continue;
}
}
// Skip missing docs:
if (fv.exists(doc)) {
@ -125,6 +128,8 @@ public class DoubleRangeFacetCounts extends RangeFacetCounts {
} else {
missingCount++;
}
doc = docs.nextDoc();
}
}

173
lucene/queries/src/java/org/apache/lucene/queries/BooleanFilter.java
View File

@ -1,173 +0,0 @@
package org.apache.lucene.queries;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.BitsFilteredDocIdSet;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Filter;
import org.apache.lucene.util.BitDocIdSet;
import org.apache.lucene.util.Bits;
/**
* A container Filter that allows Boolean composition of Filters.
* Filters are allocated into one of three logical constructs;
* SHOULD, MUST NOT, MUST
* The results Filter BitSet is constructed as follows:
* SHOULD Filters are OR'd together
* The resulting Filter is NOT'd with the NOT Filters
* The resulting Filter is AND'd with the MUST Filters
*/
public class BooleanFilter extends Filter implements Iterable<FilterClause> {
private final List<FilterClause> clauses = new ArrayList<>();
/**
* Returns the a DocIdSetIterator representing the Boolean composition
* of the filters that have been added.
*/
@Override
public DocIdSet getDocIdSet(LeafReaderContext context, Bits acceptDocs) throws IOException {
BitDocIdSet.Builder res = null;
final LeafReader reader = context.reader();
boolean hasShouldClauses = false;
for (final FilterClause fc : clauses) {
if (fc.getOccur() == Occur.SHOULD) {
hasShouldClauses = true;
final DocIdSetIterator disi = getDISI(fc.getFilter(), context);
if (disi == null) continue;
if (res == null) {
res = new BitDocIdSet.Builder(reader.maxDoc());
}
res.or(disi);
}
}
if (hasShouldClauses && res == null)
return null;
for (final FilterClause fc : clauses) {
if (fc.getOccur() == Occur.MUST_NOT) {
if (res == null) {
assert !hasShouldClauses;
res = new BitDocIdSet.Builder(reader.maxDoc(), true); // NOTE: may set bits on deleted docs
}
final DocIdSetIterator disi = getDISI(fc.getFilter(), context);
if (disi != null) {
res.andNot(disi);
}
}
}
for (final FilterClause fc : clauses) {
if (fc.getOccur() == Occur.MUST) {
final DocIdSetIterator disi = getDISI(fc.getFilter(), context);
if (disi == null) {
return null; // no documents can match
}
if (res == null) {
res = new BitDocIdSet.Builder(reader.maxDoc());
res.or(disi);
} else {
res.and(disi);
}
}
}
if (res == null) {
return null;
}
return BitsFilteredDocIdSet.wrap(res.build(), acceptDocs);
}
private static DocIdSetIterator getDISI(Filter filter, LeafReaderContext context)
throws IOException {
// we dont pass acceptDocs, we will filter at the end using an additional filter
final DocIdSet set = filter.getDocIdSet(context, null);
return set == null ? null : set.iterator();
}
/**
* Adds a new FilterClause to the Boolean Filter container
* @param filterClause A FilterClause object containing a Filter and an Occur parameter
*/
public void add(FilterClause filterClause) {
clauses.add(filterClause);
}
public final void add(Filter filter, Occur occur) {
add(new FilterClause(filter, occur));
}
/**
* Returns the list of clauses
*/
public List<FilterClause> clauses() {
return clauses;
}
/** Returns an iterator on the clauses in this query. It implements the {@link Iterable} interface to
* make it possible to do:
* <pre class="prettyprint">for (FilterClause clause : booleanFilter) {}</pre>
*/
@Override
public final Iterator<FilterClause> iterator() {
return clauses().iterator();
}
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if ((obj == null) || (obj.getClass() != this.getClass())) {
return false;
}
final BooleanFilter other = (BooleanFilter)obj;
return clauses.equals(other.clauses);
}
@Override
public int hashCode() {
return 657153718 ^ clauses.hashCode();
}
/** Prints a user-readable version of this Filter. */
@Override
public String toString(String field) {
final StringBuilder buffer = new StringBuilder("BooleanFilter(");
final int minLen = buffer.length();
for (final FilterClause c : clauses) {
if (buffer.length() > minLen) {
buffer.append(' ');
}
buffer.append(c);
}
return buffer.append(')').toString();
}
}

86
lucene/queries/src/java/org/apache/lucene/queries/FilterClause.java
View File

@ -1,86 +0,0 @@
package org.apache.lucene.queries;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.Filter;
/**
* A Filter that wrapped with an indication of how that filter
* is used when composed with another filter.
* (Follows the boolean logic in BooleanClause for composition
* of queries.)
*/
public final class FilterClause {
private final Occur occur;
private final Filter filter;
/**
* Create a new FilterClause
* @param filter A Filter object containing a BitSet
* @param occur A parameter implementation indicating SHOULD, MUST or MUST NOT
*/
public FilterClause(Filter filter, Occur occur) {
this.occur = occur;
this.filter = filter;
}
/**
* Returns this FilterClause's filter
* @return A Filter object
*/
public Filter getFilter() {
return filter;
}
/**
* Returns this FilterClause's occur parameter
* @return An Occur object
*/
public Occur getOccur() {
return occur;
}
@Override
public boolean equals(Object o) {
if (o == this)
return true;
if (o == null || !(o instanceof FilterClause))
return false;
final FilterClause other = (FilterClause)o;
return this.filter.equals(other.filter)
&& this.occur == other.occur;
}
@Override
public int hashCode() {
return filter.hashCode() ^ occur.hashCode();
}
public String toString(String field) {
return occur.toString() + filter.toString(field);
}
@Override
public String toString() {
return toString("");
}
}

370
lucene/queries/src/test/org/apache/lucene/queries/BooleanFilterTest.java
View File

@ -1,370 +0,0 @@
package org.apache.lucene.queries;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.SlowCompositeReaderWrapper;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Filter;
import org.apache.lucene.search.QueryWrapperFilter;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TermRangeFilter;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LuceneTestCase;
public class BooleanFilterTest extends LuceneTestCase {
private Directory directory;
private LeafReader reader;
@Override
public void setUp() throws Exception {
super.setUp();
directory = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), directory, new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false));
//Add series of docs with filterable fields : acces rights, prices, dates and "in-stock" flags
addDoc(writer, "admin guest", "010", "20040101", "Y");
addDoc(writer, "guest", "020", "20040101", "Y");
addDoc(writer, "guest", "020", "20050101", "Y");
addDoc(writer, "admin", "020", "20050101", "Maybe");
addDoc(writer, "admin guest", "030", "20050101", "N");
reader = SlowCompositeReaderWrapper.wrap(writer.getReader());
writer.close();
}
@Override
public void tearDown() throws Exception {
reader.close();
directory.close();
super.tearDown();
}
private void addDoc(RandomIndexWriter writer, String accessRights, String price, String date, String inStock) throws IOException {
Document doc = new Document();
doc.add(newTextField("accessRights", accessRights, Field.Store.YES));
doc.add(newTextField("price", price, Field.Store.YES));
doc.add(newTextField("date", date, Field.Store.YES));
doc.add(newTextField("inStock", inStock, Field.Store.YES));
writer.addDocument(doc);
}
private Filter getRangeFilter(String field, String lowerPrice, String upperPrice) {
Filter f = TermRangeFilter.newStringRange(field, lowerPrice, upperPrice, true, true);
return f;
}
private Filter getTermsFilter(String field, String text) {
return new TermsFilter(new Term(field, text));
}
private Filter getWrappedTermQuery(String field, String text) {
return new QueryWrapperFilter(new TermQuery(new Term(field, text)));
}
private Filter getEmptyFilter() {
return new Filter() {
@Override
public DocIdSet getDocIdSet(LeafReaderContext context, Bits acceptDocs) {
return new BitDocIdSet(new FixedBitSet(context.reader().maxDoc()));
}
@Override
public String toString(String field) {
return "emptyFilter";
}
};
}
private Filter getNullDISFilter() {
return new Filter() {
@Override
public DocIdSet getDocIdSet(LeafReaderContext context, Bits acceptDocs) {
return null;
}
@Override
public String toString(String field) {
return "nullDISFilter";
}
};
}
private Filter getNullDISIFilter() {
return new Filter() {
@Override
public DocIdSet getDocIdSet(LeafReaderContext context, Bits acceptDocs) {
return DocIdSet.EMPTY;
}
@Override
public String toString(String field) {
return "nullDISIFilter";
}
};
}
private void tstFilterCard(String mes, int expected, Filter filt)
throws Exception {
final DocIdSet docIdSet = filt.getDocIdSet(reader.getContext(), reader.getLiveDocs());
int actual = 0;
if (docIdSet != null) {
DocIdSetIterator disi = docIdSet.iterator();
while (disi.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
actual++;
}
}
assertEquals(mes, expected, actual);
}
public void testShould() throws Exception {
BooleanFilter booleanFilter = new BooleanFilter();
booleanFilter.add(getTermsFilter("price", "030"), Occur.SHOULD);
tstFilterCard("Should retrieves only 1 doc", 1, booleanFilter);
// same with a real DISI (no OpenBitSetIterator)
booleanFilter = new BooleanFilter();
booleanFilter.add(getWrappedTermQuery("price", "030"), Occur.SHOULD);
tstFilterCard("Should retrieves only 1 doc", 1, booleanFilter);
}
public void testShoulds() throws Exception {
BooleanFilter booleanFilter = new BooleanFilter();
booleanFilter.add(getRangeFilter("price", "010", "020"), Occur.SHOULD);
booleanFilter.add(getRangeFilter("price", "020", "030"), Occur.SHOULD);
tstFilterCard("Shoulds are Ored together", 5, booleanFilter);
}
public void testShouldsAndMustNot() throws Exception {
BooleanFilter booleanFilter = new BooleanFilter();
booleanFilter.add(getRangeFilter("price", "010", "020"), Occur.SHOULD);
booleanFilter.add(getRangeFilter("price", "020", "030"), Occur.SHOULD);
booleanFilter.add(getTermsFilter("inStock", "N"), Occur.MUST_NOT);
tstFilterCard("Shoulds Ored but AndNot", 4, booleanFilter);
booleanFilter.add(getTermsFilter("inStock", "Maybe"), Occur.MUST_NOT);
tstFilterCard("Shoulds Ored but AndNots", 3, booleanFilter);
// same with a real DISI (no OpenBitSetIterator)
booleanFilter = new BooleanFilter();
booleanFilter.add(getRangeFilter("price", "010", "020"), Occur.SHOULD);
booleanFilter.add(getRangeFilter("price", "020", "030"), Occur.SHOULD);
booleanFilter.add(getWrappedTermQuery("inStock", "N"), Occur.MUST_NOT);
tstFilterCard("Shoulds Ored but AndNot", 4, booleanFilter);
booleanFilter.add(getWrappedTermQuery("inStock", "Maybe"), Occur.MUST_NOT);
tstFilterCard("Shoulds Ored but AndNots", 3, booleanFilter);
}
public void testShouldsAndMust() throws Exception {
BooleanFilter booleanFilter = new BooleanFilter();
booleanFilter.add(getRangeFilter("price", "010", "020"), Occur.SHOULD);
booleanFilter.add(getRangeFilter("price", "020", "030"), Occur.SHOULD);
booleanFilter.add(getTermsFilter("accessRights", "admin"), Occur.MUST);
tstFilterCard("Shoulds Ored but MUST", 3, booleanFilter);
// same with a real DISI (no OpenBitSetIterator)
booleanFilter = new BooleanFilter();
booleanFilter.add(getRangeFilter("price", "010", "020"), Occur.SHOULD);
booleanFilter.add(getRangeFilter("price", "020", "030"), Occur.SHOULD);
booleanFilter.add(getWrappedTermQuery("accessRights", "admin"), Occur.MUST);
tstFilterCard("Shoulds Ored but MUST", 3, booleanFilter);
}
public void testShouldsAndMusts() throws Exception {
BooleanFilter booleanFilter = new BooleanFilter();
booleanFilter.add(getRangeFilter("price", "010", "020"), Occur.SHOULD);
booleanFilter.add(getRangeFilter("price", "020", "030"), Occur.SHOULD);
booleanFilter.add(getTermsFilter("accessRights", "admin"), Occur.MUST);
booleanFilter.add(getRangeFilter("date", "20040101", "20041231"), Occur.MUST);
tstFilterCard("Shoulds Ored but MUSTs ANDED", 1, booleanFilter);
}
public void testShouldsAndMustsAndMustNot() throws Exception {
BooleanFilter booleanFilter = new BooleanFilter();
booleanFilter.add(getRangeFilter("price", "030", "040"), Occur.SHOULD);
booleanFilter.add(getTermsFilter("accessRights", "admin"), Occur.MUST);
booleanFilter.add(getRangeFilter("date", "20050101", "20051231"), Occur.MUST);
booleanFilter.add(getTermsFilter("inStock", "N"), Occur.MUST_NOT);
tstFilterCard("Shoulds Ored but MUSTs ANDED and MustNot", 0, booleanFilter);
// same with a real DISI (no OpenBitSetIterator)
booleanFilter = new BooleanFilter();
booleanFilter.add(getRangeFilter("price", "030", "040"), Occur.SHOULD);
booleanFilter.add(getWrappedTermQuery("accessRights", "admin"), Occur.MUST);
booleanFilter.add(getRangeFilter("date", "20050101", "20051231"), Occur.MUST);
booleanFilter.add(getWrappedTermQuery("inStock", "N"), Occur.MUST_NOT);
tstFilterCard("Shoulds Ored but MUSTs ANDED and MustNot", 0, booleanFilter);
}
public void testJustMust() throws Exception {
BooleanFilter booleanFilter = new BooleanFilter();
booleanFilter.add(getTermsFilter("accessRights", "admin"), Occur.MUST);
tstFilterCard("MUST", 3, booleanFilter);
// same with a real DISI (no OpenBitSetIterator)
booleanFilter = new BooleanFilter();
booleanFilter.add(getWrappedTermQuery("accessRights", "admin"), Occur.MUST);
tstFilterCard("MUST", 3, booleanFilter);
}
public void testJustMustNot() throws Exception {
BooleanFilter booleanFilter = new BooleanFilter();
booleanFilter.add(getTermsFilter("inStock", "N"), Occur.MUST_NOT);
tstFilterCard("MUST_NOT", 4, booleanFilter);
// same with a real DISI (no OpenBitSetIterator)
booleanFilter = new BooleanFilter();
booleanFilter.add(getWrappedTermQuery("inStock", "N"), Occur.MUST_NOT);
tstFilterCard("MUST_NOT", 4, booleanFilter);
}
public void testMustAndMustNot() throws Exception {
BooleanFilter booleanFilter = new BooleanFilter();
booleanFilter.add(getTermsFilter("inStock", "N"), Occur.MUST);
booleanFilter.add(getTermsFilter("price", "030"), Occur.MUST_NOT);
tstFilterCard("MUST_NOT wins over MUST for same docs", 0, booleanFilter);
// same with a real DISI (no OpenBitSetIterator)
booleanFilter = new BooleanFilter();
booleanFilter.add(getWrappedTermQuery("inStock", "N"), Occur.MUST);
booleanFilter.add(getWrappedTermQuery("price", "030"), Occur.MUST_NOT);
tstFilterCard("MUST_NOT wins over MUST for same docs", 0, booleanFilter);
}
public void testEmpty() throws Exception {
BooleanFilter booleanFilter = new BooleanFilter();
tstFilterCard("empty BooleanFilter returns no results", 0, booleanFilter);
}
public void testCombinedNullDocIdSets() throws Exception {
BooleanFilter booleanFilter = new BooleanFilter();
booleanFilter.add(getTermsFilter("price", "030"), Occur.MUST);
booleanFilter.add(getNullDISFilter(), Occur.MUST);
tstFilterCard("A MUST filter that returns a null DIS should never return documents", 0, booleanFilter);
booleanFilter = new BooleanFilter();
booleanFilter.add(getTermsFilter("price", "030"), Occur.MUST);
booleanFilter.add(getNullDISIFilter(), Occur.MUST);
tstFilterCard("A MUST filter that returns a null DISI should never return documents", 0, booleanFilter);
booleanFilter = new BooleanFilter();
booleanFilter.add(getTermsFilter("price", "030"), Occur.SHOULD);
booleanFilter.add(getNullDISFilter(), Occur.SHOULD);
tstFilterCard("A SHOULD filter that returns a null DIS should be invisible", 1, booleanFilter);
booleanFilter = new BooleanFilter();
booleanFilter.add(getTermsFilter("price", "030"), Occur.SHOULD);
booleanFilter.add(getNullDISIFilter(), Occur.SHOULD);
tstFilterCard("A SHOULD filter that returns a null DISI should be invisible", 1, booleanFilter);
booleanFilter = new BooleanFilter();
booleanFilter.add(getTermsFilter("price", "030"), Occur.MUST);
booleanFilter.add(getNullDISFilter(), Occur.MUST_NOT);
tstFilterCard("A MUST_NOT filter that returns a null DIS should be invisible", 1, booleanFilter);
booleanFilter = new BooleanFilter();
booleanFilter.add(getTermsFilter("price", "030"), Occur.MUST);
booleanFilter.add(getNullDISIFilter(), Occur.MUST_NOT);
tstFilterCard("A MUST_NOT filter that returns a null DISI should be invisible", 1, booleanFilter);
}
public void testJustNullDocIdSets() throws Exception {
BooleanFilter booleanFilter = new BooleanFilter();
booleanFilter.add(getNullDISFilter(), Occur.MUST);
tstFilterCard("A MUST filter that returns a null DIS should never return documents", 0, booleanFilter);
booleanFilter = new BooleanFilter();
booleanFilter.add(getNullDISIFilter(), Occur.MUST);
tstFilterCard("A MUST filter that returns a null DISI should never return documents", 0, booleanFilter);
booleanFilter = new BooleanFilter();
booleanFilter.add(getNullDISFilter(), Occur.SHOULD);
tstFilterCard("A single SHOULD filter that returns a null DIS should never return documents", 0, booleanFilter);
booleanFilter = new BooleanFilter();
booleanFilter.add(getNullDISIFilter(), Occur.SHOULD);
tstFilterCard("A single SHOULD filter that returns a null DISI should never return documents", 0, booleanFilter);
booleanFilter = new BooleanFilter();
booleanFilter.add(getNullDISFilter(), Occur.MUST_NOT);
tstFilterCard("A single MUST_NOT filter that returns a null DIS should be invisible", 5, booleanFilter);
booleanFilter = new BooleanFilter();
booleanFilter.add(getNullDISIFilter(), Occur.MUST_NOT);
tstFilterCard("A single MUST_NOT filter that returns a null DIS should be invisible", 5, booleanFilter);
}
public void testNonMatchingShouldsAndMusts() throws Exception {
BooleanFilter booleanFilter = new BooleanFilter();
booleanFilter.add(getEmptyFilter(), Occur.SHOULD);
booleanFilter.add(getTermsFilter("accessRights", "admin"), Occur.MUST);
tstFilterCard(">0 shoulds with no matches should return no docs", 0, booleanFilter);
booleanFilter = new BooleanFilter();
booleanFilter.add(getNullDISFilter(), Occur.SHOULD);
booleanFilter.add(getTermsFilter("accessRights", "admin"), Occur.MUST);
tstFilterCard(">0 shoulds with no matches should return no docs", 0, booleanFilter);
booleanFilter = new BooleanFilter();
booleanFilter.add(getNullDISIFilter(), Occur.SHOULD);
booleanFilter.add(getTermsFilter("accessRights", "admin"), Occur.MUST);
tstFilterCard(">0 shoulds with no matches should return no docs", 0, booleanFilter);
}
public void testToStringOfBooleanFilterContainingTermsFilter() {
BooleanFilter booleanFilter = new BooleanFilter();
booleanFilter.add(getTermsFilter("inStock", "N"), Occur.MUST);
booleanFilter.add(getTermsFilter("isFragile", "Y"), Occur.MUST);
assertEquals("BooleanFilter(+inStock:N +isFragile:Y)", booleanFilter.toString());
}
public void testToStringOfWrappedBooleanFilters() {
BooleanFilter orFilter = new BooleanFilter();
BooleanFilter stockFilter = new BooleanFilter();
stockFilter.add(new FilterClause(getTermsFilter("inStock", "Y"), Occur.MUST));
stockFilter.add(new FilterClause(getTermsFilter("barCode", "12345678"), Occur.MUST));
orFilter.add(new FilterClause(stockFilter,Occur.SHOULD));
BooleanFilter productPropertyFilter = new BooleanFilter();
productPropertyFilter.add(new FilterClause(getTermsFilter("isHeavy", "N"), Occur.MUST));
productPropertyFilter.add(new FilterClause(getTermsFilter("isDamaged", "Y"), Occur.MUST));
orFilter.add(new FilterClause(productPropertyFilter,Occur.SHOULD));
BooleanFilter composedFilter = new BooleanFilter();
composedFilter.add(new FilterClause(orFilter,Occur.MUST));
assertEquals("BooleanFilter(+BooleanFilter(BooleanFilter(+inStock:Y +barCode:12345678) BooleanFilter(+isHeavy:N +isDamaged:Y)))",
composedFilter.toString());
}
}

1
lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/CorePlusExtensionsParser.java
View File

@ -50,7 +50,6 @@ public class CorePlusExtensionsParser extends CoreParser {
private CorePlusExtensionsParser(String defaultField, Analyzer analyzer, QueryParser parser) {
super(defaultField, analyzer, parser);
filterFactory.addBuilder("TermsFilter", new TermsFilterBuilder(analyzer));
filterFactory.addBuilder("BooleanFilter", new BooleanFilterBuilder(filterFactory));
filterFactory.addBuilder("DuplicateFilter", new DuplicateFilterBuilder());
String fields[] = {"contents"};
queryFactory.addBuilder("LikeThisQuery", new LikeThisQueryBuilder(analyzer, fields));

65
lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/BooleanFilterBuilder.java
View File

@ -1,65 +0,0 @@
/*
* Created on 25-Jan-2006
*/
package org.apache.lucene.queryparser.xml.builders;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.queries.BooleanFilter;
import org.apache.lucene.search.Filter;
import org.apache.lucene.queries.FilterClause;
import org.apache.lucene.queryparser.xml.DOMUtils;
import org.apache.lucene.queryparser.xml.FilterBuilder;
import org.apache.lucene.queryparser.xml.ParserException;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Builder for {@link BooleanFilter}
*/
public class BooleanFilterBuilder implements FilterBuilder {
private final FilterBuilder factory;
public BooleanFilterBuilder(FilterBuilder factory) {
this.factory = factory;
}
@Override
public Filter getFilter(Element e) throws ParserException {
BooleanFilter bf = new BooleanFilter();
NodeList nl = e.getChildNodes();
for (int i = 0; i < nl.getLength(); i++) {
Node node = nl.item(i);
if (node.getNodeName().equals("Clause")) {
Element clauseElem = (Element) node;
BooleanClause.Occur occurs = BooleanQueryBuilder.getOccursValue(clauseElem);
Element clauseFilter = DOMUtils.getFirstChildOrFail(clauseElem);
Filter f = factory.getFilter(clauseFilter);
bf.add(new FilterClause(f, occurs));
}
}
return bf;
}
}

41
lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/BooleanFilter.xml
View File

@ -1,41 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<FilteredQuery>
<Query>
<MatchAllDocsQuery/>
</Query>
<Filter>
<!--
This query illustrates how a BooleanFilter can be used to combine
multiple filters in the same way BooleanQueries can be combined
with must, should and mustnot clauses
-->
<BooleanFilter>
<Clause occurs="should">
<RangeFilter fieldName="date" lowerTerm="19870409" upperTerm="19870412"/>
</Clause>
<Clause occurs="mustNot">
<TermsFilter fieldName="contents">Emcore</TermsFilter>
</Clause>
</BooleanFilter>
</Filter>
</FilteredQuery>

5
lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestParser.java
View File

@ -181,11 +181,6 @@ public class TestParser extends LuceneTestCase {
dumpResults("MatchAllDocsQuery with range filter", q, 5);
}
public void testBooleanFilterXML() throws ParserException, IOException {
Query q = parse("BooleanFilter.xml");
dumpResults("Boolean filter", q, 5);
}
public void testNestedBooleanQuery() throws ParserException, IOException {
Query q = parse("NestedBooleanQuery.xml");
dumpResults("Nested Boolean query", q, 5);

65
lucene/test-framework/src/java/org/apache/lucene/util/BaseBitSetTestCase.java
View File

@ -204,71 +204,6 @@ public abstract class BaseBitSetTestCase<T extends BitSet> extends LuceneTestCas
testOr(random().nextFloat());
}
private void testAnd(float load) throws IOException {
final int numBits = 1 + random().nextInt(100000);
BitSet set1 = new JavaUtilBitSet(randomSet(numBits, numBits), numBits); // full
T set2 = copyOf(set1, numBits);
final int iterations = atLeast(10);
for (int iter = 0; iter < iterations; ++iter) {
// BitSets have specializations to merge with certain impls, so we randomize the impl...
DocIdSet otherSet = randomCopy(new JavaUtilBitSet(randomSet(numBits, load), numBits), numBits);
DocIdSetIterator otherIterator = otherSet.iterator();
if (otherIterator != null) {
set1.and(otherIterator);
set2.and(otherSet.iterator());
assertEquals(set1, set2, numBits);
}
}
}
/** Test {@link BitSet#and(DocIdSetIterator)} on sparse sets. */
public void testAndSparse() throws IOException {
testAnd(0.1f);
}
/** Test {@link BitSet#and(DocIdSetIterator)} on dense sets. */
public void testAndDense() throws IOException {
testAnd(0.99f);
}
/** Test {@link BitSet#and(DocIdSetIterator)} on a random density. */
public void testAndRandom() throws IOException {
testAnd(random().nextFloat());
}
private void testAndNot(float load) throws IOException {
final int numBits = 1 + random().nextInt(100000);
BitSet set1 = new JavaUtilBitSet(randomSet(numBits, numBits), numBits); // full
T set2 = copyOf(set1, numBits);
final int iterations = atLeast(10);
for (int iter = 0; iter < iterations; ++iter) {
DocIdSet otherSet = randomCopy(new JavaUtilBitSet(randomSet(numBits, load), numBits), numBits);
DocIdSetIterator otherIterator = otherSet.iterator();
if (otherIterator != null) {
set1.andNot(otherIterator);
set2.andNot(otherSet.iterator());
assertEquals(set1, set2, numBits);
}
}
}
/** Test {@link BitSet#andNot(DocIdSetIterator)} on sparse sets. */
public void testAndNotSparse() throws IOException {
testAndNot(0.01f);
}
/** Test {@link BitSet#andNot(DocIdSetIterator)} on dense sets. */
public void testAndNotDense() throws IOException {
testAndNot(0.9f);
}
/** Test {@link BitSet#andNot(DocIdSetIterator)} on a random density. */
public void testAndNotRandom() throws IOException {
testAndNot(random().nextFloat());
}
private static class JavaUtilBitSet extends BitSet {
private final java.util.BitSet bitSet;

11
solr/core/src/java/org/apache/solr/schema/CurrencyField.java
View File

@ -37,10 +37,10 @@ import org.apache.lucene.analysis.util.ResourceLoader;
import org.apache.lucene.analysis.util.ResourceLoaderAware;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.StorableField;
import org.apache.lucene.queries.BooleanFilter;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.FieldValueQuery;
import org.apache.lucene.search.Filter;
import org.apache.lucene.search.Query;
@ -335,12 +335,11 @@ public class CurrencyField extends FieldType implements SchemaAware, ResourceLoa
p1 == null ? null : p1.getAmount() + "",
p2 == null ? null : p2.getAmount() + "",
minInclusive, maxInclusive);
final BooleanFilter docsInRange = new BooleanFilter();
docsInRange.add(docsWithValues, Occur.MUST);
docsInRange.add(vsRangeFilter, Occur.MUST);
final BooleanQuery docsInRange = new BooleanQuery();
docsInRange.add(docsWithValues, Occur.FILTER);
docsInRange.add(vsRangeFilter, Occur.FILTER);
return new SolrConstantScoreQuery(docsInRange);
return new SolrConstantScoreQuery(new QueryWrapperFilter(docsInRange));
}
@Override