Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-5440: decouple OpenBitSet from DocSet and move to use FixedBitSet 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1568737 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Shai Erera 2014-02-16 07:36:37 +00:00
parent fecedd2349
commit 87f987082b
26 changed files with 552 additions and 418 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

102
lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java
View File

@ -28,6 +28,8 @@ import org.apache.lucene.search.DocIdSetIterator;
* long[], accessed with an int index, implementing {@link Bits} and
* {@link DocIdSet}. If you need to manage more than 2.1B bits, use
* {@link LongBitSet}.
*
* @lucene.internal
*/
public final class FixedBitSet extends DocIdSet implements Bits {
@ -41,6 +43,12 @@ public final class FixedBitSet extends DocIdSet implements Bits {
final long[] bits;
int doc = -1;
/** Creates an iterator over the given {@link FixedBitSet}. */
public FixedBitSetIterator(FixedBitSet bits) {
this(bits.bits, bits.numBits, bits.numWords);
}
/** Creates an iterator over the given array of bits. */
public FixedBitSetIterator(long[] bits, int numBits, int wordLength) {
this.bits = bits;
this.numBits = numBits;
@ -48,7 +56,7 @@ public final class FixedBitSet extends DocIdSet implements Bits {
}
@Override
public int nextDoc() throws IOException {
public int nextDoc() {
if (doc == NO_MORE_DOCS || ++doc >= numBits) {
return doc = NO_MORE_DOCS;
}
@ -81,7 +89,7 @@ public final class FixedBitSet extends DocIdSet implements Bits {
}
@Override
public int advance(int target) throws IOException {
public int advance(int target) {
if (doc == NO_MORE_DOCS || target >= numBits) {
return doc = NO_MORE_DOCS;
}
@ -103,10 +111,6 @@ public final class FixedBitSet extends DocIdSet implements Bits {
return doc = NO_MORE_DOCS;
}
}
private final long[] bits;
private final int numBits;
private final int wordLength;
/**
* If the given {@link FixedBitSet} is large enough to hold {@code numBits},
@ -140,15 +144,53 @@ public final class FixedBitSet extends DocIdSet implements Bits {
return numLong;
}
/**
* Returns the popcount or cardinality of the intersection of the two sets.
* Neither set is modified.
*/
public static long intersectionCount(FixedBitSet a, FixedBitSet b) {
return BitUtil.pop_intersect(a.bits, b.bits, 0, Math.min(a.numWords, b.numWords));
}
/**
* Returns the popcount or cardinality of the union of the two sets. Neither
* set is modified.
*/
public static long unionCount(FixedBitSet a, FixedBitSet b) {
long tot = BitUtil.pop_union(a.bits, b.bits, 0, Math.min(a.numWords, b.numWords));
if (a.numWords < b.numWords) {
tot += BitUtil.pop_array(b.bits, a.numWords, b.numWords - a.numWords);
} else if (a.numWords > b.numWords) {
tot += BitUtil.pop_array(a.bits, b.numWords, a.numWords - b.numWords);
}
return tot;
}
/**
* Returns the popcount or cardinality of "a and not b" or
* "intersection(a, not(b))". Neither set is modified.
*/
public static long andNotCount(FixedBitSet a, FixedBitSet b) {
long tot = BitUtil.pop_andnot(a.bits, b.bits, 0, Math.min(a.numWords, b.numWords));
if (a.numWords > b.numWords) {
tot += BitUtil.pop_array(a.bits, b.numWords, a.numWords - b.numWords);
}
return tot;
}
final long[] bits;
final int numBits;
final int numWords;
public FixedBitSet(int numBits) {
this.numBits = numBits;
bits = new long[bits2words(numBits)];
wordLength = bits.length;
numWords = bits.length;
}
public FixedBitSet(long[] storedBits, int numBits) {
this.wordLength = bits2words(numBits);
if (wordLength > storedBits.length) {
this.numWords = bits2words(numBits);
if (numWords > storedBits.length) {
throw new IllegalArgumentException("The given long array is too small to hold " + numBits + " bits");
}
this.numBits = numBits;
@ -157,7 +199,7 @@ public final class FixedBitSet extends DocIdSet implements Bits {
@Override
public DocIdSetIterator iterator() {
return new FixedBitSetIterator(bits, numBits, wordLength);
return new FixedBitSetIterator(bits, numBits, numWords);
}
@Override
@ -190,7 +232,7 @@ public final class FixedBitSet extends DocIdSet implements Bits {
@Override
public boolean get(int index) {
assert index >= 0 && index < numBits: "index=" + index;
assert index >= 0 && index < numBits: "index=" + index + ", numBits=" + numBits;
int i = index >> 6; // div 64
// signed shift will keep a negative index and force an
// array-index-out-of-bounds-exception, removing the need for an explicit check.
@ -200,7 +242,7 @@ public final class FixedBitSet extends DocIdSet implements Bits {
}
public void set(int index) {
assert index >= 0 && index < numBits: "index=" + index + " numBits=" + numBits;
assert index >= 0 && index < numBits: "index=" + index + ", numBits=" + numBits;
int wordNum = index >> 6; // div 64
int bit = index & 0x3f; // mod 64
long bitmask = 1L << bit;
@ -239,7 +281,7 @@ public final class FixedBitSet extends DocIdSet implements Bits {
* -1 is returned if there are no more set bits.
*/
public int nextSetBit(int index) {
assert index >= 0 && index < numBits;
assert index >= 0 && index < numBits : "index=" + index + ", numBits=" + numBits;
int i = index >> 6;
final int subIndex = index & 0x3f; // index within the word
long word = bits[i] >> subIndex; // skip all the bits to the right of index
@ -248,7 +290,7 @@ public final class FixedBitSet extends DocIdSet implements Bits {
return index + Long.numberOfTrailingZeros(word);
}
while(++i < wordLength) {
while(++i < numWords) {
word = bits[i];
if (word != 0) {
return (i<<6) + Long.numberOfTrailingZeros(word);
@ -306,12 +348,13 @@ public final class FixedBitSet extends DocIdSet implements Bits {
/** this = this OR other */
public void or(FixedBitSet other) {
or(other.bits, other.wordLength);
or(other.bits, other.numWords);
}
private void or(final long[] otherArr, final int otherLen) {
private void or(final long[] otherArr, final int otherNumWords) {
assert otherNumWords <= numWords : "numWords=" + numWords + ", otherNumWords=" + otherNumWords;
final long[] thisArr = this.bits;
int pos = Math.min(wordLength, otherLen);
int pos = Math.min(numWords, otherNumWords);
while (--pos >= 0) {
thisArr[pos] |= otherArr[pos];
}
@ -319,9 +362,10 @@ public final class FixedBitSet extends DocIdSet implements Bits {
/** this = this XOR other */
public void xor(FixedBitSet other) {
assert other.numWords <= numWords : "numWords=" + numWords + ", other.numWords=" + other.numWords;
final long[] thisBits = this.bits;
final long[] otherBits = other.bits;
int pos = Math.min(wordLength, other.wordLength);
int pos = Math.min(numWords, other.numWords);
while (--pos >= 0) {
thisBits[pos] ^= otherBits[pos];
}
@ -366,7 +410,7 @@ public final class FixedBitSet extends DocIdSet implements Bits {
/** returns true if the sets have any elements in common */
public boolean intersects(FixedBitSet other) {
int pos = Math.min(wordLength, other.wordLength);
int pos = Math.min(numWords, other.numWords);
while (--pos>=0) {
if ((bits[pos] & other.bits[pos]) != 0) return true;
}
@ -375,17 +419,17 @@ public final class FixedBitSet extends DocIdSet implements Bits {
/** this = this AND other */
public void and(FixedBitSet other) {
and(other.bits, other.wordLength);
and(other.bits, other.numWords);
}
private void and(final long[] otherArr, final int otherLen) {
private void and(final long[] otherArr, final int otherNumWords) {
final long[] thisArr = this.bits;
int pos = Math.min(this.wordLength, otherLen);
int pos = Math.min(this.numWords, otherNumWords);
while(--pos >= 0) {
thisArr[pos] &= otherArr[pos];
}
if (this.wordLength > otherLen) {
Arrays.fill(thisArr, otherLen, this.wordLength, 0L);
if (this.numWords > otherNumWords) {
Arrays.fill(thisArr, otherNumWords, this.numWords, 0L);
}
}
@ -417,9 +461,9 @@ public final class FixedBitSet extends DocIdSet implements Bits {
andNot(other.bits, other.bits.length);
}
private void andNot(final long[] otherArr, final int otherLen) {
private void andNot(final long[] otherArr, final int otherNumWords) {
final long[] thisArr = this.bits;
int pos = Math.min(this.wordLength, otherLen);
int pos = Math.min(this.numWords, otherNumWords);
while(--pos >= 0) {
thisArr[pos] &= ~otherArr[pos];
}
@ -502,8 +546,8 @@ public final class FixedBitSet extends DocIdSet implements Bits {
* @param endIndex one-past the last bit to clear
*/
public void clear(int startIndex, int endIndex) {
assert startIndex >= 0 && startIndex < numBits;
assert endIndex >= 0 && endIndex <= numBits;
assert startIndex >= 0 && startIndex < numBits : "startIndex=" + startIndex + ", numBits=" + numBits;
assert endIndex >= 0 && endIndex <= numBits : "endIndex=" + endIndex + ", numBits=" + numBits;
if (endIndex <= startIndex) {
return;
}
@ -554,7 +598,7 @@ public final class FixedBitSet extends DocIdSet implements Bits {
@Override
public int hashCode() {
long h = 0;
for (int i = wordLength; --i>=0;) {
for (int i = numWords; --i>=0;) {
h ^= bits[i];
h = (h << 1) | (h >>> 63); // rotate left
}

4
lucene/core/src/java/org/apache/lucene/util/LongBitSet.java
View File

@ -23,6 +23,8 @@ import java.util.Arrays;
* BitSet of fixed length (numBits), backed by accessible ({@link #getBits})
* long[], accessed with a long index. Use it only if you intend to store more
* than 2.1B bits, otherwise you should use {@link FixedBitSet}.
*
* @lucene.internal
*/
public final class LongBitSet {
@ -188,6 +190,7 @@ public final class LongBitSet {
/** this = this OR other */
public void or(LongBitSet other) {
assert other.numWords <= numWords : "numWords=" + numWords + ", other.numWords=" + other.numWords;
int pos = Math.min(numWords, other.numWords);
while (--pos >= 0) {
bits[pos] |= other.bits[pos];
@ -196,6 +199,7 @@ public final class LongBitSet {
/** this = this XOR other */
public void xor(LongBitSet other) {
assert other.numWords <= numWords : "numWords=" + numWords + ", other.numWords=" + other.numWords;
int pos = Math.min(numWords, other.numWords);
while (--pos >= 0) {
bits[pos] ^= other.bits[pos];

2
lucene/grouping/src/java/org/apache/lucene/search/grouping/AbstractAllGroupHeadsCollector.java
View File

@ -44,7 +44,7 @@ public abstract class AbstractAllGroupHeadsCollector<GH extends AbstractAllGroup
/**
* @param maxDoc The maxDoc of the top level {@link IndexReader}.
* @return an {@link FixedBitSet} containing all group heads.
* @return a {@link FixedBitSet} containing all group heads.
*/
public FixedBitSet retrieveGroupHeads(int maxDoc) {
FixedBitSet bitSet = new FixedBitSet(maxDoc);

4
solr/CHANGES.txt
View File

@ -362,6 +362,10 @@ Optimizations
* SOLR-5624: Enable QueryResultCache for CollapsingQParserPlugin.
(David Boychuck, Joel Bernstein)
* LUCENE-5440: DocSet decoupled from OpenBitSet. DocSetBase moved to use
FixedBitSet instead of OpenBitSet. As a result BitDocSet now only works
with FixedBitSet. (Shai Erera)
Other Changes
---------------------

32
solr/core/src/java/org/apache/solr/handler/component/FacetComponent.java
View File

@ -17,7 +17,17 @@
package org.apache.solr.handler.component;
import org.apache.lucene.util.OpenBitSet;
import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import org.apache.lucene.util.FixedBitSet;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.params.CommonParams;
@ -35,10 +45,6 @@ import org.apache.solr.search.SyntaxError;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.net.URL;
import java.util.*;
/**
* TODO!
*
@ -444,8 +450,8 @@ public class FacetComponent extends SearchComponent
// and if it is >= smallestCount, then flag for refinement
long maxCount = sfc.count;
for (int shardNum=0; shardNum<rb.shards.length; shardNum++) {
OpenBitSet obs = dff.counted[shardNum];
if (obs!=null && !obs.get(sfc.termNum)) { // obs can be null if a shard request failed
FixedBitSet fbs = dff.counted[shardNum];
if (fbs!=null && !fbs.get(sfc.termNum)) { // fbs can be null if a shard request failed
// if missing from this shard, add the max it could be
maxCount += dff.maxPossible(sfc,shardNum);
}
@ -459,8 +465,8 @@ public class FacetComponent extends SearchComponent
if (needRefinement) {
// add a query for each shard missing the term that needs refinement
for (int shardNum=0; shardNum<rb.shards.length; shardNum++) {
OpenBitSet obs = dff.counted[shardNum];
if(obs!=null && !obs.get(sfc.termNum) && dff.maxPossible(sfc,shardNum)>0) {
FixedBitSet fbs = dff.counted[shardNum];
if(fbs!=null && !fbs.get(sfc.termNum) && dff.maxPossible(sfc,shardNum)>0) {
dff.needRefinements = true;
List<String> lst = dff._toRefine[shardNum];
if (lst == null) {
@ -759,7 +765,7 @@ public class FacetComponent extends SearchComponent
public long missingMaxPossible;
// the max possible count for a missing term for each shard (indexed by shardNum)
public long[] missingMax;
public OpenBitSet[] counted; // a bitset for each shard, keeping track of which terms seen
public FixedBitSet[] counted; // a bitset for each shard, keeping track of which terms seen
public HashMap<String,ShardFacetCount> counts = new HashMap<String,ShardFacetCount>(128);
public int termNum;
@ -772,7 +778,7 @@ public class FacetComponent extends SearchComponent
super(rb, facetStr);
// sf = rb.req.getSchema().getField(field);
missingMax = new long[rb.shards.length];
counted = new OpenBitSet[rb.shards.length];
counted = new FixedBitSet[rb.shards.length];
}
void add(int shardNum, NamedList shardCounts, int numRequested) {
@ -780,7 +786,7 @@ public class FacetComponent extends SearchComponent
int sz = shardCounts == null ? 0 : shardCounts.size();
int numReceived = sz;
OpenBitSet terms = new OpenBitSet(termNum+sz);
FixedBitSet terms = new FixedBitSet(termNum+sz);
long last = 0;
for (int i=0; i<sz; i++) {
@ -799,7 +805,7 @@ public class FacetComponent extends SearchComponent
counts.put(name, sfc);
}
sfc.count += count;
terms.fastSet(sfc.termNum);
terms.set(sfc.termNum);
last = count;
}
}

47
solr/core/src/java/org/apache/solr/request/SimpleFacets.java
View File

@ -17,6 +17,26 @@
package org.apache.solr.request;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.EnumSet;
import java.util.IdentityHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executor;
import java.util.concurrent.Future;
import java.util.concurrent.FutureTask;
import java.util.concurrent.RunnableFuture;
import java.util.concurrent.Semaphore;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.Fields;
@ -37,8 +57,6 @@ import org.apache.lucene.search.grouping.term.TermAllGroupsCollector;
import org.apache.lucene.search.grouping.term.TermGroupFacetCollector;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.OpenBitSet;
import org.apache.lucene.util.StringHelper;
import org.apache.lucene.util.UnicodeUtil;
import org.apache.solr.common.SolrException;
@ -80,26 +98,6 @@ import org.apache.solr.util.DateMathParser;
import org.apache.solr.util.DefaultSolrThreadFactory;
import org.apache.solr.util.LongPriorityQueue;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.EnumSet;
import java.util.IdentityHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executor;
import java.util.concurrent.Future;
import java.util.concurrent.FutureTask;
import java.util.concurrent.RunnableFuture;
import java.util.concurrent.Semaphore;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
/**
* A class that generates simple Facet information for a request.
*
@ -230,10 +228,7 @@ public class SimpleFacets {
}
AbstractAllGroupHeadsCollector allGroupHeadsCollector = grouping.getCommands().get(0).createAllGroupCollector();
searcher.search(new MatchAllDocsQuery(), base.getTopFilter(), allGroupHeadsCollector);
int maxDoc = searcher.maxDoc();
FixedBitSet fixedBitSet = allGroupHeadsCollector.retrieveGroupHeads(maxDoc);
long[] bits = fixedBitSet.getBits();
this.docs = new BitDocSet(new OpenBitSet(bits, bits.length));
this.docs = new BitDocSet(allGroupHeadsCollector.retrieveGroupHeads(searcher.maxDoc()));
} else {
this.docs = base;
}

13
solr/core/src/java/org/apache/solr/request/UnInvertedField.java
View File

@ -27,12 +27,11 @@ import org.apache.lucene.index.DocTermOrds;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TermRangeQuery;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.OpenBitSet;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.UnicodeUtil;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.FacetParams;
@ -44,7 +43,11 @@ import org.apache.solr.handler.component.StatsValuesFactory;
import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.schema.TrieField;
import org.apache.solr.search.*;
import org.apache.solr.search.BitDocSet;
import org.apache.solr.search.DocIterator;
import org.apache.solr.search.DocSet;
import org.apache.solr.search.SolrCache;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.util.LongPriorityQueue;
import org.apache.solr.util.PrimUtils;
@ -269,7 +272,7 @@ public class UnInvertedField extends DocTermOrds {
&& docs instanceof BitDocSet;
if (doNegative) {
OpenBitSet bs = (OpenBitSet)((BitDocSet)docs).getBits().clone();
FixedBitSet bs = ((BitDocSet)docs).getBits().clone();
bs.flip(0, maxDoc);
// TODO: when iterator across negative elements is available, use that
// instead of creating a new bitset and inverting.
@ -512,7 +515,7 @@ public class UnInvertedField extends DocTermOrds {
}
if (doNegative) {
OpenBitSet bs = (OpenBitSet) ((BitDocSet) docs).getBits().clone();
FixedBitSet bs = ((BitDocSet) docs).getBits().clone();
bs.flip(0, maxDoc);
// TODO: when iterator across negative elements is available, use that
// instead of creating a new bitset and inverting.

140
solr/core/src/java/org/apache/solr/search/BitDocSet.java
View File

@ -21,38 +21,40 @@ import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.search.BitsFilteredDocIdSet;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Filter;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.OpenBitSet;
import org.apache.lucene.util.OpenBitSetIterator;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.FixedBitSet.FixedBitSetIterator;
/**
* <code>BitDocSet</code> represents an unordered set of Lucene Document Ids
* using a BitSet. A set bit represents inclusion in the set for that document.
*
*
* @since solr 0.9
*/
public class BitDocSet extends DocSetBase {
final OpenBitSet bits;
final FixedBitSet bits;
int size; // number of docs in the set (cached for perf)
public BitDocSet() {
bits = new OpenBitSet();
bits = new FixedBitSet(64);
}
/** Construct a BitDocSet.
* The capacity of the OpenBitSet should be at least maxDoc() */
public BitDocSet(OpenBitSet bits) {
/**
* Construct a BitDocSet. The capacity of the {@link FixedBitSet} should be at
* least maxDoc()
*/
public BitDocSet(FixedBitSet bits) {
this.bits = bits;
size=-1;
}
/** Construct a BitDocSet, and provides the number of set bits.
* The capacity of the OpenBitSet should be at least maxDoc()
/**
* Construct a BitDocSet, and provides the number of set bits. The capacity of
* the {@link FixedBitSet} should be at least maxDoc()
*/
public BitDocSet(OpenBitSet bits, int size) {
public BitDocSet(FixedBitSet bits, int size) {
this.bits = bits;
this.size = size;
}
@ -89,7 +91,7 @@ public class BitDocSet extends DocSetBase {
@Override
public DocIterator iterator() {
return new DocIterator() {
private final OpenBitSetIterator iter = new OpenBitSetIterator(bits);
private final FixedBitSetIterator iter = new FixedBitSetIterator(bits);
private int pos = iter.nextDoc();
@Override
public boolean hasNext() {
@ -120,13 +122,11 @@ public class BitDocSet extends DocSetBase {
};
}
/**
*
* @return the <b>internal</b> OpenBitSet that should <b>not</b> be modified.
* @return the <b>internal</b> {@link FixedBitSet} that should <b>not</b> be modified.
*/
@Override
public OpenBitSet getBits() {
public FixedBitSet getBits() {
return bits;
}
@ -145,7 +145,7 @@ public class BitDocSet extends DocSetBase {
@Override
public int size() {
if (size!=-1) return size;
return size=(int)bits.cardinality();
return size = bits.cardinality();
}
/**
@ -156,18 +156,19 @@ public class BitDocSet extends DocSetBase {
size=-1;
}
/** Returns true of the doc exists in the set.
* Should only be called when doc < OpenBitSet.size()
/**
* Returns true of the doc exists in the set. Should only be called when doc <
* {@link FixedBitSet#length()}.
*/
@Override
public boolean exists(int doc) {
return bits.fastGet(doc);
return bits.get(doc);
}
@Override
public int intersectionSize(DocSet other) {
if (other instanceof BitDocSet) {
return (int)OpenBitSet.intersectionCount(this.bits, ((BitDocSet)other).bits);
return (int) FixedBitSet.intersectionCount(this.bits, ((BitDocSet) other).bits);
} else {
// they had better not call us back!
return other.intersectionSize(this);
@ -189,7 +190,7 @@ public class BitDocSet extends DocSetBase {
if (other instanceof BitDocSet) {
// if we don't know our current size, this is faster than
// size + other.size - intersection_size
return (int)OpenBitSet.unionCount(this.bits, ((BitDocSet)other).bits);
return (int) FixedBitSet.unionCount(this.bits, ((BitDocSet)other).bits);
} else {
// they had better not call us back!
return other.unionSize(this);
@ -201,42 +202,56 @@ public class BitDocSet extends DocSetBase {
if (other instanceof BitDocSet) {
// if we don't know our current size, this is faster than
// size - intersection_size
return (int)OpenBitSet.andNotCount(this.bits, ((BitDocSet)other).bits);
return (int) FixedBitSet.andNotCount(this.bits, ((BitDocSet)other).bits);
} else {
return super.andNotSize(other);
}
}
@Override
public void setBitsOn(OpenBitSet target) {
target.union(bits);
public void addAllTo(DocSet target) {
if (target instanceof BitDocSet) {
((BitDocSet) target).bits.or(bits);
} else {
super.addAllTo(target);
}
}
@Override
public DocSet andNot(DocSet other) {
OpenBitSet newbits = (OpenBitSet)(bits.clone());
if (other instanceof BitDocSet) {
newbits.andNot(((BitDocSet)other).bits);
} else {
DocIterator iter = other.iterator();
while (iter.hasNext()) newbits.clear(iter.nextDoc());
}
return new BitDocSet(newbits);
public DocSet andNot(DocSet other) {
FixedBitSet newbits = bits.clone();
if (other instanceof BitDocSet) {
newbits.andNot(((BitDocSet) other).bits);
} else {
DocIterator iter = other.iterator();
while (iter.hasNext()) {
int doc = iter.nextDoc();
if (doc < newbits.length()) {
newbits.clear(doc);
}
}
}
return new BitDocSet(newbits);
}
@Override
public DocSet union(DocSet other) {
OpenBitSet newbits = (OpenBitSet)(bits.clone());
if (other instanceof BitDocSet) {
newbits.union(((BitDocSet)other).bits);
} else {
DocIterator iter = other.iterator();
while (iter.hasNext()) newbits.set(iter.nextDoc());
}
return new BitDocSet(newbits);
public DocSet union(DocSet other) {
FixedBitSet newbits = bits.clone();
if (other instanceof BitDocSet) {
BitDocSet otherDocSet = (BitDocSet) other;
newbits = FixedBitSet.ensureCapacity(newbits, otherDocSet.bits.length());
newbits.or(otherDocSet.bits);
} else {
DocIterator iter = other.iterator();
while (iter.hasNext()) {
int doc = iter.nextDoc();
newbits = FixedBitSet.ensureCapacity(newbits, doc);
newbits.set(doc);
}
}
return new BitDocSet(newbits);
}
@Override
public long memSize() {
return (bits.getBits().length << 3) + 16;
@ -244,12 +259,12 @@ public class BitDocSet extends DocSetBase {
@Override
protected BitDocSet clone() {
return new BitDocSet((OpenBitSet)bits.clone(), size);
return new BitDocSet(bits.clone(), size);
}
@Override
public Filter getTopFilter() {
final OpenBitSet bs = bits;
final FixedBitSet bs = bits;
// TODO: if cardinality isn't cached, do a quick measure of sparseness
// and return null from bits() if too sparse.
@ -272,8 +287,8 @@ public class BitDocSet extends DocSetBase {
@Override
public DocIdSetIterator iterator() {
return new DocIdSetIterator() {
int pos=base-1;
int adjustedDoc=-1;
int pos = base - 1;
int adjustedDoc = -1;
@Override
public int docID() {
@ -282,15 +297,24 @@ public class BitDocSet extends DocSetBase {
@Override
public int nextDoc() {
pos = bs.nextSetBit(pos+1);
return adjustedDoc = (pos>=0 && pos<max) ? pos-base : NO_MORE_DOCS;
if (pos >= bs.length() - 1) {
return adjustedDoc = NO_MORE_DOCS;
} else {
pos = bs.nextSetBit(pos + 1);
return adjustedDoc = (pos >= 0 && pos < max) ? pos - base : NO_MORE_DOCS;
}
}
@Override
public int advance(int target) {
if (target==NO_MORE_DOCS) return adjustedDoc=NO_MORE_DOCS;
pos = bs.nextSetBit(target+base);
return adjustedDoc = (pos>=0 && pos<max) ? pos-base : NO_MORE_DOCS;
if (target == NO_MORE_DOCS) return adjustedDoc = NO_MORE_DOCS;
int adjusted = target + base;
if (adjusted >= bs.length()) {
return adjustedDoc = NO_MORE_DOCS;
} else {
pos = bs.nextSetBit(adjusted);
return adjustedDoc = (pos >= 0 && pos < max) ? pos - base : NO_MORE_DOCS;
}
}
@Override
@ -298,7 +322,7 @@ public class BitDocSet extends DocSetBase {
// we don't want to actually compute cardinality, but
// if its already been computed, we use it (pro-rated for the segment)
if (size != -1) {
return (long)(size * ((OpenBitSet.bits2words(maxDoc)<<6) / (float)bs.capacity()));
return (long)(size * ((FixedBitSet.bits2words(maxDoc)<<6) / (float)bs.length()));
} else {
return maxDoc;
}
@ -316,7 +340,7 @@ public class BitDocSet extends DocSetBase {
return new Bits() {
@Override
public boolean get(int index) {
return bs.fastGet(index + base);
return bs.get(index + base);
}
@Override

100
solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
View File

@ -17,45 +17,49 @@
package org.apache.solr.search;
import org.apache.lucene.queries.function.FunctionQuery;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.util.BytesRef;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrRequestInfo;
import org.apache.solr.schema.TrieFloatField;
import org.apache.solr.schema.TrieIntField;
import org.apache.solr.schema.TrieLongField;
import org.apache.solr.schema.FieldType;
import org.apache.solr.handler.component.QueryElevationComponent;
import org.apache.lucene.index.AtomicReaderContext;
import java.io.IOException;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.search.*;
import org.apache.lucene.util.OpenBitSet;
import org.apache.lucene.queries.function.FunctionQuery;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.handler.component.QueryElevationComponent;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestInfo;
import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.schema.TrieFloatField;
import org.apache.solr.schema.TrieIntField;
import org.apache.solr.schema.TrieLongField;
import com.carrotsearch.hppc.FloatArrayList;
import com.carrotsearch.hppc.IntOpenHashSet;
import com.carrotsearch.hppc.cursors.IntCursor;
import java.io.IOException;
import java.util.Arrays;
import java.util.Map;
import java.util.Set;
import java.util.HashSet;
import java.util.List;
import java.util.Iterator;
/**
The <b>CollapsingQParserPlugin</b> is a PostFilter that performs field collapsing.
@ -419,7 +423,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
private class CollapsingScoreCollector extends DelegatingCollector {
private AtomicReaderContext[] contexts;
private OpenBitSet collapsedSet;
private FixedBitSet collapsedSet;
private SortedDocValues values;
private int[] ords;
private float[] scores;
@ -438,14 +442,14 @@ public class CollapsingQParserPlugin extends QParserPlugin {
IntOpenHashSet boostDocs) {
this.maxDoc = maxDoc;
this.contexts = new AtomicReaderContext[segments];
this.collapsedSet = new OpenBitSet(maxDoc);
this.collapsedSet = new FixedBitSet(maxDoc);
this.boostDocs = boostDocs;
if(this.boostDocs != null) {
//Set the elevated docs now.
Iterator<IntCursor> it = this.boostDocs.iterator();
while(it.hasNext()) {
IntCursor cursor = it.next();
this.collapsedSet.fastSet(cursor.value);
this.collapsedSet.set(cursor.value);
}
}
this.values = values;
@ -460,16 +464,19 @@ public class CollapsingQParserPlugin extends QParserPlugin {
}
}
@Override
public boolean acceptsDocsOutOfOrder() {
//Documents must be sent in order to this collector.
return false;
}
@Override
public void setNextReader(AtomicReaderContext context) throws IOException {
this.contexts[context.ord] = context;
this.docBase = context.docBase;
}
@Override
public void collect(int docId) throws IOException {
int globalDoc = docId+this.docBase;
int ord = values.getOrd(globalDoc);
@ -479,7 +486,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
ords[ord] = globalDoc;
scores[ord] = score;
}
} else if (this.collapsedSet.fastGet(globalDoc)) {
} else if (this.collapsedSet.get(globalDoc)) {
//The doc is elevated so score does not matter
//We just want to be sure it doesn't fall into the null policy
} else if(nullPolicy == CollapsingPostFilter.NULL_POLICY_COLLAPSE) {
@ -489,24 +496,25 @@ public class CollapsingQParserPlugin extends QParserPlugin {
nullDoc = globalDoc;
}
} else if(nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) {
collapsedSet.fastSet(globalDoc);
collapsedSet.set(globalDoc);
nullScores.add(scorer.score());
}
}
@Override
public void finish() throws IOException {
if(contexts.length == 0) {
return;
}
if(nullScore > 0) {
this.collapsedSet.fastSet(nullDoc);
this.collapsedSet.set(nullDoc);
}
for(int i=0; i<ords.length; i++) {
int doc = ords[i];
if(doc > -1) {
collapsedSet.fastSet(doc);
collapsedSet.set(doc);
}
}
@ -677,7 +685,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
protected FloatArrayList nullScores;
protected float nullScore;
protected float[] scores;
protected OpenBitSet collapsedSet;
protected FixedBitSet collapsedSet;
protected IntOpenHashSet boostDocs;
protected int nullDoc = -1;
protected boolean needsScores;
@ -697,26 +705,26 @@ public class CollapsingQParserPlugin extends QParserPlugin {
this.nullPolicy = nullPolicy;
this.max = max;
this.needsScores = needsScores;
this.collapsedSet = new OpenBitSet(maxDoc);
this.collapsedSet = new FixedBitSet(maxDoc);
this.boostDocs = boostDocs;
if(this.boostDocs != null) {
Iterator<IntCursor> it = boostDocs.iterator();
while(it.hasNext()) {
IntCursor cursor = it.next();
this.collapsedSet.fastSet(cursor.value);
this.collapsedSet.set(cursor.value);
}
}
}
public OpenBitSet getCollapsedSet() {
public FixedBitSet getCollapsedSet() {
if(nullDoc > -1) {
this.collapsedSet.fastSet(nullDoc);
this.collapsedSet.set(nullDoc);
}
for(int i=0; i<ords.length; i++) {
int doc = ords[i];
if(doc > -1) {
collapsedSet.fastSet(doc);
collapsedSet.set(doc);
}
}
@ -790,7 +798,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
scores[ord] = scorer.score();
}
}
} else if(this.collapsedSet.fastGet(globalDoc)) {
} else if(this.collapsedSet.get(globalDoc)) {
// Elevated doc so do nothing.
} else if(this.nullPolicy == CollapsingPostFilter.NULL_POLICY_COLLAPSE) {
if(comp.test(val, nullVal)) {
@ -801,7 +809,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
}
}
} else if(this.nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) {
this.collapsedSet.fastSet(globalDoc);
this.collapsedSet.set(globalDoc);
if(needsScores) {
nullScores.add(scorer.score());
}
@ -858,7 +866,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
scores[ord] = scorer.score();
}
}
} else if (this.collapsedSet.fastGet(globalDoc)) {
} else if (this.collapsedSet.get(globalDoc)) {
//Elevated doc so do nothing
} else if(this.nullPolicy == CollapsingPostFilter.NULL_POLICY_COLLAPSE) {
if(comp.test(val, nullVal)) {
@ -869,7 +877,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
}
}
} else if(this.nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) {
this.collapsedSet.fastSet(globalDoc);
this.collapsedSet.set(globalDoc);
if(needsScores) {
nullScores.add(scorer.score());
}
@ -927,7 +935,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
scores[ord] = scorer.score();
}
}
} else if (this.collapsedSet.fastGet(globalDoc)) {
} else if (this.collapsedSet.get(globalDoc)) {
//Elevated doc so do nothing
} else if(this.nullPolicy == CollapsingPostFilter.NULL_POLICY_COLLAPSE) {
if(comp.test(val, nullVal)) {
@ -938,7 +946,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
}
}
} else if(this.nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) {
this.collapsedSet.fastSet(globalDoc);
this.collapsedSet.set(globalDoc);
if(needsScores) {
nullScores.add(scorer.score());
}
@ -1015,7 +1023,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
scores[ord] = score;
}
}
} else if (this.collapsedSet.fastGet(globalDoc)) {
} else if (this.collapsedSet.get(globalDoc)) {
//Elevated doc so do nothing
} else if(this.nullPolicy == CollapsingPostFilter.NULL_POLICY_COLLAPSE) {
if(comp.test(val, nullVal)) {
@ -1026,7 +1034,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
}
}
} else if(this.nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) {
this.collapsedSet.fastSet(globalDoc);
this.collapsedSet.set(globalDoc);
if(needsScores) {
nullScores.add(score);
}

19
solr/core/src/java/org/apache/solr/search/DocSet.java
View File

@ -18,7 +18,6 @@
package org.apache.solr.search;
import org.apache.lucene.search.Filter;
import org.apache.lucene.util.OpenBitSet;
import org.apache.solr.common.SolrException;
/**
@ -29,7 +28,6 @@ import org.apache.solr.common.SolrException;
* a cache and could be shared.
* </p>
*
*
* @since solr 0.9
*/
public interface DocSet /* extends Collection<Integer> */ {
@ -77,16 +75,6 @@ public interface DocSet /* extends Collection<Integer> */ {
*/
public DocIterator iterator();
/**
* Returns a BitSet view of the DocSet. Any changes to this BitSet <b>may</b>
* be reflected in the DocSet, hence if the DocSet is shared or was returned from
* a SolrIndexSearcher method, it's not safe to modify the BitSet.
*
* @return
* An OpenBitSet with the bit number of every docid set in the set.
*/
public OpenBitSet getBits();
/**
* Returns the approximate amount of memory taken by this DocSet.
* This is only an approximation and doesn't take into account java object overhead.
@ -145,10 +133,11 @@ public interface DocSet /* extends Collection<Integer> */ {
public Filter getTopFilter();
/**
* Takes the docs from this set and sets those bits on the target OpenBitSet.
* The target should be sized large enough to accommodate all of the documents before calling this method.
* Adds all the docs from this set to the target set. The target should be
* sized large enough to accommodate all of the documents before calling this
* method.
*/
public void setBitsOn(OpenBitSet target);
public void addAllTo(DocSet target);
public static DocSet EMPTY = new SortedIntDocSet(new int[0], 0);
}

70
solr/core/src/java/org/apache/solr/search/DocSetBase.java
View File

@ -18,18 +18,32 @@
package org.apache.solr.search;
import org.apache.lucene.index.AtomicReader;
import org.apache.solr.common.SolrException;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.OpenBitSet;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.Filter;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.BitsFilteredDocIdSet;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.search.BitsFilteredDocIdSet;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Filter;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitSet;
import org.apache.solr.common.SolrException;
/** A base class that may be usefull for implementing DocSets */
abstract class DocSetBase implements DocSet {
public static FixedBitSet toBitSet(DocSet set) {
if (set instanceof DocSetBase) {
return ((DocSetBase) set).getBits();
} else {
FixedBitSet bits = new FixedBitSet(64);
for (DocIterator iter = set.iterator(); iter.hasNext();) {
int nextDoc = iter.nextDoc();
bits = FixedBitSet.ensureCapacity(bits, nextDoc);
bits.set(nextDoc);
}
return bits;
}
}
// Not implemented efficiently... for testing purposes only
@Override
public boolean equals(Object obj) {
@ -49,7 +63,7 @@ abstract class DocSetBase implements DocSet {
}
// if (this.size() != other.size()) return false;
return this.getBits().equals(other.getBits());
return this.getBits().equals(toBitSet(other));
}
/**
@ -69,18 +83,20 @@ abstract class DocSetBase implements DocSet {
}
/**
* Inefficient base implementation.
*
* @see BitDocSet#getBits
* Return a {@link FixedBitSet} with a bit set for every document in this
* {@link DocSet}. The default implementation iterates on all docs and sets
* the relevant bits. You should override if you can provide a more efficient
* implementation.
*/
@Override
public OpenBitSet getBits() {
OpenBitSet bits = new OpenBitSet();
protected FixedBitSet getBits() {
FixedBitSet bits = new FixedBitSet(64);
for (DocIterator iter = iterator(); iter.hasNext();) {
bits.set(iter.nextDoc());
int nextDoc = iter.nextDoc();
bits = FixedBitSet.ensureCapacity(bits, nextDoc);
bits.set(nextDoc);
}
return bits;
};
}
@Override
public DocSet intersection(DocSet other) {
@ -91,8 +107,8 @@ abstract class DocSetBase implements DocSet {
}
// Default... handle with bitsets.
OpenBitSet newbits = (OpenBitSet)(this.getBits().clone());
newbits.and(other.getBits());
FixedBitSet newbits = getBits().clone();
newbits.and(toBitSet(other));
return new BitDocSet(newbits);
}
@ -107,11 +123,11 @@ abstract class DocSetBase implements DocSet {
return intersectionSize(other) > 0;
}
@Override
public DocSet union(DocSet other) {
OpenBitSet newbits = (OpenBitSet)(this.getBits().clone());
newbits.or(other.getBits());
FixedBitSet otherBits = toBitSet(other);
FixedBitSet newbits = FixedBitSet.ensureCapacity(getBits().clone(), otherBits.length());
newbits.or(otherBits);
return new BitDocSet(newbits);
}
@ -133,8 +149,8 @@ abstract class DocSetBase implements DocSet {
@Override
public DocSet andNot(DocSet other) {
OpenBitSet newbits = (OpenBitSet)(this.getBits().clone());
newbits.andNot(other.getBits());
FixedBitSet newbits = getBits().clone();
newbits.andNot(toBitSet(other));
return new BitDocSet(newbits);
}
@ -145,7 +161,7 @@ abstract class DocSetBase implements DocSet {
@Override
public Filter getTopFilter() {
final OpenBitSet bs = getBits();
final FixedBitSet bs = getBits();
return new Filter() {
@Override
@ -189,7 +205,7 @@ abstract class DocSetBase implements DocSet {
@Override
public long cost() {
return bs.capacity();
return bs.length();
}
};
}
@ -211,10 +227,10 @@ abstract class DocSetBase implements DocSet {
}
@Override
public void setBitsOn(OpenBitSet target) {
public void addAllTo(DocSet target) {
DocIterator iter = iterator();
while (iter.hasNext()) {
target.fastSet(iter.nextDoc());
target.add(iter.nextDoc());
}
}

15
solr/core/src/java/org/apache/solr/search/DocSetCollector.java
View File

@ -17,13 +17,12 @@ package org.apache.solr.search;
* limitations under the License.
*/
import org.apache.lucene.index.IndexReader;
import java.io.IOException;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.search.Collector;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.util.OpenBitSet;
import java.io.IOException;
import org.apache.lucene.util.FixedBitSet;
/**
*
@ -31,7 +30,7 @@ import java.io.IOException;
public class DocSetCollector extends Collector {
int pos=0;
OpenBitSet bits;
FixedBitSet bits;
final int maxDoc;
final int smallSetSize;
int base;
@ -62,8 +61,8 @@ public class DocSetCollector extends Collector {
} else {
// this conditional could be removed if BitSet was preallocated, but that
// would take up more memory, and add more GC time...
if (bits==null) bits = new OpenBitSet(maxDoc);
bits.fastSet(doc);
if (bits==null) bits = new FixedBitSet(maxDoc);
bits.set(doc);
}
pos++;
@ -75,7 +74,7 @@ public class DocSetCollector extends Collector {
return new SortedIntDocSet(scratch, pos);
} else {
// set the bits for ids that were collected in the array
for (int i=0; i<scratch.length; i++) bits.fastSet(scratch[i]);
for (int i=0; i<scratch.length; i++) bits.set(scratch[i]);
return new BitDocSet(bits,pos);
}
}

13
solr/core/src/java/org/apache/solr/search/DocSetDelegateCollector.java
View File

@ -17,13 +17,12 @@ package org.apache.solr.search;
* limitations under the License.
*/
import org.apache.lucene.index.IndexReader;
import java.io.IOException;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.search.Collector;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.util.OpenBitSet;
import java.io.IOException;
import org.apache.lucene.util.FixedBitSet;
/**
*
@ -53,8 +52,8 @@ public class DocSetDelegateCollector extends DocSetCollector {
} else {
// this conditional could be removed if BitSet was preallocated, but that
// would take up more memory, and add more GC time...
if (bits==null) bits = new OpenBitSet(maxDoc);
bits.fastSet(doc);
if (bits==null) bits = new FixedBitSet(maxDoc);
bits.set(doc);
}
pos++;
@ -67,7 +66,7 @@ public class DocSetDelegateCollector extends DocSetCollector {
return new SortedIntDocSet(scratch, pos);
} else {
// set the bits for ids that were collected in the array
for (int i=0; i<scratch.length; i++) bits.fastSet(scratch[i]);
for (int i=0; i<scratch.length; i++) bits.set(scratch[i]);
return new BitDocSet(bits,pos);
}
}

39
solr/core/src/java/org/apache/solr/search/Grouping.java
View File

@ -17,13 +17,38 @@
package org.apache.solr.search;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import org.apache.commons.lang.ArrayUtils;
import org.apache.lucene.index.StorableField;
import org.apache.lucene.queries.function.FunctionQuery;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.valuesource.QueryValueSource;
import org.apache.lucene.search.*;
import org.apache.lucene.search.grouping.*;
import org.apache.lucene.search.CachingCollector;
import org.apache.lucene.search.Collector;
import org.apache.lucene.search.Filter;
import org.apache.lucene.search.MultiCollector;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TimeLimitingCollector;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.TopDocsCollector;
import org.apache.lucene.search.TopFieldCollector;
import org.apache.lucene.search.TopScoreDocCollector;
import org.apache.lucene.search.TotalHitCountCollector;
import org.apache.lucene.search.grouping.AbstractAllGroupHeadsCollector;
import org.apache.lucene.search.grouping.GroupDocs;
import org.apache.lucene.search.grouping.SearchGroup;
import org.apache.lucene.search.grouping.TopGroups;
import org.apache.lucene.search.grouping.function.FunctionAllGroupHeadsCollector;
import org.apache.lucene.search.grouping.function.FunctionAllGroupsCollector;
import org.apache.lucene.search.grouping.function.FunctionFirstPassGroupingCollector;
@ -33,8 +58,6 @@ import org.apache.lucene.search.grouping.term.TermAllGroupsCollector;
import org.apache.lucene.search.grouping.term.TermFirstPassGroupingCollector;
import org.apache.lucene.search.grouping.term.TermSecondPassGroupingCollector;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.OpenBitSet;
import org.apache.lucene.util.mutable.MutableValue;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.util.NamedList;
@ -47,9 +70,6 @@ import org.apache.solr.search.grouping.collector.FilterCollector;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.*;
/**
* Basic Solr Grouping infrastructure.
* Warning NOT thread save!
@ -353,10 +373,7 @@ public class Grouping {
}
if (getGroupedDocSet && allGroupHeadsCollector != null) {
FixedBitSet fixedBitSet = allGroupHeadsCollector.retrieveGroupHeads(maxDoc);
long[] bits = fixedBitSet.getBits();
OpenBitSet openBitSet = new OpenBitSet(bits, bits.length);
qr.setDocSet(new BitDocSet(openBitSet));
qr.setDocSet(new BitDocSet(allGroupHeadsCollector.retrieveGroupHeads(maxDoc)));
} else if (getDocSet) {
qr.setDocSet(setCollector.getDocSet());
}

51
solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java
View File

@ -16,12 +16,33 @@
*/
package org.apache.solr.search;
import org.apache.lucene.index.*;
import org.apache.lucene.search.*;
import java.io.Closeable;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Set;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.Fields;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.MultiDocsEnum;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.ComplexExplanation;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.Filter;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.Weight;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.OpenBitSet;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.StringHelper;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.SolrParams;
@ -36,13 +57,6 @@ import org.apache.solr.request.SolrRequestInfo;
import org.apache.solr.schema.TrieField;
import org.apache.solr.util.RefCounted;
import java.io.Closeable;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Set;
public class JoinQParserPlugin extends QParserPlugin {
public static final String NAME = "join";
@ -270,7 +284,7 @@ class JoinQuery extends Query {
public DocSet getDocSet() throws IOException {
OpenBitSet resultBits = null;
FixedBitSet resultBits = null;
// minimum docFreq to use the cache
int minDocFreqFrom = Math.max(5, fromSearcher.maxDoc() >> 13);
@ -387,7 +401,7 @@ class JoinQuery extends Query {
int df = toTermsEnum.docFreq();
toTermHitsTotalDf += df;
if (resultBits==null && df + resultListDocs > maxSortedIntSize && resultList.size() > 0) {
resultBits = new OpenBitSet(toSearcher.maxDoc());
resultBits = new FixedBitSet(toSearcher.maxDoc());
}
// if we don't have a bitset yet, or if the resulting set will be too large
@ -397,10 +411,10 @@ class JoinQuery extends Query {
DocSet toTermSet = toSearcher.getDocSet(toDeState);
resultListDocs += toTermSet.size();
if (resultBits != null) {
toTermSet.setBitsOn(resultBits);
toTermSet.addAllTo(new BitDocSet(resultBits));
} else {
if (toTermSet instanceof BitDocSet) {
resultBits = (OpenBitSet)((BitDocSet)toTermSet).bits.clone();
resultBits = ((BitDocSet)toTermSet).bits.clone();
} else {
resultList.add(toTermSet);
}
@ -422,14 +436,14 @@ class JoinQuery extends Query {
int docid;
while ((docid = sub.docsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
resultListDocs++;
resultBits.fastSet(docid + base);
resultBits.set(docid + base);
}
}
} else {
int docid;
while ((docid = docsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
resultListDocs++;
resultBits.fastSet(docid);
resultBits.set(docid);
}
}
}
@ -443,10 +457,11 @@ class JoinQuery extends Query {
smallSetsDeferred = resultList.size();
if (resultBits != null) {
BitDocSet bitSet = new BitDocSet(resultBits);
for (DocSet set : resultList) {
set.setBitsOn(resultBits);
set.addAllTo(bitSet);
}
return new BitDocSet(resultBits);
return bitSet;
}
if (resultList.size()==0) {

23
solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
View File

@ -66,11 +66,11 @@ import org.apache.lucene.search.ConstantScoreQuery;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.FieldDoc;
import org.apache.lucene.search.Filter;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.FieldDoc;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.Sort;
@ -78,16 +78,16 @@ import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TimeLimitingCollector;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.TopFieldDocs;
import org.apache.lucene.search.TopDocsCollector;
import org.apache.lucene.search.TopFieldCollector;
import org.apache.lucene.search.TopFieldDocs;
import org.apache.lucene.search.TopScoreDocCollector;
import org.apache.lucene.search.TotalHitCountCollector;
import org.apache.lucene.search.Weight;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.OpenBitSet;
import org.apache.lucene.util.FixedBitSet;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.params.ModifiableSolrParams;
@ -105,7 +105,6 @@ import org.apache.solr.request.UnInvertedField;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.spelling.QueryConverter;
import org.apache.solr.update.SolrIndexConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -1088,7 +1087,7 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable,SolrIn
final int[] docs = deState.scratch;
int upto = 0;
int bitsSet = 0;
OpenBitSet obs = null;
FixedBitSet fbs = null;
DocsEnum docsEnum = deState.termsEnum.docs(deState.liveDocs, deState.docsEnum, DocsEnum.FLAG_NONE);
if (deState.docsEnum == null) {
@ -1105,9 +1104,9 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable,SolrIn
int docid;
if (largestPossible > docs.length) {
if (obs == null) obs = new OpenBitSet(maxDoc());
if (fbs == null) fbs = new FixedBitSet(maxDoc());
while ((docid = sub.docsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
obs.fastSet(docid + base);
fbs.set(docid + base);
bitsSet++;
}
} else {
@ -1119,9 +1118,9 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable,SolrIn
} else {
int docid;
if (largestPossible > docs.length) {
if (obs == null) obs = new OpenBitSet(maxDoc());
if (fbs == null) fbs = new FixedBitSet(maxDoc());
while ((docid = docsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
obs.fastSet(docid);
fbs.set(docid);
bitsSet++;
}
} else {
@ -1132,12 +1131,12 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable,SolrIn
}
DocSet result;
if (obs != null) {
if (fbs != null) {
for (int i=0; i<upto; i++) {
obs.fastSet(docs[i]);
fbs.set(docs[i]);
}
bitsSet += upto;
result = new BitDocSet(obs, bitsSet);
result = new BitDocSet(fbs, bitsSet);
} else {
result = upto==0 ? DocSet.EMPTY : new SortedIntDocSet(Arrays.copyOf(docs, upto));
}

18
solr/core/src/java/org/apache/solr/search/SortedIntDocSet.java
View File

@ -18,13 +18,13 @@
package org.apache.solr.search;
import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.OpenBitSet;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.search.BitsFilteredDocIdSet;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Filter;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitSet;
/**
* <code>SortedIntDocSet</code> represents a sorted set of Lucene Document Ids.
@ -563,13 +563,12 @@ public class SortedIntDocSet extends DocSetBase {
}
@Override
public void setBitsOn(OpenBitSet target) {
public void addAllTo(DocSet target) {
for (int doc : docs) {
target.fastSet(doc);
target.add(doc);
}
}
@Override
public boolean exists(int doc) {
// this could be faster by estimating where in the list the doc is likely to appear,
@ -630,16 +629,15 @@ public class SortedIntDocSet extends DocSetBase {
}
@Override
public OpenBitSet getBits() {
public FixedBitSet getBits() {
int maxDoc = size() > 0 ? docs[size()-1] : 0;
OpenBitSet bs = new OpenBitSet(maxDoc+1);
FixedBitSet bs = new FixedBitSet(maxDoc+1);
for (int doc : docs) {
bs.fastSet(doc);
bs.set(doc);
}
return bs;
}
public static int findIndex(int[] arr, int value, int low, int high) {
// binary search
while (low <= high) {

37
solr/core/src/java/org/apache/solr/search/grouping/CommandHandler.java
View File

@ -17,22 +17,31 @@ package org.apache.solr.search.grouping;
* limitations under the License.
*/
import org.apache.lucene.search.*;
import org.apache.lucene.search.grouping.AbstractAllGroupHeadsCollector;
import org.apache.lucene.search.grouping.term.TermAllGroupHeadsCollector;
import org.apache.lucene.util.OpenBitSet;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.search.*;
import org.apache.solr.search.SolrIndexSearcher.ProcessedFilter;
import org.apache.solr.search.QueryUtils;
import org.apache.solr.search.grouping.distributed.shardresultserializer.ShardResultTransformer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.lucene.search.Collector;
import org.apache.lucene.search.Filter;
import org.apache.lucene.search.MultiCollector;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TimeLimitingCollector;
import org.apache.lucene.search.TotalHitCountCollector;
import org.apache.lucene.search.grouping.AbstractAllGroupHeadsCollector;
import org.apache.lucene.search.grouping.term.TermAllGroupHeadsCollector;
import org.apache.lucene.util.FixedBitSet;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.search.BitDocSet;
import org.apache.solr.search.DocSet;
import org.apache.solr.search.DocSetCollector;
import org.apache.solr.search.DocSetDelegateCollector;
import org.apache.solr.search.QueryUtils;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.search.SolrIndexSearcher.ProcessedFilter;
import org.apache.solr.search.grouping.distributed.shardresultserializer.ShardResultTransformer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Responsible for executing a search with a number of {@link Command} instances.
* A typical search can have more then one {@link Command} instances.
@ -159,9 +168,7 @@ public class CommandHandler {
searchWithTimeLimiter(query, filter, MultiCollector.wrap(collectors.toArray(new Collector[collectors.size()])));
}
int maxDoc = searcher.maxDoc();
long[] bits = termAllGroupHeadsCollector.retrieveGroupHeads(maxDoc).getBits();
return new BitDocSet(new OpenBitSet(bits, bits.length));
return new BitDocSet(termAllGroupHeadsCollector.retrieveGroupHeads(searcher.maxDoc()));
}
private DocSet computeDocSet(Query query, ProcessedFilter filter, List<Collector> collectors) throws IOException {

14
solr/core/src/java/org/apache/solr/search/join/BitSetSlice.java
View File

@ -17,30 +17,30 @@
package org.apache.solr.search.join;
import org.apache.lucene.util.OpenBitSet;
import org.apache.lucene.util.FixedBitSet;
class BitSetSlice {
private final OpenBitSet obs;
private final FixedBitSet fbs;
private final int off;
private final int len;
BitSetSlice(OpenBitSet obs, int off, int len) {
this.obs = obs;
BitSetSlice(FixedBitSet fbs, int off, int len) {
this.fbs = fbs;
this.off = off;
this.len = len;
}
public boolean get(int pos) {
return obs.get(pos + off);
return fbs.get(pos + off);
}
public int prevSetBit(int pos) {
int result = obs.prevSetBit(pos + off) - off;
int result = fbs.prevSetBit(pos + off) - off;
return (result < 0) ? -1 : result;
}
public int nextSetBit(int pos) {
int result = obs.nextSetBit(pos + off) - off;
int result = fbs.nextSetBit(pos + off) - off;
return (result < 0 || result >= len) ? -1 : result;
}
}

4
solr/core/src/java/org/apache/solr/store/blockcache/BlockLocks.java
View File

@ -19,7 +19,7 @@ package org.apache.solr.store.blockcache;
import java.util.concurrent.atomic.AtomicLongArray;
import org.apache.lucene.util.OpenBitSet;
import org.apache.lucene.util.LongBitSet;
public class BlockLocks {
@ -27,7 +27,7 @@ public class BlockLocks {
private int wlen;
public BlockLocks(long numBits) {
int length = OpenBitSet.bits2words(numBits);
int length = LongBitSet.bits2words(numBits);
bits = new AtomicLongArray(length);
wlen = length;
}

36
solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java
View File

@ -17,6 +17,10 @@
package org.apache.solr.update;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.DocsEnum;
@ -26,11 +30,12 @@ import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.OpenBitSet;
import org.apache.solr.common.cloud.CompositeIdRouter;
import org.apache.solr.common.cloud.DocRouter;
import org.apache.solr.common.cloud.HashBasedRouter;
@ -41,10 +46,6 @@ import org.apache.solr.util.RefCounted;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
public class SolrIndexSplitter {
public static Logger log = LoggerFactory.getLogger(SolrIndexSplitter.class);
@ -89,13 +90,13 @@ public class SolrIndexSplitter {
public void split() throws IOException {
List<AtomicReaderContext> leaves = searcher.getTopReaderContext().leaves();
List<OpenBitSet[]> segmentDocSets = new ArrayList<OpenBitSet[]>(leaves.size());
List<FixedBitSet[]> segmentDocSets = new ArrayList<FixedBitSet[]>(leaves.size());
log.info("SolrIndexSplitter: partitions=" + numPieces + " segments="+leaves.size());
for (AtomicReaderContext readerContext : leaves) {
assert readerContext.ordInParent == segmentDocSets.size(); // make sure we're going in order
OpenBitSet[] docSets = split(readerContext);
FixedBitSet[] docSets = split(readerContext);
segmentDocSets.add( docSets );
}
@ -150,11 +151,11 @@ public class SolrIndexSplitter {
OpenBitSet[] split(AtomicReaderContext readerContext) throws IOException {
FixedBitSet[] split(AtomicReaderContext readerContext) throws IOException {
AtomicReader reader = readerContext.reader();
OpenBitSet[] docSets = new OpenBitSet[numPieces];
FixedBitSet[] docSets = new FixedBitSet[numPieces];
for (int i=0; i<docSets.length; i++) {
docSets[i] = new OpenBitSet(reader.maxDoc());
docSets[i] = new FixedBitSet(reader.maxDoc());
}
Bits liveDocs = reader.getLiveDocs();
@ -196,14 +197,14 @@ public class SolrIndexSplitter {
docsEnum = termsEnum.docs(liveDocs, docsEnum, DocsEnum.FLAG_NONE);
for (;;) {
int doc = docsEnum.nextDoc();
if (doc == DocsEnum.NO_MORE_DOCS) break;
if (doc == DocIdSetIterator.NO_MORE_DOCS) break;
if (ranges == null) {
docSets[currPartition].fastSet(doc);
docSets[currPartition].set(doc);
currPartition = (currPartition + 1) % numPieces;
} else {
for (int i=0; i<rangesArr.length; i++) { // inner-loop: use array here for extra speed.
if (rangesArr[i].includes(hash)) {
docSets[i].fastSet(doc);
docSets[i].set(doc);
}
}
}
@ -232,13 +233,13 @@ public class SolrIndexSplitter {
// change livedocs on the reader to delete those docs we don't want
static class LiveDocsReader extends FilterAtomicReader {
final OpenBitSet liveDocs;
final FixedBitSet liveDocs;
final int numDocs;
public LiveDocsReader(AtomicReaderContext context, OpenBitSet liveDocs) throws IOException {
public LiveDocsReader(AtomicReaderContext context, FixedBitSet liveDocs) throws IOException {
super(context.reader());
this.liveDocs = liveDocs;
this.numDocs = (int)liveDocs.cardinality();
this.numDocs = liveDocs.cardinality();
}
@Override
@ -253,6 +254,3 @@ public class SolrIndexSplitter {
}
}

28
solr/core/src/test/org/apache/solr/search/DocSetPerf.java
View File

@ -17,13 +17,10 @@
package org.apache.solr.search;
import org.apache.solr.search.BitDocSet;
import org.apache.solr.search.HashDocSet;
import org.apache.solr.search.DocSet;
import org.apache.lucene.util.OpenBitSet;
import java.util.Random;
import java.util.BitSet;
import java.util.Random;
import org.apache.lucene.util.FixedBitSet;
/**
*/
@ -38,21 +35,20 @@ public class DocSetPerf {
static Random rand = new Random();
static OpenBitSet bs;
static FixedBitSet bs;
static BitDocSet bds;
static HashDocSet hds;
static int[] ids; // not unique
static void generate(int maxSize, int bitsToSet) {
bs = new OpenBitSet(maxSize);
bs = new FixedBitSet(maxSize);
ids = new int[bitsToSet];
int count=0;
if (maxSize>0) {
for (int i=0; i<bitsToSet; i++) {
int id=rand.nextInt(maxSize);
if (!bs.get(id)) {
bs.fastSet(id);
bs.set(id);
ids[count++]=id;
}
}
@ -80,7 +76,7 @@ public class DocSetPerf {
int ret=0;
OpenBitSet[] sets = new OpenBitSet[numSets];
FixedBitSet[] sets = new FixedBitSet[numSets];
DocSet[] bset = new DocSet[numSets];
DocSet[] hset = new DocSet[numSets];
BitSet scratch=new BitSet();
@ -97,14 +93,14 @@ public class DocSetPerf {
if ("test".equals(test)) {
for (int it=0; it<iter; it++) {
generate(randSize ? rand.nextInt(bitSetSize) : bitSetSize, numBitsSet);
OpenBitSet bs1=bs;
FixedBitSet bs1=bs;
BitDocSet bds1=bds;
HashDocSet hds1=hds;
generate(randSize ? rand.nextInt(bitSetSize) : bitSetSize, numBitsSet);
OpenBitSet res = ((OpenBitSet)bs1.clone());
FixedBitSet res = bs1.clone();
res.and(bs);
int icount = (int)res.cardinality();
int icount = res.cardinality();
test(bds1.intersection(bds).size() == icount);
test(bds1.intersectionSize(bds) == icount);
@ -167,8 +163,6 @@ public class DocSetPerf {
}
}
long end = System.currentTimeMillis();
System.out.println("TIME="+(end-start));
@ -176,6 +170,4 @@ public class DocSetPerf {
System.out.println("ret="+ret);
}
}

54
solr/core/src/test/org/apache/solr/search/TestDocSet.java
View File

@ -39,9 +39,9 @@ import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Filter;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.FixedBitSet.FixedBitSetIterator;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.OpenBitSet;
import org.apache.lucene.util.OpenBitSetIterator;
/**
*
@ -56,46 +56,45 @@ public class TestDocSet extends LuceneTestCase {
rand = random();
}
public OpenBitSet getRandomSet(int sz, int bitsToSet) {
OpenBitSet bs = new OpenBitSet(sz);
public FixedBitSet getRandomSet(int sz, int bitsToSet) {
FixedBitSet bs = new FixedBitSet(sz);
if (sz==0) return bs;
for (int i=0; i<bitsToSet; i++) {
bs.fastSet(rand.nextInt(sz));
bs.set(rand.nextInt(sz));
}
return bs;
}
public DocSet getHashDocSet(OpenBitSet bs) {
int[] docs = new int[(int)bs.cardinality()];
OpenBitSetIterator iter = new OpenBitSetIterator(bs);
public DocSet getHashDocSet(FixedBitSet bs) {
int[] docs = new int[bs.cardinality()];
FixedBitSetIterator iter = new FixedBitSetIterator(bs);
for (int i=0; i<docs.length; i++) {
docs[i] = iter.nextDoc();
}
return new HashDocSet(docs,0,docs.length);
}
public DocSet getIntDocSet(OpenBitSet bs) {
int[] docs = new int[(int)bs.cardinality()];
OpenBitSetIterator iter = new OpenBitSetIterator(bs);
public DocSet getIntDocSet(FixedBitSet bs) {
int[] docs = new int[bs.cardinality()];
FixedBitSetIterator iter = new FixedBitSetIterator(bs);
for (int i=0; i<docs.length; i++) {
docs[i] = iter.nextDoc();
}
return new SortedIntDocSet(docs);
}
public DocSet getBitDocSet(OpenBitSet bs) {
public DocSet getBitDocSet(FixedBitSet bs) {
return new BitDocSet(bs);
}
public DocSet getDocSlice(OpenBitSet bs) {
int len = (int)bs.cardinality();
public DocSet getDocSlice(FixedBitSet bs) {
int len = bs.cardinality();
int[] arr = new int[len+5];
arr[0]=10; arr[1]=20; arr[2]=30; arr[arr.length-1]=1; arr[arr.length-2]=2;
int offset = 3;
int end = offset + len;
OpenBitSetIterator iter = new OpenBitSetIterator(bs);
FixedBitSetIterator iter = new FixedBitSetIterator(bs);
// put in opposite order... DocLists are not ordered.
for (int i=end-1; i>=offset; i--) {
arr[i] = iter.nextDoc();
@ -105,7 +104,7 @@ public class TestDocSet extends LuceneTestCase {
}
public DocSet getDocSet(OpenBitSet bs) {
public DocSet getDocSet(FixedBitSet bs) {
switch(rand.nextInt(10)) {
case 0: return getHashDocSet(bs);
@ -124,7 +123,7 @@ public class TestDocSet extends LuceneTestCase {
return null;
}
public void checkEqual(OpenBitSet bs, DocSet set) {
public void checkEqual(FixedBitSet bs, DocSet set) {
for (int i=0; i<set.size(); i++) {
assertEquals(bs.get(i), set.exists(i));
}
@ -152,8 +151,8 @@ public class TestDocSet extends LuceneTestCase {
protected void doSingle(int maxSize) {
int sz = rand.nextInt(maxSize+1);
int sz2 = rand.nextInt(maxSize);
OpenBitSet bs1 = getRandomSet(sz, rand.nextInt(sz+1));
OpenBitSet bs2 = getRandomSet(sz, rand.nextInt(sz2+1));
FixedBitSet bs1 = getRandomSet(sz, rand.nextInt(sz+1));
FixedBitSet bs2 = getRandomSet(sz, rand.nextInt(sz2+1));
DocSet a1 = new BitDocSet(bs1);
DocSet a2 = new BitDocSet(bs2);
@ -166,10 +165,10 @@ public class TestDocSet extends LuceneTestCase {
iter(a1,b1);
iter(a2,b2);
OpenBitSet a_and = (OpenBitSet) bs1.clone(); a_and.and(bs2);
OpenBitSet a_or = (OpenBitSet) bs1.clone(); a_or.or(bs2);
// OpenBitSet a_xor = (OpenBitSet)bs1.clone(); a_xor.xor(bs2);
OpenBitSet a_andn = (OpenBitSet) bs1.clone(); a_andn.andNot(bs2);
FixedBitSet a_and = bs1.clone(); a_and.and(bs2);
FixedBitSet a_or = bs1.clone(); a_or.or(bs2);
// FixedBitSet a_xor = bs1.clone(); a_xor.xor(bs2);
FixedBitSet a_andn = bs1.clone(); a_andn.andNot(bs2);
checkEqual(a_and, b1.intersection(b2));
checkEqual(a_or, b1.union(b2));
@ -180,7 +179,6 @@ public class TestDocSet extends LuceneTestCase {
assertEquals(a_andn.cardinality(), b1.andNotSize(b2));
}
public void doMany(int maxSz, int iter) {
for (int i=0; i<iter; i++) {
doSingle(maxSz);
@ -197,7 +195,7 @@ public class TestDocSet extends LuceneTestCase {
}
public DocSet getRandomDocSet(int n, int maxDoc) {
OpenBitSet obs = new OpenBitSet(maxDoc);
FixedBitSet obs = new FixedBitSet(maxDoc);
int[] a = new int[n];
for (int i=0; i<n; i++) {
for(;;) {
@ -264,7 +262,7 @@ public class TestDocSet extends LuceneTestCase {
}
***/
public static int smallSetType = 0; // 0==sortedint, 1==hash, 2==openbitset
public static int smallSetType = 0; // 0==sortedint, 1==hash, 2==FixedBitSet
public static int smallSetCuttoff=3000;
/***
@ -474,7 +472,7 @@ public class TestDocSet extends LuceneTestCase {
public void doFilterTest(IndexReader reader) throws IOException {
IndexReaderContext topLevelContext = reader.getContext();
OpenBitSet bs = getRandomSet(reader.maxDoc(), rand.nextInt(reader.maxDoc()+1));
FixedBitSet bs = getRandomSet(reader.maxDoc(), rand.nextInt(reader.maxDoc()+1));
DocSet a = new BitDocSet(bs);
DocSet b = getIntDocSet(bs);

38
solr/core/src/test/org/apache/solr/search/TestFiltering.java
View File

@ -18,7 +18,7 @@
package org.apache.solr.search;
import org.apache.lucene.util.OpenBitSet;
import org.apache.lucene.util.FixedBitSet;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.SolrException;
@ -114,18 +114,18 @@ public class TestFiltering extends SolrTestCaseJ4 {
class Model {
int indexSize;
OpenBitSet answer;
OpenBitSet multiSelect;
OpenBitSet facetQuery;
FixedBitSet answer;
FixedBitSet multiSelect;
FixedBitSet facetQuery;
void clear() {
answer = new OpenBitSet(indexSize);
answer = new FixedBitSet(indexSize);
answer.set(0, indexSize);
multiSelect = new OpenBitSet(indexSize);
multiSelect = new FixedBitSet(indexSize);
multiSelect.set(0, indexSize);
facetQuery = new OpenBitSet(indexSize);
facetQuery = new FixedBitSet(indexSize);
facetQuery.set(0, indexSize);
}
}
@ -161,8 +161,8 @@ public class TestFiltering extends SolrTestCaseJ4 {
boolean positive = random().nextBoolean();
boolean exclude = facetQuery ? false : random().nextBoolean(); // can't exclude a facet query from faceting
OpenBitSet[] sets = facetQuery ? new OpenBitSet[]{model.facetQuery} :
(exclude ? new OpenBitSet[]{model.answer, model.facetQuery} : new OpenBitSet[]{model.answer, model.multiSelect, model.facetQuery});
FixedBitSet[] sets = facetQuery ? new FixedBitSet[]{model.facetQuery} :
(exclude ? new FixedBitSet[]{model.answer, model.facetQuery} : new FixedBitSet[]{model.answer, model.multiSelect, model.facetQuery});
if (random().nextInt(100) < 50) {
// frange
@ -183,33 +183,36 @@ public class TestFiltering extends SolrTestCaseJ4 {
}
}
for (OpenBitSet set : sets) {
for (FixedBitSet set : sets) {
set.clear(0,l);
set.clear(u+1, model.indexSize);
if (u + 1 < model.indexSize) {
set.clear(u+1, model.indexSize);
}
}
} else {
// negative frange.. make it relatively small
l = random().nextInt(model.indexSize);
u = Math.max(model.indexSize-1, l+random().nextInt(Math.max(model.indexSize / 10, 2)));
for (OpenBitSet set : sets) {
set.clear(l,u+1);
for (FixedBitSet set : sets) {
int end = Math.min(u+1, set.length());
set.clear(l,end);
}
}
return frangeStr(!positive, l, u, cache, cost, exclude);
} else {
// term or boolean query
OpenBitSet pset = new OpenBitSet(model.indexSize);
FixedBitSet pset = new FixedBitSet(model.indexSize);
for (int i=0; i<pset.getBits().length; i++) {
pset.getBits()[i] = random().nextLong(); // set 50% of the bits on average
}
if (positive) {
for (OpenBitSet set : sets) {
for (FixedBitSet set : sets) {
set.and(pset);
}
} else {
for (OpenBitSet set : sets) {
for (FixedBitSet set : sets) {
set.andNot(pset);
}
}
@ -217,8 +220,9 @@ public class TestFiltering extends SolrTestCaseJ4 {
StringBuilder sb = new StringBuilder();
for (int doc=-1;;) {
if (doc+1 >= model.indexSize) break;
doc = pset.nextSetBit(doc+1);
if (doc < 0 || doc >= model.indexSize) break;
if (doc < 0) break;
sb.append((positive ? " ":" -") + f+":"+doc);
}

27
solr/core/src/test/org/apache/solr/search/TestSort.java
View File

@ -18,7 +18,11 @@
package org.apache.solr.search;
import java.io.IOException;
import java.util.*;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Random;
import org.apache.lucene.analysis.core.SimpleAnalyzer;
import org.apache.lucene.document.Document;
@ -28,16 +32,27 @@ import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.search.*;
import org.apache.lucene.search.BitsFilteredDocIdSet;
import org.apache.lucene.search.Collector;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.Filter;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.SortField.Type;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.TopFieldCollector;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.OpenBitSet;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util._TestUtil;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.schema.SchemaField;
import org.junit.BeforeClass;
public class TestSort extends SolrTestCaseJ4 {
@ -320,10 +335,10 @@ public class TestSort extends SolrTestCaseJ4 {
}
public DocIdSet randSet(int sz) {
OpenBitSet obs = new OpenBitSet(sz);
FixedBitSet obs = new FixedBitSet(sz);
int n = r.nextInt(sz);
for (int i=0; i<n; i++) {
obs.fastSet(r.nextInt(sz));
obs.set(r.nextInt(sz));
}
return obs;
}

40
solr/core/src/test/org/apache/solr/util/BitSetPerf.java
View File

@ -17,13 +17,13 @@
package org.apache.solr.util;
import java.util.Random;
import java.util.BitSet;
import java.util.Random;
import org.apache.lucene.util.OpenBitSet;
import org.apache.lucene.util.OpenBitSetIterator;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.FixedBitSet.FixedBitSetIterator;
/** Performance tester for OpenBitSet.
/** Performance tester for FixedBitSet.
* Use -Xbatch for more predictable results, and run tests such that the duration
* is at least 10 seconds for better accuracy. Close browsers on your system (javascript
* or flash may be running and cause more erratic results).
@ -33,7 +33,7 @@ import org.apache.lucene.util.OpenBitSetIterator;
public class BitSetPerf {
static Random rand = new Random(0);
static void randomSets(int maxSize, int bitsToSet, BitSet target1, OpenBitSet target2) {
static void randomSets(int maxSize, int bitsToSet, BitSet target1, FixedBitSet target2) {
for (int i=0; i<bitsToSet; i++) {
int idx;
do {
@ -50,7 +50,7 @@ public class BitSetPerf {
public static void main(String[] args) {
if (args.length<5) {
System.out.println("BitSetTest <bitSetSize> <numSets> <numBitsSet> <testName> <iter> <impl>");
System.out.println(" impl => open for OpenBitSet");
System.out.println(" impl => open for FixedBitSet");
}
int bitSetSize = Integer.parseInt(args[0]);
int numSets = Integer.parseInt(args[1]);
@ -60,16 +60,16 @@ public class BitSetPerf {
String impl = args.length>5 ? args[5].intern() : "bit";
BitSet[] sets = new BitSet[numSets];
OpenBitSet[] osets = new OpenBitSet[numSets];
FixedBitSet[] osets = new FixedBitSet[numSets];
for (int i=0; i<numSets; i++) {
sets[i] = new BitSet(bitSetSize);
osets[i] = new OpenBitSet(bitSetSize);
osets[i] = new FixedBitSet(bitSetSize);
randomSets(bitSetSize, numBitsSet, sets[i], osets[i]);
}
BitSet bs = new BitSet(bitSetSize);
OpenBitSet obs = new OpenBitSet(bitSetSize);
FixedBitSet obs = new FixedBitSet(bitSetSize);
randomSets(bitSetSize, numBitsSet, bs, obs);
@ -82,8 +82,8 @@ public class BitSetPerf {
for (int it=0; it<iter; it++) {
for (int i=0; i<numSets; i++) {
if (impl=="open") {
OpenBitSet other=osets[i];
obs.union(other);
FixedBitSet other=osets[i];
obs.or(other);
} else {
BitSet other=sets[i];
bs.or(other);
@ -108,8 +108,8 @@ public class BitSetPerf {
for (int it=0; it<iter; it++) {
for (int i=0; i<numSets; i++) {
if (impl=="open") {
OpenBitSet oset = osets[i];
for (int k=0; k<bitSetSize; k++) if (oset.fastGet(k)) ret++;
FixedBitSet oset = osets[i];
for (int k=0; k<bitSetSize; k++) if (oset.get(k)) ret++;
} else {
BitSet bset = sets[i];
for (int k=0; k<bitSetSize; k++) if (bset.get(k)) ret++;
@ -122,9 +122,9 @@ public class BitSetPerf {
for (int it=0; it<iter; it++) {
for (int i=0; i<numSets-1; i++) {
if (impl=="open") {
OpenBitSet a=osets[i];
OpenBitSet b=osets[i+1];
ret += OpenBitSet.intersectionCount(a,b);
FixedBitSet a=osets[i];
FixedBitSet b=osets[i+1];
ret += FixedBitSet.intersectionCount(a,b);
} else {
BitSet a=sets[i];
BitSet b=sets[i+1];
@ -140,7 +140,7 @@ public class BitSetPerf {
for (int it=0; it<iter; it++) {
for (int i=0; i<numSets; i++) {
if (impl=="open") {
osets[i] = (OpenBitSet)osets[i].clone();
osets[i] = osets[i].clone();
} else {
sets[i] = (BitSet)sets[i].clone();
}
@ -152,7 +152,7 @@ public class BitSetPerf {
for (int it=0; it<iter; it++) {
for (int i=0; i<numSets; i++) {
if (impl=="open") {
final OpenBitSet set = osets[i];
final FixedBitSet set = osets[i];
for(int next=set.nextSetBit(0); next>=0; next=set.nextSetBit(next+1)) {
ret += next;
}
@ -171,8 +171,8 @@ public class BitSetPerf {
for (int it=0; it<iter; it++) {
for (int i=0; i<numSets; i++) {
if (impl=="open") {
final OpenBitSet set = osets[i];
final OpenBitSetIterator iterator = new OpenBitSetIterator(set);
final FixedBitSet set = osets[i];
final FixedBitSetIterator iterator = new FixedBitSetIterator(set);
for(int next=iterator.nextDoc(); next>=0; next=iterator.nextDoc()) {
ret += next;
}