mirror of https://github.com/apache/lucene.git
LUCENE-5440: decouple OpenBitSet from DocSet and move to use FixedBitSet
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1568737 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
fecedd2349
commit
87f987082b
|
@ -28,6 +28,8 @@ import org.apache.lucene.search.DocIdSetIterator;
|
|||
* long[], accessed with an int index, implementing {@link Bits} and
|
||||
* {@link DocIdSet}. If you need to manage more than 2.1B bits, use
|
||||
* {@link LongBitSet}.
|
||||
*
|
||||
* @lucene.internal
|
||||
*/
|
||||
public final class FixedBitSet extends DocIdSet implements Bits {
|
||||
|
||||
|
@ -41,6 +43,12 @@ public final class FixedBitSet extends DocIdSet implements Bits {
|
|||
final long[] bits;
|
||||
int doc = -1;
|
||||
|
||||
/** Creates an iterator over the given {@link FixedBitSet}. */
|
||||
public FixedBitSetIterator(FixedBitSet bits) {
|
||||
this(bits.bits, bits.numBits, bits.numWords);
|
||||
}
|
||||
|
||||
/** Creates an iterator over the given array of bits. */
|
||||
public FixedBitSetIterator(long[] bits, int numBits, int wordLength) {
|
||||
this.bits = bits;
|
||||
this.numBits = numBits;
|
||||
|
@ -48,7 +56,7 @@ public final class FixedBitSet extends DocIdSet implements Bits {
|
|||
}
|
||||
|
||||
@Override
|
||||
public int nextDoc() throws IOException {
|
||||
public int nextDoc() {
|
||||
if (doc == NO_MORE_DOCS || ++doc >= numBits) {
|
||||
return doc = NO_MORE_DOCS;
|
||||
}
|
||||
|
@ -81,7 +89,7 @@ public final class FixedBitSet extends DocIdSet implements Bits {
|
|||
}
|
||||
|
||||
@Override
|
||||
public int advance(int target) throws IOException {
|
||||
public int advance(int target) {
|
||||
if (doc == NO_MORE_DOCS || target >= numBits) {
|
||||
return doc = NO_MORE_DOCS;
|
||||
}
|
||||
|
@ -103,10 +111,6 @@ public final class FixedBitSet extends DocIdSet implements Bits {
|
|||
return doc = NO_MORE_DOCS;
|
||||
}
|
||||
}
|
||||
|
||||
private final long[] bits;
|
||||
private final int numBits;
|
||||
private final int wordLength;
|
||||
|
||||
/**
|
||||
* If the given {@link FixedBitSet} is large enough to hold {@code numBits},
|
||||
|
@ -140,15 +144,53 @@ public final class FixedBitSet extends DocIdSet implements Bits {
|
|||
return numLong;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the popcount or cardinality of the intersection of the two sets.
|
||||
* Neither set is modified.
|
||||
*/
|
||||
public static long intersectionCount(FixedBitSet a, FixedBitSet b) {
|
||||
return BitUtil.pop_intersect(a.bits, b.bits, 0, Math.min(a.numWords, b.numWords));
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the popcount or cardinality of the union of the two sets. Neither
|
||||
* set is modified.
|
||||
*/
|
||||
public static long unionCount(FixedBitSet a, FixedBitSet b) {
|
||||
long tot = BitUtil.pop_union(a.bits, b.bits, 0, Math.min(a.numWords, b.numWords));
|
||||
if (a.numWords < b.numWords) {
|
||||
tot += BitUtil.pop_array(b.bits, a.numWords, b.numWords - a.numWords);
|
||||
} else if (a.numWords > b.numWords) {
|
||||
tot += BitUtil.pop_array(a.bits, b.numWords, a.numWords - b.numWords);
|
||||
}
|
||||
return tot;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the popcount or cardinality of "a and not b" or
|
||||
* "intersection(a, not(b))". Neither set is modified.
|
||||
*/
|
||||
public static long andNotCount(FixedBitSet a, FixedBitSet b) {
|
||||
long tot = BitUtil.pop_andnot(a.bits, b.bits, 0, Math.min(a.numWords, b.numWords));
|
||||
if (a.numWords > b.numWords) {
|
||||
tot += BitUtil.pop_array(a.bits, b.numWords, a.numWords - b.numWords);
|
||||
}
|
||||
return tot;
|
||||
}
|
||||
|
||||
final long[] bits;
|
||||
final int numBits;
|
||||
final int numWords;
|
||||
|
||||
public FixedBitSet(int numBits) {
|
||||
this.numBits = numBits;
|
||||
bits = new long[bits2words(numBits)];
|
||||
wordLength = bits.length;
|
||||
numWords = bits.length;
|
||||
}
|
||||
|
||||
public FixedBitSet(long[] storedBits, int numBits) {
|
||||
this.wordLength = bits2words(numBits);
|
||||
if (wordLength > storedBits.length) {
|
||||
this.numWords = bits2words(numBits);
|
||||
if (numWords > storedBits.length) {
|
||||
throw new IllegalArgumentException("The given long array is too small to hold " + numBits + " bits");
|
||||
}
|
||||
this.numBits = numBits;
|
||||
|
@ -157,7 +199,7 @@ public final class FixedBitSet extends DocIdSet implements Bits {
|
|||
|
||||
@Override
|
||||
public DocIdSetIterator iterator() {
|
||||
return new FixedBitSetIterator(bits, numBits, wordLength);
|
||||
return new FixedBitSetIterator(bits, numBits, numWords);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -190,7 +232,7 @@ public final class FixedBitSet extends DocIdSet implements Bits {
|
|||
|
||||
@Override
|
||||
public boolean get(int index) {
|
||||
assert index >= 0 && index < numBits: "index=" + index;
|
||||
assert index >= 0 && index < numBits: "index=" + index + ", numBits=" + numBits;
|
||||
int i = index >> 6; // div 64
|
||||
// signed shift will keep a negative index and force an
|
||||
// array-index-out-of-bounds-exception, removing the need for an explicit check.
|
||||
|
@ -200,7 +242,7 @@ public final class FixedBitSet extends DocIdSet implements Bits {
|
|||
}
|
||||
|
||||
public void set(int index) {
|
||||
assert index >= 0 && index < numBits: "index=" + index + " numBits=" + numBits;
|
||||
assert index >= 0 && index < numBits: "index=" + index + ", numBits=" + numBits;
|
||||
int wordNum = index >> 6; // div 64
|
||||
int bit = index & 0x3f; // mod 64
|
||||
long bitmask = 1L << bit;
|
||||
|
@ -239,7 +281,7 @@ public final class FixedBitSet extends DocIdSet implements Bits {
|
|||
* -1 is returned if there are no more set bits.
|
||||
*/
|
||||
public int nextSetBit(int index) {
|
||||
assert index >= 0 && index < numBits;
|
||||
assert index >= 0 && index < numBits : "index=" + index + ", numBits=" + numBits;
|
||||
int i = index >> 6;
|
||||
final int subIndex = index & 0x3f; // index within the word
|
||||
long word = bits[i] >> subIndex; // skip all the bits to the right of index
|
||||
|
@ -248,7 +290,7 @@ public final class FixedBitSet extends DocIdSet implements Bits {
|
|||
return index + Long.numberOfTrailingZeros(word);
|
||||
}
|
||||
|
||||
while(++i < wordLength) {
|
||||
while(++i < numWords) {
|
||||
word = bits[i];
|
||||
if (word != 0) {
|
||||
return (i<<6) + Long.numberOfTrailingZeros(word);
|
||||
|
@ -306,12 +348,13 @@ public final class FixedBitSet extends DocIdSet implements Bits {
|
|||
|
||||
/** this = this OR other */
|
||||
public void or(FixedBitSet other) {
|
||||
or(other.bits, other.wordLength);
|
||||
or(other.bits, other.numWords);
|
||||
}
|
||||
|
||||
private void or(final long[] otherArr, final int otherLen) {
|
||||
private void or(final long[] otherArr, final int otherNumWords) {
|
||||
assert otherNumWords <= numWords : "numWords=" + numWords + ", otherNumWords=" + otherNumWords;
|
||||
final long[] thisArr = this.bits;
|
||||
int pos = Math.min(wordLength, otherLen);
|
||||
int pos = Math.min(numWords, otherNumWords);
|
||||
while (--pos >= 0) {
|
||||
thisArr[pos] |= otherArr[pos];
|
||||
}
|
||||
|
@ -319,9 +362,10 @@ public final class FixedBitSet extends DocIdSet implements Bits {
|
|||
|
||||
/** this = this XOR other */
|
||||
public void xor(FixedBitSet other) {
|
||||
assert other.numWords <= numWords : "numWords=" + numWords + ", other.numWords=" + other.numWords;
|
||||
final long[] thisBits = this.bits;
|
||||
final long[] otherBits = other.bits;
|
||||
int pos = Math.min(wordLength, other.wordLength);
|
||||
int pos = Math.min(numWords, other.numWords);
|
||||
while (--pos >= 0) {
|
||||
thisBits[pos] ^= otherBits[pos];
|
||||
}
|
||||
|
@ -366,7 +410,7 @@ public final class FixedBitSet extends DocIdSet implements Bits {
|
|||
|
||||
/** returns true if the sets have any elements in common */
|
||||
public boolean intersects(FixedBitSet other) {
|
||||
int pos = Math.min(wordLength, other.wordLength);
|
||||
int pos = Math.min(numWords, other.numWords);
|
||||
while (--pos>=0) {
|
||||
if ((bits[pos] & other.bits[pos]) != 0) return true;
|
||||
}
|
||||
|
@ -375,17 +419,17 @@ public final class FixedBitSet extends DocIdSet implements Bits {
|
|||
|
||||
/** this = this AND other */
|
||||
public void and(FixedBitSet other) {
|
||||
and(other.bits, other.wordLength);
|
||||
and(other.bits, other.numWords);
|
||||
}
|
||||
|
||||
private void and(final long[] otherArr, final int otherLen) {
|
||||
private void and(final long[] otherArr, final int otherNumWords) {
|
||||
final long[] thisArr = this.bits;
|
||||
int pos = Math.min(this.wordLength, otherLen);
|
||||
int pos = Math.min(this.numWords, otherNumWords);
|
||||
while(--pos >= 0) {
|
||||
thisArr[pos] &= otherArr[pos];
|
||||
}
|
||||
if (this.wordLength > otherLen) {
|
||||
Arrays.fill(thisArr, otherLen, this.wordLength, 0L);
|
||||
if (this.numWords > otherNumWords) {
|
||||
Arrays.fill(thisArr, otherNumWords, this.numWords, 0L);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -417,9 +461,9 @@ public final class FixedBitSet extends DocIdSet implements Bits {
|
|||
andNot(other.bits, other.bits.length);
|
||||
}
|
||||
|
||||
private void andNot(final long[] otherArr, final int otherLen) {
|
||||
private void andNot(final long[] otherArr, final int otherNumWords) {
|
||||
final long[] thisArr = this.bits;
|
||||
int pos = Math.min(this.wordLength, otherLen);
|
||||
int pos = Math.min(this.numWords, otherNumWords);
|
||||
while(--pos >= 0) {
|
||||
thisArr[pos] &= ~otherArr[pos];
|
||||
}
|
||||
|
@ -502,8 +546,8 @@ public final class FixedBitSet extends DocIdSet implements Bits {
|
|||
* @param endIndex one-past the last bit to clear
|
||||
*/
|
||||
public void clear(int startIndex, int endIndex) {
|
||||
assert startIndex >= 0 && startIndex < numBits;
|
||||
assert endIndex >= 0 && endIndex <= numBits;
|
||||
assert startIndex >= 0 && startIndex < numBits : "startIndex=" + startIndex + ", numBits=" + numBits;
|
||||
assert endIndex >= 0 && endIndex <= numBits : "endIndex=" + endIndex + ", numBits=" + numBits;
|
||||
if (endIndex <= startIndex) {
|
||||
return;
|
||||
}
|
||||
|
@ -554,7 +598,7 @@ public final class FixedBitSet extends DocIdSet implements Bits {
|
|||
@Override
|
||||
public int hashCode() {
|
||||
long h = 0;
|
||||
for (int i = wordLength; --i>=0;) {
|
||||
for (int i = numWords; --i>=0;) {
|
||||
h ^= bits[i];
|
||||
h = (h << 1) | (h >>> 63); // rotate left
|
||||
}
|
||||
|
|
|
@ -23,6 +23,8 @@ import java.util.Arrays;
|
|||
* BitSet of fixed length (numBits), backed by accessible ({@link #getBits})
|
||||
* long[], accessed with a long index. Use it only if you intend to store more
|
||||
* than 2.1B bits, otherwise you should use {@link FixedBitSet}.
|
||||
*
|
||||
* @lucene.internal
|
||||
*/
|
||||
public final class LongBitSet {
|
||||
|
||||
|
@ -188,6 +190,7 @@ public final class LongBitSet {
|
|||
|
||||
/** this = this OR other */
|
||||
public void or(LongBitSet other) {
|
||||
assert other.numWords <= numWords : "numWords=" + numWords + ", other.numWords=" + other.numWords;
|
||||
int pos = Math.min(numWords, other.numWords);
|
||||
while (--pos >= 0) {
|
||||
bits[pos] |= other.bits[pos];
|
||||
|
@ -196,6 +199,7 @@ public final class LongBitSet {
|
|||
|
||||
/** this = this XOR other */
|
||||
public void xor(LongBitSet other) {
|
||||
assert other.numWords <= numWords : "numWords=" + numWords + ", other.numWords=" + other.numWords;
|
||||
int pos = Math.min(numWords, other.numWords);
|
||||
while (--pos >= 0) {
|
||||
bits[pos] ^= other.bits[pos];
|
||||
|
|
|
@ -44,7 +44,7 @@ public abstract class AbstractAllGroupHeadsCollector<GH extends AbstractAllGroup
|
|||
|
||||
/**
|
||||
* @param maxDoc The maxDoc of the top level {@link IndexReader}.
|
||||
* @return an {@link FixedBitSet} containing all group heads.
|
||||
* @return a {@link FixedBitSet} containing all group heads.
|
||||
*/
|
||||
public FixedBitSet retrieveGroupHeads(int maxDoc) {
|
||||
FixedBitSet bitSet = new FixedBitSet(maxDoc);
|
||||
|
|
|
@ -362,6 +362,10 @@ Optimizations
|
|||
* SOLR-5624: Enable QueryResultCache for CollapsingQParserPlugin.
|
||||
(David Boychuck, Joel Bernstein)
|
||||
|
||||
* LUCENE-5440: DocSet decoupled from OpenBitSet. DocSetBase moved to use
|
||||
FixedBitSet instead of OpenBitSet. As a result BitDocSet now only works
|
||||
with FixedBitSet. (Shai Erera)
|
||||
|
||||
Other Changes
|
||||
---------------------
|
||||
|
||||
|
|
|
@ -17,7 +17,17 @@
|
|||
|
||||
package org.apache.solr.handler.component;
|
||||
|
||||
import org.apache.lucene.util.OpenBitSet;
|
||||
import java.io.IOException;
|
||||
import java.net.URL;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.lucene.util.FixedBitSet;
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.common.SolrException.ErrorCode;
|
||||
import org.apache.solr.common.params.CommonParams;
|
||||
|
@ -35,10 +45,6 @@ import org.apache.solr.search.SyntaxError;
|
|||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URL;
|
||||
import java.util.*;
|
||||
|
||||
/**
|
||||
* TODO!
|
||||
*
|
||||
|
@ -444,8 +450,8 @@ public class FacetComponent extends SearchComponent
|
|||
// and if it is >= smallestCount, then flag for refinement
|
||||
long maxCount = sfc.count;
|
||||
for (int shardNum=0; shardNum<rb.shards.length; shardNum++) {
|
||||
OpenBitSet obs = dff.counted[shardNum];
|
||||
if (obs!=null && !obs.get(sfc.termNum)) { // obs can be null if a shard request failed
|
||||
FixedBitSet fbs = dff.counted[shardNum];
|
||||
if (fbs!=null && !fbs.get(sfc.termNum)) { // fbs can be null if a shard request failed
|
||||
// if missing from this shard, add the max it could be
|
||||
maxCount += dff.maxPossible(sfc,shardNum);
|
||||
}
|
||||
|
@ -459,8 +465,8 @@ public class FacetComponent extends SearchComponent
|
|||
if (needRefinement) {
|
||||
// add a query for each shard missing the term that needs refinement
|
||||
for (int shardNum=0; shardNum<rb.shards.length; shardNum++) {
|
||||
OpenBitSet obs = dff.counted[shardNum];
|
||||
if(obs!=null && !obs.get(sfc.termNum) && dff.maxPossible(sfc,shardNum)>0) {
|
||||
FixedBitSet fbs = dff.counted[shardNum];
|
||||
if(fbs!=null && !fbs.get(sfc.termNum) && dff.maxPossible(sfc,shardNum)>0) {
|
||||
dff.needRefinements = true;
|
||||
List<String> lst = dff._toRefine[shardNum];
|
||||
if (lst == null) {
|
||||
|
@ -759,7 +765,7 @@ public class FacetComponent extends SearchComponent
|
|||
public long missingMaxPossible;
|
||||
// the max possible count for a missing term for each shard (indexed by shardNum)
|
||||
public long[] missingMax;
|
||||
public OpenBitSet[] counted; // a bitset for each shard, keeping track of which terms seen
|
||||
public FixedBitSet[] counted; // a bitset for each shard, keeping track of which terms seen
|
||||
public HashMap<String,ShardFacetCount> counts = new HashMap<String,ShardFacetCount>(128);
|
||||
public int termNum;
|
||||
|
||||
|
@ -772,7 +778,7 @@ public class FacetComponent extends SearchComponent
|
|||
super(rb, facetStr);
|
||||
// sf = rb.req.getSchema().getField(field);
|
||||
missingMax = new long[rb.shards.length];
|
||||
counted = new OpenBitSet[rb.shards.length];
|
||||
counted = new FixedBitSet[rb.shards.length];
|
||||
}
|
||||
|
||||
void add(int shardNum, NamedList shardCounts, int numRequested) {
|
||||
|
@ -780,7 +786,7 @@ public class FacetComponent extends SearchComponent
|
|||
int sz = shardCounts == null ? 0 : shardCounts.size();
|
||||
int numReceived = sz;
|
||||
|
||||
OpenBitSet terms = new OpenBitSet(termNum+sz);
|
||||
FixedBitSet terms = new FixedBitSet(termNum+sz);
|
||||
|
||||
long last = 0;
|
||||
for (int i=0; i<sz; i++) {
|
||||
|
@ -799,7 +805,7 @@ public class FacetComponent extends SearchComponent
|
|||
counts.put(name, sfc);
|
||||
}
|
||||
sfc.count += count;
|
||||
terms.fastSet(sfc.termNum);
|
||||
terms.set(sfc.termNum);
|
||||
last = count;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -17,6 +17,26 @@
|
|||
|
||||
package org.apache.solr.request;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Date;
|
||||
import java.util.EnumSet;
|
||||
import java.util.IdentityHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.Callable;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.Executor;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.FutureTask;
|
||||
import java.util.concurrent.RunnableFuture;
|
||||
import java.util.concurrent.Semaphore;
|
||||
import java.util.concurrent.SynchronousQueue;
|
||||
import java.util.concurrent.ThreadPoolExecutor;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import org.apache.lucene.index.AtomicReader;
|
||||
import org.apache.lucene.index.DocsEnum;
|
||||
import org.apache.lucene.index.Fields;
|
||||
|
@ -37,8 +57,6 @@ import org.apache.lucene.search.grouping.term.TermAllGroupsCollector;
|
|||
import org.apache.lucene.search.grouping.term.TermGroupFacetCollector;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.apache.lucene.util.CharsRef;
|
||||
import org.apache.lucene.util.FixedBitSet;
|
||||
import org.apache.lucene.util.OpenBitSet;
|
||||
import org.apache.lucene.util.StringHelper;
|
||||
import org.apache.lucene.util.UnicodeUtil;
|
||||
import org.apache.solr.common.SolrException;
|
||||
|
@ -80,26 +98,6 @@ import org.apache.solr.util.DateMathParser;
|
|||
import org.apache.solr.util.DefaultSolrThreadFactory;
|
||||
import org.apache.solr.util.LongPriorityQueue;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Date;
|
||||
import java.util.EnumSet;
|
||||
import java.util.IdentityHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.Callable;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.Executor;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.FutureTask;
|
||||
import java.util.concurrent.RunnableFuture;
|
||||
import java.util.concurrent.Semaphore;
|
||||
import java.util.concurrent.SynchronousQueue;
|
||||
import java.util.concurrent.ThreadPoolExecutor;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
/**
|
||||
* A class that generates simple Facet information for a request.
|
||||
*
|
||||
|
@ -230,10 +228,7 @@ public class SimpleFacets {
|
|||
}
|
||||
AbstractAllGroupHeadsCollector allGroupHeadsCollector = grouping.getCommands().get(0).createAllGroupCollector();
|
||||
searcher.search(new MatchAllDocsQuery(), base.getTopFilter(), allGroupHeadsCollector);
|
||||
int maxDoc = searcher.maxDoc();
|
||||
FixedBitSet fixedBitSet = allGroupHeadsCollector.retrieveGroupHeads(maxDoc);
|
||||
long[] bits = fixedBitSet.getBits();
|
||||
this.docs = new BitDocSet(new OpenBitSet(bits, bits.length));
|
||||
this.docs = new BitDocSet(allGroupHeadsCollector.retrieveGroupHeads(searcher.maxDoc()));
|
||||
} else {
|
||||
this.docs = base;
|
||||
}
|
||||
|
|
|
@ -27,12 +27,11 @@ import org.apache.lucene.index.DocTermOrds;
|
|||
import org.apache.lucene.index.SortedDocValues;
|
||||
import org.apache.lucene.index.Term;
|
||||
import org.apache.lucene.index.TermsEnum;
|
||||
import org.apache.lucene.search.FieldCache;
|
||||
import org.apache.lucene.search.TermQuery;
|
||||
import org.apache.lucene.search.TermRangeQuery;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.apache.lucene.util.CharsRef;
|
||||
import org.apache.lucene.util.OpenBitSet;
|
||||
import org.apache.lucene.util.FixedBitSet;
|
||||
import org.apache.lucene.util.UnicodeUtil;
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.common.params.FacetParams;
|
||||
|
@ -44,7 +43,11 @@ import org.apache.solr.handler.component.StatsValuesFactory;
|
|||
import org.apache.solr.schema.FieldType;
|
||||
import org.apache.solr.schema.SchemaField;
|
||||
import org.apache.solr.schema.TrieField;
|
||||
import org.apache.solr.search.*;
|
||||
import org.apache.solr.search.BitDocSet;
|
||||
import org.apache.solr.search.DocIterator;
|
||||
import org.apache.solr.search.DocSet;
|
||||
import org.apache.solr.search.SolrCache;
|
||||
import org.apache.solr.search.SolrIndexSearcher;
|
||||
import org.apache.solr.util.LongPriorityQueue;
|
||||
import org.apache.solr.util.PrimUtils;
|
||||
|
||||
|
@ -269,7 +272,7 @@ public class UnInvertedField extends DocTermOrds {
|
|||
&& docs instanceof BitDocSet;
|
||||
|
||||
if (doNegative) {
|
||||
OpenBitSet bs = (OpenBitSet)((BitDocSet)docs).getBits().clone();
|
||||
FixedBitSet bs = ((BitDocSet)docs).getBits().clone();
|
||||
bs.flip(0, maxDoc);
|
||||
// TODO: when iterator across negative elements is available, use that
|
||||
// instead of creating a new bitset and inverting.
|
||||
|
@ -512,7 +515,7 @@ public class UnInvertedField extends DocTermOrds {
|
|||
}
|
||||
|
||||
if (doNegative) {
|
||||
OpenBitSet bs = (OpenBitSet) ((BitDocSet) docs).getBits().clone();
|
||||
FixedBitSet bs = ((BitDocSet) docs).getBits().clone();
|
||||
bs.flip(0, maxDoc);
|
||||
// TODO: when iterator across negative elements is available, use that
|
||||
// instead of creating a new bitset and inverting.
|
||||
|
|
|
@ -21,38 +21,40 @@ import org.apache.lucene.index.AtomicReader;
|
|||
import org.apache.lucene.index.AtomicReaderContext;
|
||||
import org.apache.lucene.search.BitsFilteredDocIdSet;
|
||||
import org.apache.lucene.search.DocIdSet;
|
||||
import org.apache.lucene.search.DocIdSetIterator;
|
||||
import org.apache.lucene.search.Filter;
|
||||
import org.apache.lucene.util.Bits;
|
||||
import org.apache.lucene.util.OpenBitSet;
|
||||
import org.apache.lucene.util.OpenBitSetIterator;
|
||||
import org.apache.lucene.search.DocIdSetIterator;
|
||||
import org.apache.lucene.util.FixedBitSet;
|
||||
import org.apache.lucene.util.FixedBitSet.FixedBitSetIterator;
|
||||
|
||||
/**
|
||||
* <code>BitDocSet</code> represents an unordered set of Lucene Document Ids
|
||||
* using a BitSet. A set bit represents inclusion in the set for that document.
|
||||
*
|
||||
*
|
||||
* @since solr 0.9
|
||||
*/
|
||||
public class BitDocSet extends DocSetBase {
|
||||
final OpenBitSet bits;
|
||||
final FixedBitSet bits;
|
||||
int size; // number of docs in the set (cached for perf)
|
||||
|
||||
public BitDocSet() {
|
||||
bits = new OpenBitSet();
|
||||
bits = new FixedBitSet(64);
|
||||
}
|
||||
|
||||
/** Construct a BitDocSet.
|
||||
* The capacity of the OpenBitSet should be at least maxDoc() */
|
||||
public BitDocSet(OpenBitSet bits) {
|
||||
/**
|
||||
* Construct a BitDocSet. The capacity of the {@link FixedBitSet} should be at
|
||||
* least maxDoc()
|
||||
*/
|
||||
public BitDocSet(FixedBitSet bits) {
|
||||
this.bits = bits;
|
||||
size=-1;
|
||||
}
|
||||
|
||||
/** Construct a BitDocSet, and provides the number of set bits.
|
||||
* The capacity of the OpenBitSet should be at least maxDoc()
|
||||
/**
|
||||
* Construct a BitDocSet, and provides the number of set bits. The capacity of
|
||||
* the {@link FixedBitSet} should be at least maxDoc()
|
||||
*/
|
||||
public BitDocSet(OpenBitSet bits, int size) {
|
||||
public BitDocSet(FixedBitSet bits, int size) {
|
||||
this.bits = bits;
|
||||
this.size = size;
|
||||
}
|
||||
|
@ -89,7 +91,7 @@ public class BitDocSet extends DocSetBase {
|
|||
@Override
|
||||
public DocIterator iterator() {
|
||||
return new DocIterator() {
|
||||
private final OpenBitSetIterator iter = new OpenBitSetIterator(bits);
|
||||
private final FixedBitSetIterator iter = new FixedBitSetIterator(bits);
|
||||
private int pos = iter.nextDoc();
|
||||
@Override
|
||||
public boolean hasNext() {
|
||||
|
@ -120,13 +122,11 @@ public class BitDocSet extends DocSetBase {
|
|||
};
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
*
|
||||
* @return the <b>internal</b> OpenBitSet that should <b>not</b> be modified.
|
||||
* @return the <b>internal</b> {@link FixedBitSet} that should <b>not</b> be modified.
|
||||
*/
|
||||
@Override
|
||||
public OpenBitSet getBits() {
|
||||
public FixedBitSet getBits() {
|
||||
return bits;
|
||||
}
|
||||
|
||||
|
@ -145,7 +145,7 @@ public class BitDocSet extends DocSetBase {
|
|||
@Override
|
||||
public int size() {
|
||||
if (size!=-1) return size;
|
||||
return size=(int)bits.cardinality();
|
||||
return size = bits.cardinality();
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -156,18 +156,19 @@ public class BitDocSet extends DocSetBase {
|
|||
size=-1;
|
||||
}
|
||||
|
||||
/** Returns true of the doc exists in the set.
|
||||
* Should only be called when doc < OpenBitSet.size()
|
||||
/**
|
||||
* Returns true of the doc exists in the set. Should only be called when doc <
|
||||
* {@link FixedBitSet#length()}.
|
||||
*/
|
||||
@Override
|
||||
public boolean exists(int doc) {
|
||||
return bits.fastGet(doc);
|
||||
return bits.get(doc);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int intersectionSize(DocSet other) {
|
||||
if (other instanceof BitDocSet) {
|
||||
return (int)OpenBitSet.intersectionCount(this.bits, ((BitDocSet)other).bits);
|
||||
return (int) FixedBitSet.intersectionCount(this.bits, ((BitDocSet) other).bits);
|
||||
} else {
|
||||
// they had better not call us back!
|
||||
return other.intersectionSize(this);
|
||||
|
@ -189,7 +190,7 @@ public class BitDocSet extends DocSetBase {
|
|||
if (other instanceof BitDocSet) {
|
||||
// if we don't know our current size, this is faster than
|
||||
// size + other.size - intersection_size
|
||||
return (int)OpenBitSet.unionCount(this.bits, ((BitDocSet)other).bits);
|
||||
return (int) FixedBitSet.unionCount(this.bits, ((BitDocSet)other).bits);
|
||||
} else {
|
||||
// they had better not call us back!
|
||||
return other.unionSize(this);
|
||||
|
@ -201,42 +202,56 @@ public class BitDocSet extends DocSetBase {
|
|||
if (other instanceof BitDocSet) {
|
||||
// if we don't know our current size, this is faster than
|
||||
// size - intersection_size
|
||||
return (int)OpenBitSet.andNotCount(this.bits, ((BitDocSet)other).bits);
|
||||
return (int) FixedBitSet.andNotCount(this.bits, ((BitDocSet)other).bits);
|
||||
} else {
|
||||
return super.andNotSize(other);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setBitsOn(OpenBitSet target) {
|
||||
target.union(bits);
|
||||
public void addAllTo(DocSet target) {
|
||||
if (target instanceof BitDocSet) {
|
||||
((BitDocSet) target).bits.or(bits);
|
||||
} else {
|
||||
super.addAllTo(target);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public DocSet andNot(DocSet other) {
|
||||
OpenBitSet newbits = (OpenBitSet)(bits.clone());
|
||||
if (other instanceof BitDocSet) {
|
||||
newbits.andNot(((BitDocSet)other).bits);
|
||||
} else {
|
||||
DocIterator iter = other.iterator();
|
||||
while (iter.hasNext()) newbits.clear(iter.nextDoc());
|
||||
}
|
||||
return new BitDocSet(newbits);
|
||||
public DocSet andNot(DocSet other) {
|
||||
FixedBitSet newbits = bits.clone();
|
||||
if (other instanceof BitDocSet) {
|
||||
newbits.andNot(((BitDocSet) other).bits);
|
||||
} else {
|
||||
DocIterator iter = other.iterator();
|
||||
while (iter.hasNext()) {
|
||||
int doc = iter.nextDoc();
|
||||
if (doc < newbits.length()) {
|
||||
newbits.clear(doc);
|
||||
}
|
||||
}
|
||||
}
|
||||
return new BitDocSet(newbits);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public DocSet union(DocSet other) {
|
||||
OpenBitSet newbits = (OpenBitSet)(bits.clone());
|
||||
if (other instanceof BitDocSet) {
|
||||
newbits.union(((BitDocSet)other).bits);
|
||||
} else {
|
||||
DocIterator iter = other.iterator();
|
||||
while (iter.hasNext()) newbits.set(iter.nextDoc());
|
||||
}
|
||||
return new BitDocSet(newbits);
|
||||
public DocSet union(DocSet other) {
|
||||
FixedBitSet newbits = bits.clone();
|
||||
if (other instanceof BitDocSet) {
|
||||
BitDocSet otherDocSet = (BitDocSet) other;
|
||||
newbits = FixedBitSet.ensureCapacity(newbits, otherDocSet.bits.length());
|
||||
newbits.or(otherDocSet.bits);
|
||||
} else {
|
||||
DocIterator iter = other.iterator();
|
||||
while (iter.hasNext()) {
|
||||
int doc = iter.nextDoc();
|
||||
newbits = FixedBitSet.ensureCapacity(newbits, doc);
|
||||
newbits.set(doc);
|
||||
}
|
||||
}
|
||||
return new BitDocSet(newbits);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Override
|
||||
public long memSize() {
|
||||
return (bits.getBits().length << 3) + 16;
|
||||
|
@ -244,12 +259,12 @@ public class BitDocSet extends DocSetBase {
|
|||
|
||||
@Override
|
||||
protected BitDocSet clone() {
|
||||
return new BitDocSet((OpenBitSet)bits.clone(), size);
|
||||
return new BitDocSet(bits.clone(), size);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Filter getTopFilter() {
|
||||
final OpenBitSet bs = bits;
|
||||
final FixedBitSet bs = bits;
|
||||
// TODO: if cardinality isn't cached, do a quick measure of sparseness
|
||||
// and return null from bits() if too sparse.
|
||||
|
||||
|
@ -272,8 +287,8 @@ public class BitDocSet extends DocSetBase {
|
|||
@Override
|
||||
public DocIdSetIterator iterator() {
|
||||
return new DocIdSetIterator() {
|
||||
int pos=base-1;
|
||||
int adjustedDoc=-1;
|
||||
int pos = base - 1;
|
||||
int adjustedDoc = -1;
|
||||
|
||||
@Override
|
||||
public int docID() {
|
||||
|
@ -282,15 +297,24 @@ public class BitDocSet extends DocSetBase {
|
|||
|
||||
@Override
|
||||
public int nextDoc() {
|
||||
pos = bs.nextSetBit(pos+1);
|
||||
return adjustedDoc = (pos>=0 && pos<max) ? pos-base : NO_MORE_DOCS;
|
||||
if (pos >= bs.length() - 1) {
|
||||
return adjustedDoc = NO_MORE_DOCS;
|
||||
} else {
|
||||
pos = bs.nextSetBit(pos + 1);
|
||||
return adjustedDoc = (pos >= 0 && pos < max) ? pos - base : NO_MORE_DOCS;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public int advance(int target) {
|
||||
if (target==NO_MORE_DOCS) return adjustedDoc=NO_MORE_DOCS;
|
||||
pos = bs.nextSetBit(target+base);
|
||||
return adjustedDoc = (pos>=0 && pos<max) ? pos-base : NO_MORE_DOCS;
|
||||
if (target == NO_MORE_DOCS) return adjustedDoc = NO_MORE_DOCS;
|
||||
int adjusted = target + base;
|
||||
if (adjusted >= bs.length()) {
|
||||
return adjustedDoc = NO_MORE_DOCS;
|
||||
} else {
|
||||
pos = bs.nextSetBit(adjusted);
|
||||
return adjustedDoc = (pos >= 0 && pos < max) ? pos - base : NO_MORE_DOCS;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -298,7 +322,7 @@ public class BitDocSet extends DocSetBase {
|
|||
// we don't want to actually compute cardinality, but
|
||||
// if its already been computed, we use it (pro-rated for the segment)
|
||||
if (size != -1) {
|
||||
return (long)(size * ((OpenBitSet.bits2words(maxDoc)<<6) / (float)bs.capacity()));
|
||||
return (long)(size * ((FixedBitSet.bits2words(maxDoc)<<6) / (float)bs.length()));
|
||||
} else {
|
||||
return maxDoc;
|
||||
}
|
||||
|
@ -316,7 +340,7 @@ public class BitDocSet extends DocSetBase {
|
|||
return new Bits() {
|
||||
@Override
|
||||
public boolean get(int index) {
|
||||
return bs.fastGet(index + base);
|
||||
return bs.get(index + base);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
@ -17,45 +17,49 @@
|
|||
|
||||
package org.apache.solr.search;
|
||||
|
||||
import org.apache.lucene.queries.function.FunctionQuery;
|
||||
import org.apache.lucene.queries.function.FunctionValues;
|
||||
import org.apache.lucene.queries.function.ValueSource;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.apache.solr.common.params.ModifiableSolrParams;
|
||||
import org.apache.solr.request.LocalSolrQueryRequest;
|
||||
import org.apache.solr.request.SolrRequestInfo;
|
||||
import org.apache.solr.schema.TrieFloatField;
|
||||
import org.apache.solr.schema.TrieIntField;
|
||||
import org.apache.solr.schema.TrieLongField;
|
||||
import org.apache.solr.schema.FieldType;
|
||||
import org.apache.solr.handler.component.QueryElevationComponent;
|
||||
import org.apache.lucene.index.AtomicReaderContext;
|
||||
import java.io.IOException;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashSet;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.lucene.index.AtomicReader;
|
||||
import org.apache.lucene.index.AtomicReaderContext;
|
||||
import org.apache.lucene.index.DocsEnum;
|
||||
import org.apache.lucene.index.SortedDocValues;
|
||||
import org.apache.lucene.index.Terms;
|
||||
import org.apache.lucene.index.TermsEnum;
|
||||
import org.apache.lucene.index.DocsEnum;
|
||||
import org.apache.lucene.search.*;
|
||||
import org.apache.lucene.util.OpenBitSet;
|
||||
import org.apache.lucene.queries.function.FunctionQuery;
|
||||
import org.apache.lucene.queries.function.FunctionValues;
|
||||
import org.apache.lucene.queries.function.ValueSource;
|
||||
import org.apache.lucene.search.DocIdSetIterator;
|
||||
import org.apache.lucene.search.FieldCache;
|
||||
import org.apache.lucene.search.IndexSearcher;
|
||||
import org.apache.lucene.search.Query;
|
||||
import org.apache.lucene.search.Scorer;
|
||||
import org.apache.lucene.util.Bits;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.apache.lucene.util.FixedBitSet;
|
||||
import org.apache.solr.common.params.ModifiableSolrParams;
|
||||
import org.apache.solr.common.params.SolrParams;
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
import org.apache.solr.common.util.NamedList;
|
||||
import org.apache.solr.handler.component.QueryElevationComponent;
|
||||
import org.apache.solr.request.LocalSolrQueryRequest;
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
import org.apache.solr.request.SolrRequestInfo;
|
||||
import org.apache.solr.schema.FieldType;
|
||||
import org.apache.solr.schema.IndexSchema;
|
||||
import org.apache.solr.schema.SchemaField;
|
||||
import org.apache.solr.schema.TrieFloatField;
|
||||
import org.apache.solr.schema.TrieIntField;
|
||||
import org.apache.solr.schema.TrieLongField;
|
||||
|
||||
import com.carrotsearch.hppc.FloatArrayList;
|
||||
import com.carrotsearch.hppc.IntOpenHashSet;
|
||||
import com.carrotsearch.hppc.cursors.IntCursor;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Arrays;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Iterator;
|
||||
|
||||
/**
|
||||
|
||||
The <b>CollapsingQParserPlugin</b> is a PostFilter that performs field collapsing.
|
||||
|
@ -419,7 +423,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
|
|||
private class CollapsingScoreCollector extends DelegatingCollector {
|
||||
|
||||
private AtomicReaderContext[] contexts;
|
||||
private OpenBitSet collapsedSet;
|
||||
private FixedBitSet collapsedSet;
|
||||
private SortedDocValues values;
|
||||
private int[] ords;
|
||||
private float[] scores;
|
||||
|
@ -438,14 +442,14 @@ public class CollapsingQParserPlugin extends QParserPlugin {
|
|||
IntOpenHashSet boostDocs) {
|
||||
this.maxDoc = maxDoc;
|
||||
this.contexts = new AtomicReaderContext[segments];
|
||||
this.collapsedSet = new OpenBitSet(maxDoc);
|
||||
this.collapsedSet = new FixedBitSet(maxDoc);
|
||||
this.boostDocs = boostDocs;
|
||||
if(this.boostDocs != null) {
|
||||
//Set the elevated docs now.
|
||||
Iterator<IntCursor> it = this.boostDocs.iterator();
|
||||
while(it.hasNext()) {
|
||||
IntCursor cursor = it.next();
|
||||
this.collapsedSet.fastSet(cursor.value);
|
||||
this.collapsedSet.set(cursor.value);
|
||||
}
|
||||
}
|
||||
this.values = values;
|
||||
|
@ -460,16 +464,19 @@ public class CollapsingQParserPlugin extends QParserPlugin {
|
|||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean acceptsDocsOutOfOrder() {
|
||||
//Documents must be sent in order to this collector.
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setNextReader(AtomicReaderContext context) throws IOException {
|
||||
this.contexts[context.ord] = context;
|
||||
this.docBase = context.docBase;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void collect(int docId) throws IOException {
|
||||
int globalDoc = docId+this.docBase;
|
||||
int ord = values.getOrd(globalDoc);
|
||||
|
@ -479,7 +486,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
|
|||
ords[ord] = globalDoc;
|
||||
scores[ord] = score;
|
||||
}
|
||||
} else if (this.collapsedSet.fastGet(globalDoc)) {
|
||||
} else if (this.collapsedSet.get(globalDoc)) {
|
||||
//The doc is elevated so score does not matter
|
||||
//We just want to be sure it doesn't fall into the null policy
|
||||
} else if(nullPolicy == CollapsingPostFilter.NULL_POLICY_COLLAPSE) {
|
||||
|
@ -489,24 +496,25 @@ public class CollapsingQParserPlugin extends QParserPlugin {
|
|||
nullDoc = globalDoc;
|
||||
}
|
||||
} else if(nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) {
|
||||
collapsedSet.fastSet(globalDoc);
|
||||
collapsedSet.set(globalDoc);
|
||||
nullScores.add(scorer.score());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void finish() throws IOException {
|
||||
if(contexts.length == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
if(nullScore > 0) {
|
||||
this.collapsedSet.fastSet(nullDoc);
|
||||
this.collapsedSet.set(nullDoc);
|
||||
}
|
||||
|
||||
for(int i=0; i<ords.length; i++) {
|
||||
int doc = ords[i];
|
||||
if(doc > -1) {
|
||||
collapsedSet.fastSet(doc);
|
||||
collapsedSet.set(doc);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -677,7 +685,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
|
|||
protected FloatArrayList nullScores;
|
||||
protected float nullScore;
|
||||
protected float[] scores;
|
||||
protected OpenBitSet collapsedSet;
|
||||
protected FixedBitSet collapsedSet;
|
||||
protected IntOpenHashSet boostDocs;
|
||||
protected int nullDoc = -1;
|
||||
protected boolean needsScores;
|
||||
|
@ -697,26 +705,26 @@ public class CollapsingQParserPlugin extends QParserPlugin {
|
|||
this.nullPolicy = nullPolicy;
|
||||
this.max = max;
|
||||
this.needsScores = needsScores;
|
||||
this.collapsedSet = new OpenBitSet(maxDoc);
|
||||
this.collapsedSet = new FixedBitSet(maxDoc);
|
||||
this.boostDocs = boostDocs;
|
||||
if(this.boostDocs != null) {
|
||||
Iterator<IntCursor> it = boostDocs.iterator();
|
||||
while(it.hasNext()) {
|
||||
IntCursor cursor = it.next();
|
||||
this.collapsedSet.fastSet(cursor.value);
|
||||
this.collapsedSet.set(cursor.value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public OpenBitSet getCollapsedSet() {
|
||||
public FixedBitSet getCollapsedSet() {
|
||||
if(nullDoc > -1) {
|
||||
this.collapsedSet.fastSet(nullDoc);
|
||||
this.collapsedSet.set(nullDoc);
|
||||
}
|
||||
|
||||
for(int i=0; i<ords.length; i++) {
|
||||
int doc = ords[i];
|
||||
if(doc > -1) {
|
||||
collapsedSet.fastSet(doc);
|
||||
collapsedSet.set(doc);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -790,7 +798,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
|
|||
scores[ord] = scorer.score();
|
||||
}
|
||||
}
|
||||
} else if(this.collapsedSet.fastGet(globalDoc)) {
|
||||
} else if(this.collapsedSet.get(globalDoc)) {
|
||||
// Elevated doc so do nothing.
|
||||
} else if(this.nullPolicy == CollapsingPostFilter.NULL_POLICY_COLLAPSE) {
|
||||
if(comp.test(val, nullVal)) {
|
||||
|
@ -801,7 +809,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
|
|||
}
|
||||
}
|
||||
} else if(this.nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) {
|
||||
this.collapsedSet.fastSet(globalDoc);
|
||||
this.collapsedSet.set(globalDoc);
|
||||
if(needsScores) {
|
||||
nullScores.add(scorer.score());
|
||||
}
|
||||
|
@ -858,7 +866,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
|
|||
scores[ord] = scorer.score();
|
||||
}
|
||||
}
|
||||
} else if (this.collapsedSet.fastGet(globalDoc)) {
|
||||
} else if (this.collapsedSet.get(globalDoc)) {
|
||||
//Elevated doc so do nothing
|
||||
} else if(this.nullPolicy == CollapsingPostFilter.NULL_POLICY_COLLAPSE) {
|
||||
if(comp.test(val, nullVal)) {
|
||||
|
@ -869,7 +877,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
|
|||
}
|
||||
}
|
||||
} else if(this.nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) {
|
||||
this.collapsedSet.fastSet(globalDoc);
|
||||
this.collapsedSet.set(globalDoc);
|
||||
if(needsScores) {
|
||||
nullScores.add(scorer.score());
|
||||
}
|
||||
|
@ -927,7 +935,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
|
|||
scores[ord] = scorer.score();
|
||||
}
|
||||
}
|
||||
} else if (this.collapsedSet.fastGet(globalDoc)) {
|
||||
} else if (this.collapsedSet.get(globalDoc)) {
|
||||
//Elevated doc so do nothing
|
||||
} else if(this.nullPolicy == CollapsingPostFilter.NULL_POLICY_COLLAPSE) {
|
||||
if(comp.test(val, nullVal)) {
|
||||
|
@ -938,7 +946,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
|
|||
}
|
||||
}
|
||||
} else if(this.nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) {
|
||||
this.collapsedSet.fastSet(globalDoc);
|
||||
this.collapsedSet.set(globalDoc);
|
||||
if(needsScores) {
|
||||
nullScores.add(scorer.score());
|
||||
}
|
||||
|
@ -1015,7 +1023,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
|
|||
scores[ord] = score;
|
||||
}
|
||||
}
|
||||
} else if (this.collapsedSet.fastGet(globalDoc)) {
|
||||
} else if (this.collapsedSet.get(globalDoc)) {
|
||||
//Elevated doc so do nothing
|
||||
} else if(this.nullPolicy == CollapsingPostFilter.NULL_POLICY_COLLAPSE) {
|
||||
if(comp.test(val, nullVal)) {
|
||||
|
@ -1026,7 +1034,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
|
|||
}
|
||||
}
|
||||
} else if(this.nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) {
|
||||
this.collapsedSet.fastSet(globalDoc);
|
||||
this.collapsedSet.set(globalDoc);
|
||||
if(needsScores) {
|
||||
nullScores.add(score);
|
||||
}
|
||||
|
|
|
@ -18,7 +18,6 @@
|
|||
package org.apache.solr.search;
|
||||
|
||||
import org.apache.lucene.search.Filter;
|
||||
import org.apache.lucene.util.OpenBitSet;
|
||||
import org.apache.solr.common.SolrException;
|
||||
|
||||
/**
|
||||
|
@ -29,7 +28,6 @@ import org.apache.solr.common.SolrException;
|
|||
* a cache and could be shared.
|
||||
* </p>
|
||||
*
|
||||
*
|
||||
* @since solr 0.9
|
||||
*/
|
||||
public interface DocSet /* extends Collection<Integer> */ {
|
||||
|
@ -77,16 +75,6 @@ public interface DocSet /* extends Collection<Integer> */ {
|
|||
*/
|
||||
public DocIterator iterator();
|
||||
|
||||
/**
|
||||
* Returns a BitSet view of the DocSet. Any changes to this BitSet <b>may</b>
|
||||
* be reflected in the DocSet, hence if the DocSet is shared or was returned from
|
||||
* a SolrIndexSearcher method, it's not safe to modify the BitSet.
|
||||
*
|
||||
* @return
|
||||
* An OpenBitSet with the bit number of every docid set in the set.
|
||||
*/
|
||||
public OpenBitSet getBits();
|
||||
|
||||
/**
|
||||
* Returns the approximate amount of memory taken by this DocSet.
|
||||
* This is only an approximation and doesn't take into account java object overhead.
|
||||
|
@ -145,10 +133,11 @@ public interface DocSet /* extends Collection<Integer> */ {
|
|||
public Filter getTopFilter();
|
||||
|
||||
/**
|
||||
* Takes the docs from this set and sets those bits on the target OpenBitSet.
|
||||
* The target should be sized large enough to accommodate all of the documents before calling this method.
|
||||
* Adds all the docs from this set to the target set. The target should be
|
||||
* sized large enough to accommodate all of the documents before calling this
|
||||
* method.
|
||||
*/
|
||||
public void setBitsOn(OpenBitSet target);
|
||||
public void addAllTo(DocSet target);
|
||||
|
||||
public static DocSet EMPTY = new SortedIntDocSet(new int[0], 0);
|
||||
}
|
||||
|
|
|
@ -18,18 +18,32 @@
|
|||
package org.apache.solr.search;
|
||||
|
||||
import org.apache.lucene.index.AtomicReader;
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.lucene.util.Bits;
|
||||
import org.apache.lucene.util.OpenBitSet;
|
||||
import org.apache.lucene.search.DocIdSet;
|
||||
import org.apache.lucene.search.Filter;
|
||||
import org.apache.lucene.search.DocIdSetIterator;
|
||||
import org.apache.lucene.search.BitsFilteredDocIdSet;
|
||||
import org.apache.lucene.index.AtomicReaderContext;
|
||||
import org.apache.lucene.search.BitsFilteredDocIdSet;
|
||||
import org.apache.lucene.search.DocIdSet;
|
||||
import org.apache.lucene.search.DocIdSetIterator;
|
||||
import org.apache.lucene.search.Filter;
|
||||
import org.apache.lucene.util.Bits;
|
||||
import org.apache.lucene.util.FixedBitSet;
|
||||
import org.apache.solr.common.SolrException;
|
||||
|
||||
/** A base class that may be usefull for implementing DocSets */
|
||||
abstract class DocSetBase implements DocSet {
|
||||
|
||||
public static FixedBitSet toBitSet(DocSet set) {
|
||||
if (set instanceof DocSetBase) {
|
||||
return ((DocSetBase) set).getBits();
|
||||
} else {
|
||||
FixedBitSet bits = new FixedBitSet(64);
|
||||
for (DocIterator iter = set.iterator(); iter.hasNext();) {
|
||||
int nextDoc = iter.nextDoc();
|
||||
bits = FixedBitSet.ensureCapacity(bits, nextDoc);
|
||||
bits.set(nextDoc);
|
||||
}
|
||||
return bits;
|
||||
}
|
||||
}
|
||||
|
||||
// Not implemented efficiently... for testing purposes only
|
||||
@Override
|
||||
public boolean equals(Object obj) {
|
||||
|
@ -49,7 +63,7 @@ abstract class DocSetBase implements DocSet {
|
|||
}
|
||||
|
||||
// if (this.size() != other.size()) return false;
|
||||
return this.getBits().equals(other.getBits());
|
||||
return this.getBits().equals(toBitSet(other));
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -69,18 +83,20 @@ abstract class DocSetBase implements DocSet {
|
|||
}
|
||||
|
||||
/**
|
||||
* Inefficient base implementation.
|
||||
*
|
||||
* @see BitDocSet#getBits
|
||||
* Return a {@link FixedBitSet} with a bit set for every document in this
|
||||
* {@link DocSet}. The default implementation iterates on all docs and sets
|
||||
* the relevant bits. You should override if you can provide a more efficient
|
||||
* implementation.
|
||||
*/
|
||||
@Override
|
||||
public OpenBitSet getBits() {
|
||||
OpenBitSet bits = new OpenBitSet();
|
||||
protected FixedBitSet getBits() {
|
||||
FixedBitSet bits = new FixedBitSet(64);
|
||||
for (DocIterator iter = iterator(); iter.hasNext();) {
|
||||
bits.set(iter.nextDoc());
|
||||
int nextDoc = iter.nextDoc();
|
||||
bits = FixedBitSet.ensureCapacity(bits, nextDoc);
|
||||
bits.set(nextDoc);
|
||||
}
|
||||
return bits;
|
||||
};
|
||||
}
|
||||
|
||||
@Override
|
||||
public DocSet intersection(DocSet other) {
|
||||
|
@ -91,8 +107,8 @@ abstract class DocSetBase implements DocSet {
|
|||
}
|
||||
|
||||
// Default... handle with bitsets.
|
||||
OpenBitSet newbits = (OpenBitSet)(this.getBits().clone());
|
||||
newbits.and(other.getBits());
|
||||
FixedBitSet newbits = getBits().clone();
|
||||
newbits.and(toBitSet(other));
|
||||
return new BitDocSet(newbits);
|
||||
}
|
||||
|
||||
|
@ -107,11 +123,11 @@ abstract class DocSetBase implements DocSet {
|
|||
return intersectionSize(other) > 0;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public DocSet union(DocSet other) {
|
||||
OpenBitSet newbits = (OpenBitSet)(this.getBits().clone());
|
||||
newbits.or(other.getBits());
|
||||
FixedBitSet otherBits = toBitSet(other);
|
||||
FixedBitSet newbits = FixedBitSet.ensureCapacity(getBits().clone(), otherBits.length());
|
||||
newbits.or(otherBits);
|
||||
return new BitDocSet(newbits);
|
||||
}
|
||||
|
||||
|
@ -133,8 +149,8 @@ abstract class DocSetBase implements DocSet {
|
|||
|
||||
@Override
|
||||
public DocSet andNot(DocSet other) {
|
||||
OpenBitSet newbits = (OpenBitSet)(this.getBits().clone());
|
||||
newbits.andNot(other.getBits());
|
||||
FixedBitSet newbits = getBits().clone();
|
||||
newbits.andNot(toBitSet(other));
|
||||
return new BitDocSet(newbits);
|
||||
}
|
||||
|
||||
|
@ -145,7 +161,7 @@ abstract class DocSetBase implements DocSet {
|
|||
|
||||
@Override
|
||||
public Filter getTopFilter() {
|
||||
final OpenBitSet bs = getBits();
|
||||
final FixedBitSet bs = getBits();
|
||||
|
||||
return new Filter() {
|
||||
@Override
|
||||
|
@ -189,7 +205,7 @@ abstract class DocSetBase implements DocSet {
|
|||
|
||||
@Override
|
||||
public long cost() {
|
||||
return bs.capacity();
|
||||
return bs.length();
|
||||
}
|
||||
};
|
||||
}
|
||||
|
@ -211,10 +227,10 @@ abstract class DocSetBase implements DocSet {
|
|||
}
|
||||
|
||||
@Override
|
||||
public void setBitsOn(OpenBitSet target) {
|
||||
public void addAllTo(DocSet target) {
|
||||
DocIterator iter = iterator();
|
||||
while (iter.hasNext()) {
|
||||
target.fastSet(iter.nextDoc());
|
||||
target.add(iter.nextDoc());
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -17,13 +17,12 @@ package org.apache.solr.search;
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import org.apache.lucene.index.IndexReader;
|
||||
import java.io.IOException;
|
||||
|
||||
import org.apache.lucene.index.AtomicReaderContext;
|
||||
import org.apache.lucene.search.Collector;
|
||||
import org.apache.lucene.search.Scorer;
|
||||
import org.apache.lucene.util.OpenBitSet;
|
||||
|
||||
import java.io.IOException;
|
||||
import org.apache.lucene.util.FixedBitSet;
|
||||
|
||||
/**
|
||||
*
|
||||
|
@ -31,7 +30,7 @@ import java.io.IOException;
|
|||
|
||||
public class DocSetCollector extends Collector {
|
||||
int pos=0;
|
||||
OpenBitSet bits;
|
||||
FixedBitSet bits;
|
||||
final int maxDoc;
|
||||
final int smallSetSize;
|
||||
int base;
|
||||
|
@ -62,8 +61,8 @@ public class DocSetCollector extends Collector {
|
|||
} else {
|
||||
// this conditional could be removed if BitSet was preallocated, but that
|
||||
// would take up more memory, and add more GC time...
|
||||
if (bits==null) bits = new OpenBitSet(maxDoc);
|
||||
bits.fastSet(doc);
|
||||
if (bits==null) bits = new FixedBitSet(maxDoc);
|
||||
bits.set(doc);
|
||||
}
|
||||
|
||||
pos++;
|
||||
|
@ -75,7 +74,7 @@ public class DocSetCollector extends Collector {
|
|||
return new SortedIntDocSet(scratch, pos);
|
||||
} else {
|
||||
// set the bits for ids that were collected in the array
|
||||
for (int i=0; i<scratch.length; i++) bits.fastSet(scratch[i]);
|
||||
for (int i=0; i<scratch.length; i++) bits.set(scratch[i]);
|
||||
return new BitDocSet(bits,pos);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -17,13 +17,12 @@ package org.apache.solr.search;
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import org.apache.lucene.index.IndexReader;
|
||||
import java.io.IOException;
|
||||
|
||||
import org.apache.lucene.index.AtomicReaderContext;
|
||||
import org.apache.lucene.search.Collector;
|
||||
import org.apache.lucene.search.Scorer;
|
||||
import org.apache.lucene.util.OpenBitSet;
|
||||
|
||||
import java.io.IOException;
|
||||
import org.apache.lucene.util.FixedBitSet;
|
||||
|
||||
/**
|
||||
*
|
||||
|
@ -53,8 +52,8 @@ public class DocSetDelegateCollector extends DocSetCollector {
|
|||
} else {
|
||||
// this conditional could be removed if BitSet was preallocated, but that
|
||||
// would take up more memory, and add more GC time...
|
||||
if (bits==null) bits = new OpenBitSet(maxDoc);
|
||||
bits.fastSet(doc);
|
||||
if (bits==null) bits = new FixedBitSet(maxDoc);
|
||||
bits.set(doc);
|
||||
}
|
||||
|
||||
pos++;
|
||||
|
@ -67,7 +66,7 @@ public class DocSetDelegateCollector extends DocSetCollector {
|
|||
return new SortedIntDocSet(scratch, pos);
|
||||
} else {
|
||||
// set the bits for ids that were collected in the array
|
||||
for (int i=0; i<scratch.length; i++) bits.fastSet(scratch[i]);
|
||||
for (int i=0; i<scratch.length; i++) bits.set(scratch[i]);
|
||||
return new BitDocSet(bits,pos);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -17,13 +17,38 @@
|
|||
|
||||
package org.apache.solr.search;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.commons.lang.ArrayUtils;
|
||||
import org.apache.lucene.index.StorableField;
|
||||
import org.apache.lucene.queries.function.FunctionQuery;
|
||||
import org.apache.lucene.queries.function.ValueSource;
|
||||
import org.apache.lucene.queries.function.valuesource.QueryValueSource;
|
||||
import org.apache.lucene.search.*;
|
||||
import org.apache.lucene.search.grouping.*;
|
||||
import org.apache.lucene.search.CachingCollector;
|
||||
import org.apache.lucene.search.Collector;
|
||||
import org.apache.lucene.search.Filter;
|
||||
import org.apache.lucene.search.MultiCollector;
|
||||
import org.apache.lucene.search.Query;
|
||||
import org.apache.lucene.search.ScoreDoc;
|
||||
import org.apache.lucene.search.Sort;
|
||||
import org.apache.lucene.search.SortField;
|
||||
import org.apache.lucene.search.TimeLimitingCollector;
|
||||
import org.apache.lucene.search.TopDocs;
|
||||
import org.apache.lucene.search.TopDocsCollector;
|
||||
import org.apache.lucene.search.TopFieldCollector;
|
||||
import org.apache.lucene.search.TopScoreDocCollector;
|
||||
import org.apache.lucene.search.TotalHitCountCollector;
|
||||
import org.apache.lucene.search.grouping.AbstractAllGroupHeadsCollector;
|
||||
import org.apache.lucene.search.grouping.GroupDocs;
|
||||
import org.apache.lucene.search.grouping.SearchGroup;
|
||||
import org.apache.lucene.search.grouping.TopGroups;
|
||||
import org.apache.lucene.search.grouping.function.FunctionAllGroupHeadsCollector;
|
||||
import org.apache.lucene.search.grouping.function.FunctionAllGroupsCollector;
|
||||
import org.apache.lucene.search.grouping.function.FunctionFirstPassGroupingCollector;
|
||||
|
@ -33,8 +58,6 @@ import org.apache.lucene.search.grouping.term.TermAllGroupsCollector;
|
|||
import org.apache.lucene.search.grouping.term.TermFirstPassGroupingCollector;
|
||||
import org.apache.lucene.search.grouping.term.TermSecondPassGroupingCollector;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.apache.lucene.util.FixedBitSet;
|
||||
import org.apache.lucene.util.OpenBitSet;
|
||||
import org.apache.lucene.util.mutable.MutableValue;
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.common.util.NamedList;
|
||||
|
@ -47,9 +70,6 @@ import org.apache.solr.search.grouping.collector.FilterCollector;
|
|||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.*;
|
||||
|
||||
/**
|
||||
* Basic Solr Grouping infrastructure.
|
||||
* Warning NOT thread save!
|
||||
|
@ -353,10 +373,7 @@ public class Grouping {
|
|||
}
|
||||
|
||||
if (getGroupedDocSet && allGroupHeadsCollector != null) {
|
||||
FixedBitSet fixedBitSet = allGroupHeadsCollector.retrieveGroupHeads(maxDoc);
|
||||
long[] bits = fixedBitSet.getBits();
|
||||
OpenBitSet openBitSet = new OpenBitSet(bits, bits.length);
|
||||
qr.setDocSet(new BitDocSet(openBitSet));
|
||||
qr.setDocSet(new BitDocSet(allGroupHeadsCollector.retrieveGroupHeads(maxDoc)));
|
||||
} else if (getDocSet) {
|
||||
qr.setDocSet(setCollector.getDocSet());
|
||||
}
|
||||
|
|
|
@ -16,12 +16,33 @@
|
|||
*/
|
||||
package org.apache.solr.search;
|
||||
|
||||
import org.apache.lucene.index.*;
|
||||
import org.apache.lucene.search.*;
|
||||
import java.io.Closeable;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.lucene.index.AtomicReaderContext;
|
||||
import org.apache.lucene.index.DocsEnum;
|
||||
import org.apache.lucene.index.Fields;
|
||||
import org.apache.lucene.index.IndexReader;
|
||||
import org.apache.lucene.index.MultiDocsEnum;
|
||||
import org.apache.lucene.index.Terms;
|
||||
import org.apache.lucene.index.TermsEnum;
|
||||
import org.apache.lucene.search.ComplexExplanation;
|
||||
import org.apache.lucene.search.DocIdSet;
|
||||
import org.apache.lucene.search.DocIdSetIterator;
|
||||
import org.apache.lucene.search.Explanation;
|
||||
import org.apache.lucene.search.Filter;
|
||||
import org.apache.lucene.search.IndexSearcher;
|
||||
import org.apache.lucene.search.Query;
|
||||
import org.apache.lucene.search.Scorer;
|
||||
import org.apache.lucene.search.Weight;
|
||||
import org.apache.lucene.search.similarities.Similarity;
|
||||
import org.apache.lucene.util.Bits;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.apache.lucene.util.OpenBitSet;
|
||||
import org.apache.lucene.util.FixedBitSet;
|
||||
import org.apache.lucene.util.StringHelper;
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.common.params.SolrParams;
|
||||
|
@ -36,13 +57,6 @@ import org.apache.solr.request.SolrRequestInfo;
|
|||
import org.apache.solr.schema.TrieField;
|
||||
import org.apache.solr.util.RefCounted;
|
||||
|
||||
import java.io.Closeable;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
|
||||
public class JoinQParserPlugin extends QParserPlugin {
|
||||
public static final String NAME = "join";
|
||||
|
@ -270,7 +284,7 @@ class JoinQuery extends Query {
|
|||
|
||||
|
||||
public DocSet getDocSet() throws IOException {
|
||||
OpenBitSet resultBits = null;
|
||||
FixedBitSet resultBits = null;
|
||||
|
||||
// minimum docFreq to use the cache
|
||||
int minDocFreqFrom = Math.max(5, fromSearcher.maxDoc() >> 13);
|
||||
|
@ -387,7 +401,7 @@ class JoinQuery extends Query {
|
|||
int df = toTermsEnum.docFreq();
|
||||
toTermHitsTotalDf += df;
|
||||
if (resultBits==null && df + resultListDocs > maxSortedIntSize && resultList.size() > 0) {
|
||||
resultBits = new OpenBitSet(toSearcher.maxDoc());
|
||||
resultBits = new FixedBitSet(toSearcher.maxDoc());
|
||||
}
|
||||
|
||||
// if we don't have a bitset yet, or if the resulting set will be too large
|
||||
|
@ -397,10 +411,10 @@ class JoinQuery extends Query {
|
|||
DocSet toTermSet = toSearcher.getDocSet(toDeState);
|
||||
resultListDocs += toTermSet.size();
|
||||
if (resultBits != null) {
|
||||
toTermSet.setBitsOn(resultBits);
|
||||
toTermSet.addAllTo(new BitDocSet(resultBits));
|
||||
} else {
|
||||
if (toTermSet instanceof BitDocSet) {
|
||||
resultBits = (OpenBitSet)((BitDocSet)toTermSet).bits.clone();
|
||||
resultBits = ((BitDocSet)toTermSet).bits.clone();
|
||||
} else {
|
||||
resultList.add(toTermSet);
|
||||
}
|
||||
|
@ -422,14 +436,14 @@ class JoinQuery extends Query {
|
|||
int docid;
|
||||
while ((docid = sub.docsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
|
||||
resultListDocs++;
|
||||
resultBits.fastSet(docid + base);
|
||||
resultBits.set(docid + base);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
int docid;
|
||||
while ((docid = docsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
|
||||
resultListDocs++;
|
||||
resultBits.fastSet(docid);
|
||||
resultBits.set(docid);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -443,10 +457,11 @@ class JoinQuery extends Query {
|
|||
smallSetsDeferred = resultList.size();
|
||||
|
||||
if (resultBits != null) {
|
||||
BitDocSet bitSet = new BitDocSet(resultBits);
|
||||
for (DocSet set : resultList) {
|
||||
set.setBitsOn(resultBits);
|
||||
set.addAllTo(bitSet);
|
||||
}
|
||||
return new BitDocSet(resultBits);
|
||||
return bitSet;
|
||||
}
|
||||
|
||||
if (resultList.size()==0) {
|
||||
|
|
|
@ -66,11 +66,11 @@ import org.apache.lucene.search.ConstantScoreQuery;
|
|||
import org.apache.lucene.search.DocIdSet;
|
||||
import org.apache.lucene.search.DocIdSetIterator;
|
||||
import org.apache.lucene.search.Explanation;
|
||||
import org.apache.lucene.search.FieldDoc;
|
||||
import org.apache.lucene.search.Filter;
|
||||
import org.apache.lucene.search.IndexSearcher;
|
||||
import org.apache.lucene.search.MatchAllDocsQuery;
|
||||
import org.apache.lucene.search.Query;
|
||||
import org.apache.lucene.search.FieldDoc;
|
||||
import org.apache.lucene.search.ScoreDoc;
|
||||
import org.apache.lucene.search.Scorer;
|
||||
import org.apache.lucene.search.Sort;
|
||||
|
@ -78,16 +78,16 @@ import org.apache.lucene.search.SortField;
|
|||
import org.apache.lucene.search.TermQuery;
|
||||
import org.apache.lucene.search.TimeLimitingCollector;
|
||||
import org.apache.lucene.search.TopDocs;
|
||||
import org.apache.lucene.search.TopFieldDocs;
|
||||
import org.apache.lucene.search.TopDocsCollector;
|
||||
import org.apache.lucene.search.TopFieldCollector;
|
||||
import org.apache.lucene.search.TopFieldDocs;
|
||||
import org.apache.lucene.search.TopScoreDocCollector;
|
||||
import org.apache.lucene.search.TotalHitCountCollector;
|
||||
import org.apache.lucene.search.Weight;
|
||||
import org.apache.lucene.store.Directory;
|
||||
import org.apache.lucene.util.Bits;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.apache.lucene.util.OpenBitSet;
|
||||
import org.apache.lucene.util.FixedBitSet;
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.common.SolrException.ErrorCode;
|
||||
import org.apache.solr.common.params.ModifiableSolrParams;
|
||||
|
@ -105,7 +105,6 @@ import org.apache.solr.request.UnInvertedField;
|
|||
import org.apache.solr.response.SolrQueryResponse;
|
||||
import org.apache.solr.schema.IndexSchema;
|
||||
import org.apache.solr.schema.SchemaField;
|
||||
import org.apache.solr.spelling.QueryConverter;
|
||||
import org.apache.solr.update.SolrIndexConfig;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
@ -1088,7 +1087,7 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable,SolrIn
|
|||
final int[] docs = deState.scratch;
|
||||
int upto = 0;
|
||||
int bitsSet = 0;
|
||||
OpenBitSet obs = null;
|
||||
FixedBitSet fbs = null;
|
||||
|
||||
DocsEnum docsEnum = deState.termsEnum.docs(deState.liveDocs, deState.docsEnum, DocsEnum.FLAG_NONE);
|
||||
if (deState.docsEnum == null) {
|
||||
|
@ -1105,9 +1104,9 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable,SolrIn
|
|||
int docid;
|
||||
|
||||
if (largestPossible > docs.length) {
|
||||
if (obs == null) obs = new OpenBitSet(maxDoc());
|
||||
if (fbs == null) fbs = new FixedBitSet(maxDoc());
|
||||
while ((docid = sub.docsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
|
||||
obs.fastSet(docid + base);
|
||||
fbs.set(docid + base);
|
||||
bitsSet++;
|
||||
}
|
||||
} else {
|
||||
|
@ -1119,9 +1118,9 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable,SolrIn
|
|||
} else {
|
||||
int docid;
|
||||
if (largestPossible > docs.length) {
|
||||
if (obs == null) obs = new OpenBitSet(maxDoc());
|
||||
if (fbs == null) fbs = new FixedBitSet(maxDoc());
|
||||
while ((docid = docsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
|
||||
obs.fastSet(docid);
|
||||
fbs.set(docid);
|
||||
bitsSet++;
|
||||
}
|
||||
} else {
|
||||
|
@ -1132,12 +1131,12 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable,SolrIn
|
|||
}
|
||||
|
||||
DocSet result;
|
||||
if (obs != null) {
|
||||
if (fbs != null) {
|
||||
for (int i=0; i<upto; i++) {
|
||||
obs.fastSet(docs[i]);
|
||||
fbs.set(docs[i]);
|
||||
}
|
||||
bitsSet += upto;
|
||||
result = new BitDocSet(obs, bitsSet);
|
||||
result = new BitDocSet(fbs, bitsSet);
|
||||
} else {
|
||||
result = upto==0 ? DocSet.EMPTY : new SortedIntDocSet(Arrays.copyOf(docs, upto));
|
||||
}
|
||||
|
|
|
@ -18,13 +18,13 @@
|
|||
package org.apache.solr.search;
|
||||
|
||||
import org.apache.lucene.index.AtomicReader;
|
||||
import org.apache.lucene.util.Bits;
|
||||
import org.apache.lucene.util.OpenBitSet;
|
||||
import org.apache.lucene.index.AtomicReaderContext;
|
||||
import org.apache.lucene.search.BitsFilteredDocIdSet;
|
||||
import org.apache.lucene.search.DocIdSet;
|
||||
import org.apache.lucene.search.DocIdSetIterator;
|
||||
import org.apache.lucene.search.Filter;
|
||||
import org.apache.lucene.index.AtomicReaderContext;
|
||||
import org.apache.lucene.util.Bits;
|
||||
import org.apache.lucene.util.FixedBitSet;
|
||||
|
||||
/**
|
||||
* <code>SortedIntDocSet</code> represents a sorted set of Lucene Document Ids.
|
||||
|
@ -563,13 +563,12 @@ public class SortedIntDocSet extends DocSetBase {
|
|||
}
|
||||
|
||||
@Override
|
||||
public void setBitsOn(OpenBitSet target) {
|
||||
public void addAllTo(DocSet target) {
|
||||
for (int doc : docs) {
|
||||
target.fastSet(doc);
|
||||
target.add(doc);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public boolean exists(int doc) {
|
||||
// this could be faster by estimating where in the list the doc is likely to appear,
|
||||
|
@ -630,16 +629,15 @@ public class SortedIntDocSet extends DocSetBase {
|
|||
}
|
||||
|
||||
@Override
|
||||
public OpenBitSet getBits() {
|
||||
public FixedBitSet getBits() {
|
||||
int maxDoc = size() > 0 ? docs[size()-1] : 0;
|
||||
OpenBitSet bs = new OpenBitSet(maxDoc+1);
|
||||
FixedBitSet bs = new FixedBitSet(maxDoc+1);
|
||||
for (int doc : docs) {
|
||||
bs.fastSet(doc);
|
||||
bs.set(doc);
|
||||
}
|
||||
return bs;
|
||||
}
|
||||
|
||||
|
||||
public static int findIndex(int[] arr, int value, int low, int high) {
|
||||
// binary search
|
||||
while (low <= high) {
|
||||
|
|
|
@ -17,22 +17,31 @@ package org.apache.solr.search.grouping;
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import org.apache.lucene.search.*;
|
||||
import org.apache.lucene.search.grouping.AbstractAllGroupHeadsCollector;
|
||||
import org.apache.lucene.search.grouping.term.TermAllGroupHeadsCollector;
|
||||
import org.apache.lucene.util.OpenBitSet;
|
||||
import org.apache.solr.common.util.NamedList;
|
||||
import org.apache.solr.search.*;
|
||||
import org.apache.solr.search.SolrIndexSearcher.ProcessedFilter;
|
||||
import org.apache.solr.search.QueryUtils;
|
||||
import org.apache.solr.search.grouping.distributed.shardresultserializer.ShardResultTransformer;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.lucene.search.Collector;
|
||||
import org.apache.lucene.search.Filter;
|
||||
import org.apache.lucene.search.MultiCollector;
|
||||
import org.apache.lucene.search.Query;
|
||||
import org.apache.lucene.search.TimeLimitingCollector;
|
||||
import org.apache.lucene.search.TotalHitCountCollector;
|
||||
import org.apache.lucene.search.grouping.AbstractAllGroupHeadsCollector;
|
||||
import org.apache.lucene.search.grouping.term.TermAllGroupHeadsCollector;
|
||||
import org.apache.lucene.util.FixedBitSet;
|
||||
import org.apache.solr.common.util.NamedList;
|
||||
import org.apache.solr.search.BitDocSet;
|
||||
import org.apache.solr.search.DocSet;
|
||||
import org.apache.solr.search.DocSetCollector;
|
||||
import org.apache.solr.search.DocSetDelegateCollector;
|
||||
import org.apache.solr.search.QueryUtils;
|
||||
import org.apache.solr.search.SolrIndexSearcher;
|
||||
import org.apache.solr.search.SolrIndexSearcher.ProcessedFilter;
|
||||
import org.apache.solr.search.grouping.distributed.shardresultserializer.ShardResultTransformer;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
/**
|
||||
* Responsible for executing a search with a number of {@link Command} instances.
|
||||
* A typical search can have more then one {@link Command} instances.
|
||||
|
@ -159,9 +168,7 @@ public class CommandHandler {
|
|||
searchWithTimeLimiter(query, filter, MultiCollector.wrap(collectors.toArray(new Collector[collectors.size()])));
|
||||
}
|
||||
|
||||
int maxDoc = searcher.maxDoc();
|
||||
long[] bits = termAllGroupHeadsCollector.retrieveGroupHeads(maxDoc).getBits();
|
||||
return new BitDocSet(new OpenBitSet(bits, bits.length));
|
||||
return new BitDocSet(termAllGroupHeadsCollector.retrieveGroupHeads(searcher.maxDoc()));
|
||||
}
|
||||
|
||||
private DocSet computeDocSet(Query query, ProcessedFilter filter, List<Collector> collectors) throws IOException {
|
||||
|
|
|
@ -17,30 +17,30 @@
|
|||
|
||||
package org.apache.solr.search.join;
|
||||
|
||||
import org.apache.lucene.util.OpenBitSet;
|
||||
import org.apache.lucene.util.FixedBitSet;
|
||||
|
||||
class BitSetSlice {
|
||||
private final OpenBitSet obs;
|
||||
private final FixedBitSet fbs;
|
||||
private final int off;
|
||||
private final int len;
|
||||
|
||||
BitSetSlice(OpenBitSet obs, int off, int len) {
|
||||
this.obs = obs;
|
||||
BitSetSlice(FixedBitSet fbs, int off, int len) {
|
||||
this.fbs = fbs;
|
||||
this.off = off;
|
||||
this.len = len;
|
||||
}
|
||||
|
||||
public boolean get(int pos) {
|
||||
return obs.get(pos + off);
|
||||
return fbs.get(pos + off);
|
||||
}
|
||||
|
||||
public int prevSetBit(int pos) {
|
||||
int result = obs.prevSetBit(pos + off) - off;
|
||||
int result = fbs.prevSetBit(pos + off) - off;
|
||||
return (result < 0) ? -1 : result;
|
||||
}
|
||||
|
||||
public int nextSetBit(int pos) {
|
||||
int result = obs.nextSetBit(pos + off) - off;
|
||||
int result = fbs.nextSetBit(pos + off) - off;
|
||||
return (result < 0 || result >= len) ? -1 : result;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -19,7 +19,7 @@ package org.apache.solr.store.blockcache;
|
|||
|
||||
import java.util.concurrent.atomic.AtomicLongArray;
|
||||
|
||||
import org.apache.lucene.util.OpenBitSet;
|
||||
import org.apache.lucene.util.LongBitSet;
|
||||
|
||||
public class BlockLocks {
|
||||
|
||||
|
@ -27,7 +27,7 @@ public class BlockLocks {
|
|||
private int wlen;
|
||||
|
||||
public BlockLocks(long numBits) {
|
||||
int length = OpenBitSet.bits2words(numBits);
|
||||
int length = LongBitSet.bits2words(numBits);
|
||||
bits = new AtomicLongArray(length);
|
||||
wlen = length;
|
||||
}
|
||||
|
|
|
@ -17,6 +17,10 @@
|
|||
|
||||
package org.apache.solr.update;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.lucene.index.AtomicReader;
|
||||
import org.apache.lucene.index.AtomicReaderContext;
|
||||
import org.apache.lucene.index.DocsEnum;
|
||||
|
@ -26,11 +30,12 @@ import org.apache.lucene.index.IndexReader;
|
|||
import org.apache.lucene.index.IndexWriter;
|
||||
import org.apache.lucene.index.Terms;
|
||||
import org.apache.lucene.index.TermsEnum;
|
||||
import org.apache.lucene.search.DocIdSetIterator;
|
||||
import org.apache.lucene.util.Bits;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.apache.lucene.util.CharsRef;
|
||||
import org.apache.lucene.util.FixedBitSet;
|
||||
import org.apache.lucene.util.IOUtils;
|
||||
import org.apache.lucene.util.OpenBitSet;
|
||||
import org.apache.solr.common.cloud.CompositeIdRouter;
|
||||
import org.apache.solr.common.cloud.DocRouter;
|
||||
import org.apache.solr.common.cloud.HashBasedRouter;
|
||||
|
@ -41,10 +46,6 @@ import org.apache.solr.util.RefCounted;
|
|||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class SolrIndexSplitter {
|
||||
public static Logger log = LoggerFactory.getLogger(SolrIndexSplitter.class);
|
||||
|
||||
|
@ -89,13 +90,13 @@ public class SolrIndexSplitter {
|
|||
public void split() throws IOException {
|
||||
|
||||
List<AtomicReaderContext> leaves = searcher.getTopReaderContext().leaves();
|
||||
List<OpenBitSet[]> segmentDocSets = new ArrayList<OpenBitSet[]>(leaves.size());
|
||||
List<FixedBitSet[]> segmentDocSets = new ArrayList<FixedBitSet[]>(leaves.size());
|
||||
|
||||
log.info("SolrIndexSplitter: partitions=" + numPieces + " segments="+leaves.size());
|
||||
|
||||
for (AtomicReaderContext readerContext : leaves) {
|
||||
assert readerContext.ordInParent == segmentDocSets.size(); // make sure we're going in order
|
||||
OpenBitSet[] docSets = split(readerContext);
|
||||
FixedBitSet[] docSets = split(readerContext);
|
||||
segmentDocSets.add( docSets );
|
||||
}
|
||||
|
||||
|
@ -150,11 +151,11 @@ public class SolrIndexSplitter {
|
|||
|
||||
|
||||
|
||||
OpenBitSet[] split(AtomicReaderContext readerContext) throws IOException {
|
||||
FixedBitSet[] split(AtomicReaderContext readerContext) throws IOException {
|
||||
AtomicReader reader = readerContext.reader();
|
||||
OpenBitSet[] docSets = new OpenBitSet[numPieces];
|
||||
FixedBitSet[] docSets = new FixedBitSet[numPieces];
|
||||
for (int i=0; i<docSets.length; i++) {
|
||||
docSets[i] = new OpenBitSet(reader.maxDoc());
|
||||
docSets[i] = new FixedBitSet(reader.maxDoc());
|
||||
}
|
||||
Bits liveDocs = reader.getLiveDocs();
|
||||
|
||||
|
@ -196,14 +197,14 @@ public class SolrIndexSplitter {
|
|||
docsEnum = termsEnum.docs(liveDocs, docsEnum, DocsEnum.FLAG_NONE);
|
||||
for (;;) {
|
||||
int doc = docsEnum.nextDoc();
|
||||
if (doc == DocsEnum.NO_MORE_DOCS) break;
|
||||
if (doc == DocIdSetIterator.NO_MORE_DOCS) break;
|
||||
if (ranges == null) {
|
||||
docSets[currPartition].fastSet(doc);
|
||||
docSets[currPartition].set(doc);
|
||||
currPartition = (currPartition + 1) % numPieces;
|
||||
} else {
|
||||
for (int i=0; i<rangesArr.length; i++) { // inner-loop: use array here for extra speed.
|
||||
if (rangesArr[i].includes(hash)) {
|
||||
docSets[i].fastSet(doc);
|
||||
docSets[i].set(doc);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -232,13 +233,13 @@ public class SolrIndexSplitter {
|
|||
|
||||
// change livedocs on the reader to delete those docs we don't want
|
||||
static class LiveDocsReader extends FilterAtomicReader {
|
||||
final OpenBitSet liveDocs;
|
||||
final FixedBitSet liveDocs;
|
||||
final int numDocs;
|
||||
|
||||
public LiveDocsReader(AtomicReaderContext context, OpenBitSet liveDocs) throws IOException {
|
||||
public LiveDocsReader(AtomicReaderContext context, FixedBitSet liveDocs) throws IOException {
|
||||
super(context.reader());
|
||||
this.liveDocs = liveDocs;
|
||||
this.numDocs = (int)liveDocs.cardinality();
|
||||
this.numDocs = liveDocs.cardinality();
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -253,6 +254,3 @@ public class SolrIndexSplitter {
|
|||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
|
|
@ -17,13 +17,10 @@
|
|||
|
||||
package org.apache.solr.search;
|
||||
|
||||
import org.apache.solr.search.BitDocSet;
|
||||
import org.apache.solr.search.HashDocSet;
|
||||
import org.apache.solr.search.DocSet;
|
||||
import org.apache.lucene.util.OpenBitSet;
|
||||
|
||||
import java.util.Random;
|
||||
import java.util.BitSet;
|
||||
import java.util.Random;
|
||||
|
||||
import org.apache.lucene.util.FixedBitSet;
|
||||
|
||||
/**
|
||||
*/
|
||||
|
@ -38,21 +35,20 @@ public class DocSetPerf {
|
|||
|
||||
static Random rand = new Random();
|
||||
|
||||
|
||||
static OpenBitSet bs;
|
||||
static FixedBitSet bs;
|
||||
static BitDocSet bds;
|
||||
static HashDocSet hds;
|
||||
static int[] ids; // not unique
|
||||
|
||||
static void generate(int maxSize, int bitsToSet) {
|
||||
bs = new OpenBitSet(maxSize);
|
||||
bs = new FixedBitSet(maxSize);
|
||||
ids = new int[bitsToSet];
|
||||
int count=0;
|
||||
if (maxSize>0) {
|
||||
for (int i=0; i<bitsToSet; i++) {
|
||||
int id=rand.nextInt(maxSize);
|
||||
if (!bs.get(id)) {
|
||||
bs.fastSet(id);
|
||||
bs.set(id);
|
||||
ids[count++]=id;
|
||||
}
|
||||
}
|
||||
|
@ -80,7 +76,7 @@ public class DocSetPerf {
|
|||
|
||||
int ret=0;
|
||||
|
||||
OpenBitSet[] sets = new OpenBitSet[numSets];
|
||||
FixedBitSet[] sets = new FixedBitSet[numSets];
|
||||
DocSet[] bset = new DocSet[numSets];
|
||||
DocSet[] hset = new DocSet[numSets];
|
||||
BitSet scratch=new BitSet();
|
||||
|
@ -97,14 +93,14 @@ public class DocSetPerf {
|
|||
if ("test".equals(test)) {
|
||||
for (int it=0; it<iter; it++) {
|
||||
generate(randSize ? rand.nextInt(bitSetSize) : bitSetSize, numBitsSet);
|
||||
OpenBitSet bs1=bs;
|
||||
FixedBitSet bs1=bs;
|
||||
BitDocSet bds1=bds;
|
||||
HashDocSet hds1=hds;
|
||||
generate(randSize ? rand.nextInt(bitSetSize) : bitSetSize, numBitsSet);
|
||||
|
||||
OpenBitSet res = ((OpenBitSet)bs1.clone());
|
||||
FixedBitSet res = bs1.clone();
|
||||
res.and(bs);
|
||||
int icount = (int)res.cardinality();
|
||||
int icount = res.cardinality();
|
||||
|
||||
test(bds1.intersection(bds).size() == icount);
|
||||
test(bds1.intersectionSize(bds) == icount);
|
||||
|
@ -167,8 +163,6 @@ public class DocSetPerf {
|
|||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
long end = System.currentTimeMillis();
|
||||
System.out.println("TIME="+(end-start));
|
||||
|
||||
|
@ -176,6 +170,4 @@ public class DocSetPerf {
|
|||
System.out.println("ret="+ret);
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
|
|
@ -39,9 +39,9 @@ import org.apache.lucene.search.DocIdSet;
|
|||
import org.apache.lucene.search.DocIdSetIterator;
|
||||
import org.apache.lucene.search.Filter;
|
||||
import org.apache.lucene.util.Bits;
|
||||
import org.apache.lucene.util.FixedBitSet;
|
||||
import org.apache.lucene.util.FixedBitSet.FixedBitSetIterator;
|
||||
import org.apache.lucene.util.LuceneTestCase;
|
||||
import org.apache.lucene.util.OpenBitSet;
|
||||
import org.apache.lucene.util.OpenBitSetIterator;
|
||||
|
||||
/**
|
||||
*
|
||||
|
@ -56,46 +56,45 @@ public class TestDocSet extends LuceneTestCase {
|
|||
rand = random();
|
||||
}
|
||||
|
||||
public OpenBitSet getRandomSet(int sz, int bitsToSet) {
|
||||
OpenBitSet bs = new OpenBitSet(sz);
|
||||
public FixedBitSet getRandomSet(int sz, int bitsToSet) {
|
||||
FixedBitSet bs = new FixedBitSet(sz);
|
||||
if (sz==0) return bs;
|
||||
for (int i=0; i<bitsToSet; i++) {
|
||||
bs.fastSet(rand.nextInt(sz));
|
||||
bs.set(rand.nextInt(sz));
|
||||
}
|
||||
return bs;
|
||||
}
|
||||
|
||||
public DocSet getHashDocSet(OpenBitSet bs) {
|
||||
int[] docs = new int[(int)bs.cardinality()];
|
||||
OpenBitSetIterator iter = new OpenBitSetIterator(bs);
|
||||
public DocSet getHashDocSet(FixedBitSet bs) {
|
||||
int[] docs = new int[bs.cardinality()];
|
||||
FixedBitSetIterator iter = new FixedBitSetIterator(bs);
|
||||
for (int i=0; i<docs.length; i++) {
|
||||
docs[i] = iter.nextDoc();
|
||||
}
|
||||
return new HashDocSet(docs,0,docs.length);
|
||||
}
|
||||
|
||||
public DocSet getIntDocSet(OpenBitSet bs) {
|
||||
int[] docs = new int[(int)bs.cardinality()];
|
||||
OpenBitSetIterator iter = new OpenBitSetIterator(bs);
|
||||
public DocSet getIntDocSet(FixedBitSet bs) {
|
||||
int[] docs = new int[bs.cardinality()];
|
||||
FixedBitSetIterator iter = new FixedBitSetIterator(bs);
|
||||
for (int i=0; i<docs.length; i++) {
|
||||
docs[i] = iter.nextDoc();
|
||||
}
|
||||
return new SortedIntDocSet(docs);
|
||||
}
|
||||
|
||||
|
||||
public DocSet getBitDocSet(OpenBitSet bs) {
|
||||
public DocSet getBitDocSet(FixedBitSet bs) {
|
||||
return new BitDocSet(bs);
|
||||
}
|
||||
|
||||
public DocSet getDocSlice(OpenBitSet bs) {
|
||||
int len = (int)bs.cardinality();
|
||||
public DocSet getDocSlice(FixedBitSet bs) {
|
||||
int len = bs.cardinality();
|
||||
int[] arr = new int[len+5];
|
||||
arr[0]=10; arr[1]=20; arr[2]=30; arr[arr.length-1]=1; arr[arr.length-2]=2;
|
||||
int offset = 3;
|
||||
int end = offset + len;
|
||||
|
||||
OpenBitSetIterator iter = new OpenBitSetIterator(bs);
|
||||
FixedBitSetIterator iter = new FixedBitSetIterator(bs);
|
||||
// put in opposite order... DocLists are not ordered.
|
||||
for (int i=end-1; i>=offset; i--) {
|
||||
arr[i] = iter.nextDoc();
|
||||
|
@ -105,7 +104,7 @@ public class TestDocSet extends LuceneTestCase {
|
|||
}
|
||||
|
||||
|
||||
public DocSet getDocSet(OpenBitSet bs) {
|
||||
public DocSet getDocSet(FixedBitSet bs) {
|
||||
switch(rand.nextInt(10)) {
|
||||
case 0: return getHashDocSet(bs);
|
||||
|
||||
|
@ -124,7 +123,7 @@ public class TestDocSet extends LuceneTestCase {
|
|||
return null;
|
||||
}
|
||||
|
||||
public void checkEqual(OpenBitSet bs, DocSet set) {
|
||||
public void checkEqual(FixedBitSet bs, DocSet set) {
|
||||
for (int i=0; i<set.size(); i++) {
|
||||
assertEquals(bs.get(i), set.exists(i));
|
||||
}
|
||||
|
@ -152,8 +151,8 @@ public class TestDocSet extends LuceneTestCase {
|
|||
protected void doSingle(int maxSize) {
|
||||
int sz = rand.nextInt(maxSize+1);
|
||||
int sz2 = rand.nextInt(maxSize);
|
||||
OpenBitSet bs1 = getRandomSet(sz, rand.nextInt(sz+1));
|
||||
OpenBitSet bs2 = getRandomSet(sz, rand.nextInt(sz2+1));
|
||||
FixedBitSet bs1 = getRandomSet(sz, rand.nextInt(sz+1));
|
||||
FixedBitSet bs2 = getRandomSet(sz, rand.nextInt(sz2+1));
|
||||
|
||||
DocSet a1 = new BitDocSet(bs1);
|
||||
DocSet a2 = new BitDocSet(bs2);
|
||||
|
@ -166,10 +165,10 @@ public class TestDocSet extends LuceneTestCase {
|
|||
iter(a1,b1);
|
||||
iter(a2,b2);
|
||||
|
||||
OpenBitSet a_and = (OpenBitSet) bs1.clone(); a_and.and(bs2);
|
||||
OpenBitSet a_or = (OpenBitSet) bs1.clone(); a_or.or(bs2);
|
||||
// OpenBitSet a_xor = (OpenBitSet)bs1.clone(); a_xor.xor(bs2);
|
||||
OpenBitSet a_andn = (OpenBitSet) bs1.clone(); a_andn.andNot(bs2);
|
||||
FixedBitSet a_and = bs1.clone(); a_and.and(bs2);
|
||||
FixedBitSet a_or = bs1.clone(); a_or.or(bs2);
|
||||
// FixedBitSet a_xor = bs1.clone(); a_xor.xor(bs2);
|
||||
FixedBitSet a_andn = bs1.clone(); a_andn.andNot(bs2);
|
||||
|
||||
checkEqual(a_and, b1.intersection(b2));
|
||||
checkEqual(a_or, b1.union(b2));
|
||||
|
@ -180,7 +179,6 @@ public class TestDocSet extends LuceneTestCase {
|
|||
assertEquals(a_andn.cardinality(), b1.andNotSize(b2));
|
||||
}
|
||||
|
||||
|
||||
public void doMany(int maxSz, int iter) {
|
||||
for (int i=0; i<iter; i++) {
|
||||
doSingle(maxSz);
|
||||
|
@ -197,7 +195,7 @@ public class TestDocSet extends LuceneTestCase {
|
|||
}
|
||||
|
||||
public DocSet getRandomDocSet(int n, int maxDoc) {
|
||||
OpenBitSet obs = new OpenBitSet(maxDoc);
|
||||
FixedBitSet obs = new FixedBitSet(maxDoc);
|
||||
int[] a = new int[n];
|
||||
for (int i=0; i<n; i++) {
|
||||
for(;;) {
|
||||
|
@ -264,7 +262,7 @@ public class TestDocSet extends LuceneTestCase {
|
|||
}
|
||||
***/
|
||||
|
||||
public static int smallSetType = 0; // 0==sortedint, 1==hash, 2==openbitset
|
||||
public static int smallSetType = 0; // 0==sortedint, 1==hash, 2==FixedBitSet
|
||||
public static int smallSetCuttoff=3000;
|
||||
|
||||
/***
|
||||
|
@ -474,7 +472,7 @@ public class TestDocSet extends LuceneTestCase {
|
|||
|
||||
public void doFilterTest(IndexReader reader) throws IOException {
|
||||
IndexReaderContext topLevelContext = reader.getContext();
|
||||
OpenBitSet bs = getRandomSet(reader.maxDoc(), rand.nextInt(reader.maxDoc()+1));
|
||||
FixedBitSet bs = getRandomSet(reader.maxDoc(), rand.nextInt(reader.maxDoc()+1));
|
||||
DocSet a = new BitDocSet(bs);
|
||||
DocSet b = getIntDocSet(bs);
|
||||
|
||||
|
|
|
@ -18,7 +18,7 @@
|
|||
package org.apache.solr.search;
|
||||
|
||||
|
||||
import org.apache.lucene.util.OpenBitSet;
|
||||
import org.apache.lucene.util.FixedBitSet;
|
||||
import org.apache.solr.SolrTestCaseJ4;
|
||||
import org.apache.solr.common.params.SolrParams;
|
||||
import org.apache.solr.common.SolrException;
|
||||
|
@ -114,18 +114,18 @@ public class TestFiltering extends SolrTestCaseJ4 {
|
|||
|
||||
class Model {
|
||||
int indexSize;
|
||||
OpenBitSet answer;
|
||||
OpenBitSet multiSelect;
|
||||
OpenBitSet facetQuery;
|
||||
FixedBitSet answer;
|
||||
FixedBitSet multiSelect;
|
||||
FixedBitSet facetQuery;
|
||||
|
||||
void clear() {
|
||||
answer = new OpenBitSet(indexSize);
|
||||
answer = new FixedBitSet(indexSize);
|
||||
answer.set(0, indexSize);
|
||||
|
||||
multiSelect = new OpenBitSet(indexSize);
|
||||
multiSelect = new FixedBitSet(indexSize);
|
||||
multiSelect.set(0, indexSize);
|
||||
|
||||
facetQuery = new OpenBitSet(indexSize);
|
||||
facetQuery = new FixedBitSet(indexSize);
|
||||
facetQuery.set(0, indexSize);
|
||||
}
|
||||
}
|
||||
|
@ -161,8 +161,8 @@ public class TestFiltering extends SolrTestCaseJ4 {
|
|||
boolean positive = random().nextBoolean();
|
||||
boolean exclude = facetQuery ? false : random().nextBoolean(); // can't exclude a facet query from faceting
|
||||
|
||||
OpenBitSet[] sets = facetQuery ? new OpenBitSet[]{model.facetQuery} :
|
||||
(exclude ? new OpenBitSet[]{model.answer, model.facetQuery} : new OpenBitSet[]{model.answer, model.multiSelect, model.facetQuery});
|
||||
FixedBitSet[] sets = facetQuery ? new FixedBitSet[]{model.facetQuery} :
|
||||
(exclude ? new FixedBitSet[]{model.answer, model.facetQuery} : new FixedBitSet[]{model.answer, model.multiSelect, model.facetQuery});
|
||||
|
||||
if (random().nextInt(100) < 50) {
|
||||
// frange
|
||||
|
@ -183,33 +183,36 @@ public class TestFiltering extends SolrTestCaseJ4 {
|
|||
}
|
||||
}
|
||||
|
||||
for (OpenBitSet set : sets) {
|
||||
for (FixedBitSet set : sets) {
|
||||
set.clear(0,l);
|
||||
set.clear(u+1, model.indexSize);
|
||||
if (u + 1 < model.indexSize) {
|
||||
set.clear(u+1, model.indexSize);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// negative frange.. make it relatively small
|
||||
l = random().nextInt(model.indexSize);
|
||||
u = Math.max(model.indexSize-1, l+random().nextInt(Math.max(model.indexSize / 10, 2)));
|
||||
|
||||
for (OpenBitSet set : sets) {
|
||||
set.clear(l,u+1);
|
||||
for (FixedBitSet set : sets) {
|
||||
int end = Math.min(u+1, set.length());
|
||||
set.clear(l,end);
|
||||
}
|
||||
}
|
||||
|
||||
return frangeStr(!positive, l, u, cache, cost, exclude);
|
||||
} else {
|
||||
// term or boolean query
|
||||
OpenBitSet pset = new OpenBitSet(model.indexSize);
|
||||
FixedBitSet pset = new FixedBitSet(model.indexSize);
|
||||
for (int i=0; i<pset.getBits().length; i++) {
|
||||
pset.getBits()[i] = random().nextLong(); // set 50% of the bits on average
|
||||
}
|
||||
if (positive) {
|
||||
for (OpenBitSet set : sets) {
|
||||
for (FixedBitSet set : sets) {
|
||||
set.and(pset);
|
||||
}
|
||||
} else {
|
||||
for (OpenBitSet set : sets) {
|
||||
for (FixedBitSet set : sets) {
|
||||
set.andNot(pset);
|
||||
}
|
||||
}
|
||||
|
@ -217,8 +220,9 @@ public class TestFiltering extends SolrTestCaseJ4 {
|
|||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
for (int doc=-1;;) {
|
||||
if (doc+1 >= model.indexSize) break;
|
||||
doc = pset.nextSetBit(doc+1);
|
||||
if (doc < 0 || doc >= model.indexSize) break;
|
||||
if (doc < 0) break;
|
||||
sb.append((positive ? " ":" -") + f+":"+doc);
|
||||
}
|
||||
|
||||
|
|
|
@ -18,7 +18,11 @@
|
|||
package org.apache.solr.search;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.*;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.Random;
|
||||
|
||||
import org.apache.lucene.analysis.core.SimpleAnalyzer;
|
||||
import org.apache.lucene.document.Document;
|
||||
|
@ -28,16 +32,27 @@ import org.apache.lucene.index.AtomicReaderContext;
|
|||
import org.apache.lucene.index.DirectoryReader;
|
||||
import org.apache.lucene.index.IndexWriter;
|
||||
import org.apache.lucene.index.IndexWriterConfig;
|
||||
import org.apache.lucene.search.*;
|
||||
import org.apache.lucene.search.BitsFilteredDocIdSet;
|
||||
import org.apache.lucene.search.Collector;
|
||||
import org.apache.lucene.search.DocIdSet;
|
||||
import org.apache.lucene.search.Filter;
|
||||
import org.apache.lucene.search.IndexSearcher;
|
||||
import org.apache.lucene.search.MatchAllDocsQuery;
|
||||
import org.apache.lucene.search.ScoreDoc;
|
||||
import org.apache.lucene.search.Scorer;
|
||||
import org.apache.lucene.search.Sort;
|
||||
import org.apache.lucene.search.SortField;
|
||||
import org.apache.lucene.search.SortField.Type;
|
||||
import org.apache.lucene.search.TopDocs;
|
||||
import org.apache.lucene.search.TopFieldCollector;
|
||||
import org.apache.lucene.store.Directory;
|
||||
import org.apache.lucene.store.RAMDirectory;
|
||||
import org.apache.lucene.util.Bits;
|
||||
import org.apache.lucene.util.OpenBitSet;
|
||||
import org.apache.lucene.util.FixedBitSet;
|
||||
import org.apache.lucene.util._TestUtil;
|
||||
import org.apache.solr.SolrTestCaseJ4;
|
||||
import org.apache.solr.schema.SchemaField;
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
import org.apache.solr.schema.SchemaField;
|
||||
import org.junit.BeforeClass;
|
||||
|
||||
public class TestSort extends SolrTestCaseJ4 {
|
||||
|
@ -320,10 +335,10 @@ public class TestSort extends SolrTestCaseJ4 {
|
|||
}
|
||||
|
||||
public DocIdSet randSet(int sz) {
|
||||
OpenBitSet obs = new OpenBitSet(sz);
|
||||
FixedBitSet obs = new FixedBitSet(sz);
|
||||
int n = r.nextInt(sz);
|
||||
for (int i=0; i<n; i++) {
|
||||
obs.fastSet(r.nextInt(sz));
|
||||
obs.set(r.nextInt(sz));
|
||||
}
|
||||
return obs;
|
||||
}
|
||||
|
|
|
@ -17,13 +17,13 @@
|
|||
|
||||
package org.apache.solr.util;
|
||||
|
||||
import java.util.Random;
|
||||
import java.util.BitSet;
|
||||
import java.util.Random;
|
||||
|
||||
import org.apache.lucene.util.OpenBitSet;
|
||||
import org.apache.lucene.util.OpenBitSetIterator;
|
||||
import org.apache.lucene.util.FixedBitSet;
|
||||
import org.apache.lucene.util.FixedBitSet.FixedBitSetIterator;
|
||||
|
||||
/** Performance tester for OpenBitSet.
|
||||
/** Performance tester for FixedBitSet.
|
||||
* Use -Xbatch for more predictable results, and run tests such that the duration
|
||||
* is at least 10 seconds for better accuracy. Close browsers on your system (javascript
|
||||
* or flash may be running and cause more erratic results).
|
||||
|
@ -33,7 +33,7 @@ import org.apache.lucene.util.OpenBitSetIterator;
|
|||
public class BitSetPerf {
|
||||
static Random rand = new Random(0);
|
||||
|
||||
static void randomSets(int maxSize, int bitsToSet, BitSet target1, OpenBitSet target2) {
|
||||
static void randomSets(int maxSize, int bitsToSet, BitSet target1, FixedBitSet target2) {
|
||||
for (int i=0; i<bitsToSet; i++) {
|
||||
int idx;
|
||||
do {
|
||||
|
@ -50,7 +50,7 @@ public class BitSetPerf {
|
|||
public static void main(String[] args) {
|
||||
if (args.length<5) {
|
||||
System.out.println("BitSetTest <bitSetSize> <numSets> <numBitsSet> <testName> <iter> <impl>");
|
||||
System.out.println(" impl => open for OpenBitSet");
|
||||
System.out.println(" impl => open for FixedBitSet");
|
||||
}
|
||||
int bitSetSize = Integer.parseInt(args[0]);
|
||||
int numSets = Integer.parseInt(args[1]);
|
||||
|
@ -60,16 +60,16 @@ public class BitSetPerf {
|
|||
String impl = args.length>5 ? args[5].intern() : "bit";
|
||||
|
||||
BitSet[] sets = new BitSet[numSets];
|
||||
OpenBitSet[] osets = new OpenBitSet[numSets];
|
||||
FixedBitSet[] osets = new FixedBitSet[numSets];
|
||||
|
||||
for (int i=0; i<numSets; i++) {
|
||||
sets[i] = new BitSet(bitSetSize);
|
||||
osets[i] = new OpenBitSet(bitSetSize);
|
||||
osets[i] = new FixedBitSet(bitSetSize);
|
||||
randomSets(bitSetSize, numBitsSet, sets[i], osets[i]);
|
||||
}
|
||||
|
||||
BitSet bs = new BitSet(bitSetSize);
|
||||
OpenBitSet obs = new OpenBitSet(bitSetSize);
|
||||
FixedBitSet obs = new FixedBitSet(bitSetSize);
|
||||
randomSets(bitSetSize, numBitsSet, bs, obs);
|
||||
|
||||
|
||||
|
@ -82,8 +82,8 @@ public class BitSetPerf {
|
|||
for (int it=0; it<iter; it++) {
|
||||
for (int i=0; i<numSets; i++) {
|
||||
if (impl=="open") {
|
||||
OpenBitSet other=osets[i];
|
||||
obs.union(other);
|
||||
FixedBitSet other=osets[i];
|
||||
obs.or(other);
|
||||
} else {
|
||||
BitSet other=sets[i];
|
||||
bs.or(other);
|
||||
|
@ -108,8 +108,8 @@ public class BitSetPerf {
|
|||
for (int it=0; it<iter; it++) {
|
||||
for (int i=0; i<numSets; i++) {
|
||||
if (impl=="open") {
|
||||
OpenBitSet oset = osets[i];
|
||||
for (int k=0; k<bitSetSize; k++) if (oset.fastGet(k)) ret++;
|
||||
FixedBitSet oset = osets[i];
|
||||
for (int k=0; k<bitSetSize; k++) if (oset.get(k)) ret++;
|
||||
} else {
|
||||
BitSet bset = sets[i];
|
||||
for (int k=0; k<bitSetSize; k++) if (bset.get(k)) ret++;
|
||||
|
@ -122,9 +122,9 @@ public class BitSetPerf {
|
|||
for (int it=0; it<iter; it++) {
|
||||
for (int i=0; i<numSets-1; i++) {
|
||||
if (impl=="open") {
|
||||
OpenBitSet a=osets[i];
|
||||
OpenBitSet b=osets[i+1];
|
||||
ret += OpenBitSet.intersectionCount(a,b);
|
||||
FixedBitSet a=osets[i];
|
||||
FixedBitSet b=osets[i+1];
|
||||
ret += FixedBitSet.intersectionCount(a,b);
|
||||
} else {
|
||||
BitSet a=sets[i];
|
||||
BitSet b=sets[i+1];
|
||||
|
@ -140,7 +140,7 @@ public class BitSetPerf {
|
|||
for (int it=0; it<iter; it++) {
|
||||
for (int i=0; i<numSets; i++) {
|
||||
if (impl=="open") {
|
||||
osets[i] = (OpenBitSet)osets[i].clone();
|
||||
osets[i] = osets[i].clone();
|
||||
} else {
|
||||
sets[i] = (BitSet)sets[i].clone();
|
||||
}
|
||||
|
@ -152,7 +152,7 @@ public class BitSetPerf {
|
|||
for (int it=0; it<iter; it++) {
|
||||
for (int i=0; i<numSets; i++) {
|
||||
if (impl=="open") {
|
||||
final OpenBitSet set = osets[i];
|
||||
final FixedBitSet set = osets[i];
|
||||
for(int next=set.nextSetBit(0); next>=0; next=set.nextSetBit(next+1)) {
|
||||
ret += next;
|
||||
}
|
||||
|
@ -171,8 +171,8 @@ public class BitSetPerf {
|
|||
for (int it=0; it<iter; it++) {
|
||||
for (int i=0; i<numSets; i++) {
|
||||
if (impl=="open") {
|
||||
final OpenBitSet set = osets[i];
|
||||
final OpenBitSetIterator iterator = new OpenBitSetIterator(set);
|
||||
final FixedBitSet set = osets[i];
|
||||
final FixedBitSetIterator iterator = new FixedBitSetIterator(set);
|
||||
for(int next=iterator.nextDoc(); next>=0; next=iterator.nextDoc()) {
|
||||
ret += next;
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue