Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-6021: Make SparseFixedBitSet and FixedBitSet share a common "BitSet" interface. 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1634012 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Adrien Grand 2014-10-24 07:32:19 +00:00
parent dbe6c7ff05
commit c179a9bf94
61 changed files with 931 additions and 556 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
lucene/CHANGES.txt
View File

@ -171,6 +171,9 @@ API Changes
FieldInfo, since it's redundant with IndexOptions != null. (Robert
Muir, Mike McCandless)
* LUCENE-6021: FixedBitSet.nextSetBit now returns DocIdSetIterator.NO_MORE_DOCS
instead of -1 when there are no more bits which are set. (Adrien Grand)
Bug Fixes
* LUCENE-5650: Enforce read-only access to any path outside the temporary

3
lucene/codecs/src/java/org/apache/lucene/codecs/bloom/FuzzySet.java
View File

@ -19,6 +19,7 @@ package org.apache.lucene.codecs.bloom;
import java.io.IOException;
import java.util.Collections;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.DataOutput;
import org.apache.lucene.util.Accountable;
@ -272,7 +273,7 @@ public class FuzzySet implements Accountable {
int bitIndex = 0;
do {
bitIndex = filter.nextSetBit(bitIndex);
if (bitIndex >= 0) {
if (bitIndex != DocIdSetIterator.NO_MORE_DOCS) {
// Project the larger number into a smaller one effectively
// modulo-ing by using the target bitset size as a mask
int downSizedBitIndex = bitIndex & rightSizedBitSetSize;

4
lucene/core/src/java/org/apache/lucene/search/DocValuesDocIdSet.java
View File

@ -19,7 +19,7 @@ package org.apache.lucene.search;
import java.io.IOException;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitDocIdSet;
import org.apache.lucene.util.BitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
/**
@ -116,7 +116,7 @@ public abstract class DocValuesDocIdSet extends DocIdSet {
} else if (acceptDocs instanceof FixedBitSet) {
// special case for FixedBitSet: use the iterator and filter it
// (used e.g. when Filters are chained by FilteredQuery)
return new FilteredDocIdSetIterator(new FixedBitDocIdSet((FixedBitSet) acceptDocs).iterator()) {
return new FilteredDocIdSetIterator(new BitDocIdSet((FixedBitSet) acceptDocs).iterator()) {
@Override
protected boolean match(int doc) {
return DocValuesDocIdSet.this.matchDoc(doc);

2
lucene/core/src/java/org/apache/lucene/search/SloppyPhraseScorer.java
View File

@ -511,7 +511,7 @@ final class SloppyPhraseScorer extends Scorer {
Term[] t = tord.keySet().toArray(new Term[0]);
for (int i=0; i<bb.size(); i++) { // i is the group no.
FixedBitSet bits = bb.get(i);
for (int ord = bits.nextSetBit(0); ord != -1; ord = ord + 1 >= bits.length() ? -1 : bits.nextSetBit(ord + 1)) {
for (int ord = bits.nextSetBit(0); ord != DocIdSetIterator.NO_MORE_DOCS; ord = ord + 1 >= bits.length() ? DocIdSetIterator.NO_MORE_DOCS : bits.nextSetBit(ord + 1)) {
tg.put(t[ord],i);
}
}

23
lucene/core/src/java/org/apache/lucene/util/FixedBitDocIdSet.java → lucene/core/src/java/org/apache/lucene/util/BitDocIdSet.java
View File

@ -19,17 +19,16 @@ package org.apache.lucene.util;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.util.FixedBitSet.FixedBitSetIterator;
/**
* Implementation of the {@link DocIdSet} interface on top of a {@link FixedBitSet}.
* Implementation of the {@link DocIdSet} interface on top of a {@link BitSet}.
* @lucene.internal
*/
public class FixedBitDocIdSet extends DocIdSet {
public class BitDocIdSet extends DocIdSet {
private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(SparseFixedBitDocIdSet.class);
private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(BitDocIdSet.class);
private final FixedBitSet set;
private final BitSet set;
private final long cost;
/**
@ -37,26 +36,26 @@ public class FixedBitDocIdSet extends DocIdSet {
* {@link FixedBitSet} should not be modified after having wrapped as a
* {@link DocIdSet}.
*/
public FixedBitDocIdSet(FixedBitSet set, long cost) {
public BitDocIdSet(BitSet set, long cost) {
this.set = set;
this.cost = cost;
}
/**
* Same as {@link #FixedBitDocIdSet(FixedBitSet, long)} but uses the set
* {@link FixedBitSet#cardinality() cardinality} as a cost.
* Same as {@link #BitDocIdSet(BitSet, long)} but uses the set's
* {@link BitSet#approximateCardinality() approximate cardinality} as a cost.
*/
public FixedBitDocIdSet(FixedBitSet set) {
this(set, set.cardinality());
public BitDocIdSet(BitSet set) {
this(set, set.approximateCardinality());
}
@Override
public DocIdSetIterator iterator() {
return new FixedBitSetIterator(set, cost);
return new BitSetIterator(set, cost);
}
@Override
public FixedBitSet bits() {
public BitSet bits() {
return set;
}

94
lucene/core/src/java/org/apache/lucene/util/BitSet.java Normal file
View File

@ -0,0 +1,94 @@
package org.apache.lucene.util;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import org.apache.lucene.search.DocIdSetIterator;
/**
* Base implementation for a bit set.
* @lucene.internal
*/
public abstract class BitSet implements MutableBits, Accountable {
/** Set the bit at <code>i</code>. */
public abstract void set(int i);
/** Clears a range of bits.
*
* @param startIndex lower index
* @param endIndex one-past the last bit to clear
*/
public abstract void clear(int startIndex, int endIndex);
/**
* Return the number of bits that are set.
* NOTE: this method is likely to run in linear time
*/
public abstract int cardinality();
/**
* Return an approximation of the cardinality of this set. Some
* implementations may trade accuracy for speed if they have the ability to
* estimate the cardinality of the set without iterating over all the data.
* The default implementation returns {@link #cardinality()}.
*/
public int approximateCardinality() {
return cardinality();
}
/** Returns the index of the first set bit starting at the index specified.
* {@link DocIdSetIterator#NO_MORE_DOCS} is returned if there are no more set bits.
*/
public abstract int nextSetBit(int i);
/** Does in-place OR of the bits provided by the
* iterator. */
public void or(DocIdSetIterator iter) throws IOException {
for (int doc = iter.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = iter.nextDoc()) {
set(doc);
}
}
/** Does in-place AND of the bits provided by the
* iterator. */
public void and(DocIdSetIterator iter) throws IOException {
final int length = length();
if (length == 0) {
return;
}
int disiDoc, bitSetDoc = nextSetBit(0);
while (bitSetDoc != DocIdSetIterator.NO_MORE_DOCS && (disiDoc = iter.advance(bitSetDoc)) < length) {
clear(bitSetDoc, disiDoc);
disiDoc++;
bitSetDoc = (disiDoc < length) ? nextSetBit(disiDoc) : DocIdSetIterator.NO_MORE_DOCS;
}
if (bitSetDoc != DocIdSetIterator.NO_MORE_DOCS) {
clear(bitSetDoc, length);
}
}
/** this = this AND NOT other */
public void andNot(DocIdSetIterator iter) throws IOException {
for (int doc = iter.nextDoc(), len = length(); doc < len; doc = iter.nextDoc()) {
clear(doc);
}
}
}

80
lucene/core/src/java/org/apache/lucene/util/BitSetIterator.java Normal file
View File

@ -0,0 +1,80 @@
package org.apache.lucene.util;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import org.apache.lucene.search.DocIdSetIterator;
/**
* A {@link DocIdSetIterator} which iterates over set bits in a
* bit set.
* @lucene.internal
*/
public class BitSetIterator extends DocIdSetIterator {
private static <T extends BitSet> T getBitSet(DocIdSetIterator iterator, Class<? extends T> clazz) {
if (iterator instanceof BitSetIterator) {
BitSet bits = ((BitSetIterator) iterator).bits;
assert bits != null;
if (clazz.isInstance(bits)) {
return clazz.cast(bits);
}
}
return null;
}
/** If the provided iterator wraps a {@link FixedBitSet}, returns it, otherwise returns null. */
public static FixedBitSet getFixedBitSetOrNull(DocIdSetIterator iterator) {
return getBitSet(iterator, FixedBitSet.class);
}
private final BitSet bits;
private final int length;
private final long cost;
private int doc = -1;
/** Sole constructor. */
public BitSetIterator(BitSet bits, long cost) {
this.bits = bits;
this.length = bits.length();
this.cost = cost;
}
@Override
public int docID() {
return doc;
}
@Override
public int nextDoc() {
return advance(doc + 1);
}
@Override
public int advance(int target) {
if (target >= length) {
return doc = NO_MORE_DOCS;
}
return doc = bits.nextSetBit(target);
}
@Override
public long cost() {
return cost;
}
}

42
lucene/core/src/java/org/apache/lucene/util/DocIdSetBuilder.java
View File

@ -21,7 +21,6 @@ import java.io.IOException;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.util.SparseFixedBitSet.SparseFixedBitSetIterator;
/**
* A builder of {@link DocIdSet}s that supports random access.
@ -38,10 +37,19 @@ public final class DocIdSetBuilder {
// to re-compute approximateCardinality on the sparse set every time
private long costUpperBound;
/** Sole constructor. */
public DocIdSetBuilder(int maxDoc) {
/** Create a new instance that can hold <code>maxDoc</code> documents and is optionally <code>full</code>. */
public DocIdSetBuilder(int maxDoc, boolean full) {
this.maxDoc = maxDoc;
threshold = maxDoc >>> 10;
if (full) {
denseSet = new FixedBitSet(maxDoc);
denseSet.set(0, maxDoc);
}
}
/** Create a new empty instance. */
public DocIdSetBuilder(int maxDoc) {
this(maxDoc, false);
}
/**
@ -64,7 +72,7 @@ public final class DocIdSetBuilder {
denseSet = new FixedBitSet(maxDoc);
denseSet.or(it);
if (sparseSet != null) {
denseSet.or(new SparseFixedBitSetIterator(sparseSet, 0L));
denseSet.or(new BitSetIterator(sparseSet, 0L));
}
return;
}
@ -77,6 +85,28 @@ public final class DocIdSetBuilder {
sparseSet.or(it);
}
/**
* Removes from this builder documents that are not contained in <code>it</code>.
*/
public void and(DocIdSetIterator it) throws IOException {
if (denseSet != null) {
denseSet.and(it);
} else if (sparseSet != null) {
sparseSet.and(it);
}
}
/**
* Removes from this builder documents that are contained in <code>it</code>.
*/
public void andNot(DocIdSetIterator it) throws IOException {
if (denseSet != null) {
denseSet.andNot(it);
} else if (denseSet != null) {
denseSet.andNot(it);
}
}
/**
* Build a {@link DocIdSet} that contains all doc ids that have been added.
* This method may return <tt>null</tt> if no documents were addded to this
@ -87,9 +117,9 @@ public final class DocIdSetBuilder {
public DocIdSet build() {
final DocIdSet result;
if (denseSet != null) {
result = new FixedBitDocIdSet(denseSet, denseSet.cardinality());
result = new BitDocIdSet(denseSet);
} else if (sparseSet != null) {
result = new SparseFixedBitDocIdSet(sparseSet, sparseSet.approximateCardinality());
result = new BitDocIdSet(sparseSet);
} else {
result = null;
}

143
lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java
View File

@ -32,72 +32,10 @@ import org.apache.lucene.search.DocIdSetIterator;
*
* @lucene.internal
*/
public final class FixedBitSet implements MutableBits, Accountable {
public final class FixedBitSet extends BitSet implements MutableBits, Accountable {
private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(FixedBitSet.class);
/**
* A {@link DocIdSetIterator} which iterates over set bits in a
* {@link FixedBitSet}.
*/
public static final class FixedBitSetIterator extends DocIdSetIterator {
final int numBits, numWords;
final long[] bits;
final long cost;
int doc = -1;
/** Creates an iterator over the given {@link FixedBitSet}. */
public FixedBitSetIterator(FixedBitSet bits, long cost) {
this(bits.bits, bits.numBits, bits.numWords, cost);
}
/** Creates an iterator over the given array of bits. */
public FixedBitSetIterator(long[] bits, int numBits, int wordLength, long cost) {
this.bits = bits;
this.numBits = numBits;
this.numWords = wordLength;
this.cost = cost;
}
@Override
public int nextDoc() {
return advance(doc + 1);
}
@Override
public int docID() {
return doc;
}
@Override
public long cost() {
return cost;
}
@Override
public int advance(int target) {
if (target >= numBits) {
return doc = NO_MORE_DOCS;
}
int i = target >> 6;
long word = bits[i] >> target; // skip all the bits to the right of index
if (word != 0) {
return doc = target + Long.numberOfTrailingZeros(word);
}
while (++i < numWords) {
word = bits[i];
if (word != 0) {
return doc = (i << 6) + Long.numberOfTrailingZeros(word);
}
}
return doc = NO_MORE_DOCS;
}
}
/**
* If the given {@link FixedBitSet} is large enough to hold {@code numBits},
* returns the given bits, otherwise returns a new {@link FixedBitSet} which
@ -203,9 +141,7 @@ public final class FixedBitSet implements MutableBits, Accountable {
return bits;
}
/** Returns number of set bits. NOTE: this visits every
* long in the backing bits array, and the result is not
* internally cached! */
@Override
public int cardinality() {
return (int) BitUtil.pop_array(bits, 0, bits.length);
}
@ -236,6 +172,7 @@ public final class FixedBitSet implements MutableBits, Accountable {
return val;
}
@Override
public void clear(int index) {
assert index >= 0 && index < numBits;
int wordNum = index >> 6;
@ -252,9 +189,7 @@ public final class FixedBitSet implements MutableBits, Accountable {
return val;
}
/** Returns the index of the first set bit starting at the index specified.
* -1 is returned if there are no more set bits.
*/
@Override
public int nextSetBit(int index) {
assert index >= 0 && index < numBits : "index=" + index + ", numBits=" + numBits;
int i = index >> 6;
@ -271,7 +206,7 @@ public final class FixedBitSet implements MutableBits, Accountable {
}
}
return -1;
return DocIdSetIterator.NO_MORE_DOCS;
}
/** Returns the index of the last set bit before or on the index specified.
@ -297,20 +232,16 @@ public final class FixedBitSet implements MutableBits, Accountable {
return -1;
}
/** Does in-place OR of the bits provided by the
* iterator. */
@Override
public void or(DocIdSetIterator iter) throws IOException {
if (iter instanceof FixedBitSetIterator && iter.docID() == -1) {
final FixedBitSetIterator fbs = (FixedBitSetIterator) iter;
or(fbs.bits, fbs.numWords);
if (BitSetIterator.getFixedBitSetOrNull(iter) != null && iter.docID() == -1) {
final FixedBitSet bits = BitSetIterator.getFixedBitSetOrNull(iter);
or(bits);
// advance after last doc that would be accepted if standard
// iteration is used (to exhaust it):
fbs.advance(numBits);
iter.advance(numBits);
} else {
int doc;
while ((doc = iter.nextDoc()) < numBits) {
set(doc);
}
super.or(iter);
}
}
@ -335,12 +266,12 @@ public final class FixedBitSet implements MutableBits, Accountable {
/** Does in-place XOR of the bits provided by the iterator. */
public void xor(DocIdSetIterator iter) throws IOException {
if (iter instanceof FixedBitSetIterator && iter.docID() == -1) {
final FixedBitSetIterator fbs = (FixedBitSetIterator) iter;
xor(fbs.bits, fbs.numWords);
if (BitSetIterator.getFixedBitSetOrNull(iter) != null && iter.docID() == -1) {
final FixedBitSet bits = BitSetIterator.getFixedBitSetOrNull(iter);
xor(bits);
// advance after last doc that would be accepted if standard
// iteration is used (to exhaust it):
fbs.advance(numBits);
iter.advance(numBits);
} else {
int doc;
while ((doc = iter.nextDoc()) < numBits) {
@ -358,26 +289,16 @@ public final class FixedBitSet implements MutableBits, Accountable {
}
}
/** Does in-place AND of the bits provided by the
* iterator. */
@Override
public void and(DocIdSetIterator iter) throws IOException {
if (iter instanceof FixedBitSetIterator && iter.docID() == -1) {
final FixedBitSetIterator fbs = (FixedBitSetIterator) iter;
and(fbs.bits, fbs.numWords);
if (BitSetIterator.getFixedBitSetOrNull(iter) != null && iter.docID() == -1) {
final FixedBitSet bits = BitSetIterator.getFixedBitSetOrNull(iter);
and(bits);
// advance after last doc that would be accepted if standard
// iteration is used (to exhaust it):
fbs.advance(numBits);
iter.advance(numBits);
} else {
if (numBits == 0) return;
int disiDoc, bitSetDoc = nextSetBit(0);
while (bitSetDoc != -1 && (disiDoc = iter.advance(bitSetDoc)) < numBits) {
clear(bitSetDoc, disiDoc);
disiDoc++;
bitSetDoc = (disiDoc < numBits) ? nextSetBit(disiDoc) : -1;
}
if (bitSetDoc != -1) {
clear(bitSetDoc, numBits);
}
super.and(iter);
}
}
@ -406,20 +327,16 @@ public final class FixedBitSet implements MutableBits, Accountable {
}
}
/** Does in-place AND NOT of the bits provided by the
* iterator. */
@Override
public void andNot(DocIdSetIterator iter) throws IOException {
if (iter instanceof FixedBitSetIterator && iter.docID() == -1) {
final FixedBitSetIterator fbs = (FixedBitSetIterator) iter;
andNot(fbs.bits, fbs.numWords);
if (BitSetIterator.getFixedBitSetOrNull(iter) != null && iter.docID() == -1) {
final FixedBitSet bits = BitSetIterator.getFixedBitSetOrNull(iter);
andNot(bits);
// advance after last doc that would be accepted if standard
// iteration is used (to exhaust it):
fbs.advance(numBits);
iter.advance(numBits);
} else {
int doc;
while ((doc = iter.nextDoc()) < numBits) {
clear(doc);
}
super.andNot(iter);
}
}
@ -516,11 +433,7 @@ public final class FixedBitSet implements MutableBits, Accountable {
bits[endWord] |= endmask;
}
/** Clears a range of bits.
*
* @param startIndex lower index
* @param endIndex one-past the last bit to clear
*/
@Override
public void clear(int startIndex, int endIndex) {
assert startIndex >= 0 && startIndex < numBits : "startIndex=" + startIndex + ", numBits=" + numBits;
assert endIndex >= 0 && endIndex <= numBits : "endIndex=" + endIndex + ", numBits=" + numBits;

6
lucene/core/src/java/org/apache/lucene/util/RoaringDocIdSet.java
View File

@ -85,14 +85,14 @@ public class RoaringDocIdSet extends DocIdSet {
int excludedDoc = -1;
for (int i = 0; i < excludedDocs.length; ++i) {
excludedDoc = denseBuffer.nextSetBit(excludedDoc + 1);
assert excludedDoc != -1;
assert excludedDoc != DocIdSetIterator.NO_MORE_DOCS;
excludedDocs[i] = (short) excludedDoc;
}
assert excludedDoc + 1 == denseBuffer.length() || denseBuffer.nextSetBit(excludedDoc + 1) == -1;
assert excludedDoc + 1 == denseBuffer.length() || denseBuffer.nextSetBit(excludedDoc + 1) == DocIdSetIterator.NO_MORE_DOCS;
sets[currentBlock] = new NotDocIdSet(BLOCK_SIZE, new ShortArrayDocIdSet(excludedDocs));
} else {
// Neither sparse nor super dense, use a fixed bit set
sets[currentBlock] = new FixedBitDocIdSet(denseBuffer, currentBlockCardinality);
sets[currentBlock] = new BitDocIdSet(denseBuffer, currentBlockCardinality);
}
denseBuffer = null;
}

71
lucene/core/src/java/org/apache/lucene/util/SparseFixedBitDocIdSet.java
View File

@ -1,71 +0,0 @@
package org.apache.lucene.util;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.util.SparseFixedBitSet.SparseFixedBitSetIterator;
/**
* Implementation of the {@link DocIdSet} interface on top of a {@link SparseFixedBitSet}.
* @lucene.internal
*/
public class SparseFixedBitDocIdSet extends DocIdSet {
private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(SparseFixedBitDocIdSet.class);
private final SparseFixedBitSet set;
private final long cost;
/**
* Wrap the given {@link FixedBitSet} as a {@link DocIdSet}. The provided
* {@link FixedBitSet} should not be modified after having wrapped as a
* {@link DocIdSet}.
*/
public SparseFixedBitDocIdSet(SparseFixedBitSet set, long cost) {
this.set = set;
this.cost = cost;
}
@Override
public SparseFixedBitSet bits() {
return set;
}
@Override
public boolean isCacheable() {
return true;
}
@Override
public DocIdSetIterator iterator() throws IOException {
return new SparseFixedBitSetIterator(set, cost);
}
@Override
public long ramBytesUsed() {
return BASE_RAM_BYTES_USED + set.ramBytesUsed();
}
@Override
public String toString() {
return getClass().getSimpleName() + "(set=" + set + ",cost=" + cost + ")";
}
}

284
lucene/core/src/java/org/apache/lucene/util/SparseFixedBitSet.java
View File

@ -33,10 +33,11 @@ import org.apache.lucene.search.DocIdSetIterator;
*
* @lucene.internal
*/
public class SparseFixedBitSet implements Bits, Accountable {
public class SparseFixedBitSet extends BitSet implements Bits, Accountable {
private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(SparseFixedBitSet.class);
private static final long SINGLE_ELEMENT_ARRAY_BYTES_USED = RamUsageEstimator.sizeOf(new long[1]);
private static final int MASK_4096 = (1 << 12) - 1;
private static int blockCount(int length) {
int blockCount = length >>> 12;
@ -78,10 +79,7 @@ public class SparseFixedBitSet implements Bits, Accountable {
return true;
}
/**
* Compute the cardinality of this set.
* NOTE: this operation runs in linear time.
*/
@Override
public int cardinality() {
int cardinality = 0;
for (long[] bitArray : bits) {
@ -94,12 +92,11 @@ public class SparseFixedBitSet implements Bits, Accountable {
return cardinality;
}
/**
* Return an approximation of the cardinality of this set, assuming that bits
* are uniformly distributed. This operation runs in constant time.
*/
@Override
public int approximateCardinality() {
// this is basically the linear counting algorithm
// we are assuming that bits are uniformly set and use the linear counting
// algorithm to estimate the number of bits that are set based on the number
// of longs that are different from zero
final int totalLongs = (length + 63) >>> 6; // total number of longs in the space
assert totalLongs >= nonZeroLongCount;
final int zeroLongs = totalLongs - nonZeroLongCount; // number of longs that are zeros
@ -193,9 +190,160 @@ public class SparseFixedBitSet implements Bits, Accountable {
}
/**
* Add the documents contained in the provided {@link DocIdSetIterator} to
* this bit set.
* Clear the bit at index <tt>i</tt>.
*/
public void clear(int i) {
assert consistent(i);
final int i4096 = i >>> 12;
final int i64 = i >>> 6;
clearWithinLong(i4096, i64, ~(1L << i));
}
private void clearWithinLong(int i4096, int i64, long mask) {
final long index = indices[i4096];
if ((index & (1L << i64)) != 0) {
// offset of the long bits we are interested in in the array
final int o = Long.bitCount(index & ((1L << i64) - 1));
long bits = this.bits[i4096][o] & mask;
if (bits == 0) {
removeLong(i4096, i64, index, o);
} else {
this.bits[i4096][o] = bits;
}
}
}
private void removeLong(int i4096, int i64, long index, int o) {
index &= ~(1L << i64);
indices[i4096] = index;
if (index == 0) {
// release memory, there is nothing in this block anymore
this.bits[i4096] = null;
} else {
final int length = Long.bitCount(index);
final long[] bitArray = bits[i4096];
System.arraycopy(bitArray, o + 1, bitArray, o, length - o);
bitArray[length] = 0L;
}
}
@Override
public void clear(int from, int to) {
assert from >= 0;
assert to <= length;
if (from >= to) {
return;
}
final int firstBlock = from >>> 12;
final int lastBlock = (to - 1) >>> 12;
if (firstBlock == lastBlock) {
clearWithinBlock(firstBlock, from & MASK_4096, (to - 1) & MASK_4096);
} else {
clearWithinBlock(firstBlock, from & MASK_4096, MASK_4096);
for (int i = firstBlock + 1; i < lastBlock; ++i) {
indices[i] = 0;
bits[i] = null;
}
clearWithinBlock(lastBlock, 0, (to - 1) & MASK_4096);
}
}
// create a long that has bits set to one between from and to
private static long mask(int from, int to) {
return ((1L << (to - from) << 1) - 1) << from;
}
private void clearWithinBlock(int i4096, int from, int to) {
int firstLong = from >>> 6;
int lastLong = to >>> 6;
if (firstLong == lastLong) {
clearWithinLong(i4096, firstLong, ~mask(from, to));
} else {
assert firstLong < lastLong;
clearWithinLong(i4096, lastLong, ~mask(0, to));
for (int i = lastLong - 1; i >= firstLong + 1; --i) {
clearWithinLong(i4096, i, 0L);
}
clearWithinLong(i4096, firstLong, ~mask(from, 63));
}
}
/** Return the first document that occurs on or after the provided block index. */
private int firstDoc(int i4096) {
long index = 0;
while (i4096 < indices.length) {
index = indices[i4096];
if (index != 0) {
final int i64 = Long.numberOfTrailingZeros(index);
return (i4096 << 12) | (i64 << 6) | Long.numberOfTrailingZeros(bits[i4096][0]);
}
i4096 += 1;
}
return DocIdSetIterator.NO_MORE_DOCS;
}
@Override
public int nextSetBit(int i) {
assert i < length;
final int i4096 = i >>> 12;
final long index = indices[i4096];
int i64 = i >>> 6;
long indexBits = index >>> i64;
if (indexBits == 0) {
// if the index is zero, it means that there is no value in the
// current block, so return the first document of the next block
// or
// if neither the i64-th bit or any other bit on its left is set then
// it means that there are no more documents in this block, go to the
// next one
return firstDoc(i4096 + 1);
} else {
// We know we still have some 64-bits blocks that have bits set, let's
// advance to the next one by skipping trailing zeros of the index
int i1 = i & 0x3F;
int trailingZeros = Long.numberOfTrailingZeros(indexBits);
if (trailingZeros != 0) {
// no bits in the current long, go to the next one
i64 += trailingZeros;
i1 = 0;
}
// So now we are on a sub 64-bits block that has values
assert (index & (1L << i64)) != 0;
// we count the number of ones on the left of i64 to figure out the
// index of the long that contains the bits we are interested in
int longIndex = Long.bitCount(index & ((1L << i64) - 1)); // shifts are mod 64 in java
final long[] longArray = bits[i4096];
assert longArray[longIndex] != 0;
long bits = longArray[longIndex] >>> i1; // shifts are mod 64 in java
if (bits != 0L) {
// hurray, we found some non-zero bits, this gives us the next document:
i1 += Long.numberOfTrailingZeros(bits);
return (i4096 << 12) | ((i64 & 0x3F) << 6) | i1;
}
// otherwise it means that although we were on a sub-64 block that contains
// documents, all documents of this sub-block have already been consumed
// so two cases:
indexBits = index >>> i64 >>> 1; // we don't shift by (i64+1) otherwise we might shift by a multiple of 64 which is a no-op
if (indexBits == 0) {
// Case 1: this was the last long of the block of 4096 bits, then go
// to the next block
return firstDoc(i4096 + 1);
}
// Case 2: go to the next sub 64-bits block in the current block of 4096 bits
// by skipping trailing zeros of the index
trailingZeros = Long.numberOfTrailingZeros(indexBits);
i64 += 1 + trailingZeros;
bits = longArray[longIndex + 1];
assert bits != 0;
i1 = Long.numberOfTrailingZeros(bits);
return (i4096 << 12) | ((i64 & 0x3F) << 6) | i1;
}
}
@Override
public void or(DocIdSetIterator it) throws IOException {
for (int doc = it.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = it.nextDoc()) {
set(doc);
@ -211,116 +359,4 @@ public class SparseFixedBitSet implements Bits, Accountable {
public Iterable<? extends Accountable> getChildResources() {
return Collections.emptyList();
}
/**
* A {@link DocIdSetIterator} which iterates over set bits in a
* {@link SparseFixedBitSet}.
*/
public static class SparseFixedBitSetIterator extends DocIdSetIterator {
private final long[] indices;
private final long[][] bits;
private final long cost;
private int doc = -1;
/** Sole constructor. */
public SparseFixedBitSetIterator(SparseFixedBitSet set, long cost) {
indices = set.indices;
bits = set.bits;
this.cost = cost;
}
@Override
public int docID() {
return doc;
}
/** Return the first document that occurs on or after the provided block index. */
private int firstDoc(int i4096) {
long index = 0;
while (i4096 < indices.length) {
index = indices[i4096];
if (index != 0) {
final int i64 = Long.numberOfTrailingZeros(index);
return doc = (i4096 << 12) | (i64 << 6) | Long.numberOfTrailingZeros(bits[i4096][0]);
}
i4096 += 1;
}
return doc = NO_MORE_DOCS;
}
@Override
public int nextDoc() throws IOException {
return advance(doc + 1);
}
@Override
public int advance(int target) throws IOException {
final int i4096 = target >>> 12;
if (i4096 >= indices.length) {
return doc = NO_MORE_DOCS;
}
final long index = indices[i4096];
int i64 = target >>> 6;
long indexBits = index >>> i64;
if (indexBits == 0) {
// if the index is zero, it means that there is no value in the
// current block, so return the first document of the next block
// or
// if neither the i64-th bit or any other bit on its left is set then
// it means that there are no more documents in this block, go to the
// next one
return firstDoc(i4096 + 1);
} else {
// We know we still have some 64-bits blocks that have bits set, let's
// advance to the next one by skipping trailing zeros of the index
int i1 = target & 0x3F;
int trailingZeros = Long.numberOfTrailingZeros(indexBits);
if (trailingZeros != 0) {
// no bits in the current long, go to the next one
i64 += trailingZeros;
i1 = 0;
}
// So now we are on a sub 64-bits block that has values
assert (index & (1L << i64)) != 0;
// we count the number of ones on the left of i64 to figure out the
// index of the long that contains the bits we are interested in
int longIndex = Long.bitCount(index & ((1L << i64) - 1)); // shifts are mod 64 in java
final long[] longArray = bits[i4096];
assert longArray[longIndex] != 0;
long bits = longArray[longIndex] >>> i1; // shifts are mod 64 in java
if (bits != 0L) {
// hurray, we found some non-zero bits, this gives us the next document:
i1 += Long.numberOfTrailingZeros(bits);
return doc = (i4096 << 12) | ((i64 & 0x3F) << 6) | i1;
}
// otherwise it means that although we were on a sub-64 block that contains
// documents, all documents of this sub-block have already been consumed
// so two cases:
indexBits = index >>> i64 >>> 1; // we don't shift by (i64+1) otherwise we might shift by a multiple of 64 which is a no-op
if (indexBits == 0) {
// Case 1: this was the last long of the block of 4096 bits, then go
// to the next block
return firstDoc(i4096 + 1);
}
// Case 2: go to the next sub 64-bits block in the current block of 4096 bits
// by skipping trailing zeros of the index
trailingZeros = Long.numberOfTrailingZeros(indexBits);
i64 += 1 + trailingZeros;
bits = longArray[longIndex + 1];
assert bits != 0;
i1 = Long.numberOfTrailingZeros(bits);
return doc = (i4096 << 12) | ((i64 & 0x3F) << 6) | i1;
}
}
@Override
public long cost() {
return cost;
}
}
}

4
lucene/core/src/test/org/apache/lucene/search/MockFilter.java
View File

@ -18,7 +18,7 @@ package org.apache.lucene.search;
*/
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.util.FixedBitDocIdSet;
import org.apache.lucene.util.BitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.Bits;
@ -29,7 +29,7 @@ public class MockFilter extends Filter {
public DocIdSet getDocIdSet(LeafReaderContext context, Bits acceptDocs) {
wasCalled = true;
FixedBitSet bits = new FixedBitSet(context.reader().maxDoc());
return new FixedBitDocIdSet(bits);
return new BitDocIdSet(bits);
}
public void clear() {

4
lucene/core/src/test/org/apache/lucene/search/SingleDocTestFilter.java
View File

@ -21,7 +21,7 @@ import java.io.IOException;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitDocIdSet;
import org.apache.lucene.util.BitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
public class SingleDocTestFilter extends Filter {
@ -36,6 +36,6 @@ public class SingleDocTestFilter extends Filter {
FixedBitSet bits = new FixedBitSet(context.reader().maxDoc());
bits.set(doc);
if (acceptDocs != null && !acceptDocs.get(doc)) bits.clear(doc);
return new FixedBitDocIdSet(bits);
return new BitDocIdSet(bits);
}
}

4
lucene/core/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java
View File

@ -32,7 +32,7 @@ import org.apache.lucene.index.SlowCompositeReaderWrapper;
import org.apache.lucene.index.Term;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitDocIdSet;
import org.apache.lucene.util.BitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
@ -265,7 +265,7 @@ public class TestCachingWrapperFilter extends LuceneTestCase {
assertDocIdSetCacheable(reader, new Filter() {
@Override
public DocIdSet getDocIdSet(LeafReaderContext context, Bits acceptDocs) {
return new FixedBitDocIdSet(new FixedBitSet(context.reader().maxDoc()));
return new BitDocIdSet(new FixedBitSet(context.reader().maxDoc()));
}
}, true);

6
lucene/core/src/test/org/apache/lucene/search/TestFilteredQuery.java
View File

@ -34,7 +34,7 @@ import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.FilteredQuery.FilterStrategy;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitDocIdSet;
import org.apache.lucene.util.BitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
@ -105,7 +105,7 @@ public class TestFilteredQuery extends LuceneTestCase {
FixedBitSet bitset = new FixedBitSet(context.reader().maxDoc());
if (acceptDocs.get(1)) bitset.set(1);
if (acceptDocs.get(3)) bitset.set(3);
return new FixedBitDocIdSet(bitset);
return new BitDocIdSet(bitset);
}
};
}
@ -186,7 +186,7 @@ public class TestFilteredQuery extends LuceneTestCase {
assertNull("acceptDocs should be null, as we have an index without deletions", acceptDocs);
FixedBitSet bitset = new FixedBitSet(context.reader().maxDoc());
bitset.set(0, Math.min(5, bitset.length()));
return new FixedBitDocIdSet(bitset);
return new BitDocIdSet(bitset);
}
};
}

4
lucene/core/src/test/org/apache/lucene/search/TestFilteredSearch.java
View File

@ -31,7 +31,7 @@ import org.apache.lucene.index.Term;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitDocIdSet;
import org.apache.lucene.util.BitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
@ -99,7 +99,7 @@ public class TestFilteredSearch extends LuceneTestCase {
set.set(docId-docBase);
}
}
return set.cardinality() == 0 ? null : new FixedBitDocIdSet(set);
return set.cardinality() == 0 ? null : new BitDocIdSet(set);
}
}

4
lucene/core/src/test/org/apache/lucene/search/TestScorerPerf.java
View File

@ -14,7 +14,7 @@ import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.Term;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitDocIdSet;
import org.apache.lucene.util.BitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LuceneTestCase;
@ -146,7 +146,7 @@ public class TestScorerPerf extends LuceneTestCase {
@Override
public DocIdSet getDocIdSet (LeafReaderContext context, Bits acceptDocs) {
assertNull("acceptDocs should be null, as we have an index without deletions", acceptDocs);
return new FixedBitDocIdSet(rnd);
return new BitDocIdSet(rnd);
}
});
bq.add(q, BooleanClause.Occur.MUST);

4
lucene/core/src/test/org/apache/lucene/search/TestSortRandom.java
View File

@ -39,7 +39,7 @@ import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitDocIdSet;
import org.apache.lucene.util.BitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
@ -261,7 +261,7 @@ public class TestSortRandom extends LuceneTestCase {
}
}
return new FixedBitDocIdSet(bits);
return new BitDocIdSet(bits);
}
}
}

18
lucene/core/src/test/org/apache/lucene/util/TestDocIdSetBuilder.java
View File

@ -45,6 +45,16 @@ public class TestDocIdSetBuilder extends LuceneTestCase {
}
}
public void testFull() throws IOException {
final int maxDoc = 1 + random().nextInt(1000);
DocIdSetBuilder builder = new DocIdSetBuilder(maxDoc, true);
DocIdSet set = builder.build();
DocIdSetIterator it = set.iterator();
for (int i = 0; i < maxDoc; ++i) {
assertEquals(i, it.nextDoc());
}
}
public void testSparse() throws IOException {
final int maxDoc = 1000000 + random().nextInt(1000000);
DocIdSetBuilder builder = new DocIdSetBuilder(maxDoc);
@ -60,8 +70,8 @@ public class TestDocIdSetBuilder extends LuceneTestCase {
builder.or(b.build().iterator());
}
DocIdSet result = builder.build();
assertTrue(result instanceof SparseFixedBitDocIdSet);
assertEquals(new FixedBitDocIdSet(ref), result);
assertTrue(result instanceof BitDocIdSet);
assertEquals(new BitDocIdSet(ref), result);
}
public void testDense() throws IOException {
@ -84,8 +94,8 @@ public class TestDocIdSetBuilder extends LuceneTestCase {
builder.or(b.build().iterator());
}
DocIdSet result = builder.build();
assertTrue(result instanceof FixedBitDocIdSet);
assertEquals(new FixedBitDocIdSet(ref), result);
assertTrue(result instanceof BitDocIdSet);
assertEquals(new BitDocIdSet(ref), result);
}
}

34
lucene/core/src/test/org/apache/lucene/util/TestFixedBitDocIdSet.java Normal file
View File

@ -0,0 +1,34 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.util;
import java.io.IOException;
import java.util.BitSet;
public class TestFixedBitDocIdSet extends BaseDocIdSetTestCase<BitDocIdSet> {
@Override
public BitDocIdSet copyOf(BitSet bs, int length) throws IOException {
final FixedBitSet set = new FixedBitSet(length);
for (int doc = bs.nextSetBit(0); doc != -1; doc = bs.nextSetBit(doc + 1)) {
set.set(doc);
}
return new BitDocIdSet(set);
}
}

65
lucene/core/src/test/org/apache/lucene/util/TestFixedBitSet.java
View File

@ -1,3 +1,5 @@
package org.apache.lucene.util;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@ -15,26 +17,23 @@
* limitations under the License.
*/
package org.apache.lucene.util;
import java.io.IOException;
import java.util.BitSet;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.util.FixedBitSet.FixedBitSetIterator;
public class TestFixedBitSet extends BaseDocIdSetTestCase<FixedBitDocIdSet> {
public class TestFixedBitSet extends BaseBitSetTestCase<FixedBitSet> {
@Override
public FixedBitDocIdSet copyOf(BitSet bs, int length) throws IOException {
public FixedBitSet copyOf(BitSet bs, int length) throws IOException {
final FixedBitSet set = new FixedBitSet(length);
for (int doc = bs.nextSetBit(0); doc != -1; doc = bs.nextSetBit(doc + 1)) {
for (int doc = bs.nextSetBit(0); doc != DocIdSetIterator.NO_MORE_DOCS; doc = doc + 1 >= length ? DocIdSetIterator.NO_MORE_DOCS : bs.nextSetBit(doc + 1)) {
set.set(doc);
}
return new FixedBitDocIdSet(set);
return set;
}
void doGet(BitSet a, FixedBitSet b) {
void doGet(java.util.BitSet a, FixedBitSet b) {
int max = b.length();
for (int i=0; i<max; i++) {
if (a.get(i) != b.get(i)) {
@ -43,16 +42,19 @@ public class TestFixedBitSet extends BaseDocIdSetTestCase<FixedBitDocIdSet> {
}
}
void doNextSetBit(BitSet a, FixedBitSet b) {
void doNextSetBit(java.util.BitSet a, FixedBitSet b) {
int aa=-1,bb=-1;
do {
aa = a.nextSetBit(aa+1);
bb = bb < b.length()-1 ? b.nextSetBit(bb+1) : -1;
if (aa == -1) {
aa = DocIdSetIterator.NO_MORE_DOCS;
}
bb = bb < b.length()-1 ? b.nextSetBit(bb+1) : DocIdSetIterator.NO_MORE_DOCS;
assertEquals(aa,bb);
} while (aa>=0);
} while (aa != DocIdSetIterator.NO_MORE_DOCS);
}
void doPrevSetBit(BitSet a, FixedBitSet b) {
void doPrevSetBit(java.util.BitSet a, FixedBitSet b) {
int aa = a.size() + random().nextInt(100);
int bb = aa;
do {
@ -75,14 +77,14 @@ public class TestFixedBitSet extends BaseDocIdSetTestCase<FixedBitDocIdSet> {
}
// test interleaving different FixedBitSetIterator.next()/skipTo()
void doIterate(BitSet a, FixedBitSet b, int mode) throws IOException {
void doIterate(java.util.BitSet a, FixedBitSet b, int mode) throws IOException {
if (mode==1) doIterate1(a, b);
if (mode==2) doIterate2(a, b);
}
void doIterate1(BitSet a, FixedBitSet b) throws IOException {
void doIterate1(java.util.BitSet a, FixedBitSet b) throws IOException {
int aa=-1,bb=-1;
DocIdSetIterator iterator = new FixedBitSetIterator(b, 0);
DocIdSetIterator iterator = new BitSetIterator(b, 0);
do {
aa = a.nextSetBit(aa+1);
bb = (bb < b.length() && random().nextBoolean()) ? iterator.nextDoc() : iterator.advance(bb + 1);
@ -90,9 +92,9 @@ public class TestFixedBitSet extends BaseDocIdSetTestCase<FixedBitDocIdSet> {
} while (aa>=0);
}
void doIterate2(BitSet a, FixedBitSet b) throws IOException {
void doIterate2(java.util.BitSet a, FixedBitSet b) throws IOException {
int aa=-1,bb=-1;
DocIdSetIterator iterator = new FixedBitSetIterator(b, 0);
DocIdSetIterator iterator = new BitSetIterator(b, 0);
do {
aa = a.nextSetBit(aa+1);
bb = random().nextBoolean() ? iterator.nextDoc() : iterator.advance(bb + 1);
@ -101,12 +103,12 @@ public class TestFixedBitSet extends BaseDocIdSetTestCase<FixedBitDocIdSet> {
}
void doRandomSets(int maxSize, int iter, int mode) throws IOException {
BitSet a0=null;
java.util.BitSet a0=null;
FixedBitSet b0=null;
for (int i=0; i<iter; i++) {
int sz = TestUtil.nextInt(random(), 2, maxSize);
BitSet a = new BitSet(sz);
java.util.BitSet a = new java.util.BitSet(sz);
FixedBitSet b = new FixedBitSet(sz);
// test the various ways of setting bits
@ -148,14 +150,14 @@ public class TestFixedBitSet extends BaseDocIdSetTestCase<FixedBitDocIdSet> {
int fromIndex, toIndex;
fromIndex = random().nextInt(sz/2);
toIndex = fromIndex + random().nextInt(sz - fromIndex);
BitSet aa = (BitSet)a.clone(); aa.flip(fromIndex,toIndex);
java.util.BitSet aa = (java.util.BitSet)a.clone(); aa.flip(fromIndex,toIndex);
FixedBitSet bb = b.clone(); bb.flip(fromIndex,toIndex);
doIterate(aa,bb, mode); // a problem here is from flip or doIterate
fromIndex = random().nextInt(sz/2);
toIndex = fromIndex + random().nextInt(sz - fromIndex);
aa = (BitSet)a.clone(); aa.clear(fromIndex,toIndex);
aa = (java.util.BitSet)a.clone(); aa.clear(fromIndex,toIndex);
bb = b.clone(); bb.clear(fromIndex,toIndex);
doNextSetBit(aa,bb); // a problem here is from clear() or nextSetBit
@ -164,7 +166,7 @@ public class TestFixedBitSet extends BaseDocIdSetTestCase<FixedBitDocIdSet> {
fromIndex = random().nextInt(sz/2);
toIndex = fromIndex + random().nextInt(sz - fromIndex);
aa = (BitSet)a.clone(); aa.set(fromIndex,toIndex);
aa = (java.util.BitSet)a.clone(); aa.set(fromIndex,toIndex);
bb = b.clone(); bb.set(fromIndex,toIndex);
doNextSetBit(aa,bb); // a problem here is from set() or nextSetBit
@ -174,10 +176,10 @@ public class TestFixedBitSet extends BaseDocIdSetTestCase<FixedBitDocIdSet> {
if (b0 != null && b0.length() <= b.length()) {
assertEquals(a.cardinality(), b.cardinality());
BitSet a_and = (BitSet)a.clone(); a_and.and(a0);
BitSet a_or = (BitSet)a.clone(); a_or.or(a0);
BitSet a_xor = (BitSet)a.clone(); a_xor.xor(a0);
BitSet a_andn = (BitSet)a.clone(); a_andn.andNot(a0);
java.util.BitSet a_and = (java.util.BitSet)a.clone(); a_and.and(a0);
java.util.BitSet a_or = (java.util.BitSet)a.clone(); a_or.or(a0);
java.util.BitSet a_xor = (java.util.BitSet)a.clone(); a_xor.xor(a0);
java.util.BitSet a_andn = (java.util.BitSet)a.clone(); a_andn.andNot(a0);
FixedBitSet b_and = b.clone(); assertEquals(b,b_and); b_and.and(b0);
FixedBitSet b_or = b.clone(); b_or.or(b0);
@ -297,8 +299,8 @@ public class TestFixedBitSet extends BaseDocIdSetTestCase<FixedBitDocIdSet> {
return bs;
}
private BitSet makeBitSet(int[] a) {
BitSet bs = new BitSet();
private java.util.BitSet makeBitSet(int[] a) {
java.util.BitSet bs = new java.util.BitSet();
for (int e: a) {
bs.set(e);
}
@ -307,7 +309,7 @@ public class TestFixedBitSet extends BaseDocIdSetTestCase<FixedBitDocIdSet> {
private void checkPrevSetBitArray(int [] a, int numBits) {
FixedBitSet obs = makeFixedBitSet(a, numBits);
BitSet bs = makeBitSet(a);
java.util.BitSet bs = makeBitSet(a);
doPrevSetBit(bs, obs);
}
@ -320,7 +322,7 @@ public class TestFixedBitSet extends BaseDocIdSetTestCase<FixedBitDocIdSet> {
private void checkNextSetBitArray(int [] a, int numBits) {
FixedBitSet obs = makeFixedBitSet(a, numBits);
BitSet bs = makeBitSet(a);
java.util.BitSet bs = makeBitSet(a);
doNextSetBit(bs, obs);
}
@ -360,5 +362,4 @@ public class TestFixedBitSet extends BaseDocIdSetTestCase<FixedBitDocIdSet> {
assertTrue(bits.get(1));
assertFalse(newBits.get(1));
}
}

4
lucene/core/src/test/org/apache/lucene/util/TestNotDocIdSet.java
View File

@ -30,7 +30,7 @@ public class TestNotDocIdSet extends BaseDocIdSetTestCase<NotDocIdSet> {
for (int doc = bs.nextClearBit(0); doc < length; doc = bs.nextClearBit(doc + 1)) {
set.set(doc);
}
return new NotDocIdSet(length, new FixedBitDocIdSet(set));
return new NotDocIdSet(length, new BitDocIdSet(set));
}
@Override
@ -48,7 +48,7 @@ public class TestNotDocIdSet extends BaseDocIdSetTestCase<NotDocIdSet> {
public void testBits() throws IOException {
assertNull(new NotDocIdSet(3, DocIdSet.EMPTY).bits());
assertNotNull(new NotDocIdSet(3, new FixedBitDocIdSet(new FixedBitSet(3))).bits());
assertNotNull(new NotDocIdSet(3, new BitDocIdSet(new FixedBitSet(3))).bits());
}
}

60
lucene/core/src/test/org/apache/lucene/util/TestSparseFixedBitDocIdSet.java Normal file
View File

@ -0,0 +1,60 @@
package org.apache.lucene.util;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import java.util.ArrayList;
import java.util.BitSet;
import java.util.Collections;
import java.util.List;
public class TestSparseFixedBitDocIdSet extends BaseDocIdSetTestCase<BitDocIdSet> {
@Override
public BitDocIdSet copyOf(BitSet bs, int length) throws IOException {
final SparseFixedBitSet set = new SparseFixedBitSet(length);
// SparseFixedBitSet can be sensitive to the order of insertion so
// randomize insertion a bit
List<Integer> buffer = new ArrayList<>();
for (int doc = bs.nextSetBit(0); doc != -1; doc = bs.nextSetBit(doc + 1)) {
buffer.add(doc);
if (buffer.size() >= 100000) {
Collections.shuffle(buffer, random());
for (int i : buffer) {
set.set(i);
}
buffer.clear();
}
}
Collections.shuffle(buffer, random());
for (int i : buffer) {
set.set(i);
}
return new BitDocIdSet(set, set.approximateCardinality());
}
@Override
public void assertEquals(int numBits, BitSet ds1, BitDocIdSet ds2) throws IOException {
for (int i = 0; i < numBits; ++i) {
assertEquals(ds1.get(i), ds2.bits().get(i));
}
assertEquals(ds1.cardinality(), ds2.bits().cardinality());
super.assertEquals(numBits, ds1, ds2);
}
}

42
lucene/core/src/test/org/apache/lucene/util/TestSparseFixedBitSet.java
View File

@ -18,43 +18,18 @@ package org.apache.lucene.util;
*/
import java.io.IOException;
import java.util.ArrayList;
import java.util.BitSet;
import java.util.Collections;
import java.util.List;
public class TestSparseFixedBitSet extends BaseDocIdSetTestCase<SparseFixedBitDocIdSet> {
import org.apache.lucene.search.DocIdSetIterator;
public class TestSparseFixedBitSet extends BaseBitSetTestCase<SparseFixedBitSet> {
@Override
public SparseFixedBitDocIdSet copyOf(BitSet bs, int length) throws IOException {
public SparseFixedBitSet copyOf(BitSet bs, int length) throws IOException {
final SparseFixedBitSet set = new SparseFixedBitSet(length);
// SparseFixedBitSet can be sensitive to the order of insertion so
// randomize insertion a bit
List<Integer> buffer = new ArrayList<>();
for (int doc = bs.nextSetBit(0); doc != -1; doc = bs.nextSetBit(doc + 1)) {
buffer.add(doc);
if (buffer.size() >= 100000) {
Collections.shuffle(buffer, random());
for (int i : buffer) {
set.set(i);
}
buffer.clear();
}
for (int doc = bs.nextSetBit(0); doc != DocIdSetIterator.NO_MORE_DOCS; doc = doc + 1 >= length ? DocIdSetIterator.NO_MORE_DOCS : bs.nextSetBit(doc + 1)) {
set.set(doc);
}
Collections.shuffle(buffer, random());
for (int i : buffer) {
set.set(i);
}
return new SparseFixedBitDocIdSet(set, set.approximateCardinality());
}
@Override
public void assertEquals(int numBits, BitSet ds1, SparseFixedBitDocIdSet ds2) throws IOException {
for (int i = 0; i < numBits; ++i) {
assertEquals(ds1.get(i), ds2.bits().get(i));
}
assertEquals(ds1.cardinality(), ds2.bits().cardinality());
super.assertEquals(numBits, ds1, ds2);
return set;
}
public void testApproximateCardinality() {
@ -70,12 +45,11 @@ public class TestSparseFixedBitSet extends BaseDocIdSetTestCase<SparseFixedBitDo
public void testApproximateCardinalityOnDenseSet() {
// this tests that things work as expected in approximateCardinality when
// all longs are different than 0, in which case we divide by zero
final int numDocs = 70;//TestUtil.nextInt(random(), 1, 10000);
final int numDocs = TestUtil.nextInt(random(), 1, 10000);
final SparseFixedBitSet set = new SparseFixedBitSet(numDocs);
for (int i = 0; i < set.length(); ++i) {
set.set(i);
}
assertEquals(numDocs, set.approximateCardinality());
}
}

2
lucene/facet/src/java/org/apache/lucene/facet/DrillSidewaysScorer.java
View File

@ -330,7 +330,7 @@ class DrillSidewaysScorer extends BulkScorer {
// Fold in baseScorer, using advance:
int filledCount = 0;
int slot0 = 0;
while (slot0 < CHUNK && (slot0 = seen.nextSetBit(slot0)) != -1) {
while (slot0 < CHUNK && (slot0 = seen.nextSetBit(slot0)) != DocIdSetIterator.NO_MORE_DOCS) {
int ddDocID = docIDs[slot0];
assert ddDocID != -1;

4
lucene/facet/src/java/org/apache/lucene/facet/FacetsCollector.java
View File

@ -40,7 +40,7 @@ import org.apache.lucene.search.TopFieldCollector;
import org.apache.lucene.search.TopFieldDocs;
import org.apache.lucene.search.TopScoreDocCollector;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.FixedBitDocIdSet;
import org.apache.lucene.util.BitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
/** Collects hits for subsequent faceting. Once you've run
@ -129,7 +129,7 @@ public class FacetsCollector extends SimpleCollector {
@Override
public DocIdSet getDocIdSet() {
return new FixedBitDocIdSet(bits);
return new BitDocIdSet(bits);
}
};
}

4
lucene/facet/src/java/org/apache/lucene/facet/RandomSamplingFacetsCollector.java
View File

@ -26,7 +26,7 @@ import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.util.FixedBitDocIdSet;
import org.apache.lucene.util.BitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
/**
@ -213,7 +213,7 @@ public class RandomSamplingFacetsCollector extends FacetsCollector {
}
}
return new MatchingDocs(docs.context, new FixedBitDocIdSet(sampleDocs), docs.totalHits, null);
return new MatchingDocs(docs.context, new BitDocIdSet(sampleDocs), docs.totalHits, null);
} catch (IOException e) {
throw new RuntimeException();
}

4
lucene/facet/src/test/org/apache/lucene/facet/TestDrillSideways.java
View File

@ -58,7 +58,7 @@ import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitDocIdSet;
import org.apache.lucene.util.BitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.InPlaceMergeSorter;
@ -658,7 +658,7 @@ public class TestDrillSideways extends FacetTestCase {
bits.set(docID);
}
}
return new FixedBitDocIdSet(bits);
return new BitDocIdSet(bits);
}
};
} else {

4
lucene/facet/src/test/org/apache/lucene/facet/range/TestRangeFacetCounts.java
View File

@ -65,7 +65,7 @@ import org.apache.lucene.search.NumericRangeFilter;
import org.apache.lucene.search.NumericRangeQuery;
import org.apache.lucene.search.QueryWrapperFilter;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.FixedBitDocIdSet;
import org.apache.lucene.util.BitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.TestUtil;
@ -927,7 +927,7 @@ public class TestRangeFacetCounts extends FacetTestCase {
final FixedBitSet cached = new FixedBitSet(reader.maxDoc());
filterWasUsed.set(true);
cached.or(iterator);
return new FixedBitDocIdSet(cached);
return new BitDocIdSet(cached);
}
};
} else {

3
lucene/grouping/src/test/org/apache/lucene/search/grouping/AllGroupHeadsCollectorTest.java
View File

@ -43,6 +43,7 @@ import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.valuesource.BytesRefFieldSource;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.Sort;
@ -405,7 +406,7 @@ public class AllGroupHeadsCollectorTest extends LuceneTestCase {
expected.set(expectedDoc);
}
for (int docId = expected.nextSetBit(0); docId != -1; docId = docId + 1 >= expected.length() ? -1 : expected.nextSetBit(docId + 1)) {
for (int docId = expected.nextSetBit(0); docId != DocIdSetIterator.NO_MORE_DOCS; docId = docId + 1 >= expected.length() ? DocIdSetIterator.NO_MORE_DOCS : expected.nextSetBit(docId + 1)) {
if (!actual.get(docId)) {
return false;
}

3
lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java
View File

@ -35,6 +35,7 @@ import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.PhraseQuery;
import org.apache.lucene.search.Query;
@ -143,7 +144,7 @@ public class HighlighterPhraseTest extends LuceneTestCase {
final Highlighter highlighter = new Highlighter(
new SimpleHTMLFormatter(), new SimpleHTMLEncoder(),
new QueryScorer(phraseQuery));
for (int position = bitset.nextSetBit(0); position >= 0 && position < maxDoc-1; position = bitset
for (int position = bitset.nextSetBit(0); position < maxDoc-1; position = bitset
.nextSetBit(position + 1)) {
assertEquals(0, position);
final TokenStream tokenStream = TokenSources.getTokenStream(

6
lucene/join/src/java/org/apache/lucene/search/join/FixedBitSetCachingWrapperFilter.java
View File

@ -26,7 +26,7 @@ import org.apache.lucene.search.CachingWrapperFilter;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Filter;
import org.apache.lucene.util.FixedBitDocIdSet;
import org.apache.lucene.util.BitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
/** A {@link CachingWrapperFilter} that caches sets using a {@link FixedBitSet},
@ -43,7 +43,7 @@ public final class FixedBitSetCachingWrapperFilter extends CachingWrapperFilter
throws IOException {
if (docIdSet == null) {
return EMPTY;
} else if (docIdSet instanceof FixedBitDocIdSet) {
} else if (docIdSet instanceof BitDocIdSet) {
// this is different from CachingWrapperFilter: even when the DocIdSet is
// cacheable, we convert it to a FixedBitSet since we require all the
// cached filters to be FixedBitSets
@ -55,7 +55,7 @@ public final class FixedBitSetCachingWrapperFilter extends CachingWrapperFilter
} else {
final FixedBitSet copy = new FixedBitSet(reader.maxDoc());
copy.or(it);
return new FixedBitDocIdSet(copy);
return new BitDocIdSet(copy);
}
}
}

4
lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java
View File

@ -36,11 +36,11 @@ import org.apache.lucene.search.LeafCollector;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.Weight;
import org.apache.lucene.util.BitSetIterator;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefHash;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.FixedBitSet.FixedBitSetIterator;
class TermsIncludingScoreQuery extends Query {
@ -329,7 +329,7 @@ class TermsIncludingScoreQuery extends Query {
FixedBitSet matchingDocs = new FixedBitSet(maxDoc);
this.scores = new float[maxDoc];
fillDocsAndScores(matchingDocs, acceptDocs, termsEnum);
this.matchingDocsIterator = new FixedBitSetIterator(matchingDocs, cost);
this.matchingDocsIterator = new BitSetIterator(matchingDocs, cost);
this.cost = cost;
}

8
lucene/join/src/java/org/apache/lucene/search/join/ToChildBlockJoinQuery.java
View File

@ -33,7 +33,7 @@ import org.apache.lucene.search.Query;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.Weight;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitDocIdSet;
import org.apache.lucene.util.BitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
/**
@ -145,11 +145,11 @@ public class ToChildBlockJoinQuery extends Query {
// No matches
return null;
}
if (!(parents instanceof FixedBitDocIdSet)) {
throw new IllegalStateException("parentFilter must return FixedBitSet; got " + parents);
if (!(parents.bits() instanceof FixedBitSet)) {
throw new IllegalStateException("parentFilter must return FixedBitSet; got " + parents.bits());
}
return new ToChildBlockJoinScorer(this, parentScorer, ((FixedBitDocIdSet) parents).bits(), doScores, acceptDocs);
return new ToChildBlockJoinScorer(this, parentScorer, (FixedBitSet) parents.bits(), doScores, acceptDocs);
}
@Override

33
lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinFieldComparator.java
View File

@ -24,7 +24,6 @@ import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.FieldComparator;
import org.apache.lucene.search.Filter;
import org.apache.lucene.util.FixedBitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
/**
@ -70,8 +69,8 @@ public abstract class ToParentBlockJoinFieldComparator extends FieldComparator<O
DocIdSet innerDocuments = childFilter.getDocIdSet(context, null);
if (isEmpty(innerDocuments)) {
this.childDocuments = null;
} else if (innerDocuments instanceof FixedBitDocIdSet) {
this.childDocuments = ((FixedBitDocIdSet) innerDocuments).bits();
} else if (innerDocuments.bits() instanceof FixedBitSet) {
this.childDocuments = (FixedBitSet) innerDocuments.bits();
} else {
DocIdSetIterator iterator = innerDocuments.iterator();
if (iterator != null) {
@ -83,8 +82,8 @@ public abstract class ToParentBlockJoinFieldComparator extends FieldComparator<O
DocIdSet rootDocuments = parentFilter.getDocIdSet(context, null);
if (isEmpty(rootDocuments)) {
this.parentDocuments = null;
} else if (rootDocuments instanceof FixedBitDocIdSet) {
this.parentDocuments = ((FixedBitDocIdSet) rootDocuments).bits();
} else if (rootDocuments.bits() instanceof FixedBitSet) {
this.parentDocuments = (FixedBitSet) rootDocuments.bits();
} else {
DocIdSetIterator iterator = rootDocuments.iterator();
if (iterator != null) {
@ -144,7 +143,7 @@ public abstract class ToParentBlockJoinFieldComparator extends FieldComparator<O
// We need to copy the lowest value from all child docs into slot.
int prevParentDoc = parentDocuments.prevSetBit(parentDoc - 1);
int childDoc = childDocuments.nextSetBit(prevParentDoc + 1);
if (childDoc >= parentDoc || childDoc == -1) {
if (childDoc >= parentDoc || childDoc == DocIdSetIterator.NO_MORE_DOCS) {
return 0;
}
@ -156,7 +155,7 @@ public abstract class ToParentBlockJoinFieldComparator extends FieldComparator<O
while (true) {
childDoc = childDocuments.nextSetBit(childDoc + 1);
if (childDoc >= parentDoc || childDoc == -1) {
if (childDoc >= parentDoc || childDoc == DocIdSetIterator.NO_MORE_DOCS) {
return cmp;
}
int cmp1 = wrappedComparator.compareBottom(childDoc);
@ -179,7 +178,7 @@ public abstract class ToParentBlockJoinFieldComparator extends FieldComparator<O
// We need to copy the lowest value from all child docs into slot.
int prevParentDoc = parentDocuments.prevSetBit(parentDoc - 1);
int childDoc = childDocuments.nextSetBit(prevParentDoc + 1);
if (childDoc >= parentDoc || childDoc == -1) {
if (childDoc >= parentDoc || childDoc == DocIdSetIterator.NO_MORE_DOCS) {
return;
}
wrappedComparator.copy(spareSlot, childDoc);
@ -187,7 +186,7 @@ public abstract class ToParentBlockJoinFieldComparator extends FieldComparator<O
while (true) {
childDoc = childDocuments.nextSetBit(childDoc + 1);
if (childDoc >= parentDoc || childDoc == -1) {
if (childDoc >= parentDoc || childDoc == DocIdSetIterator.NO_MORE_DOCS) {
return;
}
wrappedComparator.copy(spareSlot, childDoc);
@ -207,7 +206,7 @@ public abstract class ToParentBlockJoinFieldComparator extends FieldComparator<O
// We need to copy the lowest value from all nested docs into slot.
int prevParentDoc = parentDocuments.prevSetBit(parentDoc - 1);
int childDoc = childDocuments.nextSetBit(prevParentDoc + 1);
if (childDoc >= parentDoc || childDoc == -1) {
if (childDoc >= parentDoc || childDoc == DocIdSetIterator.NO_MORE_DOCS) {
return 0;
}
@ -219,7 +218,7 @@ public abstract class ToParentBlockJoinFieldComparator extends FieldComparator<O
while (true) {
childDoc = childDocuments.nextSetBit(childDoc + 1);
if (childDoc >= parentDoc || childDoc == -1) {
if (childDoc >= parentDoc || childDoc == DocIdSetIterator.NO_MORE_DOCS) {
return cmp;
}
int cmp1 = wrappedComparator.compareTop(childDoc);
@ -262,7 +261,7 @@ public abstract class ToParentBlockJoinFieldComparator extends FieldComparator<O
int prevParentDoc = parentDocuments.prevSetBit(parentDoc - 1);
int childDoc = childDocuments.nextSetBit(prevParentDoc + 1);
if (childDoc >= parentDoc || childDoc == -1) {
if (childDoc >= parentDoc || childDoc == DocIdSetIterator.NO_MORE_DOCS) {
return 0;
}
@ -273,7 +272,7 @@ public abstract class ToParentBlockJoinFieldComparator extends FieldComparator<O
while (true) {
childDoc = childDocuments.nextSetBit(childDoc + 1);
if (childDoc >= parentDoc || childDoc == -1) {
if (childDoc >= parentDoc || childDoc == DocIdSetIterator.NO_MORE_DOCS) {
return cmp;
}
int cmp1 = wrappedComparator.compareBottom(childDoc);
@ -295,7 +294,7 @@ public abstract class ToParentBlockJoinFieldComparator extends FieldComparator<O
int prevParentDoc = parentDocuments.prevSetBit(parentDoc - 1);
int childDoc = childDocuments.nextSetBit(prevParentDoc + 1);
if (childDoc >= parentDoc || childDoc == -1) {
if (childDoc >= parentDoc || childDoc == DocIdSetIterator.NO_MORE_DOCS) {
return;
}
wrappedComparator.copy(spareSlot, childDoc);
@ -303,7 +302,7 @@ public abstract class ToParentBlockJoinFieldComparator extends FieldComparator<O
while (true) {
childDoc = childDocuments.nextSetBit(childDoc + 1);
if (childDoc >= parentDoc || childDoc == -1) {
if (childDoc >= parentDoc || childDoc == DocIdSetIterator.NO_MORE_DOCS) {
return;
}
wrappedComparator.copy(spareSlot, childDoc);
@ -322,7 +321,7 @@ public abstract class ToParentBlockJoinFieldComparator extends FieldComparator<O
int prevParentDoc = parentDocuments.prevSetBit(parentDoc - 1);
int childDoc = childDocuments.nextSetBit(prevParentDoc + 1);
if (childDoc >= parentDoc || childDoc == -1) {
if (childDoc >= parentDoc || childDoc == DocIdSetIterator.NO_MORE_DOCS) {
return 0;
}
@ -333,7 +332,7 @@ public abstract class ToParentBlockJoinFieldComparator extends FieldComparator<O
while (true) {
childDoc = childDocuments.nextSetBit(childDoc + 1);
if (childDoc >= parentDoc || childDoc == -1) {
if (childDoc >= parentDoc || childDoc == DocIdSetIterator.NO_MORE_DOCS) {
return cmp;
}
int cmp1 = wrappedComparator.compareTop(childDoc);

9
lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinQuery.java
View File

@ -39,7 +39,6 @@ import org.apache.lucene.search.Weight;
import org.apache.lucene.search.grouping.TopGroups;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
/**
@ -184,11 +183,11 @@ public class ToParentBlockJoinQuery extends Query {
// No matches
return null;
}
if (!(parents instanceof FixedBitDocIdSet)) {
throw new IllegalStateException("parentFilter must return FixedBitSet; got " + parents);
if (!(parents.bits() instanceof FixedBitSet)) {
throw new IllegalStateException("parentFilter must return FixedBitSet; got " + parents.bits());
}
return new BlockJoinScorer(this, childScorer, ((FixedBitDocIdSet) parents).bits(), firstChildDoc, scoreMode, acceptDocs);
return new BlockJoinScorer(this, childScorer, (FixedBitSet) parents.bits(), firstChildDoc, scoreMode, acceptDocs);
}
@Override
@ -288,7 +287,7 @@ public class ToParentBlockJoinQuery extends Query {
}
//System.out.println(" parentDoc=" + parentDoc);
assert parentDoc != -1;
assert parentDoc != DocIdSetIterator.NO_MORE_DOCS;
//System.out.println(" nextChildDoc=" + nextChildDoc);
if (acceptDocs != null && !acceptDocs.get(parentDoc)) {

6
lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java
View File

@ -64,10 +64,10 @@ import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.TopScoreDocCollector;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BitSetIterator;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.FixedBitSet.FixedBitSetIterator;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
import org.junit.Test;
@ -505,12 +505,12 @@ public class TestJoinUtil extends LuceneTestCase {
// Asserting bit set...
if (VERBOSE) {
System.out.println("expected cardinality:" + expectedResult.cardinality());
DocIdSetIterator iterator = new FixedBitSetIterator(expectedResult, expectedResult.cardinality());
DocIdSetIterator iterator = new BitSetIterator(expectedResult, expectedResult.cardinality());
for (int doc = iterator.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = iterator.nextDoc()) {
System.out.println(String.format(Locale.ROOT, "Expected doc[%d] with id value %s", doc, indexSearcher.doc(doc).get("id")));
}
System.out.println("actual cardinality:" + actualResult.cardinality());
iterator = new FixedBitSetIterator(actualResult, actualResult.cardinality());
iterator = new BitSetIterator(actualResult, actualResult.cardinality());
for (int doc = iterator.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = iterator.nextDoc()) {
System.out.println(String.format(Locale.ROOT, "Actual doc[%d] with id value %s", doc, indexSearcher.doc(doc).get("id")));
}

4
lucene/misc/src/java/org/apache/lucene/index/PKIndexSplitter.java
View File

@ -26,9 +26,9 @@ import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Filter;
import org.apache.lucene.search.TermRangeFilter;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BitSetIterator;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.FixedBitSet.FixedBitSetIterator;
import org.apache.lucene.util.IOUtils;
/**
@ -142,7 +142,7 @@ public class PKIndexSplitter {
if (in.hasDeletions()) {
final Bits oldLiveDocs = in.getLiveDocs();
assert oldLiveDocs != null;
final DocIdSetIterator it = new FixedBitSetIterator(bits, 0L); // the cost is not useful here
final DocIdSetIterator it = new BitSetIterator(bits, 0L); // the cost is not useful here
for (int i = it.nextDoc(); i < maxDoc; i = it.nextDoc()) {
if (!oldLiveDocs.get(i)) {
// we can safely modify the current bit, as the iterator already stepped over it:

4
lucene/misc/src/java/org/apache/lucene/index/sorter/BlockJoinComparatorSource.java
View File

@ -30,7 +30,7 @@ import org.apache.lucene.search.ScoreDoc; // javadocs
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.util.FixedBitDocIdSet;
import org.apache.lucene.util.BitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
/**
@ -149,7 +149,7 @@ public class BlockJoinComparatorSource extends FieldComparatorSource {
if (parents == null) {
throw new IllegalStateException("LeafReader " + context.reader() + " contains no parents!");
}
if (!(parents instanceof FixedBitDocIdSet)) {
if (!(parents instanceof BitDocIdSet)) {
throw new IllegalStateException("parentFilter must return FixedBitSet; got " + parents);
}
parentBits = (FixedBitSet) parents.bits();

4
lucene/misc/src/test/org/apache/lucene/index/sorter/TestBlockJoinSorter.java
View File

@ -41,7 +41,7 @@ import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.FixedBitDocIdSet;
import org.apache.lucene.util.BitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LuceneTestCase;
@ -58,7 +58,7 @@ public class TestBlockJoinSorter extends LuceneTestCase {
throws IOException {
final FixedBitSet cached = new FixedBitSet(reader.maxDoc());
cached.or(iterator);
return new FixedBitDocIdSet(cached);
return new BitDocIdSet(cached);
}
}

4
lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheSortRandom.java
View File

@ -52,7 +52,7 @@ import org.apache.lucene.store.Directory;
import org.apache.lucene.uninverting.UninvertingReader.Type;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitDocIdSet;
import org.apache.lucene.util.BitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
@ -291,7 +291,7 @@ public class TestFieldCacheSortRandom extends LuceneTestCase {
}
}
return new FixedBitDocIdSet(bits);
return new BitDocIdSet(bits);
}
}
}

14
lucene/queries/src/java/org/apache/lucene/queries/BooleanFilter.java
View File

@ -30,8 +30,7 @@ import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Filter;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.DocIdSetBuilder;
/**
* A container Filter that allows Boolean composition of Filters.
@ -52,7 +51,7 @@ public class BooleanFilter extends Filter implements Iterable<FilterClause> {
*/
@Override
public DocIdSet getDocIdSet(LeafReaderContext context, Bits acceptDocs) throws IOException {
FixedBitSet res = null;
DocIdSetBuilder res = null;
final LeafReader reader = context.reader();
boolean hasShouldClauses = false;
@ -62,7 +61,7 @@ public class BooleanFilter extends Filter implements Iterable<FilterClause> {
final DocIdSetIterator disi = getDISI(fc.getFilter(), context);
if (disi == null) continue;
if (res == null) {
res = new FixedBitSet(reader.maxDoc());
res = new DocIdSetBuilder(reader.maxDoc());
}
res.or(disi);
}
@ -74,8 +73,7 @@ public class BooleanFilter extends Filter implements Iterable<FilterClause> {
if (fc.getOccur() == Occur.MUST_NOT) {
if (res == null) {
assert !hasShouldClauses;
res = new FixedBitSet(reader.maxDoc());
res.set(0, reader.maxDoc()); // NOTE: may set bits on deleted docs
res = new DocIdSetBuilder(reader.maxDoc(), true); // NOTE: may set bits on deleted docs
}
final DocIdSetIterator disi = getDISI(fc.getFilter(), context);
if (disi != null) {
@ -91,7 +89,7 @@ public class BooleanFilter extends Filter implements Iterable<FilterClause> {
return null; // no documents can match
}
if (res == null) {
res = new FixedBitSet(reader.maxDoc());
res = new DocIdSetBuilder(reader.maxDoc());
res.or(disi);
} else {
res.and(disi);
@ -102,7 +100,7 @@ public class BooleanFilter extends Filter implements Iterable<FilterClause> {
if (res == null) {
return null;
}
return BitsFilteredDocIdSet.wrap(new FixedBitDocIdSet(res), acceptDocs);
return BitsFilteredDocIdSet.wrap(res.build(), acceptDocs);
}
private static DocIdSetIterator getDISI(Filter filter, LeafReaderContext context)

4
lucene/queries/src/test/org/apache/lucene/queries/BooleanFilterTest.java
View File

@ -37,7 +37,7 @@ import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TermRangeFilter;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitDocIdSet;
import org.apache.lucene.util.BitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LuceneTestCase;
@ -94,7 +94,7 @@ public class BooleanFilterTest extends LuceneTestCase {
return new Filter() {
@Override
public DocIdSet getDocIdSet(LeafReaderContext context, Bits acceptDocs) {
return new FixedBitDocIdSet(new FixedBitSet(context.reader().maxDoc()));
return new BitDocIdSet(new FixedBitSet(context.reader().maxDoc()));
}
};
}

16
lucene/queries/src/test/org/apache/lucene/queries/TermsFilterTest.java
View File

@ -47,7 +47,7 @@ import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitDocIdSet;
import org.apache.lucene.util.BitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
@ -82,19 +82,19 @@ public class TermsFilterTest extends LuceneTestCase {
List<Term> terms = new ArrayList<>();
terms.add(new Term(fieldName, "19"));
FixedBitDocIdSet bits = (FixedBitDocIdSet) termsFilter(random().nextBoolean(), terms).getDocIdSet(context, context.reader().getLiveDocs());
BitDocIdSet bits = (BitDocIdSet) termsFilter(random().nextBoolean(), terms).getDocIdSet(context, context.reader().getLiveDocs());
assertNull("Must match nothing", bits);
terms.add(new Term(fieldName, "20"));
bits = (FixedBitDocIdSet) termsFilter(random().nextBoolean(), terms).getDocIdSet(context, context.reader().getLiveDocs());
bits = (BitDocIdSet) termsFilter(random().nextBoolean(), terms).getDocIdSet(context, context.reader().getLiveDocs());
assertEquals("Must match 1", 1, bits.bits().cardinality());
terms.add(new Term(fieldName, "10"));
bits = (FixedBitDocIdSet) termsFilter(random().nextBoolean(), terms).getDocIdSet(context, context.reader().getLiveDocs());
bits = (BitDocIdSet) termsFilter(random().nextBoolean(), terms).getDocIdSet(context, context.reader().getLiveDocs());
assertEquals("Must match 2", 2, bits.bits().cardinality());
terms.add(new Term(fieldName, "00"));
bits = (FixedBitDocIdSet) termsFilter(random().nextBoolean(), terms).getDocIdSet(context, context.reader().getLiveDocs());
bits = (BitDocIdSet) termsFilter(random().nextBoolean(), terms).getDocIdSet(context, context.reader().getLiveDocs());
assertEquals("Must match 2", 2, bits.bits().cardinality());
reader.close();
@ -127,7 +127,7 @@ public class TermsFilterTest extends LuceneTestCase {
if (context.reader().docFreq(new Term(fieldName, "content1")) == 0) {
assertNull(docIdSet);
} else {
FixedBitDocIdSet bits = (FixedBitDocIdSet) docIdSet;
BitDocIdSet bits = (BitDocIdSet) docIdSet;
assertTrue("Must be >= 0", bits.bits().cardinality() >= 0);
}
}
@ -164,7 +164,7 @@ public class TermsFilterTest extends LuceneTestCase {
LeafReaderContext context = reader.leaves().get(0);
TermsFilter tf = new TermsFilter(terms);
FixedBitDocIdSet bits = (FixedBitDocIdSet) tf.getDocIdSet(context, context.reader().getLiveDocs());
BitDocIdSet bits = (BitDocIdSet) tf.getDocIdSet(context, context.reader().getLiveDocs());
assertEquals("Must be num fields - 1 since we skip only one field", num-1, bits.bits().cardinality());
reader.close();
dir.close();
@ -200,7 +200,7 @@ public class TermsFilterTest extends LuceneTestCase {
LeafReaderContext context = reader.leaves().get(0);
TermsFilter tf = new TermsFilter(new ArrayList<>(terms));
FixedBitDocIdSet bits = (FixedBitDocIdSet) tf.getDocIdSet(context, context.reader().getLiveDocs());
BitDocIdSet bits = (BitDocIdSet) tf.getDocIdSet(context, context.reader().getLiveDocs());
assertEquals(context.reader().numDocs(), bits.bits().cardinality());
reader.close();
dir.close();

7
lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/DuplicateFilter.java
View File

@ -27,11 +27,10 @@ import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Filter;
import org.apache.lucene.util.BitDocIdSet;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.SparseFixedBitDocIdSet;
import org.apache.lucene.util.SparseFixedBitSet;
/**
@ -127,7 +126,7 @@ public class DuplicateFilter extends Filter {
}
}
}
return new SparseFixedBitDocIdSet(bits, bits.approximateCardinality());
return new BitDocIdSet(bits, bits.approximateCardinality());
}
private DocIdSet fastBits(LeafReader reader, Bits acceptDocs) throws IOException {
@ -172,7 +171,7 @@ public class DuplicateFilter extends Filter {
}
}
return new FixedBitDocIdSet(bits);
return new BitDocIdSet(bits);
}
public String getFieldName() {

4
lucene/sandbox/src/test/org/apache/lucene/search/TestTermAutomatonQuery.java
View File

@ -46,7 +46,7 @@ import org.apache.lucene.index.Term;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitDocIdSet;
import org.apache.lucene.util.BitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
@ -638,7 +638,7 @@ public class TestTermAutomatonQuery extends LuceneTestCase {
}
}
return new FixedBitDocIdSet(bits);
return new BitDocIdSet(bits);
}
}
}

4
lucene/spatial/src/java/org/apache/lucene/spatial/prefix/IntersectsPrefixTreeFilter.java
View File

@ -25,7 +25,7 @@ import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.spatial.prefix.tree.Cell;
import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitDocIdSet;
import org.apache.lucene.util.BitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
import java.io.IOException;
@ -73,7 +73,7 @@ public class IntersectsPrefixTreeFilter extends AbstractVisitingPrefixTreeFilter
@Override
protected DocIdSet finish() {
return new FixedBitDocIdSet(results);
return new BitDocIdSet(results);
}
@Override

4
lucene/spatial/src/java/org/apache/lucene/spatial/prefix/WithinPrefixTreeFilter.java
View File

@ -25,7 +25,7 @@ import org.apache.lucene.spatial.prefix.tree.Cell;
import org.apache.lucene.spatial.prefix.tree.CellIterator;
import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitDocIdSet;
import org.apache.lucene.util.BitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
import com.spatial4j.core.context.SpatialContext;
@ -136,7 +136,7 @@ public class WithinPrefixTreeFilter extends AbstractVisitingPrefixTreeFilter {
@Override
protected DocIdSet finish() {
inside.andNot(outside);
return new FixedBitDocIdSet(inside);
return new BitDocIdSet(inside);
}
@Override

235
lucene/test-framework/src/java/org/apache/lucene/util/BaseBitSetTestCase.java Normal file
View File

@ -0,0 +1,235 @@
package org.apache.lucene.util;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import java.util.Collections;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
import org.junit.Ignore;
/**
* Base test case for BitSets.
*/
@Ignore
public abstract class BaseBitSetTestCase<T extends BitSet> extends LuceneTestCase {
/** Create a copy of the given {@link BitSet} which has <code>length</code> bits. */
public abstract T copyOf(BitSet bs, int length) throws IOException;
/** Create a random set which has <code>numBitsSet</code> of its <code>numBits</code> bits set. */
static java.util.BitSet randomSet(int numBits, int numBitsSet) {
assert numBitsSet <= numBits;
final java.util.BitSet set = new java.util.BitSet(numBits);
if (numBitsSet == numBits) {
set.set(0, numBits);
} else {
for (int i = 0; i < numBitsSet; ++i) {
while (true) {
final int o = random().nextInt(numBits);
if (!set.get(o)) {
set.set(o);
break;
}
}
}
}
return set;
}
/** Same as {@link #randomSet(int, int)} but given a load factor. */
static java.util.BitSet randomSet(int numBits, float percentSet) {
return randomSet(numBits, (int) (percentSet * numBits));
}
private void assertEquals(BitSet set1, BitSet set2, int maxDoc) {
for (int i = 0; i < maxDoc; ++i) {
assertEquals("Different at " + i, set1.get(i), set2.get(i));
}
}
/** Test the {@link BitSet#cardinality()} method. */
public void testCardinality() throws IOException {
final int numBits = 1 + random().nextInt(100000);
for (float percentSet : new float[] {0, 0.01f, 0.1f, 0.5f, 0.9f, 0.99f, 1f}) {
BitSet set1 = new JavaUtilBitSet(randomSet(numBits, percentSet), numBits);
T set2 = copyOf(set1, numBits);
assertEquals(set1.cardinality(), set2.cardinality());
}
}
/** Test the {@link BitSet#set} method. */
public void testSet() throws IOException {
final int numBits = 1 + random().nextInt(100000);
for (float percentSet : new float[] {0, 0.01f, 0.1f, 0.5f, 0.9f, 0.99f, 1f}) {
BitSet set1 = new JavaUtilBitSet(randomSet(numBits, percentSet), numBits);
T set2 = copyOf(set1, numBits);
assertEquals(set1, set2, numBits);
}
}
/** Test the {@link BitSet#clear(int)} method. */
public void testClear() throws IOException {
final int numBits = 1 + random().nextInt(100000);
for (float percentSet : new float[] {0, 0.01f, 0.1f, 0.5f, 0.9f, 0.99f, 1f}) {
BitSet set1 = new JavaUtilBitSet(randomSet(numBits, percentSet), numBits);
T set2 = copyOf(set1, numBits);
final int iters = 1 + random().nextInt(numBits * 2);
for (int i = 0; i < iters; ++i) {
final int index = random().nextInt(numBits);
set1.clear(index);
set2.clear(index);
}
assertEquals(set1, set2, numBits);
}
}
/** Test the {@link BitSet#clear(int,int)} method. */
public void testClearRange() throws IOException {
final int numBits = 1 + random().nextInt(100000);
for (float percentSet : new float[] {0, 0.01f, 0.1f, 0.5f, 0.9f, 0.99f, 1f}) {
BitSet set1 = new JavaUtilBitSet(randomSet(numBits, percentSet), numBits);
T set2 = copyOf(set1, numBits);
final int iters = 1 + random().nextInt(100);
for (int i = 0; i < iters; ++i) {
final int from = random().nextInt(numBits);
final int to = random().nextInt(numBits + 1);
set1.clear(from, to);
set2.clear(from, to);
assertEquals(set1, set2, numBits);
}
}
}
private DocIdSet randomCopy(BitSet set, int numBits) throws IOException {
if (random().nextBoolean()) {
return new BitDocIdSet(copyOf(set, numBits), set.cardinality());
} else if (random().nextBoolean()) {
final RoaringDocIdSet.Builder builder = new RoaringDocIdSet.Builder(numBits);
for (int i = set.nextSetBit(0); i != DocIdSetIterator.NO_MORE_DOCS; i = i + 1 >= numBits ? DocIdSetIterator.NO_MORE_DOCS : set.nextSetBit(i + 1)) {
builder.add(i);
}
return builder.build();
} else {
return new BitDocIdSet(set, set.cardinality());
}
}
/** Test the {@link BitSet#and}, {@link BitSet#or} and {@link BitSet#andNot} methods. */
public void testBulkOperations() throws IOException {
final int numBits = 1 + random().nextInt(100000);
BitSet set1 = new JavaUtilBitSet(randomSet(numBits, 0), numBits);
T set2 = copyOf(set1, numBits);
final int iters = 10 + random().nextInt(100);
for (int i = 0; i < iters; ++i) {
// make extreme percents more likely
float percentSet2 = (float) Math.pow(random().nextDouble(), 2);
if (random().nextBoolean()) {
percentSet2 = 1 - percentSet2;
}
BitSet bulkSet = new JavaUtilBitSet(randomSet(numBits, percentSet2), numBits);
// operations are sometimes specialized based on the impl, so randomize the impl
final DocIdSet bulkSetCopy = randomCopy(bulkSet, numBits);
// now randomize the operation
if (bulkSetCopy.iterator() == null) {
continue;
}
switch (random().nextInt(3)) {
case 0:
set1.or(bulkSetCopy.iterator());
set2.or(bulkSetCopy.iterator());
break;
case 1:
set1.and(bulkSetCopy.iterator());
set2.and(bulkSetCopy.iterator());
break;
default:
set1.andNot(bulkSetCopy.iterator());
set2.andNot(bulkSetCopy.iterator());
break;
}
assertEquals(set1, set2, numBits);
}
}
private static class JavaUtilBitSet extends BitSet {
private final java.util.BitSet bitSet;
private final int numBits;
JavaUtilBitSet(java.util.BitSet bitSet, int numBits) {
this.bitSet = bitSet;
this.numBits = numBits;
}
@Override
public void clear(int index) {
bitSet.clear(index);
}
@Override
public boolean get(int index) {
return bitSet.get(index);
}
@Override
public int length() {
return numBits;
}
@Override
public long ramBytesUsed() {
return -1;
}
@Override
public Iterable<? extends Accountable> getChildResources() {
return Collections.emptyList();
}
@Override
public void set(int i) {
bitSet.set(i);
}
@Override
public void clear(int startIndex, int endIndex) {
if (startIndex >= endIndex) {
return;
}
bitSet.clear(startIndex, endIndex);
}
@Override
public int cardinality() {
return bitSet.cardinality();
}
@Override
public int nextSetBit(int i) {
int next = bitSet.nextSetBit(i);
if (next == -1) {
next = DocIdSetIterator.NO_MORE_DOCS;
}
return next;
}
}
}

27
lucene/test-framework/src/java/org/apache/lucene/util/BaseDocIdSetTestCase.java
View File

@ -17,6 +17,8 @@ package org.apache.lucene.util;
* limitations under the License.
*/
import static org.apache.lucene.util.BaseBitSetTestCase.randomSet;
import java.io.IOException;
import java.util.BitSet;
@ -29,31 +31,6 @@ public abstract class BaseDocIdSetTestCase<T extends DocIdSet> extends LuceneTes
/** Create a copy of the given {@link BitSet} which has <code>length</code> bits. */
public abstract T copyOf(BitSet bs, int length) throws IOException;
/** Create a random set which has <code>numBitsSet</code> of its <code>numBits</code> bits set. */
protected static BitSet randomSet(int numBits, int numBitsSet) {
assert numBitsSet <= numBits;
final BitSet set = new BitSet(numBits);
if (numBitsSet == numBits) {
set.set(0, numBits);
} else {
for (int i = 0; i < numBitsSet; ++i) {
while (true) {
final int o = random().nextInt(numBits);
if (!set.get(o)) {
set.set(o);
break;
}
}
}
}
return set;
}
/** Same as {@link #randomSet(int, int)} but given a load factor. */
protected static BitSet randomSet(int numBits, float percentSet) {
return randomSet(numBits, (int) (percentSet * numBits));
}
/** Test length=0. */
public void testNoBit() throws IOException {
final BitSet bs = new BitSet(1);

4
solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java
View File

@ -40,10 +40,10 @@ import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.TopDocsCollector;
import org.apache.lucene.search.TopFieldCollector;
import org.apache.lucene.search.TopScoreDocCollector;
import org.apache.lucene.util.BitSetIterator;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRefBuilder;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.FixedBitSet.FixedBitSetIterator;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.params.ExpandParams;
import org.apache.solr.common.params.ShardParams;
@ -308,7 +308,7 @@ public class ExpandComponent extends SearchComponent implements PluginInfoInitia
public GroupExpandCollector(SortedDocValues docValues, FixedBitSet groupBits, IntOpenHashSet collapsedSet, int limit, Sort sort) throws IOException {
int numGroups = collapsedSet.size();
groups = new IntObjectOpenHashMap<>(numGroups * 2);
DocIdSetIterator iterator = new FixedBitSetIterator(groupBits, 0); // cost is not useful here
DocIdSetIterator iterator = new BitSetIterator(groupBits, 0); // cost is not useful here
int group;
while ((group = iterator.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
Collector collector = (sort == null) ? TopScoreDocCollector.create(limit, true) : TopFieldCollector.create(sort, limit, false, false, false, true);

4
solr/core/src/java/org/apache/solr/response/SortingResponseWriter.java
View File

@ -32,10 +32,10 @@ import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BitSetIterator;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRefBuilder;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.FixedBitSet.FixedBitSetIterator;
import org.apache.lucene.util.LongValues;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
@ -135,7 +135,7 @@ public class SortingResponseWriter implements QueryResponseWriter {
SortDoc top = queue.top();
for(int i=0; i<leaves.size(); i++) {
sortDoc.setNextReader(leaves.get(i));
DocIdSetIterator it = new FixedBitSetIterator(sets[i], 0); // cost is not useful here
DocIdSetIterator it = new BitSetIterator(sets[i], 0); // cost is not useful here
int docId = -1;
while((docId = it.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
sortDoc.setValues(docId);

12
solr/core/src/java/org/apache/solr/search/BitDocSet.java
View File

@ -23,10 +23,10 @@ import org.apache.lucene.search.BitsFilteredDocIdSet;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Filter;
import org.apache.lucene.util.BitDocIdSet;
import org.apache.lucene.util.BitSetIterator;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.FixedBitSet.FixedBitSetIterator;
import org.apache.lucene.util.FixedBitDocIdSet;
/**
* <code>BitDocSet</code> represents an unordered set of Lucene Document Ids
@ -92,7 +92,7 @@ public class BitDocSet extends DocSetBase {
@Override
public DocIterator iterator() {
return new DocIterator() {
private final FixedBitSetIterator iter = new FixedBitSetIterator(bits, 0L); // cost is not useful here
private final BitSetIterator iter = new BitSetIterator(bits, 0L); // cost is not useful here
private int pos = iter.nextDoc();
@Override
public boolean hasNext() {
@ -277,7 +277,7 @@ public class BitDocSet extends DocSetBase {
final Bits acceptDocs2 = acceptDocs == null ? null : (reader.getLiveDocs() == acceptDocs ? null : acceptDocs);
if (context.isTopLevel) {
return BitsFilteredDocIdSet.wrap(new FixedBitDocIdSet(bs), acceptDocs);
return BitsFilteredDocIdSet.wrap(new BitDocIdSet(bs), acceptDocs);
}
final int base = context.docBase;
@ -302,7 +302,7 @@ public class BitDocSet extends DocSetBase {
return adjustedDoc = NO_MORE_DOCS;
} else {
pos = bs.nextSetBit(pos + 1);
return adjustedDoc = (pos >= 0 && pos < max) ? pos - base : NO_MORE_DOCS;
return adjustedDoc = pos < max ? pos - base : NO_MORE_DOCS;
}
}
@ -314,7 +314,7 @@ public class BitDocSet extends DocSetBase {
return adjustedDoc = NO_MORE_DOCS;
} else {
pos = bs.nextSetBit(adjusted);
return adjustedDoc = (pos >= 0 && pos < max) ? pos - base : NO_MORE_DOCS;
return adjustedDoc = pos < max ? pos - base : NO_MORE_DOCS;
}
}

6
solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
View File

@ -33,9 +33,9 @@ import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.util.BitSetIterator;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.FixedBitSet.FixedBitSetIterator;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
@ -491,7 +491,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
leafDelegate = delegate.getLeafCollector(contexts[currentContext]);
DummyScorer dummy = new DummyScorer();
leafDelegate.setScorer(dummy);
DocIdSetIterator it = new FixedBitSetIterator(collapsedSet, 0L); // cost is not useful here
DocIdSetIterator it = new BitSetIterator(collapsedSet, 0L); // cost is not useful here
int docId = -1;
int nullScoreIndex = 0;
while((docId = it.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
@ -604,7 +604,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
leafDelegate = delegate.getLeafCollector(contexts[currentContext]);
DummyScorer dummy = new DummyScorer();
leafDelegate.setScorer(dummy);
DocIdSetIterator it = new FixedBitSetIterator(fieldValueCollapse.getCollapsedSet(), 0); // cost is not useful here
DocIdSetIterator it = new BitSetIterator(fieldValueCollapse.getCollapsedSet(), 0); // cost is not useful here
int docId = -1;
int nullScoreIndex = 0;
float[] scores = fieldValueCollapse.getScores();

8
solr/core/src/java/org/apache/solr/search/DocSetBase.java
View File

@ -24,7 +24,7 @@ import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Filter;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitDocIdSet;
import org.apache.lucene.util.BitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
import org.apache.solr.common.SolrException;
@ -172,7 +172,7 @@ abstract class DocSetBase implements DocSet {
final Bits acceptDocs2 = acceptDocs == null ? null : (reader.getLiveDocs() == acceptDocs ? null : acceptDocs);
if (context.isTopLevel) {
return BitsFilteredDocIdSet.wrap(new FixedBitDocIdSet(bs), acceptDocs);
return BitsFilteredDocIdSet.wrap(new BitDocIdSet(bs), acceptDocs);
}
final int base = context.docBase;
@ -194,14 +194,14 @@ abstract class DocSetBase implements DocSet {
@Override
public int nextDoc() {
pos = bs.nextSetBit(pos+1);
return adjustedDoc = (pos>=0 && pos<max) ? pos-base : NO_MORE_DOCS;
return adjustedDoc = pos<max ? pos-base : NO_MORE_DOCS;
}
@Override
public int advance(int target) {
if (target==NO_MORE_DOCS) return adjustedDoc=NO_MORE_DOCS;
pos = bs.nextSetBit(target+base);
return adjustedDoc = (pos>=0 && pos<max) ? pos-base : NO_MORE_DOCS;
return adjustedDoc = pos<max ? pos-base : NO_MORE_DOCS;
}
@Override

2
solr/core/src/java/org/apache/solr/search/join/BitSetSlice.java
View File

@ -41,6 +41,6 @@ class BitSetSlice {
public int nextSetBit(int pos) {
int result = fbs.nextSetBit(pos + off) - off;
return (result < 0 || result >= len) ? -1 : result;
return (result >= len) ? -1 : result;
}
}

8
solr/core/src/test/org/apache/solr/search/TestDocSet.java
View File

@ -39,9 +39,9 @@ import org.apache.lucene.index.StoredFieldVisitor;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Filter;
import org.apache.lucene.util.BitSetIterator;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.FixedBitSet.FixedBitSetIterator;
import org.apache.lucene.util.LuceneTestCase;
/**
@ -68,7 +68,7 @@ public class TestDocSet extends LuceneTestCase {
public DocSet getHashDocSet(FixedBitSet bs) {
int[] docs = new int[bs.cardinality()];
FixedBitSetIterator iter = new FixedBitSetIterator(bs, 0);
BitSetIterator iter = new BitSetIterator(bs, 0);
for (int i=0; i<docs.length; i++) {
docs[i] = iter.nextDoc();
}
@ -77,7 +77,7 @@ public class TestDocSet extends LuceneTestCase {
public DocSet getIntDocSet(FixedBitSet bs) {
int[] docs = new int[bs.cardinality()];
FixedBitSetIterator iter = new FixedBitSetIterator(bs, 0);
BitSetIterator iter = new BitSetIterator(bs, 0);
for (int i=0; i<docs.length; i++) {
docs[i] = iter.nextDoc();
}
@ -95,7 +95,7 @@ public class TestDocSet extends LuceneTestCase {
int offset = 3;
int end = offset + len;
FixedBitSetIterator iter = new FixedBitSetIterator(bs, 0);
BitSetIterator iter = new BitSetIterator(bs, 0);
// put in opposite order... DocLists are not ordered.
for (int i=end-1; i>=offset; i--) {
arr[i] = iter.nextDoc();

3
solr/core/src/test/org/apache/solr/search/TestFiltering.java
View File

@ -18,6 +18,7 @@
package org.apache.solr.search;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.util.FixedBitSet;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.params.SolrParams;
@ -222,7 +223,7 @@ public class TestFiltering extends SolrTestCaseJ4 {
for (int doc=-1;;) {
if (doc+1 >= model.indexSize) break;
doc = pset.nextSetBit(doc+1);
if (doc < 0) break;
if (doc == DocIdSetIterator.NO_MORE_DOCS) break;
sb.append((positive ? " ":" -") + f+":"+doc);
}

4
solr/core/src/test/org/apache/solr/search/TestSort.java
View File

@ -53,7 +53,7 @@ import org.apache.lucene.store.Directory;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.uninverting.UninvertingReader;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitDocIdSet;
import org.apache.lucene.util.BitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.TestUtil;
import org.apache.solr.SolrTestCaseJ4;
@ -340,7 +340,7 @@ public class TestSort extends SolrTestCaseJ4 {
for (int i=0; i<n; i++) {
obs.set(r.nextInt(sz));
}
return new FixedBitDocIdSet(obs);
return new BitDocIdSet(obs);
}

7
solr/core/src/test/org/apache/solr/util/BitSetPerf.java
View File

@ -20,8 +20,9 @@ package org.apache.solr.util;
import java.util.BitSet;
import java.util.Random;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.util.BitSetIterator;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.FixedBitSet.FixedBitSetIterator;
/** Performance tester for FixedBitSet.
* Use -Xbatch for more predictable results, and run tests such that the duration
@ -153,7 +154,7 @@ public class BitSetPerf {
for (int i=0; i<numSets; i++) {
if (impl=="open") {
final FixedBitSet set = osets[i];
for(int next=set.nextSetBit(0); next>=0; next=set.nextSetBit(next+1)) {
for(int next=set.nextSetBit(0); next != DocIdSetIterator.NO_MORE_DOCS; next=set.nextSetBit(next+1)) {
ret += next;
}
} else {
@ -172,7 +173,7 @@ public class BitSetPerf {
for (int i=0; i<numSets; i++) {
if (impl=="open") {
final FixedBitSet set = osets[i];
final FixedBitSetIterator iterator = new FixedBitSetIterator(set, 0);
final BitSetIterator iterator = new BitSetIterator(set, 0);
for(int next=iterator.nextDoc(); next>=0; next=iterator.nextDoc()) {
ret += next;
}