Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-5441: Decouple (Sparse)FixedBitSet from DocIdSet. 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1633628 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Adrien Grand 2014-10-22 14:19:19 +00:00
parent 6d6ed37027
commit 9e9812cbb0
50 changed files with 435 additions and 256 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
lucene/core/src/java/org/apache/lucene/search/DocValuesDocIdSet.java
View File

@ -19,6 +19,7 @@ package org.apache.lucene.search;
import java.io.IOException;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
/**
@ -116,7 +117,7 @@ public abstract class DocValuesDocIdSet extends DocIdSet {
} else if (acceptDocs instanceof FixedBitSet) {
// special case for FixedBitSet: use the iterator and filter it
// (used e.g. when Filters are chained by FilteredQuery)
return new FilteredDocIdSetIterator(((DocIdSet) acceptDocs).iterator()) {
return new FilteredDocIdSetIterator(new FixedBitDocIdSet((FixedBitSet) acceptDocs).iterator()) {
@Override
protected boolean match(int doc) {
return DocValuesDocIdSet.this.matchDoc(doc);

5
lucene/core/src/java/org/apache/lucene/search/SloppyPhraseScorer.java
View File

@ -510,9 +510,8 @@ final class SloppyPhraseScorer extends Scorer {
HashMap<Term,Integer> tg = new HashMap<>();
Term[] t = tord.keySet().toArray(new Term[0]);
for (int i=0; i<bb.size(); i++) { // i is the group no.
DocIdSetIterator bits = bb.get(i).iterator();
int ord;
while ((ord=bits.nextDoc())!=NO_MORE_DOCS) {
FixedBitSet bits = bb.get(i);
for (int ord = bits.nextSetBit(0); ord != -1; ord = ord + 1 >= bits.length() ? -1 : bits.nextSetBit(ord + 1)) {
tg.put(t[ord],i);
}
}

12
lucene/core/src/java/org/apache/lucene/util/DocIdSetBuilder.java
View File

@ -21,6 +21,7 @@ import java.io.IOException;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.util.SparseFixedBitSet.SparseFixedBitSetIterator;
/**
* A builder of {@link DocIdSet}s that supports random access.
@ -63,7 +64,7 @@ public final class DocIdSetBuilder {
denseSet = new FixedBitSet(maxDoc);
denseSet.or(it);
if (sparseSet != null) {
denseSet.or(sparseSet.iterator());
denseSet.or(new SparseFixedBitSetIterator(sparseSet, 0L));
}
return;
}
@ -84,7 +85,14 @@ public final class DocIdSetBuilder {
* anymore after this method has been called.
*/
public DocIdSet build() {
final DocIdSet result = denseSet != null ? denseSet : sparseSet;
final DocIdSet result;
if (denseSet != null) {
result = new FixedBitDocIdSet(denseSet, denseSet.cardinality());
} else if (sparseSet != null) {
result = new SparseFixedBitDocIdSet(sparseSet, sparseSet.approximateCardinality());
} else {
result = null;
}
denseSet = null;
sparseSet = null;
costUpperBound = 0;

79
lucene/core/src/java/org/apache/lucene/util/FixedBitDocIdSet.java Normal file
View File

@ -0,0 +1,79 @@
package org.apache.lucene.util;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.util.FixedBitSet.FixedBitSetIterator;
/**
* Implementation of the {@link DocIdSet} interface on top of a {@link FixedBitSet}.
* @lucene.internal
*/
public class FixedBitDocIdSet extends DocIdSet {
private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(SparseFixedBitDocIdSet.class);
private final FixedBitSet set;
private final long cost;
/**
* Wrap the given {@link FixedBitSet} as a {@link DocIdSet}. The provided
* {@link FixedBitSet} should not be modified after having wrapped as a
* {@link DocIdSet}.
*/
public FixedBitDocIdSet(FixedBitSet set, long cost) {
this.set = set;
this.cost = cost;
}
/**
* Same as {@link #FixedBitDocIdSet(FixedBitSet, long)} but uses the set
* {@link FixedBitSet#cardinality() cardinality} as a cost.
*/
public FixedBitDocIdSet(FixedBitSet set) {
this(set, set.cardinality());
}
@Override
public DocIdSetIterator iterator() {
return new FixedBitSetIterator(set, cost);
}
@Override
public FixedBitSet bits() {
return set;
}
/** This DocIdSet implementation is cacheable. */
@Override
public boolean isCacheable() {
return true;
}
@Override
public long ramBytesUsed() {
return BASE_RAM_BYTES_USED + set.ramBytesUsed();
}
@Override
public String toString() {
return getClass().getSimpleName() + "(set=" + set + ",cost=" + cost + ")";
}
}

34
lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java
View File

@ -19,6 +19,7 @@ package org.apache.lucene.util;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
@ -31,7 +32,7 @@ import org.apache.lucene.search.DocIdSetIterator;
*
* @lucene.internal
*/
public final class FixedBitSet extends DocIdSet implements MutableBits {
public final class FixedBitSet implements MutableBits, Accountable {
private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(FixedBitSet.class);
@ -43,18 +44,20 @@ public final class FixedBitSet extends DocIdSet implements MutableBits {
final int numBits, numWords;
final long[] bits;
final long cost;
int doc = -1;
/** Creates an iterator over the given {@link FixedBitSet}. */
public FixedBitSetIterator(FixedBitSet bits) {
this(bits.bits, bits.numBits, bits.numWords);
public FixedBitSetIterator(FixedBitSet bits, long cost) {
this(bits.bits, bits.numBits, bits.numWords, cost);
}
/** Creates an iterator over the given array of bits. */
public FixedBitSetIterator(long[] bits, int numBits, int wordLength) {
public FixedBitSetIterator(long[] bits, int numBits, int wordLength, long cost) {
this.bits = bits;
this.numBits = numBits;
this.numWords = wordLength;
this.cost = cost;
}
@Override
@ -69,7 +72,7 @@ public final class FixedBitSet extends DocIdSet implements MutableBits {
@Override
public long cost() {
return numBits;
return cost;
}
@Override
@ -180,32 +183,21 @@ public final class FixedBitSet extends DocIdSet implements MutableBits {
this.bits = storedBits;
}
@Override
public DocIdSetIterator iterator() {
return new FixedBitSetIterator(bits, numBits, numWords);
}
@Override
public Bits bits() {
return this;
}
@Override
public int length() {
return numBits;
}
/** This DocIdSet implementation is cacheable. */
@Override
public boolean isCacheable() {
return true;
}
@Override
public long ramBytesUsed() {
return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(bits);
}
@Override
public Iterable<? extends Accountable> getChildResources() {
return Collections.emptyList();
}
/** Expert. */
public long[] getBits() {
return bits;

2
lucene/core/src/java/org/apache/lucene/util/RoaringDocIdSet.java
View File

@ -92,7 +92,7 @@ public class RoaringDocIdSet extends DocIdSet {
sets[currentBlock] = new NotDocIdSet(BLOCK_SIZE, new ShortArrayDocIdSet(excludedDocs));
} else {
// Neither sparse nor super dense, use a fixed bit set
sets[currentBlock] = denseBuffer;
sets[currentBlock] = new FixedBitDocIdSet(denseBuffer, currentBlockCardinality);
}
denseBuffer = null;
}

71
lucene/core/src/java/org/apache/lucene/util/SparseFixedBitDocIdSet.java Normal file
View File

@ -0,0 +1,71 @@
package org.apache.lucene.util;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.util.SparseFixedBitSet.SparseFixedBitSetIterator;
/**
* Implementation of the {@link DocIdSet} interface on top of a {@link SparseFixedBitSet}.
* @lucene.internal
*/
public class SparseFixedBitDocIdSet extends DocIdSet {
private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(SparseFixedBitDocIdSet.class);
private final SparseFixedBitSet set;
private final long cost;
/**
* Wrap the given {@link FixedBitSet} as a {@link DocIdSet}. The provided
* {@link FixedBitSet} should not be modified after having wrapped as a
* {@link DocIdSet}.
*/
public SparseFixedBitDocIdSet(SparseFixedBitSet set, long cost) {
this.set = set;
this.cost = cost;
}
@Override
public SparseFixedBitSet bits() {
return set;
}
@Override
public boolean isCacheable() {
return true;
}
@Override
public DocIdSetIterator iterator() throws IOException {
return new SparseFixedBitSetIterator(set, cost);
}
@Override
public long ramBytesUsed() {
return BASE_RAM_BYTES_USED + set.ramBytesUsed();
}
@Override
public String toString() {
return getClass().getSimpleName() + "(set=" + set + ",cost=" + cost + ")";
}
}

42
lucene/core/src/java/org/apache/lucene/util/SparseFixedBitSet.java
View File

@ -18,8 +18,8 @@ package org.apache.lucene.util;
*/
import java.io.IOException;
import java.util.Collections;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
/**
@ -33,7 +33,7 @@ import org.apache.lucene.search.DocIdSetIterator;
*
* @lucene.internal
*/
public class SparseFixedBitSet extends DocIdSet implements Bits {
public class SparseFixedBitSet implements Bits, Accountable {
private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(SparseFixedBitSet.class);
private static final long SINGLE_ELEMENT_ARRAY_BYTES_USED = RamUsageEstimator.sizeOf(new long[1]);
@ -68,21 +68,11 @@ public class SparseFixedBitSet extends DocIdSet implements Bits {
+ RamUsageEstimator.shallowSizeOf(bits);
}
@Override
public boolean isCacheable() {
return true;
}
@Override
public int length() {
return length;
}
@Override
public Bits bits() throws IOException {
return this;
}
private boolean consistent(int index) {
assert index >= 0 && index < length : "index=" + index + ",length=" + length;
return true;
@ -218,14 +208,27 @@ public class SparseFixedBitSet extends DocIdSet implements Bits {
}
@Override
public DocIdSetIterator iterator() throws IOException {
return new Iterator();
public Iterable<? extends Accountable> getChildResources() {
return Collections.emptyList();
}
class Iterator extends DocIdSetIterator {
/**
* A {@link DocIdSetIterator} which iterates over set bits in a
* {@link SparseFixedBitSet}.
*/
public static class SparseFixedBitSetIterator extends DocIdSetIterator {
private final long[] indices;
private final long[][] bits;
private final long cost;
private int doc = -1;
private int cost = -1;
/** Sole constructor. */
public SparseFixedBitSetIterator(SparseFixedBitSet set, long cost) {
indices = set.indices;
bits = set.bits;
this.cost = cost;
}
@Override
public int docID() {
@ -315,13 +318,6 @@ public class SparseFixedBitSet extends DocIdSet implements Bits {
@Override
public long cost() {
// although constant-time, approximateCardinality is a bit expensive so
// we cache it to avoid performance traps eg. when sorting iterators by
// cost
if (cost < 0) {
cost = approximateCardinality();
}
assert cost >= 0;
return cost;
}

4
lucene/core/src/test/org/apache/lucene/search/MockFilter.java
View File

@ -18,6 +18,7 @@ package org.apache.lucene.search;
*/
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.util.FixedBitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.Bits;
@ -27,7 +28,8 @@ public class MockFilter extends Filter {
@Override
public DocIdSet getDocIdSet(LeafReaderContext context, Bits acceptDocs) {
wasCalled = true;
return new FixedBitSet(context.reader().maxDoc());
FixedBitSet bits = new FixedBitSet(context.reader().maxDoc());
return new FixedBitDocIdSet(bits);
}
public void clear() {

7
lucene/core/src/test/org/apache/lucene/search/SingleDocTestFilter.java
View File

@ -17,12 +17,13 @@ package org.apache.lucene.search;
* limitations under the License.
*/
import java.io.IOException;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
import java.io.IOException;
public class SingleDocTestFilter extends Filter {
private int doc;
@ -35,6 +36,6 @@ public class SingleDocTestFilter extends Filter {
FixedBitSet bits = new FixedBitSet(context.reader().maxDoc());
bits.set(doc);
if (acceptDocs != null && !acceptDocs.get(doc)) bits.clear(doc);
return bits;
return new FixedBitDocIdSet(bits);
}
}

3
lucene/core/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java
View File

@ -32,6 +32,7 @@ import org.apache.lucene.index.SlowCompositeReaderWrapper;
import org.apache.lucene.index.Term;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
@ -264,7 +265,7 @@ public class TestCachingWrapperFilter extends LuceneTestCase {
assertDocIdSetCacheable(reader, new Filter() {
@Override
public DocIdSet getDocIdSet(LeafReaderContext context, Bits acceptDocs) {
return new FixedBitSet(context.reader().maxDoc());
return new FixedBitDocIdSet(new FixedBitSet(context.reader().maxDoc()));
}
}, true);

5
lucene/core/src/test/org/apache/lucene/search/TestFilteredQuery.java
View File

@ -34,6 +34,7 @@ import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.FilteredQuery.FilterStrategy;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
@ -104,7 +105,7 @@ public class TestFilteredQuery extends LuceneTestCase {
FixedBitSet bitset = new FixedBitSet(context.reader().maxDoc());
if (acceptDocs.get(1)) bitset.set(1);
if (acceptDocs.get(3)) bitset.set(3);
return bitset;
return new FixedBitDocIdSet(bitset);
}
};
}
@ -185,7 +186,7 @@ public class TestFilteredQuery extends LuceneTestCase {
assertNull("acceptDocs should be null, as we have an index without deletions", acceptDocs);
FixedBitSet bitset = new FixedBitSet(context.reader().maxDoc());
bitset.set(0, Math.min(5, bitset.length()));
return bitset;
return new FixedBitDocIdSet(bitset);
}
};
}

3
lucene/core/src/test/org/apache/lucene/search/TestFilteredSearch.java
View File

@ -31,6 +31,7 @@ import org.apache.lucene.index.Term;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
@ -98,7 +99,7 @@ public class TestFilteredSearch extends LuceneTestCase {
set.set(docId-docBase);
}
}
return set.cardinality() == 0 ? null:set;
return set.cardinality() == 0 ? null : new FixedBitDocIdSet(set);
}
}

3
lucene/core/src/test/org/apache/lucene/search/TestScorerPerf.java
View File

@ -14,6 +14,7 @@ import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.Term;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LuceneTestCase;
@ -145,7 +146,7 @@ public class TestScorerPerf extends LuceneTestCase {
@Override
public DocIdSet getDocIdSet (LeafReaderContext context, Bits acceptDocs) {
assertNull("acceptDocs should be null, as we have an index without deletions", acceptDocs);
return rnd;
return new FixedBitDocIdSet(rnd);
}
});
bq.add(q, BooleanClause.Occur.MUST);

3
lucene/core/src/test/org/apache/lucene/search/TestSortRandom.java
View File

@ -39,6 +39,7 @@ import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
@ -260,7 +261,7 @@ public class TestSortRandom extends LuceneTestCase {
}
}
return bits;
return new FixedBitDocIdSet(bits);
}
}
}

8
lucene/core/src/test/org/apache/lucene/util/TestDocIdSetBuilder.java
View File

@ -60,8 +60,8 @@ public class TestDocIdSetBuilder extends LuceneTestCase {
builder.or(b.build().iterator());
}
DocIdSet result = builder.build();
assertTrue(result instanceof SparseFixedBitSet);
assertEquals(ref, result);
assertTrue(result instanceof SparseFixedBitDocIdSet);
assertEquals(new FixedBitDocIdSet(ref), result);
}
public void testDense() throws IOException {
@ -84,8 +84,8 @@ public class TestDocIdSetBuilder extends LuceneTestCase {
builder.or(b.build().iterator());
}
DocIdSet result = builder.build();
assertTrue(result instanceof FixedBitSet);
assertEquals(ref, result);
assertTrue(result instanceof FixedBitDocIdSet);
assertEquals(new FixedBitDocIdSet(ref), result);
}
}

11
lucene/core/src/test/org/apache/lucene/util/TestFixedBitSet.java
View File

@ -21,16 +21,17 @@ import java.io.IOException;
import java.util.BitSet;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.util.FixedBitSet.FixedBitSetIterator;
public class TestFixedBitSet extends BaseDocIdSetTestCase<FixedBitSet> {
public class TestFixedBitSet extends BaseDocIdSetTestCase<FixedBitDocIdSet> {
@Override
public FixedBitSet copyOf(BitSet bs, int length) throws IOException {
public FixedBitDocIdSet copyOf(BitSet bs, int length) throws IOException {
final FixedBitSet set = new FixedBitSet(length);
for (int doc = bs.nextSetBit(0); doc != -1; doc = bs.nextSetBit(doc + 1)) {
set.set(doc);
}
return set;
return new FixedBitDocIdSet(set);
}
void doGet(BitSet a, FixedBitSet b) {
@ -81,7 +82,7 @@ public class TestFixedBitSet extends BaseDocIdSetTestCase<FixedBitSet> {
void doIterate1(BitSet a, FixedBitSet b) throws IOException {
int aa=-1,bb=-1;
DocIdSetIterator iterator = b.iterator();
DocIdSetIterator iterator = new FixedBitSetIterator(b, 0);
do {
aa = a.nextSetBit(aa+1);
bb = (bb < b.length() && random().nextBoolean()) ? iterator.nextDoc() : iterator.advance(bb + 1);
@ -91,7 +92,7 @@ public class TestFixedBitSet extends BaseDocIdSetTestCase<FixedBitSet> {
void doIterate2(BitSet a, FixedBitSet b) throws IOException {
int aa=-1,bb=-1;
DocIdSetIterator iterator = b.iterator();
DocIdSetIterator iterator = new FixedBitSetIterator(b, 0);
do {
aa = a.nextSetBit(aa+1);
bb = random().nextBoolean() ? iterator.nextDoc() : iterator.advance(bb + 1);

4
lucene/core/src/test/org/apache/lucene/util/TestNotDocIdSet.java
View File

@ -30,7 +30,7 @@ public class TestNotDocIdSet extends BaseDocIdSetTestCase<NotDocIdSet> {
for (int doc = bs.nextClearBit(0); doc < length; doc = bs.nextClearBit(doc + 1)) {
set.set(doc);
}
return new NotDocIdSet(length, set);
return new NotDocIdSet(length, new FixedBitDocIdSet(set));
}
@Override
@ -48,7 +48,7 @@ public class TestNotDocIdSet extends BaseDocIdSetTestCase<NotDocIdSet> {
public void testBits() throws IOException {
assertNull(new NotDocIdSet(3, DocIdSet.EMPTY).bits());
assertNotNull(new NotDocIdSet(3, new FixedBitSet(3)).bits());
assertNotNull(new NotDocIdSet(3, new FixedBitDocIdSet(new FixedBitSet(3))).bits());
}
}

12
lucene/core/src/test/org/apache/lucene/util/TestSparseFixedBitSet.java
View File

@ -23,10 +23,10 @@ import java.util.BitSet;
import java.util.Collections;
import java.util.List;
public class TestSparseFixedBitSet extends BaseDocIdSetTestCase<SparseFixedBitSet> {
public class TestSparseFixedBitSet extends BaseDocIdSetTestCase<SparseFixedBitDocIdSet> {
@Override
public SparseFixedBitSet copyOf(BitSet bs, int length) throws IOException {
public SparseFixedBitDocIdSet copyOf(BitSet bs, int length) throws IOException {
final SparseFixedBitSet set = new SparseFixedBitSet(length);
// SparseFixedBitSet can be sensitive to the order of insertion so
// randomize insertion a bit
@ -45,15 +45,15 @@ public class TestSparseFixedBitSet extends BaseDocIdSetTestCase<SparseFixedBitSe
for (int i : buffer) {
set.set(i);
}
return set;
return new SparseFixedBitDocIdSet(set, set.approximateCardinality());
}
@Override
public void assertEquals(int numBits, BitSet ds1, SparseFixedBitSet ds2) throws IOException {
public void assertEquals(int numBits, BitSet ds1, SparseFixedBitDocIdSet ds2) throws IOException {
for (int i = 0; i < numBits; ++i) {
assertEquals(ds1.get(i), ds2.get(i));
assertEquals(ds1.get(i), ds2.bits().get(i));
}
assertEquals(ds1.cardinality(), ds2.cardinality());
assertEquals(ds1.cardinality(), ds2.bits().cardinality());
super.assertEquals(numBits, ds1, ds2);
}

3
lucene/facet/src/java/org/apache/lucene/facet/FacetsCollector.java
View File

@ -40,6 +40,7 @@ import org.apache.lucene.search.TopFieldCollector;
import org.apache.lucene.search.TopFieldDocs;
import org.apache.lucene.search.TopScoreDocCollector;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.FixedBitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
/** Collects hits for subsequent faceting. Once you've run
@ -128,7 +129,7 @@ public class FacetsCollector extends SimpleCollector {
@Override
public DocIdSet getDocIdSet() {
return bits;
return new FixedBitDocIdSet(bits);
}
};
}

3
lucene/facet/src/java/org/apache/lucene/facet/RandomSamplingFacetsCollector.java
View File

@ -26,6 +26,7 @@ import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.util.FixedBitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
/**
@ -212,7 +213,7 @@ public class RandomSamplingFacetsCollector extends FacetsCollector {
}
}
return new MatchingDocs(docs.context, sampleDocs, docs.totalHits, null);
return new MatchingDocs(docs.context, new FixedBitDocIdSet(sampleDocs), docs.totalHits, null);
} catch (IOException e) {
throw new RuntimeException();
}

3
lucene/facet/src/test/org/apache/lucene/facet/TestDrillSideways.java
View File

@ -58,6 +58,7 @@ import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.InPlaceMergeSorter;
@ -657,7 +658,7 @@ public class TestDrillSideways extends FacetTestCase {
bits.set(docID);
}
}
return bits;
return new FixedBitDocIdSet(bits);
}
};
} else {

3
lucene/facet/src/test/org/apache/lucene/facet/range/TestRangeFacetCounts.java
View File

@ -65,6 +65,7 @@ import org.apache.lucene.search.NumericRangeFilter;
import org.apache.lucene.search.NumericRangeQuery;
import org.apache.lucene.search.QueryWrapperFilter;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.FixedBitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.TestUtil;
@ -926,7 +927,7 @@ public class TestRangeFacetCounts extends FacetTestCase {
final FixedBitSet cached = new FixedBitSet(reader.maxDoc());
filterWasUsed.set(true);
cached.or(iterator);
return cached;
return new FixedBitDocIdSet(cached);
}
};
} else {

5
lucene/grouping/src/test/org/apache/lucene/search/grouping/AllGroupHeadsCollectorTest.java
View File

@ -43,7 +43,6 @@ import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.valuesource.BytesRefFieldSource;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.Sort;
@ -406,9 +405,7 @@ public class AllGroupHeadsCollectorTest extends LuceneTestCase {
expected.set(expectedDoc);
}
int docId;
DocIdSetIterator iterator = expected.iterator();
while ((docId = iterator.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
for (int docId = expected.nextSetBit(0); docId != -1; docId = docId + 1 >= expected.length() ? -1 : expected.nextSetBit(docId + 1)) {
if (!actual.get(docId)) {
return false;
}

5
lucene/join/src/java/org/apache/lucene/search/join/FixedBitSetCachingWrapperFilter.java
View File

@ -26,6 +26,7 @@ import org.apache.lucene.search.CachingWrapperFilter;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Filter;
import org.apache.lucene.util.FixedBitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
/** A {@link CachingWrapperFilter} that caches sets using a {@link FixedBitSet},
@ -42,7 +43,7 @@ public final class FixedBitSetCachingWrapperFilter extends CachingWrapperFilter
throws IOException {
if (docIdSet == null) {
return EMPTY;
} else if (docIdSet instanceof FixedBitSet) {
} else if (docIdSet instanceof FixedBitDocIdSet) {
// this is different from CachingWrapperFilter: even when the DocIdSet is
// cacheable, we convert it to a FixedBitSet since we require all the
// cached filters to be FixedBitSets
@ -54,7 +55,7 @@ public final class FixedBitSetCachingWrapperFilter extends CachingWrapperFilter
} else {
final FixedBitSet copy = new FixedBitSet(reader.maxDoc());
copy.or(it);
return copy;
return new FixedBitDocIdSet(copy);
}
}
}

9
lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java
View File

@ -21,25 +21,26 @@ import java.io.IOException;
import java.util.Locale;
import java.util.Set;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.LeafCollector;
import org.apache.lucene.search.BulkScorer;
import org.apache.lucene.search.ComplexExplanation;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.LeafCollector;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.BulkScorer;
import org.apache.lucene.search.Weight;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefHash;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.FixedBitSet.FixedBitSetIterator;
class TermsIncludingScoreQuery extends Query {
@ -328,7 +329,7 @@ class TermsIncludingScoreQuery extends Query {
FixedBitSet matchingDocs = new FixedBitSet(maxDoc);
this.scores = new float[maxDoc];
fillDocsAndScores(matchingDocs, acceptDocs, termsEnum);
this.matchingDocsIterator = matchingDocs.iterator();
this.matchingDocsIterator = new FixedBitSetIterator(matchingDocs, cost);
this.cost = cost;
}

7
lucene/join/src/java/org/apache/lucene/search/join/ToChildBlockJoinQuery.java
View File

@ -22,8 +22,8 @@ import java.util.Collection;
import java.util.Collections;
import java.util.Set;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.Explanation;
@ -33,6 +33,7 @@ import org.apache.lucene.search.Query;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.Weight;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
/**
@ -144,11 +145,11 @@ public class ToChildBlockJoinQuery extends Query {
// No matches
return null;
}
if (!(parents instanceof FixedBitSet)) {
if (!(parents instanceof FixedBitDocIdSet)) {
throw new IllegalStateException("parentFilter must return FixedBitSet; got " + parents);
}
return new ToChildBlockJoinScorer(this, parentScorer, (FixedBitSet) parents, doScores, acceptDocs);
return new ToChildBlockJoinScorer(this, parentScorer, ((FixedBitDocIdSet) parents).bits(), doScores, acceptDocs);
}
@Override

13
lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinFieldComparator.java
View File

@ -17,15 +17,16 @@ package org.apache.lucene.search.join;
* limitations under the License.
*/
import java.io.IOException;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.FieldComparator;
import org.apache.lucene.search.Filter;
import org.apache.lucene.util.FixedBitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
import java.io.IOException;
/**
* A field comparator that allows parent documents to be sorted by fields
* from the nested / child documents.
@ -69,8 +70,8 @@ public abstract class ToParentBlockJoinFieldComparator extends FieldComparator<O
DocIdSet innerDocuments = childFilter.getDocIdSet(context, null);
if (isEmpty(innerDocuments)) {
this.childDocuments = null;
} else if (innerDocuments instanceof FixedBitSet) {
this.childDocuments = (FixedBitSet) innerDocuments;
} else if (innerDocuments instanceof FixedBitDocIdSet) {
this.childDocuments = ((FixedBitDocIdSet) innerDocuments).bits();
} else {
DocIdSetIterator iterator = innerDocuments.iterator();
if (iterator != null) {
@ -82,8 +83,8 @@ public abstract class ToParentBlockJoinFieldComparator extends FieldComparator<O
DocIdSet rootDocuments = parentFilter.getDocIdSet(context, null);
if (isEmpty(rootDocuments)) {
this.parentDocuments = null;
} else if (rootDocuments instanceof FixedBitSet) {
this.parentDocuments = (FixedBitSet) rootDocuments;
} else if (rootDocuments instanceof FixedBitDocIdSet) {
this.parentDocuments = ((FixedBitDocIdSet) rootDocuments).bits();
} else {
DocIdSetIterator iterator = rootDocuments.iterator();
if (iterator != null) {

7
lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinQuery.java
View File

@ -23,9 +23,9 @@ import java.util.Collections;
import java.util.Locale;
import java.util.Set;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.ComplexExplanation;
import org.apache.lucene.search.DocIdSet;
@ -39,6 +39,7 @@ import org.apache.lucene.search.Weight;
import org.apache.lucene.search.grouping.TopGroups;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
/**
@ -183,11 +184,11 @@ public class ToParentBlockJoinQuery extends Query {
// No matches
return null;
}
if (!(parents instanceof FixedBitSet)) {
if (!(parents instanceof FixedBitDocIdSet)) {
throw new IllegalStateException("parentFilter must return FixedBitSet; got " + parents);
}
return new BlockJoinScorer(this, childScorer, (FixedBitSet) parents, firstChildDoc, scoreMode, acceptDocs);
return new BlockJoinScorer(this, childScorer, ((FixedBitDocIdSet) parents).bits(), firstChildDoc, scoreMode, acceptDocs);
}
@Override

2
lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java
View File

@ -359,7 +359,7 @@ public class TestBlockJoin extends LuceneTestCase {
final List<LeafReaderContext> leaves = reader.leaves();
final int subIndex = ReaderUtil.subIndex(childDocID, leaves);
final LeafReaderContext leaf = leaves.get(subIndex);
final FixedBitSet bits = (FixedBitSet) parents.getDocIdSet(leaf, null);
final FixedBitSet bits = (FixedBitSet) parents.getDocIdSet(leaf, null).bits();
return leaf.reader().document(bits.nextSetBit(childDocID - leaf.docBase));
}

9
lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java
View File

@ -37,12 +37,12 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.MultiFields;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.SlowCompositeReaderWrapper;
@ -67,6 +67,7 @@ import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.FixedBitSet.FixedBitSetIterator;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
import org.junit.Test;
@ -504,12 +505,12 @@ public class TestJoinUtil extends LuceneTestCase {
// Asserting bit set...
if (VERBOSE) {
System.out.println("expected cardinality:" + expectedResult.cardinality());
DocIdSetIterator iterator = expectedResult.iterator();
DocIdSetIterator iterator = new FixedBitSetIterator(expectedResult, expectedResult.cardinality());
for (int doc = iterator.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = iterator.nextDoc()) {
System.out.println(String.format(Locale.ROOT, "Expected doc[%d] with id value %s", doc, indexSearcher.doc(doc).get("id")));
}
System.out.println("actual cardinality:" + actualResult.cardinality());
iterator = actualResult.iterator();
iterator = new FixedBitSetIterator(actualResult, actualResult.cardinality());
for (int doc = iterator.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = iterator.nextDoc()) {
System.out.println(String.format(Locale.ROOT, "Actual doc[%d] with id value %s", doc, indexSearcher.doc(doc).get("id")));
}

3
lucene/misc/src/java/org/apache/lucene/index/PKIndexSplitter.java
View File

@ -28,6 +28,7 @@ import org.apache.lucene.search.TermRangeFilter;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.FixedBitSet.FixedBitSetIterator;
import org.apache.lucene.util.IOUtils;
/**
@ -141,7 +142,7 @@ public class PKIndexSplitter {
if (in.hasDeletions()) {
final Bits oldLiveDocs = in.getLiveDocs();
assert oldLiveDocs != null;
final DocIdSetIterator it = bits.iterator();
final DocIdSetIterator it = new FixedBitSetIterator(bits, 0L); // the cost is not useful here
for (int i = it.nextDoc(); i < maxDoc; i = it.nextDoc()) {
if (!oldLiveDocs.get(i)) {
// we can safely modify the current bit, as the iterator already stepped over it:

5
lucene/misc/src/java/org/apache/lucene/index/sorter/BlockJoinComparatorSource.java
View File

@ -30,6 +30,7 @@ import org.apache.lucene.search.ScoreDoc; // javadocs
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.util.FixedBitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
/**
@ -148,10 +149,10 @@ public class BlockJoinComparatorSource extends FieldComparatorSource {
if (parents == null) {
throw new IllegalStateException("LeafReader " + context.reader() + " contains no parents!");
}
if (!(parents instanceof FixedBitSet)) {
if (!(parents instanceof FixedBitDocIdSet)) {
throw new IllegalStateException("parentFilter must return FixedBitSet; got " + parents);
}
parentBits = (FixedBitSet) parents;
parentBits = (FixedBitSet) parents.bits();
for (int i = 0; i < parentComparators.length; i++) {
parentComparators[i] = parentComparators[i].setNextReader(context);
}

5
lucene/misc/src/test/org/apache/lucene/index/sorter/TestBlockJoinSorter.java
View File

@ -41,6 +41,7 @@ import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.FixedBitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LuceneTestCase;
@ -57,7 +58,7 @@ public class TestBlockJoinSorter extends LuceneTestCase {
throws IOException {
final FixedBitSet cached = new FixedBitSet(reader.maxDoc());
cached.or(iterator);
return cached;
return new FixedBitDocIdSet(cached);
}
}
@ -90,7 +91,7 @@ public class TestBlockJoinSorter extends LuceneTestCase {
final LeafReader reader = getOnlySegmentReader(indexReader);
final Filter parentsFilter = new FixedBitSetCachingWrapperFilter(new QueryWrapperFilter(new TermQuery(new Term("parent", "true"))));
final FixedBitSet parentBits = (FixedBitSet) parentsFilter.getDocIdSet(reader.getContext(), null);
final FixedBitSet parentBits = (FixedBitSet) parentsFilter.getDocIdSet(reader.getContext(), null).bits();
final NumericDocValues parentValues = reader.getNumericDocValues("parent_val");
final NumericDocValues childValues = reader.getNumericDocValues("child_val");

3
lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheSortRandom.java
View File

@ -52,6 +52,7 @@ import org.apache.lucene.store.Directory;
import org.apache.lucene.uninverting.UninvertingReader.Type;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
@ -290,7 +291,7 @@ public class TestFieldCacheSortRandom extends LuceneTestCase {
}
}
return bits;
return new FixedBitDocIdSet(bits);
}
}
}

6
lucene/queries/src/java/org/apache/lucene/queries/BooleanFilter.java
View File

@ -30,6 +30,7 @@ import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Filter;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
/**
@ -98,7 +99,10 @@ public class BooleanFilter extends Filter implements Iterable<FilterClause> {
}
}
return BitsFilteredDocIdSet.wrap(res, acceptDocs);
if (res == null) {
return null;
}
return BitsFilteredDocIdSet.wrap(new FixedBitDocIdSet(res), acceptDocs);
}
private static DocIdSetIterator getDISI(Filter filter, LeafReaderContext context)

13
lucene/queries/src/test/org/apache/lucene/queries/BooleanFilterTest.java
View File

@ -17,6 +17,8 @@ package org.apache.lucene.queries;
* limitations under the License.
*/
import java.io.IOException;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.document.Document;
@ -27,19 +29,18 @@ import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.SlowCompositeReaderWrapper;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Filter;
import org.apache.lucene.search.TermRangeFilter;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.QueryWrapperFilter;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TermRangeFilter;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LuceneTestCase;
import java.io.IOException;
public class BooleanFilterTest extends LuceneTestCase {
private Directory directory;
private LeafReader reader;
@ -93,7 +94,7 @@ public class BooleanFilterTest extends LuceneTestCase {
return new Filter() {
@Override
public DocIdSet getDocIdSet(LeafReaderContext context, Bits acceptDocs) {
return new FixedBitSet(context.reader().maxDoc());
return new FixedBitDocIdSet(new FixedBitSet(context.reader().maxDoc()));
}
};
}

27
lucene/queries/src/test/org/apache/lucene/queries/TermsFilterTest.java
View File

@ -47,6 +47,7 @@ import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
@ -81,20 +82,20 @@ public class TermsFilterTest extends LuceneTestCase {
List<Term> terms = new ArrayList<>();
terms.add(new Term(fieldName, "19"));
FixedBitSet bits = (FixedBitSet) termsFilter(random().nextBoolean(), terms).getDocIdSet(context, context.reader().getLiveDocs());
FixedBitDocIdSet bits = (FixedBitDocIdSet) termsFilter(random().nextBoolean(), terms).getDocIdSet(context, context.reader().getLiveDocs());
assertNull("Must match nothing", bits);
terms.add(new Term(fieldName, "20"));
bits = (FixedBitSet) termsFilter(random().nextBoolean(), terms).getDocIdSet(context, context.reader().getLiveDocs());
assertEquals("Must match 1", 1, bits.cardinality());
bits = (FixedBitDocIdSet) termsFilter(random().nextBoolean(), terms).getDocIdSet(context, context.reader().getLiveDocs());
assertEquals("Must match 1", 1, bits.bits().cardinality());
terms.add(new Term(fieldName, "10"));
bits = (FixedBitSet) termsFilter(random().nextBoolean(), terms).getDocIdSet(context, context.reader().getLiveDocs());
assertEquals("Must match 2", 2, bits.cardinality());
bits = (FixedBitDocIdSet) termsFilter(random().nextBoolean(), terms).getDocIdSet(context, context.reader().getLiveDocs());
assertEquals("Must match 2", 2, bits.bits().cardinality());
terms.add(new Term(fieldName, "00"));
bits = (FixedBitSet) termsFilter(random().nextBoolean(), terms).getDocIdSet(context, context.reader().getLiveDocs());
assertEquals("Must match 2", 2, bits.cardinality());
bits = (FixedBitDocIdSet) termsFilter(random().nextBoolean(), terms).getDocIdSet(context, context.reader().getLiveDocs());
assertEquals("Must match 2", 2, bits.bits().cardinality());
reader.close();
rd.close();
@ -126,8 +127,8 @@ public class TermsFilterTest extends LuceneTestCase {
if (context.reader().docFreq(new Term(fieldName, "content1")) == 0) {
assertNull(docIdSet);
} else {
FixedBitSet bits = (FixedBitSet) docIdSet;
assertTrue("Must be >= 0", bits.cardinality() >= 0);
FixedBitDocIdSet bits = (FixedBitDocIdSet) docIdSet;
assertTrue("Must be >= 0", bits.bits().cardinality() >= 0);
}
}
multi.close();
@ -163,8 +164,8 @@ public class TermsFilterTest extends LuceneTestCase {
LeafReaderContext context = reader.leaves().get(0);
TermsFilter tf = new TermsFilter(terms);
FixedBitSet bits = (FixedBitSet) tf.getDocIdSet(context, context.reader().getLiveDocs());
assertEquals("Must be num fields - 1 since we skip only one field", num-1, bits.cardinality());
FixedBitDocIdSet bits = (FixedBitDocIdSet) tf.getDocIdSet(context, context.reader().getLiveDocs());
assertEquals("Must be num fields - 1 since we skip only one field", num-1, bits.bits().cardinality());
reader.close();
dir.close();
}
@ -199,8 +200,8 @@ public class TermsFilterTest extends LuceneTestCase {
LeafReaderContext context = reader.leaves().get(0);
TermsFilter tf = new TermsFilter(new ArrayList<>(terms));
FixedBitSet bits = (FixedBitSet) tf.getDocIdSet(context, context.reader().getLiveDocs());
assertEquals(context.reader().numDocs(), bits.cardinality());
FixedBitDocIdSet bits = (FixedBitDocIdSet) tf.getDocIdSet(context, context.reader().getLiveDocs());
assertEquals(context.reader().numDocs(), bits.bits().cardinality());
reader.close();
dir.close();
}

22
lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/DuplicateFilter.java
View File

@ -29,7 +29,9 @@ import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Filter;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.SparseFixedBitDocIdSet;
import org.apache.lucene.util.SparseFixedBitSet;
/**
@ -93,14 +95,11 @@ public class DuplicateFilter extends Filter {
}
}
private SparseFixedBitSet correctBits(LeafReader reader, Bits acceptDocs) throws IOException {
private DocIdSet correctBits(LeafReader reader, Bits acceptDocs) throws IOException {
SparseFixedBitSet bits = new SparseFixedBitSet(reader.maxDoc()); //assume all are INvalid
Terms terms = reader.fields().terms(fieldName);
if (terms == null) {
return bits;
}
if (terms != null) {
TermsEnum termsEnum = terms.iterator(null);
DocsEnum docs = null;
while (true) {
@ -127,18 +126,16 @@ public class DuplicateFilter extends Filter {
}
}
}
return bits;
}
return new SparseFixedBitDocIdSet(bits, bits.approximateCardinality());
}
private FixedBitSet fastBits(LeafReader reader, Bits acceptDocs) throws IOException {
private DocIdSet fastBits(LeafReader reader, Bits acceptDocs) throws IOException {
FixedBitSet bits = new FixedBitSet(reader.maxDoc());
bits.set(0, reader.maxDoc()); //assume all are valid
Terms terms = reader.fields().terms(fieldName);
if (terms == null) {
return bits;
}
if (terms != null) {
TermsEnum termsEnum = terms.iterator(null);
DocsEnum docs = null;
while (true) {
@ -173,8 +170,9 @@ public class DuplicateFilter extends Filter {
}
}
}
}
return bits;
return new FixedBitDocIdSet(bits);
}
public String getFieldName() {

3
lucene/sandbox/src/test/org/apache/lucene/search/TestTermAutomatonQuery.java
View File

@ -46,6 +46,7 @@ import org.apache.lucene.index.Term;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
@ -637,7 +638,7 @@ public class TestTermAutomatonQuery extends LuceneTestCase {
}
}
return bits;
return new FixedBitDocIdSet(bits);
}
}
}

4
lucene/spatial/src/java/org/apache/lucene/spatial/prefix/IntersectsPrefixTreeFilter.java
View File

@ -19,11 +19,13 @@ package org.apache.lucene.spatial.prefix;
import com.spatial4j.core.shape.Shape;
import com.spatial4j.core.shape.SpatialRelation;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.spatial.prefix.tree.Cell;
import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
import java.io.IOException;
@ -71,7 +73,7 @@ public class IntersectsPrefixTreeFilter extends AbstractVisitingPrefixTreeFilter
@Override
protected DocIdSet finish() {
return results;
return new FixedBitDocIdSet(results);
}
@Override

22
lucene/spatial/src/java/org/apache/lucene/spatial/prefix/WithinPrefixTreeFilter.java
View File

@ -17,6 +17,17 @@ package org.apache.lucene.spatial.prefix;
* limitations under the License.
*/
import java.io.IOException;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.spatial.prefix.tree.Cell;
import org.apache.lucene.spatial.prefix.tree.CellIterator;
import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
import com.spatial4j.core.context.SpatialContext;
import com.spatial4j.core.distance.DistanceUtils;
import com.spatial4j.core.shape.Circle;
@ -24,15 +35,6 @@ import com.spatial4j.core.shape.Point;
import com.spatial4j.core.shape.Rectangle;
import com.spatial4j.core.shape.Shape;
import com.spatial4j.core.shape.SpatialRelation;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.spatial.prefix.tree.Cell;
import org.apache.lucene.spatial.prefix.tree.CellIterator;
import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitSet;
import java.io.IOException;
/**
* Finds docs where its indexed shape is {@link org.apache.lucene.spatial.query.SpatialOperation#IsWithin
@ -134,7 +136,7 @@ public class WithinPrefixTreeFilter extends AbstractVisitingPrefixTreeFilter {
@Override
protected DocIdSet finish() {
inside.andNot(outside);
return inside;
return new FixedBitDocIdSet(inside);
}
@Override

29
solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java
View File

@ -17,15 +17,17 @@
package org.apache.solr.handler.component;
import com.carrotsearch.hppc.IntObjectMap;
import com.carrotsearch.hppc.IntObjectOpenHashMap;
import com.carrotsearch.hppc.IntOpenHashSet;
import com.carrotsearch.hppc.cursors.IntObjectCursor;
import com.carrotsearch.hppc.cursors.ObjectCursor;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.search.Collector;
import org.apache.lucene.search.DocIdSetIterator;
@ -41,6 +43,7 @@ import org.apache.lucene.search.TopScoreDocCollector;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRefBuilder;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.FixedBitSet.FixedBitSetIterator;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.params.ExpandParams;
import org.apache.solr.common.params.ShardParams;
@ -60,13 +63,11 @@ import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.util.plugin.PluginInfoInitialized;
import org.apache.solr.util.plugin.SolrCoreAware;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import com.carrotsearch.hppc.IntObjectMap;
import com.carrotsearch.hppc.IntObjectOpenHashMap;
import com.carrotsearch.hppc.IntOpenHashSet;
import com.carrotsearch.hppc.cursors.IntObjectCursor;
import com.carrotsearch.hppc.cursors.ObjectCursor;
/**
* The ExpandComponent is designed to work with the CollapsingPostFilter.
@ -307,7 +308,7 @@ public class ExpandComponent extends SearchComponent implements PluginInfoInitia
public GroupExpandCollector(SortedDocValues docValues, FixedBitSet groupBits, IntOpenHashSet collapsedSet, int limit, Sort sort) throws IOException {
int numGroups = collapsedSet.size();
groups = new IntObjectOpenHashMap<>(numGroups * 2);
DocIdSetIterator iterator = groupBits.iterator();
DocIdSetIterator iterator = new FixedBitSetIterator(groupBits, 0); // cost is not useful here
int group;
while ((group = iterator.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
Collector collector = (sort == null) ? TopScoreDocCollector.create(limit, true) : TopFieldCollector.create(sort, limit, false, false, false, true);

27
solr/core/src/java/org/apache/solr/response/SortingResponseWriter.java
View File

@ -17,11 +17,16 @@
package org.apache.solr.response;
import org.apache.lucene.index.LeafReaderContext;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.Writer;
import java.util.List;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.MultiDocValues;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Sort;
@ -29,31 +34,27 @@ import org.apache.lucene.search.SortField;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRefBuilder;
import org.apache.lucene.util.LongValues;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.FixedBitSet.FixedBitSetIterator;
import org.apache.lucene.util.LongValues;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestInfo;
import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.TrieFloatField;
import org.apache.solr.schema.StrField;
import org.apache.solr.schema.TrieDoubleField;
import org.apache.solr.schema.TrieFloatField;
import org.apache.solr.schema.TrieIntField;
import org.apache.solr.schema.TrieLongField;
import org.apache.solr.schema.StrField;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.search.SortSpec;
import org.apache.solr.search.SyntaxError;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.Writer;
import java.io.PrintWriter;
import java.util.List;
public class SortingResponseWriter implements QueryResponseWriter {
@ -127,8 +128,6 @@ public class SortingResponseWriter implements QueryResponseWriter {
SortQueue queue = new SortQueue(queueSize, sortDoc);
SortDoc[] outDocs = new SortDoc[queueSize];
long total = 0;
while(count < totalHits) {
//long begin = System.nanoTime();
boolean commaNeeded = false;
@ -136,7 +135,7 @@ public class SortingResponseWriter implements QueryResponseWriter {
SortDoc top = queue.top();
for(int i=0; i<leaves.size(); i++) {
sortDoc.setNextReader(leaves.get(i));
DocIdSetIterator it = sets[i].iterator();
DocIdSetIterator it = new FixedBitSetIterator(sets[i], 0); // cost is not useful here
int docId = -1;
while((docId = it.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
sortDoc.setValues(docId);

5
solr/core/src/java/org/apache/solr/search/BitDocSet.java
View File

@ -26,6 +26,7 @@ import org.apache.lucene.search.Filter;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.FixedBitSet.FixedBitSetIterator;
import org.apache.lucene.util.FixedBitDocIdSet;
/**
* <code>BitDocSet</code> represents an unordered set of Lucene Document Ids
@ -91,7 +92,7 @@ public class BitDocSet extends DocSetBase {
@Override
public DocIterator iterator() {
return new DocIterator() {
private final FixedBitSetIterator iter = new FixedBitSetIterator(bits);
private final FixedBitSetIterator iter = new FixedBitSetIterator(bits, 0L); // cost is not useful here
private int pos = iter.nextDoc();
@Override
public boolean hasNext() {
@ -276,7 +277,7 @@ public class BitDocSet extends DocSetBase {
final Bits acceptDocs2 = acceptDocs == null ? null : (reader.getLiveDocs() == acceptDocs ? null : acceptDocs);
if (context.isTopLevel) {
return BitsFilteredDocIdSet.wrap(bs, acceptDocs);
return BitsFilteredDocIdSet.wrap(new FixedBitDocIdSet(bs), acceptDocs);
}
final int base = context.docBase;

9
solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
View File

@ -22,8 +22,8 @@ import java.util.Arrays;
import java.util.Iterator;
import java.util.Map;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.queries.function.FunctionQuery;
@ -35,6 +35,7 @@ import org.apache.lucene.search.Query;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.FixedBitSet.FixedBitSetIterator;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
@ -49,8 +50,8 @@ import org.apache.solr.schema.TrieIntField;
import org.apache.solr.schema.TrieLongField;
import com.carrotsearch.hppc.FloatArrayList;
import com.carrotsearch.hppc.IntOpenHashSet;
import com.carrotsearch.hppc.IntIntOpenHashMap;
import com.carrotsearch.hppc.IntOpenHashSet;
import com.carrotsearch.hppc.cursors.IntIntCursor;
/**
@ -490,7 +491,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
leafDelegate = delegate.getLeafCollector(contexts[currentContext]);
DummyScorer dummy = new DummyScorer();
leafDelegate.setScorer(dummy);
DocIdSetIterator it = collapsedSet.iterator();
DocIdSetIterator it = new FixedBitSetIterator(collapsedSet, 0L); // cost is not useful here
int docId = -1;
int nullScoreIndex = 0;
while((docId = it.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
@ -603,7 +604,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
leafDelegate = delegate.getLeafCollector(contexts[currentContext]);
DummyScorer dummy = new DummyScorer();
leafDelegate.setScorer(dummy);
DocIdSetIterator it = fieldValueCollapse.getCollapsedSet().iterator();
DocIdSetIterator it = new FixedBitSetIterator(fieldValueCollapse.getCollapsedSet(), 0); // cost is not useful here
int docId = -1;
int nullScoreIndex = 0;
float[] scores = fieldValueCollapse.getScores();

3
solr/core/src/java/org/apache/solr/search/DocSetBase.java
View File

@ -24,6 +24,7 @@ import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Filter;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
import org.apache.solr.common.SolrException;
@ -171,7 +172,7 @@ abstract class DocSetBase implements DocSet {
final Bits acceptDocs2 = acceptDocs == null ? null : (reader.getLiveDocs() == acceptDocs ? null : acceptDocs);
if (context.isTopLevel) {
return BitsFilteredDocIdSet.wrap(bs, acceptDocs);
return BitsFilteredDocIdSet.wrap(new FixedBitDocIdSet(bs), acceptDocs);
}
final int base = context.docBase;

10
solr/core/src/test/org/apache/solr/search/TestDocSet.java
View File

@ -22,14 +22,14 @@ import java.util.Arrays;
import java.util.List;
import java.util.Random;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.Fields;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexReaderContext;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.MultiReader;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SortedDocValues;
@ -68,7 +68,7 @@ public class TestDocSet extends LuceneTestCase {
public DocSet getHashDocSet(FixedBitSet bs) {
int[] docs = new int[bs.cardinality()];
FixedBitSetIterator iter = new FixedBitSetIterator(bs);
FixedBitSetIterator iter = new FixedBitSetIterator(bs, 0);
for (int i=0; i<docs.length; i++) {
docs[i] = iter.nextDoc();
}
@ -77,7 +77,7 @@ public class TestDocSet extends LuceneTestCase {
public DocSet getIntDocSet(FixedBitSet bs) {
int[] docs = new int[bs.cardinality()];
FixedBitSetIterator iter = new FixedBitSetIterator(bs);
FixedBitSetIterator iter = new FixedBitSetIterator(bs, 0);
for (int i=0; i<docs.length; i++) {
docs[i] = iter.nextDoc();
}
@ -95,7 +95,7 @@ public class TestDocSet extends LuceneTestCase {
int offset = 3;
int end = offset + len;
FixedBitSetIterator iter = new FixedBitSetIterator(bs);
FixedBitSetIterator iter = new FixedBitSetIterator(bs, 0);
// put in opposite order... DocLists are not ordered.
for (int i=end-1; i>=offset; i--) {
arr[i] = iter.nextDoc();

9
solr/core/src/test/org/apache/solr/search/TestSort.java
View File

@ -30,18 +30,18 @@ import org.apache.lucene.analysis.core.SimpleAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.StringField;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.search.LeafCollector;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.BitsFilteredDocIdSet;
import org.apache.lucene.search.Collector;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.Filter;
import org.apache.lucene.search.FilterLeafCollector;
import org.apache.lucene.search.FilterCollector;
import org.apache.lucene.search.FilterLeafCollector;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.LeafCollector;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.Sort;
@ -53,6 +53,7 @@ import org.apache.lucene.store.Directory;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.uninverting.UninvertingReader;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.TestUtil;
import org.apache.solr.SolrTestCaseJ4;
@ -339,7 +340,7 @@ public class TestSort extends SolrTestCaseJ4 {
for (int i=0; i<n; i++) {
obs.set(r.nextInt(sz));
}
return obs;
return new FixedBitDocIdSet(obs);
}

2
solr/core/src/test/org/apache/solr/util/BitSetPerf.java
View File

@ -172,7 +172,7 @@ public class BitSetPerf {
for (int i=0; i<numSets; i++) {
if (impl=="open") {
final FixedBitSet set = osets[i];
final FixedBitSetIterator iterator = new FixedBitSetIterator(set);
final FixedBitSetIterator iterator = new FixedBitSetIterator(set, 0);
for(int next=iterator.nextDoc(); next>=0; next=iterator.nextDoc()) {
ret += next;
}