Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-6010: Remove dead code. 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1632305 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Adrien Grand 2014-10-16 13:41:08 +00:00
parent 6a790d5b25
commit e1b7594cd0
14 changed files with 8 additions and 1951 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java
View File

@ -46,7 +46,6 @@ import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.Bits; import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder; import org.apache.lucene.util.BytesRefBuilder;
import org.apache.lucene.util.DoubleBarrelLRUCache;
import org.apache.lucene.util.RamUsageEstimator; import org.apache.lucene.util.RamUsageEstimator;
/** Handles a terms dict, but decouples all details of /** Handles a terms dict, but decouples all details of
@ -76,7 +75,7 @@ public class BlockTermsReader extends FieldsProducer {
private TermsIndexReaderBase indexReader; private TermsIndexReaderBase indexReader;
// Used as key for the terms cache // Used as key for the terms cache
private static class FieldAndTerm extends DoubleBarrelLRUCache.CloneableKey { private static class FieldAndTerm implements Cloneable {
String field; String field;
BytesRef term; BytesRef term;

23
lucene/core/src/java/org/apache/lucene/index/FilterDirectoryReader.java
View File

@ -61,32 +61,9 @@ public abstract class FilterDirectoryReader extends DirectoryReader {
} }
/**
* A no-op SubReaderWrapper that simply returns the parent
* DirectoryReader's original subreaders.
*/
public static class StandardReaderWrapper extends SubReaderWrapper {
/** Constructor */
public StandardReaderWrapper() {}
@Override
public LeafReader wrap(LeafReader reader) {
return reader;
}
}
/** The filtered DirectoryReader */ /** The filtered DirectoryReader */
protected final DirectoryReader in; protected final DirectoryReader in;
/**
* Create a new FilterDirectoryReader that filters a passed in DirectoryReader.
* @param in the DirectoryReader to filter
*/
public FilterDirectoryReader(DirectoryReader in) {
this(in, new StandardReaderWrapper());
}
/** /**
* Create a new FilterDirectoryReader that filters a passed in DirectoryReader, * Create a new FilterDirectoryReader that filters a passed in DirectoryReader,
* using the supplied SubReaderWrapper to wrap its subreader. * using the supplied SubReaderWrapper to wrap its subreader.

5
lucene/core/src/java/org/apache/lucene/search/DocValuesDocIdSet.java
View File

@ -20,7 +20,6 @@ import java.io.IOException;
import org.apache.lucene.util.Bits; import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitSet; import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.OpenBitSet;
/** /**
* Base class for DocIdSet to be used with DocValues. The implementation * Base class for DocIdSet to be used with DocValues. The implementation
@ -114,8 +113,8 @@ public abstract class DocValuesDocIdSet extends DocIdSet {
return maxDoc; return maxDoc;
} }
}; };
} else if (acceptDocs instanceof FixedBitSet || acceptDocs instanceof OpenBitSet) { } else if (acceptDocs instanceof FixedBitSet) {
// special case for FixedBitSet / OpenBitSet: use the iterator and filter it // special case for FixedBitSet: use the iterator and filter it
// (used e.g. when Filters are chained by FilteredQuery) // (used e.g. when Filters are chained by FilteredQuery)
return new FilteredDocIdSetIterator(((DocIdSet) acceptDocs).iterator()) { return new FilteredDocIdSetIterator(((DocIdSet) acceptDocs).iterator()) {
@Override @Override

122
lucene/core/src/java/org/apache/lucene/util/DoubleBarrelLRUCache.java
View File

@ -1,122 +0,0 @@
package org.apache.lucene.util;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.Map;
/**
* Simple concurrent LRU cache, using a "double barrel"
* approach where two ConcurrentHashMaps record entries.
*
* <p>At any given time, one hash is primary and the other
* is secondary. {@link #get} first checks primary, and if
* that's a miss, checks secondary. If secondary has the
* entry, it's promoted to primary (<b>NOTE</b>: the key is
* cloned at this point). Once primary is full, the
* secondary is cleared and the two are swapped.</p>
*
* <p>This is not as space efficient as other possible
* concurrent approaches (see LUCENE-2075): to achieve
* perfect LRU(N) it requires 2*N storage. But, this
* approach is relatively simple and seems in practice to
* not grow unbounded in size when under hideously high
* load.</p>
*
* @lucene.internal
*/
final public class DoubleBarrelLRUCache<K extends DoubleBarrelLRUCache.CloneableKey,V> {
/** Object providing clone(); the key class must subclass this. */
public static abstract class CloneableKey {
@Override
abstract public CloneableKey clone();
}
private final Map<K,V> cache1;
private final Map<K,V> cache2;
private final AtomicInteger countdown;
private volatile boolean swapped;
private final int maxSize;
public DoubleBarrelLRUCache(int maxSize) {
this.maxSize = maxSize;
countdown = new AtomicInteger(maxSize);
cache1 = new ConcurrentHashMap<>();
cache2 = new ConcurrentHashMap<>();
}
@SuppressWarnings("unchecked")
public V get(K key) {
final Map<K,V> primary;
final Map<K,V> secondary;
if (swapped) {
primary = cache2;
secondary = cache1;
} else {
primary = cache1;
secondary = cache2;
}
// Try primary first
V result = primary.get(key);
if (result == null) {
// Not found -- try secondary
result = secondary.get(key);
if (result != null) {
// Promote to primary
put((K) key.clone(), result);
}
}
return result;
}
public void put(K key, V value) {
final Map<K,V> primary;
final Map<K,V> secondary;
if (swapped) {
primary = cache2;
secondary = cache1;
} else {
primary = cache1;
secondary = cache2;
}
primary.put(key, value);
if (countdown.decrementAndGet() == 0) {
// Time to swap
// NOTE: there is saturation risk here, that the
// thread that's doing the clear() takes too long to
// do so, while other threads continue to add to
// primary, but in practice this seems not to be an
// issue (see LUCENE-2075 for benchmark & details)
// First, clear secondary
secondary.clear();
// Second, swap
swapped = !swapped;
// Third, reset countdown
countdown.set(maxSize);
}
}
}

24
lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java
View File

@ -325,13 +325,7 @@ public final class FixedBitSet extends DocIdSet implements MutableBits {
/** Does in-place OR of the bits provided by the /** Does in-place OR of the bits provided by the
* iterator. */ * iterator. */
public void or(DocIdSetIterator iter) throws IOException { public void or(DocIdSetIterator iter) throws IOException {
if (iter instanceof OpenBitSetIterator && iter.docID() == -1) { if (iter instanceof FixedBitSetIterator && iter.docID() == -1) {
final OpenBitSetIterator obs = (OpenBitSetIterator) iter;
or(obs.arr, obs.words);
// advance after last doc that would be accepted if standard
// iteration is used (to exhaust it):
obs.advance(numBits);
} else if (iter instanceof FixedBitSetIterator && iter.docID() == -1) {
final FixedBitSetIterator fbs = (FixedBitSetIterator) iter; final FixedBitSetIterator fbs = (FixedBitSetIterator) iter;
or(fbs.bits, fbs.numWords); or(fbs.bits, fbs.numWords);
// advance after last doc that would be accepted if standard // advance after last doc that would be accepted if standard
@ -381,13 +375,7 @@ public final class FixedBitSet extends DocIdSet implements MutableBits {
/** Does in-place AND of the bits provided by the /** Does in-place AND of the bits provided by the
* iterator. */ * iterator. */
public void and(DocIdSetIterator iter) throws IOException { public void and(DocIdSetIterator iter) throws IOException {
if (iter instanceof OpenBitSetIterator && iter.docID() == -1) { if (iter instanceof FixedBitSetIterator && iter.docID() == -1) {
final OpenBitSetIterator obs = (OpenBitSetIterator) iter;
and(obs.arr, obs.words);
// advance after last doc that would be accepted if standard
// iteration is used (to exhaust it):
obs.advance(numBits);
} else if (iter instanceof FixedBitSetIterator && iter.docID() == -1) {
final FixedBitSetIterator fbs = (FixedBitSetIterator) iter; final FixedBitSetIterator fbs = (FixedBitSetIterator) iter;
and(fbs.bits, fbs.numWords); and(fbs.bits, fbs.numWords);
// advance after last doc that would be accepted if standard // advance after last doc that would be accepted if standard
@ -435,13 +423,7 @@ public final class FixedBitSet extends DocIdSet implements MutableBits {
/** Does in-place AND NOT of the bits provided by the /** Does in-place AND NOT of the bits provided by the
* iterator. */ * iterator. */
public void andNot(DocIdSetIterator iter) throws IOException { public void andNot(DocIdSetIterator iter) throws IOException {
if (iter instanceof OpenBitSetIterator && iter.docID() == -1) { if (iter instanceof FixedBitSetIterator && iter.docID() == -1) {
final OpenBitSetIterator obs = (OpenBitSetIterator) iter;
andNot(obs.arr, obs.words);
// advance after last doc that would be accepted if standard
// iteration is used (to exhaust it):
obs.advance(numBits);
} else if (iter instanceof FixedBitSetIterator && iter.docID() == -1) {
final FixedBitSetIterator fbs = (FixedBitSetIterator) iter; final FixedBitSetIterator fbs = (FixedBitSetIterator) iter;
andNot(fbs.bits, fbs.numWords); andNot(fbs.bits, fbs.numWords);
// advance after last doc that would be accepted if standard // advance after last doc that would be accepted if standard

890
lucene/core/src/java/org/apache/lucene/util/OpenBitSet.java
View File

@ -1,890 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.util;
import java.util.Arrays;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
/** An "open" BitSet implementation that allows direct access to the array of words
* storing the bits.
* <p/>
* Unlike java.util.bitset, the fact that bits are packed into an array of longs
* is part of the interface. This allows efficient implementation of other algorithms
* by someone other than the author. It also allows one to efficiently implement
* alternate serialization or interchange formats.
* <p/>
* <code>OpenBitSet</code> is faster than <code>java.util.BitSet</code> in most operations
* and *much* faster at calculating cardinality of sets and results of set operations.
* It can also handle sets of larger cardinality (up to 64 * 2**32-1)
* <p/>
* The goals of <code>OpenBitSet</code> are the fastest implementation possible, and
* maximum code reuse. Extra safety and encapsulation
* may always be built on top, but if that's built in, the cost can never be removed (and
* hence people re-implement their own version in order to get better performance).
* If you want a "safe", totally encapsulated (and slower and limited) BitSet
* class, use <code>java.util.BitSet</code>.
* <p/>
* <h3>Performance Results</h3>
*
Test system: Pentium 4, Sun Java 1.5_06 -server -Xbatch -Xmx64M
<br/>BitSet size = 1,000,000
<br/>Results are java.util.BitSet time divided by OpenBitSet time.
<table border="1">
<tr>
<th></th> <th>cardinality</th> <th>intersect_count</th> <th>union</th> <th>nextSetBit</th> <th>get</th> <th>iterator</th>
</tr>
<tr>
<th>50% full</th> <td>3.36</td> <td>3.96</td> <td>1.44</td> <td>1.46</td> <td>1.99</td> <td>1.58</td>
</tr>
<tr>
<th>1% full</th> <td>3.31</td> <td>3.90</td> <td>&nbsp;</td> <td>1.04</td> <td>&nbsp;</td> <td>0.99</td>
</tr>
</table>
<br/>
Test system: AMD Opteron, 64 bit linux, Sun Java 1.5_06 -server -Xbatch -Xmx64M
<br/>BitSet size = 1,000,000
<br/>Results are java.util.BitSet time divided by OpenBitSet time.
<table border="1">
<tr>
<th></th> <th>cardinality</th> <th>intersect_count</th> <th>union</th> <th>nextSetBit</th> <th>get</th> <th>iterator</th>
</tr>
<tr>
<th>50% full</th> <td>2.50</td> <td>3.50</td> <td>1.00</td> <td>1.03</td> <td>1.12</td> <td>1.25</td>
</tr>
<tr>
<th>1% full</th> <td>2.51</td> <td>3.49</td> <td>&nbsp;</td> <td>1.00</td> <td>&nbsp;</td> <td>1.02</td>
</tr>
</table>
*/
public class OpenBitSet extends DocIdSet implements Bits, Cloneable {
private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(OpenBitSet.class);
protected long[] bits;
protected int wlen; // number of words (elements) used in the array
// Used only for assert:
private long numBits;
/** Constructs an OpenBitSet large enough to hold {@code numBits}. */
public OpenBitSet(long numBits) {
this.numBits = numBits;
bits = new long[bits2words(numBits)];
wlen = bits.length;
}
/** Constructor: allocates enough space for 64 bits. */
public OpenBitSet() {
this(64);
}
/**
* Constructs an OpenBitSet from an existing long[].
* <p>
* The first 64 bits are in long[0], with bit index 0 at the least significant
* bit, and bit index 63 at the most significant. Given a bit index, the word
* containing it is long[index/64], and it is at bit number index%64 within
* that word.
* <p>
* numWords are the number of elements in the array that contain set bits
* (non-zero longs). numWords should be &lt= bits.length, and any existing
* words in the array at position &gt= numWords should be zero.
*
*/
public OpenBitSet(long[] bits, int numWords) {
if (numWords > bits.length) {
throw new IllegalArgumentException("numWords cannot exceed bits.length");
}
this.bits = bits;
this.wlen = numWords;
this.numBits = wlen * 64;
}
@Override
public DocIdSetIterator iterator() {
return new OpenBitSetIterator(bits, wlen);
}
@Override
public Bits bits() {
return this;
}
/** This DocIdSet implementation is cacheable. */
@Override
public boolean isCacheable() {
return true;
}
@Override
public long ramBytesUsed() {
return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(bits);
}
/** Returns the current capacity in bits (1 greater than the index of the last bit) */
public long capacity() { return bits.length << 6; }
/**
* Returns the current capacity of this set. Included for
* compatibility. This is *not* equal to {@link #cardinality}
*/
public long size() {
return capacity();
}
@Override
public int length() {
return bits.length << 6;
}
/** Returns true if there are no set bits */
public boolean isEmpty() { return cardinality()==0; }
/** Expert: returns the long[] storing the bits */
public long[] getBits() { return bits; }
/** Expert: gets the number of longs in the array that are in use */
public int getNumWords() { return wlen; }
/** Returns true or false for the specified bit index. */
@Override
public boolean get(int index) {
int i = index >> 6; // div 64
// signed shift will keep a negative index and force an
// array-index-out-of-bounds-exception, removing the need for an explicit check.
if (i>=bits.length) return false;
long bitmask = 1L << index;
return (bits[i] & bitmask) != 0;
}
/** Returns true or false for the specified bit index.
* The index should be less than the OpenBitSet size
*/
public boolean fastGet(int index) {
assert index >= 0 && index < numBits;
int i = index >> 6; // div 64
// signed shift will keep a negative index and force an
// array-index-out-of-bounds-exception, removing the need for an explicit check.
long bitmask = 1L << index;
return (bits[i] & bitmask) != 0;
}
/** Returns true or false for the specified bit index
*/
public boolean get(long index) {
int i = (int)(index >> 6); // div 64
if (i>=bits.length) return false;
long bitmask = 1L << index;
return (bits[i] & bitmask) != 0;
}
/** Returns true or false for the specified bit index.
* The index should be less than the OpenBitSet size.
*/
public boolean fastGet(long index) {
assert index >= 0 && index < numBits;
int i = (int)(index >> 6); // div 64
long bitmask = 1L << index;
return (bits[i] & bitmask) != 0;
}
/*
// alternate implementation of get()
public boolean get1(int index) {
int i = index >> 6; // div 64
int bit = index & 0x3f; // mod 64
return ((bits[i]>>>bit) & 0x01) != 0;
// this does a long shift and a bittest (on x86) vs
// a long shift, and a long AND, (the test for zero is prob a no-op)
// testing on a P4 indicates this is slower than (bits[i] & bitmask) != 0;
}
*/
/** returns 1 if the bit is set, 0 if not.
* The index should be less than the OpenBitSet size
*/
public int getBit(int index) {
assert index >= 0 && index < numBits;
int i = index >> 6; // div 64
return ((int)(bits[i]>>>index)) & 0x01;
}
/*
public boolean get2(int index) {
int word = index >> 6; // div 64
int bit = index & 0x0000003f; // mod 64
return (bits[word] << bit) < 0; // hmmm, this would work if bit order were reversed
// we could right shift and check for parity bit, if it was available to us.
}
*/
/** sets a bit, expanding the set size if necessary */
public void set(long index) {
int wordNum = expandingWordNum(index);
long bitmask = 1L << index;
bits[wordNum] |= bitmask;
}
/** Sets the bit at the specified index.
* The index should be less than the OpenBitSet size.
*/
public void fastSet(int index) {
assert index >= 0 && index < numBits;
int wordNum = index >> 6; // div 64
long bitmask = 1L << index;
bits[wordNum] |= bitmask;
}
/** Sets the bit at the specified index.
* The index should be less than the OpenBitSet size.
*/
public void fastSet(long index) {
assert index >= 0 && index < numBits;
int wordNum = (int)(index >> 6);
long bitmask = 1L << index;
bits[wordNum] |= bitmask;
}
/** Sets a range of bits, expanding the set size if necessary
*
* @param startIndex lower index
* @param endIndex one-past the last bit to set
*/
public void set(long startIndex, long endIndex) {
if (endIndex <= startIndex) return;
int startWord = (int)(startIndex>>6);
// since endIndex is one past the end, this is index of the last
// word to be changed.
int endWord = expandingWordNum(endIndex-1);
long startmask = -1L << startIndex;
long endmask = -1L >>> -endIndex; // 64-(endIndex&0x3f) is the same as -endIndex due to wrap
if (startWord == endWord) {
bits[startWord] |= (startmask & endmask);
return;
}
bits[startWord] |= startmask;
Arrays.fill(bits, startWord+1, endWord, -1L);
bits[endWord] |= endmask;
}
protected int expandingWordNum(long index) {
int wordNum = (int)(index >> 6);
if (wordNum >= wlen) {
ensureCapacity(index + 1);
}
return wordNum;
}
/** clears a bit.
* The index should be less than the OpenBitSet size.
*/
public void fastClear(int index) {
assert index >= 0 && index < numBits;
int wordNum = index >> 6;
long bitmask = 1L << index;
bits[wordNum] &= ~bitmask;
// hmmm, it takes one more instruction to clear than it does to set... any
// way to work around this? If there were only 63 bits per word, we could
// use a right shift of 10111111...111 in binary to position the 0 in the
// correct place (using sign extension).
// Could also use Long.rotateRight() or rotateLeft() *if* they were converted
// by the JVM into a native instruction.
// bits[word] &= Long.rotateLeft(0xfffffffe,bit);
}
/** clears a bit.
* The index should be less than the OpenBitSet size.
*/
public void fastClear(long index) {
assert index >= 0 && index < numBits;
int wordNum = (int)(index >> 6); // div 64
long bitmask = 1L << index;
bits[wordNum] &= ~bitmask;
}
/** clears a bit, allowing access beyond the current set size without changing the size.*/
public void clear(long index) {
int wordNum = (int)(index >> 6); // div 64
if (wordNum>=wlen) return;
long bitmask = 1L << index;
bits[wordNum] &= ~bitmask;
}
/** Clears a range of bits. Clearing past the end does not change the size of the set.
*
* @param startIndex lower index
* @param endIndex one-past the last bit to clear
*/
public void clear(int startIndex, int endIndex) {
if (endIndex <= startIndex) return;
int startWord = (startIndex>>6);
if (startWord >= wlen) return;
// since endIndex is one past the end, this is index of the last
// word to be changed.
int endWord = ((endIndex-1)>>6);
long startmask = -1L << startIndex;
long endmask = -1L >>> -endIndex; // 64-(endIndex&0x3f) is the same as -endIndex due to wrap
// invert masks since we are clearing
startmask = ~startmask;
endmask = ~endmask;
if (startWord == endWord) {
bits[startWord] &= (startmask | endmask);
return;
}
bits[startWord] &= startmask;
int middle = Math.min(wlen, endWord);
Arrays.fill(bits, startWord+1, middle, 0L);
if (endWord < wlen) {
bits[endWord] &= endmask;
}
}
/** Clears a range of bits. Clearing past the end does not change the size of the set.
*
* @param startIndex lower index
* @param endIndex one-past the last bit to clear
*/
public void clear(long startIndex, long endIndex) {
if (endIndex <= startIndex) return;
int startWord = (int)(startIndex>>6);
if (startWord >= wlen) return;
// since endIndex is one past the end, this is index of the last
// word to be changed.
int endWord = (int)((endIndex-1)>>6);
long startmask = -1L << startIndex;
long endmask = -1L >>> -endIndex; // 64-(endIndex&0x3f) is the same as -endIndex due to wrap
// invert masks since we are clearing
startmask = ~startmask;
endmask = ~endmask;
if (startWord == endWord) {
bits[startWord] &= (startmask | endmask);
return;
}
bits[startWord] &= startmask;
int middle = Math.min(wlen, endWord);
Arrays.fill(bits, startWord+1, middle, 0L);
if (endWord < wlen) {
bits[endWord] &= endmask;
}
}
/** Sets a bit and returns the previous value.
* The index should be less than the OpenBitSet size.
*/
public boolean getAndSet(int index) {
assert index >= 0 && index < numBits;
int wordNum = index >> 6; // div 64
long bitmask = 1L << index;
boolean val = (bits[wordNum] & bitmask) != 0;
bits[wordNum] |= bitmask;
return val;
}
/** Sets a bit and returns the previous value.
* The index should be less than the OpenBitSet size.
*/
public boolean getAndSet(long index) {
assert index >= 0 && index < numBits;
int wordNum = (int)(index >> 6); // div 64
long bitmask = 1L << index;
boolean val = (bits[wordNum] & bitmask) != 0;
bits[wordNum] |= bitmask;
return val;
}
/** flips a bit.
* The index should be less than the OpenBitSet size.
*/
public void fastFlip(int index) {
assert index >= 0 && index < numBits;
int wordNum = index >> 6; // div 64
long bitmask = 1L << index;
bits[wordNum] ^= bitmask;
}
/** flips a bit.
* The index should be less than the OpenBitSet size.
*/
public void fastFlip(long index) {
assert index >= 0 && index < numBits;
int wordNum = (int)(index >> 6); // div 64
long bitmask = 1L << index;
bits[wordNum] ^= bitmask;
}
/** flips a bit, expanding the set size if necessary */
public void flip(long index) {
int wordNum = expandingWordNum(index);
long bitmask = 1L << index;
bits[wordNum] ^= bitmask;
}
/** flips a bit and returns the resulting bit value.
* The index should be less than the OpenBitSet size.
*/
public boolean flipAndGet(int index) {
assert index >= 0 && index < numBits;
int wordNum = index >> 6; // div 64
long bitmask = 1L << index;
bits[wordNum] ^= bitmask;
return (bits[wordNum] & bitmask) != 0;
}
/** flips a bit and returns the resulting bit value.
* The index should be less than the OpenBitSet size.
*/
public boolean flipAndGet(long index) {
assert index >= 0 && index < numBits;
int wordNum = (int)(index >> 6); // div 64
long bitmask = 1L << index;
bits[wordNum] ^= bitmask;
return (bits[wordNum] & bitmask) != 0;
}
/** Flips a range of bits, expanding the set size if necessary
*
* @param startIndex lower index
* @param endIndex one-past the last bit to flip
*/
public void flip(long startIndex, long endIndex) {
if (endIndex <= startIndex) return;
int startWord = (int)(startIndex>>6);
// since endIndex is one past the end, this is index of the last
// word to be changed.
int endWord = expandingWordNum(endIndex-1);
/*** Grrr, java shifting wraps around so -1L>>>64 == -1
* for that reason, make sure not to use endmask if the bits to flip will
* be zero in the last word (redefine endWord to be the last changed...)
long startmask = -1L << (startIndex & 0x3f); // example: 11111...111000
long endmask = -1L >>> (64-(endIndex & 0x3f)); // example: 00111...111111
***/
long startmask = -1L << startIndex;
long endmask = -1L >>> -endIndex; // 64-(endIndex&0x3f) is the same as -endIndex due to wrap
if (startWord == endWord) {
bits[startWord] ^= (startmask & endmask);
return;
}
bits[startWord] ^= startmask;
for (int i=startWord+1; i<endWord; i++) {
bits[i] = ~bits[i];
}
bits[endWord] ^= endmask;
}
/*
public static int pop(long v0, long v1, long v2, long v3) {
// derived from pop_array by setting last four elems to 0.
// exchanges one pop() call for 10 elementary operations
// saving about 7 instructions... is there a better way?
long twosA=v0 & v1;
long ones=v0^v1;
long u2=ones^v2;
long twosB =(ones&v2)|(u2&v3);
ones=u2^v3;
long fours=(twosA&twosB);
long twos=twosA^twosB;
return (pop(fours)<<2)
+ (pop(twos)<<1)
+ pop(ones);
}
*/
/** @return the number of set bits */
public long cardinality() {
return BitUtil.pop_array(bits,0,wlen);
}
/** Returns the popcount or cardinality of the intersection of the two sets.
* Neither set is modified.
*/
public static long intersectionCount(OpenBitSet a, OpenBitSet b) {
return BitUtil.pop_intersect(a.bits, b.bits, 0, Math.min(a.wlen, b.wlen));
}
/** Returns the popcount or cardinality of the union of the two sets.
* Neither set is modified.
*/
public static long unionCount(OpenBitSet a, OpenBitSet b) {
long tot = BitUtil.pop_union(a.bits, b.bits, 0, Math.min(a.wlen, b.wlen));
if (a.wlen < b.wlen) {
tot += BitUtil.pop_array(b.bits, a.wlen, b.wlen-a.wlen);
} else if (a.wlen > b.wlen) {
tot += BitUtil.pop_array(a.bits, b.wlen, a.wlen-b.wlen);
}
return tot;
}
/** Returns the popcount or cardinality of "a and not b"
* or "intersection(a, not(b))".
* Neither set is modified.
*/
public static long andNotCount(OpenBitSet a, OpenBitSet b) {
long tot = BitUtil.pop_andnot(a.bits, b.bits, 0, Math.min(a.wlen, b.wlen));
if (a.wlen > b.wlen) {
tot += BitUtil.pop_array(a.bits, b.wlen, a.wlen-b.wlen);
}
return tot;
}
/** Returns the popcount or cardinality of the exclusive-or of the two sets.
* Neither set is modified.
*/
public static long xorCount(OpenBitSet a, OpenBitSet b) {
long tot = BitUtil.pop_xor(a.bits, b.bits, 0, Math.min(a.wlen, b.wlen));
if (a.wlen < b.wlen) {
tot += BitUtil.pop_array(b.bits, a.wlen, b.wlen-a.wlen);
} else if (a.wlen > b.wlen) {
tot += BitUtil.pop_array(a.bits, b.wlen, a.wlen-b.wlen);
}
return tot;
}
/** Returns the index of the first set bit starting at the index specified.
* -1 is returned if there are no more set bits.
*/
public int nextSetBit(int index) {
int i = index>>6;
if (i>=wlen) return -1;
int subIndex = index & 0x3f; // index within the word
long word = bits[i] >> subIndex; // skip all the bits to the right of index
if (word!=0) {
return (i<<6) + subIndex + Long.numberOfTrailingZeros(word);
}
while(++i < wlen) {
word = bits[i];
if (word!=0) return (i<<6) + Long.numberOfTrailingZeros(word);
}
return -1;
}
/** Returns the index of the first set bit starting at the index specified.
* -1 is returned if there are no more set bits.
*/
public long nextSetBit(long index) {
int i = (int)(index>>>6);
if (i>=wlen) return -1;
int subIndex = (int)index & 0x3f; // index within the word
long word = bits[i] >>> subIndex; // skip all the bits to the right of index
if (word!=0) {
return (((long)i)<<6) + (subIndex + Long.numberOfTrailingZeros(word));
}
while(++i < wlen) {
word = bits[i];
if (word!=0) return (((long)i)<<6) + Long.numberOfTrailingZeros(word);
}
return -1;
}
/** Returns the index of the first set bit starting downwards at
* the index specified.
* -1 is returned if there are no more set bits.
*/
public int prevSetBit(int index) {
int i = index >> 6;
final int subIndex;
long word;
if (i >= wlen) {
i = wlen - 1;
if (i < 0) return -1;
subIndex = 63; // last possible bit
word = bits[i];
} else {
if (i < 0) return -1;
subIndex = index & 0x3f; // index within the word
word = (bits[i] << (63-subIndex)); // skip all the bits to the left of index
}
if (word != 0) {
return (i << 6) + subIndex - Long.numberOfLeadingZeros(word); // See LUCENE-3197
}
while (--i >= 0) {
word = bits[i];
if (word !=0 ) {
return (i << 6) + 63 - Long.numberOfLeadingZeros(word);
}
}
return -1;
}
/** Returns the index of the first set bit starting downwards at
* the index specified.
* -1 is returned if there are no more set bits.
*/
public long prevSetBit(long index) {
int i = (int) (index >> 6);
final int subIndex;
long word;
if (i >= wlen) {
i = wlen - 1;
if (i < 0) return -1;
subIndex = 63; // last possible bit
word = bits[i];
} else {
if (i < 0) return -1;
subIndex = (int)index & 0x3f; // index within the word
word = (bits[i] << (63-subIndex)); // skip all the bits to the left of index
}
if (word != 0) {
return (((long)i)<<6) + subIndex - Long.numberOfLeadingZeros(word); // See LUCENE-3197
}
while (--i >= 0) {
word = bits[i];
if (word !=0 ) {
return (((long)i)<<6) + 63 - Long.numberOfLeadingZeros(word);
}
}
return -1;
}
@Override
public OpenBitSet clone() {
try {
OpenBitSet obs = (OpenBitSet)super.clone();
obs.bits = obs.bits.clone(); // hopefully an array clone is as fast(er) than arraycopy
return obs;
} catch (CloneNotSupportedException e) {
throw new RuntimeException(e);
}
}
/** this = this AND other */
public void intersect(OpenBitSet other) {
int newLen= Math.min(this.wlen,other.wlen);
long[] thisArr = this.bits;
long[] otherArr = other.bits;
// testing against zero can be more efficient
int pos=newLen;
while(--pos>=0) {
thisArr[pos] &= otherArr[pos];
}
if (this.wlen > newLen) {
// fill zeros from the new shorter length to the old length
Arrays.fill(bits,newLen,this.wlen,0);
}
this.wlen = newLen;
}
/** this = this OR other */
public void union(OpenBitSet other) {
int newLen = Math.max(wlen,other.wlen);
ensureCapacityWords(newLen);
assert (numBits = Math.max(other.numBits, numBits)) >= 0;
long[] thisArr = this.bits;
long[] otherArr = other.bits;
int pos=Math.min(wlen,other.wlen);
while(--pos>=0) {
thisArr[pos] |= otherArr[pos];
}
if (this.wlen < newLen) {
System.arraycopy(otherArr, this.wlen, thisArr, this.wlen, newLen-this.wlen);
}
this.wlen = newLen;
}
/** Remove all elements set in other. this = this AND_NOT other */
public void remove(OpenBitSet other) {
int idx = Math.min(wlen,other.wlen);
long[] thisArr = this.bits;
long[] otherArr = other.bits;
while(--idx>=0) {
thisArr[idx] &= ~otherArr[idx];
}
}
/** this = this XOR other */
public void xor(OpenBitSet other) {
int newLen = Math.max(wlen,other.wlen);
ensureCapacityWords(newLen);
assert (numBits = Math.max(other.numBits, numBits)) >= 0;
long[] thisArr = this.bits;
long[] otherArr = other.bits;
int pos=Math.min(wlen,other.wlen);
while(--pos>=0) {
thisArr[pos] ^= otherArr[pos];
}
if (this.wlen < newLen) {
System.arraycopy(otherArr, this.wlen, thisArr, this.wlen, newLen-this.wlen);
}
this.wlen = newLen;
}
// some BitSet compatability methods
//** see {@link intersect} */
public void and(OpenBitSet other) {
intersect(other);
}
//** see {@link union} */
public void or(OpenBitSet other) {
union(other);
}
//** see {@link andNot} */
public void andNot(OpenBitSet other) {
remove(other);
}
/** returns true if the sets have any elements in common */
public boolean intersects(OpenBitSet other) {
int pos = Math.min(this.wlen, other.wlen);
long[] thisArr = this.bits;
long[] otherArr = other.bits;
while (--pos>=0) {
if ((thisArr[pos] & otherArr[pos])!=0) return true;
}
return false;
}
/** Expand the long[] with the size given as a number of words (64 bit longs). */
public void ensureCapacityWords(int numWords) {
bits = ArrayUtil.grow(bits, numWords);
wlen = numWords;
assert (this.numBits = Math.max(this.numBits, numWords << 6)) >= 0;
}
/**
* Ensure that the long[] is big enough to hold numBits, expanding it if
* necessary.
*/
public void ensureCapacity(long numBits) {
ensureCapacityWords(bits2words(numBits));
// ensureCapacityWords sets numBits to a multiple of 64, but we want to set
// it to exactly what the app asked.
assert (this.numBits = Math.max(this.numBits, numBits)) >= 0;
}
/** Lowers numWords, the number of words in use,
* by checking for trailing zero words.
*/
public void trimTrailingZeros() {
int idx = wlen-1;
while (idx>=0 && bits[idx]==0) idx--;
wlen = idx+1;
}
/** returns the number of 64 bit words it would take to hold numBits */
public static int bits2words(long numBits) {
return (int)(((numBits-1)>>>6)+1);
}
/** returns true if both sets have the same bits set */
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (!(o instanceof OpenBitSet)) return false;
OpenBitSet a;
OpenBitSet b = (OpenBitSet)o;
// make a the larger set.
if (b.wlen > this.wlen) {
a = b; b=this;
} else {
a=this;
}
// check for any set bits out of the range of b
for (int i=a.wlen-1; i>=b.wlen; i--) {
if (a.bits[i]!=0) return false;
}
for (int i=b.wlen-1; i>=0; i--) {
if (a.bits[i] != b.bits[i]) return false;
}
return true;
}
@Override
public int hashCode() {
// Start with a zero hash and use a mix that results in zero if the input is zero.
// This effectively truncates trailing zeros without an explicit check.
long h = 0;
for (int i = bits.length; --i>=0;) {
h ^= bits[i];
h = (h << 1) | (h >>> 63); // rotate left
}
// fold leftmost bits into right and add a constant to prevent
// empty sets from returning 0, which is too common.
return (int)((h>>32) ^ h) + 0x98761234;
}
}

104
lucene/core/src/java/org/apache/lucene/util/OpenBitSetDISI.java
View File

@ -1,104 +0,0 @@
package org.apache.lucene.util;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import org.apache.lucene.search.DocIdSetIterator;
/** OpenBitSet with added methods to bulk-update the bits
* from a {@link DocIdSetIterator}. */
public class OpenBitSetDISI extends OpenBitSet {
/** Construct an OpenBitSetDISI with its bits set
* from the doc ids of the given DocIdSetIterator.
* Also give a maximum size one larger than the largest doc id for which a
* bit may ever be set on this OpenBitSetDISI.
*/
public OpenBitSetDISI(DocIdSetIterator disi, int maxSize) throws IOException {
super(maxSize);
inPlaceOr(disi);
}
/** Construct an OpenBitSetDISI with no bits set, and a given maximum size
* one larger than the largest doc id for which a bit may ever be set
* on this OpenBitSetDISI.
*/
public OpenBitSetDISI(int maxSize) {
super(maxSize);
}
/**
* Perform an inplace OR with the doc ids from a given DocIdSetIterator,
* setting the bit for each such doc id.
* These doc ids should be smaller than the maximum size passed to the
* constructor.
*/
public void inPlaceOr(DocIdSetIterator disi) throws IOException {
int doc;
long size = size();
while ((doc = disi.nextDoc()) < size) {
fastSet(doc);
}
}
/**
* Perform an inplace AND with the doc ids from a given DocIdSetIterator,
* leaving only the bits set for which the doc ids are in common.
* These doc ids should be smaller than the maximum size passed to the
* constructor.
*/
public void inPlaceAnd(DocIdSetIterator disi) throws IOException {
int bitSetDoc = nextSetBit(0);
int disiDoc;
while (bitSetDoc != -1 && (disiDoc = disi.advance(bitSetDoc)) != DocIdSetIterator.NO_MORE_DOCS) {
clear(bitSetDoc, disiDoc);
bitSetDoc = nextSetBit(disiDoc + 1);
}
if (bitSetDoc != -1) {
clear(bitSetDoc, size());
}
}
/**
* Perform an inplace NOT with the doc ids from a given DocIdSetIterator,
* clearing all the bits for each such doc id.
* These doc ids should be smaller than the maximum size passed to the
* constructor.
*/
public void inPlaceNot(DocIdSetIterator disi) throws IOException {
int doc;
long size = size();
while ((doc = disi.nextDoc()) < size) {
fastClear(doc);
}
}
/**
* Perform an inplace XOR with the doc ids from a given DocIdSetIterator,
* flipping all the bits for each such doc id.
* These doc ids should be smaller than the maximum size passed to the
* constructor.
*/
public void inPlaceXor(DocIdSetIterator disi) throws IOException {
int doc;
long size = size();
while ((doc = disi.nextDoc()) < size) {
fastFlip(doc);
}
}
}

148
lucene/core/src/java/org/apache/lucene/util/OpenBitSetIterator.java
View File

@ -1,148 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.util;
import org.apache.lucene.search.DocIdSetIterator;
/** An iterator to iterate over set bits in an OpenBitSet.
* This is faster than nextSetBit() for iterating over the complete set of bits,
* especially when the density of the bits set is high.
*/
public class OpenBitSetIterator extends DocIdSetIterator {
// hmmm, what about an iterator that finds zeros though,
// or a reverse iterator... should they be separate classes
// for efficiency, or have a common root interface? (or
// maybe both? could ask for a SetBitsIterator, etc...
final long[] arr;
final int words;
private int i=-1;
private long word;
private int wordShift;
private int indexArray;
private int curDocId = -1;
public OpenBitSetIterator(OpenBitSet obs) {
this(obs.getBits(), obs.getNumWords());
}
public OpenBitSetIterator(long[] bits, int numWords) {
arr = bits;
words = numWords;
}
// 64 bit shifts
private void shift() {
if ((int)word ==0) {wordShift +=32; word = word >>>32; }
if ((word & 0x0000FFFF) == 0) { wordShift +=16; word >>>=16; }
if ((word & 0x000000FF) == 0) { wordShift +=8; word >>>=8; }
indexArray = BitUtil.bitList((byte) word);
}
/***** alternate shift implementations
// 32 bit shifts, but a long shift needed at the end
private void shift2() {
int y = (int)word;
if (y==0) {wordShift +=32; y = (int)(word >>>32); }
if ((y & 0x0000FFFF) == 0) { wordShift +=16; y>>>=16; }
if ((y & 0x000000FF) == 0) { wordShift +=8; y>>>=8; }
indexArray = bitlist[y & 0xff];
word >>>= (wordShift +1);
}
private void shift3() {
int lower = (int)word;
int lowByte = lower & 0xff;
if (lowByte != 0) {
indexArray=bitlist[lowByte];
return;
}
shift();
}
******/
@Override
public int nextDoc() {
if (indexArray == 0) {
if (word != 0) {
word >>>= 8;
wordShift += 8;
}
while (word == 0) {
if (++i >= words) {
return curDocId = NO_MORE_DOCS;
}
word = arr[i];
wordShift = -1; // loop invariant code motion should move this
}
// after the first time, should I go with a linear search, or
// stick with the binary search in shift?
shift();
}
int bitIndex = (indexArray & 0x0f) + wordShift;
indexArray >>>= 4;
// should i<<6 be cached as a separate variable?
// it would only save one cycle in the best circumstances.
return curDocId = (i<<6) + bitIndex;
}
@Override
public int advance(int target) {
indexArray = 0;
i = target >> 6;
if (i >= words) {
word = 0; // setup so next() will also return -1
return curDocId = NO_MORE_DOCS;
}
wordShift = target & 0x3f;
word = arr[i] >>> wordShift;
if (word != 0) {
wordShift--; // compensate for 1 based arrIndex
} else {
while (word == 0) {
if (++i >= words) {
return curDocId = NO_MORE_DOCS;
}
word = arr[i];
}
wordShift = -1;
}
shift();
int bitIndex = (indexArray & 0x0f) + wordShift;
indexArray >>>= 4;
// should i<<6 be cached as a separate variable?
// it would only save one cycle in the best circumstances.
return curDocId = (i<<6) + bitIndex;
}
@Override
public int docID() {
return curDocId;
}
@Override
public long cost() {
return words / 64;
}
}

10
lucene/core/src/java/org/apache/lucene/util/fst/FST.java
View File

@ -1376,16 +1376,6 @@ public final class FST<T> implements Accountable {
public abstract boolean reversed(); public abstract boolean reversed();
} }
private static class ArcAndState<T> {
final Arc<T> arc;
final IntsRef chain;
public ArcAndState(Arc<T> arc, IntsRef chain) {
this.arc = arc;
this.chain = chain;
}
}
/* /*
public void countSingleChains() throws IOException { public void countSingleChains() throws IOException {
// TODO: must assert this FST was built with // TODO: must assert this FST was built with

69
lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java
View File

@ -815,22 +815,6 @@ public class PackedInts {
throw new AssertionError("Unknown Writer format: " + format); throw new AssertionError("Unknown Writer format: " + format);
} }
} }
/**
* Expert: Restore a {@link Reader} from a stream without reading metadata at
* the beginning of the stream. This method is useful to restore data when
* metadata has been previously read using {@link #readHeader(DataInput)}.
*
* @param in the stream to read data from, positioned at the beginning of the packed values
* @param header metadata result from <code>readHeader()</code>
* @return a Reader
* @throws IOException If there is a low-level I/O error
* @see #readHeader(DataInput)
* @lucene.internal
*/
public static Reader getReaderNoHeader(DataInput in, Header header) throws IOException {
return getReaderNoHeader(in, header.format, header.version, header.valueCount, header.bitsPerValue);
}
/** /**
* Restore a {@link Reader} from a stream. * Restore a {@link Reader} from a stream.
@ -943,23 +927,6 @@ public class PackedInts {
throw new AssertionError("Unknwown format: " + format); throw new AssertionError("Unknwown format: " + format);
} }
} }
/**
* Expert: Construct a direct {@link Reader} from an {@link IndexInput}
* without reading metadata at the beginning of the stream. This method is
* useful to restore data when metadata has been previously read using
* {@link #readHeader(DataInput)}.
*
* @param in the stream to read data from, positioned at the beginning of the packed values
* @param header metadata result from <code>readHeader()</code>
* @return a Reader
* @throws IOException If there is a low-level I/O error
* @see #readHeader(DataInput)
* @lucene.internal
*/
public static Reader getDirectReaderNoHeader(IndexInput in, Header header) throws IOException {
return getDirectReaderNoHeader(in, header.format, header.version, header.valueCount, header.bitsPerValue);
}
/** /**
* Construct a direct {@link Reader} from an {@link IndexInput}. This method * Construct a direct {@link Reader} from an {@link IndexInput}. This method
@ -1216,42 +1183,6 @@ public class PackedInts {
} }
} }
/**
* Expert: reads only the metadata from a stream. This is useful to later
* restore a stream or open a direct reader via
* {@link #getReaderNoHeader(DataInput, Header)}
* or {@link #getDirectReaderNoHeader(IndexInput, Header)}.
* @param in the stream to read data
* @return packed integer metadata.
* @throws IOException If there is a low-level I/O error
* @see #getReaderNoHeader(DataInput, Header)
* @see #getDirectReaderNoHeader(IndexInput, Header)
*/
public static Header readHeader(DataInput in) throws IOException {
final int version = CodecUtil.checkHeader(in, CODEC_NAME, VERSION_START, VERSION_CURRENT);
final int bitsPerValue = in.readVInt();
assert bitsPerValue > 0 && bitsPerValue <= 64: "bitsPerValue=" + bitsPerValue;
final int valueCount = in.readVInt();
final Format format = Format.byId(in.readVInt());
return new Header(format, valueCount, bitsPerValue, version);
}
/** Header identifying the structure of a packed integer array. */
public static class Header {
private final Format format;
private final int valueCount;
private final int bitsPerValue;
private final int version;
public Header(Format format, int valueCount, int bitsPerValue, int version) {
this.format = format;
this.valueCount = valueCount;
this.bitsPerValue = bitsPerValue;
this.version = version;
}
}
/** Check that the block size is a power of 2, in the right bounds, and return /** Check that the block size is a power of 2, in the right bounds, and return
* its log in base 2. */ * its log in base 2. */
static int checkBlockSize(int blockSize, int minBlockSize, int maxBlockSize) { static int checkBlockSize(int blockSize, int minBlockSize, int maxBlockSize) {

2
lucene/core/src/java/overview.html
View File

@ -118,7 +118,7 @@ which implements files as memory-resident data structures.</li>
<li> <li>
<b>{@link org.apache.lucene.util}</b> <b>{@link org.apache.lucene.util}</b>
contains a few handy data structures and util classes, ie {@link org.apache.lucene.util.OpenBitSet OpenBitSet} contains a few handy data structures and util classes, ie {@link org.apache.lucene.util.FixedBitSet FixedBitSet}
and {@link org.apache.lucene.util.PriorityQueue PriorityQueue}.</li> and {@link org.apache.lucene.util.PriorityQueue PriorityQueue}.</li>
</ul> </ul>
To use Lucene, an application should: To use Lucene, an application should:

188
lucene/core/src/test/org/apache/lucene/util/TestDoubleBarrelLRUCache.java
View File

@ -1,188 +0,0 @@
package org.apache.lucene.util;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import org.apache.lucene.util.LuceneTestCase;
public class TestDoubleBarrelLRUCache extends LuceneTestCase {
private void testCache(DoubleBarrelLRUCache<CloneableInteger,Object> cache, int n) throws Exception {
Object dummy = new Object();
for (int i = 0; i < n; i++) {
cache.put(new CloneableInteger(i), dummy);
}
// access every 2nd item in cache
for (int i = 0; i < n; i+=2) {
assertNotNull(cache.get(new CloneableInteger(i)));
}
// add n/2 elements to cache, the ones that weren't
// touched in the previous loop should now be thrown away
for (int i = n; i < n + (n / 2); i++) {
cache.put(new CloneableInteger(i), dummy);
}
// access every 4th item in cache
for (int i = 0; i < n; i+=4) {
assertNotNull(cache.get(new CloneableInteger(i)));
}
// add 3/4n elements to cache, the ones that weren't
// touched in the previous loops should now be thrown away
for (int i = n; i < n + (n * 3 / 4); i++) {
cache.put(new CloneableInteger(i), dummy);
}
// access every 4th item in cache
for (int i = 0; i < n; i+=4) {
assertNotNull(cache.get(new CloneableInteger(i)));
}
}
public void testLRUCache() throws Exception {
final int n = 100;
testCache(new DoubleBarrelLRUCache<CloneableInteger,Object>(n), n);
}
private class CacheThread extends Thread {
private final CloneableObject[] objs;
private final DoubleBarrelLRUCache<CloneableObject,Object> c;
private final long endTime;
volatile boolean failed;
public CacheThread(DoubleBarrelLRUCache<CloneableObject,Object> c,
CloneableObject[] objs, long endTime) {
this.c = c;
this.objs = objs;
this.endTime = endTime;
}
@Override
public void run() {
try {
long count = 0;
long miss = 0;
long hit = 0;
final int limit = objs.length;
while(true) {
final CloneableObject obj = objs[(int) ((count/2) % limit)];
Object v = c.get(obj);
if (v == null) {
c.put(new CloneableObject(obj), obj);
miss++;
} else {
assert obj == v;
hit++;
}
if ((++count % 10000) == 0) {
if (System.currentTimeMillis() >= endTime) {
break;
}
}
}
addResults(miss, hit);
} catch (Throwable t) {
failed = true;
throw new RuntimeException(t);
}
}
}
long totMiss, totHit;
void addResults(long miss, long hit) {
totMiss += miss;
totHit += hit;
}
public void testThreadCorrectness() throws Exception {
final int NUM_THREADS = 4;
final int CACHE_SIZE = 512;
final int OBJ_COUNT = 3*CACHE_SIZE;
DoubleBarrelLRUCache<CloneableObject,Object> c = new DoubleBarrelLRUCache<>(1024);
CloneableObject[] objs = new CloneableObject[OBJ_COUNT];
for(int i=0;i<OBJ_COUNT;i++) {
objs[i] = new CloneableObject(new Object());
}
final CacheThread[] threads = new CacheThread[NUM_THREADS];
final long endTime = System.currentTimeMillis()+1000L;
for(int i=0;i<NUM_THREADS;i++) {
threads[i] = new CacheThread(c, objs, endTime);
threads[i].start();
}
for(int i=0;i<NUM_THREADS;i++) {
threads[i].join();
assert !threads[i].failed;
}
//System.out.println("hits=" + totHit + " misses=" + totMiss);
}
private static class CloneableObject extends DoubleBarrelLRUCache.CloneableKey {
private Object value;
public CloneableObject(Object value) {
this.value = value;
}
@Override
public boolean equals(Object other) {
return this.value.equals(((CloneableObject) other).value);
}
@Override
public int hashCode() {
return value.hashCode();
}
@Override
public CloneableObject clone() {
return new CloneableObject(value);
}
}
protected static class CloneableInteger extends DoubleBarrelLRUCache.CloneableKey {
private Integer value;
public CloneableInteger(Integer value) {
this.value = value;
}
@Override
public boolean equals(Object other) {
return this.value.equals(((CloneableInteger) other).value);
}
@Override
public int hashCode() {
return value.hashCode();
}
@Override
public CloneableInteger clone() {
return new CloneableInteger(value);
}
}
}

369
lucene/core/src/test/org/apache/lucene/util/TestOpenBitSet.java
View File

@ -1,369 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.util;
import java.io.IOException;
import java.util.BitSet;
import org.apache.lucene.search.DocIdSetIterator;
public class TestOpenBitSet extends BaseDocIdSetTestCase<OpenBitSet> {
@Override
public OpenBitSet copyOf(BitSet bs, int length) throws IOException {
final OpenBitSet set = new OpenBitSet(length);
for (int doc = bs.nextSetBit(0); doc != -1; doc = bs.nextSetBit(doc + 1)) {
set.set(doc);
}
return set;
}
void doGet(BitSet a, OpenBitSet b) {
int max = a.size();
for (int i=0; i<max; i++) {
if (a.get(i) != b.get(i)) {
fail("mismatch: BitSet=["+i+"]="+a.get(i));
}
if (a.get(i) != b.get((long) i)) {
fail("mismatch: BitSet=["+i+"]="+a.get(i));
}
}
}
void doGetFast(BitSet a, OpenBitSet b, int max) {
for (int i=0; i<max; i++) {
if (a.get(i) != b.fastGet(i)) {
fail("mismatch: BitSet=["+i+"]="+a.get(i));
}
if (a.get(i) != b.fastGet((long) i)) {
fail("mismatch: BitSet=["+i+"]="+a.get(i));
}
}
}
void doNextSetBit(BitSet a, OpenBitSet b) {
int aa=-1,bb=-1;
do {
aa = a.nextSetBit(aa+1);
bb = b.nextSetBit(bb+1);
assertEquals(aa,bb);
} while (aa>=0);
}
void doNextSetBitLong(BitSet a, OpenBitSet b) {
int aa=-1,bb=-1;
do {
aa = a.nextSetBit(aa+1);
bb = (int) b.nextSetBit((long) (bb+1));
assertEquals(aa,bb);
} while (aa>=0);
}
void doPrevSetBit(BitSet a, OpenBitSet b) {
int aa = a.size() + random().nextInt(100);
int bb = aa;
do {
// aa = a.prevSetBit(aa-1);
aa--;
while ((aa >= 0) && (! a.get(aa))) {
aa--;
}
bb = b.prevSetBit(bb-1);
assertEquals(aa,bb);
} while (aa>=0);
}
void doPrevSetBitLong(BitSet a, OpenBitSet b) {
int aa = a.size() + random().nextInt(100);
int bb = aa;
do {
// aa = a.prevSetBit(aa-1);
aa--;
while ((aa >= 0) && (! a.get(aa))) {
aa--;
}
bb = (int) b.prevSetBit((long) (bb-1));
assertEquals(aa,bb);
} while (aa>=0);
}
// test interleaving different OpenBitSetIterator.next()/skipTo()
void doIterate(BitSet a, OpenBitSet b, int mode) {
if (mode==1) doIterate1(a, b);
if (mode==2) doIterate2(a, b);
}
void doIterate1(BitSet a, OpenBitSet b) {
int aa=-1,bb=-1;
OpenBitSetIterator iterator = new OpenBitSetIterator(b);
do {
aa = a.nextSetBit(aa+1);
bb = random().nextBoolean() ? iterator.nextDoc() : iterator.advance(bb + 1);
assertEquals(aa == -1 ? DocIdSetIterator.NO_MORE_DOCS : aa, bb);
} while (aa>=0);
}
void doIterate2(BitSet a, OpenBitSet b) {
int aa=-1,bb=-1;
OpenBitSetIterator iterator = new OpenBitSetIterator(b);
do {
aa = a.nextSetBit(aa+1);
bb = random().nextBoolean() ? iterator.nextDoc() : iterator.advance(bb + 1);
assertEquals(aa == -1 ? DocIdSetIterator.NO_MORE_DOCS : aa, bb);
} while (aa>=0);
}
void doRandomSets(int maxSize, int iter, int mode) {
BitSet a0=null;
OpenBitSet b0=null;
for (int i=0; i<iter; i++) {
int sz = random().nextInt(maxSize);
BitSet a = new BitSet(sz);
OpenBitSet b = new OpenBitSet(sz);
// test the various ways of setting bits
if (sz>0) {
int nOper = random().nextInt(sz);
for (int j=0; j<nOper; j++) {
int idx;
idx = random().nextInt(sz);
a.set(idx);
b.fastSet(idx);
idx = random().nextInt(sz);
a.set(idx);
b.fastSet((long) idx);
idx = random().nextInt(sz);
a.clear(idx);
b.fastClear(idx);
idx = random().nextInt(sz);
a.clear(idx);
b.fastClear((long) idx);
idx = random().nextInt(sz);
a.flip(idx);
b.fastFlip(idx);
boolean val = b.flipAndGet(idx);
boolean val2 = b.flipAndGet(idx);
assertTrue(val != val2);
idx = random().nextInt(sz);
a.flip(idx);
b.fastFlip((long) idx);
val = b.flipAndGet((long) idx);
val2 = b.flipAndGet((long) idx);
assertTrue(val != val2);
val = b.getAndSet(idx);
assertTrue(val2 == val);
assertTrue(b.get(idx));
if (!val) b.fastClear(idx);
assertTrue(b.get(idx) == val);
}
}
// test that the various ways of accessing the bits are equivalent
doGet(a,b);
doGetFast(a, b, sz);
// test ranges, including possible extension
int fromIndex, toIndex;
fromIndex = random().nextInt(sz+80);
toIndex = fromIndex + random().nextInt((sz>>1)+1);
BitSet aa = (BitSet)a.clone(); aa.flip(fromIndex,toIndex);
OpenBitSet bb = b.clone(); bb.flip(fromIndex,toIndex);
doIterate(aa,bb, mode); // a problem here is from flip or doIterate
fromIndex = random().nextInt(sz+80);
toIndex = fromIndex + random().nextInt((sz>>1)+1);
aa = (BitSet)a.clone(); aa.clear(fromIndex,toIndex);
bb = b.clone(); bb.clear(fromIndex,toIndex);
doNextSetBit(aa,bb); // a problem here is from clear() or nextSetBit
doNextSetBitLong(aa,bb);
doPrevSetBit(aa,bb);
doPrevSetBitLong(aa,bb);
fromIndex = random().nextInt(sz+80);
toIndex = fromIndex + random().nextInt((sz>>1)+1);
aa = (BitSet)a.clone(); aa.set(fromIndex,toIndex);
bb = b.clone(); bb.set(fromIndex,toIndex);
doNextSetBit(aa,bb); // a problem here is from set() or nextSetBit
doNextSetBitLong(aa,bb);
doPrevSetBit(aa,bb);
doPrevSetBitLong(aa,bb);
if (a0 != null) {
assertEquals( a.equals(a0), b.equals(b0));
assertEquals(a.cardinality(), b.cardinality());
BitSet a_and = (BitSet)a.clone(); a_and.and(a0);
BitSet a_or = (BitSet)a.clone(); a_or.or(a0);
BitSet a_xor = (BitSet)a.clone(); a_xor.xor(a0);
BitSet a_andn = (BitSet)a.clone(); a_andn.andNot(a0);
OpenBitSet b_and = b.clone(); assertEquals(b,b_and); b_and.and(b0);
OpenBitSet b_or = b.clone(); b_or.or(b0);
OpenBitSet b_xor = b.clone(); b_xor.xor(b0);
OpenBitSet b_andn = b.clone(); b_andn.andNot(b0);
doIterate(a_and,b_and, mode);
doIterate(a_or,b_or, mode);
doIterate(a_xor,b_xor, mode);
doIterate(a_andn,b_andn, mode);
assertEquals(a_and.cardinality(), b_and.cardinality());
assertEquals(a_or.cardinality(), b_or.cardinality());
assertEquals(a_xor.cardinality(), b_xor.cardinality());
assertEquals(a_andn.cardinality(), b_andn.cardinality());
// test non-mutating popcounts
assertEquals(b_and.cardinality(), OpenBitSet.intersectionCount(b,b0));
assertEquals(b_or.cardinality(), OpenBitSet.unionCount(b,b0));
assertEquals(b_xor.cardinality(), OpenBitSet.xorCount(b,b0));
assertEquals(b_andn.cardinality(), OpenBitSet.andNotCount(b,b0));
}
a0=a;
b0=b;
}
}
// large enough to flush obvious bugs, small enough to run in <.5 sec as part of a
// larger testsuite.
public void testSmall() {
doRandomSets(atLeast(1200), atLeast(1000), 1);
doRandomSets(atLeast(1200), atLeast(1000), 2);
}
// uncomment to run a bigger test (~2 minutes).
/*
public void testBig() {
doRandomSets(2000,200000, 1);
doRandomSets(2000,200000, 2);
}
*/
public void testEquals() {
OpenBitSet b1 = new OpenBitSet(1111);
OpenBitSet b2 = new OpenBitSet(2222);
assertTrue(b1.equals(b2));
assertTrue(b2.equals(b1));
b1.set(10);
assertFalse(b1.equals(b2));
assertFalse(b2.equals(b1));
b2.set(10);
assertTrue(b1.equals(b2));
assertTrue(b2.equals(b1));
b2.set(2221);
assertFalse(b1.equals(b2));
assertFalse(b2.equals(b1));
b1.set(2221);
assertTrue(b1.equals(b2));
assertTrue(b2.equals(b1));
// try different type of object
assertFalse(b1.equals(new Object()));
}
public void testHashCodeEquals() {
OpenBitSet bs1 = new OpenBitSet(200);
OpenBitSet bs2 = new OpenBitSet(64);
bs1.set(3);
bs2.set(3);
assertEquals(bs1, bs2);
assertEquals(bs1.hashCode(), bs2.hashCode());
}
private OpenBitSet makeOpenBitSet(int[] a) {
OpenBitSet bs = new OpenBitSet();
for (int e: a) {
bs.set(e);
}
return bs;
}
private BitSet makeBitSet(int[] a) {
BitSet bs = new BitSet();
for (int e: a) {
bs.set(e);
}
return bs;
}
private void checkPrevSetBitArray(int [] a) {
OpenBitSet obs = makeOpenBitSet(a);
BitSet bs = makeBitSet(a);
doPrevSetBit(bs, obs);
}
public void testPrevSetBit() {
checkPrevSetBitArray(new int[] {});
checkPrevSetBitArray(new int[] {0});
checkPrevSetBitArray(new int[] {0,2});
}
public void testEnsureCapacity() {
OpenBitSet bits = new OpenBitSet(1);
int bit = random().nextInt(100) + 10;
bits.ensureCapacity(bit); // make room for more bits
bits.fastSet(bit-1);
assertTrue(bits.fastGet(bit-1));
bits.ensureCapacity(bit + 1);
bits.fastSet(bit);
assertTrue(bits.fastGet(bit));
bits.ensureCapacity(3); // should not change numBits nor grow the array
bits.fastSet(3);
assertTrue(bits.fastGet(3));
bits.fastSet(bit-1);
assertTrue(bits.fastGet(bit-1));
// test ensureCapacityWords
int numWords = random().nextInt(10) + 2; // make sure we grow the array (at least 128 bits)
bits.ensureCapacityWords(numWords);
bit = TestUtil.nextInt(random(), 127, (numWords << 6) - 1); // pick a bit >= to 128, but still within range
bits.fastSet(bit);
assertTrue(bits.fastGet(bit));
bits.fastClear(bit);
assertFalse(bits.fastGet(bit));
bits.fastFlip(bit);
assertTrue(bits.fastGet(bit));
bits.ensureCapacityWords(2); // should not change numBits nor grow the array
bits.fastSet(3);
assertTrue(bits.fastGet(3));
bits.fastSet(bit-1);
assertTrue(bits.fastGet(bit-1));
}
}

2
solr/core/src/java/org/apache/solr/schema/BoolField.java
View File

@ -172,7 +172,7 @@ public class BoolField extends PrimitiveFieldType {
} }
} }
// TODO - this can be much more efficient - use OpenBitSet or Bits // TODO - this can be much more efficient - use FixedBitSet or Bits
class BoolFieldSource extends ValueSource { class BoolFieldSource extends ValueSource {
protected String field; protected String field;