Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-5034: Make (Monotonic)AppendingLongBuffer's block size configurable. 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1490469 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Adrien Grand 2013-06-06 23:30:15 +00:00
parent 309dd8191d
commit 3ff9c73184
4 changed files with 66 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
lucene/core/src/java/org/apache/lucene/util/packed/AbstractAppendingLongBuffer.java
View File

@ -17,6 +17,8 @@ package org.apache.lucene.util.packed;
* limitations under the License.
*/
import static org.apache.lucene.util.packed.PackedInts.checkBlockSize;
import java.util.Arrays;
import org.apache.lucene.util.ArrayUtil;
@ -25,33 +27,37 @@ import org.apache.lucene.util.RamUsageEstimator;
/** Common functionality shared by {@link AppendingLongBuffer} and {@link MonotonicAppendingLongBuffer}. */
abstract class AbstractAppendingLongBuffer {
static final int BLOCK_BITS = 10;
static final int MAX_PENDING_COUNT = 1 << BLOCK_BITS;
static final int BLOCK_MASK = MAX_PENDING_COUNT - 1;
static final int MIN_PAGE_SIZE = 64;
// More than 1M doesn't really makes sense with these appending buffers
// since their goal is to try to have small numbers of bits per value
static final int MAX_PAGE_SIZE = 1 << 20;
final int pageShift, pageMask;
long[] minValues;
PackedInts.Reader[] deltas;
private long deltasBytes;
int valuesOff;
long[] pending;
final long[] pending;
int pendingOff;
AbstractAppendingLongBuffer(int initialBlockCount) {
minValues = new long[16];
deltas = new PackedInts.Reader[16];
pending = new long[MAX_PENDING_COUNT];
AbstractAppendingLongBuffer(int initialBlockCount, int pageSize) {
minValues = new long[initialBlockCount];
deltas = new PackedInts.Reader[initialBlockCount];
pending = new long[pageSize];
pageShift = checkBlockSize(pageSize, MIN_PAGE_SIZE, MAX_PAGE_SIZE);
pageMask = pageSize - 1;
valuesOff = 0;
pendingOff = 0;
}
/** Get the number of values that have been added to the buffer. */
public final long size() {
return valuesOff * (long) MAX_PENDING_COUNT + pendingOff;
return valuesOff * (long) pending.length + pendingOff;
}
/** Append a value to this buffer. */
public final void add(long l) {
if (pendingOff == MAX_PENDING_COUNT) {
if (pendingOff == pending.length) {
// check size
if (deltas.length == valuesOff) {
final int newLength = ArrayUtil.oversize(valuesOff + 1, 8);
@ -80,8 +86,8 @@ abstract class AbstractAppendingLongBuffer {
if (index < 0 || index >= size()) {
throw new IndexOutOfBoundsException("" + index);
}
int block = (int) (index >> BLOCK_BITS);
int element = (int) (index & BLOCK_MASK);
final int block = (int) (index >> pageShift);
final int element = (int) (index & pageMask);
return get(block, element);
}
@ -99,7 +105,7 @@ abstract class AbstractAppendingLongBuffer {
if (valuesOff == 0) {
currentValues = pending;
} else {
currentValues = new long[MAX_PENDING_COUNT];
currentValues = new long[pending.length];
fillValues();
}
}
@ -115,7 +121,7 @@ abstract class AbstractAppendingLongBuffer {
public final long next() {
assert hasNext();
long result = currentValues[pOff++];
if (pOff == MAX_PENDING_COUNT) {
if (pOff == pending.length) {
vOff += 1;
pOff = 0;
if (vOff <= valuesOff) {
@ -139,6 +145,7 @@ abstract class AbstractAppendingLongBuffer {
public long ramBytesUsed() {
// TODO: this is called per-doc-per-norms/dv-field, can we optimize this?
long bytesUsed = RamUsageEstimator.alignObjectSize(baseRamBytesUsed())
+ 2 * RamUsageEstimator.NUM_BYTES_INT // pageShift, pageMask
+ RamUsageEstimator.NUM_BYTES_LONG // valuesBytes
+ RamUsageEstimator.sizeOf(pending)
+ RamUsageEstimator.sizeOf(minValues)

24
lucene/core/src/java/org/apache/lucene/util/packed/AppendingLongBuffer.java
View File

@ -27,9 +27,16 @@ import java.util.Arrays;
*/
public final class AppendingLongBuffer extends AbstractAppendingLongBuffer {
/** Sole constructor. */
/** @param initialPageCount the initial number of pages
* @param pageSize the size of a single page */
public AppendingLongBuffer(int initialPageCount, int pageSize) {
super(initialPageCount, pageSize);
}
/** Create an {@link AppendingLongBuffer} with initialPageCount=16 and
* pageSize=1024. */
public AppendingLongBuffer() {
super(16);
this(16, 1024);
}
@Override
@ -43,8 +50,9 @@ public final class AppendingLongBuffer extends AbstractAppendingLongBuffer {
}
}
@Override
void packPendingValues() {
assert pendingOff == MAX_PENDING_COUNT;
assert pendingOff == pending.length;
// compute max delta
long minValue = pending[0];
@ -71,6 +79,7 @@ public final class AppendingLongBuffer extends AbstractAppendingLongBuffer {
}
/** Return an iterator over the values of this buffer. */
@Override
public Iterator iterator() {
return new Iterator();
}
@ -78,20 +87,21 @@ public final class AppendingLongBuffer extends AbstractAppendingLongBuffer {
/** A long iterator. */
public final class Iterator extends AbstractAppendingLongBuffer.Iterator {
private Iterator() {
Iterator() {
super();
}
@Override
void fillValues() {
if (vOff == valuesOff) {
currentValues = pending;
} else if (deltas[vOff] == null) {
Arrays.fill(currentValues, minValues[vOff]);
} else {
for (int k = 0; k < MAX_PENDING_COUNT; ) {
k += deltas[vOff].get(k, currentValues, k, MAX_PENDING_COUNT - k);
for (int k = 0; k < pending.length; ) {
k += deltas[vOff].get(k, currentValues, k, pending.length - k);
}
for (int k = 0; k < MAX_PENDING_COUNT; ++k) {
for (int k = 0; k < pending.length; ++k) {
currentValues[k] += minValues[vOff];
}
}

36
lucene/core/src/java/org/apache/lucene/util/packed/MonotonicAppendingLongBuffer.java
View File

@ -37,14 +37,22 @@ public final class MonotonicAppendingLongBuffer extends AbstractAppendingLongBuf
return (n >> 63) ^ (n << 1);
}
private float[] averages;
float[] averages;
/** Sole constructor. */
public MonotonicAppendingLongBuffer() {
super(16);
averages = new float[16];
/** @param initialPageCount the initial number of pages
* @param pageSize the size of a single page */
public MonotonicAppendingLongBuffer(int initialPageCount, int pageSize) {
super(initialPageCount, pageSize);
averages = new float[pending.length];
}
/** Create an {@link MonotonicAppendingLongBuffer} with initialPageCount=16
* and pageSize=1024. */
public MonotonicAppendingLongBuffer() {
this(16, 1024);
}
@Override
long get(int block, int element) {
if (block == valuesOff) {
return pending[element];
@ -66,16 +74,16 @@ public final class MonotonicAppendingLongBuffer extends AbstractAppendingLongBuf
@Override
void packPendingValues() {
assert pendingOff == MAX_PENDING_COUNT;
assert pendingOff == pending.length;
minValues[valuesOff] = pending[0];
averages[valuesOff] = (float) (pending[BLOCK_MASK] - pending[0]) / BLOCK_MASK;
averages[valuesOff] = (float) (pending[pending.length - 1] - pending[0]) / (pending.length - 1);
for (int i = 0; i < MAX_PENDING_COUNT; ++i) {
for (int i = 0; i < pending.length; ++i) {
pending[i] = zigZagEncode(pending[i] - minValues[valuesOff] - (long) (averages[valuesOff] * (long) i));
}
long maxDelta = 0;
for (int i = 0; i < MAX_PENDING_COUNT; ++i) {
for (int i = 0; i < pending.length; ++i) {
if (pending[i] < 0) {
maxDelta = -1;
break;
@ -94,6 +102,7 @@ public final class MonotonicAppendingLongBuffer extends AbstractAppendingLongBuf
}
/** Return an iterator over the values of this buffer. */
@Override
public Iterator iterator() {
return new Iterator();
}
@ -105,18 +114,19 @@ public final class MonotonicAppendingLongBuffer extends AbstractAppendingLongBuf
super();
}
@Override
void fillValues() {
if (vOff == valuesOff) {
currentValues = pending;
} else if (deltas[vOff] == null) {
for (int k = 0; k < MAX_PENDING_COUNT; ++k) {
for (int k = 0; k < pending.length; ++k) {
currentValues[k] = minValues[vOff] + (long) (averages[vOff] * (long) k);
}
} else {
for (int k = 0; k < MAX_PENDING_COUNT; ) {
k += deltas[vOff].get(k, currentValues, k, MAX_PENDING_COUNT - k);
for (int k = 0; k < pending.length; ) {
k += deltas[vOff].get(k, currentValues, k, pending.length - k);
}
for (int k = 0; k < MAX_PENDING_COUNT; ++k) {
for (int k = 0; k < pending.length; ++k) {
currentValues[k] = minValues[vOff] + (long) (averages[vOff] * (long) k) + zigZagDecode(currentValues[k]);
}
}

6
lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java
View File

@ -863,13 +863,15 @@ public class TestPackedInts extends LuceneTestCase {
final long[] arr = new long[RandomInts.randomIntBetween(random(), 1, 1000000)];
for (int bpv : new int[] {0, 1, 63, 64, RandomInts.randomIntBetween(random(), 2, 62)}) {
for (boolean monotonic : new boolean[] {true, false}) {
final int pageSize = 1 << _TestUtil.nextInt(random(), 6, 20);
final int initialPageCount = _TestUtil.nextInt(random(), 0, 16);
AbstractAppendingLongBuffer buf;
final int inc;
if (monotonic) {
buf = new MonotonicAppendingLongBuffer();
buf = new MonotonicAppendingLongBuffer(initialPageCount, pageSize);
inc = _TestUtil.nextInt(random(), -1000, 1000);
} else {
buf = new AppendingLongBuffer();
buf = new AppendingLongBuffer(initialPageCount, pageSize);
inc = 0;
}
if (bpv == 0) {