mirror of https://github.com/apache/lucene.git
[Minor] Improvements to slice pools (#12795)
1. Remove rest method used only in tests. 2. Update Javadocs. 3. Make interleaved slices test a bit more evil by adding pool resets.
This commit is contained in:
parent
14196cfcf7
commit
a70432c110
|
@ -22,11 +22,11 @@ import org.apache.lucene.store.DataOutput;
|
||||||
import org.apache.lucene.util.BitUtil;
|
import org.apache.lucene.util.BitUtil;
|
||||||
import org.apache.lucene.util.ByteBlockPool;
|
import org.apache.lucene.util.ByteBlockPool;
|
||||||
|
|
||||||
/* IndexInput that knows how to read the byte slices written
|
/**
|
||||||
* by Posting and PostingVector. We read the bytes in
|
* IndexInput that knows how to read the byte slices written by Posting and PostingVector. We read
|
||||||
* each slice until we hit the end of that slice at which
|
* the bytes in each slice until we hit the end of that slice at which point we read the forwarding
|
||||||
* point we read the forwarding address of the next slice
|
* address of the next slice and then jump to it.
|
||||||
* and then jump to it.*/
|
*/
|
||||||
final class ByteSliceReader extends DataInput {
|
final class ByteSliceReader extends DataInput {
|
||||||
ByteBlockPool pool;
|
ByteBlockPool pool;
|
||||||
int bufferUpto;
|
int bufferUpto;
|
||||||
|
|
|
@ -132,18 +132,11 @@ public final class ByteBlockPool implements Accountable {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Resets the pool to its initial state, reusing the first buffer and filling all buffers with
|
* Expert: Resets the pool to its initial state, while optionally reusing the first buffer.
|
||||||
* {@code 0} bytes before they are reused or passed to {@link
|
* Buffers that are not reused are reclaimed by {@link Allocator#recycleByteBlocks(byte[][], int,
|
||||||
* Allocator#recycleByteBlocks(byte[][], int, int)}. Calling {@link ByteBlockPool#nextBuffer()} is
|
* int)}. Buffers can be filled with zeros before recycling them. This is useful if a slice pool
|
||||||
* not needed after reset.
|
* works on top of this byte pool and relies on the buffers being filled with zeros to find the
|
||||||
*/
|
* non-zero end of slices.
|
||||||
public void reset() {
|
|
||||||
reset(true, true);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Expert: Resets the pool to its initial state, while reusing the first buffer. Calling {@link
|
|
||||||
* ByteBlockPool#nextBuffer()} is not needed after reset.
|
|
||||||
*
|
*
|
||||||
* @param zeroFillBuffers if {@code true} the buffers are filled with {@code 0}. This should be
|
* @param zeroFillBuffers if {@code true} the buffers are filled with {@code 0}. This should be
|
||||||
* set to {@code true} if this pool is used with slices.
|
* set to {@code true} if this pool is used with slices.
|
||||||
|
@ -188,7 +181,8 @@ public final class ByteBlockPool implements Accountable {
|
||||||
/**
|
/**
|
||||||
* Allocates a new buffer and advances the pool to it. This method should be called once after the
|
* Allocates a new buffer and advances the pool to it. This method should be called once after the
|
||||||
* constructor to initialize the pool. In contrast to the constructor, a {@link
|
* constructor to initialize the pool. In contrast to the constructor, a {@link
|
||||||
* ByteBlockPool#reset()} call will advance the pool to its first buffer immediately.
|
* ByteBlockPool#reset(boolean, boolean)} call will advance the pool to its first buffer
|
||||||
|
* immediately.
|
||||||
*/
|
*/
|
||||||
public void nextBuffer() {
|
public void nextBuffer() {
|
||||||
if (1 + bufferUpto == buffers.length) {
|
if (1 + bufferUpto == buffers.length) {
|
||||||
|
|
|
@ -94,15 +94,11 @@ public class IntBlockPool {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Resets the pool to its initial state reusing the first buffer. Calling {@link
|
* Expert: Resets the pool to its initial state, while optionally reusing the first buffer.
|
||||||
* IntBlockPool#nextBuffer()} is not needed after reset.
|
* Buffers that are not reused are reclaimed by {@link
|
||||||
*/
|
* ByteBlockPool.Allocator#recycleByteBlocks(byte[][], int, int)}. Buffers can be filled with
|
||||||
public void reset() {
|
* zeros before recycling them. This is useful if a slice pool works on top of this int pool and
|
||||||
this.reset(true, true);
|
* relies on the buffers being filled with zeros to find the non-zero end of slices.
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Expert: Resets the pool to its initial state reusing the first buffer.
|
|
||||||
*
|
*
|
||||||
* @param zeroFillBuffers if <code>true</code> the buffers are filled with <code>0</code>.
|
* @param zeroFillBuffers if <code>true</code> the buffers are filled with <code>0</code>.
|
||||||
* @param reuseFirst if <code>true</code> the first buffer will be reused and calling {@link
|
* @param reuseFirst if <code>true</code> the first buffer will be reused and calling {@link
|
||||||
|
@ -145,8 +141,8 @@ public class IntBlockPool {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Advances the pool to its next buffer. This method should be called once after the constructor
|
* Advances the pool to its next buffer. This method should be called once after the constructor
|
||||||
* to initialize the pool. In contrast to the constructor a {@link IntBlockPool#reset()} call will
|
* to initialize the pool. In contrast to the constructor a {@link IntBlockPool#reset(boolean,
|
||||||
* advance the pool to its first buffer immediately.
|
* boolean)} call will advance the pool to its first buffer immediately.
|
||||||
*/
|
*/
|
||||||
public void nextBuffer() {
|
public void nextBuffer() {
|
||||||
if (1 + bufferUpto == buffers.length) {
|
if (1 + bufferUpto == buffers.length) {
|
||||||
|
|
|
@ -21,6 +21,7 @@ import org.apache.lucene.tests.util.TestUtil;
|
||||||
import org.apache.lucene.util.BitUtil;
|
import org.apache.lucene.util.BitUtil;
|
||||||
import org.apache.lucene.util.ByteBlockPool;
|
import org.apache.lucene.util.ByteBlockPool;
|
||||||
import org.apache.lucene.util.Counter;
|
import org.apache.lucene.util.Counter;
|
||||||
|
import org.apache.lucene.util.RecyclingByteBlockAllocator;
|
||||||
|
|
||||||
public class TestByteSlicePool extends LuceneTestCase {
|
public class TestByteSlicePool extends LuceneTestCase {
|
||||||
public void testAllocKnownSizeSlice() {
|
public void testAllocKnownSizeSlice() {
|
||||||
|
@ -223,57 +224,66 @@ public class TestByteSlicePool extends LuceneTestCase {
|
||||||
* that we read back the same data we wrote.
|
* that we read back the same data we wrote.
|
||||||
*/
|
*/
|
||||||
public void testRandomInterleavedSlices() {
|
public void testRandomInterleavedSlices() {
|
||||||
ByteBlockPool blockPool = new ByteBlockPool(new ByteBlockPool.DirectAllocator());
|
ByteBlockPool blockPool = new ByteBlockPool(new RecyclingByteBlockAllocator());
|
||||||
ByteSlicePool slicePool = new ByteSlicePool(blockPool);
|
ByteSlicePool slicePool = new ByteSlicePool(blockPool);
|
||||||
|
|
||||||
int n = TestUtil.nextInt(random(), 2, 3); // 2 or 3 writers and readers
|
int nIterations =
|
||||||
SliceWriter[] sliceWriters = new SliceWriter[n];
|
TestUtil.nextInt(random(), 1, 3); // 1-3 iterations with buffer resets in between
|
||||||
SliceReader[] sliceReaders = new SliceReader[n];
|
for (int iter = 0; iter < nIterations; iter++) {
|
||||||
|
int n = TestUtil.nextInt(random(), 2, 3); // 2 or 3 writers and readers
|
||||||
|
SliceWriter[] sliceWriters = new SliceWriter[n];
|
||||||
|
SliceReader[] sliceReaders = new SliceReader[n];
|
||||||
|
|
||||||
// Init slice writers
|
// Init slice writers
|
||||||
for (int i = 0; i < n; i++) {
|
for (int i = 0; i < n; i++) {
|
||||||
sliceWriters[i] = new SliceWriter(slicePool);
|
sliceWriters[i] = new SliceWriter(slicePool);
|
||||||
}
|
|
||||||
|
|
||||||
// Write slices
|
|
||||||
while (true) {
|
|
||||||
int i = random().nextInt(n);
|
|
||||||
boolean succeeded = sliceWriters[i].writeSlice();
|
|
||||||
if (succeeded == false) {
|
|
||||||
for (int j = 0; j < n; j++) {
|
|
||||||
while (sliceWriters[j].writeSlice())
|
|
||||||
;
|
|
||||||
}
|
|
||||||
break;
|
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// Init slice readers
|
// Write slices
|
||||||
for (int i = 0; i < n; i++) {
|
while (true) {
|
||||||
sliceReaders[i] =
|
int i = random().nextInt(n);
|
||||||
new SliceReader(
|
boolean succeeded = sliceWriters[i].writeSlice();
|
||||||
slicePool,
|
if (succeeded == false) {
|
||||||
sliceWriters[i].size,
|
for (int j = 0; j < n; j++) {
|
||||||
sliceWriters[i].firstSliceOffset,
|
while (sliceWriters[j].writeSlice())
|
||||||
sliceWriters[i].firstSlice);
|
;
|
||||||
}
|
}
|
||||||
|
break;
|
||||||
// Read slices
|
|
||||||
while (true) {
|
|
||||||
int i = random().nextInt(n);
|
|
||||||
boolean succeeded = sliceReaders[i].readSlice();
|
|
||||||
if (succeeded == false) {
|
|
||||||
for (int j = 0; j < n; j++) {
|
|
||||||
while (sliceReaders[j].readSlice())
|
|
||||||
;
|
|
||||||
}
|
}
|
||||||
break;
|
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// Compare written data with read data
|
// Init slice readers
|
||||||
for (int i = 0; i < n; i++) {
|
for (int i = 0; i < n; i++) {
|
||||||
assertArrayEquals(sliceWriters[i].randomData, sliceReaders[i].readData);
|
sliceReaders[i] =
|
||||||
|
new SliceReader(
|
||||||
|
slicePool,
|
||||||
|
sliceWriters[i].size,
|
||||||
|
sliceWriters[i].firstSliceOffset,
|
||||||
|
sliceWriters[i].firstSlice);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read slices
|
||||||
|
while (true) {
|
||||||
|
int i = random().nextInt(n);
|
||||||
|
boolean succeeded = sliceReaders[i].readSlice();
|
||||||
|
if (succeeded == false) {
|
||||||
|
for (int j = 0; j < n; j++) {
|
||||||
|
while (sliceReaders[j].readSlice())
|
||||||
|
;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Compare written data with read data
|
||||||
|
for (int i = 0; i < n; i++) {
|
||||||
|
assertArrayEquals(sliceWriters[i].randomData, sliceReaders[i].readData);
|
||||||
|
}
|
||||||
|
|
||||||
|
// We don't rely on the buffers being filled with zeros because the SliceWriter keeps the
|
||||||
|
// slice length as state, but ByteSlicePool.allocKnownSizeSlice asserts on zeros in the
|
||||||
|
// buffer.
|
||||||
|
blockPool.reset(true, random().nextBoolean());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -50,7 +50,7 @@ public class TestIntBlockPool extends LuceneTestCase {
|
||||||
|
|
||||||
// Reset and fill with zeros, then check there is no data left
|
// Reset and fill with zeros, then check there is no data left
|
||||||
pool.intUpto = count;
|
pool.intUpto = count;
|
||||||
pool.reset();
|
pool.reset(true, true);
|
||||||
for (int i = 0; i < count; i++) {
|
for (int i = 0; i < count; i++) {
|
||||||
assertEquals(0, pool.buffers[0][i]);
|
assertEquals(0, pool.buffers[0][i]);
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue