mirror of https://github.com/apache/lucene.git
[Minor] Improvements to slice pools (#12795)
1. Remove rest method used only in tests. 2. Update Javadocs. 3. Make interleaved slices test a bit more evil by adding pool resets.
This commit is contained in:
parent
14196cfcf7
commit
a70432c110
|
@ -22,11 +22,11 @@ import org.apache.lucene.store.DataOutput;
|
|||
import org.apache.lucene.util.BitUtil;
|
||||
import org.apache.lucene.util.ByteBlockPool;
|
||||
|
||||
/* IndexInput that knows how to read the byte slices written
|
||||
* by Posting and PostingVector. We read the bytes in
|
||||
* each slice until we hit the end of that slice at which
|
||||
* point we read the forwarding address of the next slice
|
||||
* and then jump to it.*/
|
||||
/**
|
||||
* IndexInput that knows how to read the byte slices written by Posting and PostingVector. We read
|
||||
* the bytes in each slice until we hit the end of that slice at which point we read the forwarding
|
||||
* address of the next slice and then jump to it.
|
||||
*/
|
||||
final class ByteSliceReader extends DataInput {
|
||||
ByteBlockPool pool;
|
||||
int bufferUpto;
|
||||
|
|
|
@ -132,18 +132,11 @@ public final class ByteBlockPool implements Accountable {
|
|||
}
|
||||
|
||||
/**
|
||||
* Resets the pool to its initial state, reusing the first buffer and filling all buffers with
|
||||
* {@code 0} bytes before they are reused or passed to {@link
|
||||
* Allocator#recycleByteBlocks(byte[][], int, int)}. Calling {@link ByteBlockPool#nextBuffer()} is
|
||||
* not needed after reset.
|
||||
*/
|
||||
public void reset() {
|
||||
reset(true, true);
|
||||
}
|
||||
|
||||
/**
|
||||
* Expert: Resets the pool to its initial state, while reusing the first buffer. Calling {@link
|
||||
* ByteBlockPool#nextBuffer()} is not needed after reset.
|
||||
* Expert: Resets the pool to its initial state, while optionally reusing the first buffer.
|
||||
* Buffers that are not reused are reclaimed by {@link Allocator#recycleByteBlocks(byte[][], int,
|
||||
* int)}. Buffers can be filled with zeros before recycling them. This is useful if a slice pool
|
||||
* works on top of this byte pool and relies on the buffers being filled with zeros to find the
|
||||
* non-zero end of slices.
|
||||
*
|
||||
* @param zeroFillBuffers if {@code true} the buffers are filled with {@code 0}. This should be
|
||||
* set to {@code true} if this pool is used with slices.
|
||||
|
@ -188,7 +181,8 @@ public final class ByteBlockPool implements Accountable {
|
|||
/**
|
||||
* Allocates a new buffer and advances the pool to it. This method should be called once after the
|
||||
* constructor to initialize the pool. In contrast to the constructor, a {@link
|
||||
* ByteBlockPool#reset()} call will advance the pool to its first buffer immediately.
|
||||
* ByteBlockPool#reset(boolean, boolean)} call will advance the pool to its first buffer
|
||||
* immediately.
|
||||
*/
|
||||
public void nextBuffer() {
|
||||
if (1 + bufferUpto == buffers.length) {
|
||||
|
|
|
@ -94,15 +94,11 @@ public class IntBlockPool {
|
|||
}
|
||||
|
||||
/**
|
||||
* Resets the pool to its initial state reusing the first buffer. Calling {@link
|
||||
* IntBlockPool#nextBuffer()} is not needed after reset.
|
||||
*/
|
||||
public void reset() {
|
||||
this.reset(true, true);
|
||||
}
|
||||
|
||||
/**
|
||||
* Expert: Resets the pool to its initial state reusing the first buffer.
|
||||
* Expert: Resets the pool to its initial state, while optionally reusing the first buffer.
|
||||
* Buffers that are not reused are reclaimed by {@link
|
||||
* ByteBlockPool.Allocator#recycleByteBlocks(byte[][], int, int)}. Buffers can be filled with
|
||||
* zeros before recycling them. This is useful if a slice pool works on top of this int pool and
|
||||
* relies on the buffers being filled with zeros to find the non-zero end of slices.
|
||||
*
|
||||
* @param zeroFillBuffers if <code>true</code> the buffers are filled with <code>0</code>.
|
||||
* @param reuseFirst if <code>true</code> the first buffer will be reused and calling {@link
|
||||
|
@ -145,8 +141,8 @@ public class IntBlockPool {
|
|||
|
||||
/**
|
||||
* Advances the pool to its next buffer. This method should be called once after the constructor
|
||||
* to initialize the pool. In contrast to the constructor a {@link IntBlockPool#reset()} call will
|
||||
* advance the pool to its first buffer immediately.
|
||||
* to initialize the pool. In contrast to the constructor a {@link IntBlockPool#reset(boolean,
|
||||
* boolean)} call will advance the pool to its first buffer immediately.
|
||||
*/
|
||||
public void nextBuffer() {
|
||||
if (1 + bufferUpto == buffers.length) {
|
||||
|
|
|
@ -21,6 +21,7 @@ import org.apache.lucene.tests.util.TestUtil;
|
|||
import org.apache.lucene.util.BitUtil;
|
||||
import org.apache.lucene.util.ByteBlockPool;
|
||||
import org.apache.lucene.util.Counter;
|
||||
import org.apache.lucene.util.RecyclingByteBlockAllocator;
|
||||
|
||||
public class TestByteSlicePool extends LuceneTestCase {
|
||||
public void testAllocKnownSizeSlice() {
|
||||
|
@ -223,57 +224,66 @@ public class TestByteSlicePool extends LuceneTestCase {
|
|||
* that we read back the same data we wrote.
|
||||
*/
|
||||
public void testRandomInterleavedSlices() {
|
||||
ByteBlockPool blockPool = new ByteBlockPool(new ByteBlockPool.DirectAllocator());
|
||||
ByteBlockPool blockPool = new ByteBlockPool(new RecyclingByteBlockAllocator());
|
||||
ByteSlicePool slicePool = new ByteSlicePool(blockPool);
|
||||
|
||||
int n = TestUtil.nextInt(random(), 2, 3); // 2 or 3 writers and readers
|
||||
SliceWriter[] sliceWriters = new SliceWriter[n];
|
||||
SliceReader[] sliceReaders = new SliceReader[n];
|
||||
int nIterations =
|
||||
TestUtil.nextInt(random(), 1, 3); // 1-3 iterations with buffer resets in between
|
||||
for (int iter = 0; iter < nIterations; iter++) {
|
||||
int n = TestUtil.nextInt(random(), 2, 3); // 2 or 3 writers and readers
|
||||
SliceWriter[] sliceWriters = new SliceWriter[n];
|
||||
SliceReader[] sliceReaders = new SliceReader[n];
|
||||
|
||||
// Init slice writers
|
||||
for (int i = 0; i < n; i++) {
|
||||
sliceWriters[i] = new SliceWriter(slicePool);
|
||||
}
|
||||
|
||||
// Write slices
|
||||
while (true) {
|
||||
int i = random().nextInt(n);
|
||||
boolean succeeded = sliceWriters[i].writeSlice();
|
||||
if (succeeded == false) {
|
||||
for (int j = 0; j < n; j++) {
|
||||
while (sliceWriters[j].writeSlice())
|
||||
;
|
||||
}
|
||||
break;
|
||||
// Init slice writers
|
||||
for (int i = 0; i < n; i++) {
|
||||
sliceWriters[i] = new SliceWriter(slicePool);
|
||||
}
|
||||
}
|
||||
|
||||
// Init slice readers
|
||||
for (int i = 0; i < n; i++) {
|
||||
sliceReaders[i] =
|
||||
new SliceReader(
|
||||
slicePool,
|
||||
sliceWriters[i].size,
|
||||
sliceWriters[i].firstSliceOffset,
|
||||
sliceWriters[i].firstSlice);
|
||||
}
|
||||
|
||||
// Read slices
|
||||
while (true) {
|
||||
int i = random().nextInt(n);
|
||||
boolean succeeded = sliceReaders[i].readSlice();
|
||||
if (succeeded == false) {
|
||||
for (int j = 0; j < n; j++) {
|
||||
while (sliceReaders[j].readSlice())
|
||||
;
|
||||
// Write slices
|
||||
while (true) {
|
||||
int i = random().nextInt(n);
|
||||
boolean succeeded = sliceWriters[i].writeSlice();
|
||||
if (succeeded == false) {
|
||||
for (int j = 0; j < n; j++) {
|
||||
while (sliceWriters[j].writeSlice())
|
||||
;
|
||||
}
|
||||
break;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Compare written data with read data
|
||||
for (int i = 0; i < n; i++) {
|
||||
assertArrayEquals(sliceWriters[i].randomData, sliceReaders[i].readData);
|
||||
// Init slice readers
|
||||
for (int i = 0; i < n; i++) {
|
||||
sliceReaders[i] =
|
||||
new SliceReader(
|
||||
slicePool,
|
||||
sliceWriters[i].size,
|
||||
sliceWriters[i].firstSliceOffset,
|
||||
sliceWriters[i].firstSlice);
|
||||
}
|
||||
|
||||
// Read slices
|
||||
while (true) {
|
||||
int i = random().nextInt(n);
|
||||
boolean succeeded = sliceReaders[i].readSlice();
|
||||
if (succeeded == false) {
|
||||
for (int j = 0; j < n; j++) {
|
||||
while (sliceReaders[j].readSlice())
|
||||
;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Compare written data with read data
|
||||
for (int i = 0; i < n; i++) {
|
||||
assertArrayEquals(sliceWriters[i].randomData, sliceReaders[i].readData);
|
||||
}
|
||||
|
||||
// We don't rely on the buffers being filled with zeros because the SliceWriter keeps the
|
||||
// slice length as state, but ByteSlicePool.allocKnownSizeSlice asserts on zeros in the
|
||||
// buffer.
|
||||
blockPool.reset(true, random().nextBoolean());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -50,7 +50,7 @@ public class TestIntBlockPool extends LuceneTestCase {
|
|||
|
||||
// Reset and fill with zeros, then check there is no data left
|
||||
pool.intUpto = count;
|
||||
pool.reset();
|
||||
pool.reset(true, true);
|
||||
for (int i = 0; i < count; i++) {
|
||||
assertEquals(0, pool.buffers[0][i]);
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue