[Minor] Improvements to slice pools (#12795)

1. Remove rest method used only in tests.
2. Update Javadocs.
3. Make interleaved slices test a bit more evil by adding pool resets.
This commit is contained in:
Stefan Vodita 2023-11-13 14:42:01 +00:00 committed by GitHub
parent 14196cfcf7
commit a70432c110
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 73 additions and 73 deletions

View File

@ -22,11 +22,11 @@ import org.apache.lucene.store.DataOutput;
import org.apache.lucene.util.BitUtil;
import org.apache.lucene.util.ByteBlockPool;
/* IndexInput that knows how to read the byte slices written
* by Posting and PostingVector. We read the bytes in
* each slice until we hit the end of that slice at which
* point we read the forwarding address of the next slice
* and then jump to it.*/
/**
* IndexInput that knows how to read the byte slices written by Posting and PostingVector. We read
* the bytes in each slice until we hit the end of that slice at which point we read the forwarding
* address of the next slice and then jump to it.
*/
final class ByteSliceReader extends DataInput {
ByteBlockPool pool;
int bufferUpto;

View File

@ -132,18 +132,11 @@ public final class ByteBlockPool implements Accountable {
}
/**
* Resets the pool to its initial state, reusing the first buffer and filling all buffers with
* {@code 0} bytes before they are reused or passed to {@link
* Allocator#recycleByteBlocks(byte[][], int, int)}. Calling {@link ByteBlockPool#nextBuffer()} is
* not needed after reset.
*/
public void reset() {
reset(true, true);
}
/**
* Expert: Resets the pool to its initial state, while reusing the first buffer. Calling {@link
* ByteBlockPool#nextBuffer()} is not needed after reset.
* Expert: Resets the pool to its initial state, while optionally reusing the first buffer.
* Buffers that are not reused are reclaimed by {@link Allocator#recycleByteBlocks(byte[][], int,
* int)}. Buffers can be filled with zeros before recycling them. This is useful if a slice pool
* works on top of this byte pool and relies on the buffers being filled with zeros to find the
* non-zero end of slices.
*
* @param zeroFillBuffers if {@code true} the buffers are filled with {@code 0}. This should be
* set to {@code true} if this pool is used with slices.
@ -188,7 +181,8 @@ public final class ByteBlockPool implements Accountable {
/**
* Allocates a new buffer and advances the pool to it. This method should be called once after the
* constructor to initialize the pool. In contrast to the constructor, a {@link
* ByteBlockPool#reset()} call will advance the pool to its first buffer immediately.
* ByteBlockPool#reset(boolean, boolean)} call will advance the pool to its first buffer
* immediately.
*/
public void nextBuffer() {
if (1 + bufferUpto == buffers.length) {

View File

@ -94,15 +94,11 @@ public class IntBlockPool {
}
/**
* Resets the pool to its initial state reusing the first buffer. Calling {@link
* IntBlockPool#nextBuffer()} is not needed after reset.
*/
public void reset() {
this.reset(true, true);
}
/**
* Expert: Resets the pool to its initial state reusing the first buffer.
* Expert: Resets the pool to its initial state, while optionally reusing the first buffer.
* Buffers that are not reused are reclaimed by {@link
* ByteBlockPool.Allocator#recycleByteBlocks(byte[][], int, int)}. Buffers can be filled with
* zeros before recycling them. This is useful if a slice pool works on top of this int pool and
* relies on the buffers being filled with zeros to find the non-zero end of slices.
*
* @param zeroFillBuffers if <code>true</code> the buffers are filled with <code>0</code>.
* @param reuseFirst if <code>true</code> the first buffer will be reused and calling {@link
@ -145,8 +141,8 @@ public class IntBlockPool {
/**
* Advances the pool to its next buffer. This method should be called once after the constructor
* to initialize the pool. In contrast to the constructor a {@link IntBlockPool#reset()} call will
* advance the pool to its first buffer immediately.
* to initialize the pool. In contrast to the constructor a {@link IntBlockPool#reset(boolean,
* boolean)} call will advance the pool to its first buffer immediately.
*/
public void nextBuffer() {
if (1 + bufferUpto == buffers.length) {

View File

@ -21,6 +21,7 @@ import org.apache.lucene.tests.util.TestUtil;
import org.apache.lucene.util.BitUtil;
import org.apache.lucene.util.ByteBlockPool;
import org.apache.lucene.util.Counter;
import org.apache.lucene.util.RecyclingByteBlockAllocator;
public class TestByteSlicePool extends LuceneTestCase {
public void testAllocKnownSizeSlice() {
@ -223,57 +224,66 @@ public class TestByteSlicePool extends LuceneTestCase {
* that we read back the same data we wrote.
*/
public void testRandomInterleavedSlices() {
ByteBlockPool blockPool = new ByteBlockPool(new ByteBlockPool.DirectAllocator());
ByteBlockPool blockPool = new ByteBlockPool(new RecyclingByteBlockAllocator());
ByteSlicePool slicePool = new ByteSlicePool(blockPool);
int n = TestUtil.nextInt(random(), 2, 3); // 2 or 3 writers and readers
SliceWriter[] sliceWriters = new SliceWriter[n];
SliceReader[] sliceReaders = new SliceReader[n];
int nIterations =
TestUtil.nextInt(random(), 1, 3); // 1-3 iterations with buffer resets in between
for (int iter = 0; iter < nIterations; iter++) {
int n = TestUtil.nextInt(random(), 2, 3); // 2 or 3 writers and readers
SliceWriter[] sliceWriters = new SliceWriter[n];
SliceReader[] sliceReaders = new SliceReader[n];
// Init slice writers
for (int i = 0; i < n; i++) {
sliceWriters[i] = new SliceWriter(slicePool);
}
// Write slices
while (true) {
int i = random().nextInt(n);
boolean succeeded = sliceWriters[i].writeSlice();
if (succeeded == false) {
for (int j = 0; j < n; j++) {
while (sliceWriters[j].writeSlice())
;
}
break;
// Init slice writers
for (int i = 0; i < n; i++) {
sliceWriters[i] = new SliceWriter(slicePool);
}
}
// Init slice readers
for (int i = 0; i < n; i++) {
sliceReaders[i] =
new SliceReader(
slicePool,
sliceWriters[i].size,
sliceWriters[i].firstSliceOffset,
sliceWriters[i].firstSlice);
}
// Read slices
while (true) {
int i = random().nextInt(n);
boolean succeeded = sliceReaders[i].readSlice();
if (succeeded == false) {
for (int j = 0; j < n; j++) {
while (sliceReaders[j].readSlice())
;
// Write slices
while (true) {
int i = random().nextInt(n);
boolean succeeded = sliceWriters[i].writeSlice();
if (succeeded == false) {
for (int j = 0; j < n; j++) {
while (sliceWriters[j].writeSlice())
;
}
break;
}
break;
}
}
// Compare written data with read data
for (int i = 0; i < n; i++) {
assertArrayEquals(sliceWriters[i].randomData, sliceReaders[i].readData);
// Init slice readers
for (int i = 0; i < n; i++) {
sliceReaders[i] =
new SliceReader(
slicePool,
sliceWriters[i].size,
sliceWriters[i].firstSliceOffset,
sliceWriters[i].firstSlice);
}
// Read slices
while (true) {
int i = random().nextInt(n);
boolean succeeded = sliceReaders[i].readSlice();
if (succeeded == false) {
for (int j = 0; j < n; j++) {
while (sliceReaders[j].readSlice())
;
}
break;
}
}
// Compare written data with read data
for (int i = 0; i < n; i++) {
assertArrayEquals(sliceWriters[i].randomData, sliceReaders[i].readData);
}
// We don't rely on the buffers being filled with zeros because the SliceWriter keeps the
// slice length as state, but ByteSlicePool.allocKnownSizeSlice asserts on zeros in the
// buffer.
blockPool.reset(true, random().nextBoolean());
}
}
}

View File

@ -50,7 +50,7 @@ public class TestIntBlockPool extends LuceneTestCase {
// Reset and fill with zeros, then check there is no data left
pool.intUpto = count;
pool.reset();
pool.reset(true, true);
for (int i = 0; i < count; i++) {
assertEquals(0, pool.buffers[0][i]);
}