Make `IndexInput#prefetch` take an offset. (#13363)

This makes `IndexInput#prefetch` take an offset instead of being relative to
the current position. This avoids requiring callers to seek only to call
`prefetch()`.
This commit is contained in:
Adrien Grand 2024-05-14 18:43:21 +02:00 committed by GitHub
parent 838b23ebed
commit 46f1f95ceb
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 22 additions and 26 deletions

View File

@ -199,7 +199,8 @@ public abstract class IndexInput extends DataInput implements Closeable {
*
* <p>The default implementation is a no-op.
*
* @param offset start offset
* @param length the number of bytes to prefetch
*/
public void prefetch(long length) throws IOException {}
public void prefetch(long offset, long length) throws IOException {}
}

View File

@ -313,30 +313,29 @@ abstract class MemorySegmentIndexInput extends IndexInput implements RandomAcces
}
@Override
public void prefetch(long length) throws IOException {
public void prefetch(long offset, long length) throws IOException {
ensureOpen();
Objects.checkFromIndexSize(getFilePointer(), length, length());
Objects.checkFromIndexSize(offset, length, length());
if (NATIVE_ACCESS.isEmpty()) {
return;
}
final NativeAccess nativeAccess = NATIVE_ACCESS.get();
// If at the boundary between two chunks, move to the next one.
seek(getFilePointer());
try {
final MemorySegment segment = segments[(int) (offset >> chunkSizePower)];
offset &= chunkSizeMask;
// Compute the intersection of the current segment and the region that should be prefetched.
long offset = curPosition;
if (offset + length > curSegment.byteSize()) {
if (offset + length > segment.byteSize()) {
// Only prefetch bytes that are stored in the current segment. There may be bytes on the
// next segment but this case is rare enough that we don't try to optimize it and keep
// things simple instead.
length = curSegment.byteSize() - curPosition;
length = segment.byteSize() - offset;
}
// Now align offset with the page size, this is required for madvise.
// Compute the offset of the current position in the OS's page.
final long offsetInPage = (curSegment.address() + offset) % nativeAccess.getPageSize();
final long offsetInPage = (segment.address() + offset) % nativeAccess.getPageSize();
offset -= offsetInPage;
length += offsetInPage;
if (offset < 0) {
@ -344,7 +343,7 @@ abstract class MemorySegmentIndexInput extends IndexInput implements RandomAcces
return;
}
final MemorySegment prefetchSlice = curSegment.asSlice(offset, length);
final MemorySegment prefetchSlice = segment.asSlice(offset, length);
nativeAccess.madviseWillNeed(prefetchSlice);
} catch (
@SuppressWarnings("unused")

View File

@ -1540,36 +1540,32 @@ public abstract class BaseDirectoryTestCase extends LuceneTestCase {
in = orig.slice("slice", startOffset, totalLength - startOffset);
}
for (int i = 0; i < 10_000; ++i) {
final int startPointer = (int) in.getFilePointer();
assertTrue(startPointer < in.length());
int offset = TestUtil.nextInt(random(), 0, (int) in.length() - 1);
if (random().nextBoolean()) {
final long prefetchLength = TestUtil.nextLong(random(), 1, in.length() - startPointer);
in.prefetch(prefetchLength);
final long prefetchLength = TestUtil.nextLong(random(), 1, in.length() - offset);
in.prefetch(offset, prefetchLength);
}
assertEquals(startPointer, in.getFilePointer());
in.seek(offset);
assertEquals(offset, in.getFilePointer());
switch (random().nextInt(100)) {
case 0:
assertEquals(arr[startOffset + startPointer], in.readByte());
assertEquals(arr[startOffset + offset], in.readByte());
break;
case 1:
if (in.length() - startPointer >= Long.BYTES) {
if (in.length() - offset >= Long.BYTES) {
assertEquals(
(long) BitUtil.VH_LE_LONG.get(arr, startOffset + startPointer), in.readLong());
(long) BitUtil.VH_LE_LONG.get(arr, startOffset + offset), in.readLong());
}
break;
default:
final int readLength =
TestUtil.nextInt(
random(), 1, (int) Math.min(temp.length, in.length() - startPointer));
TestUtil.nextInt(random(), 1, (int) Math.min(temp.length, in.length() - offset));
in.readBytes(temp, 0, readLength);
assertArrayEquals(
ArrayUtil.copyOfSubArray(
arr, startOffset + startPointer, startOffset + startPointer + readLength),
arr, startOffset + offset, startOffset + offset + readLength),
ArrayUtil.copyOfSubArray(temp, 0, readLength));
}
if (in.getFilePointer() == in.length() || random().nextBoolean()) {
in.seek(TestUtil.nextInt(random(), 0, (int) in.length() - 1));
}
}
}
}

View File

@ -131,9 +131,9 @@ public class MockIndexInputWrapper extends FilterIndexInput {
}
@Override
public void prefetch(long length) throws IOException {
public void prefetch(long offset, long length) throws IOException {
ensureOpen();
in.prefetch(length);
in.prefetch(offset, length);
}
@Override