mirror of https://github.com/apache/lucene.git
Make `IndexInput#prefetch` take an offset. (#13363)
This makes `IndexInput#prefetch` take an offset instead of being relative to the current position. This avoids requiring callers to seek only to call `prefetch()`.
This commit is contained in:
parent
838b23ebed
commit
46f1f95ceb
|
@ -199,7 +199,8 @@ public abstract class IndexInput extends DataInput implements Closeable {
|
|||
*
|
||||
* <p>The default implementation is a no-op.
|
||||
*
|
||||
* @param offset start offset
|
||||
* @param length the number of bytes to prefetch
|
||||
*/
|
||||
public void prefetch(long length) throws IOException {}
|
||||
public void prefetch(long offset, long length) throws IOException {}
|
||||
}
|
||||
|
|
|
@ -313,30 +313,29 @@ abstract class MemorySegmentIndexInput extends IndexInput implements RandomAcces
|
|||
}
|
||||
|
||||
@Override
|
||||
public void prefetch(long length) throws IOException {
|
||||
public void prefetch(long offset, long length) throws IOException {
|
||||
ensureOpen();
|
||||
|
||||
Objects.checkFromIndexSize(getFilePointer(), length, length());
|
||||
Objects.checkFromIndexSize(offset, length, length());
|
||||
|
||||
if (NATIVE_ACCESS.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
final NativeAccess nativeAccess = NATIVE_ACCESS.get();
|
||||
|
||||
// If at the boundary between two chunks, move to the next one.
|
||||
seek(getFilePointer());
|
||||
try {
|
||||
final MemorySegment segment = segments[(int) (offset >> chunkSizePower)];
|
||||
offset &= chunkSizeMask;
|
||||
// Compute the intersection of the current segment and the region that should be prefetched.
|
||||
long offset = curPosition;
|
||||
if (offset + length > curSegment.byteSize()) {
|
||||
if (offset + length > segment.byteSize()) {
|
||||
// Only prefetch bytes that are stored in the current segment. There may be bytes on the
|
||||
// next segment but this case is rare enough that we don't try to optimize it and keep
|
||||
// things simple instead.
|
||||
length = curSegment.byteSize() - curPosition;
|
||||
length = segment.byteSize() - offset;
|
||||
}
|
||||
// Now align offset with the page size, this is required for madvise.
|
||||
// Compute the offset of the current position in the OS's page.
|
||||
final long offsetInPage = (curSegment.address() + offset) % nativeAccess.getPageSize();
|
||||
final long offsetInPage = (segment.address() + offset) % nativeAccess.getPageSize();
|
||||
offset -= offsetInPage;
|
||||
length += offsetInPage;
|
||||
if (offset < 0) {
|
||||
|
@ -344,7 +343,7 @@ abstract class MemorySegmentIndexInput extends IndexInput implements RandomAcces
|
|||
return;
|
||||
}
|
||||
|
||||
final MemorySegment prefetchSlice = curSegment.asSlice(offset, length);
|
||||
final MemorySegment prefetchSlice = segment.asSlice(offset, length);
|
||||
nativeAccess.madviseWillNeed(prefetchSlice);
|
||||
} catch (
|
||||
@SuppressWarnings("unused")
|
||||
|
|
|
@ -1540,36 +1540,32 @@ public abstract class BaseDirectoryTestCase extends LuceneTestCase {
|
|||
in = orig.slice("slice", startOffset, totalLength - startOffset);
|
||||
}
|
||||
for (int i = 0; i < 10_000; ++i) {
|
||||
final int startPointer = (int) in.getFilePointer();
|
||||
assertTrue(startPointer < in.length());
|
||||
int offset = TestUtil.nextInt(random(), 0, (int) in.length() - 1);
|
||||
if (random().nextBoolean()) {
|
||||
final long prefetchLength = TestUtil.nextLong(random(), 1, in.length() - startPointer);
|
||||
in.prefetch(prefetchLength);
|
||||
final long prefetchLength = TestUtil.nextLong(random(), 1, in.length() - offset);
|
||||
in.prefetch(offset, prefetchLength);
|
||||
}
|
||||
assertEquals(startPointer, in.getFilePointer());
|
||||
in.seek(offset);
|
||||
assertEquals(offset, in.getFilePointer());
|
||||
switch (random().nextInt(100)) {
|
||||
case 0:
|
||||
assertEquals(arr[startOffset + startPointer], in.readByte());
|
||||
assertEquals(arr[startOffset + offset], in.readByte());
|
||||
break;
|
||||
case 1:
|
||||
if (in.length() - startPointer >= Long.BYTES) {
|
||||
if (in.length() - offset >= Long.BYTES) {
|
||||
assertEquals(
|
||||
(long) BitUtil.VH_LE_LONG.get(arr, startOffset + startPointer), in.readLong());
|
||||
(long) BitUtil.VH_LE_LONG.get(arr, startOffset + offset), in.readLong());
|
||||
}
|
||||
break;
|
||||
default:
|
||||
final int readLength =
|
||||
TestUtil.nextInt(
|
||||
random(), 1, (int) Math.min(temp.length, in.length() - startPointer));
|
||||
TestUtil.nextInt(random(), 1, (int) Math.min(temp.length, in.length() - offset));
|
||||
in.readBytes(temp, 0, readLength);
|
||||
assertArrayEquals(
|
||||
ArrayUtil.copyOfSubArray(
|
||||
arr, startOffset + startPointer, startOffset + startPointer + readLength),
|
||||
arr, startOffset + offset, startOffset + offset + readLength),
|
||||
ArrayUtil.copyOfSubArray(temp, 0, readLength));
|
||||
}
|
||||
if (in.getFilePointer() == in.length() || random().nextBoolean()) {
|
||||
in.seek(TestUtil.nextInt(random(), 0, (int) in.length() - 1));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -131,9 +131,9 @@ public class MockIndexInputWrapper extends FilterIndexInput {
|
|||
}
|
||||
|
||||
@Override
|
||||
public void prefetch(long length) throws IOException {
|
||||
public void prefetch(long offset, long length) throws IOException {
|
||||
ensureOpen();
|
||||
in.prefetch(length);
|
||||
in.prefetch(offset, length);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
Loading…
Reference in New Issue