Add IndexInput isLoaded (#13998)

This commit adds IndexInput::isLoaded to help determine if the contents of an input is resident in physical memory.

The intent of this new method is to help build inspection and diagnostic infrastructure on top.

lucene/core/src/java/org/apache/lucene/store/IndexInput.java

@@ -18,6 +18,7 @@ package org.apache.lucene.store;
 
 import java.io.Closeable;
 import java.io.IOException;
+import java.util.Optional;
 import org.apache.lucene.codecs.CompoundFormat;
 
 /**
@@ -234,4 +235,19 @@ public abstract class IndexInput extends DataInput implements Closeable {
    * <p>The default implementation is a no-op.
    */
   public void updateReadAdvice(ReadAdvice readAdvice) throws IOException {}
+
+  /**
+   * Returns a hint whether all the contents of this input are resident in physical memory. It's a
+   * hint because the operating system may have paged out some of the data by the time this method
+   * returns. If the optional is true, then it's likely that the contents of this input are resident
+   * in physical memory. A value of false does not imply that the contents are not resident in
+   * physical memory. An empty optional is returned if it is not possible to determine.
+   *
+   * <p>This runs in linear time with the {@link #length()} of this input / page size.
+   *
+   * <p>The default implementation returns an empty optional.
+   */
+  public Optional<Boolean> isLoaded() {
+    return Optional.empty();
+  }
 }

lucene/core/src/java/org/apache/lucene/store/RandomAccessInput.java

@@ -17,6 +17,7 @@
 package org.apache.lucene.store;
 
 import java.io.IOException;
+import java.util.Optional;
 import org.apache.lucene.util.BitUtil; // javadocs
 
 /**
@@ -77,4 +78,13 @@ public interface RandomAccessInput {
    * @see IndexInput#prefetch
    */
   default void prefetch(long offset, long length) throws IOException {}
+
+  /**
+   * Returns a hint whether all the contents of this input are resident in physical memory.
+   *
+   * @see IndexInput#isLoaded()
+   */
+  default Optional<Boolean> isLoaded() {
+    return Optional.empty();
+  }
 }

lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInput.java

@@ -420,6 +420,16 @@ abstract class MemorySegmentIndexInput extends IndexInput
     }
   }
 
+  @Override
+  public Optional<Boolean> isLoaded() {
+    for (MemorySegment seg : segments) {
+      if (seg.isLoaded() == false) {
+        return Optional.of(Boolean.FALSE);
+      }
+    }
+    return Optional.of(Boolean.TRUE);
+  }
+
   @Override
   public byte readByte(long pos) throws IOException {
     try {

lucene/test-framework/src/java/org/apache/lucene/tests/store/BaseDirectoryTestCase.java

@@ -51,9 +51,11 @@ import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.store.ChecksumIndexInput;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.store.FilterDirectory;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.store.MMapDirectory;
 import org.apache.lucene.store.RandomAccessInput;
 import org.apache.lucene.store.ReadAdvice;
 import org.apache.lucene.tests.mockfile.ExtrasFS;
@@ -1636,4 +1638,44 @@ public abstract class BaseDirectoryTestCase extends LuceneTestCase {
       }
     }
   }
+
+  public void testIsLoaded() throws IOException {
+    testIsLoaded(0);
+  }
+
+  public void testIsLoadedOnSlice() throws IOException {
+    testIsLoaded(TestUtil.nextInt(random(), 1, 1024));
+  }
+
+  private void testIsLoaded(int startOffset) throws IOException {
+    try (Directory dir = getDirectory(createTempDir())) {
+      if (FilterDirectory.unwrap(dir) instanceof MMapDirectory mMapDirectory) {
+        mMapDirectory.setPreload(MMapDirectory.ALL_FILES);
+      }
+      final int totalLength = startOffset + TestUtil.nextInt(random(), 16384, 65536);
+      byte[] arr = new byte[totalLength];
+      random().nextBytes(arr);
+      try (IndexOutput out = dir.createOutput("temp.bin", IOContext.DEFAULT)) {
+        out.writeBytes(arr, arr.length);
+      }
+
+      try (IndexInput orig = dir.openInput("temp.bin", IOContext.DEFAULT)) {
+        IndexInput in;
+        if (startOffset == 0) {
+          in = orig.clone();
+        } else {
+          in = orig.slice("slice", startOffset, totalLength - startOffset);
+        }
+        var loaded = in.isLoaded();
+        if (FilterDirectory.unwrap(dir) instanceof MMapDirectory
+            // direct IO wraps MMap but does not support isLoaded
+            && !(dir.getClass().getName().contains("DirectIO"))) {
+          assertTrue(loaded.isPresent());
+          assertTrue(loaded.get());
+        } else {
+          assertFalse(loaded.isPresent());
+        }
+      }
+    }
+  }
 }

lucene/test-framework/src/java/org/apache/lucene/tests/store/MockIndexInputWrapper.java

@@ -19,6 +19,7 @@ package org.apache.lucene.tests.store;
 import java.io.Closeable;
 import java.io.IOException;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import org.apache.lucene.internal.tests.TestSecrets;
 import org.apache.lucene.store.FilterIndexInput;
@@ -184,6 +185,13 @@ public class MockIndexInputWrapper extends FilterIndexInput {
     in.prefetch(offset, length);
   }
 
+  @Override
+  public Optional<Boolean> isLoaded() {
+    ensureOpen();
+    ensureAccessible();
+    return in.isLoaded();
+  }
+
   @Override
   public void updateReadAdvice(ReadAdvice readAdvice) throws IOException {
     ensureOpen();

lucene/test-framework/src/java/org/apache/lucene/tests/store/SerialIOCountingDirectory.java

@@ -17,6 +17,7 @@
 package org.apache.lucene.tests.store;
 
 import java.io.IOException;
+import java.util.Optional;
 import java.util.concurrent.atomic.LongAdder;
 import org.apache.lucene.internal.hppc.LongHashSet;
 import org.apache.lucene.store.ChecksumIndexInput;
@@ -206,5 +207,10 @@ public class SerialIOCountingDirectory extends FilterDirectory {
       IndexInput clone = in.clone();
       return new SerializedIOCountingIndexInput(clone, readAdvice, sliceOffset, sliceLength);
     }
+
+    @Override
+    public Optional<Boolean> isLoaded() {
+      return in.isLoaded();
+    }
   }
 }