Enable MemorySegment in MMapDirectory for Java 22+ and Vectorization (incubation) for exact Java 22 (#12706)

gradle/generation/extract-jdk-apis.gradle

@@ -20,7 +20,7 @@ def resources = scriptResources(buildscript)
 configure(rootProject) {
   ext {
     // also change this in extractor tool: ExtractForeignAPI
-    vectorIncubatorJavaVersions = [ JavaVersion.VERSION_20, JavaVersion.VERSION_21 ] as Set
+    vectorIncubatorJavaVersions = [ JavaVersion.VERSION_20, JavaVersion.VERSION_21, JavaVersion.VERSION_22 ] as Set
   }
 }
 

lucene/CHANGES.txt

@@ -37,6 +37,17 @@ New Features
 * GITHUB#12336: Index additional data per facet label in the taxonomy. (Shai Erera, Egor Potemkin, Mike McCandless,
   Stefan Vodita)
 
+* GITHUB#12706: Add support for the final release of Java foreign memory API in Java 22 (and later).
+  Lucene's MMapDirectory will now mmap Lucene indexes in chunks of 16 GiB (instead of 1 GiB) starting
+  from Java 19. Indexes closed while queries are running can no longer crash the JVM.
+  Support for vectorized implementations of VectorUtil based on jdk.incubator.vector APIs was added
+  for eactly Java 22. Therefore, applications started with command line parameter
+  "java --add-modules jdk.incubator.vector" will automatically use the new vectorized implementations
+  if running on a supported platform (Java 20/21/22 on x86 CPUs with AVX2 or later or ARM NEON CPUs).
+  This is an opt-in feature and requires explicit Java command line flag! When enabled, Lucene logs
+  a notice using java.util.logging. Please test thoroughly and report bugs/slowness to Lucene's mailing
+  list.  (Uwe Schindler, Chris Hegarty)
+
 Improvements
 ---------------------
 

lucene/core/src/java/org/apache/lucene/internal/vectorization/VectorizationProvider.java

@@ -103,7 +103,7 @@ public abstract class VectorizationProvider {
   // visible for tests
   static VectorizationProvider lookup(boolean testMode) {
     final int runtimeVersion = Runtime.version().feature();
-    if (runtimeVersion >= 20 && runtimeVersion <= 21) {
+    if (runtimeVersion >= 20 && runtimeVersion <= 22) {
       // is locale sane (only buggy in Java 20)
       if (isAffectedByJDK8301190()) {
         LOG.warning(
@@ -169,9 +169,9 @@ public abstract class VectorizationProvider {
       } catch (ClassNotFoundException cnfe) {
         throw new LinkageError("PanamaVectorizationProvider is missing in Lucene JAR file", cnfe);
       }
-    } else if (runtimeVersion >= 22) {
+    } else if (runtimeVersion >= 23) {
       LOG.warning(
-          "You are running with Java 22 or later. To make full use of the Vector API, please update Apache Lucene.");
+          "You are running with Java 23 or later. To make full use of the Vector API, please update Apache Lucene.");
     } else if (lookupVectorModule().isPresent()) {
       LOG.warning(
           "Java vector incubator module was enabled by command line flags, but your Java version is too old: "

lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java

@@ -417,7 +417,7 @@ public class MMapDirectory extends FSDirectory {
     }
     final var lookup = MethodHandles.lookup();
     final int runtimeVersion = Runtime.version().feature();
-    if (runtimeVersion >= 19 && runtimeVersion <= 21) {
+    if (runtimeVersion >= 19) {
       try {
         final var cls = lookup.findClass("org.apache.lucene.store.MemorySegmentIndexInputProvider");
         // we use method handles, so we do not need to deal with setAccessible as we have private
@@ -437,9 +437,6 @@ public class MMapDirectory extends FSDirectory {
         throw new LinkageError(
             "MemorySegmentIndexInputProvider is missing in Lucene JAR file", cnfe);
       }
-    } else if (runtimeVersion >= 22) {
-      LOG.warning(
-          "You are running with Java 22 or later. To make full use of MMapDirectory, please update Apache Lucene.");
     }
     return new MappedByteBufferIndexInputProvider();
   }

lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInput.java

@@ -108,10 +108,16 @@ abstract class MemorySegmentIndexInput extends IndexInput implements RandomAcces
     if (this.curSegment == null) {
       return new AlreadyClosedException("Already closed: " + this);
     }
-    // ISE can be thrown by MemorySegment and contains "closed" in message:
+    // in Java 22 or later we can check the isAlive status of all segments
+    // (see https://bugs.openjdk.org/browse/JDK-8310644):
+    if (Arrays.stream(segments).allMatch(s -> s.scope().isAlive()) == false) {
+      return new AlreadyClosedException("Already closed: " + this);
+    }
+    // fallback for Java 21: ISE can be thrown by MemorySegment and contains "closed" in message:
     if (e instanceof IllegalStateException
         && e.getMessage() != null
         && e.getMessage().contains("closed")) {
+      // the check is on message only, so preserve original cause for debugging:
       return new AlreadyClosedException("Already closed: " + this, e);
     }
     // otherwise rethrow unmodified NPE/ISE (as it possibly a bug with passing a null parameter to

lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInputProvider.java

@@ -33,7 +33,7 @@ final class MemorySegmentIndexInputProvider implements MMapDirectory.MMapIndexIn
   public MemorySegmentIndexInputProvider() {
     var log = Logger.getLogger(getClass().getName());
     log.info(
-        "Using MemorySegmentIndexInput with Java 21; to disable start with -D"
+        "Using MemorySegmentIndexInput with Java 21 or later; to disable start with -D"
             + MMapDirectory.ENABLE_MEMORY_SEGMENTS_SYSPROP
             + "=false");
   }

lucene/core/src/test/org/apache/lucene/store/TestMMapDirectory.java

@@ -48,9 +48,9 @@ public class TestMMapDirectory extends BaseDirectoryTestCase {
 
   public void testCorrectImplementation() {
     final int runtimeVersion = Runtime.version().feature();
-    if (runtimeVersion >= 19 && runtimeVersion <= 21) {
+    if (runtimeVersion >= 19) {
       assertTrue(
-          "on Java 19, 20, and 21 we should use MemorySegmentIndexInputProvider to create mmap IndexInputs",
+          "on Java 19 or later we should use MemorySegmentIndexInputProvider to create mmap IndexInputs",
           isMemorySegmentImpl());
     } else {
       assertSame(MappedByteBufferIndexInputProvider.class, MMapDirectory.PROVIDER.getClass());