Patch class files for Java 19 code to no longer have the "preview" flag (this enables Java 19 memory segments by default) (#12033)

build.gradle

@@ -119,6 +119,7 @@ apply from: file('gradle/ide/eclipse.gradle')
 // (java, tests)
 apply from: file('gradle/java/folder-layout.gradle')
 apply from: file('gradle/java/javac.gradle')
+apply from: file('gradle/java/memorysegment-mrjar.gradle')
 apply from: file('gradle/testing/defaults-tests.gradle')
 apply from: file('gradle/testing/randomization.gradle')
 apply from: file('gradle/testing/fail-on-no-tests.gradle')

gradle/generation/local-settings.gradle

@@ -101,7 +101,7 @@ tests.jvms=${testsJvms}
 org.gradle.java.installations.auto-download=true
 
 # Set these to enable automatic JVM location discovery.
-org.gradle.java.installations.fromEnv=JAVA17_HOME,JAVA19_HOME
+org.gradle.java.installations.fromEnv=JAVA17_HOME,JAVA19_HOME,JAVA20_HOME,JAVA21_HOME,RUNTIME_JAVA_HOME
 #org.gradle.java.installations.paths=(custom paths)
 
 """, "UTF-8")

gradle/java/javac.gradle

@@ -79,48 +79,3 @@ allprojects {
     }
   }
 }
-
-configure(project(":lucene:core")) {
-  plugins.withType(JavaPlugin) {
-    sourceSets {
-      main19 {
-        java {
-          srcDirs = ['src/java19']
-        }
-      }
-    }
-
-    configurations {
-      // Inherit any dependencies from the main source set.
-      main19Implementation.extendsFrom implementation
-    }
-    
-    dependencies {
-      // We need the main classes to compile our Java 19 pieces.
-      main19Implementation sourceSets.main.output
-    }
-
-    tasks.named('compileMain19Java').configure {
-      javaCompiler = javaToolchains.compilerFor {
-        languageVersion = JavaLanguageVersion.of(19)
-      }
-      
-      // undo alternative JDK support:
-      options.forkOptions.javaHome = null
-      
-      sourceCompatibility = 19
-      targetCompatibility = 19
-      options.compilerArgs += ["--release", 19 as String, "--enable-preview"]
-    }
-    
-    tasks.named('jar').configure {
-      into('META-INF/versions/19') {
-        from sourceSets.main19.output
-      }
-
-      manifest.attributes(
-        'Multi-Release': 'true'
-      )
-    }
-  }
-}

gradle/java/memorysegment-mrjar.gradle

@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Produce an MR-JAR with Java 19 MemorySegment implementation for MMapDirectory
+
+configure(project(":lucene:core")) {
+  plugins.withType(JavaPlugin) {
+    sourceSets {
+      main19 {
+        java {
+          srcDirs = ['src/java19']
+        }
+      }
+    }
+
+    configurations {
+      // Inherit any dependencies from the main source set.
+      main19Implementation.extendsFrom implementation
+    }
+    
+    dependencies {
+      // We need the main classes to compile our Java 19 pieces.
+      main19Implementation sourceSets.main.output
+    }
+
+    def patchClassFiles = { DirectoryProperty destinationDirectory, int expectedMajor ->
+      destinationDirectory.getAsFileTree().matching(new PatternSet().include('**/*.class')).visit{ details ->
+        if (!details.directory) {
+          logger.info("Patching: ${details.file}")
+          new RandomAccessFile(details.file, 'rw').withCloseable { f ->
+            int magic = f.readInt();
+            if (magic != (int)0xCAFEBABE) {
+              throw new GradleException("Invalid Java class file magic ${String.format("0x%08X", magic)}: ${details.file}")
+            }
+            f.seek(6L)
+            short major = f.readShort()
+            if (major != expectedMajor) {
+              throw new GradleException("Invalid Java class file version ${major}: ${details.file}")
+            }
+            // patch the minor version to 0 (remove preview flag):
+            f.seek(4L)
+            f.writeShort(0)
+          }
+        }
+      }
+    }
+
+    tasks.named('compileMain19Java').configure {
+      javaCompiler = javaToolchains.compilerFor {
+        languageVersion = JavaLanguageVersion.of(19)
+      }
+      
+      // undo alternative JDK support:
+      options.forkOptions.javaHome = null
+      
+      sourceCompatibility = 19
+      targetCompatibility = 19
+      options.compilerArgs += ["--release", 19 as String, "--enable-preview"]
+      
+      doLast {
+        patchClassFiles(destinationDirectory, 63)
+      }
+    }
+    
+    tasks.named('jar').configure {
+      into('META-INF/versions/19') {
+        from sourceSets.main19.output
+      }
+
+      manifest.attributes(
+        'Multi-Release': 'true'
+      )
+    }
+  }
+}

gradle/testing/defaults-tests.gradle

@@ -124,10 +124,6 @@ allprojects {
       // (if the runner JVM does not support them, it will fail tests):
       jvmArgs '--add-modules', 'jdk.unsupported,jdk.management'
 
-      if (rootProject.runtimeJavaVersion == JavaVersion.VERSION_19) {
-        jvmArgs '--enable-preview'
-      }
-
       systemProperty 'java.util.logging.config.file', file("${resources}/logging.properties")
       systemProperty 'java.awt.headless', 'true'
       systemProperty 'jdk.map.althashing.threshold', '0'

lucene/CHANGES.txt

@@ -157,6 +157,11 @@ New Features
   use numeric fields that perform well both for filtering and sorting.
   (Francisco Fernández Castaño)
 
+* GITHUB#12033: Support for Java 19 foreign memory support is now enabled by default,
+  no need to pass "--enable-preview" on the command line. If exactly Java 19 is used,
+  MMapDirectory will mmap Lucene indexes in chunks of 16 GiB (instead of 1 GiB) and
+  indexes closed while queries are running can no longer crash the JVM.  (Uwe Schindler)
+
 Improvements
 ---------------------
 * GITHUB#11778: Detailed part-of-speech information for particle(조사) and ending(어미) on Nori

lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java

@@ -61,8 +61,8 @@ import org.apache.lucene.util.Constants;
  * the workaround will be automatically enabled (with no guarantees; if you discover any problems,
  * you can disable it).
  *
- * <p>On <b>Java 19</b> with {@code --enable-preview} command line setting, this class will use the
- * modern {@code MemorySegment} API which allows to safely unmap.
+ * <p>On exactly <b>Java 19</b> this class will use the modern {@code MemorySegment} API which
+ * allows to safely unmap.
  *
  * <p><b>NOTE:</b> Accessing this class either directly or indirectly from a thread while it's
  * interrupted can close the underlying channel immediately if at the same time the thread is
@@ -106,8 +106,7 @@ public class MMapDirectory extends FSDirectory {
    * Default max chunk size:
    *
    * <ul>
-   *   <li>16 GiBytes for 64 bit <b>Java 19</b> JVMs running with {@code --enable-preview} as
-   *       command line parameter
+   *   <li>16 GiBytes for 64 bit <b>Java 19</b> JVMs
    *   <li>1 GiBytes for other 64 bit JVMs
    *   <li>256 MiBytes for 32 bit JVMs
    * </ul>
@@ -189,9 +188,8 @@ public class MMapDirectory extends FSDirectory {
    * non-Oracle/OpenJDK JVMs. It forcefully unmaps the buffer on close by using an undocumented
    * internal cleanup functionality.
    *
-   * <p>On Java 19 with {@code --enable-preview} command line setting, this class will use the
-   * modern {@code MemorySegment} API which allows to safely unmap. <em>The following warnings no
-   * longer apply in that case!</em>
+   * <p>On exactly Java 19 this class will use the modern {@code MemorySegment} API which allows to
+   * safely unmap. <em>The following warnings no longer apply in that case!</em>
    *
    * <p><b>NOTE:</b> Enabling this is completely unsupported by Java and may lead to JVM crashes if
    * <code>IndexInput</code> is closed while another thread is still accessing it (SIGSEGV).
@@ -351,39 +349,33 @@ public class MMapDirectory extends FSDirectory {
 
   private static MMapIndexInputProvider lookupProvider() {
     final var lookup = MethodHandles.lookup();
-    try {
-      final var cls = lookup.findClass("org.apache.lucene.store.MemorySegmentIndexInputProvider");
-      // we use method handles, so we do not need to deal with setAccessible as we have private
-      // access through the lookup:
-      final var constr = lookup.findConstructor(cls, MethodType.methodType(void.class));
+    final int runtimeVersion = Runtime.version().feature();
+    if (runtimeVersion == 19) {
       try {
-        return (MMapIndexInputProvider) constr.invoke();
-      } catch (RuntimeException | Error e) {
-        throw e;
-      } catch (Throwable th) {
-        throw new AssertionError(th);
+        final var cls = lookup.findClass("org.apache.lucene.store.MemorySegmentIndexInputProvider");
+        // we use method handles, so we do not need to deal with setAccessible as we have private
+        // access through the lookup:
+        final var constr = lookup.findConstructor(cls, MethodType.methodType(void.class));
+        try {
+          return (MMapIndexInputProvider) constr.invoke();
+        } catch (RuntimeException | Error e) {
+          throw e;
+        } catch (Throwable th) {
+          throw new AssertionError(th);
+        }
+      } catch (NoSuchMethodException | IllegalAccessException e) {
+        throw new LinkageError(
+            "MemorySegmentIndexInputProvider is missing correctly typed constructor", e);
+      } catch (ClassNotFoundException cnfe) {
+        throw new LinkageError(
+            "MemorySegmentIndexInputProvider is missing in Lucene JAR file", cnfe);
       }
-    } catch (
-        @SuppressWarnings("unused")
-        ClassNotFoundException e) {
-      // we're before Java 19
-      return new MappedByteBufferIndexInputProvider();
-    } catch (
-        @SuppressWarnings("unused")
-        UnsupportedClassVersionError e) {
+    } else if (runtimeVersion >= 20) {
       var log = Logger.getLogger(lookup.lookupClass().getName());
-      if (Runtime.version().feature() == 19) {
-        log.warning(
-            "You are running with Java 19. To make full use of MMapDirectory, please pass '--enable-preview' to the Java command line.");
-      } else {
-        log.warning(
-            "You are running with Java 20 or later. To make full use of MMapDirectory, please update Apache Lucene.");
-      }
-      return new MappedByteBufferIndexInputProvider();
-    } catch (NoSuchMethodException | IllegalAccessException e) {
-      throw new LinkageError(
-          "MemorySegmentIndexInputProvider is missing correctly typed constructor", e);
+      log.warning(
+          "You are running with Java 20 or later. To make full use of MMapDirectory, please update Apache Lucene.");
     }
+    return new MappedByteBufferIndexInputProvider();
   }
 
   static {

lucene/core/src/test/org/apache/lucene/store/TestMmapDirectory.java

@@ -52,8 +52,6 @@ public class TestMmapDirectory extends BaseDirectoryTestCase {
       assertTrue(
           "on Java 19 we should use MemorySegmentIndexInputProvider to create mmap IndexInputs",
           isMemorySegmentImpl());
-    } else if (runtimeVersion > 19) {
-      // TODO: We don't know how this is handled in later Java versions, so make no assumptions!
     } else {
       assertSame(MappedByteBufferIndexInputProvider.class, MMapDirectory.PROVIDER.getClass());
     }