Patch class files for Java 19 code to no longer have the "preview" flag (this enables Java 19 memory segments by default) (#12033)

This commit is contained in:
Uwe Schindler 2022-12-26 10:07:44 +01:00 committed by GitHub
parent 92f08aff9f
commit c9401bf064
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 124 additions and 88 deletions

View File

@ -119,6 +119,7 @@ apply from: file('gradle/ide/eclipse.gradle')
// (java, tests)
apply from: file('gradle/java/folder-layout.gradle')
apply from: file('gradle/java/javac.gradle')
apply from: file('gradle/java/memorysegment-mrjar.gradle')
apply from: file('gradle/testing/defaults-tests.gradle')
apply from: file('gradle/testing/randomization.gradle')
apply from: file('gradle/testing/fail-on-no-tests.gradle')

View File

@ -101,7 +101,7 @@ tests.jvms=${testsJvms}
org.gradle.java.installations.auto-download=true
# Set these to enable automatic JVM location discovery.
org.gradle.java.installations.fromEnv=JAVA17_HOME,JAVA19_HOME
org.gradle.java.installations.fromEnv=JAVA17_HOME,JAVA19_HOME,JAVA20_HOME,JAVA21_HOME,RUNTIME_JAVA_HOME
#org.gradle.java.installations.paths=(custom paths)
""", "UTF-8")

View File

@ -79,48 +79,3 @@ allprojects {
}
}
}
configure(project(":lucene:core")) {
plugins.withType(JavaPlugin) {
sourceSets {
main19 {
java {
srcDirs = ['src/java19']
}
}
}
configurations {
// Inherit any dependencies from the main source set.
main19Implementation.extendsFrom implementation
}
dependencies {
// We need the main classes to compile our Java 19 pieces.
main19Implementation sourceSets.main.output
}
tasks.named('compileMain19Java').configure {
javaCompiler = javaToolchains.compilerFor {
languageVersion = JavaLanguageVersion.of(19)
}
// undo alternative JDK support:
options.forkOptions.javaHome = null
sourceCompatibility = 19
targetCompatibility = 19
options.compilerArgs += ["--release", 19 as String, "--enable-preview"]
}
tasks.named('jar').configure {
into('META-INF/versions/19') {
from sourceSets.main19.output
}
manifest.attributes(
'Multi-Release': 'true'
)
}
}
}

View File

@ -0,0 +1,89 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Produce an MR-JAR with Java 19 MemorySegment implementation for MMapDirectory
configure(project(":lucene:core")) {
plugins.withType(JavaPlugin) {
sourceSets {
main19 {
java {
srcDirs = ['src/java19']
}
}
}
configurations {
// Inherit any dependencies from the main source set.
main19Implementation.extendsFrom implementation
}
dependencies {
// We need the main classes to compile our Java 19 pieces.
main19Implementation sourceSets.main.output
}
def patchClassFiles = { DirectoryProperty destinationDirectory, int expectedMajor ->
destinationDirectory.getAsFileTree().matching(new PatternSet().include('**/*.class')).visit{ details ->
if (!details.directory) {
logger.info("Patching: ${details.file}")
new RandomAccessFile(details.file, 'rw').withCloseable { f ->
int magic = f.readInt();
if (magic != (int)0xCAFEBABE) {
throw new GradleException("Invalid Java class file magic ${String.format("0x%08X", magic)}: ${details.file}")
}
f.seek(6L)
short major = f.readShort()
if (major != expectedMajor) {
throw new GradleException("Invalid Java class file version ${major}: ${details.file}")
}
// patch the minor version to 0 (remove preview flag):
f.seek(4L)
f.writeShort(0)
}
}
}
}
tasks.named('compileMain19Java').configure {
javaCompiler = javaToolchains.compilerFor {
languageVersion = JavaLanguageVersion.of(19)
}
// undo alternative JDK support:
options.forkOptions.javaHome = null
sourceCompatibility = 19
targetCompatibility = 19
options.compilerArgs += ["--release", 19 as String, "--enable-preview"]
doLast {
patchClassFiles(destinationDirectory, 63)
}
}
tasks.named('jar').configure {
into('META-INF/versions/19') {
from sourceSets.main19.output
}
manifest.attributes(
'Multi-Release': 'true'
)
}
}
}

View File

@ -124,10 +124,6 @@ allprojects {
// (if the runner JVM does not support them, it will fail tests):
jvmArgs '--add-modules', 'jdk.unsupported,jdk.management'
if (rootProject.runtimeJavaVersion == JavaVersion.VERSION_19) {
jvmArgs '--enable-preview'
}
systemProperty 'java.util.logging.config.file', file("${resources}/logging.properties")
systemProperty 'java.awt.headless', 'true'
systemProperty 'jdk.map.althashing.threshold', '0'

View File

@ -157,6 +157,11 @@ New Features
use numeric fields that perform well both for filtering and sorting.
(Francisco Fernández Castaño)
* GITHUB#12033: Support for Java 19 foreign memory support is now enabled by default,
no need to pass "--enable-preview" on the command line. If exactly Java 19 is used,
MMapDirectory will mmap Lucene indexes in chunks of 16 GiB (instead of 1 GiB) and
indexes closed while queries are running can no longer crash the JVM. (Uwe Schindler)
Improvements
---------------------
* GITHUB#11778: Detailed part-of-speech information for particle(조사) and ending(어미) on Nori

View File

@ -61,8 +61,8 @@ import org.apache.lucene.util.Constants;
* the workaround will be automatically enabled (with no guarantees; if you discover any problems,
* you can disable it).
*
* <p>On <b>Java 19</b> with {@code --enable-preview} command line setting, this class will use the
* modern {@code MemorySegment} API which allows to safely unmap.
* <p>On exactly <b>Java 19</b> this class will use the modern {@code MemorySegment} API which
* allows to safely unmap.
*
* <p><b>NOTE:</b> Accessing this class either directly or indirectly from a thread while it's
* interrupted can close the underlying channel immediately if at the same time the thread is
@ -106,8 +106,7 @@ public class MMapDirectory extends FSDirectory {
* Default max chunk size:
*
* <ul>
* <li>16 GiBytes for 64 bit <b>Java 19</b> JVMs running with {@code --enable-preview} as
* command line parameter
* <li>16 GiBytes for 64 bit <b>Java 19</b> JVMs
* <li>1 GiBytes for other 64 bit JVMs
* <li>256 MiBytes for 32 bit JVMs
* </ul>
@ -189,9 +188,8 @@ public class MMapDirectory extends FSDirectory {
* non-Oracle/OpenJDK JVMs. It forcefully unmaps the buffer on close by using an undocumented
* internal cleanup functionality.
*
* <p>On Java 19 with {@code --enable-preview} command line setting, this class will use the
* modern {@code MemorySegment} API which allows to safely unmap. <em>The following warnings no
* longer apply in that case!</em>
* <p>On exactly Java 19 this class will use the modern {@code MemorySegment} API which allows to
* safely unmap. <em>The following warnings no longer apply in that case!</em>
*
* <p><b>NOTE:</b> Enabling this is completely unsupported by Java and may lead to JVM crashes if
* <code>IndexInput</code> is closed while another thread is still accessing it (SIGSEGV).
@ -351,39 +349,33 @@ public class MMapDirectory extends FSDirectory {
private static MMapIndexInputProvider lookupProvider() {
final var lookup = MethodHandles.lookup();
try {
final var cls = lookup.findClass("org.apache.lucene.store.MemorySegmentIndexInputProvider");
// we use method handles, so we do not need to deal with setAccessible as we have private
// access through the lookup:
final var constr = lookup.findConstructor(cls, MethodType.methodType(void.class));
final int runtimeVersion = Runtime.version().feature();
if (runtimeVersion == 19) {
try {
return (MMapIndexInputProvider) constr.invoke();
} catch (RuntimeException | Error e) {
throw e;
} catch (Throwable th) {
throw new AssertionError(th);
final var cls = lookup.findClass("org.apache.lucene.store.MemorySegmentIndexInputProvider");
// we use method handles, so we do not need to deal with setAccessible as we have private
// access through the lookup:
final var constr = lookup.findConstructor(cls, MethodType.methodType(void.class));
try {
return (MMapIndexInputProvider) constr.invoke();
} catch (RuntimeException | Error e) {
throw e;
} catch (Throwable th) {
throw new AssertionError(th);
}
} catch (NoSuchMethodException | IllegalAccessException e) {
throw new LinkageError(
"MemorySegmentIndexInputProvider is missing correctly typed constructor", e);
} catch (ClassNotFoundException cnfe) {
throw new LinkageError(
"MemorySegmentIndexInputProvider is missing in Lucene JAR file", cnfe);
}
} catch (
@SuppressWarnings("unused")
ClassNotFoundException e) {
// we're before Java 19
return new MappedByteBufferIndexInputProvider();
} catch (
@SuppressWarnings("unused")
UnsupportedClassVersionError e) {
} else if (runtimeVersion >= 20) {
var log = Logger.getLogger(lookup.lookupClass().getName());
if (Runtime.version().feature() == 19) {
log.warning(
"You are running with Java 19. To make full use of MMapDirectory, please pass '--enable-preview' to the Java command line.");
} else {
log.warning(
"You are running with Java 20 or later. To make full use of MMapDirectory, please update Apache Lucene.");
}
return new MappedByteBufferIndexInputProvider();
} catch (NoSuchMethodException | IllegalAccessException e) {
throw new LinkageError(
"MemorySegmentIndexInputProvider is missing correctly typed constructor", e);
log.warning(
"You are running with Java 20 or later. To make full use of MMapDirectory, please update Apache Lucene.");
}
return new MappedByteBufferIndexInputProvider();
}
static {

View File

@ -52,8 +52,6 @@ public class TestMmapDirectory extends BaseDirectoryTestCase {
assertTrue(
"on Java 19 we should use MemorySegmentIndexInputProvider to create mmap IndexInputs",
isMemorySegmentImpl());
} else if (runtimeVersion > 19) {
// TODO: We don't know how this is handled in later Java versions, so make no assumptions!
} else {
assertSame(MappedByteBufferIndexInputProvider.class, MMapDirectory.PROVIDER.getClass());
}