diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index e9a0b8555a4..50547028f3d 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -113,6 +113,13 @@ Improvements CorruptIndexException instead of the more confusing EOFException (Mike Drob via Mike McCandless) +* LUCENE-6989: Make MMapDirectory's unmap hack work with Java 9 EA (b150+): + Unmapping uses new sun.misc.Unsafe#invokeCleaner(ByteBuffer). + Java 9 now needs same permissions like Java 8; + RuntimePermission("accessClassInPackage.jdk.internal.ref") + is no longer needed. Support for older Java 9 builds was removed. + (Uwe Schindler) + Optimizations * LUCENE-7568: Optimize merging when index sorting is used but the diff --git a/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java b/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java index be08a1663a6..0487400c776 100644 --- a/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java +++ b/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java @@ -34,6 +34,7 @@ import java.util.Locale; import java.util.Objects; import java.util.concurrent.Future; import java.lang.invoke.MethodHandle; +import java.lang.reflect.Field; import java.lang.reflect.Method; import org.apache.lucene.store.ByteBufferGuard.BufferCleaner; @@ -174,14 +175,13 @@ public class MMapDirectory extends FSDirectory { * is closed while another thread is still accessing it (SIGSEGV). *

To enable the hack, the following requirements need to be * fulfilled: The used JVM must be Oracle Java / OpenJDK 8 - * (preliminary support for Java 9 was added with Lucene 6). + * (preliminary support for Java 9 EA build 150+ was added with Lucene 6.4). * In addition, the following permissions need to be granted * to {@code lucene-core.jar} in your * policy file: *

* @throws IllegalArgumentException if {@link #UNMAP_SUPPORTED} * is false and the workaround cannot be enabled. @@ -335,64 +335,82 @@ public class MMapDirectory extends FSDirectory { } } - @SuppressForbidden(reason = "Needs access to private APIs in DirectBuffer and sun.misc.Cleaner to enable hack") + @SuppressForbidden(reason = "Needs access to private APIs in DirectBuffer, sun.misc.Cleaner, and sun.misc.Unsafe to enable hack") private static Object unmapHackImpl() { final Lookup lookup = lookup(); try { - final Class directBufferClass = Class.forName("java.nio.DirectByteBuffer"); - - final Method m = directBufferClass.getMethod("cleaner"); - m.setAccessible(true); - MethodHandle directBufferCleanerMethod = lookup.unreflect(m); - Class cleanerClass = directBufferCleanerMethod.type().returnType(); - - final MethodHandle cleanMethod; - if (Runnable.class.isAssignableFrom(cleanerClass)) { - // early Java 9 impl using Runnable (we do the security check early that the Runnable does at runtime): - final SecurityManager sm = System.getSecurityManager(); - if (sm != null) { - sm.checkPackageAccess("jdk.internal.ref"); - } - // cast return value of cleaner() to Runnable: - directBufferCleanerMethod = directBufferCleanerMethod.asType(directBufferCleanerMethod.type().changeReturnType(Runnable.class)); - cleanerClass = Runnable.class; - // lookup run() method on the interface instead of Cleaner: - cleanMethod = lookup.findVirtual(cleanerClass, "run", methodType(void.class)); - } else { - // can be either the old internal "sun.misc.Cleaner" or - // the new Java 9 "java.lang.ref.Cleaner$Cleanable": - cleanMethod = lookup.findVirtual(cleanerClass, "clean", methodType(void.class)); + try { + // *** sun.misc.Unsafe unmapping (Java 9+) *** + final Class unsafeClass = Class.forName("sun.misc.Unsafe"); + // first check if Unsafe has the right method, otherwise we can give up + // without doing any security critical stuff: + final MethodHandle unmapper = lookup.findVirtual(unsafeClass, "invokeCleaner", + methodType(void.class, ByteBuffer.class)); + // fetch the unsafe instance and bind it to the virtual MH: + final Field f = unsafeClass.getDeclaredField("theUnsafe"); + f.setAccessible(true); + final Object theUnsafe = f.get(null); + return newBufferCleaner(ByteBuffer.class, unmapper.bindTo(theUnsafe)); + } catch (SecurityException se) { + // rethrow to report errors correctly (we need to catch it here, as we also catch RuntimeException below!): + throw se; + } catch (ReflectiveOperationException | RuntimeException e) { + // *** sun.misc.Cleaner unmapping (Java 8) *** + final Class directBufferClass = Class.forName("java.nio.DirectByteBuffer"); + + final Method m = directBufferClass.getMethod("cleaner"); + m.setAccessible(true); + final MethodHandle directBufferCleanerMethod = lookup.unreflect(m); + final Class cleanerClass = directBufferCleanerMethod.type().returnType(); + + /* "Compile" a MH that basically is equivalent to the following code: + * void unmapper(ByteBuffer byteBuffer) { + * sun.misc.Cleaner cleaner = ((java.nio.DirectByteBuffer) byteBuffer).cleaner(); + * if (Objects.nonNull(cleaner)) { + * cleaner.clean(); + * } else { + * noop(cleaner); // the noop is needed because MethodHandles#guardWithTest always needs ELSE + * } + * } + */ + final MethodHandle cleanMethod = lookup.findVirtual(cleanerClass, "clean", methodType(void.class)); + final MethodHandle nonNullTest = lookup.findStatic(Objects.class, "nonNull", methodType(boolean.class, Object.class)) + .asType(methodType(boolean.class, cleanerClass)); + final MethodHandle noop = dropArguments(constant(Void.class, null).asType(methodType(void.class)), 0, cleanerClass); + final MethodHandle unmapper = filterReturnValue(directBufferCleanerMethod, guardWithTest(nonNullTest, cleanMethod, noop)) + .asType(methodType(void.class, ByteBuffer.class)); + return newBufferCleaner(directBufferClass, unmapper); } - - final MethodHandle nonNullTest = lookup.findStatic(Objects.class, "nonNull", methodType(boolean.class, Object.class)) - .asType(methodType(boolean.class, cleanerClass)); - final MethodHandle noop = dropArguments(constant(Void.class, null).asType(methodType(void.class)), 0, cleanerClass); - final MethodHandle unmapper = filterReturnValue(directBufferCleanerMethod, guardWithTest(nonNullTest, cleanMethod, noop)) - .asType(methodType(void.class, ByteBuffer.class)); - - return (BufferCleaner) (String resourceDescription, ByteBuffer buffer) -> { - if (directBufferClass.isInstance(buffer)) { - final Throwable error = AccessController.doPrivileged((PrivilegedAction) () -> { - try { - unmapper.invokeExact(buffer); - return null; - } catch (Throwable t) { - return t; - } - }); - if (error != null) { - throw new IOException("Unable to unmap the mapped buffer: " + resourceDescription, error); - } - } - }; - } catch (SecurityException e) { - return "Unmapping is not supported, because not all required permissions are given to the Lucene JAR file: " + e + - " [Please grant at least the following permissions: RuntimePermission(\"accessClassInPackage.sun.misc\"), " + - "RuntimePermission(\"accessClassInPackage.jdk.internal.ref\"), and " + - "ReflectPermission(\"suppressAccessChecks\")]"; + } catch (SecurityException se) { + return "Unmapping is not supported, because not all required permissions are given to the Lucene JAR file: " + se + + " [Please grant at least the following permissions: RuntimePermission(\"accessClassInPackage.sun.misc\") " + + " and ReflectPermission(\"suppressAccessChecks\")]"; } catch (ReflectiveOperationException | RuntimeException e) { return "Unmapping is not supported on this platform, because internal Java APIs are not compatible to this Lucene version: " + e; } } + private static BufferCleaner newBufferCleaner(final Class unmappableBufferClass, final MethodHandle unmapper) { + assert Objects.equals(methodType(void.class, ByteBuffer.class), unmapper.type()); + return (String resourceDescription, ByteBuffer buffer) -> { + if (!buffer.isDirect()) { + throw new IllegalArgumentException("unmapping only works with direct buffers"); + } + if (!unmappableBufferClass.isInstance(buffer)) { + throw new IllegalArgumentException("buffer is not an instance of " + unmappableBufferClass.getName()); + } + final Throwable error = AccessController.doPrivileged((PrivilegedAction) () -> { + try { + unmapper.invokeExact(buffer); + return null; + } catch (Throwable t) { + return t; + } + }); + if (error != null) { + throw new IOException("Unable to unmap the mapped buffer: " + resourceDescription, error); + } + }; + } + } diff --git a/lucene/tools/junit4/tests.policy b/lucene/tools/junit4/tests.policy index 2dde5c6f329..b351b172c26 100644 --- a/lucene/tools/junit4/tests.policy +++ b/lucene/tools/junit4/tests.policy @@ -63,7 +63,6 @@ grant { permission java.lang.RuntimePermission "createClassLoader"; // needed to test unmap hack on platforms that support it permission java.lang.RuntimePermission "accessClassInPackage.sun.misc"; - permission java.lang.RuntimePermission "accessClassInPackage.jdk.internal.ref"; // needed by cyberneko usage by benchmarks on J9 permission java.lang.RuntimePermission "accessClassInPackage.org.apache.xerces.util"; // needed by jacoco to dump coverage