From 7b6df2542d345ef1815693d43c9e18c6e64726bd Mon Sep 17 00:00:00 2001 From: Uwe Schindler Date: Mon, 15 Feb 2016 19:57:39 +0100 Subject: [PATCH] LUCENE-6989: Add preliminary support for MMapDirectory unmapping in Java 9 --- lucene/CHANGES.txt | 3 + .../org/apache/lucene/index/IndexWriter.java | 6 +- .../apache/lucene/store/MMapDirectory.java | 130 ++++++++++++------ .../lucene/store/TestMmapDirectory.java | 3 +- .../apache/lucene/store/TestMultiMMap.java | 3 +- lucene/tools/junit4/tests.policy | 1 + 6 files changed, 103 insertions(+), 43 deletions(-) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index c3f305d97a7..fc95bb2aad3 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -61,6 +61,9 @@ New Features * LUCENE-6975: Add ExactPointQuery, to match a single N-dimensional point (Robert Muir, Mike McCandless) +* LUCENE-6989: Add preliminary support for MMapDirectory unmapping in Java 9. + (Uwe Schindler, Chris Hegarty, Peter Levart) + API Changes * LUCENE-6067: Accountable.getChildResources has a default diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java index 8886ab12730..57ce3ddea6d 100644 --- a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java +++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java @@ -1055,7 +1055,11 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable { "index=" + segString() + "\n" + "version=" + Version.LATEST.toString() + "\n" + config.toString()); - infoStream.message("IW", "MMapDirectory.UNMAP_SUPPORTED=" + MMapDirectory.UNMAP_SUPPORTED); + final StringBuilder unmapInfo = new StringBuilder(Boolean.toString(MMapDirectory.UNMAP_SUPPORTED)); + if (!MMapDirectory.UNMAP_SUPPORTED) { + unmapInfo.append(" (").append(MMapDirectory.UNMAP_NOT_SUPPORTED_REASON).append(")"); + } + infoStream.message("IW", "MMapDirectory.UNMAP_SUPPORTED=" + unmapInfo); } } diff --git a/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java b/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java index dca843de6cc..da8cf704aa8 100644 --- a/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java +++ b/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java @@ -17,6 +17,9 @@ package org.apache.lucene.store; +import static java.lang.invoke.MethodHandles.*; +import static java.lang.invoke.MethodType.methodType; + import java.io.IOException; import java.nio.ByteBuffer; import java.nio.MappedByteBuffer; @@ -27,10 +30,10 @@ import java.nio.file.Path; import java.nio.file.StandardOpenOption; import java.security.AccessController; import java.security.PrivilegedAction; -import java.security.PrivilegedExceptionAction; -import java.security.PrivilegedActionException; import java.util.Locale; +import java.util.Objects; import java.util.concurrent.Future; +import java.lang.invoke.MethodHandle; import java.lang.reflect.Method; import org.apache.lucene.store.ByteBufferIndexInput.BufferCleaner; @@ -160,24 +163,6 @@ public class MMapDirectory extends FSDirectory { assert this.chunkSizePower >= 0 && this.chunkSizePower <= 30; } - /** - * true, if this platform supports unmapping mmapped files. - */ - public static final boolean UNMAP_SUPPORTED = - AccessController.doPrivileged((PrivilegedAction) MMapDirectory::checkUnmapSupported); - - @SuppressForbidden(reason = "Java 9 Jigsaw whitelists access to sun.misc.Cleaner, so setAccessible works") - private static boolean checkUnmapSupported() { - try { - Class clazz = Class.forName("java.nio.DirectByteBuffer"); - Method method = clazz.getMethod("cleaner"); - method.setAccessible(true); - return true; - } catch (Exception e) { - return false; - } - } - /** * This method enables the workaround for unmapping the buffers * from address space after closing {@link IndexInput}, that is @@ -191,8 +176,9 @@ public class MMapDirectory extends FSDirectory { * is false and the workaround cannot be enabled. */ public void setUseUnmap(final boolean useUnmapHack) { - if (useUnmapHack && !UNMAP_SUPPORTED) - throw new IllegalArgumentException("Unmap hack not supported on this platform!"); + if (useUnmapHack && !UNMAP_SUPPORTED) { + throw new IllegalArgumentException(UNMAP_NOT_SUPPORTED_REASON); + } this.useUnmapHack=useUnmapHack; } @@ -310,23 +296,87 @@ public class MMapDirectory extends FSDirectory { return newIoe; } - private static final BufferCleaner CLEANER = (final ByteBufferIndexInput parent, final ByteBuffer buffer) -> { - try { - AccessController.doPrivileged(new PrivilegedExceptionAction() { - @Override - @SuppressForbidden(reason = "Java 9 Jigsaw whitelists access to sun.misc.Cleaner, so setAccessible works") - public Void run() throws Exception { - final Method getCleanerMethod = buffer.getClass().getMethod("cleaner"); - getCleanerMethod.setAccessible(true); - final Object cleaner = getCleanerMethod.invoke(buffer); - if (cleaner != null) { - cleaner.getClass().getMethod("clean").invoke(cleaner); - } - return null; - } - }); - } catch (PrivilegedActionException e) { - throw new IOException("Unable to unmap the mapped buffer: " + parent.toString(), e.getCause()); + /** + * true, if this platform supports unmapping mmapped files. + */ + public static final boolean UNMAP_SUPPORTED; + + /** + * if {@link #UNMAP_SUPPORTED} is {@code false}, this contains the reason why unmapping is not supported. + */ + public static final String UNMAP_NOT_SUPPORTED_REASON; + + /** Reference to a BufferCleaner that does unmapping; {@code null} if not supported. */ + private static final BufferCleaner CLEANER; + + static { + final Object hack = AccessController.doPrivileged((PrivilegedAction) MMapDirectory::initUnmapHack); + if (hack instanceof BufferCleaner) { + CLEANER = (BufferCleaner) hack; + UNMAP_SUPPORTED = true; + UNMAP_NOT_SUPPORTED_REASON = null; + } else { + CLEANER = null; + UNMAP_SUPPORTED = false; + UNMAP_NOT_SUPPORTED_REASON = hack.toString(); } - }; + } + + @SuppressForbidden(reason = "Needs access to private APIs in DirectBuffer and sun.misc.Cleaner to enable hack") + private static Object initUnmapHack() { + final Lookup lookup = lookup(); + try { + final Class directBufferClass = Class.forName("java.nio.DirectByteBuffer"); + + final Method m = directBufferClass.getMethod("cleaner"); + m.setAccessible(true); + final MethodHandle directBufferCleanerMethod = lookup.unreflect(m); + final Class cleanerClass = directBufferCleanerMethod.type().returnType(); + + final MethodHandle cleanMethod; + if (Runnable.class.isAssignableFrom(cleanerClass)) { + // early Java 9 impl using Runnable (we do the security check early that the Runnable does at runtime): + final SecurityManager sm = System.getSecurityManager(); + if (sm != null) { + sm.checkPackageAccess("jdk.internal.ref"); + } + cleanMethod = explicitCastArguments(lookup.findVirtual(Runnable.class, "run", methodType(void.class)), + methodType(void.class, cleanerClass)); + } else { + // can be either the old internal "sun.misc.Cleaner" or + // the new Java 9 "java.lang.ref.Cleaner$Cleanable": + cleanMethod = lookup.findVirtual(cleanerClass, "clean", methodType(void.class)); + } + + final MethodHandle nonNullTest = explicitCastArguments(lookup.findStatic(Objects.class, "nonNull", methodType(boolean.class, Object.class)), + methodType(boolean.class, cleanerClass)); + final MethodHandle noop = dropArguments(explicitCastArguments(constant(Void.class, null), methodType(void.class)), 0, cleanerClass); + final MethodHandle unmapper = explicitCastArguments(filterReturnValue(directBufferCleanerMethod, guardWithTest(nonNullTest, cleanMethod, noop)), + methodType(void.class, ByteBuffer.class)); + + return (BufferCleaner) (ByteBufferIndexInput parent, ByteBuffer buffer) -> { + if (directBufferClass.isInstance(buffer)) { + final Throwable error = AccessController.doPrivileged((PrivilegedAction) () -> { + try { + unmapper.invokeExact(buffer); + return null; + } catch (Throwable t) { + return t; + } + }); + if (error != null) { + throw new IOException("Unable to unmap the mapped buffer: " + parent.toString(), error); + } + } + }; + } catch (ReflectiveOperationException e) { + return "Unmapping is not supported on this platform, because internal Java APIs are not compatible to this Lucene version: " + e; + } catch (SecurityException e) { + return "Unmapping is not supported, because not all required permissions are given to the Lucene JAR file. " + + "Please grant at least the following permissions: RuntimePermission(\"accessClassInPackage.sun.misc\"), " + + "RuntimePermission(\"accessClassInPackage.jdk.internal.ref\"), and " + + "ReflectPermission(\"suppressAccessChecks\")"; + } + } + } diff --git a/lucene/core/src/test/org/apache/lucene/store/TestMmapDirectory.java b/lucene/core/src/test/org/apache/lucene/store/TestMmapDirectory.java index 44e6de263d7..153cc5e6d30 100644 --- a/lucene/core/src/test/org/apache/lucene/store/TestMmapDirectory.java +++ b/lucene/core/src/test/org/apache/lucene/store/TestMmapDirectory.java @@ -35,7 +35,8 @@ public class TestMmapDirectory extends BaseDirectoryTestCase { @Override public void setUp() throws Exception { super.setUp(); - assumeTrue("test requires a jre that supports unmapping", MMapDirectory.UNMAP_SUPPORTED); + assumeTrue("test requires a jre that supports unmapping: " + MMapDirectory.UNMAP_NOT_SUPPORTED_REASON, + MMapDirectory.UNMAP_SUPPORTED); } } diff --git a/lucene/core/src/test/org/apache/lucene/store/TestMultiMMap.java b/lucene/core/src/test/org/apache/lucene/store/TestMultiMMap.java index adea8ff6a62..3b6ef22e456 100644 --- a/lucene/core/src/test/org/apache/lucene/store/TestMultiMMap.java +++ b/lucene/core/src/test/org/apache/lucene/store/TestMultiMMap.java @@ -46,7 +46,8 @@ public class TestMultiMMap extends BaseDirectoryTestCase { @Override public void setUp() throws Exception { super.setUp(); - assumeTrue("test requires a jre that supports unmapping", MMapDirectory.UNMAP_SUPPORTED); + assumeTrue("test requires a jre that supports unmapping: " + MMapDirectory.UNMAP_NOT_SUPPORTED_REASON, + MMapDirectory.UNMAP_SUPPORTED); } public void testCloneSafety() throws Exception { diff --git a/lucene/tools/junit4/tests.policy b/lucene/tools/junit4/tests.policy index a6305138e77..a579fe251d3 100644 --- a/lucene/tools/junit4/tests.policy +++ b/lucene/tools/junit4/tests.policy @@ -64,6 +64,7 @@ grant { permission java.lang.RuntimePermission "createClassLoader"; // needed to test unmap hack on platforms that support it permission java.lang.RuntimePermission "accessClassInPackage.sun.misc"; + permission java.lang.RuntimePermission "accessClassInPackage.jdk.internal.ref"; // needed by cyberneko usage by benchmarks on J9 permission java.lang.RuntimePermission "accessClassInPackage.org.apache.xerces.util"; // needed by jacoco to dump coverage