mirror of https://github.com/apache/lucene.git
LUCENE-6989: Add preliminary support for MMapDirectory unmapping in Java 9
This commit is contained in:
parent
a9dc40294e
commit
7b6df2542d
|
@ -61,6 +61,9 @@ New Features
|
|||
* LUCENE-6975: Add ExactPointQuery, to match a single N-dimensional
|
||||
point (Robert Muir, Mike McCandless)
|
||||
|
||||
* LUCENE-6989: Add preliminary support for MMapDirectory unmapping in Java 9.
|
||||
(Uwe Schindler, Chris Hegarty, Peter Levart)
|
||||
|
||||
API Changes
|
||||
|
||||
* LUCENE-6067: Accountable.getChildResources has a default
|
||||
|
|
|
@ -1055,7 +1055,11 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
|
|||
"index=" + segString() + "\n" +
|
||||
"version=" + Version.LATEST.toString() + "\n" +
|
||||
config.toString());
|
||||
infoStream.message("IW", "MMapDirectory.UNMAP_SUPPORTED=" + MMapDirectory.UNMAP_SUPPORTED);
|
||||
final StringBuilder unmapInfo = new StringBuilder(Boolean.toString(MMapDirectory.UNMAP_SUPPORTED));
|
||||
if (!MMapDirectory.UNMAP_SUPPORTED) {
|
||||
unmapInfo.append(" (").append(MMapDirectory.UNMAP_NOT_SUPPORTED_REASON).append(")");
|
||||
}
|
||||
infoStream.message("IW", "MMapDirectory.UNMAP_SUPPORTED=" + unmapInfo);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -17,6 +17,9 @@
|
|||
package org.apache.lucene.store;
|
||||
|
||||
|
||||
import static java.lang.invoke.MethodHandles.*;
|
||||
import static java.lang.invoke.MethodType.methodType;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.MappedByteBuffer;
|
||||
|
@ -27,10 +30,10 @@ import java.nio.file.Path;
|
|||
import java.nio.file.StandardOpenOption;
|
||||
import java.security.AccessController;
|
||||
import java.security.PrivilegedAction;
|
||||
import java.security.PrivilegedExceptionAction;
|
||||
import java.security.PrivilegedActionException;
|
||||
import java.util.Locale;
|
||||
import java.util.Objects;
|
||||
import java.util.concurrent.Future;
|
||||
import java.lang.invoke.MethodHandle;
|
||||
import java.lang.reflect.Method;
|
||||
|
||||
import org.apache.lucene.store.ByteBufferIndexInput.BufferCleaner;
|
||||
|
@ -160,24 +163,6 @@ public class MMapDirectory extends FSDirectory {
|
|||
assert this.chunkSizePower >= 0 && this.chunkSizePower <= 30;
|
||||
}
|
||||
|
||||
/**
|
||||
* <code>true</code>, if this platform supports unmapping mmapped files.
|
||||
*/
|
||||
public static final boolean UNMAP_SUPPORTED =
|
||||
AccessController.doPrivileged((PrivilegedAction<Boolean>) MMapDirectory::checkUnmapSupported);
|
||||
|
||||
@SuppressForbidden(reason = "Java 9 Jigsaw whitelists access to sun.misc.Cleaner, so setAccessible works")
|
||||
private static boolean checkUnmapSupported() {
|
||||
try {
|
||||
Class<?> clazz = Class.forName("java.nio.DirectByteBuffer");
|
||||
Method method = clazz.getMethod("cleaner");
|
||||
method.setAccessible(true);
|
||||
return true;
|
||||
} catch (Exception e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* This method enables the workaround for unmapping the buffers
|
||||
* from address space after closing {@link IndexInput}, that is
|
||||
|
@ -191,8 +176,9 @@ public class MMapDirectory extends FSDirectory {
|
|||
* is <code>false</code> and the workaround cannot be enabled.
|
||||
*/
|
||||
public void setUseUnmap(final boolean useUnmapHack) {
|
||||
if (useUnmapHack && !UNMAP_SUPPORTED)
|
||||
throw new IllegalArgumentException("Unmap hack not supported on this platform!");
|
||||
if (useUnmapHack && !UNMAP_SUPPORTED) {
|
||||
throw new IllegalArgumentException(UNMAP_NOT_SUPPORTED_REASON);
|
||||
}
|
||||
this.useUnmapHack=useUnmapHack;
|
||||
}
|
||||
|
||||
|
@ -310,23 +296,87 @@ public class MMapDirectory extends FSDirectory {
|
|||
return newIoe;
|
||||
}
|
||||
|
||||
private static final BufferCleaner CLEANER = (final ByteBufferIndexInput parent, final ByteBuffer buffer) -> {
|
||||
try {
|
||||
AccessController.doPrivileged(new PrivilegedExceptionAction<Void>() {
|
||||
@Override
|
||||
@SuppressForbidden(reason = "Java 9 Jigsaw whitelists access to sun.misc.Cleaner, so setAccessible works")
|
||||
public Void run() throws Exception {
|
||||
final Method getCleanerMethod = buffer.getClass().getMethod("cleaner");
|
||||
getCleanerMethod.setAccessible(true);
|
||||
final Object cleaner = getCleanerMethod.invoke(buffer);
|
||||
if (cleaner != null) {
|
||||
cleaner.getClass().getMethod("clean").invoke(cleaner);
|
||||
/**
|
||||
* <code>true</code>, if this platform supports unmapping mmapped files.
|
||||
*/
|
||||
public static final boolean UNMAP_SUPPORTED;
|
||||
|
||||
/**
|
||||
* if {@link #UNMAP_SUPPORTED} is {@code false}, this contains the reason why unmapping is not supported.
|
||||
*/
|
||||
public static final String UNMAP_NOT_SUPPORTED_REASON;
|
||||
|
||||
/** Reference to a BufferCleaner that does unmapping; {@code null} if not supported. */
|
||||
private static final BufferCleaner CLEANER;
|
||||
|
||||
static {
|
||||
final Object hack = AccessController.doPrivileged((PrivilegedAction<Object>) MMapDirectory::initUnmapHack);
|
||||
if (hack instanceof BufferCleaner) {
|
||||
CLEANER = (BufferCleaner) hack;
|
||||
UNMAP_SUPPORTED = true;
|
||||
UNMAP_NOT_SUPPORTED_REASON = null;
|
||||
} else {
|
||||
CLEANER = null;
|
||||
UNMAP_SUPPORTED = false;
|
||||
UNMAP_NOT_SUPPORTED_REASON = hack.toString();
|
||||
}
|
||||
}
|
||||
|
||||
@SuppressForbidden(reason = "Needs access to private APIs in DirectBuffer and sun.misc.Cleaner to enable hack")
|
||||
private static Object initUnmapHack() {
|
||||
final Lookup lookup = lookup();
|
||||
try {
|
||||
final Class<?> directBufferClass = Class.forName("java.nio.DirectByteBuffer");
|
||||
|
||||
final Method m = directBufferClass.getMethod("cleaner");
|
||||
m.setAccessible(true);
|
||||
final MethodHandle directBufferCleanerMethod = lookup.unreflect(m);
|
||||
final Class<?> cleanerClass = directBufferCleanerMethod.type().returnType();
|
||||
|
||||
final MethodHandle cleanMethod;
|
||||
if (Runnable.class.isAssignableFrom(cleanerClass)) {
|
||||
// early Java 9 impl using Runnable (we do the security check early that the Runnable does at runtime):
|
||||
final SecurityManager sm = System.getSecurityManager();
|
||||
if (sm != null) {
|
||||
sm.checkPackageAccess("jdk.internal.ref");
|
||||
}
|
||||
cleanMethod = explicitCastArguments(lookup.findVirtual(Runnable.class, "run", methodType(void.class)),
|
||||
methodType(void.class, cleanerClass));
|
||||
} else {
|
||||
// can be either the old internal "sun.misc.Cleaner" or
|
||||
// the new Java 9 "java.lang.ref.Cleaner$Cleanable":
|
||||
cleanMethod = lookup.findVirtual(cleanerClass, "clean", methodType(void.class));
|
||||
}
|
||||
|
||||
final MethodHandle nonNullTest = explicitCastArguments(lookup.findStatic(Objects.class, "nonNull", methodType(boolean.class, Object.class)),
|
||||
methodType(boolean.class, cleanerClass));
|
||||
final MethodHandle noop = dropArguments(explicitCastArguments(constant(Void.class, null), methodType(void.class)), 0, cleanerClass);
|
||||
final MethodHandle unmapper = explicitCastArguments(filterReturnValue(directBufferCleanerMethod, guardWithTest(nonNullTest, cleanMethod, noop)),
|
||||
methodType(void.class, ByteBuffer.class));
|
||||
|
||||
return (BufferCleaner) (ByteBufferIndexInput parent, ByteBuffer buffer) -> {
|
||||
if (directBufferClass.isInstance(buffer)) {
|
||||
final Throwable error = AccessController.doPrivileged((PrivilegedAction<Throwable>) () -> {
|
||||
try {
|
||||
unmapper.invokeExact(buffer);
|
||||
return null;
|
||||
} catch (Throwable t) {
|
||||
return t;
|
||||
}
|
||||
});
|
||||
} catch (PrivilegedActionException e) {
|
||||
throw new IOException("Unable to unmap the mapped buffer: " + parent.toString(), e.getCause());
|
||||
if (error != null) {
|
||||
throw new IOException("Unable to unmap the mapped buffer: " + parent.toString(), error);
|
||||
}
|
||||
}
|
||||
};
|
||||
} catch (ReflectiveOperationException e) {
|
||||
return "Unmapping is not supported on this platform, because internal Java APIs are not compatible to this Lucene version: " + e;
|
||||
} catch (SecurityException e) {
|
||||
return "Unmapping is not supported, because not all required permissions are given to the Lucene JAR file. " +
|
||||
"Please grant at least the following permissions: RuntimePermission(\"accessClassInPackage.sun.misc\"), " +
|
||||
"RuntimePermission(\"accessClassInPackage.jdk.internal.ref\"), and " +
|
||||
"ReflectPermission(\"suppressAccessChecks\")";
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -35,7 +35,8 @@ public class TestMmapDirectory extends BaseDirectoryTestCase {
|
|||
@Override
|
||||
public void setUp() throws Exception {
|
||||
super.setUp();
|
||||
assumeTrue("test requires a jre that supports unmapping", MMapDirectory.UNMAP_SUPPORTED);
|
||||
assumeTrue("test requires a jre that supports unmapping: " + MMapDirectory.UNMAP_NOT_SUPPORTED_REASON,
|
||||
MMapDirectory.UNMAP_SUPPORTED);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -46,7 +46,8 @@ public class TestMultiMMap extends BaseDirectoryTestCase {
|
|||
@Override
|
||||
public void setUp() throws Exception {
|
||||
super.setUp();
|
||||
assumeTrue("test requires a jre that supports unmapping", MMapDirectory.UNMAP_SUPPORTED);
|
||||
assumeTrue("test requires a jre that supports unmapping: " + MMapDirectory.UNMAP_NOT_SUPPORTED_REASON,
|
||||
MMapDirectory.UNMAP_SUPPORTED);
|
||||
}
|
||||
|
||||
public void testCloneSafety() throws Exception {
|
||||
|
|
|
@ -64,6 +64,7 @@ grant {
|
|||
permission java.lang.RuntimePermission "createClassLoader";
|
||||
// needed to test unmap hack on platforms that support it
|
||||
permission java.lang.RuntimePermission "accessClassInPackage.sun.misc";
|
||||
permission java.lang.RuntimePermission "accessClassInPackage.jdk.internal.ref";
|
||||
// needed by cyberneko usage by benchmarks on J9
|
||||
permission java.lang.RuntimePermission "accessClassInPackage.org.apache.xerces.util";
|
||||
// needed by jacoco to dump coverage
|
||||
|
|
Loading…
Reference in New Issue