Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-6989: Make MMapDirectory's unmap hack work with Java 9 EA (b150+): Unmapping uses new sun.misc.Unsafe#invokeCleaner(ByteBuffer).

This commit is contained in:
Uwe Schindler 2016-12-21 19:49:22 +01:00
parent 5cda93afb2
commit 1d92eed93f
3 changed files with 78 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
lucene/CHANGES.txt
View File

@ -113,6 +113,13 @@ Improvements
CorruptIndexException instead of the more confusing EOFException
(Mike Drob via Mike McCandless)
* LUCENE-6989: Make MMapDirectory's unmap hack work with Java 9 EA (b150+):
Unmapping uses new sun.misc.Unsafe#invokeCleaner(ByteBuffer).
Java 9 now needs same permissions like Java 8;
RuntimePermission("accessClassInPackage.jdk.internal.ref")
is no longer needed. Support for older Java 9 builds was removed.
(Uwe Schindler)
Optimizations
* LUCENE-7568: Optimize merging when index sorting is used but the

124
lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java
View File

@ -34,6 +34,7 @@ import java.util.Locale;
import java.util.Objects;
import java.util.concurrent.Future;
import java.lang.invoke.MethodHandle;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import org.apache.lucene.store.ByteBufferGuard.BufferCleaner;
@ -174,14 +175,13 @@ public class MMapDirectory extends FSDirectory {
* is closed while another thread is still accessing it (SIGSEGV).
* <p>To enable the hack, the following requirements need to be
* fulfilled: The used JVM must be Oracle Java / OpenJDK 8
* <em>(preliminary support for Java 9 was added with Lucene 6)</em>.
* <em>(preliminary support for Java 9 EA build 150+ was added with Lucene 6.4)</em>.
* In addition, the following permissions need to be granted
* to {@code lucene-core.jar} in your
* <a href="http://docs.oracle.com/javase/8/docs/technotes/guides/security/PolicyFiles.html">policy file</a>:
* <ul>
* <li>{@code permission java.lang.reflect.ReflectPermission "suppressAccessChecks";}</li>
* <li>{@code permission java.lang.RuntimePermission "accessClassInPackage.sun.misc";}</li>
* <li>{@code permission java.lang.RuntimePermission "accessClassInPackage.jdk.internal.ref";}</li>
* </ul>
* @throws IllegalArgumentException if {@link #UNMAP_SUPPORTED}
* is <code>false</code> and the workaround cannot be enabled.
@ -335,64 +335,82 @@ public class MMapDirectory extends FSDirectory {
}
}
@SuppressForbidden(reason = "Needs access to private APIs in DirectBuffer and sun.misc.Cleaner to enable hack")
@SuppressForbidden(reason = "Needs access to private APIs in DirectBuffer, sun.misc.Cleaner, and sun.misc.Unsafe to enable hack")
private static Object unmapHackImpl() {
final Lookup lookup = lookup();
try {
final Class<?> directBufferClass = Class.forName("java.nio.DirectByteBuffer");
final Method m = directBufferClass.getMethod("cleaner");
m.setAccessible(true);
MethodHandle directBufferCleanerMethod = lookup.unreflect(m);
Class<?> cleanerClass = directBufferCleanerMethod.type().returnType();
final MethodHandle cleanMethod;
if (Runnable.class.isAssignableFrom(cleanerClass)) {
// early Java 9 impl using Runnable (we do the security check early that the Runnable does at runtime):
final SecurityManager sm = System.getSecurityManager();
if (sm != null) {
sm.checkPackageAccess("jdk.internal.ref");
}
// cast return value of cleaner() to Runnable:
directBufferCleanerMethod = directBufferCleanerMethod.asType(directBufferCleanerMethod.type().changeReturnType(Runnable.class));
cleanerClass = Runnable.class;
// lookup run() method on the interface instead of Cleaner:
cleanMethod = lookup.findVirtual(cleanerClass, "run", methodType(void.class));
} else {
// can be either the old internal "sun.misc.Cleaner" or
// the new Java 9 "java.lang.ref.Cleaner$Cleanable":
cleanMethod = lookup.findVirtual(cleanerClass, "clean", methodType(void.class));
try {
// *** sun.misc.Unsafe unmapping (Java 9+) ***
final Class<?> unsafeClass = Class.forName("sun.misc.Unsafe");
// first check if Unsafe has the right method, otherwise we can give up
// without doing any security critical stuff:
final MethodHandle unmapper = lookup.findVirtual(unsafeClass, "invokeCleaner",
methodType(void.class, ByteBuffer.class));
// fetch the unsafe instance and bind it to the virtual MH:
final Field f = unsafeClass.getDeclaredField("theUnsafe");
f.setAccessible(true);
final Object theUnsafe = f.get(null);
return newBufferCleaner(ByteBuffer.class, unmapper.bindTo(theUnsafe));
} catch (SecurityException se) {
// rethrow to report errors correctly (we need to catch it here, as we also catch RuntimeException below!):
throw se;
} catch (ReflectiveOperationException | RuntimeException e) {
// *** sun.misc.Cleaner unmapping (Java 8) ***
final Class<?> directBufferClass = Class.forName("java.nio.DirectByteBuffer");
final Method m = directBufferClass.getMethod("cleaner");
m.setAccessible(true);
final MethodHandle directBufferCleanerMethod = lookup.unreflect(m);
final Class<?> cleanerClass = directBufferCleanerMethod.type().returnType();
/* "Compile" a MH that basically is equivalent to the following code:
* void unmapper(ByteBuffer byteBuffer) {
* sun.misc.Cleaner cleaner = ((java.nio.DirectByteBuffer) byteBuffer).cleaner();
* if (Objects.nonNull(cleaner)) {
* cleaner.clean();
* } else {
* noop(cleaner); // the noop is needed because MethodHandles#guardWithTest always needs ELSE
* }
* }
*/
final MethodHandle cleanMethod = lookup.findVirtual(cleanerClass, "clean", methodType(void.class));
final MethodHandle nonNullTest = lookup.findStatic(Objects.class, "nonNull", methodType(boolean.class, Object.class))
.asType(methodType(boolean.class, cleanerClass));
final MethodHandle noop = dropArguments(constant(Void.class, null).asType(methodType(void.class)), 0, cleanerClass);
final MethodHandle unmapper = filterReturnValue(directBufferCleanerMethod, guardWithTest(nonNullTest, cleanMethod, noop))
.asType(methodType(void.class, ByteBuffer.class));
return newBufferCleaner(directBufferClass, unmapper);
}
final MethodHandle nonNullTest = lookup.findStatic(Objects.class, "nonNull", methodType(boolean.class, Object.class))
.asType(methodType(boolean.class, cleanerClass));
final MethodHandle noop = dropArguments(constant(Void.class, null).asType(methodType(void.class)), 0, cleanerClass);
final MethodHandle unmapper = filterReturnValue(directBufferCleanerMethod, guardWithTest(nonNullTest, cleanMethod, noop))
.asType(methodType(void.class, ByteBuffer.class));
return (BufferCleaner) (String resourceDescription, ByteBuffer buffer) -> {
if (directBufferClass.isInstance(buffer)) {
final Throwable error = AccessController.doPrivileged((PrivilegedAction<Throwable>) () -> {
try {
unmapper.invokeExact(buffer);
return null;
} catch (Throwable t) {
return t;
}
});
if (error != null) {
throw new IOException("Unable to unmap the mapped buffer: " + resourceDescription, error);
}
}
};
} catch (SecurityException e) {
return "Unmapping is not supported, because not all required permissions are given to the Lucene JAR file: " + e +
" [Please grant at least the following permissions: RuntimePermission(\"accessClassInPackage.sun.misc\"), " +
"RuntimePermission(\"accessClassInPackage.jdk.internal.ref\"), and " +
"ReflectPermission(\"suppressAccessChecks\")]";
} catch (SecurityException se) {
return "Unmapping is not supported, because not all required permissions are given to the Lucene JAR file: " + se +
" [Please grant at least the following permissions: RuntimePermission(\"accessClassInPackage.sun.misc\") " +
" and ReflectPermission(\"suppressAccessChecks\")]";
} catch (ReflectiveOperationException | RuntimeException e) {
return "Unmapping is not supported on this platform, because internal Java APIs are not compatible to this Lucene version: " + e;
}
}
private static BufferCleaner newBufferCleaner(final Class<?> unmappableBufferClass, final MethodHandle unmapper) {
assert Objects.equals(methodType(void.class, ByteBuffer.class), unmapper.type());
return (String resourceDescription, ByteBuffer buffer) -> {
if (!buffer.isDirect()) {
throw new IllegalArgumentException("unmapping only works with direct buffers");
}
if (!unmappableBufferClass.isInstance(buffer)) {
throw new IllegalArgumentException("buffer is not an instance of " + unmappableBufferClass.getName());
}
final Throwable error = AccessController.doPrivileged((PrivilegedAction<Throwable>) () -> {
try {
unmapper.invokeExact(buffer);
return null;
} catch (Throwable t) {
return t;
}
});
if (error != null) {
throw new IOException("Unable to unmap the mapped buffer: " + resourceDescription, error);
}
};
}
}

1
lucene/tools/junit4/tests.policy
View File

@ -63,7 +63,6 @@ grant {
permission java.lang.RuntimePermission "createClassLoader";
// needed to test unmap hack on platforms that support it
permission java.lang.RuntimePermission "accessClassInPackage.sun.misc";
permission java.lang.RuntimePermission "accessClassInPackage.jdk.internal.ref";
// needed by cyberneko usage by benchmarks on J9
permission java.lang.RuntimePermission "accessClassInPackage.org.apache.xerces.util";
// needed by jacoco to dump coverage