Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove ByteBufferIndexInput and update all Panama implementations (MMap and Vector) to Java 21 (#13146)

This commit is contained in:
Uwe Schindler 2024-02-29 19:38:37 +01:00 committed by GitHub
parent dfce6ee8d2
commit e446904c61
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
40 changed files with 60 additions and 3085 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
gradle/generation/extract-jdk-apis.gradle
View File

@ -20,14 +20,14 @@ def resources = scriptResources(buildscript)
configure(rootProject) {
ext {
// also change this in extractor tool: ExtractForeignAPI
vectorIncubatorJavaVersions = [ JavaVersion.VERSION_20, JavaVersion.VERSION_21, JavaVersion.VERSION_22 ] as Set
vectorIncubatorJavaVersions = [ JavaVersion.VERSION_21, JavaVersion.VERSION_22 ] as Set
}
}
configure(project(":lucene:core")) {
ext {
apijars = layout.projectDirectory.dir("src/generated/jdk")
mrjarJavaVersions = [ 19, 20, 21 ]
mrjarJavaVersions = [ 21 ]
}
configurations {

10
gradle/generation/extract-jdk-apis/ExtractJdkApis.java
View File

@ -54,9 +54,7 @@ public final class ExtractJdkApis {
private static final String PATTERN_VECTOR_VM_INTERNALS = "java.base/jdk/internal/vm/vector/VectorSupport{,$Vector,$VectorMask,$VectorPayload,$VectorShuffle}";
static final Map<Integer,List<String>> CLASSFILE_PATTERNS = Map.of(
19, List.of(PATTERN_PANAMA_FOREIGN),
20, List.of(PATTERN_PANAMA_FOREIGN, PATTERN_VECTOR_VM_INTERNALS, PATTERN_VECTOR_INCUBATOR),
21, List.of(PATTERN_PANAMA_FOREIGN)
21, List.of(PATTERN_PANAMA_FOREIGN, PATTERN_VECTOR_VM_INTERNALS, PATTERN_VECTOR_INCUBATOR)
);
public static void main(String... args) throws IOException {
@ -143,7 +141,7 @@ public final class ExtractJdkApis {
@Override
public void visit(int version, int access, String name, String signature, String superName, String[] interfaces) {
super.visit(Opcodes.V11, access, name, signature, superName, interfaces);
super.visit(Opcodes.V21, access, name, signature, superName, interfaces);
if (isVisible(access)) {
classesToInclude.add(name);
}
@ -188,10 +186,6 @@ public final class ExtractJdkApis {
}
}
@Override
public void visitPermittedSubclass(String c) {
}
}
}

2
gradle/testing/defaults-tests.gradle
View File

@ -126,7 +126,7 @@ allprojects {
// Lucene needs to optional modules at runtime, which we want to enforce for testing
// (if the runner JVM does not support them, it will fail tests):
jvmArgs '--add-modules', 'jdk.unsupported,jdk.management'
jvmArgs '--add-modules', 'jdk.management'
// Enable the vector incubator module on supported Java versions:
if (rootProject.vectorIncubatorJavaVersions.contains(rootProject.runtimeJavaVersion)) {

3
gradle/testing/randomization/policies/replicator-tests.policy
View File

@ -54,9 +54,6 @@ grant {
permission java.lang.RuntimePermission "getStackTrace";
// needed for mock filesystems in tests
permission java.lang.RuntimePermission "fileSystemProvider";
// needed to test unmap hack on platforms that support it
permission java.lang.RuntimePermission "accessClassInPackage.sun.misc";
permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
};
// Permissions to support ant build

3
gradle/testing/randomization/policies/tests.policy
View File

@ -50,9 +50,6 @@ grant {
permission java.lang.RuntimePermission "getStackTrace";
// needed for mock filesystems in tests
permission java.lang.RuntimePermission "fileSystemProvider";
// needed to test unmap hack on platforms that support it
permission java.lang.RuntimePermission "accessClassInPackage.sun.misc";
permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
// needed by cyberneko usage by benchmarks on J9
permission java.lang.RuntimePermission "accessClassInPackage.org.apache.xerces.util";

34
lucene/core.tests/src/test/org/apache/lucene/core/tests/TestMMap.java
View File

@ -1,34 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.core.tests;
import org.apache.lucene.store.MMapDirectory;
import org.apache.lucene.tests.util.LuceneTestCase;
import org.junit.Assert;
public class TestMMap extends LuceneTestCase {
public void testUnmapSupported() {
final Module module = MMapDirectory.class.getModule();
Assert.assertTrue("Lucene Core is not loaded as module", module.isNamed());
Assert.assertTrue(
"Lucene Core can't read 'jdk.unsupported' module",
module.getLayer().findModule("jdk.unsupported").map(module::canRead).orElse(false));
// check that MMapDirectory can unmap by running the autodetection logic:
Assert.assertTrue(MMapDirectory.UNMAP_NOT_SUPPORTED_REASON, MMapDirectory.UNMAP_SUPPORTED);
}
}

BIN
lucene/core/src/generated/jdk/jdk19.apijar
View File

Binary file not shown.

BIN
lucene/core/src/generated/jdk/jdk20.apijar
View File

Binary file not shown.

BIN
lucene/core/src/generated/jdk/jdk21.apijar
View File

Binary file not shown.

1
lucene/core/src/java/module-info.java
View File

@ -21,7 +21,6 @@ import org.apache.lucene.codecs.lucene99.Lucene99Codec;
@SuppressWarnings("module") // the test framework is compiled after the core...
module org.apache.lucene.core {
requires java.logging;
requires static jdk.unsupported; // this is optional but without it MMapDirectory won't be enabled
requires static jdk.management; // this is optional but explicit declaration is recommended
exports org.apache.lucene.analysis;

7
lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
View File

@ -74,7 +74,6 @@ import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.Lock;
import org.apache.lucene.store.LockObtainFailedException;
import org.apache.lucene.store.LockValidatingDirectoryWrapper;
import org.apache.lucene.store.MMapDirectory;
import org.apache.lucene.store.MergeInfo;
import org.apache.lucene.store.TrackingDirectoryWrapper;
import org.apache.lucene.util.Accountable;
@ -1303,12 +1302,6 @@ public class IndexWriter
+ Version.LATEST.toString()
+ "\n"
+ config.toString());
final StringBuilder unmapInfo =
new StringBuilder(Boolean.toString(MMapDirectory.UNMAP_SUPPORTED));
if (!MMapDirectory.UNMAP_SUPPORTED) {
unmapInfo.append(" (").append(MMapDirectory.UNMAP_NOT_SUPPORTED_REASON).append(")");
}
infoStream.message("IW", "MMapDirectory.UNMAP_SUPPORTED=" + unmapInfo);
}
}

34
lucene/core/src/java/org/apache/lucene/internal/vectorization/VectorizationProvider.java
View File

@ -17,11 +17,9 @@
package org.apache.lucene.internal.vectorization;
import java.lang.Runtime.Version;
import java.lang.StackWalker.StackFrame;
import java.lang.invoke.MethodHandles;
import java.lang.invoke.MethodType;
import java.util.Locale;
import java.util.Objects;
import java.util.Optional;
import java.util.OptionalInt;
@ -97,20 +95,11 @@ public abstract class VectorizationProvider {
private static final Logger LOG = Logger.getLogger(VectorizationProvider.class.getName());
/** The minimal version of Java that has the bugfix for JDK-8301190. */
private static final Version VERSION_JDK8301190_FIXED = Version.parse("20.0.2");
// visible for tests
static VectorizationProvider lookup(boolean testMode) {
final int runtimeVersion = Runtime.version().feature();
if (runtimeVersion >= 20 && runtimeVersion <= 22) {
// is locale sane (only buggy in Java 20)
if (isAffectedByJDK8301190()) {
LOG.warning(
"Java runtime is using a buggy default locale; Java vector incubator API can't be enabled: "
+ Locale.getDefault());
return new DefaultVectorizationProvider();
}
assert runtimeVersion >= 21;
if (runtimeVersion <= 22) {
// only use vector module with Hotspot VM
if (!Constants.IS_HOTSPOT_VM) {
LOG.warning(
@ -169,13 +158,11 @@ public abstract class VectorizationProvider {
} catch (ClassNotFoundException cnfe) {
throw new LinkageError("PanamaVectorizationProvider is missing in Lucene JAR file", cnfe);
}
} else if (runtimeVersion >= 23) {
} else {
LOG.warning(
"You are running with Java 23 or later. To make full use of the Vector API, please update Apache Lucene.");
} else if (lookupVectorModule().isPresent()) {
LOG.warning(
"Java vector incubator module was enabled by command line flags, but your Java version is too old: "
+ runtimeVersion);
"You are running with unsupported Java "
+ runtimeVersion
+ ". To make full use of the Vector API, please update Apache Lucene.");
}
return new DefaultVectorizationProvider();
}
@ -189,15 +176,6 @@ public abstract class VectorizationProvider {
.findModule("jdk.incubator.vector");
}
/**
* Check if runtime is affected by JDK-8301190 (avoids assertion when default language is say
* "tr").
*/
private static boolean isAffectedByJDK8301190() {
return VERSION_JDK8301190_FIXED.compareToIgnoreOptional(Runtime.version()) > 0
&& !Objects.equals("I", "i".toUpperCase(Locale.getDefault()));
}
// add all possible callers here as FQCN:
private static final Set<String> VALID_CALLERS = Set.of("org.apache.lucene.util.VectorUtil");

156
lucene/core/src/java/org/apache/lucene/store/ByteBufferGuard.java
View File

@ -1,156 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.store;
import java.io.IOException;
import java.lang.invoke.VarHandle;
import java.nio.ByteBuffer;
import java.nio.FloatBuffer;
import java.nio.IntBuffer;
import java.nio.LongBuffer;
/**
* A guard that is created for every {@link ByteBufferIndexInput} that tries on best effort to
* reject any access to the {@link ByteBuffer} behind, once it is unmapped. A single instance of
* this is used for the original and all clones, so once the original is closed and unmapped all
* clones also throw {@link AlreadyClosedException}, triggered by a {@link NullPointerException}.
*
* <p>This code tries to hopefully flush any CPU caches using a store-store barrier. It also yields
* the current thread to give other threads a chance to finish in-flight requests...
*/
final class ByteBufferGuard {
/**
* Pass in an implementation of this interface to cleanup ByteBuffers. MMapDirectory implements
* this to allow unmapping of bytebuffers with private Java APIs.
*/
@FunctionalInterface
interface BufferCleaner {
void freeBuffer(String resourceDescription, ByteBuffer b) throws IOException;
}
private final String resourceDescription;
private final BufferCleaner cleaner;
/** Not volatile; see comments on visibility below! */
private boolean invalidated = false;
/**
* Creates an instance to be used for a single {@link ByteBufferIndexInput} which must be shared
* by all of its clones.
*/
public ByteBufferGuard(String resourceDescription, BufferCleaner cleaner) {
this.resourceDescription = resourceDescription;
this.cleaner = cleaner;
}
/** Invalidates this guard and unmaps (if supported). */
public void invalidateAndUnmap(ByteBuffer... bufs) throws IOException {
if (cleaner != null) {
invalidated = true;
// This call should hopefully flush any CPU caches and as a result make
// the "invalidated" field update visible to other threads. We specifically
// don't make "invalidated" field volatile for performance reasons, hoping the
// JVM won't optimize away reads of that field and hardware should ensure
// caches are in sync after this call.
// For previous implementation (based on `AtomicInteger#lazySet(0)`) see LUCENE-7409.
VarHandle.fullFence();
// we give other threads a bit of time to finish reads on their ByteBuffer...:
Thread.yield();
// finally unmap the ByteBuffers:
for (ByteBuffer b : bufs) {
cleaner.freeBuffer(resourceDescription, b);
}
}
}
public boolean isInvalidated() {
return invalidated;
}
private void ensureValid() {
if (invalidated) {
// this triggers an AlreadyClosedException in ByteBufferIndexInput:
throw new NullPointerException();
}
}
public void getBytes(ByteBuffer receiver, int pos, byte[] dst, int offset, int length) {
ensureValid();
receiver.get(pos, dst, offset, length);
}
public void getBytes(ByteBuffer receiver, byte[] dst, int offset, int length) {
ensureValid();
receiver.get(dst, offset, length);
}
public byte getByte(ByteBuffer receiver) {
ensureValid();
return receiver.get();
}
public short getShort(ByteBuffer receiver) {
ensureValid();
return receiver.getShort();
}
public int getInt(ByteBuffer receiver) {
ensureValid();
return receiver.getInt();
}
public long getLong(ByteBuffer receiver) {
ensureValid();
return receiver.getLong();
}
public byte getByte(ByteBuffer receiver, int pos) {
ensureValid();
return receiver.get(pos);
}
public short getShort(ByteBuffer receiver, int pos) {
ensureValid();
return receiver.getShort(pos);
}
public int getInt(ByteBuffer receiver, int pos) {
ensureValid();
return receiver.getInt(pos);
}
public long getLong(ByteBuffer receiver, int pos) {
ensureValid();
return receiver.getLong(pos);
}
public void getLongs(LongBuffer receiver, long[] dst, int offset, int length) {
ensureValid();
receiver.get(dst, offset, length);
}
public void getInts(IntBuffer receiver, int[] dst, int offset, int length) {
ensureValid();
receiver.get(dst, offset, length);
}
public void getFloats(FloatBuffer receiver, float[] dst, int offset, int length) {
ensureValid();
receiver.get(dst, offset, length);
}
}

732
lucene/core/src/java/org/apache/lucene/store/ByteBufferIndexInput.java
View File

@ -1,732 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.store;
import java.io.EOFException;
import java.io.IOException;
import java.nio.BufferUnderflowException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.FloatBuffer;
import java.nio.IntBuffer;
import java.nio.LongBuffer;
/**
* Base IndexInput implementation that uses an array of ByteBuffers to represent a file.
*
* <p>Because Java's ByteBuffer uses an int to address the values, it's necessary to access a file
* greater Integer.MAX_VALUE in size using multiple byte buffers.
*
* <p>For efficiency, this class requires that the buffers are a power-of-two (<code>chunkSizePower
* </code>).
*
* @deprecated This class was made public for internal reasons ({@code instanceof} checks). In
* {@link MMapDirectory} it was replaced by {@code MemorySegment} based {@link IndexInput}
* implementations and will be no longer required in Lucene 10.
* @lucene.internal
*/
@Deprecated
public abstract class ByteBufferIndexInput extends IndexInput implements RandomAccessInput {
private static final FloatBuffer EMPTY_FLOATBUFFER = FloatBuffer.allocate(0);
private static final LongBuffer EMPTY_LONGBUFFER = LongBuffer.allocate(0);
private static final IntBuffer EMPTY_INTBUFFER = IntBuffer.allocate(0);
protected final long length;
protected final long chunkSizeMask;
protected final int chunkSizePower;
protected final ByteBufferGuard guard;
protected ByteBuffer[] buffers;
protected int curBufIndex = -1;
protected ByteBuffer curBuf; // redundant for speed: buffers[curBufIndex]
private LongBuffer[] curLongBufferViews;
private IntBuffer[] curIntBufferViews;
private FloatBuffer[] curFloatBufferViews;
protected boolean isClone = false;
public static ByteBufferIndexInput newInstance(
String resourceDescription,
ByteBuffer[] buffers,
long length,
int chunkSizePower,
ByteBufferGuard guard) {
if (buffers.length == 1) {
return new SingleBufferImpl(resourceDescription, buffers[0], length, chunkSizePower, guard);
} else {
return new MultiBufferImpl(resourceDescription, buffers, 0, length, chunkSizePower, guard);
}
}
ByteBufferIndexInput(
String resourceDescription,
ByteBuffer[] buffers,
long length,
int chunkSizePower,
ByteBufferGuard guard) {
super(resourceDescription);
this.buffers = buffers;
this.length = length;
this.chunkSizePower = chunkSizePower;
this.chunkSizeMask = (1L << chunkSizePower) - 1L;
this.guard = guard;
assert chunkSizePower >= 0 && chunkSizePower <= 30;
assert (length >>> chunkSizePower) < Integer.MAX_VALUE;
}
protected void setCurBuf(ByteBuffer curBuf) {
this.curBuf = curBuf;
curLongBufferViews = null;
curFloatBufferViews = null;
curIntBufferViews = null;
}
// the unused parameter is just to silence javac about unused variables
RuntimeException handlePositionalIOOBE(RuntimeException unused, String action, long pos)
throws IOException {
if (pos < 0L) {
return new IllegalArgumentException(action + " negative position (pos=" + pos + "): " + this);
} else {
throw new EOFException(action + " past EOF (pos=" + pos + "): " + this);
}
}
AlreadyClosedException alreadyClosed(NullPointerException npe) {
// we use NPE to signal if this input is closed (to not have checks everywhere). If NPE happens,
// we check the "is closed" condition explicitly by checking that our "buffers" are null or
// the guard was invalidated.
if (this.buffers == null || this.curBuf == null || guard.isInvalidated()) {
return new AlreadyClosedException("Already closed: " + this);
}
// otherwise rethrow unmodified NPE (as it possibly a bug with passing a null parameter to the
// IndexInput method):
throw npe;
}
@Override
public final byte readByte() throws IOException {
try {
return guard.getByte(curBuf);
} catch (
@SuppressWarnings("unused")
BufferUnderflowException e) {
do {
curBufIndex++;
if (curBufIndex >= buffers.length) {
throw new EOFException("read past EOF: " + this);
}
setCurBuf(buffers[curBufIndex]);
curBuf.position(0);
} while (!curBuf.hasRemaining());
return guard.getByte(curBuf);
} catch (NullPointerException e) {
throw alreadyClosed(e);
}
}
@Override
public final void readBytes(byte[] b, int offset, int len) throws IOException {
try {
guard.getBytes(curBuf, b, offset, len);
} catch (
@SuppressWarnings("unused")
BufferUnderflowException e) {
int curAvail = curBuf.remaining();
while (len > curAvail) {
guard.getBytes(curBuf, b, offset, curAvail);
len -= curAvail;
offset += curAvail;
curBufIndex++;
if (curBufIndex >= buffers.length) {
throw new EOFException("read past EOF: " + this);
}
setCurBuf(buffers[curBufIndex]);
curBuf.position(0);
curAvail = curBuf.remaining();
}
guard.getBytes(curBuf, b, offset, len);
} catch (NullPointerException e) {
throw alreadyClosed(e);
}
}
@Override
public final void readLongs(long[] dst, int offset, int length) throws IOException {
// ByteBuffer#getLong could work but it has some per-long overhead and there
// is no ByteBuffer#getLongs to read multiple longs at once. So we use the
// below trick in order to be able to leverage LongBuffer#get(long[]) to
// read multiple longs at once with as little overhead as possible.
if (curLongBufferViews == null) {
// readLELongs is only used for postings today, so we compute the long
// views lazily so that other data-structures don't have to pay for the
// associated initialization/memory overhead.
curLongBufferViews = new LongBuffer[Long.BYTES];
for (int i = 0; i < Long.BYTES; ++i) {
// Compute a view for each possible alignment. We cache these views
// because #asLongBuffer() has some cost that we don't want to pay on
// each invocation of #readLELongs.
if (i < curBuf.limit()) {
curLongBufferViews[i] =
curBuf.duplicate().position(i).order(ByteOrder.LITTLE_ENDIAN).asLongBuffer();
} else {
curLongBufferViews[i] = EMPTY_LONGBUFFER;
}
}
}
try {
final int position = curBuf.position();
guard.getLongs(
curLongBufferViews[position & 0x07].position(position >>> 3), dst, offset, length);
// if the above call succeeded, then we know the below sum cannot overflow
curBuf.position(position + (length << 3));
} catch (
@SuppressWarnings("unused")
BufferUnderflowException e) {
super.readLongs(dst, offset, length);
} catch (NullPointerException e) {
throw alreadyClosed(e);
}
}
@Override
public void readInts(int[] dst, int offset, int length) throws IOException {
// See notes about readLongs above
if (curIntBufferViews == null) {
curIntBufferViews = new IntBuffer[Integer.BYTES];
for (int i = 0; i < Integer.BYTES; ++i) {
if (i < curBuf.limit()) {
curIntBufferViews[i] =
curBuf.duplicate().position(i).order(ByteOrder.LITTLE_ENDIAN).asIntBuffer();
} else {
curIntBufferViews[i] = EMPTY_INTBUFFER;
}
}
}
try {
final int position = curBuf.position();
guard.getInts(
curIntBufferViews[position & 0x03].position(position >>> 2), dst, offset, length);
// if the above call succeeded, then we know the below sum cannot overflow
curBuf.position(position + (length << 2));
} catch (
@SuppressWarnings("unused")
BufferUnderflowException e) {
super.readInts(dst, offset, length);
} catch (NullPointerException e) {
throw alreadyClosed(e);
}
}
@Override
public final void readFloats(float[] floats, int offset, int len) throws IOException {
// See notes about readLongs above
if (curFloatBufferViews == null) {
curFloatBufferViews = new FloatBuffer[Float.BYTES];
for (int i = 0; i < Float.BYTES; ++i) {
// Compute a view for each possible alignment.
if (i < curBuf.limit()) {
ByteBuffer dup = curBuf.duplicate().order(ByteOrder.LITTLE_ENDIAN);
dup.position(i);
curFloatBufferViews[i] = dup.asFloatBuffer();
} else {
curFloatBufferViews[i] = EMPTY_FLOATBUFFER;
}
}
}
try {
final int position = curBuf.position();
FloatBuffer floatBuffer = curFloatBufferViews[position & 0x03];
floatBuffer.position(position >>> 2);
guard.getFloats(floatBuffer, floats, offset, len);
// if the above call succeeded, then we know the below sum cannot overflow
curBuf.position(position + (len << 2));
} catch (
@SuppressWarnings("unused")
BufferUnderflowException e) {
super.readFloats(floats, offset, len);
} catch (NullPointerException e) {
throw alreadyClosed(e);
}
}
@Override
public final short readShort() throws IOException {
try {
return guard.getShort(curBuf);
} catch (
@SuppressWarnings("unused")
BufferUnderflowException e) {
return super.readShort();
} catch (NullPointerException e) {
throw alreadyClosed(e);
}
}
@Override
public final int readInt() throws IOException {
try {
return guard.getInt(curBuf);
} catch (
@SuppressWarnings("unused")
BufferUnderflowException e) {
return super.readInt();
} catch (NullPointerException e) {
throw alreadyClosed(e);
}
}
@Override
public final int readVInt() throws IOException {
// this can make JVM less confused (see LUCENE-10366)
return super.readVInt();
}
@Override
public final long readVLong() throws IOException {
// this can make JVM less confused (see LUCENE-10366)
return super.readVLong();
}
@Override
public final long readLong() throws IOException {
try {
return guard.getLong(curBuf);
} catch (
@SuppressWarnings("unused")
BufferUnderflowException e) {
return super.readLong();
} catch (NullPointerException e) {
throw alreadyClosed(e);
}
}
@Override
public long getFilePointer() {
try {
return (((long) curBufIndex) << chunkSizePower) + curBuf.position();
} catch (NullPointerException e) {
throw alreadyClosed(e);
}
}
@Override
public void seek(long pos) throws IOException {
// we use >> here to preserve negative, so we will catch AIOOBE,
// in case pos + offset overflows.
final int bi = (int) (pos >> chunkSizePower);
try {
if (bi == curBufIndex) {
curBuf.position((int) (pos & chunkSizeMask));
} else {
final ByteBuffer b = buffers[bi];
b.position((int) (pos & chunkSizeMask));
// write values, on exception all is unchanged
this.curBufIndex = bi;
setCurBuf(b);
}
} catch (ArrayIndexOutOfBoundsException | IllegalArgumentException e) {
throw handlePositionalIOOBE(e, "seek", pos);
} catch (NullPointerException e) {
throw alreadyClosed(e);
}
}
@Override
public byte readByte(long pos) throws IOException {
try {
final int bi = (int) (pos >> chunkSizePower);
return guard.getByte(buffers[bi], (int) (pos & chunkSizeMask));
} catch (IndexOutOfBoundsException ioobe) {
throw handlePositionalIOOBE(ioobe, "read", pos);
} catch (NullPointerException e) {
throw alreadyClosed(e);
}
}
// used only by random access methods to handle reads across boundaries
private void setPos(long pos, int bi) throws IOException {
try {
final ByteBuffer b = buffers[bi];
b.position((int) (pos & chunkSizeMask));
this.curBufIndex = bi;
setCurBuf(b);
} catch (ArrayIndexOutOfBoundsException | IllegalArgumentException aioobe) {
throw handlePositionalIOOBE(aioobe, "read", pos);
} catch (NullPointerException e) {
throw alreadyClosed(e);
}
}
@Override
public void readBytes(long pos, byte[] bytes, int offset, int len) throws IOException {
int bi = (int) (pos >> chunkSizePower);
int bufferPos = (int) (pos & chunkSizeMask);
try {
int curAvail = Math.min(buffers[bi].capacity() - bufferPos, len);
while (len > curAvail) {
guard.getBytes(buffers[bi], bufferPos, bytes, offset, curAvail);
len -= curAvail;
offset += curAvail;
bi++;
if (bi >= buffers.length) {
throw new EOFException("read past EOF: " + this);
}
bufferPos = 0;
curAvail = Math.min(len, buffers[bi].capacity());
}
guard.getBytes(buffers[bi], bufferPos, bytes, offset, curAvail);
} catch (NullPointerException e) {
throw alreadyClosed(e);
}
}
@Override
public short readShort(long pos) throws IOException {
final int bi = (int) (pos >> chunkSizePower);
try {
return guard.getShort(buffers[bi], (int) (pos & chunkSizeMask));
} catch (
@SuppressWarnings("unused")
IndexOutOfBoundsException ioobe) {
// either it's a boundary, or read past EOF, fall back:
setPos(pos, bi);
return readShort();
} catch (NullPointerException e) {
throw alreadyClosed(e);
}
}
@Override
public int readInt(long pos) throws IOException {
final int bi = (int) (pos >> chunkSizePower);
try {
return guard.getInt(buffers[bi], (int) (pos & chunkSizeMask));
} catch (
@SuppressWarnings("unused")
IndexOutOfBoundsException ioobe) {
// either it's a boundary, or read past EOF, fall back:
setPos(pos, bi);
return readInt();
} catch (NullPointerException e) {
throw alreadyClosed(e);
}
}
@Override
public long readLong(long pos) throws IOException {
final int bi = (int) (pos >> chunkSizePower);
try {
return guard.getLong(buffers[bi], (int) (pos & chunkSizeMask));
} catch (
@SuppressWarnings("unused")
IndexOutOfBoundsException ioobe) {
// either it's a boundary, or read past EOF, fall back:
setPos(pos, bi);
return readLong();
} catch (NullPointerException e) {
throw alreadyClosed(e);
}
}
@Override
public final long length() {
return length;
}
@Override
public final ByteBufferIndexInput clone() {
final ByteBufferIndexInput clone = buildSlice((String) null, 0L, this.length);
try {
clone.seek(getFilePointer());
} catch (IOException ioe) {
throw new AssertionError(ioe);
}
return clone;
}
/**
* Creates a slice of this index input, with the given description, offset, and length. The slice
* is seeked to the beginning.
*/
@Override
public final ByteBufferIndexInput slice(String sliceDescription, long offset, long length) {
if (offset < 0 || length < 0 || offset + length > this.length) {
throw new IllegalArgumentException(
"slice() "
+ sliceDescription
+ " out of bounds: offset="
+ offset
+ ",length="
+ length
+ ",fileLength="
+ this.length
+ ": "
+ this);
}
return buildSlice(sliceDescription, offset, length);
}
/** Builds the actual sliced IndexInput (may apply extra offset in subclasses). * */
protected ByteBufferIndexInput buildSlice(String sliceDescription, long offset, long length) {
if (buffers == null || guard.isInvalidated()) {
throw alreadyClosed(null);
}
final ByteBuffer[] newBuffers = buildSlice(buffers, offset, length);
final int ofs = (int) (offset & chunkSizeMask);
final ByteBufferIndexInput clone =
newCloneInstance(getFullSliceDescription(sliceDescription), newBuffers, ofs, length);
clone.isClone = true;
return clone;
}
/**
* Factory method that creates a suitable implementation of this class for the given ByteBuffers.
*/
@SuppressWarnings("resource")
protected ByteBufferIndexInput newCloneInstance(
String newResourceDescription, ByteBuffer[] newBuffers, int offset, long length) {
if (newBuffers.length == 1) {
newBuffers[0].position(offset);
return new SingleBufferImpl(
newResourceDescription,
newBuffers[0].slice().order(ByteOrder.LITTLE_ENDIAN),
length,
chunkSizePower,
this.guard);
} else {
return new MultiBufferImpl(
newResourceDescription, newBuffers, offset, length, chunkSizePower, guard);
}
}
/**
* Returns a sliced view from a set of already-existing buffers: the last buffer's limit() will be
* correct, but you must deal with offset separately (the first buffer will not be adjusted)
*/
private ByteBuffer[] buildSlice(ByteBuffer[] buffers, long offset, long length) {
final long sliceEnd = offset + length;
final int startIndex = (int) (offset >>> chunkSizePower);
final int endIndex = (int) (sliceEnd >>> chunkSizePower);
// we always allocate one more slice, the last one may be a 0 byte one
final ByteBuffer[] slices = new ByteBuffer[endIndex - startIndex + 1];
for (int i = 0; i < slices.length; i++) {
slices[i] = buffers[startIndex + i].duplicate().order(ByteOrder.LITTLE_ENDIAN);
}
// set the last buffer's limit for the sliced view.
slices[slices.length - 1].limit((int) (sliceEnd & chunkSizeMask));
return slices;
}
@Override
public final void close() throws IOException {
try {
if (buffers == null) return;
// make local copy, then un-set early
final ByteBuffer[] bufs = buffers;
unsetBuffers();
if (isClone) return;
// tell the guard to invalidate and later unmap the bytebuffers (if supported):
guard.invalidateAndUnmap(bufs);
} finally {
unsetBuffers();
}
}
/** Called to remove all references to byte buffers, so we can throw AlreadyClosed on NPE. */
private void unsetBuffers() {
buffers = null;
curBuf = null;
curBufIndex = 0;
curLongBufferViews = null;
curIntBufferViews = null;
}
/** Optimization of ByteBufferIndexInput for when there is only one buffer */
static final class SingleBufferImpl extends ByteBufferIndexInput {
SingleBufferImpl(
String resourceDescription,
ByteBuffer buffer,
long length,
int chunkSizePower,
ByteBufferGuard guard) {
super(resourceDescription, new ByteBuffer[] {buffer}, length, chunkSizePower, guard);
this.curBufIndex = 0;
assert buffer.order() == ByteOrder.LITTLE_ENDIAN;
setCurBuf(buffer);
buffer.position(0);
}
// TODO: investigate optimizing readByte() & Co?
@Override
public void seek(long pos) throws IOException {
try {
curBuf.position((int) pos);
} catch (IllegalArgumentException e) {
throw handlePositionalIOOBE(e, "seek", pos);
} catch (NullPointerException e) {
throw alreadyClosed(e);
}
}
@Override
public long getFilePointer() {
try {
return curBuf.position();
} catch (NullPointerException e) {
throw alreadyClosed(e);
}
}
@Override
public byte readByte(long pos) throws IOException {
try {
return guard.getByte(curBuf, (int) pos);
} catch (IllegalArgumentException e) {
throw handlePositionalIOOBE(e, "read", pos);
} catch (NullPointerException e) {
throw alreadyClosed(e);
}
}
@Override
public void readBytes(long pos, byte[] bytes, int offset, int len) throws IOException {
try {
guard.getBytes(curBuf, (int) pos, bytes, offset, len);
} catch (IllegalArgumentException e) {
throw handlePositionalIOOBE(e, "read", pos);
} catch (NullPointerException e) {
throw alreadyClosed(e);
}
}
@Override
public short readShort(long pos) throws IOException {
try {
return guard.getShort(curBuf, (int) pos);
} catch (IllegalArgumentException e) {
throw handlePositionalIOOBE(e, "read", pos);
} catch (NullPointerException e) {
throw alreadyClosed(e);
}
}
@Override
public int readInt(long pos) throws IOException {
try {
return guard.getInt(curBuf, (int) pos);
} catch (IllegalArgumentException e) {
throw handlePositionalIOOBE(e, "read", pos);
} catch (NullPointerException e) {
throw alreadyClosed(e);
}
}
@Override
public long readLong(long pos) throws IOException {
try {
return guard.getLong(curBuf, (int) pos);
} catch (IllegalArgumentException e) {
throw handlePositionalIOOBE(e, "read", pos);
} catch (NullPointerException e) {
throw alreadyClosed(e);
}
}
}
/** This class adds offset support to ByteBufferIndexInput, which is needed for slices. */
static final class MultiBufferImpl extends ByteBufferIndexInput {
private final int offset;
MultiBufferImpl(
String resourceDescription,
ByteBuffer[] buffers,
int offset,
long length,
int chunkSizePower,
ByteBufferGuard guard) {
super(resourceDescription, buffers, length, chunkSizePower, guard);
this.offset = offset;
try {
seek(0L);
} catch (IOException ioe) {
throw new AssertionError(ioe);
}
}
@Override
RuntimeException handlePositionalIOOBE(RuntimeException unused, String action, long pos)
throws IOException {
return super.handlePositionalIOOBE(unused, action, pos - offset);
}
@Override
public void seek(long pos) throws IOException {
assert pos >= 0L : "negative position";
super.seek(pos + offset);
}
@Override
public long getFilePointer() {
return super.getFilePointer() - offset;
}
@Override
public byte readByte(long pos) throws IOException {
return super.readByte(pos + offset);
}
@Override
public void readBytes(long pos, byte[] bytes, int offset, int len) throws IOException {
super.readBytes(pos + this.offset, bytes, offset, len);
}
@Override
public short readShort(long pos) throws IOException {
return super.readShort(pos + offset);
}
@Override
public int readInt(long pos) throws IOException {
return super.readInt(pos + offset);
}
@Override
public long readLong(long pos) throws IOException {
return super.readLong(pos + offset);
}
@Override
protected ByteBufferIndexInput buildSlice(String sliceDescription, long ofs, long length) {
return super.buildSlice(sliceDescription, this.offset + ofs, length);
}
}
}

41
lucene/core/src/java/org/apache/lucene/store/ByteBuffersDirectory.java
View File

@ -25,7 +25,6 @@ import java.nio.file.NoSuchFileException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.Objects;
import java.util.Set;
@ -36,7 +35,6 @@ import java.util.function.Function;
import java.util.function.Supplier;
import java.util.zip.CRC32;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.util.BitUtil;
/**
* A {@link ByteBuffer}-based {@link Directory} implementation that can be used to store index files
@ -82,45 +80,6 @@ public final class ByteBuffersDirectory extends BaseDirectory {
public static final BiFunction<String, ByteBuffersDataOutput, IndexInput> OUTPUT_AS_BYTE_ARRAY =
OUTPUT_AS_ONE_BUFFER;
/**
* Use {@link ByteBufferIndexInput} for reading, otherwise identical to {@link
* #OUTPUT_AS_MANY_BUFFERS}.
*
* @deprecated Use {@link #OUTPUT_AS_MANY_BUFFERS} instead.
*/
@Deprecated
public static final BiFunction<String, ByteBuffersDataOutput, IndexInput>
OUTPUT_AS_MANY_BUFFERS_LUCENE =
(fileName, output) -> {
List<ByteBuffer> bufferList = output.toBufferList();
bufferList.add(ByteBuffer.allocate(0).order(ByteOrder.LITTLE_ENDIAN));
int chunkSizePower;
int blockSize = ByteBuffersDataInput.determineBlockPage(bufferList);
if (blockSize == 0) {
chunkSizePower = 30;
} else {
chunkSizePower =
Integer.numberOfTrailingZeros(BitUtil.nextHighestPowerOfTwo(blockSize));
}
String inputName =
String.format(
Locale.ROOT,
"%s (file=%s)",
ByteBuffersDirectory.class.getSimpleName(),
fileName);
ByteBufferGuard guard =
new ByteBufferGuard("none", (String resourceDescription, ByteBuffer b) -> {});
return ByteBufferIndexInput.newInstance(
inputName,
bufferList.toArray(new ByteBuffer[bufferList.size()]),
output.size(),
chunkSizePower,
guard);
};
private final Function<String, String> tempFileName =
new Function<String, String>() {
private final AtomicLong counter = new AtomicLong();

34
lucene/core/src/java/org/apache/lucene/store/FSDirectory.java
View File

@ -44,31 +44,27 @@ import org.apache.lucene.util.IOUtils;
/**
* Base class for Directory implementations that store index files in the file system. <a
* id="subclasses"></a> There are currently three core subclasses:
* id="subclasses"></a> There are currently two core subclasses:
*
* <ul>
* <li>{@link MMapDirectory} uses memory-mapped IO when reading. This is a good choice if you have
* plenty of virtual memory relative to your index size, eg if you are running on a 64 bit
* JRE, or you are running on a 32 bit JRE but your index sizes are small enough to fit into
* the virtual memory space. This class will use the modern {@link
* java.lang.foreign.MemorySegment} API available since Java 21 which allows to safely unmap
* previously mmapped files after closing the {@link IndexInput}s. There is no need to enable
* the "preview feature" of your Java version; it works out of box with some compilation
* tricks. For more information about the foreign memory API read documentation of the {@link
* java.lang.foreign} package and <a
* href="https://blog.thetaphi.de/2012/07/use-lucenes-mmapdirectory-on-64bit.html">Uwe's blog
* post</a>.
* <li>{@link NIOFSDirectory} uses java.nio's FileChannel's positional io when reading to avoid
* synchronization when reading from the same file. Unfortunately, due to a Windows-only <a
* href="http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6265734">Sun JRE bug</a> this is a
* poor choice for Windows, but on all other platforms this is the preferred choice.
* Applications using {@link Thread#interrupt()} or {@link Future#cancel(boolean)} should use
* {@code RAFDirectory} instead. See {@link NIOFSDirectory} java doc for details.
* <li>{@link MMapDirectory} uses memory-mapped IO when reading. This is a good choice if you have
* plenty of virtual memory relative to your index size, eg if you are running on a 64 bit
* JRE, or you are running on a 32 bit JRE but your index sizes are small enough to fit into
* the virtual memory space. Java has currently the limitation of not being able to unmap
* files from user code. The files are unmapped, when GC releases the byte buffers. Due to <a
* href="http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4724038">this bug</a> in Sun's
* JRE, MMapDirectory's {@link IndexInput#close} is unable to close the underlying OS file
* handle. Only when GC finally collects the underlying objects, which could be quite some
* time later, will the file handle be closed. This will consume additional transient disk
* usage: on Windows, attempts to delete or overwrite the files will result in an exception;
* on other platforms, which typically have a &quot;delete on last close&quot; semantics,
* while such operations will succeed, the bytes are still consuming space on disk. For many
* applications this limitation is not a problem (e.g. if you have plenty of disk space, and
* you don't rely on overwriting files on Windows) but it's still an important limitation to
* be aware of. This class supplies a (possibly dangerous) workaround mentioned in the bug
* report, which may fail on non-Sun JVMs.
* {@code RAFDirectory} instead, which is provided in the {@code misc} module. See {@link
* NIOFSDirectory} javadoc for details.
* </ul>
*
* <p>Unfortunately, because of system peculiarities, there is no single overall best
@ -157,7 +153,7 @@ public abstract class FSDirectory extends BaseDirectory {
/** Just like {@link #open(Path)}, but allows you to also specify a custom {@link LockFactory}. */
public static FSDirectory open(Path path, LockFactory lockFactory) throws IOException {
if (Constants.JRE_IS_64BIT && MMapDirectory.UNMAP_SUPPORTED) {
if (Constants.JRE_IS_64BIT) {
return new MMapDirectory(path, lockFactory);
} else {
return new NIOFSDirectory(path, lockFactory);

153
lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java
View File

@ -21,15 +21,10 @@ import java.lang.invoke.MethodHandles;
import java.lang.invoke.MethodType;
import java.nio.channels.ClosedChannelException; // javadoc @link
import java.nio.file.Path;
import java.security.AccessController;
import java.security.PrivilegedAction;
import java.util.Locale;
import java.util.Optional;
import java.util.concurrent.Future;
import java.util.function.BiPredicate;
import java.util.logging.Logger;
import org.apache.lucene.util.Constants;
import org.apache.lucene.util.SuppressForbidden;
/**
* File-based {@link Directory} implementation that uses mmap for reading, and {@link
@ -40,45 +35,18 @@ import org.apache.lucene.util.SuppressForbidden;
* plenty of virtual address space, e.g. by using a 64 bit JRE, or a 32 bit JRE with indexes that
* are guaranteed to fit within the address space. On 32 bit platforms also consult {@link
* #MMapDirectory(Path, LockFactory, long)} if you have problems with mmap failing because of
* fragmented address space. If you get an OutOfMemoryException, it is recommended to reduce the
* chunk size, until it works.
* fragmented address space. If you get an {@link IOException} about mapping failed, it is
* recommended to reduce the chunk size, until it works.
*
* <p>This class supports preloading files into physical memory upon opening. This can help improve
* performance of searches on a cold page cache at the expense of slowing down opening an index. See
* {@link #setPreload(BiPredicate)} for more details.
*
* <p>Due to <a href="https://bugs.openjdk.org/browse/JDK-4724038">this bug</a> in OpenJDK,
* MMapDirectory's {@link IndexInput#close} is unable to close the underlying OS file handle. Only
* when GC finally collects the underlying objects, which could be quite some time later, will the
* file handle be closed.
*
* <p>This will consume additional transient disk usage: on Windows, attempts to delete or overwrite
* the files will result in an exception; on other platforms, which typically have a &quot;delete on
* last close&quot; semantics, while such operations will succeed, the bytes are still consuming
* space on disk. For many applications this limitation is not a problem (e.g. if you have plenty of
* disk space, and you don't rely on overwriting files on Windows) but it's still an important
* limitation to be aware of.
*
* <p>This class supplies the workaround mentioned in the bug report, which may fail on
* non-Oracle/OpenJDK JVMs. It forcefully unmaps the buffer on close by using an undocumented
* internal cleanup functionality. If {@link #UNMAP_SUPPORTED} is <code>true</code>, the workaround
* will be automatically enabled (with no guarantees; if you discover any problems, you can disable
* it by using system property {@link #ENABLE_UNMAP_HACK_SYSPROP}).
*
* <p>For the hack to work correct, the following requirements need to be fulfilled: The used JVM
* must be at least Oracle Java / OpenJDK. In addition, the following permissions need to be granted
* to {@code lucene-core.jar} in your <a
* href="http://docs.oracle.com/javase/8/docs/technotes/guides/security/PolicyFiles.html">policy
* file</a>:
*
* <ul>
* <li>{@code permission java.lang.reflect.ReflectPermission "suppressAccessChecks";}
* <li>{@code permission java.lang.RuntimePermission "accessClassInPackage.sun.misc";}
* </ul>
*
* <p>On exactly <b>Java 19 / 20 / 21</b> this class will use the modern {@code MemorySegment} API
* which allows to safely unmap (if you discover any problems with this preview API, you can disable
* it by using system property {@link #ENABLE_MEMORY_SEGMENTS_SYSPROP}).
* <p>This class will use the modern {@link java.lang.foreign.MemorySegment} API available since
* Java 21 which allows to safely unmap previously mmapped files after closing the {@link
* IndexInput}s. There is no need to enable the "preview feature" of your Java version; it works out
* of box with some compilation tricks. For more information about the foreign memory API read
* documentation of the {@link java.lang.foreign} package.
*
* <p><b>NOTE:</b> Accessing this class either directly or indirectly from a thread while it's
* interrupted can close the underlying channel immediately if at the same time the thread is
@ -91,13 +59,11 @@ import org.apache.lucene.util.SuppressForbidden;
* synchronize on the <code>MMapDirectory</code> instance as this may cause deadlock; use your own
* (non-Lucene) objects instead.
*
* @see <a href="http://blog.thetaphi.de/2012/07/use-lucenes-mmapdirectory-on-64bit.html">Blog post
* @see <a href="https://blog.thetaphi.de/2012/07/use-lucenes-mmapdirectory-on-64bit.html">Blog post
* about MMapDirectory</a>
*/
public class MMapDirectory extends FSDirectory {
private static final Logger LOG = Logger.getLogger(MMapDirectory.class.getName());
/**
* Argument for {@link #setPreload(BiPredicate)} that configures all files to be preloaded upon
* opening them.
@ -123,35 +89,12 @@ public class MMapDirectory extends FSDirectory {
* Default max chunk size:
*
* <ul>
* <li>16 GiBytes for 64 bit <b>Java 19 / 20 / 21</b> JVMs
* <li>1 GiBytes for other 64 bit JVMs
* <li>16 GiBytes for 64 bit JVMs
* <li>256 MiBytes for 32 bit JVMs
* </ul>
*/
public static final long DEFAULT_MAX_CHUNK_SIZE;
/**
* This sysprop allows to control the workaround/hack for unmapping the buffers from address space
* after closing {@link IndexInput}. By default it is enabled; set to {@code false} to disable the
* unmap hack globally. On command line pass {@code
* -Dorg.apache.lucene.store.MMapDirectory.enableUnmapHack=false} to disable.
*
* @lucene.internal
*/
public static final String ENABLE_UNMAP_HACK_SYSPROP =
"org.apache.lucene.store.MMapDirectory.enableUnmapHack";
/**
* This sysprop allows to control if {@code MemorySegment} API should be used on supported Java
* versions. By default it is enabled; set to {@code false} to use legacy {@code ByteBuffer}
* implementation. On command line pass {@code
* -Dorg.apache.lucene.store.MMapDirectory.enableMemorySegments=false} to disable.
*
* @lucene.internal
*/
public static final String ENABLE_MEMORY_SEGMENTS_SYSPROP =
"org.apache.lucene.store.MMapDirectory.enableMemorySegments";
final int chunkSizePower;
/**
@ -255,25 +198,12 @@ public class MMapDirectory extends FSDirectory {
// visible for tests:
static final MMapIndexInputProvider PROVIDER;
/** <code>true</code>, if this platform supports unmapping mmapped files. */
public static final boolean UNMAP_SUPPORTED;
/**
* if {@link #UNMAP_SUPPORTED} is {@code false}, this contains the reason why unmapping is not
* supported.
*/
public static final String UNMAP_NOT_SUPPORTED_REASON;
interface MMapIndexInputProvider {
IndexInput openInput(Path path, IOContext context, int chunkSizePower, boolean preload)
throws IOException;
long getDefaultMaxChunkSize();
boolean isUnmapSupported();
String getUnmapNotSupportedReason();
default IOException convertMapFailedIOException(
IOException ioe, String resourceDescription, long bufSize) {
final String originalMessage;
@ -317,63 +247,30 @@ public class MMapDirectory extends FSDirectory {
}
}
// Extracted to a method to be able to apply the SuppressForbidden annotation
@SuppressWarnings("removal")
@SuppressForbidden(reason = "security manager")
private static <T> T doPrivileged(PrivilegedAction<T> action) {
return AccessController.doPrivileged(action);
}
private static boolean checkMemorySegmentsSysprop() {
try {
return Optional.ofNullable(System.getProperty(ENABLE_MEMORY_SEGMENTS_SYSPROP))
.map(Boolean::valueOf)
.orElse(Boolean.TRUE);
} catch (
@SuppressWarnings("unused")
SecurityException ignored) {
LOG.warning(
"Cannot read sysprop "
+ ENABLE_MEMORY_SEGMENTS_SYSPROP
+ ", so MemorySegments will be enabled by default, if possible.");
return true;
}
}
private static MMapIndexInputProvider lookupProvider() {
if (checkMemorySegmentsSysprop() == false) {
return new MappedByteBufferIndexInputProvider();
}
final var lookup = MethodHandles.lookup();
final int runtimeVersion = Runtime.version().feature();
if (runtimeVersion >= 19) {
try {
final var cls = lookup.findClass("org.apache.lucene.store.MemorySegmentIndexInputProvider");
// we use method handles, so we do not need to deal with setAccessible as we have private
// access through the lookup:
final var constr = lookup.findConstructor(cls, MethodType.methodType(void.class));
try {
final var cls = lookup.findClass("org.apache.lucene.store.MemorySegmentIndexInputProvider");
// we use method handles, so we do not need to deal with setAccessible as we have private
// access through the lookup:
final var constr = lookup.findConstructor(cls, MethodType.methodType(void.class));
try {
return (MMapIndexInputProvider) constr.invoke();
} catch (RuntimeException | Error e) {
throw e;
} catch (Throwable th) {
throw new AssertionError(th);
}
} catch (NoSuchMethodException | IllegalAccessException e) {
throw new LinkageError(
"MemorySegmentIndexInputProvider is missing correctly typed constructor", e);
} catch (ClassNotFoundException cnfe) {
throw new LinkageError(
"MemorySegmentIndexInputProvider is missing in Lucene JAR file", cnfe);
return (MMapIndexInputProvider) constr.invoke();
} catch (RuntimeException | Error e) {
throw e;
} catch (Throwable th) {
throw new AssertionError(th);
}
} catch (NoSuchMethodException | IllegalAccessException e) {
throw new LinkageError(
"MemorySegmentIndexInputProvider is missing correctly typed constructor", e);
} catch (ClassNotFoundException cnfe) {
throw new LinkageError("MemorySegmentIndexInputProvider is missing in Lucene JAR file", cnfe);
}
return new MappedByteBufferIndexInputProvider();
}
static {
PROVIDER = doPrivileged(MMapDirectory::lookupProvider);
PROVIDER = lookupProvider();
DEFAULT_MAX_CHUNK_SIZE = PROVIDER.getDefaultMaxChunkSize();
UNMAP_SUPPORTED = PROVIDER.isUnmapSupported();
UNMAP_NOT_SUPPORTED_REASON = PROVIDER.getUnmapNotSupportedReason();
}
}

205
lucene/core/src/java/org/apache/lucene/store/MappedByteBufferIndexInputProvider.java
View File

@ -1,205 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.store;
import static java.lang.invoke.MethodHandles.lookup;
import static java.lang.invoke.MethodType.methodType;
import java.io.IOException;
import java.lang.invoke.MethodHandle;
import java.lang.invoke.MethodHandles.Lookup;
import java.lang.reflect.Field;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.channels.FileChannel.MapMode;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.Objects;
import java.util.Optional;
import java.util.logging.Logger;
import org.apache.lucene.store.ByteBufferGuard.BufferCleaner;
import org.apache.lucene.util.Constants;
import org.apache.lucene.util.SuppressForbidden;
final class MappedByteBufferIndexInputProvider implements MMapDirectory.MMapIndexInputProvider {
private static final Logger LOG =
Logger.getLogger(MappedByteBufferIndexInputProvider.class.getName());
private final BufferCleaner cleaner;
private final boolean unmapSupported;
private final String unmapNotSupportedReason;
public MappedByteBufferIndexInputProvider() {
final Object hack = unmapHackImpl();
if (hack instanceof BufferCleaner) {
cleaner = (BufferCleaner) hack;
unmapSupported = true;
unmapNotSupportedReason = null;
} else {
cleaner = null;
unmapSupported = false;
unmapNotSupportedReason = hack.toString();
LOG.warning(unmapNotSupportedReason);
}
}
@Override
public IndexInput openInput(Path path, IOContext context, int chunkSizePower, boolean preload)
throws IOException {
if (chunkSizePower > 30) {
throw new IllegalArgumentException(
"ByteBufferIndexInput cannot use a chunk size of >1 GiBytes.");
}
final String resourceDescription = "ByteBufferIndexInput(path=\"" + path.toString() + "\")";
try (var fc = FileChannel.open(path, StandardOpenOption.READ)) {
final long fileSize = fc.size();
return ByteBufferIndexInput.newInstance(
resourceDescription,
map(resourceDescription, fc, chunkSizePower, preload, fileSize),
fileSize,
chunkSizePower,
new ByteBufferGuard(resourceDescription, cleaner));
}
}
@Override
public long getDefaultMaxChunkSize() {
return Constants.JRE_IS_64BIT ? (1L << 30) : (1L << 28);
}
@Override
public boolean isUnmapSupported() {
return unmapSupported;
}
@Override
public String getUnmapNotSupportedReason() {
return unmapNotSupportedReason;
}
/** Maps a file into a set of buffers */
final ByteBuffer[] map(
String resourceDescription, FileChannel fc, int chunkSizePower, boolean preload, long length)
throws IOException {
if ((length >>> chunkSizePower) >= Integer.MAX_VALUE)
throw new IllegalArgumentException(
"RandomAccessFile too big for chunk size: " + resourceDescription);
final long chunkSize = 1L << chunkSizePower;
// we always allocate one more buffer, the last one may be a 0 byte one
final int nrBuffers = (int) (length >>> chunkSizePower) + 1;
final ByteBuffer[] buffers = new ByteBuffer[nrBuffers];
long startOffset = 0L;
for (int bufNr = 0; bufNr < nrBuffers; bufNr++) {
final int bufSize =
(int) ((length > (startOffset + chunkSize)) ? chunkSize : (length - startOffset));
final MappedByteBuffer buffer;
try {
buffer = fc.map(MapMode.READ_ONLY, startOffset, bufSize);
buffer.order(ByteOrder.LITTLE_ENDIAN);
} catch (IOException ioe) {
throw convertMapFailedIOException(ioe, resourceDescription, bufSize);
}
if (preload) {
buffer.load();
}
buffers[bufNr] = buffer;
startOffset += bufSize;
}
return buffers;
}
private static boolean checkUnmapHackSysprop() {
try {
return Optional.ofNullable(System.getProperty(MMapDirectory.ENABLE_UNMAP_HACK_SYSPROP))
.map(Boolean::valueOf)
.orElse(Boolean.TRUE);
} catch (
@SuppressWarnings("unused")
SecurityException ignored) {
LOG.warning(
"Cannot read sysprop "
+ MMapDirectory.ENABLE_UNMAP_HACK_SYSPROP
+ ", so buffer unmap hack will be enabled by default, if possible.");
return true;
}
}
@SuppressForbidden(reason = "Needs access to sun.misc.Unsafe to enable hack")
private static Object unmapHackImpl() {
if (checkUnmapHackSysprop() == false) {
return "Unmapping was disabled by system property "
+ MMapDirectory.ENABLE_UNMAP_HACK_SYSPROP
+ "=false";
}
final Lookup lookup = lookup();
try {
// *** sun.misc.Unsafe unmapping (Java 9+) ***
final Class<?> unsafeClass = lookup.findClass("sun.misc.Unsafe");
// first check if Unsafe has the right method, otherwise we can give up
// without doing any security critical stuff:
final MethodHandle unmapper =
lookup.findVirtual(
unsafeClass, "invokeCleaner", methodType(void.class, ByteBuffer.class));
// fetch the unsafe instance and bind it to the virtual MH:
final Field f = unsafeClass.getDeclaredField("theUnsafe");
f.setAccessible(true);
final Object theUnsafe = f.get(null);
return newBufferCleaner(unmapper.bindTo(theUnsafe));
} catch (SecurityException se) {
return "Unmapping is not supported, because not all required permissions are given to the Lucene JAR file: "
+ se
+ " [Please grant at least the following permissions: RuntimePermission(\"accessClassInPackage.sun.misc\") "
+ " and ReflectPermission(\"suppressAccessChecks\")]";
} catch (ReflectiveOperationException | RuntimeException e) {
final Module module = MappedByteBufferIndexInputProvider.class.getModule();
final ModuleLayer layer = module.getLayer();
// classpath / unnamed module has no layer, so we need to check:
if (layer != null
&& layer.findModule("jdk.unsupported").map(module::canRead).orElse(false) == false) {
return "Unmapping is not supported, because Lucene cannot read 'jdk.unsupported' module "
+ "[please add 'jdk.unsupported' to modular application either by command line or its module descriptor]";
}
return "Unmapping is not supported on this platform, because internal Java APIs are not compatible with this Lucene version: "
+ e;
}
}
private static BufferCleaner newBufferCleaner(final MethodHandle unmapper) {
assert Objects.equals(methodType(void.class, ByteBuffer.class), unmapper.type());
return (String resourceDescription, ByteBuffer buffer) -> {
if (!buffer.isDirect()) {
throw new IllegalArgumentException("unmapping only works with direct buffers");
}
try {
unmapper.invokeExact(buffer);
} catch (Throwable t) {
throw new IOException("Unable to unmap the mapped buffer: " + resourceDescription, t);
}
};
}
}

2
lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java
View File

@ -74,7 +74,7 @@ public class NRTCachingDirectory extends FilterDirectory implements Accountable
ByteBuffersDataOutput::new,
(fileName, content) -> {
cacheSize.addAndGet(content.size());
return ByteBuffersDirectory.OUTPUT_AS_MANY_BUFFERS_LUCENE.apply(fileName, content);
return ByteBuffersDirectory.OUTPUT_AS_MANY_BUFFERS.apply(fileName, content);
});
private final long maxMergeSizeBytes;

683
lucene/core/src/java19/org/apache/lucene/store/MemorySegmentIndexInput.java
View File

@ -1,683 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.store;
import java.io.EOFException;
import java.io.IOException;
import java.lang.foreign.MemorySegment;
import java.lang.foreign.MemorySession;
import java.lang.foreign.ValueLayout;
import java.nio.ByteOrder;
import java.util.Arrays;
import java.util.Objects;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.GroupVIntUtil;
/**
* Base IndexInput implementation that uses an array of MemorySegments to represent a file.
*
* <p>For efficiency, this class requires that the segment size are a power-of-two (<code>
* chunkSizePower</code>).
*/
@SuppressWarnings("preview")
abstract class MemorySegmentIndexInput extends IndexInput implements RandomAccessInput {
static final ValueLayout.OfByte LAYOUT_BYTE = ValueLayout.JAVA_BYTE;
static final ValueLayout.OfShort LAYOUT_LE_SHORT =
ValueLayout.JAVA_SHORT.withOrder(ByteOrder.LITTLE_ENDIAN).withBitAlignment(8);
static final ValueLayout.OfInt LAYOUT_LE_INT =
ValueLayout.JAVA_INT.withOrder(ByteOrder.LITTLE_ENDIAN).withBitAlignment(8);
static final ValueLayout.OfLong LAYOUT_LE_LONG =
ValueLayout.JAVA_LONG.withOrder(ByteOrder.LITTLE_ENDIAN).withBitAlignment(8);
static final ValueLayout.OfFloat LAYOUT_LE_FLOAT =
ValueLayout.JAVA_FLOAT.withOrder(ByteOrder.LITTLE_ENDIAN).withBitAlignment(8);
final long length;
final long chunkSizeMask;
final int chunkSizePower;
final MemorySession session;
final MemorySegment[] segments;
int curSegmentIndex = -1;
MemorySegment
curSegment; // redundant for speed: segments[curSegmentIndex], also marker if closed!
long curPosition; // relative to curSegment, not globally
public static MemorySegmentIndexInput newInstance(
String resourceDescription,
MemorySession session,
MemorySegment[] segments,
long length,
int chunkSizePower) {
assert Arrays.stream(segments).map(MemorySegment::session).allMatch(session::equals);
if (segments.length == 1) {
return new SingleSegmentImpl(
resourceDescription, session, segments[0], length, chunkSizePower);
} else {
return new MultiSegmentImpl(
resourceDescription, session, segments, 0, length, chunkSizePower);
}
}
private MemorySegmentIndexInput(
String resourceDescription,
MemorySession session,
MemorySegment[] segments,
long length,
int chunkSizePower) {
super(resourceDescription);
this.session = session;
this.segments = segments;
this.length = length;
this.chunkSizePower = chunkSizePower;
this.chunkSizeMask = (1L << chunkSizePower) - 1L;
this.curSegment = segments[0];
}
void ensureOpen() {
if (curSegment == null) {
throw alreadyClosed(null);
}
}
// the unused parameter is just to silence javac about unused variables
RuntimeException handlePositionalIOOBE(RuntimeException unused, String action, long pos)
throws IOException {
if (pos < 0L) {
return new IllegalArgumentException(action + " negative position (pos=" + pos + "): " + this);
} else {
throw new EOFException(action + " past EOF (pos=" + pos + "): " + this);
}
}
AlreadyClosedException alreadyClosed(RuntimeException e) {
// we use NPE to signal if this input is closed (to not have checks everywhere). If NPE happens,
// we check the "is closed" condition explicitly by checking that our "curSegment" is null.
// Care must be taken to not leak the NPE to code outside MemorySegmentIndexInput!
if (this.curSegment == null) {
return new AlreadyClosedException("Already closed: " + this);
}
// ISE can be thrown by MemorySegment and contains "closed" in message:
if (e instanceof IllegalStateException
&& e.getMessage() != null
&& e.getMessage().contains("closed")) {
return new AlreadyClosedException("Already closed: " + this, e);
}
// otherwise rethrow unmodified NPE/ISE (as it possibly a bug with passing a null parameter to
// the IndexInput method):
throw e;
}
@Override
public final byte readByte() throws IOException {
try {
final byte v = curSegment.get(LAYOUT_BYTE, curPosition);
curPosition++;
return v;
} catch (
@SuppressWarnings("unused")
IndexOutOfBoundsException e) {
do {
curSegmentIndex++;
if (curSegmentIndex >= segments.length) {
throw new EOFException("read past EOF: " + this);
}
curSegment = segments[curSegmentIndex];
curPosition = 0L;
} while (curSegment.byteSize() == 0L);
final byte v = curSegment.get(LAYOUT_BYTE, curPosition);
curPosition++;
return v;
} catch (NullPointerException | IllegalStateException e) {
throw alreadyClosed(e);
}
}
@Override
public final void readBytes(byte[] b, int offset, int len) throws IOException {
try {
MemorySegment.copy(curSegment, LAYOUT_BYTE, curPosition, b, offset, len);
curPosition += len;
} catch (
@SuppressWarnings("unused")
IndexOutOfBoundsException e) {
readBytesBoundary(b, offset, len);
} catch (NullPointerException | IllegalStateException e) {
throw alreadyClosed(e);
}
}
private void readBytesBoundary(byte[] b, int offset, int len) throws IOException {
try {
long curAvail = curSegment.byteSize() - curPosition;
while (len > curAvail) {
MemorySegment.copy(curSegment, LAYOUT_BYTE, curPosition, b, offset, (int) curAvail);
len -= curAvail;
offset += curAvail;
curSegmentIndex++;
if (curSegmentIndex >= segments.length) {
throw new EOFException("read past EOF: " + this);
}
curSegment = segments[curSegmentIndex];
curPosition = 0L;
curAvail = curSegment.byteSize();
}
MemorySegment.copy(curSegment, LAYOUT_BYTE, curPosition, b, offset, len);
curPosition += len;
} catch (NullPointerException | IllegalStateException e) {
throw alreadyClosed(e);
}
}
@Override
public void readInts(int[] dst, int offset, int length) throws IOException {
try {
MemorySegment.copy(curSegment, LAYOUT_LE_INT, curPosition, dst, offset, length);
curPosition += Integer.BYTES * (long) length;
} catch (
@SuppressWarnings("unused")
IndexOutOfBoundsException iobe) {
super.readInts(dst, offset, length);
} catch (NullPointerException | IllegalStateException e) {
throw alreadyClosed(e);
}
}
@Override
public void readLongs(long[] dst, int offset, int length) throws IOException {
try {
MemorySegment.copy(curSegment, LAYOUT_LE_LONG, curPosition, dst, offset, length);
curPosition += Long.BYTES * (long) length;
} catch (
@SuppressWarnings("unused")
IndexOutOfBoundsException iobe) {
super.readLongs(dst, offset, length);
} catch (NullPointerException | IllegalStateException e) {
throw alreadyClosed(e);
}
}
@Override
public void readFloats(float[] dst, int offset, int length) throws IOException {
try {
MemorySegment.copy(curSegment, LAYOUT_LE_FLOAT, curPosition, dst, offset, length);
curPosition += Float.BYTES * (long) length;
} catch (
@SuppressWarnings("unused")
IndexOutOfBoundsException iobe) {
super.readFloats(dst, offset, length);
} catch (NullPointerException | IllegalStateException e) {
throw alreadyClosed(e);
}
}
@Override
public final short readShort() throws IOException {
try {
final short v = curSegment.get(LAYOUT_LE_SHORT, curPosition);
curPosition += Short.BYTES;
return v;
} catch (
@SuppressWarnings("unused")
IndexOutOfBoundsException e) {
return super.readShort();
} catch (NullPointerException | IllegalStateException e) {
throw alreadyClosed(e);
}
}
@Override
public final int readInt() throws IOException {
try {
final int v = curSegment.get(LAYOUT_LE_INT, curPosition);
curPosition += Integer.BYTES;
return v;
} catch (
@SuppressWarnings("unused")
IndexOutOfBoundsException e) {
return super.readInt();
} catch (NullPointerException | IllegalStateException e) {
throw alreadyClosed(e);
}
}
@Override
public final int readVInt() throws IOException {
// this can make JVM less confused (see LUCENE-10366)
return super.readVInt();
}
@Override
public final long readVLong() throws IOException {
// this can make JVM less confused (see LUCENE-10366)
return super.readVLong();
}
@Override
public final long readLong() throws IOException {
try {
final long v = curSegment.get(LAYOUT_LE_LONG, curPosition);
curPosition += Long.BYTES;
return v;
} catch (
@SuppressWarnings("unused")
IndexOutOfBoundsException e) {
return super.readLong();
} catch (NullPointerException | IllegalStateException e) {
throw alreadyClosed(e);
}
}
@Override
public long getFilePointer() {
ensureOpen();
return (((long) curSegmentIndex) << chunkSizePower) + curPosition;
}
@Override
public void seek(long pos) throws IOException {
ensureOpen();
// we use >> here to preserve negative, so we will catch AIOOBE,
// in case pos + offset overflows.
final int si = (int) (pos >> chunkSizePower);
try {
if (si != curSegmentIndex) {
final MemorySegment seg = segments[si];
// write values, on exception all is unchanged
this.curSegmentIndex = si;
this.curSegment = seg;
}
this.curPosition = Objects.checkIndex(pos & chunkSizeMask, curSegment.byteSize() + 1);
} catch (IndexOutOfBoundsException e) {
throw handlePositionalIOOBE(e, "seek", pos);
}
}
@Override
public byte readByte(long pos) throws IOException {
try {
final int si = (int) (pos >> chunkSizePower);
return segments[si].get(LAYOUT_BYTE, pos & chunkSizeMask);
} catch (IndexOutOfBoundsException ioobe) {
throw handlePositionalIOOBE(ioobe, "read", pos);
} catch (NullPointerException | IllegalStateException e) {
throw alreadyClosed(e);
}
}
@Override
protected void readGroupVInt(long[] dst, int offset) throws IOException {
try {
final int len =
GroupVIntUtil.readGroupVInt(
this,
curSegment.byteSize() - curPosition,
p -> curSegment.get(LAYOUT_LE_INT, p),
curPosition,
dst,
offset);
curPosition += len;
} catch (NullPointerException | IllegalStateException e) {
throw alreadyClosed(e);
}
}
@Override
public void readBytes(long pos, byte[] b, int offset, int len) throws IOException {
try {
int si = (int) (pos >> chunkSizePower);
pos = pos & chunkSizeMask;
long curAvail = segments[si].byteSize() - pos;
while (len > curAvail) {
MemorySegment.copy(segments[si], LAYOUT_BYTE, pos, b, offset, (int) curAvail);
len -= curAvail;
offset += curAvail;
si++;
if (si >= segments.length) {
throw new EOFException("read past EOF: " + this);
}
pos = 0L;
curAvail = segments[si].byteSize();
}
MemorySegment.copy(segments[si], LAYOUT_BYTE, pos, b, offset, len);
} catch (IndexOutOfBoundsException ioobe) {
throw handlePositionalIOOBE(ioobe, "read", pos);
} catch (NullPointerException | IllegalStateException e) {
throw alreadyClosed(e);
}
}
// used only by random access methods to handle reads across boundaries
private void setPos(long pos, int si) throws IOException {
try {
final MemorySegment seg = segments[si];
// write values, on exception above all is unchanged
this.curPosition = pos & chunkSizeMask;
this.curSegmentIndex = si;
this.curSegment = seg;
} catch (IndexOutOfBoundsException ioobe) {
throw handlePositionalIOOBE(ioobe, "read", pos);
} catch (NullPointerException | IllegalStateException e) {
throw alreadyClosed(e);
}
}
@Override
public short readShort(long pos) throws IOException {
final int si = (int) (pos >> chunkSizePower);
try {
return segments[si].get(LAYOUT_LE_SHORT, pos & chunkSizeMask);
} catch (
@SuppressWarnings("unused")
IndexOutOfBoundsException ioobe) {
// either it's a boundary, or read past EOF, fall back:
setPos(pos, si);
return readShort();
} catch (NullPointerException | IllegalStateException e) {
throw alreadyClosed(e);
}
}
@Override
public int readInt(long pos) throws IOException {
final int si = (int) (pos >> chunkSizePower);
try {
return segments[si].get(LAYOUT_LE_INT, pos & chunkSizeMask);
} catch (
@SuppressWarnings("unused")
IndexOutOfBoundsException ioobe) {
// either it's a boundary, or read past EOF, fall back:
setPos(pos, si);
return readInt();
} catch (NullPointerException | IllegalStateException e) {
throw alreadyClosed(e);
}
}
@Override
public long readLong(long pos) throws IOException {
final int si = (int) (pos >> chunkSizePower);
try {
return segments[si].get(LAYOUT_LE_LONG, pos & chunkSizeMask);
} catch (
@SuppressWarnings("unused")
IndexOutOfBoundsException ioobe) {
// either it's a boundary, or read past EOF, fall back:
setPos(pos, si);
return readLong();
} catch (NullPointerException | IllegalStateException e) {
throw alreadyClosed(e);
}
}
@Override
public final long length() {
return length;
}
@Override
public final MemorySegmentIndexInput clone() {
final MemorySegmentIndexInput clone = buildSlice((String) null, 0L, this.length);
try {
clone.seek(getFilePointer());
} catch (IOException ioe) {
throw new AssertionError(ioe);
}
return clone;
}
/**
* Creates a slice of this index input, with the given description, offset, and length. The slice
* is seeked to the beginning.
*/
@Override
public final MemorySegmentIndexInput slice(String sliceDescription, long offset, long length) {
if (offset < 0 || length < 0 || offset + length > this.length) {
throw new IllegalArgumentException(
"slice() "
+ sliceDescription
+ " out of bounds: offset="
+ offset
+ ",length="
+ length
+ ",fileLength="
+ this.length
+ ": "
+ this);
}
return buildSlice(sliceDescription, offset, length);
}
/** Builds the actual sliced IndexInput (may apply extra offset in subclasses). * */
MemorySegmentIndexInput buildSlice(String sliceDescription, long offset, long length) {
ensureOpen();
final long sliceEnd = offset + length;
final int startIndex = (int) (offset >>> chunkSizePower);
final int endIndex = (int) (sliceEnd >>> chunkSizePower);
// we always allocate one more slice, the last one may be a 0 byte one after truncating with
// asSlice():
final MemorySegment slices[] = ArrayUtil.copyOfSubArray(segments, startIndex, endIndex + 1);
// set the last segment's limit for the sliced view.
slices[slices.length - 1] = slices[slices.length - 1].asSlice(0L, sliceEnd & chunkSizeMask);
offset = offset & chunkSizeMask;
final String newResourceDescription = getFullSliceDescription(sliceDescription);
if (slices.length == 1) {
return new SingleSegmentImpl(
newResourceDescription,
null, // clones don't have a MemorySession, as they can't close)
slices[0].asSlice(offset, length),
length,
chunkSizePower);
} else {
return new MultiSegmentImpl(
newResourceDescription,
null, // clones don't have a MemorySession, as they can't close)
slices,
offset,
length,
chunkSizePower);
}
}
@Override
public final void close() throws IOException {
if (curSegment == null) {
return;
}
// the master IndexInput has a MemorySession and is able
// to release all resources (unmap segments) - a
// side effect is that other threads still using clones
// will throw IllegalStateException
if (session != null) {
while (session.isAlive()) {
try {
session.close();
break;
} catch (
@SuppressWarnings("unused")
IllegalStateException e) {
Thread.onSpinWait();
}
}
}
// make sure all accesses to this IndexInput instance throw NPE:
curSegment = null;
Arrays.fill(segments, null);
}
/** Optimization of MemorySegmentIndexInput for when there is only one segment. */
static final class SingleSegmentImpl extends MemorySegmentIndexInput {
SingleSegmentImpl(
String resourceDescription,
MemorySession session,
MemorySegment segment,
long length,
int chunkSizePower) {
super(resourceDescription, session, new MemorySegment[] {segment}, length, chunkSizePower);
this.curSegmentIndex = 0;
}
@Override
public void seek(long pos) throws IOException {
ensureOpen();
try {
curPosition = Objects.checkIndex(pos, length + 1);
} catch (IndexOutOfBoundsException e) {
throw handlePositionalIOOBE(e, "seek", pos);
}
}
@Override
public long getFilePointer() {
ensureOpen();
return curPosition;
}
@Override
public byte readByte(long pos) throws IOException {
try {
return curSegment.get(LAYOUT_BYTE, pos);
} catch (IndexOutOfBoundsException e) {
throw handlePositionalIOOBE(e, "read", pos);
} catch (NullPointerException | IllegalStateException e) {
throw alreadyClosed(e);
}
}
@Override
public void readBytes(long pos, byte[] bytes, int offset, int length) throws IOException {
try {
MemorySegment.copy(curSegment, LAYOUT_BYTE, pos, bytes, offset, length);
} catch (IndexOutOfBoundsException e) {
throw handlePositionalIOOBE(e, "read", pos);
} catch (NullPointerException | IllegalStateException e) {
throw alreadyClosed(e);
}
}
@Override
public short readShort(long pos) throws IOException {
try {
return curSegment.get(LAYOUT_LE_SHORT, pos);
} catch (IndexOutOfBoundsException e) {
throw handlePositionalIOOBE(e, "read", pos);
} catch (NullPointerException | IllegalStateException e) {
throw alreadyClosed(e);
}
}
@Override
public int readInt(long pos) throws IOException {
try {
return curSegment.get(LAYOUT_LE_INT, pos);
} catch (IndexOutOfBoundsException e) {
throw handlePositionalIOOBE(e, "read", pos);
} catch (NullPointerException | IllegalStateException e) {
throw alreadyClosed(e);
}
}
@Override
public long readLong(long pos) throws IOException {
try {
return curSegment.get(LAYOUT_LE_LONG, pos);
} catch (IndexOutOfBoundsException e) {
throw handlePositionalIOOBE(e, "read", pos);
} catch (NullPointerException | IllegalStateException e) {
throw alreadyClosed(e);
}
}
}
/** This class adds offset support to MemorySegmentIndexInput, which is needed for slices. */
static final class MultiSegmentImpl extends MemorySegmentIndexInput {
private final long offset;
MultiSegmentImpl(
String resourceDescription,
MemorySession session,
MemorySegment[] segments,
long offset,
long length,
int chunkSizePower) {
super(resourceDescription, session, segments, length, chunkSizePower);
this.offset = offset;
try {
seek(0L);
} catch (IOException ioe) {
throw new AssertionError(ioe);
}
assert curSegment != null && curSegmentIndex >= 0;
}
@Override
RuntimeException handlePositionalIOOBE(RuntimeException unused, String action, long pos)
throws IOException {
return super.handlePositionalIOOBE(unused, action, pos - offset);
}
@Override
public void seek(long pos) throws IOException {
assert pos >= 0L : "negative position";
super.seek(pos + offset);
}
@Override
public long getFilePointer() {
return super.getFilePointer() - offset;
}
@Override
public byte readByte(long pos) throws IOException {
return super.readByte(pos + offset);
}
@Override
public void readBytes(long pos, byte[] bytes, int offset, int length) throws IOException {
super.readBytes(pos + this.offset, bytes, offset, length);
}
@Override
public short readShort(long pos) throws IOException {
return super.readShort(pos + offset);
}
@Override
public int readInt(long pos) throws IOException {
return super.readInt(pos + offset);
}
@Override
public long readLong(long pos) throws IOException {
return super.readLong(pos + offset);
}
@Override
MemorySegmentIndexInput buildSlice(String sliceDescription, long ofs, long length) {
return super.buildSlice(sliceDescription, this.offset + ofs, length);
}
}
}

120
lucene/core/src/java19/org/apache/lucene/store/MemorySegmentIndexInputProvider.java
View File

@ -1,120 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.store;
import java.io.IOException;
import java.lang.foreign.MemorySegment;
import java.lang.foreign.MemorySession;
import java.nio.channels.FileChannel;
import java.nio.channels.FileChannel.MapMode;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.logging.Logger;
import org.apache.lucene.util.Constants;
import org.apache.lucene.util.Unwrappable;
@SuppressWarnings("preview")
final class MemorySegmentIndexInputProvider implements MMapDirectory.MMapIndexInputProvider {
public MemorySegmentIndexInputProvider() {
var log = Logger.getLogger(getClass().getName());
log.info(
"Using MemorySegmentIndexInput with Java 19; to disable start with -D"
+ MMapDirectory.ENABLE_MEMORY_SEGMENTS_SYSPROP
+ "=false");
}
@Override
public IndexInput openInput(Path path, IOContext context, int chunkSizePower, boolean preload)
throws IOException {
final String resourceDescription = "MemorySegmentIndexInput(path=\"" + path.toString() + "\")";
// Work around for JDK-8259028: we need to unwrap our test-only file system layers
path = Unwrappable.unwrapAll(path);
boolean success = false;
final MemorySession session = MemorySession.openShared();
try (var fc = FileChannel.open(path, StandardOpenOption.READ)) {
final long fileSize = fc.size();
final IndexInput in =
MemorySegmentIndexInput.newInstance(
resourceDescription,
session,
map(session, resourceDescription, fc, chunkSizePower, preload, fileSize),
fileSize,
chunkSizePower);
success = true;
return in;
} finally {
if (success == false) {
session.close();
}
}
}
@Override
public long getDefaultMaxChunkSize() {
return Constants.JRE_IS_64BIT ? (1L << 34) : (1L << 28);
}
@Override
public boolean isUnmapSupported() {
return true;
}
@Override
public String getUnmapNotSupportedReason() {
return null;
}
private final MemorySegment[] map(
MemorySession session,
String resourceDescription,
FileChannel fc,
int chunkSizePower,
boolean preload,
long length)
throws IOException {
if ((length >>> chunkSizePower) >= Integer.MAX_VALUE)
throw new IllegalArgumentException("File too big for chunk size: " + resourceDescription);
final long chunkSize = 1L << chunkSizePower;
// we always allocate one more segments, the last one may be a 0 byte one
final int nrSegments = (int) (length >>> chunkSizePower) + 1;
final MemorySegment[] segments = new MemorySegment[nrSegments];
long startOffset = 0L;
for (int segNr = 0; segNr < nrSegments; segNr++) {
final long segSize =
(length > (startOffset + chunkSize)) ? chunkSize : (length - startOffset);
final MemorySegment segment;
try {
segment = fc.map(MapMode.READ_ONLY, startOffset, segSize, session);
} catch (IOException ioe) {
throw convertMapFailedIOException(ioe, resourceDescription, segSize);
}
if (preload) {
segment.load();
}
segments[segNr] = segment;
startOffset += segSize;
}
return segments;
}
}

681
lucene/core/src/java20/org/apache/lucene/store/MemorySegmentIndexInput.java
View File

@ -1,681 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.store;
import java.io.EOFException;
import java.io.IOException;
import java.lang.foreign.Arena;
import java.lang.foreign.MemorySegment;
import java.lang.foreign.ValueLayout;
import java.nio.ByteOrder;
import java.util.Arrays;
import java.util.Objects;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.GroupVIntUtil;
/**
* Base IndexInput implementation that uses an array of MemorySegments to represent a file.
*
* <p>For efficiency, this class requires that the segment size are a power-of-two (<code>
* chunkSizePower</code>).
*/
@SuppressWarnings("preview")
abstract class MemorySegmentIndexInput extends IndexInput implements RandomAccessInput {
static final ValueLayout.OfByte LAYOUT_BYTE = ValueLayout.JAVA_BYTE;
static final ValueLayout.OfShort LAYOUT_LE_SHORT =
ValueLayout.JAVA_SHORT_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN);
static final ValueLayout.OfInt LAYOUT_LE_INT =
ValueLayout.JAVA_INT_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN);
static final ValueLayout.OfLong LAYOUT_LE_LONG =
ValueLayout.JAVA_LONG_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN);
static final ValueLayout.OfFloat LAYOUT_LE_FLOAT =
ValueLayout.JAVA_FLOAT_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN);
final long length;
final long chunkSizeMask;
final int chunkSizePower;
final Arena arena;
final MemorySegment[] segments;
int curSegmentIndex = -1;
MemorySegment
curSegment; // redundant for speed: segments[curSegmentIndex], also marker if closed!
long curPosition; // relative to curSegment, not globally
public static MemorySegmentIndexInput newInstance(
String resourceDescription,
Arena arena,
MemorySegment[] segments,
long length,
int chunkSizePower) {
assert Arrays.stream(segments).map(MemorySegment::scope).allMatch(arena.scope()::equals);
if (segments.length == 1) {
return new SingleSegmentImpl(resourceDescription, arena, segments[0], length, chunkSizePower);
} else {
return new MultiSegmentImpl(resourceDescription, arena, segments, 0, length, chunkSizePower);
}
}
private MemorySegmentIndexInput(
String resourceDescription,
Arena arena,
MemorySegment[] segments,
long length,
int chunkSizePower) {
super(resourceDescription);
this.arena = arena;
this.segments = segments;
this.length = length;
this.chunkSizePower = chunkSizePower;
this.chunkSizeMask = (1L << chunkSizePower) - 1L;
this.curSegment = segments[0];
}
void ensureOpen() {
if (curSegment == null) {
throw alreadyClosed(null);
}
}
// the unused parameter is just to silence javac about unused variables
RuntimeException handlePositionalIOOBE(RuntimeException unused, String action, long pos)
throws IOException {
if (pos < 0L) {
return new IllegalArgumentException(action + " negative position (pos=" + pos + "): " + this);
} else {
throw new EOFException(action + " past EOF (pos=" + pos + "): " + this);
}
}
AlreadyClosedException alreadyClosed(RuntimeException e) {
// we use NPE to signal if this input is closed (to not have checks everywhere). If NPE happens,
// we check the "is closed" condition explicitly by checking that our "curSegment" is null.
// Care must be taken to not leak the NPE to code outside MemorySegmentIndexInput!
if (this.curSegment == null) {
return new AlreadyClosedException("Already closed: " + this);
}
// ISE can be thrown by MemorySegment and contains "closed" in message:
if (e instanceof IllegalStateException
&& e.getMessage() != null
&& e.getMessage().contains("closed")) {
return new AlreadyClosedException("Already closed: " + this, e);
}
// otherwise rethrow unmodified NPE/ISE (as it possibly a bug with passing a null parameter to
// the IndexInput method):
throw e;
}
@Override
public final byte readByte() throws IOException {
try {
final byte v = curSegment.get(LAYOUT_BYTE, curPosition);
curPosition++;
return v;
} catch (
@SuppressWarnings("unused")
IndexOutOfBoundsException e) {
do {
curSegmentIndex++;
if (curSegmentIndex >= segments.length) {
throw new EOFException("read past EOF: " + this);
}
curSegment = segments[curSegmentIndex];
curPosition = 0L;
} while (curSegment.byteSize() == 0L);
final byte v = curSegment.get(LAYOUT_BYTE, curPosition);
curPosition++;
return v;
} catch (NullPointerException | IllegalStateException e) {
throw alreadyClosed(e);
}
}
@Override
public final void readBytes(byte[] b, int offset, int len) throws IOException {
try {
MemorySegment.copy(curSegment, LAYOUT_BYTE, curPosition, b, offset, len);
curPosition += len;
} catch (
@SuppressWarnings("unused")
IndexOutOfBoundsException e) {
readBytesBoundary(b, offset, len);
} catch (NullPointerException | IllegalStateException e) {
throw alreadyClosed(e);
}
}
private void readBytesBoundary(byte[] b, int offset, int len) throws IOException {
try {
long curAvail = curSegment.byteSize() - curPosition;
while (len > curAvail) {
MemorySegment.copy(curSegment, LAYOUT_BYTE, curPosition, b, offset, (int) curAvail);
len -= curAvail;
offset += curAvail;
curSegmentIndex++;
if (curSegmentIndex >= segments.length) {
throw new EOFException("read past EOF: " + this);
}
curSegment = segments[curSegmentIndex];
curPosition = 0L;
curAvail = curSegment.byteSize();
}
MemorySegment.copy(curSegment, LAYOUT_BYTE, curPosition, b, offset, len);
curPosition += len;
} catch (NullPointerException | IllegalStateException e) {
throw alreadyClosed(e);
}
}
@Override
public void readInts(int[] dst, int offset, int length) throws IOException {
try {
MemorySegment.copy(curSegment, LAYOUT_LE_INT, curPosition, dst, offset, length);
curPosition += Integer.BYTES * (long) length;
} catch (
@SuppressWarnings("unused")
IndexOutOfBoundsException iobe) {
super.readInts(dst, offset, length);
} catch (NullPointerException | IllegalStateException e) {
throw alreadyClosed(e);
}
}
@Override
public void readLongs(long[] dst, int offset, int length) throws IOException {
try {
MemorySegment.copy(curSegment, LAYOUT_LE_LONG, curPosition, dst, offset, length);
curPosition += Long.BYTES * (long) length;
} catch (
@SuppressWarnings("unused")
IndexOutOfBoundsException iobe) {
super.readLongs(dst, offset, length);
} catch (NullPointerException | IllegalStateException e) {
throw alreadyClosed(e);
}
}
@Override
public void readFloats(float[] dst, int offset, int length) throws IOException {
try {
MemorySegment.copy(curSegment, LAYOUT_LE_FLOAT, curPosition, dst, offset, length);
curPosition += Float.BYTES * (long) length;
} catch (
@SuppressWarnings("unused")
IndexOutOfBoundsException iobe) {
super.readFloats(dst, offset, length);
} catch (NullPointerException | IllegalStateException e) {
throw alreadyClosed(e);
}
}
@Override
public final short readShort() throws IOException {
try {
final short v = curSegment.get(LAYOUT_LE_SHORT, curPosition);
curPosition += Short.BYTES;
return v;
} catch (
@SuppressWarnings("unused")
IndexOutOfBoundsException e) {
return super.readShort();
} catch (NullPointerException | IllegalStateException e) {
throw alreadyClosed(e);
}
}
@Override
public final int readInt() throws IOException {
try {
final int v = curSegment.get(LAYOUT_LE_INT, curPosition);
curPosition += Integer.BYTES;
return v;
} catch (
@SuppressWarnings("unused")
IndexOutOfBoundsException e) {
return super.readInt();
} catch (NullPointerException | IllegalStateException e) {
throw alreadyClosed(e);
}
}
@Override
public final int readVInt() throws IOException {
// this can make JVM less confused (see LUCENE-10366)
return super.readVInt();
}
@Override
public final long readVLong() throws IOException {
// this can make JVM less confused (see LUCENE-10366)
return super.readVLong();
}
@Override
public final long readLong() throws IOException {
try {
final long v = curSegment.get(LAYOUT_LE_LONG, curPosition);
curPosition += Long.BYTES;
return v;
} catch (
@SuppressWarnings("unused")
IndexOutOfBoundsException e) {
return super.readLong();
} catch (NullPointerException | IllegalStateException e) {
throw alreadyClosed(e);
}
}
@Override
public long getFilePointer() {
ensureOpen();
return (((long) curSegmentIndex) << chunkSizePower) + curPosition;
}
@Override
public void seek(long pos) throws IOException {
ensureOpen();
// we use >> here to preserve negative, so we will catch AIOOBE,
// in case pos + offset overflows.
final int si = (int) (pos >> chunkSizePower);
try {
if (si != curSegmentIndex) {
final MemorySegment seg = segments[si];
// write values, on exception all is unchanged
this.curSegmentIndex = si;
this.curSegment = seg;
}
this.curPosition = Objects.checkIndex(pos & chunkSizeMask, curSegment.byteSize() + 1);
} catch (IndexOutOfBoundsException e) {
throw handlePositionalIOOBE(e, "seek", pos);
}
}
@Override
public byte readByte(long pos) throws IOException {
try {
final int si = (int) (pos >> chunkSizePower);
return segments[si].get(LAYOUT_BYTE, pos & chunkSizeMask);
} catch (IndexOutOfBoundsException ioobe) {
throw handlePositionalIOOBE(ioobe, "read", pos);
} catch (NullPointerException | IllegalStateException e) {
throw alreadyClosed(e);
}
}
@Override
protected void readGroupVInt(long[] dst, int offset) throws IOException {
try {
final int len =
GroupVIntUtil.readGroupVInt(
this,
curSegment.byteSize() - curPosition,
p -> curSegment.get(LAYOUT_LE_INT, p),
curPosition,
dst,
offset);
curPosition += len;
} catch (NullPointerException | IllegalStateException e) {
throw alreadyClosed(e);
}
}
@Override
public void readBytes(long pos, byte[] b, int offset, int len) throws IOException {
try {
int si = (int) (pos >> chunkSizePower);
pos = pos & chunkSizeMask;
long curAvail = segments[si].byteSize() - pos;
while (len > curAvail) {
MemorySegment.copy(segments[si], LAYOUT_BYTE, pos, b, offset, (int) curAvail);
len -= curAvail;
offset += curAvail;
si++;
if (si >= segments.length) {
throw new EOFException("read past EOF: " + this);
}
pos = 0L;
curAvail = segments[si].byteSize();
}
MemorySegment.copy(segments[si], LAYOUT_BYTE, pos, b, offset, len);
} catch (IndexOutOfBoundsException ioobe) {
throw handlePositionalIOOBE(ioobe, "read", pos);
} catch (NullPointerException | IllegalStateException e) {
throw alreadyClosed(e);
}
}
// used only by random access methods to handle reads across boundaries
private void setPos(long pos, int si) throws IOException {
try {
final MemorySegment seg = segments[si];
// write values, on exception above all is unchanged
this.curPosition = pos & chunkSizeMask;
this.curSegmentIndex = si;
this.curSegment = seg;
} catch (IndexOutOfBoundsException ioobe) {
throw handlePositionalIOOBE(ioobe, "read", pos);
} catch (NullPointerException | IllegalStateException e) {
throw alreadyClosed(e);
}
}
@Override
public short readShort(long pos) throws IOException {
final int si = (int) (pos >> chunkSizePower);
try {
return segments[si].get(LAYOUT_LE_SHORT, pos & chunkSizeMask);
} catch (
@SuppressWarnings("unused")
IndexOutOfBoundsException ioobe) {
// either it's a boundary, or read past EOF, fall back:
setPos(pos, si);
return readShort();
} catch (NullPointerException | IllegalStateException e) {
throw alreadyClosed(e);
}
}
@Override
public int readInt(long pos) throws IOException {
final int si = (int) (pos >> chunkSizePower);
try {
return segments[si].get(LAYOUT_LE_INT, pos & chunkSizeMask);
} catch (
@SuppressWarnings("unused")
IndexOutOfBoundsException ioobe) {
// either it's a boundary, or read past EOF, fall back:
setPos(pos, si);
return readInt();
} catch (NullPointerException | IllegalStateException e) {
throw alreadyClosed(e);
}
}
@Override
public long readLong(long pos) throws IOException {
final int si = (int) (pos >> chunkSizePower);
try {
return segments[si].get(LAYOUT_LE_LONG, pos & chunkSizeMask);
} catch (
@SuppressWarnings("unused")
IndexOutOfBoundsException ioobe) {
// either it's a boundary, or read past EOF, fall back:
setPos(pos, si);
return readLong();
} catch (NullPointerException | IllegalStateException e) {
throw alreadyClosed(e);
}
}
@Override
public final long length() {
return length;
}
@Override
public final MemorySegmentIndexInput clone() {
final MemorySegmentIndexInput clone = buildSlice((String) null, 0L, this.length);
try {
clone.seek(getFilePointer());
} catch (IOException ioe) {
throw new AssertionError(ioe);
}
return clone;
}
/**
* Creates a slice of this index input, with the given description, offset, and length. The slice
* is seeked to the beginning.
*/
@Override
public final MemorySegmentIndexInput slice(String sliceDescription, long offset, long length) {
if (offset < 0 || length < 0 || offset + length > this.length) {
throw new IllegalArgumentException(
"slice() "
+ sliceDescription
+ " out of bounds: offset="
+ offset
+ ",length="
+ length
+ ",fileLength="
+ this.length
+ ": "
+ this);
}
return buildSlice(sliceDescription, offset, length);
}
/** Builds the actual sliced IndexInput (may apply extra offset in subclasses). * */
MemorySegmentIndexInput buildSlice(String sliceDescription, long offset, long length) {
ensureOpen();
final long sliceEnd = offset + length;
final int startIndex = (int) (offset >>> chunkSizePower);
final int endIndex = (int) (sliceEnd >>> chunkSizePower);
// we always allocate one more slice, the last one may be a 0 byte one after truncating with
// asSlice():
final MemorySegment slices[] = ArrayUtil.copyOfSubArray(segments, startIndex, endIndex + 1);
// set the last segment's limit for the sliced view.
slices[slices.length - 1] = slices[slices.length - 1].asSlice(0L, sliceEnd & chunkSizeMask);
offset = offset & chunkSizeMask;
final String newResourceDescription = getFullSliceDescription(sliceDescription);
if (slices.length == 1) {
return new SingleSegmentImpl(
newResourceDescription,
null, // clones don't have an Arena, as they can't close)
slices[0].asSlice(offset, length),
length,
chunkSizePower);
} else {
return new MultiSegmentImpl(
newResourceDescription,
null, // clones don't have an Arena, as they can't close)
slices,
offset,
length,
chunkSizePower);
}
}
@Override
public final void close() throws IOException {
if (curSegment == null) {
return;
}
// the master IndexInput has an Arena and is able
// to release all resources (unmap segments) - a
// side effect is that other threads still using clones
// will throw IllegalStateException
if (arena != null) {
while (arena.scope().isAlive()) {
try {
arena.close();
break;
} catch (
@SuppressWarnings("unused")
IllegalStateException e) {
Thread.onSpinWait();
}
}
}
// make sure all accesses to this IndexInput instance throw NPE:
curSegment = null;
Arrays.fill(segments, null);
}
/** Optimization of MemorySegmentIndexInput for when there is only one segment. */
static final class SingleSegmentImpl extends MemorySegmentIndexInput {
SingleSegmentImpl(
String resourceDescription,
Arena arena,
MemorySegment segment,
long length,
int chunkSizePower) {
super(resourceDescription, arena, new MemorySegment[] {segment}, length, chunkSizePower);
this.curSegmentIndex = 0;
}
@Override
public void seek(long pos) throws IOException {
ensureOpen();
try {
curPosition = Objects.checkIndex(pos, length + 1);
} catch (IndexOutOfBoundsException e) {
throw handlePositionalIOOBE(e, "seek", pos);
}
}
@Override
public long getFilePointer() {
ensureOpen();
return curPosition;
}
@Override
public byte readByte(long pos) throws IOException {
try {
return curSegment.get(LAYOUT_BYTE, pos);
} catch (IndexOutOfBoundsException e) {
throw handlePositionalIOOBE(e, "read", pos);
} catch (NullPointerException | IllegalStateException e) {
throw alreadyClosed(e);
}
}
@Override
public void readBytes(long pos, byte[] bytes, int offset, int length) throws IOException {
try {
MemorySegment.copy(curSegment, LAYOUT_BYTE, pos, bytes, offset, length);
} catch (IndexOutOfBoundsException e) {
throw handlePositionalIOOBE(e, "read", pos);
} catch (NullPointerException | IllegalStateException e) {
throw alreadyClosed(e);
}
}
@Override
public short readShort(long pos) throws IOException {
try {
return curSegment.get(LAYOUT_LE_SHORT, pos);
} catch (IndexOutOfBoundsException e) {
throw handlePositionalIOOBE(e, "read", pos);
} catch (NullPointerException | IllegalStateException e) {
throw alreadyClosed(e);
}
}
@Override
public int readInt(long pos) throws IOException {
try {
return curSegment.get(LAYOUT_LE_INT, pos);
} catch (IndexOutOfBoundsException e) {
throw handlePositionalIOOBE(e, "read", pos);
} catch (NullPointerException | IllegalStateException e) {
throw alreadyClosed(e);
}
}
@Override
public long readLong(long pos) throws IOException {
try {
return curSegment.get(LAYOUT_LE_LONG, pos);
} catch (IndexOutOfBoundsException e) {
throw handlePositionalIOOBE(e, "read", pos);
} catch (NullPointerException | IllegalStateException e) {
throw alreadyClosed(e);
}
}
}
/** This class adds offset support to MemorySegmentIndexInput, which is needed for slices. */
static final class MultiSegmentImpl extends MemorySegmentIndexInput {
private final long offset;
MultiSegmentImpl(
String resourceDescription,
Arena arena,
MemorySegment[] segments,
long offset,
long length,
int chunkSizePower) {
super(resourceDescription, arena, segments, length, chunkSizePower);
this.offset = offset;
try {
seek(0L);
} catch (IOException ioe) {
throw new AssertionError(ioe);
}
assert curSegment != null && curSegmentIndex >= 0;
}
@Override
RuntimeException handlePositionalIOOBE(RuntimeException unused, String action, long pos)
throws IOException {
return super.handlePositionalIOOBE(unused, action, pos - offset);
}
@Override
public void seek(long pos) throws IOException {
assert pos >= 0L : "negative position";
super.seek(pos + offset);
}
@Override
public long getFilePointer() {
return super.getFilePointer() - offset;
}
@Override
public byte readByte(long pos) throws IOException {
return super.readByte(pos + offset);
}
@Override
public void readBytes(long pos, byte[] bytes, int offset, int length) throws IOException {
super.readBytes(pos + this.offset, bytes, offset, length);
}
@Override
public short readShort(long pos) throws IOException {
return super.readShort(pos + offset);
}
@Override
public int readInt(long pos) throws IOException {
return super.readInt(pos + offset);
}
@Override
public long readLong(long pos) throws IOException {
return super.readLong(pos + offset);
}
@Override
MemorySegmentIndexInput buildSlice(String sliceDescription, long ofs, long length) {
return super.buildSlice(sliceDescription, this.offset + ofs, length);
}
}
}

120
lucene/core/src/java20/org/apache/lucene/store/MemorySegmentIndexInputProvider.java
View File

@ -1,120 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.store;
import java.io.IOException;
import java.lang.foreign.Arena;
import java.lang.foreign.MemorySegment;
import java.nio.channels.FileChannel;
import java.nio.channels.FileChannel.MapMode;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.logging.Logger;
import org.apache.lucene.util.Constants;
import org.apache.lucene.util.Unwrappable;
@SuppressWarnings("preview")
final class MemorySegmentIndexInputProvider implements MMapDirectory.MMapIndexInputProvider {
public MemorySegmentIndexInputProvider() {
var log = Logger.getLogger(getClass().getName());
log.info(
"Using MemorySegmentIndexInput with Java 20; to disable start with -D"
+ MMapDirectory.ENABLE_MEMORY_SEGMENTS_SYSPROP
+ "=false");
}
@Override
public IndexInput openInput(Path path, IOContext context, int chunkSizePower, boolean preload)
throws IOException {
final String resourceDescription = "MemorySegmentIndexInput(path=\"" + path.toString() + "\")";
// Work around for JDK-8259028: we need to unwrap our test-only file system layers
path = Unwrappable.unwrapAll(path);
boolean success = false;
final Arena arena = Arena.openShared();
try (var fc = FileChannel.open(path, StandardOpenOption.READ)) {
final long fileSize = fc.size();
final IndexInput in =
MemorySegmentIndexInput.newInstance(
resourceDescription,
arena,
map(arena, resourceDescription, fc, chunkSizePower, preload, fileSize),
fileSize,
chunkSizePower);
success = true;
return in;
} finally {
if (success == false) {
arena.close();
}
}
}
@Override
public long getDefaultMaxChunkSize() {
return Constants.JRE_IS_64BIT ? (1L << 34) : (1L << 28);
}
@Override
public boolean isUnmapSupported() {
return true;
}
@Override
public String getUnmapNotSupportedReason() {
return null;
}
private final MemorySegment[] map(
Arena arena,
String resourceDescription,
FileChannel fc,
int chunkSizePower,
boolean preload,
long length)
throws IOException {
if ((length >>> chunkSizePower) >= Integer.MAX_VALUE)
throw new IllegalArgumentException("File too big for chunk size: " + resourceDescription);
final long chunkSize = 1L << chunkSizePower;
// we always allocate one more segments, the last one may be a 0 byte one
final int nrSegments = (int) (length >>> chunkSizePower) + 1;
final MemorySegment[] segments = new MemorySegment[nrSegments];
long startOffset = 0L;
for (int segNr = 0; segNr < nrSegments; segNr++) {
final long segSize =
(length > (startOffset + chunkSize)) ? chunkSize : (length - startOffset);
final MemorySegment segment;
try {
segment = fc.map(MapMode.READ_ONLY, startOffset, segSize, arena.scope());
} catch (IOException ioe) {
throw convertMapFailedIOException(ioe, resourceDescription, segSize);
}
if (preload) {
segment.load();
}
segments[segNr] = segment;
startOffset += segSize;
}
return segments;
}
}

0
lucene/core/src/java20/org/apache/lucene/internal/vectorization/PanamaVectorUtilSupport.java → lucene/core/src/java21/org/apache/lucene/internal/vectorization/PanamaVectorUtilSupport.java
View File

0
lucene/core/src/java20/org/apache/lucene/internal/vectorization/PanamaVectorizationProvider.java → lucene/core/src/java21/org/apache/lucene/internal/vectorization/PanamaVectorizationProvider.java
View File

19
lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInputProvider.java
View File

@ -23,21 +23,12 @@ import java.nio.channels.FileChannel;
import java.nio.channels.FileChannel.MapMode;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.logging.Logger;
import org.apache.lucene.util.Constants;
import org.apache.lucene.util.Unwrappable;
@SuppressWarnings("preview")
final class MemorySegmentIndexInputProvider implements MMapDirectory.MMapIndexInputProvider {
public MemorySegmentIndexInputProvider() {
var log = Logger.getLogger(getClass().getName());
log.info(
"Using MemorySegmentIndexInput with Java 21 or later; to disable start with -D"
+ MMapDirectory.ENABLE_MEMORY_SEGMENTS_SYSPROP
+ "=false");
}
@Override
public IndexInput openInput(Path path, IOContext context, int chunkSizePower, boolean preload)
throws IOException {
@ -71,16 +62,6 @@ final class MemorySegmentIndexInputProvider implements MMapDirectory.MMapIndexIn
return Constants.JRE_IS_64BIT ? (1L << 34) : (1L << 28);
}
@Override
public boolean isUnmapSupported() {
return true;
}
@Override
public String getUnmapNotSupportedReason() {
return null;
}
private final MemorySegment[] map(
Arena arena,
String resourceDescription,

2
lucene/core/src/java21/org/apache/lucene/util/VectorUtilPanamaProvider.txt
View File

@ -1,2 +0,0 @@
The version of VectorUtilPanamaProvider for Java 21 is identical to that of Java 20.
As such, there is no specific 21 version - the Java 20 version will be loaded from the MRJAR.

2
lucene/core/src/test/org/apache/lucene/index/Test4GBStoredFields.java
View File

@ -38,8 +38,6 @@ import org.apache.lucene.util.BytesRef;
public class Test4GBStoredFields extends LuceneTestCase {
public void test() throws Exception {
assumeWorkingMMapOnWindows();
MockDirectoryWrapper dir =
new MockDirectoryWrapper(random(), new MMapDirectory(createTempDir("4GBStoredFields")));
dir.setThrottling(MockDirectoryWrapper.Throttling.NEVER);

1
lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
View File

@ -1234,7 +1234,6 @@ public class TestIndexWriter extends LuceneTestCase {
public void testDeleteUnusedFiles() throws Exception {
assumeFalse("test relies on exact filenames", Codec.getDefault() instanceof SimpleTextCodec);
assumeWorkingMMapOnWindows();
for (int iter = 0; iter < 2; iter++) {
// relies on Windows semantics

9
lucene/core/src/test/org/apache/lucene/store/TestByteBuffersDirectory.java
View File

@ -84,15 +84,6 @@ public class TestByteBuffersDirectory extends BaseDirectoryTestCase {
ByteBuffersDirectory.OUTPUT_AS_ONE_BUFFER),
"one buffer (heap)"
},
{
(Supplier<ByteBuffersDirectory>)
() ->
new ByteBuffersDirectory(
new SingleInstanceLockFactory(),
ByteBuffersDataOutput::new,
ByteBuffersDirectory.OUTPUT_AS_MANY_BUFFERS_LUCENE),
"lucene's buffers (heap)"
},
{
(Supplier<ByteBuffersDirectory>)
() ->

7
lucene/core/src/test/org/apache/lucene/store/TestDirectory.java
View File

@ -43,9 +43,7 @@ public class TestDirectory extends LuceneTestCase {
final List<FSDirectory> dirs0 = new ArrayList<>();
dirs0.add(new NIOFSDirectory(path));
if (hasWorkingMMapOnWindows()) {
dirs0.add(new MMapDirectory(path));
}
dirs0.add(new MMapDirectory(path));
final FSDirectory[] dirs = dirs0.toArray(FSDirectory[]::new);
for (int i = 0; i < dirs.length; i++) {
@ -63,9 +61,6 @@ public class TestDirectory extends LuceneTestCase {
assertTrue(slowFileExists(d2, fname));
assertEquals(1 + largeBuffer.length, d2.fileLength(fname));
// don't do read tests if unmapping is not supported!
if (d2 instanceof MMapDirectory && !MMapDirectory.UNMAP_SUPPORTED) continue;
IndexInput input = d2.openInput(fname, newIOContext(random()));
assertEquals((byte) i, input.readByte());
// read array with buffering enabled

25
lucene/core/src/test/org/apache/lucene/store/TestMMapDirectory.java
View File

@ -18,11 +18,9 @@ package org.apache.lucene.store;
import java.io.IOException;
import java.nio.file.Path;
import java.util.Objects;
import java.util.Random;
import java.util.concurrent.CountDownLatch;
import org.apache.lucene.tests.store.BaseDirectoryTestCase;
import org.junit.BeforeClass;
/** Tests MMapDirectory */
// See: https://issues.apache.org/jira/browse/SOLR-12028 Tests cannot remove files on Windows
@ -36,30 +34,7 @@ public class TestMMapDirectory extends BaseDirectoryTestCase {
return m;
}
@BeforeClass
public static void beforeClass() throws Exception {
assertTrue(MMapDirectory.UNMAP_NOT_SUPPORTED_REASON, MMapDirectory.UNMAP_SUPPORTED);
}
private static boolean isMemorySegmentImpl() {
return Objects.equals(
"MemorySegmentIndexInputProvider", MMapDirectory.PROVIDER.getClass().getSimpleName());
}
public void testCorrectImplementation() {
final int runtimeVersion = Runtime.version().feature();
if (runtimeVersion >= 19) {
assertTrue(
"on Java 19 or later we should use MemorySegmentIndexInputProvider to create mmap IndexInputs",
isMemorySegmentImpl());
} else {
assertSame(MappedByteBufferIndexInputProvider.class, MMapDirectory.PROVIDER.getClass());
}
}
public void testAceWithThreads() throws Exception {
assumeTrue("Test requires MemorySegmentIndexInput", isMemorySegmentImpl());
final int nInts = 8 * 1024 * 1024;
try (Directory dir = getDirectory(createTempDir("testAceWithThreads"))) {

6
lucene/core/src/test/org/apache/lucene/store/TestMultiMMap.java
View File

@ -23,7 +23,6 @@ import java.util.List;
import java.util.Locale;
import org.apache.lucene.tests.store.BaseChunkedDirectoryTestCase;
import org.apache.lucene.util.BytesRef;
import org.junit.BeforeClass;
/**
* Tests MMapDirectory's MultiMMapIndexInput
@ -38,11 +37,6 @@ public class TestMultiMMap extends BaseChunkedDirectoryTestCase {
return new MMapDirectory(path, maxChunkSize);
}
@BeforeClass
public static void beforeClass() throws Exception {
assertTrue(MMapDirectory.UNMAP_NOT_SUPPORTED_REASON, MMapDirectory.UNMAP_SUPPORTED);
}
public void testSeekingExceptions() throws IOException {
final int sliceSize = 128;
try (Directory dir = getDirectory(createTempDir(), sliceSize)) {

2
lucene/core/src/test/org/apache/lucene/util/fst/Test2BFST.java
View File

@ -42,8 +42,6 @@ public class Test2BFST extends LuceneTestCase {
private static long LIMIT = 3L * 1024 * 1024 * 1024;
public void test() throws Exception {
assumeWorkingMMapOnWindows();
int[] ints = new int[7];
IntsRef input = new IntsRef(ints, 0, ints.length);
long seed = random().nextLong();

2
lucene/core/src/test/org/apache/lucene/util/fst/Test2BFSTOffHeap.java
View File

@ -44,8 +44,6 @@ public class Test2BFSTOffHeap extends LuceneTestCase {
private static long LIMIT = 3L * 1024 * 1024 * 1024;
public void test() throws Exception {
assumeWorkingMMapOnWindows();
int[] ints = new int[7];
IntsRef input = new IntsRef(ints, 0, ints.length);
long seed = random().nextLong();

11
lucene/distribution.tests/src/test/org/apache/lucene/distribution/TestModularLayer.java
View File

@ -207,7 +207,7 @@ public class TestModularLayer extends AbstractLuceneDistributionTest {
ClassLoader loader = layer.findLoader(coreModuleId);
final Set<Integer> jarVersions = Set.of(19, 20, 21);
final Set<Integer> jarVersions = Set.of(21);
for (var v : jarVersions) {
Assertions.assertThat(
loader.getResource(
@ -217,12 +217,9 @@ public class TestModularLayer extends AbstractLuceneDistributionTest {
.isNotNull();
}
final int runtimeVersion = Runtime.version().feature();
if (jarVersions.contains(runtimeVersion)) {
Assertions.assertThat(
loader.loadClass("org.apache.lucene.store.MemorySegmentIndexInput"))
.isNotNull();
}
Assertions.assertThat(
loader.loadClass("org.apache.lucene.store.MemorySegmentIndexInput"))
.isNotNull();
});
}

2
lucene/expressions/src/java/org/apache/lucene/expressions/js/JavascriptCompiler.java
View File

@ -97,7 +97,7 @@ public final class JavascriptCompiler {
private static final Lookup LOOKUP = MethodHandles.lookup();
private static final int CLASSFILE_VERSION = Opcodes.V17;
private static final int CLASSFILE_VERSION = Opcodes.V21;
private static final MethodType MT_EXPRESSION_CTOR_LOOKUP =
MethodType.methodType(void.class, String.class, String[].class);

4
lucene/suggest/src/java/org/apache/lucene/search/suggest/document/NRTSuggester.java
View File

@ -27,7 +27,6 @@ import org.apache.lucene.search.suggest.analyzing.FSTUtil;
import org.apache.lucene.search.suggest.document.CompletionPostingsFormat.FSTLoadMode;
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.ByteArrayDataOutput;
import org.apache.lucene.store.ByteBufferIndexInput;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.Bits;
@ -327,8 +326,7 @@ public final class NRTSuggester implements Accountable {
case AUTO:
// TODO: Make this less hacky to maybe expose "off-heap" feature using a marker interface on
// the IndexInput
return input instanceof ByteBufferIndexInput
|| input.getClass().getName().contains(".MemorySegmentIndexInput");
return input.getClass().getName().contains(".MemorySegmentIndexInput");
default:
throw new IllegalStateException("unknown enum constant: " + fstLoadMode);
}

2
lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseStoredFieldsFormatTestCase.java
View File

@ -723,8 +723,6 @@ public abstract class BaseStoredFieldsFormatTestCase extends BaseIndexFileFormat
@Nightly
public void testBigDocuments() throws IOException {
assumeWorkingMMapOnWindows();
// "big" as "much bigger than the chunk size"
// for this test we force an FS dir
// we can't just use newFSDirectory, because this test doesn't really index anything.

26
lucene/test-framework/src/java/org/apache/lucene/tests/util/LuceneTestCase.java
View File

@ -166,7 +166,6 @@ import org.apache.lucene.store.FileSwitchDirectory;
import org.apache.lucene.store.FlushInfo;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.LockFactory;
import org.apache.lucene.store.MMapDirectory;
import org.apache.lucene.store.MergeInfo;
import org.apache.lucene.store.NRTCachingDirectory;
import org.apache.lucene.tests.analysis.MockAnalyzer;
@ -190,7 +189,6 @@ import org.apache.lucene.tests.util.automaton.AutomatonTestUtil;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CommandLineUtil;
import org.apache.lucene.util.Constants;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.InfoStream;
import org.apache.lucene.util.NamedThreadFactory;
@ -498,31 +496,9 @@ public abstract class LuceneTestCase extends Assert {
LEAVE_TEMPORARY = defaultValue;
}
/**
* Returns true, if MMapDirectory supports unmapping on this platform (required for Windows), or
* if we are not on Windows.
*/
public static boolean hasWorkingMMapOnWindows() {
return !Constants.WINDOWS || MMapDirectory.UNMAP_SUPPORTED;
}
/**
* Assumes that the current MMapDirectory implementation supports unmapping, so the test will not
* fail on Windows.
*
* @see #hasWorkingMMapOnWindows()
*/
public static void assumeWorkingMMapOnWindows() {
assumeTrue(MMapDirectory.UNMAP_NOT_SUPPORTED_REASON, hasWorkingMMapOnWindows());
}
/** Filesystem-based {@link Directory} implementations. */
private static final List<String> FS_DIRECTORIES =
Arrays.asList(
"NIOFSDirectory",
// NIOFSDirectory as replacement for MMapDirectory if unmapping is not supported on
// Windows (to make randomization stable):
hasWorkingMMapOnWindows() ? "MMapDirectory" : "NIOFSDirectory");
Arrays.asList("NIOFSDirectory", "MMapDirectory");
/** All {@link Directory} implementations. */
private static final List<String> CORE_DIRECTORIES;