Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Only call madvise when necessary. (#13907) 

This commit tries to save calls to `madvise` which are not necessary, either
because they map to the OS' default, or because the advice would be overridden
later on anyway. I have not noticed specific problems with this, but it seems
desirable to keep calls to `madvise` to a minimum.

As a consequence:
 - Files that are open with `ReadAdvice.NORMAL` do not call `madvise` since
   this is the OS' default.
 - Compound files are always open with `ReadAdvice.NORMAL`, and the actual is
   only set when opening sub files of these compound files.

To make the latter less trappy, the `IOContext` parameter has been removed from
`CompoundFormat#getCompoundReader`.
This commit is contained in:
Adrien Grand 2024-10-15 14:14:27 +02:00
parent fbe5757d4c
commit b8bfffa368
23 changed files with 105 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50CompoundFormat.java
View File

@ -77,9 +77,8 @@ public final class Lucene50CompoundFormat extends CompoundFormat {
public Lucene50CompoundFormat() {}
@Override
public CompoundDirectory getCompoundReader(Directory dir, SegmentInfo si, IOContext context)
throws IOException {
return new Lucene50CompoundReader(dir, si, context);
public CompoundDirectory getCompoundReader(Directory dir, SegmentInfo si) throws IOException {
return new Lucene50CompoundReader(dir, si);
}
@Override

8
lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50CompoundReader.java
View File

@ -31,6 +31,7 @@ import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.ReadAdvice;
import org.apache.lucene.util.CollectionUtil;
import org.apache.lucene.util.IOUtils;
@ -57,8 +58,7 @@ final class Lucene50CompoundReader extends CompoundDirectory {
/** Create a new CompoundFileDirectory. */
// TODO: we should just pre-strip "entries" and append segment name up-front like simpletext?
// this need not be a "general purpose" directory anymore (it only writes index files)
public Lucene50CompoundReader(Directory directory, SegmentInfo si, IOContext context)
throws IOException {
public Lucene50CompoundReader(Directory directory, SegmentInfo si) throws IOException {
this.directory = directory;
this.segmentName = si.name;
String dataFileName =
@ -74,7 +74,7 @@ final class Lucene50CompoundReader extends CompoundDirectory {
}
expectedLength += CodecUtil.footerLength();
handle = directory.openInput(dataFileName, context);
handle = directory.openInput(dataFileName, IOContext.DEFAULT.withReadAdvice(ReadAdvice.NORMAL));
// DirectoryUtil.openInput(directory, dataFileName, context);
try {
CodecUtil.checkIndexHeader(
@ -170,7 +170,7 @@ final class Lucene50CompoundReader extends CompoundDirectory {
+ entries.keySet()
+ ")");
}
return handle.slice(name, entry.offset, entry.length);
return handle.slice(name, entry.offset, entry.length, context.readAdvice());
}
/** Returns an array of strings, one for each file in the directory. */

5
lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/Lucene50RWCompoundFormat.java
View File

@ -81,9 +81,8 @@ public final class Lucene50RWCompoundFormat extends CompoundFormat {
public Lucene50RWCompoundFormat() {}
@Override
public CompoundDirectory getCompoundReader(Directory dir, SegmentInfo si, IOContext context)
throws IOException {
return new Lucene50CompoundReader(dir, si, context);
public CompoundDirectory getCompoundReader(Directory dir, SegmentInfo si) throws IOException {
return new Lucene50CompoundReader(dir, si);
}
@Override

13
lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextCompoundFormat.java
View File

@ -35,6 +35,7 @@ import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.ReadAdvice;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
import org.apache.lucene.util.StringHelper;
@ -52,10 +53,10 @@ public class SimpleTextCompoundFormat extends CompoundFormat {
public SimpleTextCompoundFormat() {}
@Override
public CompoundDirectory getCompoundReader(Directory dir, SegmentInfo si, IOContext context)
throws IOException {
public CompoundDirectory getCompoundReader(Directory dir, SegmentInfo si) throws IOException {
String dataFile = IndexFileNames.segmentFileName(si.name, "", DATA_EXTENSION);
final IndexInput in = dir.openInput(dataFile, context);
final IndexInput in =
dir.openInput(dataFile, IOContext.DEFAULT.withReadAdvice(ReadAdvice.NORMAL));
BytesRefBuilder scratch = new BytesRefBuilder();
@ -135,7 +136,11 @@ public class SimpleTextCompoundFormat extends CompoundFormat {
public IndexInput openInput(String name, IOContext context) throws IOException {
ensureOpen();
int index = getIndex(name);
return in.slice(name, startOffsets[index], endOffsets[index] - startOffsets[index]);
return in.slice(
name,
startOffsets[index],
endOffsets[index] - startOffsets[index],
context.readAdvice());
}
@Override

4
lucene/core/src/java/org/apache/lucene/codecs/CompoundFormat.java
View File

@ -35,8 +35,8 @@ public abstract class CompoundFormat {
// we can add 'producer' classes.
/** Returns a Directory view (read-only) for the compound files in this segment */
public abstract CompoundDirectory getCompoundReader(
Directory dir, SegmentInfo si, IOContext context) throws IOException;
public abstract CompoundDirectory getCompoundReader(Directory dir, SegmentInfo si)
throws IOException;
/**
* Packs the provided segment's files into a compound format. All files referenced by the provided

5
lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90CompoundFormat.java
View File

@ -82,9 +82,8 @@ public final class Lucene90CompoundFormat extends CompoundFormat {
public Lucene90CompoundFormat() {}
@Override
public CompoundDirectory getCompoundReader(Directory dir, SegmentInfo si, IOContext context)
throws IOException {
return new Lucene90CompoundReader(dir, si, context);
public CompoundDirectory getCompoundReader(Directory dir, SegmentInfo si) throws IOException {
return new Lucene90CompoundReader(dir, si);
}
@Override

8
lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90CompoundReader.java
View File

@ -30,6 +30,7 @@ import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.ReadAdvice;
import org.apache.lucene.util.CollectionUtil;
import org.apache.lucene.util.IOUtils;
@ -56,8 +57,7 @@ final class Lucene90CompoundReader extends CompoundDirectory {
/** Create a new CompoundFileDirectory. */
// TODO: we should just pre-strip "entries" and append segment name up-front like simpletext?
// this need not be a "general purpose" directory anymore (it only writes index files)
public Lucene90CompoundReader(Directory directory, SegmentInfo si, IOContext context)
throws IOException {
public Lucene90CompoundReader(Directory directory, SegmentInfo si) throws IOException {
this.directory = directory;
this.segmentName = si.name;
String dataFileName =
@ -75,7 +75,7 @@ final class Lucene90CompoundReader extends CompoundDirectory {
.orElseGet(() -> CodecUtil.indexHeaderLength(Lucene90CompoundFormat.DATA_CODEC, ""))
+ CodecUtil.footerLength();
handle = directory.openInput(dataFileName, context);
handle = directory.openInput(dataFileName, IOContext.DEFAULT.withReadAdvice(ReadAdvice.NORMAL));
try {
CodecUtil.checkIndexHeader(
handle, Lucene90CompoundFormat.DATA_CODEC, version, version, si.getId(), "");
@ -169,7 +169,7 @@ final class Lucene90CompoundReader extends CompoundDirectory {
+ entries.keySet()
+ ")");
}
return handle.slice(name, entry.offset, entry.length);
return handle.slice(name, entry.offset, entry.length, context.readAdvice());
}
/** Returns an array of strings, one for each file in the directory. */

3
lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
View File

@ -1255,8 +1255,7 @@ public class IndexWriter
return reader.read(si.info.dir, si.info, segmentSuffix, IOContext.READONCE);
} else if (si.info.getUseCompoundFile()) {
// cfs
try (Directory cfs =
codec.compoundFormat().getCompoundReader(si.info.dir, si.info, IOContext.DEFAULT)) {
try (Directory cfs = codec.compoundFormat().getCompoundReader(si.info.dir, si.info)) {
return reader.read(cfs, si.info, "", IOContext.READONCE);
}
} else {

7
lucene/core/src/java/org/apache/lucene/index/PendingSoftDeletes.java
View File

@ -226,12 +226,7 @@ final class PendingSoftDeletes extends PendingDeletes {
// updates always outside of CFS
Closeable toClose;
if (segInfo.getUseCompoundFile()) {
toClose =
dir =
segInfo
.getCodec()
.compoundFormat()
.getCompoundReader(segInfo.dir, segInfo, IOContext.READONCE);
toClose = dir = segInfo.getCodec().compoundFormat().getCompoundReader(segInfo.dir, segInfo);
} else {
toClose = null;
dir = segInfo.dir;

2
lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java
View File

@ -80,7 +80,7 @@ final class SegmentCoreReaders {
try {
if (si.info.getUseCompoundFile()) {
cfsDir = cfsReader = codec.compoundFormat().getCompoundReader(dir, si.info, context);
cfsDir = cfsReader = codec.compoundFormat().getCompoundReader(dir, si.info);
} else {
cfsReader = null;
cfsDir = dir;

4
lucene/core/src/java/org/apache/lucene/store/IndexInput.java
View File

@ -127,6 +127,10 @@ public abstract class IndexInput extends DataInput implements Closeable {
* CompoundFormat} implementations to honor the {@link ReadAdvice} of each file within the
* compound file.
*
* <p><b>NOTE</b>: it is only legal to call this method if this {@link IndexInput} has been open
* with {@link ReadAdvice#NORMAL}. However, this method accepts any {@link ReadAdvice} value but
* {@code null} as a read advice for the slice.
*
* <p>The default implementation delegates to {@link #slice(String, long, long)} and ignores the
* {@link ReadAdvice}.
*/

3
lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInput.java
View File

@ -567,7 +567,8 @@ abstract class MemorySegmentIndexInput extends IndexInput
public final MemorySegmentIndexInput slice(
String sliceDescription, long offset, long length, ReadAdvice advice) throws IOException {
MemorySegmentIndexInput slice = slice(sliceDescription, offset, length);
if (NATIVE_ACCESS.isPresent()) {
if (NATIVE_ACCESS.isPresent() && advice != ReadAdvice.NORMAL) {
// No need to madvise with a normal advice, since it's the OS' default.
final NativeAccess nativeAccess = NATIVE_ACCESS.get();
slice.advise(
0,

4
lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInputProvider.java
View File

@ -129,7 +129,9 @@ final class MemorySegmentIndexInputProvider
// internal FileChannel logic)
if (preload) {
segment.load();
} else if (nativeAccess.filter(na -> segment.address() % na.getPageSize() == 0).isPresent()) {
} else if (readAdvice != ReadAdvice.NORMAL
&& nativeAccess.filter(na -> segment.address() % na.getPageSize() == 0).isPresent()) {
// No need to madvise with ReadAdvice.NORMAL since it is the OS' default read advice.
nativeAccess.get().madvise(segment, readAdvice);
}
segments[segNr] = segment;

7
lucene/core/src/java21/org/apache/lucene/store/PosixNativeAccess.java
View File

@ -122,10 +122,7 @@ final class PosixNativeAccess extends NativeAccess {
@Override
public void madvise(MemorySegment segment, ReadAdvice readAdvice) throws IOException {
final Integer advice = mapReadAdvice(readAdvice);
if (advice == null) {
return; // do nothing
}
final int advice = mapReadAdvice(readAdvice);
madvise(segment, advice);
}
@ -156,7 +153,7 @@ final class PosixNativeAccess extends NativeAccess {
}
}
private Integer mapReadAdvice(ReadAdvice readAdvice) {
private int mapReadAdvice(ReadAdvice readAdvice) {
return switch (readAdvice) {
case NORMAL -> POSIX_MADV_NORMAL;
case RANDOM -> POSIX_MADV_RANDOM;

5
lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveChecksumFooter.java
View File

@ -65,10 +65,7 @@ public class TestAllFilesHaveChecksumFooter extends LuceneTestCase {
}
if (si.info.getUseCompoundFile()) {
try (Directory cfsDir =
si.info
.getCodec()
.compoundFormat()
.getCompoundReader(dir, si.info, newIOContext(random()))) {
si.info.getCodec().compoundFormat().getCompoundReader(dir, si.info)) {
for (String cfsFile : cfsDir.listAll()) {
checkFooter(cfsDir, cfsFile);
}

5
lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveCodecHeader.java
View File

@ -70,10 +70,7 @@ public class TestAllFilesHaveCodecHeader extends LuceneTestCase {
}
if (si.info.getUseCompoundFile()) {
try (Directory cfsDir =
si.info
.getCodec()
.compoundFormat()
.getCompoundReader(dir, si.info, newIOContext(random()))) {
si.info.getCodec().compoundFormat().getCompoundReader(dir, si.info)) {
for (String cfsFile : cfsDir.listAll()) {
checkHeader(cfsDir, cfsFile, namesToExtensions, si.info.getId());
}

6
lucene/core/src/test/org/apache/lucene/index/TestIndexWriterForceMerge.java
View File

@ -40,7 +40,6 @@ import org.apache.lucene.document.LongPoint;
import org.apache.lucene.document.StringField;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.tests.analysis.MockAnalyzer;
import org.apache.lucene.tests.analysis.MockTokenizer;
import org.apache.lucene.tests.store.MockDirectoryWrapper;
@ -244,10 +243,7 @@ public class TestIndexWriterForceMerge extends LuceneTestCase {
}
if (info.info.getUseCompoundFile()) {
try (Directory cfs =
info.info
.getCodec()
.compoundFormat()
.getCompoundReader(dir, info.info, IOContext.DEFAULT)) {
info.info.getCodec().compoundFormat().getCompoundReader(dir, info.info)) {
for (String file : cfs.listAll()) {
sb.append(
String.format(

5
lucene/test-framework/src/java/org/apache/lucene/tests/codecs/cranky/CrankyCompoundFormat.java
View File

@ -34,9 +34,8 @@ class CrankyCompoundFormat extends CompoundFormat {
}
@Override
public CompoundDirectory getCompoundReader(Directory dir, SegmentInfo si, IOContext context)
throws IOException {
return delegate.getCompoundReader(dir, si, context);
public CompoundDirectory getCompoundReader(Directory dir, SegmentInfo si) throws IOException {
return delegate.getCompoundReader(dir, si);
}
@Override

33
lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseCompoundFormatTestCase.java
View File

@ -64,7 +64,7 @@ public abstract class BaseCompoundFormatTestCase extends BaseIndexFileFormatTest
SegmentInfo si = newSegmentInfo(dir, "_123");
si.setFiles(Collections.emptySet());
si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si);
assertEquals(0, cfs.listAll().length);
cfs.close();
dir.close();
@ -84,7 +84,7 @@ public abstract class BaseCompoundFormatTestCase extends BaseIndexFileFormatTest
si.setFiles(Collections.singleton(testfile));
si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si);
IndexInput expected = dir.openInput(testfile, newIOContext(random()));
IndexInput actual = cfs.openInput(testfile, newIOContext(random()));
@ -107,7 +107,7 @@ public abstract class BaseCompoundFormatTestCase extends BaseIndexFileFormatTest
si.setFiles(Arrays.asList(files));
si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si);
for (String file : files) {
IndexInput expected = dir.openInput(file, newIOContext(random()));
@ -136,7 +136,7 @@ public abstract class BaseCompoundFormatTestCase extends BaseIndexFileFormatTest
si.setFiles(Collections.singleton(testfile));
si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si);
assertEquals(1, cfs.listAll().length);
cfs.close();
cfs.close(); // second close should not throw exception
@ -215,10 +215,7 @@ public abstract class BaseCompoundFormatTestCase extends BaseIndexFileFormatTest
for (SegmentCommitInfo si : infos) {
if (si.info.getUseCompoundFile()) {
try (Directory cfsDir =
si.info
.getCodec()
.compoundFormat()
.getCompoundReader(dir, si.info, newIOContext(random()))) {
si.info.getCodec().compoundFormat().getCompoundReader(dir, si.info)) {
for (String cfsFile : cfsDir.listAll()) {
try (IndexInput cfsIn = cfsDir.openInput(cfsFile, IOContext.DEFAULT)) {
assert cfsIn != null;
@ -237,7 +234,7 @@ public abstract class BaseCompoundFormatTestCase extends BaseIndexFileFormatTest
SegmentInfo si = newSegmentInfo(dir, "_123");
si.setFiles(Collections.emptyList());
si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si);
expectThrows(
UnsupportedOperationException.class,
() -> {
@ -260,7 +257,7 @@ public abstract class BaseCompoundFormatTestCase extends BaseIndexFileFormatTest
SegmentInfo si = newSegmentInfo(dir, "_123");
si.setFiles(Collections.emptyList());
si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si);
expectThrows(
UnsupportedOperationException.class,
() -> {
@ -283,7 +280,7 @@ public abstract class BaseCompoundFormatTestCase extends BaseIndexFileFormatTest
SegmentInfo si = newSegmentInfo(dir, "_123");
si.setFiles(Collections.emptyList());
si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si);
expectThrows(
UnsupportedOperationException.class,
() -> {
@ -306,7 +303,7 @@ public abstract class BaseCompoundFormatTestCase extends BaseIndexFileFormatTest
SegmentInfo si = newSegmentInfo(dir, "_123");
si.setFiles(Collections.emptyList());
si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si);
expectThrows(
UnsupportedOperationException.class,
() -> {
@ -329,7 +326,7 @@ public abstract class BaseCompoundFormatTestCase extends BaseIndexFileFormatTest
SegmentInfo si = newSegmentInfo(dir, "_123");
si.setFiles(Collections.emptyList());
si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si);
expectThrows(
UnsupportedOperationException.class,
() -> {
@ -374,7 +371,7 @@ public abstract class BaseCompoundFormatTestCase extends BaseIndexFileFormatTest
si.setFiles(files);
si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si);
for (String file : files) {
IndexInput check = dir.openInput(file, newIOContext(random()));
@ -411,7 +408,7 @@ public abstract class BaseCompoundFormatTestCase extends BaseIndexFileFormatTest
si.setFiles(files);
si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si);
final IndexInput[] ins = new IndexInput[FILE_COUNT];
for (int fileIdx = 0; fileIdx < FILE_COUNT; fileIdx++) {
@ -793,7 +790,7 @@ public abstract class BaseCompoundFormatTestCase extends BaseIndexFileFormatTest
si.setFiles(files);
si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si);
return cfs;
}
@ -817,7 +814,7 @@ public abstract class BaseCompoundFormatTestCase extends BaseIndexFileFormatTest
si.setFiles(Collections.singletonList(subFile));
si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si);
IndexInput in = cfs.openInput(subFile, IOContext.DEFAULT);
String desc = in.toString();
assertTrue(
@ -899,7 +896,7 @@ public abstract class BaseCompoundFormatTestCase extends BaseIndexFileFormatTest
ReadBytesDirectoryWrapper readTrackingDir = new ReadBytesDirectoryWrapper(dir);
CompoundDirectory compoundDir =
si.getCodec().compoundFormat().getCompoundReader(readTrackingDir, si, IOContext.DEFAULT);
si.getCodec().compoundFormat().getCompoundReader(readTrackingDir, si);
compoundDir.checkIntegrity();
Map<String, FixedBitSet> readBytes = readTrackingDir.getReadBytes();
assertEquals(createdFiles, readBytes.keySet());

9
lucene/test-framework/src/java/org/apache/lucene/tests/store/MockDirectoryWrapper.java
View File

@ -53,6 +53,7 @@ import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.Lock;
import org.apache.lucene.store.ReadAdvice;
import org.apache.lucene.tests.util.LuceneTestCase;
import org.apache.lucene.tests.util.TestUtil;
import org.apache.lucene.tests.util.ThrottledIndexOutput;
@ -812,6 +813,8 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
false);
}
// record the read advice before randomizing the context
ReadAdvice readAdvice = context.readAdvice();
context = LuceneTestCase.newIOContext(randomState, context);
final boolean confined = context == IOContext.READONCE;
if (name.startsWith(IndexFileNames.SEGMENTS) && confined == false) {
@ -831,15 +834,15 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
System.out.println(
"MockDirectoryWrapper: using SlowClosingMockIndexInputWrapper for file " + name);
}
ii = new SlowClosingMockIndexInputWrapper(this, name, delegateInput, confined);
ii = new SlowClosingMockIndexInputWrapper(this, name, delegateInput, readAdvice, confined);
} else if (useSlowOpenClosers && randomInt == 1) {
if (LuceneTestCase.VERBOSE) {
System.out.println(
"MockDirectoryWrapper: using SlowOpeningMockIndexInputWrapper for file " + name);
}
ii = new SlowOpeningMockIndexInputWrapper(this, name, delegateInput, confined);
ii = new SlowOpeningMockIndexInputWrapper(this, name, delegateInput, readAdvice, confined);
} else {
ii = new MockIndexInputWrapper(this, name, delegateInput, null, confined);
ii = new MockIndexInputWrapper(this, name, delegateInput, null, readAdvice, confined);
}
addFileHandle(ii, name, Handle.Input);
return ii;

28
lucene/test-framework/src/java/org/apache/lucene/tests/store/MockIndexInputWrapper.java
View File

@ -23,6 +23,7 @@ import java.util.Set;
import org.apache.lucene.internal.tests.TestSecrets;
import org.apache.lucene.store.FilterIndexInput;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.ReadAdvice;
/**
* Used by MockDirectoryWrapper to create an input stream that keeps track of when it's been closed.
@ -39,6 +40,7 @@ public class MockIndexInputWrapper extends FilterIndexInput {
// Which MockIndexInputWrapper we were cloned from, or null if we are not a clone:
private final MockIndexInputWrapper parent;
private final ReadAdvice readAdvice;
private final boolean confined;
private final Thread thread;
@ -48,6 +50,7 @@ public class MockIndexInputWrapper extends FilterIndexInput {
String name,
IndexInput delegate,
MockIndexInputWrapper parent,
ReadAdvice readAdvice,
boolean confined) {
super("MockIndexInputWrapper(name=" + name + " delegate=" + delegate + ")", delegate);
@ -57,6 +60,7 @@ public class MockIndexInputWrapper extends FilterIndexInput {
this.parent = parent;
this.name = name;
this.dir = dir;
this.readAdvice = readAdvice;
this.confined = confined;
this.thread = Thread.currentThread();
}
@ -107,7 +111,8 @@ public class MockIndexInputWrapper extends FilterIndexInput {
dir.inputCloneCount.incrementAndGet();
IndexInput iiclone = in.clone();
MockIndexInputWrapper clone =
new MockIndexInputWrapper(dir, name, iiclone, parent != null ? parent : this, confined);
new MockIndexInputWrapper(
dir, name, iiclone, parent != null ? parent : this, readAdvice, confined);
// Pending resolution on LUCENE-686 we may want to
// uncomment this code so that we also track that all
// clones get closed:
@ -135,7 +140,26 @@ public class MockIndexInputWrapper extends FilterIndexInput {
IndexInput slice = in.slice(sliceDescription, offset, length);
MockIndexInputWrapper clone =
new MockIndexInputWrapper(
dir, sliceDescription, slice, parent != null ? parent : this, confined);
dir, sliceDescription, slice, parent != null ? parent : this, readAdvice, confined);
return clone;
}
@Override
public IndexInput slice(String sliceDescription, long offset, long length, ReadAdvice readAdvice)
throws IOException {
if (this.readAdvice != ReadAdvice.NORMAL) {
throw new IllegalStateException(
"slice() may only be called with a custom read advice on inputs that have been open with ReadAdvice.NORMAL");
}
ensureOpen();
if (dir.verboseClone) {
new Exception("slice: " + this).printStackTrace(System.out);
}
dir.inputCloneCount.incrementAndGet();
IndexInput slice = in.slice(sliceDescription, offset, length);
MockIndexInputWrapper clone =
new MockIndexInputWrapper(
dir, sliceDescription, slice, parent != null ? parent : this, readAdvice, confined);
return clone;
}

9
lucene/test-framework/src/java/org/apache/lucene/tests/store/SlowClosingMockIndexInputWrapper.java
View File

@ -19,6 +19,7 @@ package org.apache.lucene.tests.store;
import java.io.IOException;
import org.apache.lucene.internal.tests.TestSecrets;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.ReadAdvice;
import org.apache.lucene.util.SuppressForbidden;
import org.apache.lucene.util.ThreadInterruptedException;
@ -35,8 +36,12 @@ class SlowClosingMockIndexInputWrapper extends MockIndexInputWrapper {
}
public SlowClosingMockIndexInputWrapper(
MockDirectoryWrapper dir, String name, IndexInput delegate, boolean confined) {
super(dir, name, delegate, null, confined);
MockDirectoryWrapper dir,
String name,
IndexInput delegate,
ReadAdvice readAdvice,
boolean confined) {
super(dir, name, delegate, null, readAdvice, confined);
}
@SuppressForbidden(reason = "Thread sleep")

9
lucene/test-framework/src/java/org/apache/lucene/tests/store/SlowOpeningMockIndexInputWrapper.java
View File

@ -19,6 +19,7 @@ package org.apache.lucene.tests.store;
import java.io.IOException;
import org.apache.lucene.internal.tests.TestSecrets;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.ReadAdvice;
import org.apache.lucene.util.SuppressForbidden;
import org.apache.lucene.util.ThreadInterruptedException;
@ -35,9 +36,13 @@ class SlowOpeningMockIndexInputWrapper extends MockIndexInputWrapper {
@SuppressForbidden(reason = "Thread sleep")
public SlowOpeningMockIndexInputWrapper(
MockDirectoryWrapper dir, String name, IndexInput delegate, boolean confined)
MockDirectoryWrapper dir,
String name,
IndexInput delegate,
ReadAdvice readAdvice,
boolean confined)
throws IOException {
super(dir, name, delegate, null, confined);
super(dir, name, delegate, null, readAdvice, confined);
try {
Thread.sleep(50);
} catch (InterruptedException ie) {