mirror of https://github.com/apache/lucene.git
Only call madvise when necessary. (#13907)
This commit tries to save calls to `madvise` which are not necessary, either because they map to the OS' default, or because the advice would be overridden later on anyway. I have not noticed specific problems with this, but it seems desirable to keep calls to `madvise` to a minimum. As a consequence: - Files that are open with `ReadAdvice.NORMAL` do not call `madvise` since this is the OS' default. - Compound files are always open with `ReadAdvice.NORMAL`, and the actual is only set when opening sub files of these compound files. To make the latter less trappy, the `IOContext` parameter has been removed from `CompoundFormat#getCompoundReader`.
This commit is contained in:
parent
66bb46f71f
commit
e2636c648d
|
@ -77,9 +77,8 @@ public final class Lucene50CompoundFormat extends CompoundFormat {
|
|||
public Lucene50CompoundFormat() {}
|
||||
|
||||
@Override
|
||||
public CompoundDirectory getCompoundReader(Directory dir, SegmentInfo si, IOContext context)
|
||||
throws IOException {
|
||||
return new Lucene50CompoundReader(dir, si, context);
|
||||
public CompoundDirectory getCompoundReader(Directory dir, SegmentInfo si) throws IOException {
|
||||
return new Lucene50CompoundReader(dir, si);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
@ -31,6 +31,7 @@ import org.apache.lucene.store.ChecksumIndexInput;
|
|||
import org.apache.lucene.store.Directory;
|
||||
import org.apache.lucene.store.IOContext;
|
||||
import org.apache.lucene.store.IndexInput;
|
||||
import org.apache.lucene.store.ReadAdvice;
|
||||
import org.apache.lucene.util.CollectionUtil;
|
||||
import org.apache.lucene.util.IOUtils;
|
||||
|
||||
|
@ -57,8 +58,7 @@ final class Lucene50CompoundReader extends CompoundDirectory {
|
|||
/** Create a new CompoundFileDirectory. */
|
||||
// TODO: we should just pre-strip "entries" and append segment name up-front like simpletext?
|
||||
// this need not be a "general purpose" directory anymore (it only writes index files)
|
||||
public Lucene50CompoundReader(Directory directory, SegmentInfo si, IOContext context)
|
||||
throws IOException {
|
||||
public Lucene50CompoundReader(Directory directory, SegmentInfo si) throws IOException {
|
||||
this.directory = directory;
|
||||
this.segmentName = si.name;
|
||||
String dataFileName =
|
||||
|
@ -74,7 +74,7 @@ final class Lucene50CompoundReader extends CompoundDirectory {
|
|||
}
|
||||
expectedLength += CodecUtil.footerLength();
|
||||
|
||||
handle = directory.openInput(dataFileName, context);
|
||||
handle = directory.openInput(dataFileName, IOContext.DEFAULT.withReadAdvice(ReadAdvice.NORMAL));
|
||||
// DirectoryUtil.openInput(directory, dataFileName, context);
|
||||
try {
|
||||
CodecUtil.checkIndexHeader(
|
||||
|
@ -170,7 +170,7 @@ final class Lucene50CompoundReader extends CompoundDirectory {
|
|||
+ entries.keySet()
|
||||
+ ")");
|
||||
}
|
||||
return handle.slice(name, entry.offset, entry.length);
|
||||
return handle.slice(name, entry.offset, entry.length, context.readAdvice());
|
||||
}
|
||||
|
||||
/** Returns an array of strings, one for each file in the directory. */
|
||||
|
|
|
@ -81,9 +81,8 @@ public final class Lucene50RWCompoundFormat extends CompoundFormat {
|
|||
public Lucene50RWCompoundFormat() {}
|
||||
|
||||
@Override
|
||||
public CompoundDirectory getCompoundReader(Directory dir, SegmentInfo si, IOContext context)
|
||||
throws IOException {
|
||||
return new Lucene50CompoundReader(dir, si, context);
|
||||
public CompoundDirectory getCompoundReader(Directory dir, SegmentInfo si) throws IOException {
|
||||
return new Lucene50CompoundReader(dir, si);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
@ -35,6 +35,7 @@ import org.apache.lucene.store.Directory;
|
|||
import org.apache.lucene.store.IOContext;
|
||||
import org.apache.lucene.store.IndexInput;
|
||||
import org.apache.lucene.store.IndexOutput;
|
||||
import org.apache.lucene.store.ReadAdvice;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.apache.lucene.util.BytesRefBuilder;
|
||||
import org.apache.lucene.util.StringHelper;
|
||||
|
@ -52,10 +53,10 @@ public class SimpleTextCompoundFormat extends CompoundFormat {
|
|||
public SimpleTextCompoundFormat() {}
|
||||
|
||||
@Override
|
||||
public CompoundDirectory getCompoundReader(Directory dir, SegmentInfo si, IOContext context)
|
||||
throws IOException {
|
||||
public CompoundDirectory getCompoundReader(Directory dir, SegmentInfo si) throws IOException {
|
||||
String dataFile = IndexFileNames.segmentFileName(si.name, "", DATA_EXTENSION);
|
||||
final IndexInput in = dir.openInput(dataFile, context);
|
||||
final IndexInput in =
|
||||
dir.openInput(dataFile, IOContext.DEFAULT.withReadAdvice(ReadAdvice.NORMAL));
|
||||
|
||||
BytesRefBuilder scratch = new BytesRefBuilder();
|
||||
|
||||
|
@ -135,7 +136,11 @@ public class SimpleTextCompoundFormat extends CompoundFormat {
|
|||
public IndexInput openInput(String name, IOContext context) throws IOException {
|
||||
ensureOpen();
|
||||
int index = getIndex(name);
|
||||
return in.slice(name, startOffsets[index], endOffsets[index] - startOffsets[index]);
|
||||
return in.slice(
|
||||
name,
|
||||
startOffsets[index],
|
||||
endOffsets[index] - startOffsets[index],
|
||||
context.readAdvice());
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
@ -35,8 +35,8 @@ public abstract class CompoundFormat {
|
|||
// we can add 'producer' classes.
|
||||
|
||||
/** Returns a Directory view (read-only) for the compound files in this segment */
|
||||
public abstract CompoundDirectory getCompoundReader(
|
||||
Directory dir, SegmentInfo si, IOContext context) throws IOException;
|
||||
public abstract CompoundDirectory getCompoundReader(Directory dir, SegmentInfo si)
|
||||
throws IOException;
|
||||
|
||||
/**
|
||||
* Packs the provided segment's files into a compound format. All files referenced by the provided
|
||||
|
|
|
@ -82,9 +82,8 @@ public final class Lucene90CompoundFormat extends CompoundFormat {
|
|||
public Lucene90CompoundFormat() {}
|
||||
|
||||
@Override
|
||||
public CompoundDirectory getCompoundReader(Directory dir, SegmentInfo si, IOContext context)
|
||||
throws IOException {
|
||||
return new Lucene90CompoundReader(dir, si, context);
|
||||
public CompoundDirectory getCompoundReader(Directory dir, SegmentInfo si) throws IOException {
|
||||
return new Lucene90CompoundReader(dir, si);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
@ -30,6 +30,7 @@ import org.apache.lucene.store.ChecksumIndexInput;
|
|||
import org.apache.lucene.store.Directory;
|
||||
import org.apache.lucene.store.IOContext;
|
||||
import org.apache.lucene.store.IndexInput;
|
||||
import org.apache.lucene.store.ReadAdvice;
|
||||
import org.apache.lucene.util.CollectionUtil;
|
||||
import org.apache.lucene.util.IOUtils;
|
||||
|
||||
|
@ -56,8 +57,7 @@ final class Lucene90CompoundReader extends CompoundDirectory {
|
|||
/** Create a new CompoundFileDirectory. */
|
||||
// TODO: we should just pre-strip "entries" and append segment name up-front like simpletext?
|
||||
// this need not be a "general purpose" directory anymore (it only writes index files)
|
||||
public Lucene90CompoundReader(Directory directory, SegmentInfo si, IOContext context)
|
||||
throws IOException {
|
||||
public Lucene90CompoundReader(Directory directory, SegmentInfo si) throws IOException {
|
||||
this.directory = directory;
|
||||
this.segmentName = si.name;
|
||||
String dataFileName =
|
||||
|
@ -75,7 +75,7 @@ final class Lucene90CompoundReader extends CompoundDirectory {
|
|||
.orElseGet(() -> CodecUtil.indexHeaderLength(Lucene90CompoundFormat.DATA_CODEC, ""))
|
||||
+ CodecUtil.footerLength();
|
||||
|
||||
handle = directory.openInput(dataFileName, context);
|
||||
handle = directory.openInput(dataFileName, IOContext.DEFAULT.withReadAdvice(ReadAdvice.NORMAL));
|
||||
try {
|
||||
CodecUtil.checkIndexHeader(
|
||||
handle, Lucene90CompoundFormat.DATA_CODEC, version, version, si.getId(), "");
|
||||
|
@ -169,7 +169,7 @@ final class Lucene90CompoundReader extends CompoundDirectory {
|
|||
+ entries.keySet()
|
||||
+ ")");
|
||||
}
|
||||
return handle.slice(name, entry.offset, entry.length);
|
||||
return handle.slice(name, entry.offset, entry.length, context.readAdvice());
|
||||
}
|
||||
|
||||
/** Returns an array of strings, one for each file in the directory. */
|
||||
|
|
|
@ -1255,8 +1255,7 @@ public class IndexWriter
|
|||
return reader.read(si.info.dir, si.info, segmentSuffix, IOContext.READONCE);
|
||||
} else if (si.info.getUseCompoundFile()) {
|
||||
// cfs
|
||||
try (Directory cfs =
|
||||
codec.compoundFormat().getCompoundReader(si.info.dir, si.info, IOContext.DEFAULT)) {
|
||||
try (Directory cfs = codec.compoundFormat().getCompoundReader(si.info.dir, si.info)) {
|
||||
return reader.read(cfs, si.info, "", IOContext.READONCE);
|
||||
}
|
||||
} else {
|
||||
|
|
|
@ -226,12 +226,7 @@ final class PendingSoftDeletes extends PendingDeletes {
|
|||
// updates always outside of CFS
|
||||
Closeable toClose;
|
||||
if (segInfo.getUseCompoundFile()) {
|
||||
toClose =
|
||||
dir =
|
||||
segInfo
|
||||
.getCodec()
|
||||
.compoundFormat()
|
||||
.getCompoundReader(segInfo.dir, segInfo, IOContext.READONCE);
|
||||
toClose = dir = segInfo.getCodec().compoundFormat().getCompoundReader(segInfo.dir, segInfo);
|
||||
} else {
|
||||
toClose = null;
|
||||
dir = segInfo.dir;
|
||||
|
|
|
@ -80,7 +80,7 @@ final class SegmentCoreReaders {
|
|||
|
||||
try {
|
||||
if (si.info.getUseCompoundFile()) {
|
||||
cfsDir = cfsReader = codec.compoundFormat().getCompoundReader(dir, si.info, context);
|
||||
cfsDir = cfsReader = codec.compoundFormat().getCompoundReader(dir, si.info);
|
||||
} else {
|
||||
cfsReader = null;
|
||||
cfsDir = dir;
|
||||
|
|
|
@ -127,6 +127,10 @@ public abstract class IndexInput extends DataInput implements Closeable {
|
|||
* CompoundFormat} implementations to honor the {@link ReadAdvice} of each file within the
|
||||
* compound file.
|
||||
*
|
||||
* <p><b>NOTE</b>: it is only legal to call this method if this {@link IndexInput} has been open
|
||||
* with {@link ReadAdvice#NORMAL}. However, this method accepts any {@link ReadAdvice} value but
|
||||
* {@code null} as a read advice for the slice.
|
||||
*
|
||||
* <p>The default implementation delegates to {@link #slice(String, long, long)} and ignores the
|
||||
* {@link ReadAdvice}.
|
||||
*/
|
||||
|
|
|
@ -567,7 +567,8 @@ abstract class MemorySegmentIndexInput extends IndexInput
|
|||
public final MemorySegmentIndexInput slice(
|
||||
String sliceDescription, long offset, long length, ReadAdvice advice) throws IOException {
|
||||
MemorySegmentIndexInput slice = slice(sliceDescription, offset, length);
|
||||
if (NATIVE_ACCESS.isPresent()) {
|
||||
if (NATIVE_ACCESS.isPresent() && advice != ReadAdvice.NORMAL) {
|
||||
// No need to madvise with a normal advice, since it's the OS' default.
|
||||
final NativeAccess nativeAccess = NATIVE_ACCESS.get();
|
||||
slice.advise(
|
||||
0,
|
||||
|
|
|
@ -129,7 +129,9 @@ final class MemorySegmentIndexInputProvider
|
|||
// internal FileChannel logic)
|
||||
if (preload) {
|
||||
segment.load();
|
||||
} else if (nativeAccess.filter(na -> segment.address() % na.getPageSize() == 0).isPresent()) {
|
||||
} else if (readAdvice != ReadAdvice.NORMAL
|
||||
&& nativeAccess.filter(na -> segment.address() % na.getPageSize() == 0).isPresent()) {
|
||||
// No need to madvise with ReadAdvice.NORMAL since it is the OS' default read advice.
|
||||
nativeAccess.get().madvise(segment, readAdvice);
|
||||
}
|
||||
segments[segNr] = segment;
|
||||
|
|
|
@ -122,10 +122,7 @@ final class PosixNativeAccess extends NativeAccess {
|
|||
|
||||
@Override
|
||||
public void madvise(MemorySegment segment, ReadAdvice readAdvice) throws IOException {
|
||||
final Integer advice = mapReadAdvice(readAdvice);
|
||||
if (advice == null) {
|
||||
return; // do nothing
|
||||
}
|
||||
final int advice = mapReadAdvice(readAdvice);
|
||||
madvise(segment, advice);
|
||||
}
|
||||
|
||||
|
@ -156,7 +153,7 @@ final class PosixNativeAccess extends NativeAccess {
|
|||
}
|
||||
}
|
||||
|
||||
private Integer mapReadAdvice(ReadAdvice readAdvice) {
|
||||
private int mapReadAdvice(ReadAdvice readAdvice) {
|
||||
return switch (readAdvice) {
|
||||
case NORMAL -> POSIX_MADV_NORMAL;
|
||||
case RANDOM -> POSIX_MADV_RANDOM;
|
||||
|
|
|
@ -65,10 +65,7 @@ public class TestAllFilesHaveChecksumFooter extends LuceneTestCase {
|
|||
}
|
||||
if (si.info.getUseCompoundFile()) {
|
||||
try (Directory cfsDir =
|
||||
si.info
|
||||
.getCodec()
|
||||
.compoundFormat()
|
||||
.getCompoundReader(dir, si.info, newIOContext(random()))) {
|
||||
si.info.getCodec().compoundFormat().getCompoundReader(dir, si.info)) {
|
||||
for (String cfsFile : cfsDir.listAll()) {
|
||||
checkFooter(cfsDir, cfsFile);
|
||||
}
|
||||
|
|
|
@ -70,10 +70,7 @@ public class TestAllFilesHaveCodecHeader extends LuceneTestCase {
|
|||
}
|
||||
if (si.info.getUseCompoundFile()) {
|
||||
try (Directory cfsDir =
|
||||
si.info
|
||||
.getCodec()
|
||||
.compoundFormat()
|
||||
.getCompoundReader(dir, si.info, newIOContext(random()))) {
|
||||
si.info.getCodec().compoundFormat().getCompoundReader(dir, si.info)) {
|
||||
for (String cfsFile : cfsDir.listAll()) {
|
||||
checkHeader(cfsDir, cfsFile, namesToExtensions, si.info.getId());
|
||||
}
|
||||
|
|
|
@ -40,7 +40,6 @@ import org.apache.lucene.document.LongPoint;
|
|||
import org.apache.lucene.document.StringField;
|
||||
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
|
||||
import org.apache.lucene.store.Directory;
|
||||
import org.apache.lucene.store.IOContext;
|
||||
import org.apache.lucene.tests.analysis.MockAnalyzer;
|
||||
import org.apache.lucene.tests.analysis.MockTokenizer;
|
||||
import org.apache.lucene.tests.store.MockDirectoryWrapper;
|
||||
|
@ -244,10 +243,7 @@ public class TestIndexWriterForceMerge extends LuceneTestCase {
|
|||
}
|
||||
if (info.info.getUseCompoundFile()) {
|
||||
try (Directory cfs =
|
||||
info.info
|
||||
.getCodec()
|
||||
.compoundFormat()
|
||||
.getCompoundReader(dir, info.info, IOContext.DEFAULT)) {
|
||||
info.info.getCodec().compoundFormat().getCompoundReader(dir, info.info)) {
|
||||
for (String file : cfs.listAll()) {
|
||||
sb.append(
|
||||
String.format(
|
||||
|
|
|
@ -34,9 +34,8 @@ class CrankyCompoundFormat extends CompoundFormat {
|
|||
}
|
||||
|
||||
@Override
|
||||
public CompoundDirectory getCompoundReader(Directory dir, SegmentInfo si, IOContext context)
|
||||
throws IOException {
|
||||
return delegate.getCompoundReader(dir, si, context);
|
||||
public CompoundDirectory getCompoundReader(Directory dir, SegmentInfo si) throws IOException {
|
||||
return delegate.getCompoundReader(dir, si);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
@ -64,7 +64,7 @@ public abstract class BaseCompoundFormatTestCase extends BaseIndexFileFormatTest
|
|||
SegmentInfo si = newSegmentInfo(dir, "_123");
|
||||
si.setFiles(Collections.emptySet());
|
||||
si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
|
||||
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
|
||||
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si);
|
||||
assertEquals(0, cfs.listAll().length);
|
||||
cfs.close();
|
||||
dir.close();
|
||||
|
@ -84,7 +84,7 @@ public abstract class BaseCompoundFormatTestCase extends BaseIndexFileFormatTest
|
|||
|
||||
si.setFiles(Collections.singleton(testfile));
|
||||
si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
|
||||
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
|
||||
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si);
|
||||
|
||||
IndexInput expected = dir.openInput(testfile, newIOContext(random()));
|
||||
IndexInput actual = cfs.openInput(testfile, newIOContext(random()));
|
||||
|
@ -107,7 +107,7 @@ public abstract class BaseCompoundFormatTestCase extends BaseIndexFileFormatTest
|
|||
|
||||
si.setFiles(Arrays.asList(files));
|
||||
si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
|
||||
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
|
||||
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si);
|
||||
|
||||
for (String file : files) {
|
||||
IndexInput expected = dir.openInput(file, newIOContext(random()));
|
||||
|
@ -136,7 +136,7 @@ public abstract class BaseCompoundFormatTestCase extends BaseIndexFileFormatTest
|
|||
|
||||
si.setFiles(Collections.singleton(testfile));
|
||||
si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
|
||||
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
|
||||
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si);
|
||||
assertEquals(1, cfs.listAll().length);
|
||||
cfs.close();
|
||||
cfs.close(); // second close should not throw exception
|
||||
|
@ -215,10 +215,7 @@ public abstract class BaseCompoundFormatTestCase extends BaseIndexFileFormatTest
|
|||
for (SegmentCommitInfo si : infos) {
|
||||
if (si.info.getUseCompoundFile()) {
|
||||
try (Directory cfsDir =
|
||||
si.info
|
||||
.getCodec()
|
||||
.compoundFormat()
|
||||
.getCompoundReader(dir, si.info, newIOContext(random()))) {
|
||||
si.info.getCodec().compoundFormat().getCompoundReader(dir, si.info)) {
|
||||
for (String cfsFile : cfsDir.listAll()) {
|
||||
try (IndexInput cfsIn = cfsDir.openInput(cfsFile, IOContext.DEFAULT)) {
|
||||
assert cfsIn != null;
|
||||
|
@ -237,7 +234,7 @@ public abstract class BaseCompoundFormatTestCase extends BaseIndexFileFormatTest
|
|||
SegmentInfo si = newSegmentInfo(dir, "_123");
|
||||
si.setFiles(Collections.emptyList());
|
||||
si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
|
||||
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
|
||||
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si);
|
||||
expectThrows(
|
||||
UnsupportedOperationException.class,
|
||||
() -> {
|
||||
|
@ -260,7 +257,7 @@ public abstract class BaseCompoundFormatTestCase extends BaseIndexFileFormatTest
|
|||
SegmentInfo si = newSegmentInfo(dir, "_123");
|
||||
si.setFiles(Collections.emptyList());
|
||||
si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
|
||||
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
|
||||
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si);
|
||||
expectThrows(
|
||||
UnsupportedOperationException.class,
|
||||
() -> {
|
||||
|
@ -283,7 +280,7 @@ public abstract class BaseCompoundFormatTestCase extends BaseIndexFileFormatTest
|
|||
SegmentInfo si = newSegmentInfo(dir, "_123");
|
||||
si.setFiles(Collections.emptyList());
|
||||
si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
|
||||
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
|
||||
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si);
|
||||
expectThrows(
|
||||
UnsupportedOperationException.class,
|
||||
() -> {
|
||||
|
@ -306,7 +303,7 @@ public abstract class BaseCompoundFormatTestCase extends BaseIndexFileFormatTest
|
|||
SegmentInfo si = newSegmentInfo(dir, "_123");
|
||||
si.setFiles(Collections.emptyList());
|
||||
si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
|
||||
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
|
||||
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si);
|
||||
expectThrows(
|
||||
UnsupportedOperationException.class,
|
||||
() -> {
|
||||
|
@ -329,7 +326,7 @@ public abstract class BaseCompoundFormatTestCase extends BaseIndexFileFormatTest
|
|||
SegmentInfo si = newSegmentInfo(dir, "_123");
|
||||
si.setFiles(Collections.emptyList());
|
||||
si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
|
||||
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
|
||||
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si);
|
||||
expectThrows(
|
||||
UnsupportedOperationException.class,
|
||||
() -> {
|
||||
|
@ -374,7 +371,7 @@ public abstract class BaseCompoundFormatTestCase extends BaseIndexFileFormatTest
|
|||
|
||||
si.setFiles(files);
|
||||
si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
|
||||
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
|
||||
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si);
|
||||
|
||||
for (String file : files) {
|
||||
IndexInput check = dir.openInput(file, newIOContext(random()));
|
||||
|
@ -411,7 +408,7 @@ public abstract class BaseCompoundFormatTestCase extends BaseIndexFileFormatTest
|
|||
|
||||
si.setFiles(files);
|
||||
si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
|
||||
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
|
||||
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si);
|
||||
|
||||
final IndexInput[] ins = new IndexInput[FILE_COUNT];
|
||||
for (int fileIdx = 0; fileIdx < FILE_COUNT; fileIdx++) {
|
||||
|
@ -793,7 +790,7 @@ public abstract class BaseCompoundFormatTestCase extends BaseIndexFileFormatTest
|
|||
|
||||
si.setFiles(files);
|
||||
si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
|
||||
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
|
||||
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si);
|
||||
return cfs;
|
||||
}
|
||||
|
||||
|
@ -817,7 +814,7 @@ public abstract class BaseCompoundFormatTestCase extends BaseIndexFileFormatTest
|
|||
|
||||
si.setFiles(Collections.singletonList(subFile));
|
||||
si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
|
||||
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
|
||||
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si);
|
||||
IndexInput in = cfs.openInput(subFile, IOContext.DEFAULT);
|
||||
String desc = in.toString();
|
||||
assertTrue(
|
||||
|
@ -899,7 +896,7 @@ public abstract class BaseCompoundFormatTestCase extends BaseIndexFileFormatTest
|
|||
|
||||
ReadBytesDirectoryWrapper readTrackingDir = new ReadBytesDirectoryWrapper(dir);
|
||||
CompoundDirectory compoundDir =
|
||||
si.getCodec().compoundFormat().getCompoundReader(readTrackingDir, si, IOContext.DEFAULT);
|
||||
si.getCodec().compoundFormat().getCompoundReader(readTrackingDir, si);
|
||||
compoundDir.checkIntegrity();
|
||||
Map<String, FixedBitSet> readBytes = readTrackingDir.getReadBytes();
|
||||
assertEquals(createdFiles, readBytes.keySet());
|
||||
|
|
|
@ -53,6 +53,7 @@ import org.apache.lucene.store.IOContext;
|
|||
import org.apache.lucene.store.IndexInput;
|
||||
import org.apache.lucene.store.IndexOutput;
|
||||
import org.apache.lucene.store.Lock;
|
||||
import org.apache.lucene.store.ReadAdvice;
|
||||
import org.apache.lucene.tests.util.LuceneTestCase;
|
||||
import org.apache.lucene.tests.util.TestUtil;
|
||||
import org.apache.lucene.tests.util.ThrottledIndexOutput;
|
||||
|
@ -812,6 +813,8 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
|
|||
false);
|
||||
}
|
||||
|
||||
// record the read advice before randomizing the context
|
||||
ReadAdvice readAdvice = context.readAdvice();
|
||||
context = LuceneTestCase.newIOContext(randomState, context);
|
||||
final boolean confined = context == IOContext.READONCE;
|
||||
if (name.startsWith(IndexFileNames.SEGMENTS) && confined == false) {
|
||||
|
@ -831,15 +834,15 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
|
|||
System.out.println(
|
||||
"MockDirectoryWrapper: using SlowClosingMockIndexInputWrapper for file " + name);
|
||||
}
|
||||
ii = new SlowClosingMockIndexInputWrapper(this, name, delegateInput, confined);
|
||||
ii = new SlowClosingMockIndexInputWrapper(this, name, delegateInput, readAdvice, confined);
|
||||
} else if (useSlowOpenClosers && randomInt == 1) {
|
||||
if (LuceneTestCase.VERBOSE) {
|
||||
System.out.println(
|
||||
"MockDirectoryWrapper: using SlowOpeningMockIndexInputWrapper for file " + name);
|
||||
}
|
||||
ii = new SlowOpeningMockIndexInputWrapper(this, name, delegateInput, confined);
|
||||
ii = new SlowOpeningMockIndexInputWrapper(this, name, delegateInput, readAdvice, confined);
|
||||
} else {
|
||||
ii = new MockIndexInputWrapper(this, name, delegateInput, null, confined);
|
||||
ii = new MockIndexInputWrapper(this, name, delegateInput, null, readAdvice, confined);
|
||||
}
|
||||
addFileHandle(ii, name, Handle.Input);
|
||||
return ii;
|
||||
|
|
|
@ -23,6 +23,7 @@ import java.util.Set;
|
|||
import org.apache.lucene.internal.tests.TestSecrets;
|
||||
import org.apache.lucene.store.FilterIndexInput;
|
||||
import org.apache.lucene.store.IndexInput;
|
||||
import org.apache.lucene.store.ReadAdvice;
|
||||
|
||||
/**
|
||||
* Used by MockDirectoryWrapper to create an input stream that keeps track of when it's been closed.
|
||||
|
@ -39,6 +40,7 @@ public class MockIndexInputWrapper extends FilterIndexInput {
|
|||
|
||||
// Which MockIndexInputWrapper we were cloned from, or null if we are not a clone:
|
||||
private final MockIndexInputWrapper parent;
|
||||
private final ReadAdvice readAdvice;
|
||||
private final boolean confined;
|
||||
private final Thread thread;
|
||||
|
||||
|
@ -48,6 +50,7 @@ public class MockIndexInputWrapper extends FilterIndexInput {
|
|||
String name,
|
||||
IndexInput delegate,
|
||||
MockIndexInputWrapper parent,
|
||||
ReadAdvice readAdvice,
|
||||
boolean confined) {
|
||||
super("MockIndexInputWrapper(name=" + name + " delegate=" + delegate + ")", delegate);
|
||||
|
||||
|
@ -57,6 +60,7 @@ public class MockIndexInputWrapper extends FilterIndexInput {
|
|||
this.parent = parent;
|
||||
this.name = name;
|
||||
this.dir = dir;
|
||||
this.readAdvice = readAdvice;
|
||||
this.confined = confined;
|
||||
this.thread = Thread.currentThread();
|
||||
}
|
||||
|
@ -107,7 +111,8 @@ public class MockIndexInputWrapper extends FilterIndexInput {
|
|||
dir.inputCloneCount.incrementAndGet();
|
||||
IndexInput iiclone = in.clone();
|
||||
MockIndexInputWrapper clone =
|
||||
new MockIndexInputWrapper(dir, name, iiclone, parent != null ? parent : this, confined);
|
||||
new MockIndexInputWrapper(
|
||||
dir, name, iiclone, parent != null ? parent : this, readAdvice, confined);
|
||||
// Pending resolution on LUCENE-686 we may want to
|
||||
// uncomment this code so that we also track that all
|
||||
// clones get closed:
|
||||
|
@ -135,7 +140,26 @@ public class MockIndexInputWrapper extends FilterIndexInput {
|
|||
IndexInput slice = in.slice(sliceDescription, offset, length);
|
||||
MockIndexInputWrapper clone =
|
||||
new MockIndexInputWrapper(
|
||||
dir, sliceDescription, slice, parent != null ? parent : this, confined);
|
||||
dir, sliceDescription, slice, parent != null ? parent : this, readAdvice, confined);
|
||||
return clone;
|
||||
}
|
||||
|
||||
@Override
|
||||
public IndexInput slice(String sliceDescription, long offset, long length, ReadAdvice readAdvice)
|
||||
throws IOException {
|
||||
if (this.readAdvice != ReadAdvice.NORMAL) {
|
||||
throw new IllegalStateException(
|
||||
"slice() may only be called with a custom read advice on inputs that have been open with ReadAdvice.NORMAL");
|
||||
}
|
||||
ensureOpen();
|
||||
if (dir.verboseClone) {
|
||||
new Exception("slice: " + this).printStackTrace(System.out);
|
||||
}
|
||||
dir.inputCloneCount.incrementAndGet();
|
||||
IndexInput slice = in.slice(sliceDescription, offset, length);
|
||||
MockIndexInputWrapper clone =
|
||||
new MockIndexInputWrapper(
|
||||
dir, sliceDescription, slice, parent != null ? parent : this, readAdvice, confined);
|
||||
return clone;
|
||||
}
|
||||
|
||||
|
|
|
@ -19,6 +19,7 @@ package org.apache.lucene.tests.store;
|
|||
import java.io.IOException;
|
||||
import org.apache.lucene.internal.tests.TestSecrets;
|
||||
import org.apache.lucene.store.IndexInput;
|
||||
import org.apache.lucene.store.ReadAdvice;
|
||||
import org.apache.lucene.util.SuppressForbidden;
|
||||
import org.apache.lucene.util.ThreadInterruptedException;
|
||||
|
||||
|
@ -35,8 +36,12 @@ class SlowClosingMockIndexInputWrapper extends MockIndexInputWrapper {
|
|||
}
|
||||
|
||||
public SlowClosingMockIndexInputWrapper(
|
||||
MockDirectoryWrapper dir, String name, IndexInput delegate, boolean confined) {
|
||||
super(dir, name, delegate, null, confined);
|
||||
MockDirectoryWrapper dir,
|
||||
String name,
|
||||
IndexInput delegate,
|
||||
ReadAdvice readAdvice,
|
||||
boolean confined) {
|
||||
super(dir, name, delegate, null, readAdvice, confined);
|
||||
}
|
||||
|
||||
@SuppressForbidden(reason = "Thread sleep")
|
||||
|
|
|
@ -19,6 +19,7 @@ package org.apache.lucene.tests.store;
|
|||
import java.io.IOException;
|
||||
import org.apache.lucene.internal.tests.TestSecrets;
|
||||
import org.apache.lucene.store.IndexInput;
|
||||
import org.apache.lucene.store.ReadAdvice;
|
||||
import org.apache.lucene.util.SuppressForbidden;
|
||||
import org.apache.lucene.util.ThreadInterruptedException;
|
||||
|
||||
|
@ -35,9 +36,13 @@ class SlowOpeningMockIndexInputWrapper extends MockIndexInputWrapper {
|
|||
|
||||
@SuppressForbidden(reason = "Thread sleep")
|
||||
public SlowOpeningMockIndexInputWrapper(
|
||||
MockDirectoryWrapper dir, String name, IndexInput delegate, boolean confined)
|
||||
MockDirectoryWrapper dir,
|
||||
String name,
|
||||
IndexInput delegate,
|
||||
ReadAdvice readAdvice,
|
||||
boolean confined)
|
||||
throws IOException {
|
||||
super(dir, name, delegate, null, confined);
|
||||
super(dir, name, delegate, null, readAdvice, confined);
|
||||
try {
|
||||
Thread.sleep(50);
|
||||
} catch (InterruptedException ie) {
|
||||
|
|
Loading…
Reference in New Issue