Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Ensure to use IOContext.READONCE when reading segment files (#13574) 

This commit uses IOContext.READONCE in more places where the index input is clearly being read once by the thread opening it. We can then enforce that segment files are only opened with READONCE, in the test specific Mock directory wrapper.

Much of the changes in this PR update individual test usage, but there is one non-test change to Directory::copyFrom.
This commit is contained in:
Chris Hegarty 2024-07-17 08:56:11 +01:00 committed by GitHub
parent 5e52b8094a
commit 99488b2245
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
12 changed files with 25 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
lucene/core/src/java/org/apache/lucene/store/Directory.java
View File

@ -172,12 +172,12 @@ public abstract class Directory implements Closeable {
/**
* Copies an existing {@code src} file from directory {@code from} to a non-existent file {@code
* dest} in this directory.
* dest} in this directory. The given IOContext is only used for opening the destination file.
*/
public void copyFrom(Directory from, String src, String dest, IOContext context)
throws IOException {
boolean success = false;
try (IndexInput is = from.openInput(src, context);
try (IndexInput is = from.openInput(src, IOContext.READONCE);
IndexOutput os = createOutput(dest, context)) {
os.copyBytes(is, is.length());
success = true;

2
lucene/core/src/test/org/apache/lucene/index/TestAllFilesCheckIndexHeader.java
View File

@ -115,7 +115,7 @@ public class TestAllFilesCheckIndexHeader extends LuceneTestCase {
// time this will only require one iteration!
while (true) {
try (IndexOutput out = dirCopy.createOutput(name, IOContext.DEFAULT);
IndexInput in = dir.openInput(name, IOContext.DEFAULT)) {
IndexInput in = dir.openInput(name, IOContext.READONCE)) {
// keeps same file length, but replaces the first wrongBytes with random bytes:
byte[] bytes = new byte[wrongBytes];
random().nextBytes(bytes);

2
lucene/core/src/test/org/apache/lucene/index/TestAllFilesDetectMismatchedChecksum.java
View File

@ -122,7 +122,7 @@ public class TestAllFilesDetectMismatchedChecksum extends LuceneTestCase {
dirCopy.copyFrom(dir, name, name, IOContext.DEFAULT);
} else {
try (IndexOutput out = dirCopy.createOutput(name, IOContext.DEFAULT);
IndexInput in = dir.openInput(name, IOContext.DEFAULT)) {
IndexInput in = dir.openInput(name, IOContext.READONCE)) {
out.copyBytes(in, flipOffset);
out.writeByte((byte) (in.readByte() + TestUtil.nextInt(random(), 0x01, 0xFF)));
out.copyBytes(in, victimLength - flipOffset - 1);

2
lucene/core/src/test/org/apache/lucene/index/TestAllFilesDetectTruncation.java
View File

@ -152,7 +152,7 @@ public class TestAllFilesDetectTruncation extends LuceneTestCase {
}
try (IndexOutput out = dirCopy.createOutput(name, IOContext.DEFAULT);
IndexInput in = dir.openInput(name, IOContext.DEFAULT)) {
IndexInput in = dir.openInput(name, IOContext.READONCE)) {
out.copyBytes(in, victimLength - lostBytes);
}
}

3
lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveChecksumFooter.java
View File

@ -19,6 +19,7 @@ package org.apache.lucene.index;
import java.io.IOException;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.tests.analysis.MockAnalyzer;
import org.apache.lucene.tests.index.RandomIndexWriter;
@ -77,7 +78,7 @@ public class TestAllFilesHaveChecksumFooter extends LuceneTestCase {
}
private void checkFooter(Directory dir, String file) throws IOException {
try (IndexInput in = dir.openInput(file, newIOContext(random()))) {
try (IndexInput in = dir.openInput(file, IOContext.READONCE)) {
CodecUtil.checksumEntireFile(in);
}
}

3
lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveCodecHeader.java
View File

@ -21,6 +21,7 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.tests.analysis.MockAnalyzer;
import org.apache.lucene.tests.index.RandomIndexWriter;
@ -84,7 +85,7 @@ public class TestAllFilesHaveCodecHeader extends LuceneTestCase {
private void checkHeader(
Directory dir, String file, Map<String, String> namesToExtensions, byte[] id)
throws IOException {
try (IndexInput in = dir.openInput(file, newIOContext(random()))) {
try (IndexInput in = dir.openInput(file, IOContext.READONCE)) {
int val = CodecUtil.readBEInt(in);
assertEquals(
file + " has no codec header, instead found: " + val, CodecUtil.CODEC_MAGIC, val);

4
lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
View File

@ -1275,7 +1275,7 @@ public class TestIndexWriterExceptions extends LuceneTestCase {
assertTrue("segment generation should be > 0 but got " + gen, gen > 0);
final String segmentsFileName = SegmentInfos.getLastCommitSegmentsFileName(dir);
IndexInput in = dir.openInput(segmentsFileName, newIOContext(random()));
IndexInput in = dir.openInput(segmentsFileName, IOContext.READONCE);
IndexOutput out =
dir.createOutput(
IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", 1 + gen),
@ -1320,7 +1320,7 @@ public class TestIndexWriterExceptions extends LuceneTestCase {
String fileNameIn = SegmentInfos.getLastCommitSegmentsFileName(dir);
String fileNameOut =
IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", 1 + gen);
IndexInput in = dir.openInput(fileNameIn, newIOContext(random()));
IndexInput in = dir.openInput(fileNameIn, IOContext.READONCE);
IndexOutput out = dir.createOutput(fileNameOut, newIOContext(random()));
long length = in.length();
for (int i = 0; i < length - 1; i++) {

4
lucene/core/src/test/org/apache/lucene/index/TestSegmentInfos.java
View File

@ -367,7 +367,7 @@ public class TestSegmentInfos extends LuceneTestCase {
boolean corrupt = false;
for (String file : dir.listAll()) {
if (file.startsWith(IndexFileNames.SEGMENTS)) {
try (IndexInput in = dir.openInput(file, IOContext.DEFAULT);
try (IndexInput in = dir.openInput(file, IOContext.READONCE);
IndexOutput out = corruptDir.createOutput(file, IOContext.DEFAULT)) {
final long corruptIndex = TestUtil.nextLong(random(), 0, in.length() - 1);
out.copyBytes(in, corruptIndex);
@ -375,7 +375,7 @@ public class TestSegmentInfos extends LuceneTestCase {
out.writeByte((byte) b);
out.copyBytes(in, in.length() - in.getFilePointer());
}
try (IndexInput in = corruptDir.openInput(file, IOContext.DEFAULT)) {
try (IndexInput in = corruptDir.openInput(file, IOContext.READONCE)) {
CodecUtil.checksumEntireFile(in);
if (VERBOSE) {
System.out.println("TEST: Altering the file did not update the checksum, aborting...");

3
lucene/core/src/test/org/apache/lucene/index/TestSnapshotDeletionPolicy.java
View File

@ -26,6 +26,7 @@ import org.apache.lucene.document.Document;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.TextField;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.tests.analysis.MockAnalyzer;
import org.apache.lucene.tests.util.LuceneTestCase;
@ -230,7 +231,7 @@ public class TestSnapshotDeletionPolicy extends LuceneTestCase {
@SuppressForbidden(reason = "Thread sleep")
private void readFile(Directory dir, String name) throws Exception {
IndexInput input = dir.openInput(name, newIOContext(random()));
IndexInput input = dir.openInput(name, IOContext.READONCE);
try {
long size = dir.fileLength(name);
long bytesLeft = size;

2
lucene/luke/src/java/org/apache/lucene/luke/models/util/IndexUtils.java
View File

@ -341,7 +341,7 @@ public final class IndexUtils {
@Override
protected String doBody(String segmentFileName) throws IOException {
String format = "unknown";
try (IndexInput in = dir.openInput(segmentFileName, IOContext.DEFAULT)) {
try (IndexInput in = dir.openInput(segmentFileName, IOContext.READONCE)) {
if (CodecUtil.CODEC_MAGIC == CodecUtil.readBEInt(in)) {
int actualVersion =
CodecUtil.checkHeaderNoMagic(

8
lucene/test-framework/src/java/org/apache/lucene/tests/store/MockDirectoryWrapper.java
View File

@ -814,6 +814,14 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
context = LuceneTestCase.newIOContext(randomState, context);
final boolean confined = context == IOContext.READONCE;
if (name.startsWith(IndexFileNames.SEGMENTS) && confined == false) {
throw new RuntimeException(
"MockDirectoryWrapper: opening segments file ["
+ name
+ "] with a non-READONCE context["
+ context
+ "]");
}
IndexInput delegateInput = in.openInput(name, context);
final IndexInput ii;

2
lucene/test-framework/src/java/org/apache/lucene/tests/util/LuceneTestCase.java
View File

@ -2989,7 +2989,7 @@ public abstract class LuceneTestCase extends Assert {
*/
public static boolean slowFileExists(Directory dir, String fileName) throws IOException {
try {
dir.openInput(fileName, IOContext.DEFAULT).close();
dir.openInput(fileName, IOContext.READONCE).close();
return true;
} catch (@SuppressWarnings("unused") NoSuchFileException | FileNotFoundException e) {
return false;