mirror of https://github.com/apache/lucene.git
LUCENE-9409: Reenable TestAllFilesDetectTruncation. (#896)
- Removed dependency on LineFileDocs to improve reproducibility. - Relaxed the expected exception type: any exception is ok. - Ignore rare cases when a file still appears to have a well-formed footer after truncation.
This commit is contained in:
parent
34446c40c4
commit
62189b2e85
|
@ -67,6 +67,7 @@ public class Lucene90PointsReader extends PointsReader {
|
||||||
Lucene90PointsFormat.VERSION_CURRENT,
|
Lucene90PointsFormat.VERSION_CURRENT,
|
||||||
readState.segmentInfo.getId(),
|
readState.segmentInfo.getId(),
|
||||||
readState.segmentSuffix);
|
readState.segmentSuffix);
|
||||||
|
CodecUtil.retrieveChecksum(indexIn);
|
||||||
|
|
||||||
dataIn = readState.directory.openInput(dataFileName, readState.context);
|
dataIn = readState.directory.openInput(dataFileName, readState.context);
|
||||||
CodecUtil.checkIndexHeader(
|
CodecUtil.checkIndexHeader(
|
||||||
|
@ -76,6 +77,7 @@ public class Lucene90PointsReader extends PointsReader {
|
||||||
Lucene90PointsFormat.VERSION_CURRENT,
|
Lucene90PointsFormat.VERSION_CURRENT,
|
||||||
readState.segmentInfo.getId(),
|
readState.segmentInfo.getId(),
|
||||||
readState.segmentSuffix);
|
readState.segmentSuffix);
|
||||||
|
CodecUtil.retrieveChecksum(dataIn);
|
||||||
|
|
||||||
long indexLength = -1, dataLength = -1;
|
long indexLength = -1, dataLength = -1;
|
||||||
try (ChecksumIndexInput metaIn =
|
try (ChecksumIndexInput metaIn =
|
||||||
|
|
|
@ -16,10 +16,21 @@
|
||||||
*/
|
*/
|
||||||
package org.apache.lucene.index;
|
package org.apache.lucene.index;
|
||||||
|
|
||||||
import java.io.EOFException;
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
|
import org.apache.lucene.codecs.CodecUtil;
|
||||||
|
import org.apache.lucene.document.Document;
|
||||||
|
import org.apache.lucene.document.Field;
|
||||||
|
import org.apache.lucene.document.Field.Store;
|
||||||
|
import org.apache.lucene.document.FieldType;
|
||||||
|
import org.apache.lucene.document.KnnVectorField;
|
||||||
|
import org.apache.lucene.document.LongPoint;
|
||||||
|
import org.apache.lucene.document.NumericDocValuesField;
|
||||||
|
import org.apache.lucene.document.SortedDocValuesField;
|
||||||
|
import org.apache.lucene.document.StringField;
|
||||||
|
import org.apache.lucene.document.TextField;
|
||||||
|
import org.apache.lucene.store.ChecksumIndexInput;
|
||||||
import org.apache.lucene.store.Directory;
|
import org.apache.lucene.store.Directory;
|
||||||
import org.apache.lucene.store.IOContext;
|
import org.apache.lucene.store.IOContext;
|
||||||
import org.apache.lucene.store.IndexInput;
|
import org.apache.lucene.store.IndexInput;
|
||||||
|
@ -27,17 +38,24 @@ import org.apache.lucene.store.IndexOutput;
|
||||||
import org.apache.lucene.tests.analysis.MockAnalyzer;
|
import org.apache.lucene.tests.analysis.MockAnalyzer;
|
||||||
import org.apache.lucene.tests.index.RandomIndexWriter;
|
import org.apache.lucene.tests.index.RandomIndexWriter;
|
||||||
import org.apache.lucene.tests.store.BaseDirectoryWrapper;
|
import org.apache.lucene.tests.store.BaseDirectoryWrapper;
|
||||||
import org.apache.lucene.tests.util.LineFileDocs;
|
|
||||||
import org.apache.lucene.tests.util.LuceneTestCase;
|
import org.apache.lucene.tests.util.LuceneTestCase;
|
||||||
import org.apache.lucene.tests.util.LuceneTestCase.AwaitsFix;
|
|
||||||
import org.apache.lucene.tests.util.LuceneTestCase.SuppressFileSystems;
|
import org.apache.lucene.tests.util.LuceneTestCase.SuppressFileSystems;
|
||||||
import org.apache.lucene.tests.util.TestUtil;
|
import org.apache.lucene.tests.util.TestUtil;
|
||||||
|
import org.apache.lucene.util.BytesRef;
|
||||||
|
|
||||||
/** Test that a plain default detects index file truncation early (on opening a reader). */
|
/** Test that a plain default detects index file truncation early (on opening a reader). */
|
||||||
@SuppressFileSystems("ExtrasFS")
|
@SuppressFileSystems("ExtrasFS")
|
||||||
@AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/LUCENE-9409")
|
|
||||||
public class TestAllFilesDetectTruncation extends LuceneTestCase {
|
public class TestAllFilesDetectTruncation extends LuceneTestCase {
|
||||||
|
|
||||||
public void test() throws Exception {
|
public void test() throws Exception {
|
||||||
|
doTest(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void testCFS() throws Exception {
|
||||||
|
doTest(true);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void doTest(boolean cfs) throws Exception {
|
||||||
Directory dir = newDirectory();
|
Directory dir = newDirectory();
|
||||||
|
|
||||||
IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
|
IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
|
||||||
|
@ -45,31 +63,46 @@ public class TestAllFilesDetectTruncation extends LuceneTestCase {
|
||||||
|
|
||||||
// Disable CFS 80% of the time so we can truncate individual files, but the other 20% of the
|
// Disable CFS 80% of the time so we can truncate individual files, but the other 20% of the
|
||||||
// time we test truncation of .cfs/.cfe too:
|
// time we test truncation of .cfs/.cfe too:
|
||||||
if (random().nextInt(5) != 1) {
|
if (cfs == false) {
|
||||||
conf.setUseCompoundFile(false);
|
conf.setUseCompoundFile(false);
|
||||||
conf.getMergePolicy().setNoCFSRatio(0.0);
|
conf.getMergePolicy().setNoCFSRatio(0.0);
|
||||||
}
|
}
|
||||||
|
|
||||||
RandomIndexWriter riw = new RandomIndexWriter(random(), dir, conf);
|
RandomIndexWriter riw = new RandomIndexWriter(random(), dir, conf);
|
||||||
// Use LineFileDocs so we (hopefully) get most Lucene features
|
Document doc = new Document();
|
||||||
// tested, e.g. IntPoint was recently added to it:
|
FieldType textWithTermVectorsType = new FieldType(TextField.TYPE_STORED);
|
||||||
LineFileDocs docs = new LineFileDocs(random());
|
textWithTermVectorsType.setStoreTermVectors(true);
|
||||||
|
Field text = new Field("text", "", textWithTermVectorsType);
|
||||||
|
doc.add(text);
|
||||||
|
Field termString = new StringField("string", "", Store.YES);
|
||||||
|
doc.add(termString);
|
||||||
|
Field dvString = new SortedDocValuesField("string", new BytesRef());
|
||||||
|
doc.add(dvString);
|
||||||
|
Field pointNumber = new LongPoint("long", 0L);
|
||||||
|
doc.add(pointNumber);
|
||||||
|
Field dvNumber = new NumericDocValuesField("long", 0L);
|
||||||
|
doc.add(dvNumber);
|
||||||
|
KnnVectorField vector = new KnnVectorField("vector", new float[16]);
|
||||||
|
doc.add(vector);
|
||||||
|
|
||||||
for (int i = 0; i < 100; i++) {
|
for (int i = 0; i < 100; i++) {
|
||||||
riw.addDocument(docs.nextDoc());
|
text.setStringValue(TestUtil.randomAnalysisString(random(), 20, true));
|
||||||
if (random().nextInt(7) == 0) {
|
String randomString = TestUtil.randomSimpleString(random(), 5);
|
||||||
riw.commit();
|
termString.setStringValue(randomString);
|
||||||
}
|
dvString.setBytesValue(new BytesRef(randomString));
|
||||||
if (random().nextInt(20) == 0) {
|
long number = random().nextInt(10);
|
||||||
riw.deleteDocuments(new Term("docid", Integer.toString(i)));
|
pointNumber.setLongValue(number);
|
||||||
}
|
dvNumber.setLongValue(number);
|
||||||
if (random().nextInt(15) == 0) {
|
Arrays.fill(vector.vectorValue(), i % 4);
|
||||||
riw.updateNumericDocValue(
|
riw.addDocument(doc);
|
||||||
new Term("docid", Integer.toString(i)), "docid_intDV", Long.valueOf(i));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (TEST_NIGHTLY == false) {
|
if (TEST_NIGHTLY == false) {
|
||||||
riw.forceMerge(1);
|
riw.forceMerge(1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
riw.deleteDocuments(LongPoint.newRangeQuery("long", 0, 2));
|
||||||
|
|
||||||
riw.close();
|
riw.close();
|
||||||
checkTruncation(dir);
|
checkTruncation(dir);
|
||||||
dir.close();
|
dir.close();
|
||||||
|
@ -105,6 +138,19 @@ public class TestAllFilesDetectTruncation extends LuceneTestCase {
|
||||||
if (name.equals(victim) == false) {
|
if (name.equals(victim) == false) {
|
||||||
dirCopy.copyFrom(dir, name, name, IOContext.DEFAULT);
|
dirCopy.copyFrom(dir, name, name, IOContext.DEFAULT);
|
||||||
} else {
|
} else {
|
||||||
|
try (ChecksumIndexInput in = dir.openChecksumInput(name, IOContext.DEFAULT)) {
|
||||||
|
try {
|
||||||
|
CodecUtil.checkFooter(in);
|
||||||
|
// In some rare cases, the codec footer would still appear as correct even though the
|
||||||
|
// file has been truncated. We just skip the test is this rare case.
|
||||||
|
return;
|
||||||
|
} catch (
|
||||||
|
@SuppressWarnings("unused")
|
||||||
|
CorruptIndexException e) {
|
||||||
|
// expected
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
try (IndexOutput out = dirCopy.createOutput(name, IOContext.DEFAULT);
|
try (IndexOutput out = dirCopy.createOutput(name, IOContext.DEFAULT);
|
||||||
IndexInput in = dir.openInput(name, IOContext.DEFAULT)) {
|
IndexInput in = dir.openInput(name, IOContext.DEFAULT)) {
|
||||||
out.copyBytes(in, victimLength - lostBytes);
|
out.copyBytes(in, victimLength - lostBytes);
|
||||||
|
@ -113,16 +159,14 @@ public class TestAllFilesDetectTruncation extends LuceneTestCase {
|
||||||
dirCopy.sync(Collections.singleton(name));
|
dirCopy.sync(Collections.singleton(name));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// There needs to be an exception thrown, but we don't care about its type, it's too heroic to
|
||||||
|
// ensure that a specific exception type gets throws upon opening an index.
|
||||||
// NOTE: we .close so that if the test fails (truncation not detected) we don't also get all
|
// NOTE: we .close so that if the test fails (truncation not detected) we don't also get all
|
||||||
// these confusing errors about open files:
|
// these confusing errors about open files:
|
||||||
expectThrowsAnyOf(
|
expectThrows(Exception.class, () -> DirectoryReader.open(dirCopy).close());
|
||||||
Arrays.asList(CorruptIndexException.class, EOFException.class),
|
|
||||||
() -> DirectoryReader.open(dirCopy).close());
|
|
||||||
|
|
||||||
// CheckIndex should also fail:
|
// CheckIndex should also fail:
|
||||||
expectThrowsAnyOf(
|
expectThrows(Exception.class, () -> TestUtil.checkIndex(dirCopy, true, true, true, null));
|
||||||
Arrays.asList(CorruptIndexException.class, EOFException.class),
|
|
||||||
() -> TestUtil.checkIndex(dirCopy, true, true, true, null));
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue