Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-8048: Filesystems do not guarantee order of directories updates

This commit is contained in:
Erick Erickson 2017-12-05 09:05:22 -08:00
parent ccedbdd9d6
commit 4a5900728d
3 changed files with 23 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
lucene/CHANGES.txt
View File

@ -144,6 +144,9 @@ Optimizations
* LUCENE-8058: Large instances of TermInSetQuery are no longer eligible for * LUCENE-8058: Large instances of TermInSetQuery are no longer eligible for
caching as they could break memory accounting of the query cache. caching as they could break memory accounting of the query cache.
(Adrien Grand) (Adrien Grand)
* LUCENE-8048: Filesystems do not guarantee order of directories updates
(Nikolay Martynov, Simon Willnauer, Erick Erickson)
Tests Tests

1
lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java
View File

@ -747,6 +747,7 @@ public final class SegmentInfos implements Cloneable, Iterable<SegmentCommitInfo
if (pendingCommit) { if (pendingCommit) {
throw new IllegalStateException("prepareCommit was already called"); throw new IllegalStateException("prepareCommit was already called");
} }
dir.syncMetaData();
write(dir); write(dir);
} }

28
lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
View File

@ -885,14 +885,16 @@ public class TestIndexWriterExceptions extends LuceneTestCase {
private static class FailOnlyInCommit extends MockDirectoryWrapper.Failure { private static class FailOnlyInCommit extends MockDirectoryWrapper.Failure {
boolean failOnCommit, failOnDeleteFile; boolean failOnCommit, failOnDeleteFile, failOnSyncMetadata;
private final boolean dontFailDuringGlobalFieldMap; private final boolean dontFailDuringGlobalFieldMap;
private final boolean dontFailDuringSyncMetadata;
private static final String PREPARE_STAGE = "prepareCommit"; private static final String PREPARE_STAGE = "prepareCommit";
private static final String FINISH_STAGE = "finishCommit"; private static final String FINISH_STAGE = "finishCommit";
private final String stage; private final String stage;
public FailOnlyInCommit(boolean dontFailDuringGlobalFieldMap, String stage) { public FailOnlyInCommit(boolean dontFailDuringGlobalFieldMap, boolean dontFailDuringSyncMetadata, String stage) {
this.dontFailDuringGlobalFieldMap = dontFailDuringGlobalFieldMap; this.dontFailDuringGlobalFieldMap = dontFailDuringGlobalFieldMap;
this.dontFailDuringSyncMetadata = dontFailDuringSyncMetadata;
this.stage = stage; this.stage = stage;
} }
@ -901,9 +903,10 @@ public class TestIndexWriterExceptions extends LuceneTestCase {
StackTraceElement[] trace = new Exception().getStackTrace(); StackTraceElement[] trace = new Exception().getStackTrace();
boolean isCommit = false; boolean isCommit = false;
boolean isDelete = false; boolean isDelete = false;
boolean isSyncMetadata = false;
boolean isInGlobalFieldMap = false; boolean isInGlobalFieldMap = false;
for (int i = 0; i < trace.length; i++) { for (int i = 0; i < trace.length; i++) {
if (isCommit && isDelete && isInGlobalFieldMap) { if (isCommit && isDelete && isInGlobalFieldMap && isSyncMetadata) {
break; break;
} }
if (SegmentInfos.class.getName().equals(trace[i].getClassName()) && stage.equals(trace[i].getMethodName())) { if (SegmentInfos.class.getName().equals(trace[i].getClassName()) && stage.equals(trace[i].getMethodName())) {
@ -915,14 +918,20 @@ public class TestIndexWriterExceptions extends LuceneTestCase {
if (SegmentInfos.class.getName().equals(trace[i].getClassName()) && "writeGlobalFieldMap".equals(trace[i].getMethodName())) { if (SegmentInfos.class.getName().equals(trace[i].getClassName()) && "writeGlobalFieldMap".equals(trace[i].getMethodName())) {
isInGlobalFieldMap = true; isInGlobalFieldMap = true;
} }
if (MockDirectoryWrapper.class.getName().equals(trace[i].getClassName()) && "syncMetaData".equals(trace[i].getMethodName())) {
isSyncMetadata = true;
}
} }
if (isInGlobalFieldMap && dontFailDuringGlobalFieldMap) { if (isInGlobalFieldMap && dontFailDuringGlobalFieldMap) {
isCommit = false; isCommit = false;
} }
if (isSyncMetadata && dontFailDuringSyncMetadata) {
isCommit = false;
}
if (isCommit) { if (isCommit) {
if (!isDelete) { if (!isDelete) {
failOnCommit = true; failOnCommit = true;
failOnSyncMetadata = isSyncMetadata;
throw new RuntimeException("now fail first"); throw new RuntimeException("now fail first");
} else { } else {
failOnDeleteFile = true; failOnDeleteFile = true;
@ -935,9 +944,10 @@ public class TestIndexWriterExceptions extends LuceneTestCase {
public void testExceptionsDuringCommit() throws Throwable { public void testExceptionsDuringCommit() throws Throwable {
FailOnlyInCommit[] failures = new FailOnlyInCommit[] { FailOnlyInCommit[] failures = new FailOnlyInCommit[] {
// LUCENE-1214 // LUCENE-1214
new FailOnlyInCommit(false, FailOnlyInCommit.PREPARE_STAGE), // fail during global field map is written new FailOnlyInCommit(false, true, FailOnlyInCommit.PREPARE_STAGE), // fail during global field map is written
new FailOnlyInCommit(true, FailOnlyInCommit.PREPARE_STAGE), // fail after global field map is written new FailOnlyInCommit(true, false, FailOnlyInCommit.PREPARE_STAGE), // fail during sync metadata
new FailOnlyInCommit(false, FailOnlyInCommit.FINISH_STAGE) // fail while running finishCommit new FailOnlyInCommit(true, true, FailOnlyInCommit.PREPARE_STAGE), // fail after global field map is written
new FailOnlyInCommit(false, true, FailOnlyInCommit.FINISH_STAGE) // fail while running finishCommit
}; };
for (FailOnlyInCommit failure : failures) { for (FailOnlyInCommit failure : failures) {
@ -952,8 +962,8 @@ public class TestIndexWriterExceptions extends LuceneTestCase {
expectThrows(RuntimeException.class, () -> { expectThrows(RuntimeException.class, () -> {
w.close(); w.close();
}); });
assertTrue("failOnCommit=" + failure.failOnCommit + " failOnDeleteFile=" + failure.failOnDeleteFile
assertTrue("failOnCommit=" + failure.failOnCommit + " failOnDeleteFile=" + failure.failOnDeleteFile, failure.failOnCommit && failure.failOnDeleteFile); + " failOnSyncMetadata=" + failure.failOnSyncMetadata + "", failure.failOnCommit && (failure.failOnDeleteFile || failure.failOnSyncMetadata));
w.rollback(); w.rollback();
String files[] = dir.listAll(); String files[] = dir.listAll();
assertTrue(files.length == fileCount || (files.length == fileCount+1 && Arrays.asList(files).contains(IndexWriter.WRITE_LOCK_NAME))); assertTrue(files.length == fileCount || (files.length == fileCount+1 && Arrays.asList(files).contains(IndexWriter.WRITE_LOCK_NAME)));