mirror of https://github.com/apache/lucene.git
LUCENE-4870: Lucene deletes entire index if and exception is thrown due do TooManyOpenFiles and OpenMode.CREATE_OR_APPEND
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1459903 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
8df4265ef0
commit
d56fa5175e
|
@ -191,6 +191,11 @@ Bug Fixes
|
|||
* SOLR-4589: Fixed CPU spikes and poor performance in lazy field loading
|
||||
of multivalued fields. (hossman)
|
||||
|
||||
* LUCENE-4870: Fix bug where an entire index might be deleted by the IndexWriter
|
||||
due to false detection if an index exists in the directory when
|
||||
OpenMode.CREATE_OR_APPEND is used. This might also affect application that set
|
||||
the open mode manually using DirectoryReader#indexExists. (Simon Willnauer)
|
||||
|
||||
Optimizations
|
||||
|
||||
* LUCENE-4819: Added Sorted[Set]DocValues.termsEnum(), and optimized the
|
||||
|
|
|
@ -23,6 +23,7 @@ import java.util.ArrayList;
|
|||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.lucene.index.SegmentInfos.FindSegmentsFile;
|
||||
import org.apache.lucene.search.SearcherManager; // javadocs
|
||||
import org.apache.lucene.store.Directory;
|
||||
|
||||
|
@ -319,7 +320,24 @@ public abstract class DirectoryReader extends BaseCompositeReader<AtomicReader>
|
|||
*/
|
||||
public static boolean indexExists(Directory directory) {
|
||||
try {
|
||||
new SegmentInfos().read(directory);
|
||||
new FindSegmentsFile(directory) {
|
||||
@Override
|
||||
protected Object doBody(String segmentFileName) throws IOException {
|
||||
try {
|
||||
new SegmentInfos().read(directory, segmentFileName);
|
||||
} catch (FileNotFoundException ex) {
|
||||
if (!directory.fileExists(segmentFileName)) {
|
||||
throw ex;
|
||||
}
|
||||
/* this is ok - we might have run into a access exception here.
|
||||
* or even worse like on LUCENE-4870 this is triggered due to
|
||||
* too many open files on the system. In that case we rather report
|
||||
* a false positive here since wrongly returning false from indexExist
|
||||
* can cause data loss since IW relies on this.*/
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}.run();
|
||||
return true;
|
||||
} catch (IOException ioe) {
|
||||
return false;
|
||||
|
|
|
@ -17,6 +17,7 @@ package org.apache.lucene.index;
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.IOException;
|
||||
import java.io.Reader;
|
||||
import java.io.StringReader;
|
||||
|
@ -1627,4 +1628,75 @@ public class TestIndexWriterExceptions extends LuceneTestCase {
|
|||
iw.close();
|
||||
dir.close();
|
||||
}
|
||||
|
||||
// See LUCENE-4870 TooManyOpenFiles errors are thrown as
|
||||
// FNFExceptions which can trigger data loss.
|
||||
public void testTooManyFileException() throws Exception {
|
||||
|
||||
// Create failure that throws Too many open files exception randomly
|
||||
MockDirectoryWrapper.Failure failure = new MockDirectoryWrapper.Failure() {
|
||||
|
||||
@Override
|
||||
public MockDirectoryWrapper.Failure reset() {
|
||||
doFail = false;
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void eval(MockDirectoryWrapper dir) throws IOException {
|
||||
if (doFail) {
|
||||
if (random().nextBoolean()) {
|
||||
throw new FileNotFoundException("some/file/name.ext (Too many open files)");
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
MockDirectoryWrapper dir = newMockDirectory();
|
||||
// The exception is only thrown on open input
|
||||
dir.setFailOnOpenInput(true);
|
||||
dir.failOn(failure);
|
||||
|
||||
// Create an index with one document
|
||||
IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
|
||||
IndexWriter iw = new IndexWriter(dir, iwc);
|
||||
Document doc = new Document();
|
||||
doc.add(new StringField("foo", "bar", Field.Store.NO));
|
||||
iw.addDocument(doc); // add a document
|
||||
iw.commit();
|
||||
DirectoryReader ir = DirectoryReader.open(dir);
|
||||
assertEquals(1, ir.numDocs());
|
||||
ir.close();
|
||||
iw.close();
|
||||
|
||||
// Open and close the index a few times
|
||||
for (int i = 0; i < 10; i++) {
|
||||
failure.setDoFail();
|
||||
iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
|
||||
try {
|
||||
iw = new IndexWriter(dir, iwc);
|
||||
} catch (CorruptIndexException ex) {
|
||||
// Exceptions are fine - we are running out of file handlers here
|
||||
continue;
|
||||
} catch (FileNotFoundException ex) {
|
||||
continue;
|
||||
}
|
||||
failure.clearDoFail();
|
||||
iw.close();
|
||||
ir = DirectoryReader.open(dir);
|
||||
assertEquals("lost document after iteration: " + i, 1, ir.numDocs());
|
||||
ir.close();
|
||||
}
|
||||
|
||||
// Check if document is still there
|
||||
failure.clearDoFail();
|
||||
ir = DirectoryReader.open(dir);
|
||||
assertEquals(1, ir.numDocs());
|
||||
ir.close();
|
||||
|
||||
dir.close();
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue