LUCENE-4870: Lucene deletes entire index if and exception is thrown due do TooManyOpenFiles and OpenMode.CREATE_OR_APPEND

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1459903 13f79535-47bb-0310-9956-ffa450edef68

lucene/CHANGES.txt

@@ -191,6 +191,11 @@ Bug Fixes
 * SOLR-4589: Fixed CPU spikes and poor performance in lazy field loading 
   of multivalued fields. (hossman)
 
+* LUCENE-4870: Fix bug where an entire index might be deleted by the IndexWriter
+  due to false detection if an index exists in the directory when
+  OpenMode.CREATE_OR_APPEND is used. This might also affect application that set
+  the open mode manually using DirectoryReader#indexExists. (Simon Willnauer)
+
 Optimizations
 
 * LUCENE-4819: Added Sorted[Set]DocValues.termsEnum(), and optimized the

lucene/core/src/java/org/apache/lucene/index/DirectoryReader.java

@@ -23,6 +23,7 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 
+import org.apache.lucene.index.SegmentInfos.FindSegmentsFile;
 import org.apache.lucene.search.SearcherManager; // javadocs
 import org.apache.lucene.store.Directory;
 
@@ -319,7 +320,24 @@ public abstract class DirectoryReader extends BaseCompositeReader<AtomicReader>
    */
   public static boolean indexExists(Directory directory) {
     try {
-      new SegmentInfos().read(directory);
+      new FindSegmentsFile(directory) {
+        @Override
+        protected Object doBody(String segmentFileName) throws IOException {
+          try {
+            new SegmentInfos().read(directory, segmentFileName);
+          } catch (FileNotFoundException ex) {
+            if (!directory.fileExists(segmentFileName)) {
+              throw ex;
+            }
+            /* this is ok - we might have run into a access exception here.
+             * or even worse like on LUCENE-4870 this is triggered due to
+             * too many open files on the system. In that case we rather report
+             * a false positive here since wrongly returning false from indexExist
+             * can cause data loss since IW relies on this.*/
+          }
+          return null;
+        }
+      }.run();
       return true;
     } catch (IOException ioe) {
       return false;

lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java

@@ -17,6 +17,7 @@ package org.apache.lucene.index;
  * limitations under the License.
  */
 
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.Reader;
 import java.io.StringReader;
@@ -1627,4 +1628,75 @@ public class TestIndexWriterExceptions extends LuceneTestCase {
     iw.close();
     dir.close();
   }
+  
+  // See LUCENE-4870 TooManyOpenFiles errors are thrown as
+  // FNFExceptions which can trigger data loss.
+  public void testTooManyFileException() throws Exception {
+
+    // Create failure that throws Too many open files exception randomly
+    MockDirectoryWrapper.Failure failure = new MockDirectoryWrapper.Failure() {
+
+      @Override
+      public MockDirectoryWrapper.Failure reset() {
+        doFail = false;
+        return this;
+      }
+
+      @Override
+      public void eval(MockDirectoryWrapper dir) throws IOException {
+        if (doFail) {
+          if (random().nextBoolean()) {
+            throw new FileNotFoundException("some/file/name.ext (Too many open files)");
+          }
+        }
+      }
+    };
+
+    MockDirectoryWrapper dir = newMockDirectory();
+    // The exception is only thrown on open input
+    dir.setFailOnOpenInput(true);
+    dir.failOn(failure);
+
+    // Create an index with one document
+    IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+    IndexWriter iw = new IndexWriter(dir, iwc);
+    Document doc = new Document();
+    doc.add(new StringField("foo", "bar", Field.Store.NO));
+    iw.addDocument(doc); // add a document
+    iw.commit();
+    DirectoryReader ir = DirectoryReader.open(dir);
+    assertEquals(1, ir.numDocs());
+    ir.close();
+    iw.close();
+
+    // Open and close the index a few times
+    for (int i = 0; i < 10; i++) {
+      failure.setDoFail();
+      iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+      try {
+        iw = new IndexWriter(dir, iwc);
+      } catch (CorruptIndexException ex) {
+        // Exceptions are fine - we are running out of file handlers here
+        continue;
+      } catch (FileNotFoundException ex) {
+        continue;
+      }
+      failure.clearDoFail();
+      iw.close();
+      ir = DirectoryReader.open(dir);
+      assertEquals("lost document after iteration: " + i, 1, ir.numDocs());
+      ir.close();
+    }
+
+    // Check if document is still there
+    failure.clearDoFail();
+    ir = DirectoryReader.open(dir);
+    assertEquals(1, ir.numDocs());
+    ir.close();
+
+    dir.close();
+  }
+  
+  
+
 }