Fix issue where entire shards are lost due to too many open files exceptions and a but in Lucene's

IndexWriter / DirectoryReader where an existsing index was not detected and then deleted due to a wrong
creation mode. See LUCENE-4870

Closes #2812

src/main/java/org/elasticsearch/common/lucene/Lucene.java

@@ -22,6 +22,7 @@ package org.elasticsearch.common.lucene;
 import org.apache.lucene.analysis.core.KeywordAnalyzer;
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.index.*;
+import org.apache.lucene.index.SegmentInfos.FindSegmentsFile;
 import org.apache.lucene.search.*;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.BytesRef;
@@ -34,6 +35,7 @@ import org.elasticsearch.index.analysis.AnalyzerScope;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.lang.reflect.Field;
 
@@ -375,4 +377,28 @@ public class Lucene {
     private Lucene() {
 
     }
+    
+    // LUCENE UPGRADE this is a workaround for LUCENE-4870
+    public static final boolean indexExists(final Directory directory) {
+        try {
+            new FindSegmentsFile(directory) {
+                @Override
+                protected Object doBody(String segmentFileName) throws IOException {
+                    try {
+                        new SegmentInfos().read(directory, segmentFileName);
+                    } catch (FileNotFoundException ex) {
+                        if (!directory.fileExists(segmentFileName)) {
+                            throw ex;
+                        }
+                        // this is ok - we might have run into a access
+                        // exception here or even worse a too many open files exception.
+                    }
+                    return null;
+                }
+            }.run();
+            return true;
+        } catch (IOException ioe) {
+            return false;
+        }
+    }
 }

src/main/java/org/elasticsearch/index/engine/robin/RobinEngine.java

@@ -21,11 +21,13 @@ package org.elasticsearch.index.engine.robin;
 
 import com.google.common.collect.Lists;
 import org.apache.lucene.index.*;
+import org.apache.lucene.index.SegmentInfos.FindSegmentsFile;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.SearcherFactory;
 import org.apache.lucene.search.SearcherManager;
 import org.apache.lucene.store.AlreadyClosedException;
+import org.apache.lucene.store.Directory;
 import org.elasticsearch.ElasticSearchException;
 import org.elasticsearch.ElasticSearchIllegalStateException;
 import org.elasticsearch.cluster.routing.operation.hash.djb.DjbHashFunction;
@@ -63,6 +65,7 @@ import org.elasticsearch.indices.warmer.IndicesWarmer;
 import org.elasticsearch.indices.warmer.InternalIndicesWarmer;
 import org.elasticsearch.threadpool.ThreadPool;
 
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.util.*;
 import java.util.concurrent.ConcurrentMap;
@@ -253,7 +256,7 @@ public class RobinEngine extends AbstractIndexShardComponent implements Engine {
             try {
                 // commit on a just opened writer will commit even if there are no changes done to it
                 // we rely on that for the commit data translog id key
-                if (DirectoryReader.indexExists(store.directory())) {
+                if (Lucene.indexExists(store.directory())) {
                     Map<String, String> commitUserData = Lucene.readSegmentInfos(store.directory()).getUserData();
                     if (commitUserData.containsKey(Translog.TRANSLOG_ID_KEY)) {
                         translogIdGenerator.set(Long.parseLong(commitUserData.get(Translog.TRANSLOG_ID_KEY)));
@@ -1318,7 +1321,7 @@ public class RobinEngine extends AbstractIndexShardComponent implements Engine {
                 logger.warn("shard is locked, releasing lock");
                 IndexWriter.unlock(store.directory());
             }
-            boolean create = !DirectoryReader.indexExists(store.directory());
+            boolean create = !Lucene.indexExists(store.directory());
             IndexWriterConfig config = new IndexWriterConfig(Lucene.VERSION, analysisService.defaultIndexAnalyzer());
             config.setOpenMode(create ? IndexWriterConfig.OpenMode.CREATE : IndexWriterConfig.OpenMode.APPEND);
             config.setIndexDeletionPolicy(deletionPolicy);
@@ -1338,7 +1341,7 @@ public class RobinEngine extends AbstractIndexShardComponent implements Engine {
         }
         return indexWriter;
     }
-
+    
     public static final String INDEX_TERM_INDEX_INTERVAL = "index.term_index_interval";
     public static final String INDEX_TERM_INDEX_DIVISOR = "index.term_index_divisor";
     public static final String INDEX_INDEX_CONCURRENCY = "index.index_concurrency";

src/main/java/org/elasticsearch/index/gateway/blobstore/BlobStoreIndexShardGateway.java

@@ -607,7 +607,7 @@ public abstract class BlobStoreIndexShardGateway extends AbstractIndexShardCompo
         // read the gateway data persisted
         long version = -1;
         try {
-            if (DirectoryReader.indexExists(store.directory())) {
+            if (Lucene.indexExists(store.directory())) {
                 version = Lucene.readSegmentInfos(store.directory()).getVersion();
             }
         } catch (IOException e) {

src/main/java/org/elasticsearch/index/shard/service/InternalIndexShard.java

@@ -35,6 +35,7 @@ import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.FastByteArrayOutputStream;
+import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.lucene.search.XFilteredQuery;
 import org.elasticsearch.common.metrics.MeanMetric;
 import org.elasticsearch.common.settings.Settings;
@@ -811,7 +812,7 @@ public class InternalIndexShard extends AbstractIndexShardComponent implements I
         try {
             checkIndexTook = 0;
             long time = System.currentTimeMillis();
-            if (!DirectoryReader.indexExists(store.directory())) {
+            if (!Lucene.indexExists(store.directory())) {
                 return;
             }
             CheckIndex checkIndex = new CheckIndex(store.directory());