improve handling of memory caching with file system, only force compound file when really needed (when an extension that exists within the compound file is part of the memory cached extensions)

modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/FsStore.java

@@ -22,6 +22,7 @@ package org.elasticsearch.index.store.fs;
 import org.apache.lucene.store.*;
 import org.elasticsearch.cache.memory.ByteBufferCache;
 import org.elasticsearch.common.collect.ImmutableSet;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.io.FileSystemUtils;
 import org.elasticsearch.common.lucene.store.SwitchDirectory;
 import org.elasticsearch.common.settings.Settings;
@@ -73,7 +74,7 @@ public abstract class FsStore extends AbstractStore {
         return lockFactory;
     }
 
-    protected SwitchDirectory buildSwitchDirectoryIfNeeded(Directory fsDirectory, ByteBufferCache byteBufferCache) {
+    protected Tuple<SwitchDirectory, Boolean> buildSwitchDirectoryIfNeeded(Directory fsDirectory, ByteBufferCache byteBufferCache) {
         boolean cache = componentSettings.getAsBoolean("memory.enabled", false);
         if (!cache) {
             return null;
@@ -81,6 +82,17 @@ public abstract class FsStore extends AbstractStore {
         Directory memDir = new ByteBufferDirectory(byteBufferCache);
         // see http://lucene.apache.org/java/3_0_1/fileformats.html
         String[] primaryExtensions = componentSettings.getAsArray("memory.extensions", new String[]{"", "del", "gen"});
-        return new SwitchDirectory(ImmutableSet.copyOf(primaryExtensions), memDir, fsDirectory, true);
+        if (primaryExtensions == null || primaryExtensions.length == 0) {
+            return null;
+        }
+        Boolean forceUseCompound = null;
+        for (String extension : primaryExtensions) {
+            if (!("".equals(extension) || "del".equals(extension) || "gen".equals(extension))) {
+                // caching internal CFS extension, don't use compound file extension
+                forceUseCompound = false;
+            }
+        }
+
+        return new Tuple<SwitchDirectory, Boolean>(new SwitchDirectory(ImmutableSet.copyOf(primaryExtensions), memDir, fsDirectory, true), forceUseCompound);
     }
 }

modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/MmapFsStore.java

@@ -24,6 +24,7 @@ import org.apache.lucene.store.FSDirectory;
 import org.apache.lucene.store.LockFactory;
 import org.apache.lucene.store.MMapDirectory;
 import org.elasticsearch.cache.memory.ByteBufferCache;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.lucene.store.SwitchDirectory;
 import org.elasticsearch.common.settings.Settings;
@@ -52,16 +53,21 @@ public class MmapFsStore extends FsStore {
         location.mkdirs();
         this.fsDirectory = new MMapDirectory(location, lockFactory);
 
-        SwitchDirectory switchDirectory = buildSwitchDirectoryIfNeeded(fsDirectory, byteBufferCache);
+        boolean suggestUseCompoundFile;
+        Tuple<SwitchDirectory, Boolean> switchDirectory = buildSwitchDirectoryIfNeeded(fsDirectory, byteBufferCache);
         if (switchDirectory != null) {
-            suggestUseCompoundFile = false;
+            suggestUseCompoundFile = true;
-            logger.debug("Using [mmap_fs] Store with path [{}], cache [true] with extensions [{}]", fsDirectory.getFile(), switchDirectory.primaryExtensions());
+            if (switchDirectory.v2() != null) {
-            directory = wrapDirectory(switchDirectory);
+                suggestUseCompoundFile = switchDirectory.v2();
+            }
+            logger.debug("Using [mmap_fs] Store with path [{}], cache [true] with extensions [{}]", fsDirectory.getFile(), switchDirectory.v1().primaryExtensions());
+            directory = wrapDirectory(switchDirectory.v1());
         } else {
             suggestUseCompoundFile = true;
             directory = wrapDirectory(fsDirectory);
             logger.debug("Using [mmap_fs] Store with path [{}]", fsDirectory.getFile());
         }
+        this.suggestUseCompoundFile = suggestUseCompoundFile;
     }
 
     @Override public FSDirectory fsDirectory() {

modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/NioFsStore.java

@@ -24,6 +24,7 @@ import org.apache.lucene.store.FSDirectory;
 import org.apache.lucene.store.LockFactory;
 import org.apache.lucene.store.NIOFSDirectory;
 import org.elasticsearch.cache.memory.ByteBufferCache;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.lucene.store.SwitchDirectory;
 import org.elasticsearch.common.settings.Settings;
@@ -52,16 +53,21 @@ public class NioFsStore extends FsStore {
         location.mkdirs();
         this.fsDirectory = new NIOFSDirectory(location, lockFactory);
 
-        SwitchDirectory switchDirectory = buildSwitchDirectoryIfNeeded(fsDirectory, byteBufferCache);
+        boolean suggestUseCompoundFile;
+        Tuple<SwitchDirectory, Boolean> switchDirectory = buildSwitchDirectoryIfNeeded(fsDirectory, byteBufferCache);
         if (switchDirectory != null) {
-            suggestUseCompoundFile = false;
+            suggestUseCompoundFile = true;
-            logger.debug("Using [nio_fs] Store with path [{}], cache [true] with extensions [{}]", fsDirectory.getFile(), switchDirectory.primaryExtensions());
+            if (switchDirectory.v2() != null) {
-            directory = wrapDirectory(switchDirectory);
+                suggestUseCompoundFile = switchDirectory.v2();
+            }
+            logger.debug("Using [nio_fs] Store with path [{}], cache [true] with extensions [{}]", fsDirectory.getFile(), switchDirectory.v1().primaryExtensions());
+            directory = wrapDirectory(switchDirectory.v1());
         } else {
             suggestUseCompoundFile = true;
             directory = wrapDirectory(fsDirectory);
             logger.debug("Using [nio_fs] Store with path [{}]", fsDirectory.getFile());
         }
+        this.suggestUseCompoundFile = suggestUseCompoundFile;
     }
 
     @Override public FSDirectory fsDirectory() {

modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/SimpleFsStore.java

@@ -24,6 +24,7 @@ import org.apache.lucene.store.FSDirectory;
 import org.apache.lucene.store.LockFactory;
 import org.apache.lucene.store.SimpleFSDirectory;
 import org.elasticsearch.cache.memory.ByteBufferCache;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.lucene.store.SwitchDirectory;
 import org.elasticsearch.common.settings.Settings;
@@ -52,16 +53,21 @@ public class SimpleFsStore extends FsStore {
         location.mkdirs();
         this.fsDirectory = new SimpleFSDirectory(location, lockFactory);
 
-        SwitchDirectory switchDirectory = buildSwitchDirectoryIfNeeded(fsDirectory, byteBufferCache);
+        boolean suggestUseCompoundFile;
+        Tuple<SwitchDirectory, Boolean> switchDirectory = buildSwitchDirectoryIfNeeded(fsDirectory, byteBufferCache);
         if (switchDirectory != null) {
-            suggestUseCompoundFile = false;
+            suggestUseCompoundFile = true;
-            logger.debug("Using [simple_fs] Store with path [{}], cache [true] with extensions [{}]", fsDirectory.getFile(), switchDirectory.primaryExtensions());
+            if (switchDirectory.v2() != null) {
-            directory = wrapDirectory(switchDirectory);
+                suggestUseCompoundFile = switchDirectory.v2();
+            }
+            logger.debug("Using [simple_fs] Store with path [{}], cache [true] with extensions [{}]", fsDirectory.getFile(), switchDirectory.v1().primaryExtensions());
+            directory = wrapDirectory(switchDirectory.v1());
         } else {
             suggestUseCompoundFile = true;
             directory = wrapDirectory(fsDirectory);
             logger.debug("Using [simple_fs] Store with path [{}]", fsDirectory.getFile());
         }
+        this.suggestUseCompoundFile = suggestUseCompoundFile;
     }
 
     @Override public FSDirectory fsDirectory() {

modules/elasticsearch/src/main/java/org/elasticsearch/index/store/support/AbstractStore.java

@@ -167,7 +167,11 @@ public abstract class AbstractStore extends AbstractIndexShardComponent implemen
             synchronized (mutex) {
                 MapBuilder<String, StoreFileMetaData> builder = MapBuilder.newMapBuilder();
                 for (String file : delegate.listAll()) {
+                    try {
                         builder.put(file, new StoreFileMetaData(file, delegate.fileLength(file), delegate.fileModified(file), preComputedMd5(file)));
+                    } catch (FileNotFoundException e) {
+                        // ignore
+                    }
                 }
                 filesMetadata = builder.immutableMap();
                 files = filesMetadata.keySet().toArray(new String[filesMetadata.size()]);