Add hybridfs store type

With this commit we introduce a new store type `hybridfs` that is a
hybrid between `mmapfs` and `niofs`. This store type chooses different
strategies to read Lucene files based on the read access pattern (random
or linear) in order to optimize performance.

This store type has been available in earlier versions of Elasticsearch
as `default_fs`. We have chosen a different name now in order to convey
the intent of the store type instead of tying it to the fact whether it
is the default choice.

Relates #36668

docs/reference/index-modules/store.asciidoc

@@ -40,7 +40,7 @@ The following sections lists all the different storage types supported.
 `fs`::
 
 Default file system implementation. This will pick the best implementation
-depending on the operating environment, which is currently `mmapfs` on all
+depending on the operating environment, which is currently `hybridfs` on all
 supported systems but is subject to change.
 
 [[simplefs]]`simplefs`::
@@ -67,12 +67,22 @@ process equal to the size of the file being mapped. Before using this
 class, be sure you have allowed plenty of
 <<vm-max-map-count,virtual address space>>.
 
+[[hybridfs]]`hybridfs`::
+
+The `hybridfs` type is a hybrid of `niofs` and `mmapfs`, which chooses the best
+file system type for each type of file based on the read access pattern.
+Currently only the Lucene term dictionary, norms and doc values files are
+memory mapped. All other files are opened using Lucene `NIOFSDirectory`.
+Similarly to `mmapfs` be sure you have allowed plenty of
+<<vm-max-map-count,virtual address space>>.
+
 [[allow-mmapfs]]
-You can restrict the use of the `mmapfs` store type via the setting
+You can restrict the use of the `mmapfs` and the related `hybridfs` store type
-`node.store.allow_mmapfs`. This is a boolean setting indicating whether or not
+via the setting `node.store.allow_mmapfs`. This is a boolean setting indicating
-`mmapfs` is allowed. The default is to allow `mmapfs`. This setting is useful,
+whether or not memory-mapping is allowed. The default is to allow it. This
-for example, if you are in an environment where you can not control the ability
+setting is useful, for example, if you are in an environment where you can not
-to create a lot of memory maps so you need disable the ability to use `mmapfs`.
+control the ability to create a lot of memory maps so you need disable the
+ability to use memory-mapping.
 
 === Pre-loading data into the file system cache
 

server/src/main/java/org/elasticsearch/index/IndexModule.java

@@ -302,6 +302,7 @@ public final class IndexModule {
 
 
     public enum Type {
+        HYBRIDFS("hybridfs"),
         NIOFS("niofs"),
         MMAPFS("mmapfs"),
         SIMPLEFS("simplefs"),
@@ -330,7 +331,7 @@ public final class IndexModule {
         public static Type fromSettingsKey(final String key) {
             final Type type = TYPES.get(key);
             if (type == null) {
-                throw new IllegalArgumentException("no matching type for [" + key + "]");
+                throw new IllegalArgumentException("no matching store type for [" + key + "]");
             }
             return type;
         }
@@ -356,7 +357,7 @@ public final class IndexModule {
 
     public static Type defaultStoreType(final boolean allowMmapfs) {
         if (allowMmapfs && Constants.JRE_IS_64BIT && MMapDirectory.UNMAP_SUPPORTED) {
-            return Type.MMAPFS;
+            return Type.HYBRIDFS;
         } else if (Constants.WINDOWS) {
             return Type.SIMPLEFS;
         } else {

server/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java

@@ -20,6 +20,7 @@
 package org.elasticsearch.index.store;
 
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
 import org.apache.lucene.store.FileSwitchDirectory;
 import org.apache.lucene.store.LockFactory;
 import org.apache.lucene.store.MMapDirectory;
@@ -30,6 +31,7 @@ import org.apache.lucene.store.SimpleFSLockFactory;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
+import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.shard.ShardPath;
@@ -37,10 +39,16 @@ import org.elasticsearch.index.shard.ShardPath;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.Set;
 
 public class FsDirectoryService extends DirectoryService {
+    /*
+     * We are mmapping norms, docvalues as well as term dictionaries, all other files are served through NIOFS
+     * this provides good random access performance and does not lead to page cache thrashing.
+     */
+    private static final Set<String> PRIMARY_EXTENSIONS = Collections.unmodifiableSet(Sets.newHashSet("nvd", "dvd", "tim"));
 
     protected final IndexStore indexStore;
     public static final Setting<LockFactory> INDEX_LOCK_FACTOR_SETTING = new Setting<>("index.store.fs.fs_lock", "native", (s) -> {
@@ -78,27 +86,36 @@ public class FsDirectoryService extends DirectoryService {
     protected Directory newFSDirectory(Path location, LockFactory lockFactory) throws IOException {
         final String storeType =
                 indexSettings.getSettings().get(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), IndexModule.Type.FS.getSettingsKey());
+        IndexModule.Type type;
         if (IndexModule.Type.FS.match(storeType)) {
-            final IndexModule.Type type =
+            type = IndexModule.defaultStoreType(IndexModule.NODE_STORE_ALLOW_MMAPFS.get(indexSettings.getNodeSettings()));
-                    IndexModule.defaultStoreType(IndexModule.NODE_STORE_ALLOW_MMAPFS.get(indexSettings.getNodeSettings()));
+        } else {
-            switch (type) {
+            type = IndexModule.Type.fromSettingsKey(storeType);
-                case MMAPFS:
+        }
-                    return new MMapDirectory(location, lockFactory);
+        switch (type) {
-                case SIMPLEFS:
+            case HYBRIDFS:
-                    return new SimpleFSDirectory(location, lockFactory);
+                // Use Lucene defaults
-                case NIOFS:
+                final FSDirectory primaryDirectory = FSDirectory.open(location, lockFactory);
-                    return new NIOFSDirectory(location, lockFactory);
+                if (primaryDirectory instanceof MMapDirectory) {
-                default:
+                    return new FileSwitchDirectory(PRIMARY_EXTENSIONS, primaryDirectory, new NIOFSDirectory(location, lockFactory), true) {
-                    throw new AssertionError("unexpected built-in store type [" + type + "]");
+                        @Override
-            }
+                        public String[] listAll() throws IOException {
-        } else if (IndexModule.Type.SIMPLEFS.match(storeType)) {
+                            // Avoid doing listAll twice:
-            return new SimpleFSDirectory(location, lockFactory);
+                            return primaryDirectory.listAll();
-        } else if (IndexModule.Type.NIOFS.match(storeType)) {
+                        }
-            return new NIOFSDirectory(location, lockFactory);
+                    };
-        } else if (IndexModule.Type.MMAPFS.match(storeType)) {
+                } else {
-            return new MMapDirectory(location, lockFactory);
+                    return primaryDirectory;
+                }
+            case MMAPFS:
+                return new MMapDirectory(location, lockFactory);
+            case SIMPLEFS:
+                return new SimpleFSDirectory(location, lockFactory);
+            case NIOFS:
+                return new NIOFSDirectory(location, lockFactory);
+            default:
+                throw new AssertionError("unexpected built-in store type [" + type + "]");
         }
-        throw new IllegalArgumentException("No directory found for type [" + storeType + "]");
     }
 
     private static Directory setPreload(Directory directory, Path location, LockFactory lockFactory,

server/src/test/java/org/elasticsearch/index/store/IndexStoreTests.java

@@ -19,6 +19,7 @@
 package org.elasticsearch.index.store;
 
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FileSwitchDirectory;
 import org.apache.lucene.store.MMapDirectory;
 import org.apache.lucene.store.NIOFSDirectory;
 import org.apache.lucene.store.NoLockFactory;
@@ -64,6 +65,9 @@ public class IndexStoreTests extends ESTestCase {
             new ShardPath(false, tempDir, tempDir, new ShardId(index, 0)));
         try (Directory directory = service.newFSDirectory(tempDir, NoLockFactory.INSTANCE)) {
             switch (type) {
+                case HYBRIDFS:
+                    assertHybridDirectory(directory);
+                    break;
                 case NIOFS:
                     assertTrue(type + " " + directory.toString(), directory instanceof NIOFSDirectory);
                     break;
@@ -75,7 +79,7 @@ public class IndexStoreTests extends ESTestCase {
                     break;
                 case FS:
                     if (Constants.JRE_IS_64BIT && MMapDirectory.UNMAP_SUPPORTED) {
-                        assertTrue(directory.toString(), directory instanceof MMapDirectory);
+                        assertHybridDirectory(directory);
                     } else if (Constants.WINDOWS) {
                         assertTrue(directory.toString(), directory instanceof SimpleFSDirectory);
                     } else {
@@ -88,4 +92,9 @@ public class IndexStoreTests extends ESTestCase {
         }
     }
 
+    private void assertHybridDirectory(Directory directory) {
+        assertTrue(directory.toString(), directory instanceof FileSwitchDirectory);
+        Directory primaryDirectory = ((FileSwitchDirectory) directory).getPrimaryDir();
+        assertTrue("primary directory " +  primaryDirectory.toString(), primaryDirectory instanceof MMapDirectory);
+    }
 }