diff --git a/docs/reference/index-modules/store.asciidoc b/docs/reference/index-modules/store.asciidoc index c2b3d700e9b..a5478a5bc12 100644 --- a/docs/reference/index-modules/store.asciidoc +++ b/docs/reference/index-modules/store.asciidoc @@ -40,7 +40,7 @@ The following sections lists all the different storage types supported. `fs`:: Default file system implementation. This will pick the best implementation -depending on the operating environment, which is currently `mmapfs` on all +depending on the operating environment, which is currently `hybridfs` on all supported systems but is subject to change. [[simplefs]]`simplefs`:: @@ -67,12 +67,22 @@ process equal to the size of the file being mapped. Before using this class, be sure you have allowed plenty of <>. +[[hybridfs]]`hybridfs`:: + +The `hybridfs` type is a hybrid of `niofs` and `mmapfs`, which chooses the best +file system type for each type of file based on the read access pattern. +Currently only the Lucene term dictionary, norms and doc values files are +memory mapped. All other files are opened using Lucene `NIOFSDirectory`. +Similarly to `mmapfs` be sure you have allowed plenty of +<>. + [[allow-mmapfs]] -You can restrict the use of the `mmapfs` store type via the setting -`node.store.allow_mmapfs`. This is a boolean setting indicating whether or not -`mmapfs` is allowed. The default is to allow `mmapfs`. This setting is useful, -for example, if you are in an environment where you can not control the ability -to create a lot of memory maps so you need disable the ability to use `mmapfs`. +You can restrict the use of the `mmapfs` and the related `hybridfs` store type +via the setting `node.store.allow_mmapfs`. This is a boolean setting indicating +whether or not memory-mapping is allowed. The default is to allow it. This +setting is useful, for example, if you are in an environment where you can not +control the ability to create a lot of memory maps so you need disable the +ability to use memory-mapping. === Pre-loading data into the file system cache diff --git a/server/src/main/java/org/elasticsearch/index/IndexModule.java b/server/src/main/java/org/elasticsearch/index/IndexModule.java index 7f2eae492fd..288fed89d2c 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexModule.java +++ b/server/src/main/java/org/elasticsearch/index/IndexModule.java @@ -302,6 +302,7 @@ public final class IndexModule { public enum Type { + HYBRIDFS("hybridfs"), NIOFS("niofs"), MMAPFS("mmapfs"), SIMPLEFS("simplefs"), @@ -330,7 +331,7 @@ public final class IndexModule { public static Type fromSettingsKey(final String key) { final Type type = TYPES.get(key); if (type == null) { - throw new IllegalArgumentException("no matching type for [" + key + "]"); + throw new IllegalArgumentException("no matching store type for [" + key + "]"); } return type; } @@ -356,7 +357,7 @@ public final class IndexModule { public static Type defaultStoreType(final boolean allowMmapfs) { if (allowMmapfs && Constants.JRE_IS_64BIT && MMapDirectory.UNMAP_SUPPORTED) { - return Type.MMAPFS; + return Type.HYBRIDFS; } else if (Constants.WINDOWS) { return Type.SIMPLEFS; } else { diff --git a/server/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java b/server/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java index f95cdb3a9f6..345ef1eef41 100644 --- a/server/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java +++ b/server/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.store; import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; import org.apache.lucene.store.FileSwitchDirectory; import org.apache.lucene.store.LockFactory; import org.apache.lucene.store.MMapDirectory; @@ -30,6 +31,7 @@ import org.apache.lucene.store.SimpleFSLockFactory; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; +import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.shard.ShardPath; @@ -37,10 +39,16 @@ import org.elasticsearch.index.shard.ShardPath; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Collections; import java.util.HashSet; import java.util.Set; public class FsDirectoryService extends DirectoryService { + /* + * We are mmapping norms, docvalues as well as term dictionaries, all other files are served through NIOFS + * this provides good random access performance and does not lead to page cache thrashing. + */ + private static final Set PRIMARY_EXTENSIONS = Collections.unmodifiableSet(Sets.newHashSet("nvd", "dvd", "tim")); protected final IndexStore indexStore; public static final Setting INDEX_LOCK_FACTOR_SETTING = new Setting<>("index.store.fs.fs_lock", "native", (s) -> { @@ -78,27 +86,36 @@ public class FsDirectoryService extends DirectoryService { protected Directory newFSDirectory(Path location, LockFactory lockFactory) throws IOException { final String storeType = indexSettings.getSettings().get(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), IndexModule.Type.FS.getSettingsKey()); + IndexModule.Type type; if (IndexModule.Type.FS.match(storeType)) { - final IndexModule.Type type = - IndexModule.defaultStoreType(IndexModule.NODE_STORE_ALLOW_MMAPFS.get(indexSettings.getNodeSettings())); - switch (type) { - case MMAPFS: - return new MMapDirectory(location, lockFactory); - case SIMPLEFS: - return new SimpleFSDirectory(location, lockFactory); - case NIOFS: - return new NIOFSDirectory(location, lockFactory); - default: - throw new AssertionError("unexpected built-in store type [" + type + "]"); - } - } else if (IndexModule.Type.SIMPLEFS.match(storeType)) { - return new SimpleFSDirectory(location, lockFactory); - } else if (IndexModule.Type.NIOFS.match(storeType)) { - return new NIOFSDirectory(location, lockFactory); - } else if (IndexModule.Type.MMAPFS.match(storeType)) { - return new MMapDirectory(location, lockFactory); + type = IndexModule.defaultStoreType(IndexModule.NODE_STORE_ALLOW_MMAPFS.get(indexSettings.getNodeSettings())); + } else { + type = IndexModule.Type.fromSettingsKey(storeType); + } + switch (type) { + case HYBRIDFS: + // Use Lucene defaults + final FSDirectory primaryDirectory = FSDirectory.open(location, lockFactory); + if (primaryDirectory instanceof MMapDirectory) { + return new FileSwitchDirectory(PRIMARY_EXTENSIONS, primaryDirectory, new NIOFSDirectory(location, lockFactory), true) { + @Override + public String[] listAll() throws IOException { + // Avoid doing listAll twice: + return primaryDirectory.listAll(); + } + }; + } else { + return primaryDirectory; + } + case MMAPFS: + return new MMapDirectory(location, lockFactory); + case SIMPLEFS: + return new SimpleFSDirectory(location, lockFactory); + case NIOFS: + return new NIOFSDirectory(location, lockFactory); + default: + throw new AssertionError("unexpected built-in store type [" + type + "]"); } - throw new IllegalArgumentException("No directory found for type [" + storeType + "]"); } private static Directory setPreload(Directory directory, Path location, LockFactory lockFactory, diff --git a/server/src/test/java/org/elasticsearch/index/store/IndexStoreTests.java b/server/src/test/java/org/elasticsearch/index/store/IndexStoreTests.java index 155473c83cf..eb85219bcc2 100644 --- a/server/src/test/java/org/elasticsearch/index/store/IndexStoreTests.java +++ b/server/src/test/java/org/elasticsearch/index/store/IndexStoreTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.store; import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FileSwitchDirectory; import org.apache.lucene.store.MMapDirectory; import org.apache.lucene.store.NIOFSDirectory; import org.apache.lucene.store.NoLockFactory; @@ -64,6 +65,9 @@ public class IndexStoreTests extends ESTestCase { new ShardPath(false, tempDir, tempDir, new ShardId(index, 0))); try (Directory directory = service.newFSDirectory(tempDir, NoLockFactory.INSTANCE)) { switch (type) { + case HYBRIDFS: + assertHybridDirectory(directory); + break; case NIOFS: assertTrue(type + " " + directory.toString(), directory instanceof NIOFSDirectory); break; @@ -75,7 +79,7 @@ public class IndexStoreTests extends ESTestCase { break; case FS: if (Constants.JRE_IS_64BIT && MMapDirectory.UNMAP_SUPPORTED) { - assertTrue(directory.toString(), directory instanceof MMapDirectory); + assertHybridDirectory(directory); } else if (Constants.WINDOWS) { assertTrue(directory.toString(), directory instanceof SimpleFSDirectory); } else { @@ -88,4 +92,9 @@ public class IndexStoreTests extends ESTestCase { } } + private void assertHybridDirectory(Directory directory) { + assertTrue(directory.toString(), directory instanceof FileSwitchDirectory); + Directory primaryDirectory = ((FileSwitchDirectory) directory).getPrimaryDir(); + assertTrue("primary directory " + primaryDirectory.toString(), primaryDirectory instanceof MMapDirectory); + } }