Use `mmapfs` by default.

I case any problem was discovered, you can still enable the legacy `default`
directory instead. But the plan is to get rid of it in 6.0.

Closes #16983

core/src/main/java/org/elasticsearch/index/IndexModule.java

@@ -283,6 +283,7 @@ public final class IndexModule {
         MMAPFS,
         SIMPLEFS,
         FS,
+        @Deprecated
         DEFAULT;
 
         public String getSettingsKey() {

core/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java

@@ -110,10 +110,12 @@ public class FsDirectoryService extends DirectoryService implements StoreRateLim
 
     protected Directory newFSDirectory(Path location, LockFactory lockFactory) throws IOException {
         final String storeType = indexSettings.getSettings().get(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(),
-            IndexModule.Type.DEFAULT.getSettingsKey());
+            IndexModule.Type.FS.getSettingsKey());
-        if (IndexModule.Type.FS.match(storeType) || IndexModule.Type.DEFAULT.match(storeType)) {
+        if (IndexModule.Type.FS.match(storeType) || isDefault(storeType)) {
             final FSDirectory open = FSDirectory.open(location, lockFactory); // use lucene defaults
-            if (open instanceof MMapDirectory && Constants.WINDOWS == false) {
+            if (open instanceof MMapDirectory
+                    && isDefault(storeType)
+                    && Constants.WINDOWS == false) {
                 return newDefaultDir(location, (MMapDirectory) open, lockFactory);
             }
             return open;
@@ -127,6 +129,11 @@ public class FsDirectoryService extends DirectoryService implements StoreRateLim
         throw new IllegalArgumentException("No directory found for type [" + storeType + "]");
     }
 
+    @SuppressWarnings("deprecation")
+    private static boolean isDefault(String storeType) {
+        return IndexModule.Type.DEFAULT.match(storeType);
+    }
+
     private Directory newDefaultDir(Path location, final MMapDirectory mmapDir, LockFactory lockFactory) throws IOException {
         return new FileSwitchDirectory(PRIMARY_EXTENSIONS, mmapDir, new NIOFSDirectory(location, lockFactory), true) {
             @Override

core/src/test/java/org/elasticsearch/index/store/IndexStoreTests.java

@@ -18,7 +18,6 @@
  */
 package org.elasticsearch.index.store;
 
-import com.carrotsearch.randomizedtesting.generators.RandomPicks;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FileSwitchDirectory;
 import org.apache.lucene.store.MMapDirectory;
@@ -49,10 +48,21 @@ public class IndexStoreTests extends ESTestCase {
     public void testStoreDirectory() throws IOException {
         Index index = new Index("foo", "fooUUID");
         final Path tempDir = createTempDir().resolve(index.getUUID()).resolve("0");
-        final IndexModule.Type[] values = IndexModule.Type.values();
+        // default
-        final IndexModule.Type type = RandomPicks.randomFrom(random(), values);
+        doTestStoreDirectory(index, tempDir, null, IndexModule.Type.FS);
-        Settings settings = Settings.builder().put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), type.name().toLowerCase(Locale.ROOT))
+        // explicit directory impls
-                .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
+        for (IndexModule.Type type : IndexModule.Type.values()) {
+            doTestStoreDirectory(index, tempDir, type.name().toLowerCase(Locale.ROOT), type);
+        }
+    }
+
+    private void doTestStoreDirectory(Index index, Path tempDir, String typeSettingValue, IndexModule.Type type) throws IOException {
+        Settings.Builder settingsBuilder = Settings.builder()
+                .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT);
+        if (typeSettingValue != null) {
+            settingsBuilder.put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), typeSettingValue);
+        }
+        Settings settings = settingsBuilder.build();
         IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("foo", settings);
         FsDirectoryService service = new FsDirectoryService(indexSettings, null, new ShardPath(false, tempDir, tempDir, new ShardId(index, 0)));
         try (final Directory directory = service.newFSDirectory(tempDir, NoLockFactory.INSTANCE)) {
@@ -67,6 +77,14 @@ public class IndexStoreTests extends ESTestCase {
                     assertTrue(type + " " + directory.toString(), directory instanceof SimpleFSDirectory);
                     break;
                 case FS:
+                    if (Constants.JRE_IS_64BIT && MMapDirectory.UNMAP_SUPPORTED) {
+                        assertTrue(directory.toString(), directory instanceof MMapDirectory);
+                    } else if (Constants.WINDOWS) {
+                        assertTrue(directory.toString(), directory instanceof SimpleFSDirectory);
+                    } else {
+                        assertTrue(directory.toString(), directory instanceof NIOFSDirectory);
+                    }
+                    break;
                 case DEFAULT:
                    if (Constants.WINDOWS) {
                         if (Constants.JRE_IS_64BIT && MMapDirectory.UNMAP_SUPPORTED) {
@@ -80,21 +98,8 @@ public class IndexStoreTests extends ESTestCase {
                         assertTrue(type + " " + directory.toString(), directory instanceof NIOFSDirectory);
                     }
                     break;
-            }
+                default:
-        }
+                    fail();
-    }
-
-    public void testStoreDirectoryDefault() throws IOException {
-        Index index = new Index("bar", "foo");
-        final Path tempDir = createTempDir().resolve(index.getUUID()).resolve("0");
-        FsDirectoryService service = new FsDirectoryService(IndexSettingsModule.newIndexSettings("bar", Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build()), null, new ShardPath(false, tempDir, tempDir, new ShardId(index, 0)));
-        try (final Directory directory = service.newFSDirectory(tempDir, NoLockFactory.INSTANCE)) {
-            if (Constants.WINDOWS) {
-                assertTrue(directory.toString(), directory instanceof MMapDirectory || directory instanceof SimpleFSDirectory);
-            } else if (Constants.JRE_IS_64BIT) {
-                assertTrue(directory.toString(), directory instanceof FileSwitchDirectory);
-            } else {
-                assertTrue(directory.toString(), directory instanceof NIOFSDirectory);
             }
         }
     }

docs/reference/migration/migrate_5_0.asciidoc

@@ -40,6 +40,7 @@ way to do this is to upgrade to Elasticsearch 2.3 or later and to use the
 * <<breaking_50_java_api_changes>>
 * <<breaking_50_packaging>>
 * <<breaking_50_plugins>>
+* <<breaking_50_fs>>
 
 include::migrate_5_0/search.asciidoc[]
 
@@ -63,4 +64,4 @@ include::migrate_5_0/packaging.asciidoc[]
 
 include::migrate_5_0/plugins.asciidoc[]
 
-
+include::migrate_5_0/fs.asciidoc[]

docs/reference/migration/migrate_5_0/fs.asciidoc

@@ -0,0 +1,10 @@
+[[breaking_50_fs]]
+=== Filesystem related changes
+
+Only a subset of index files were open with `mmap` on Elasticsearch 2.x. As of
+Elasticsearch 5.0, all index files will be open with `mmap` on 64-bit systems.
+While this may increase the amount of virtual memory used by Elasticsearch,
+there is nothing to worry about since this is only address space consumption
+and the actual memory usage of Elasticsearch will stay similar to what it was
+in 2.x. See http://blog.thetaphi.de/2012/07/use-lucenes-mmapdirectory-on-64bit.html
+for more information.