From 4adc31fe115ac508cc9d8814204c66c6933161c2 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Fri, 8 Apr 2016 12:18:02 +0200 Subject: [PATCH] Use `mmapfs` by default. I case any problem was discovered, you can still enable the legacy `default` directory instead. But the plan is to get rid of it in 6.0. Closes #16983 --- .../org/elasticsearch/index/IndexModule.java | 1 + .../index/store/FsDirectoryService.java | 13 ++++-- .../index/store/IndexStoreTests.java | 45 ++++++++++--------- docs/reference/migration/migrate_5_0.asciidoc | 3 +- .../migration/migrate_5_0/fs.asciidoc | 10 +++++ 5 files changed, 48 insertions(+), 24 deletions(-) create mode 100644 docs/reference/migration/migrate_5_0/fs.asciidoc diff --git a/core/src/main/java/org/elasticsearch/index/IndexModule.java b/core/src/main/java/org/elasticsearch/index/IndexModule.java index 3bc1346e884..24f075e98f5 100644 --- a/core/src/main/java/org/elasticsearch/index/IndexModule.java +++ b/core/src/main/java/org/elasticsearch/index/IndexModule.java @@ -283,6 +283,7 @@ public final class IndexModule { MMAPFS, SIMPLEFS, FS, + @Deprecated DEFAULT; public String getSettingsKey() { diff --git a/core/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java b/core/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java index 584b98cff33..d41697cd14c 100644 --- a/core/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java +++ b/core/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java @@ -110,10 +110,12 @@ public class FsDirectoryService extends DirectoryService implements StoreRateLim protected Directory newFSDirectory(Path location, LockFactory lockFactory) throws IOException { final String storeType = indexSettings.getSettings().get(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), - IndexModule.Type.DEFAULT.getSettingsKey()); - if (IndexModule.Type.FS.match(storeType) || IndexModule.Type.DEFAULT.match(storeType)) { + IndexModule.Type.FS.getSettingsKey()); + if (IndexModule.Type.FS.match(storeType) || isDefault(storeType)) { final FSDirectory open = FSDirectory.open(location, lockFactory); // use lucene defaults - if (open instanceof MMapDirectory && Constants.WINDOWS == false) { + if (open instanceof MMapDirectory + && isDefault(storeType) + && Constants.WINDOWS == false) { return newDefaultDir(location, (MMapDirectory) open, lockFactory); } return open; @@ -127,6 +129,11 @@ public class FsDirectoryService extends DirectoryService implements StoreRateLim throw new IllegalArgumentException("No directory found for type [" + storeType + "]"); } + @SuppressWarnings("deprecation") + private static boolean isDefault(String storeType) { + return IndexModule.Type.DEFAULT.match(storeType); + } + private Directory newDefaultDir(Path location, final MMapDirectory mmapDir, LockFactory lockFactory) throws IOException { return new FileSwitchDirectory(PRIMARY_EXTENSIONS, mmapDir, new NIOFSDirectory(location, lockFactory), true) { @Override diff --git a/core/src/test/java/org/elasticsearch/index/store/IndexStoreTests.java b/core/src/test/java/org/elasticsearch/index/store/IndexStoreTests.java index f2d2afd9fc1..0e012f0d1a7 100644 --- a/core/src/test/java/org/elasticsearch/index/store/IndexStoreTests.java +++ b/core/src/test/java/org/elasticsearch/index/store/IndexStoreTests.java @@ -18,7 +18,6 @@ */ package org.elasticsearch.index.store; -import com.carrotsearch.randomizedtesting.generators.RandomPicks; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FileSwitchDirectory; import org.apache.lucene.store.MMapDirectory; @@ -49,10 +48,21 @@ public class IndexStoreTests extends ESTestCase { public void testStoreDirectory() throws IOException { Index index = new Index("foo", "fooUUID"); final Path tempDir = createTempDir().resolve(index.getUUID()).resolve("0"); - final IndexModule.Type[] values = IndexModule.Type.values(); - final IndexModule.Type type = RandomPicks.randomFrom(random(), values); - Settings settings = Settings.builder().put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), type.name().toLowerCase(Locale.ROOT)) - .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build(); + // default + doTestStoreDirectory(index, tempDir, null, IndexModule.Type.FS); + // explicit directory impls + for (IndexModule.Type type : IndexModule.Type.values()) { + doTestStoreDirectory(index, tempDir, type.name().toLowerCase(Locale.ROOT), type); + } + } + + private void doTestStoreDirectory(Index index, Path tempDir, String typeSettingValue, IndexModule.Type type) throws IOException { + Settings.Builder settingsBuilder = Settings.builder() + .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT); + if (typeSettingValue != null) { + settingsBuilder.put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), typeSettingValue); + } + Settings settings = settingsBuilder.build(); IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("foo", settings); FsDirectoryService service = new FsDirectoryService(indexSettings, null, new ShardPath(false, tempDir, tempDir, new ShardId(index, 0))); try (final Directory directory = service.newFSDirectory(tempDir, NoLockFactory.INSTANCE)) { @@ -67,6 +77,14 @@ public class IndexStoreTests extends ESTestCase { assertTrue(type + " " + directory.toString(), directory instanceof SimpleFSDirectory); break; case FS: + if (Constants.JRE_IS_64BIT && MMapDirectory.UNMAP_SUPPORTED) { + assertTrue(directory.toString(), directory instanceof MMapDirectory); + } else if (Constants.WINDOWS) { + assertTrue(directory.toString(), directory instanceof SimpleFSDirectory); + } else { + assertTrue(directory.toString(), directory instanceof NIOFSDirectory); + } + break; case DEFAULT: if (Constants.WINDOWS) { if (Constants.JRE_IS_64BIT && MMapDirectory.UNMAP_SUPPORTED) { @@ -80,21 +98,8 @@ public class IndexStoreTests extends ESTestCase { assertTrue(type + " " + directory.toString(), directory instanceof NIOFSDirectory); } break; - } - } - } - - public void testStoreDirectoryDefault() throws IOException { - Index index = new Index("bar", "foo"); - final Path tempDir = createTempDir().resolve(index.getUUID()).resolve("0"); - FsDirectoryService service = new FsDirectoryService(IndexSettingsModule.newIndexSettings("bar", Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build()), null, new ShardPath(false, tempDir, tempDir, new ShardId(index, 0))); - try (final Directory directory = service.newFSDirectory(tempDir, NoLockFactory.INSTANCE)) { - if (Constants.WINDOWS) { - assertTrue(directory.toString(), directory instanceof MMapDirectory || directory instanceof SimpleFSDirectory); - } else if (Constants.JRE_IS_64BIT) { - assertTrue(directory.toString(), directory instanceof FileSwitchDirectory); - } else { - assertTrue(directory.toString(), directory instanceof NIOFSDirectory); + default: + fail(); } } } diff --git a/docs/reference/migration/migrate_5_0.asciidoc b/docs/reference/migration/migrate_5_0.asciidoc index f4aaacee35e..29d41851f35 100644 --- a/docs/reference/migration/migrate_5_0.asciidoc +++ b/docs/reference/migration/migrate_5_0.asciidoc @@ -40,6 +40,7 @@ way to do this is to upgrade to Elasticsearch 2.3 or later and to use the * <> * <> * <> +* <> include::migrate_5_0/search.asciidoc[] @@ -63,4 +64,4 @@ include::migrate_5_0/packaging.asciidoc[] include::migrate_5_0/plugins.asciidoc[] - +include::migrate_5_0/fs.asciidoc[] diff --git a/docs/reference/migration/migrate_5_0/fs.asciidoc b/docs/reference/migration/migrate_5_0/fs.asciidoc new file mode 100644 index 00000000000..2d582702690 --- /dev/null +++ b/docs/reference/migration/migrate_5_0/fs.asciidoc @@ -0,0 +1,10 @@ +[[breaking_50_fs]] +=== Filesystem related changes + +Only a subset of index files were open with `mmap` on Elasticsearch 2.x. As of +Elasticsearch 5.0, all index files will be open with `mmap` on 64-bit systems. +While this may increase the amount of virtual memory used by Elasticsearch, +there is nothing to worry about since this is only address space consumption +and the actual memory usage of Elasticsearch will stay similar to what it was +in 2.x. See http://blog.thetaphi.de/2012/07/use-lucenes-mmapdirectory-on-64bit.html +for more information.