honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Use `mmapfs` by default. 

I case any problem was discovered, you can still enable the legacy `default`
directory instead. But the plan is to get rid of it in 6.0.

Closes #16983
This commit is contained in:
Adrien Grand 2016-04-08 12:18:02 +02:00
parent 4391594e4c
commit 4adc31fe11
5 changed files with 48 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
core/src/main/java/org/elasticsearch/index/IndexModule.java
View File

@ -283,6 +283,7 @@ public final class IndexModule {
MMAPFS,
SIMPLEFS,
FS,
@Deprecated
DEFAULT;
public String getSettingsKey() {

13
core/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java
View File

@ -110,10 +110,12 @@ public class FsDirectoryService extends DirectoryService implements StoreRateLim
protected Directory newFSDirectory(Path location, LockFactory lockFactory) throws IOException {
final String storeType = indexSettings.getSettings().get(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(),
IndexModule.Type.DEFAULT.getSettingsKey());
if (IndexModule.Type.FS.match(storeType) || IndexModule.Type.DEFAULT.match(storeType)) {
IndexModule.Type.FS.getSettingsKey());
if (IndexModule.Type.FS.match(storeType) || isDefault(storeType)) {
final FSDirectory open = FSDirectory.open(location, lockFactory); // use lucene defaults
if (open instanceof MMapDirectory && Constants.WINDOWS == false) {
if (open instanceof MMapDirectory
&& isDefault(storeType)
&& Constants.WINDOWS == false) {
return newDefaultDir(location, (MMapDirectory) open, lockFactory);
}
return open;
@ -127,6 +129,11 @@ public class FsDirectoryService extends DirectoryService implements StoreRateLim
throw new IllegalArgumentException("No directory found for type [" + storeType + "]");
}
@SuppressWarnings("deprecation")
private static boolean isDefault(String storeType) {
return IndexModule.Type.DEFAULT.match(storeType);
}
private Directory newDefaultDir(Path location, final MMapDirectory mmapDir, LockFactory lockFactory) throws IOException {
return new FileSwitchDirectory(PRIMARY_EXTENSIONS, mmapDir, new NIOFSDirectory(location, lockFactory), true) {
@Override

45
core/src/test/java/org/elasticsearch/index/store/IndexStoreTests.java
View File

@ -18,7 +18,6 @@
*/
package org.elasticsearch.index.store;
import com.carrotsearch.randomizedtesting.generators.RandomPicks;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FileSwitchDirectory;
import org.apache.lucene.store.MMapDirectory;
@ -49,10 +48,21 @@ public class IndexStoreTests extends ESTestCase {
public void testStoreDirectory() throws IOException {
Index index = new Index("foo", "fooUUID");
final Path tempDir = createTempDir().resolve(index.getUUID()).resolve("0");
final IndexModule.Type[] values = IndexModule.Type.values();
final IndexModule.Type type = RandomPicks.randomFrom(random(), values);
Settings settings = Settings.builder().put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), type.name().toLowerCase(Locale.ROOT))
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
// default
doTestStoreDirectory(index, tempDir, null, IndexModule.Type.FS);
// explicit directory impls
for (IndexModule.Type type : IndexModule.Type.values()) {
doTestStoreDirectory(index, tempDir, type.name().toLowerCase(Locale.ROOT), type);
}
}
private void doTestStoreDirectory(Index index, Path tempDir, String typeSettingValue, IndexModule.Type type) throws IOException {
Settings.Builder settingsBuilder = Settings.builder()
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT);
if (typeSettingValue != null) {
settingsBuilder.put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), typeSettingValue);
}
Settings settings = settingsBuilder.build();
IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("foo", settings);
FsDirectoryService service = new FsDirectoryService(indexSettings, null, new ShardPath(false, tempDir, tempDir, new ShardId(index, 0)));
try (final Directory directory = service.newFSDirectory(tempDir, NoLockFactory.INSTANCE)) {
@ -67,6 +77,14 @@ public class IndexStoreTests extends ESTestCase {
assertTrue(type + " " + directory.toString(), directory instanceof SimpleFSDirectory);
break;
case FS:
if (Constants.JRE_IS_64BIT && MMapDirectory.UNMAP_SUPPORTED) {
assertTrue(directory.toString(), directory instanceof MMapDirectory);
} else if (Constants.WINDOWS) {
assertTrue(directory.toString(), directory instanceof SimpleFSDirectory);
} else {
assertTrue(directory.toString(), directory instanceof NIOFSDirectory);
}
break;
case DEFAULT:
if (Constants.WINDOWS) {
if (Constants.JRE_IS_64BIT && MMapDirectory.UNMAP_SUPPORTED) {
@ -80,21 +98,8 @@ public class IndexStoreTests extends ESTestCase {
assertTrue(type + " " + directory.toString(), directory instanceof NIOFSDirectory);
}
break;
}
}
}
public void testStoreDirectoryDefault() throws IOException {
Index index = new Index("bar", "foo");
final Path tempDir = createTempDir().resolve(index.getUUID()).resolve("0");
FsDirectoryService service = new FsDirectoryService(IndexSettingsModule.newIndexSettings("bar", Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build()), null, new ShardPath(false, tempDir, tempDir, new ShardId(index, 0)));
try (final Directory directory = service.newFSDirectory(tempDir, NoLockFactory.INSTANCE)) {
if (Constants.WINDOWS) {
assertTrue(directory.toString(), directory instanceof MMapDirectory || directory instanceof SimpleFSDirectory);
} else if (Constants.JRE_IS_64BIT) {
assertTrue(directory.toString(), directory instanceof FileSwitchDirectory);
} else {
assertTrue(directory.toString(), directory instanceof NIOFSDirectory);
default:
fail();
}
}
}

3
docs/reference/migration/migrate_5_0.asciidoc
View File

@ -40,6 +40,7 @@ way to do this is to upgrade to Elasticsearch 2.3 or later and to use the
* <<breaking_50_java_api_changes>>
* <<breaking_50_packaging>>
* <<breaking_50_plugins>>
* <<breaking_50_fs>>
include::migrate_5_0/search.asciidoc[]
@ -63,4 +64,4 @@ include::migrate_5_0/packaging.asciidoc[]
include::migrate_5_0/plugins.asciidoc[]
include::migrate_5_0/fs.asciidoc[]

10
docs/reference/migration/migrate_5_0/fs.asciidoc Normal file
View File

@ -0,0 +1,10 @@
[[breaking_50_fs]]
=== Filesystem related changes
Only a subset of index files were open with `mmap` on Elasticsearch 2.x. As of
Elasticsearch 5.0, all index files will be open with `mmap` on 64-bit systems.
While this may increase the amount of virtual memory used by Elasticsearch,
there is nothing to worry about since this is only address space consumption
and the actual memory usage of Elasticsearch will stay similar to what it was
in 2.x. See http://blog.thetaphi.de/2012/07/use-lucenes-mmapdirectory-on-64bit.html
for more information.