honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Rename setting to enable mmap 

With this commit we rename `node.store.allow_mmapfs` to
`node.store.allow_mmap`. Previously this setting has controlled whether
`mmapfs` could be used as a store type. With the introduction of
`hybridfs` which also relies on memory-mapping,
`node.store.allow_mmapfs` also applies to `hybridfs` and thus we rename
it in order to convey that it is actually used to allow memory-mapping
but not a specific store type.

Relates #36668
Relates #37070
This commit is contained in:
Daniel Mitterdorfer 2019-01-03 07:10:34 +01:00 committed by GitHub
parent b2aa655f46
commit 75f3443c62
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 31 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
docs/reference/index-modules/store.asciidoc
View File

@ -76,9 +76,9 @@ memory mapped. All other files are opened using Lucene `NIOFSDirectory`.
Similarly to `mmapfs` be sure you have allowed plenty of
<<vm-max-map-count,virtual address space>>.
[[allow-mmapfs]]
[[allow-mmap]]
You can restrict the use of the `mmapfs` and the related `hybridfs` store type
via the setting `node.store.allow_mmapfs`. This is a boolean setting indicating
via the setting `node.store.allow_mmap`. This is a boolean setting indicating
whether or not memory-mapping is allowed. The default is to allow it. This
setting is useful, for example, if you are in an environment where you can not
control the ability to create a lot of memory maps so you need disable the

5
docs/reference/migration/migrate_7_0/settings.asciidoc
View File

@ -36,6 +36,11 @@ of the node id. It can still be configured explicitly in `elasticsearch.yml`.
available to keep the display output in APIs as `bulk` instead of `write`.
These fallback settings and this system property have been removed.
[float]
==== Disabling memory-mapping
* The setting `node.store.allow_mmapfs` has been renamed to `node.store.allow_mmap`.
[float]
[[remove-http-enabled]]
==== Http enabled setting removed

6
docs/reference/setup/bootstrap-checks.asciidoc
View File

@ -157,9 +157,9 @@ and is enforced on Linux only. To pass the maximum map count check, you
must configure `vm.max_map_count` via `sysctl` to be at least `262144`.
Alternatively, the maximum map count check is only needed if you are using
`mmapfs` as the <<index-modules-store,store type>> for your indices. If you
<<allow-mmapfs,do not allow>> the use of `mmapfs` then this bootstrap check will
not be enforced.
`mmapfs` or `hybridfs` as the <<index-modules-store,store type>> for your
indices. If you <<allow-mmap,do not allow>> the use of `mmap` then this
bootstrap check will not be enforced.
=== Client JVM check

4
server/src/main/java/org/elasticsearch/bootstrap/BootstrapChecks.java
View File

@ -407,8 +407,8 @@ final class BootstrapChecks {
@Override
public BootstrapCheckResult check(final BootstrapContext context) {
// we only enforce the check if mmapfs is an allowed store type
if (IndexModule.NODE_STORE_ALLOW_MMAPFS.get(context.settings())) {
// we only enforce the check if a store is allowed to use mmap at all
if (IndexModule.NODE_STORE_ALLOW_MMAP.get(context.settings())) {
if (getMaxMapCount() != -1 && getMaxMapCount() < LIMIT) {
final String message = String.format(
Locale.ROOT,

2
server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
View File

@ -284,7 +284,7 @@ public final class ClusterSettings extends AbstractScopedSettings {
HierarchyCircuitBreakerService.REQUEST_CIRCUIT_BREAKER_OVERHEAD_SETTING,
HierarchyCircuitBreakerService.ACCOUNTING_CIRCUIT_BREAKER_LIMIT_SETTING,
HierarchyCircuitBreakerService.ACCOUNTING_CIRCUIT_BREAKER_OVERHEAD_SETTING,
IndexModule.NODE_STORE_ALLOW_MMAPFS,
IndexModule.NODE_STORE_ALLOW_MMAP,
ClusterService.CLUSTER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD_SETTING,
ClusterService.USER_DEFINED_META_DATA,
SearchService.DEFAULT_SEARCH_TIMEOUT_SETTING,

14
server/src/main/java/org/elasticsearch/index/IndexModule.java
View File

@ -84,7 +84,7 @@ import java.util.function.Function;
*/
public final class IndexModule {
public static final Setting<Boolean> NODE_STORE_ALLOW_MMAPFS = Setting.boolSetting("node.store.allow_mmapfs", true, Property.NodeScope);
public static final Setting<Boolean> NODE_STORE_ALLOW_MMAP = Setting.boolSetting("node.store.allow_mmap", true, Property.NodeScope);
public static final Setting<String> INDEX_STORE_TYPE_SETTING =
new Setting<>("index.store.type", "", Function.identity(), Property.IndexScope, Property.NodeScope);
@ -355,8 +355,8 @@ public final class IndexModule {
IndexSearcherWrapper newWrapper(IndexService indexService);
}
public static Type defaultStoreType(final boolean allowMmapfs) {
if (allowMmapfs && Constants.JRE_IS_64BIT && MMapDirectory.UNMAP_SUPPORTED) {
public static Type defaultStoreType(final boolean allowMmap) {
if (allowMmap && Constants.JRE_IS_64BIT && MMapDirectory.UNMAP_SUPPORTED) {
return Type.HYBRIDFS;
} else if (Constants.WINDOWS) {
return Type.SIMPLEFS;
@ -406,9 +406,9 @@ public final class IndexModule {
final IndexSettings indexSettings, final Map<String, Function<IndexSettings, IndexStore>> indexStoreFactories) {
final String storeType = indexSettings.getValue(INDEX_STORE_TYPE_SETTING);
final Type type;
final Boolean allowMmapfs = NODE_STORE_ALLOW_MMAPFS.get(indexSettings.getNodeSettings());
final Boolean allowMmap = NODE_STORE_ALLOW_MMAP.get(indexSettings.getNodeSettings());
if (storeType.isEmpty() || Type.FS.getSettingsKey().equals(storeType)) {
type = defaultStoreType(allowMmapfs);
type = defaultStoreType(allowMmap);
} else {
if (isBuiltinType(storeType)) {
type = Type.fromSettingsKey(storeType);
@ -416,8 +416,8 @@ public final class IndexModule {
type = null;
}
}
if (type != null && type == Type.MMAPFS && allowMmapfs == false) {
throw new IllegalArgumentException("store type [mmapfs] is not allowed");
if (allowMmap == false && (type == Type.MMAPFS || type == Type.HYBRIDFS)) {
throw new IllegalArgumentException("store type [" + storeType + "] is not allowed because mmap is disabled");
}
final IndexStore store;
if (storeType.isEmpty() || isBuiltinType(storeType)) {

2
server/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java
View File

@ -88,7 +88,7 @@ public class FsDirectoryService extends DirectoryService {
indexSettings.getSettings().get(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), IndexModule.Type.FS.getSettingsKey());
IndexModule.Type type;
if (IndexModule.Type.FS.match(storeType)) {
type = IndexModule.defaultStoreType(IndexModule.NODE_STORE_ALLOW_MMAPFS.get(indexSettings.getNodeSettings()));
type = IndexModule.defaultStoreType(IndexModule.NODE_STORE_ALLOW_MMAP.get(indexSettings.getNodeSettings()));
} else {
type = IndexModule.Type.fromSettingsKey(storeType);
}

10
server/src/test/java/org/elasticsearch/bootstrap/MaxMapCountCheckTests.java
View File

@ -76,12 +76,12 @@ public class MaxMapCountCheckTests extends AbstractBootstrapCheckTestCase {
/*
* There are two ways that memory maps are allowed:
* - by default
* - mmapfs is explicitly allowed
* We want to test that if mmapfs is allowed then the max map count check is enforced.
* - mmap is explicitly allowed
* We want to test that if mmap is allowed then the max map count check is enforced.
*/
final List<Settings> settingsThatAllowMemoryMap = new ArrayList<>();
settingsThatAllowMemoryMap.add(Settings.EMPTY);
settingsThatAllowMemoryMap.add(Settings.builder().put("node.store.allow_mmapfs", true).build());
settingsThatAllowMemoryMap.add(Settings.builder().put("node.store.allow_mmap", true).build());
for (final Settings settingThatAllowsMemoryMap : settingsThatAllowMemoryMap) {
assertFailure(check.check(createTestContext(settingThatAllowsMemoryMap, MetaData.EMPTY_META_DATA)));
@ -89,8 +89,8 @@ public class MaxMapCountCheckTests extends AbstractBootstrapCheckTestCase {
}
public void testMaxMapCountCheckNotEnforcedIfMemoryMapNotAllowed() {
// nothing should happen if current vm.max_map_count is under the limit but mmapfs is not allowed
final Settings settings = Settings.builder().put("node.store.allow_mmapfs", false).build();
// nothing should happen if current vm.max_map_count is under the limit but mmap is not allowed
final Settings settings = Settings.builder().put("node.store.allow_mmap", false).build();
final BootstrapContext context = createTestContext(settings, MetaData.EMPTY_META_DATA);
final BootstrapCheck.BootstrapCheckResult result = check.check(context);
assertTrue(result.isSuccess());

9
server/src/test/java/org/elasticsearch/index/IndexModuleTests.java
View File

@ -383,19 +383,20 @@ public class IndexModuleTests extends ESTestCase {
indexService.close("simon says", false);
}
public void testMmapfsStoreTypeNotAllowed() {
public void testMmapNotAllowed() {
String storeType = randomFrom(IndexModule.Type.HYBRIDFS.getSettingsKey(), IndexModule.Type.MMAPFS.getSettingsKey());
final Settings settings = Settings.builder()
.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
.put("index.store.type", "mmapfs")
.put("index.store.type", storeType)
.build();
final Settings nodeSettings = Settings.builder()
.put(IndexModule.NODE_STORE_ALLOW_MMAPFS.getKey(), false)
.put(IndexModule.NODE_STORE_ALLOW_MMAP.getKey(), false)
.build();
final IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(new Index("foo", "_na_"), settings, nodeSettings);
final IndexModule module =
new IndexModule(indexSettings, emptyAnalysisRegistry, new InternalEngineFactory(), Collections.emptyMap());
final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> newIndexService(module));
assertThat(e, hasToString(containsString("store type [mmapfs] is not allowed")));
assertThat(e, hasToString(containsString("store type [" + storeType + "] is not allowed")));
}
class CustomQueryCache implements QueryCache {