honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Enable avoiding mmap bootstrap check (#32421) 

The maximum map count boostrap check can be a hindrance to users that do
not own the underlying platform on which they are executing
Elasticsearch. This is because addressing it requires tuning the kernel
and a platform provider might now allow this, especially on shared
infrastructure. However, this bootstrap check is not needed if mmapfs is
not in use. Today we do not have a way for the user to communicate that
they are not going to use mmapfs. This commit therefore adds a setting
that enables the user to disallow mmapfs. When mmapfs is disallowed, the
maximum map count bootstrap check is not enforced. Additionally, we
fallback to a different default index store and prevent the explicit use
of mmapfs for an index.
This commit is contained in:
Jason Tedor 2018-08-21 11:02:25 -04:00 committed by GitHub
parent 5b446b81ef
commit bdfcc326d7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 243 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
docs/reference/index-modules/store.asciidoc
View File

@ -67,6 +67,13 @@ process equal to the size of the file being mapped. Before using this
class, be sure you have allowed plenty of
<<vm-max-map-count,virtual address space>>.
[[allow-mmapfs]]
You can restrict the use of the `mmapfs` store type via the setting
`node.store.allow_mmapfs`. This is a boolean setting indicating whether or not
`mmapfs` is allowed. The default is to allow `mmapfs`. This setting is useful,
for example, if you are in an environment where you can not control the ability
to create a lot of memory maps so you need disable the ability to use `mmapfs`.
=== Pre-loading data into the file system cache
NOTE: This is an expert setting, the details of which may change in the future.

5
docs/reference/setup/bootstrap-checks.asciidoc
View File

@ -155,6 +155,11 @@ the kernel allows a process to have at least 262,144 memory-mapped areas
and is enforced on Linux only. To pass the maximum map count check, you
must configure `vm.max_map_count` via `sysctl` to be at least `262144`.
Alternatively, the maximum map count check is only needed if you are using
`mmapfs` as the <<index-modules-store,store type>> for your indices. If you
<<allow-mmapfs,do not allow>> the use of `mmapfs` then this bootstrap check will
not be enforced.
=== Client JVM check
There are two different JVMs provided by OpenJDK-derived JVMs: the

10
server/src/main/java/org/elasticsearch/bootstrap/BootstrapChecks.java
View File

@ -28,6 +28,7 @@ import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.transport.BoundTransportAddress;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.discovery.DiscoveryModule;
import org.elasticsearch.index.IndexModule;
import org.elasticsearch.monitor.jvm.JvmInfo;
import org.elasticsearch.monitor.process.ProcessProbe;
import org.elasticsearch.node.NodeValidationException;
@ -393,10 +394,12 @@ final class BootstrapChecks {
static class MaxMapCountCheck implements BootstrapCheck {
private static final long LIMIT = 1 << 18;
static final long LIMIT = 1 << 18;
@Override
public BootstrapCheckResult check(BootstrapContext context) {
public BootstrapCheckResult check(final BootstrapContext context) {
// we only enforce the check if mmapfs is an allowed store type
if (IndexModule.NODE_STORE_ALLOW_MMAPFS.get(context.settings)) {
if (getMaxMapCount() != -1 && getMaxMapCount() < LIMIT) {
final String message = String.format(
Locale.ROOT,
@ -407,6 +410,9 @@ final class BootstrapChecks {
} else {
return BootstrapCheckResult.success();
}
} else {
return BootstrapCheckResult.success();
}
}
// visible for testing

2
server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
View File

@ -63,6 +63,7 @@ import org.elasticsearch.env.Environment;
import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.gateway.GatewayService;
import org.elasticsearch.http.HttpTransportSettings;
import org.elasticsearch.index.IndexModule;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.indices.IndexingMemoryController;
import org.elasticsearch.indices.IndicesQueryCache;
@ -264,6 +265,7 @@ public final class ClusterSettings extends AbstractScopedSettings {
HierarchyCircuitBreakerService.REQUEST_CIRCUIT_BREAKER_OVERHEAD_SETTING,
HierarchyCircuitBreakerService.ACCOUNTING_CIRCUIT_BREAKER_LIMIT_SETTING,
HierarchyCircuitBreakerService.ACCOUNTING_CIRCUIT_BREAKER_OVERHEAD_SETTING,
IndexModule.NODE_STORE_ALLOW_MMAPFS,
ClusterService.CLUSTER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD_SETTING,
SearchService.DEFAULT_SEARCH_TIMEOUT_SETTING,
SearchService.DEFAULT_ALLOW_PARTIAL_SEARCH_RESULTS,

103
server/src/main/java/org/elasticsearch/index/IndexModule.java
View File

@ -21,10 +21,11 @@ package org.elasticsearch.index;
import org.apache.lucene.search.similarities.BM25Similarity;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.store.MMapDirectory;
import org.apache.lucene.util.Constants;
import org.apache.lucene.util.SetOnce;
import org.elasticsearch.Version;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.TriFunction;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.settings.Setting;
@ -59,7 +60,6 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
@ -84,6 +84,8 @@ import java.util.function.Function;
*/
public final class IndexModule {
public static final Setting<Boolean> NODE_STORE_ALLOW_MMAPFS = Setting.boolSetting("node.store.allow_mmapfs", true, Property.NodeScope);
public static final Setting<String> INDEX_STORE_TYPE_SETTING =
new Setting<>("index.store.type", "", Function.identity(), Property.IndexScope, Property.NodeScope);
@ -289,7 +291,7 @@ public final class IndexModule {
}
}
private static boolean isBuiltinType(String storeType) {
public static boolean isBuiltinType(String storeType) {
for (Type type : Type.values()) {
if (type.match(storeType)) {
return true;
@ -298,21 +300,48 @@ public final class IndexModule {
return false;
}
public enum Type {
NIOFS,
MMAPFS,
SIMPLEFS,
FS;
NIOFS("niofs"),
MMAPFS("mmapfs"),
SIMPLEFS("simplefs"),
FS("fs");
private final String settingsKey;
Type(final String settingsKey) {
this.settingsKey = settingsKey;
}
private static final Map<String, Type> TYPES;
static {
final Map<String, Type> types = new HashMap<>(4);
for (final Type type : values()) {
types.put(type.settingsKey, type);
}
TYPES = Collections.unmodifiableMap(types);
}
public String getSettingsKey() {
return this.name().toLowerCase(Locale.ROOT);
return this.settingsKey;
}
public static Type fromSettingsKey(final String key) {
final Type type = TYPES.get(key);
if (type == null) {
throw new IllegalArgumentException("no matching type for [" + key + "]");
}
return type;
}
/**
* Returns true iff this settings matches the type.
*/
public boolean match(String setting) {
return getSettingsKey().equals(setting);
}
}
/**
@ -325,6 +354,16 @@ public final class IndexModule {
IndexSearcherWrapper newWrapper(IndexService indexService);
}
public static Type defaultStoreType(final boolean allowMmapfs) {
if (allowMmapfs && Constants.JRE_IS_64BIT && MMapDirectory.UNMAP_SUPPORTED) {
return Type.MMAPFS;
} else if (Constants.WINDOWS) {
return Type.SIMPLEFS;
} else {
return Type.NIOFS;
}
}
public IndexService newIndexService(
NodeEnvironment environment,
NamedXContentRegistry xContentRegistry,
@ -343,20 +382,7 @@ public final class IndexModule {
IndexSearcherWrapperFactory searcherWrapperFactory = indexSearcherWrapper.get() == null
? (shard) -> null : indexSearcherWrapper.get();
eventListener.beforeIndexCreated(indexSettings.getIndex(), indexSettings.getSettings());
final String storeType = indexSettings.getValue(INDEX_STORE_TYPE_SETTING);
final IndexStore store;
if (Strings.isEmpty(storeType) || isBuiltinType(storeType)) {
store = new IndexStore(indexSettings);
} else {
Function<IndexSettings, IndexStore> factory = indexStoreFactories.get(storeType);
if (factory == null) {
throw new IllegalArgumentException("Unknown store type [" + storeType + "]");
}
store = factory.apply(indexSettings);
if (store == null) {
throw new IllegalStateException("store must not be null");
}
}
final IndexStore store = getIndexStore(indexSettings, indexStoreFactories);
final QueryCache queryCache;
if (indexSettings.getValue(INDEX_QUERY_CACHE_ENABLED_SETTING)) {
BiFunction<IndexSettings, IndicesQueryCache, QueryCache> queryCacheProvider = forceQueryCacheProvider.get();
@ -375,6 +401,39 @@ public final class IndexModule {
indicesFieldDataCache, searchOperationListeners, indexOperationListeners, namedWriteableRegistry);
}
private static IndexStore getIndexStore(
final IndexSettings indexSettings, final Map<String, Function<IndexSettings, IndexStore>> indexStoreFactories) {
final String storeType = indexSettings.getValue(INDEX_STORE_TYPE_SETTING);
final Type type;
final Boolean allowMmapfs = NODE_STORE_ALLOW_MMAPFS.get(indexSettings.getNodeSettings());
if (storeType.isEmpty() || Type.FS.getSettingsKey().equals(storeType)) {
type = defaultStoreType(allowMmapfs);
} else {
if (isBuiltinType(storeType)) {
type = Type.fromSettingsKey(storeType);
} else {
type = null;
}
}
if (type != null && type == Type.MMAPFS && allowMmapfs == false) {
throw new IllegalArgumentException("store type [mmapfs] is not allowed");
}
final IndexStore store;
if (storeType.isEmpty() || isBuiltinType(storeType)) {
store = new IndexStore(indexSettings);
} else {
Function<IndexSettings, IndexStore> factory = indexStoreFactories.get(storeType);
if (factory == null) {
throw new IllegalArgumentException("Unknown store type [" + storeType + "]");
}
store = factory.apply(indexSettings);
if (store == null) {
throw new IllegalStateException("store must not be null");
}
}
return store;
}
/**
* creates a new mapper service to do administrative work like mapping updates. This *should not* be used for document parsing.
* doing so will result in an exception.

18
server/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java
View File

@ -20,7 +20,6 @@
package org.elasticsearch.index.store;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.FileSwitchDirectory;
import org.apache.lucene.store.LockFactory;
import org.apache.lucene.store.MMapDirectory;
@ -77,10 +76,21 @@ public class FsDirectoryService extends DirectoryService {
}
protected Directory newFSDirectory(Path location, LockFactory lockFactory) throws IOException {
final String storeType = indexSettings.getSettings().get(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(),
IndexModule.Type.FS.getSettingsKey());
final String storeType =
indexSettings.getSettings().get(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), IndexModule.Type.FS.getSettingsKey());
if (IndexModule.Type.FS.match(storeType)) {
return FSDirectory.open(location, lockFactory); // use lucene defaults
final IndexModule.Type type =
IndexModule.defaultStoreType(IndexModule.NODE_STORE_ALLOW_MMAPFS.get(indexSettings.getNodeSettings()));
switch (type) {
case MMAPFS:
return new MMapDirectory(location, lockFactory);
case SIMPLEFS:
return new SimpleFSDirectory(location, lockFactory);
case NIOFS:
return new NIOFSDirectory(location, lockFactory);
default:
throw new AssertionError("unexpected built-in store type [" + type + "]");
}
} else if (IndexModule.Type.SIMPLEFS.match(storeType)) {
return new SimpleFSDirectory(location, lockFactory);
} else if (IndexModule.Type.NIOFS.match(storeType)) {

8
server/src/main/java/org/elasticsearch/indices/IndicesService.java
View File

@ -228,6 +228,14 @@ public class IndicesService extends AbstractLifecycleComponent
this.cacheCleaner = new CacheCleaner(indicesFieldDataCache, indicesRequestCache, logger, threadPool, this.cleanInterval);
this.metaStateService = metaStateService;
this.engineFactoryProviders = engineFactoryProviders;
// do not allow any plugin-provided index store type to conflict with a built-in type
for (final String indexStoreType : indexStoreFactories.keySet()) {
if (IndexModule.isBuiltinType(indexStoreType)) {
throw new IllegalStateException("registered index store type [" + indexStoreType + "] conflicts with a built-in type");
}
}
this.indexStoreFactories = indexStoreFactories;
}

27
server/src/test/java/org/elasticsearch/bootstrap/BootstrapChecksTests.java
View File

@ -52,7 +52,7 @@ import static org.mockito.Mockito.when;
public class BootstrapChecksTests extends ESTestCase {
private static final BootstrapContext defaultContext = new BootstrapContext(Settings.EMPTY, MetaData.EMPTY_META_DATA);
static final BootstrapContext defaultContext = new BootstrapContext(Settings.EMPTY, MetaData.EMPTY_META_DATA);
public void testNonProductionMode() throws NodeValidationException {
// nothing should happen since we are in non-production mode
@ -356,31 +356,6 @@ public class BootstrapChecksTests extends ESTestCase {
BootstrapChecks.check(defaultContext, true, Collections.singletonList(check));
}
public void testMaxMapCountCheck() throws NodeValidationException {
final int limit = 1 << 18;
final AtomicLong maxMapCount = new AtomicLong(randomIntBetween(1, limit - 1));
final BootstrapChecks.MaxMapCountCheck check = new BootstrapChecks.MaxMapCountCheck() {
@Override
long getMaxMapCount() {
return maxMapCount.get();
}
};
final NodeValidationException e = expectThrows(
NodeValidationException.class,
() -> BootstrapChecks.check(defaultContext, true, Collections.singletonList(check)));
assertThat(e.getMessage(), containsString("max virtual memory areas vm.max_map_count"));
maxMapCount.set(randomIntBetween(limit + 1, Integer.MAX_VALUE));
BootstrapChecks.check(defaultContext, true, Collections.singletonList(check));
// nothing should happen if current vm.max_map_count is not
// available
maxMapCount.set(-1);
BootstrapChecks.check(defaultContext, true, Collections.singletonList(check));
}
public void testClientJvmCheck() throws NodeValidationException {
final AtomicReference<String> vmName = new AtomicReference<>("Java HotSpot(TM) 32-Bit Client VM");
final BootstrapCheck check = new BootstrapChecks.ClientJvmCheck() {

67
server/src/test/java/org/elasticsearch/bootstrap/MaxMapCountCheckTests.java
View File

@ -24,16 +24,21 @@ import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.core.LogEvent;
import org.apache.logging.log4j.message.ParameterizedMessage;
import org.apache.lucene.util.Constants;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.common.io.PathUtils;
import org.elasticsearch.common.logging.ESLoggerFactory;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.test.MockLogAppender;
import java.io.BufferedReader;
import java.io.IOException;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Predicate;
import static org.hamcrest.CoreMatchers.equalTo;
@ -45,6 +50,66 @@ import static org.mockito.Mockito.when;
public class MaxMapCountCheckTests extends ESTestCase {
// initialize as if the max map count is under the limit, tests can override by setting maxMapCount before executing the check
private final AtomicLong maxMapCount = new AtomicLong(randomIntBetween(1, Math.toIntExact(BootstrapChecks.MaxMapCountCheck.LIMIT) - 1));
private final BootstrapChecks.MaxMapCountCheck check = new BootstrapChecks.MaxMapCountCheck() {
@Override
long getMaxMapCount() {
return maxMapCount.get();
}
};
private void assertFailure(final BootstrapCheck.BootstrapCheckResult result) {
assertTrue(result.isFailure());
assertThat(
result.getMessage(),
equalTo(
"max virtual memory areas vm.max_map_count [" + maxMapCount.get() + "] is too low, "
+ "increase to at least [" + BootstrapChecks.MaxMapCountCheck.LIMIT + "]"));
}
public void testMaxMapCountCheckBelowLimit() {
assertFailure(check.check(BootstrapChecksTests.defaultContext));
}
public void testMaxMapCountCheckBelowLimitAndMemoryMapAllowed() {
/*
* There are two ways that memory maps are allowed:
* - by default
* - mmapfs is explicitly allowed
* We want to test that if mmapfs is allowed then the max map count check is enforced.
*/
final List<Settings> settingsThatAllowMemoryMap = new ArrayList<>();
settingsThatAllowMemoryMap.add(Settings.EMPTY);
settingsThatAllowMemoryMap.add(Settings.builder().put("node.store.allow_mmapfs", true).build());
for (final Settings settingThatAllowsMemoryMap : settingsThatAllowMemoryMap) {
assertFailure(check.check(new BootstrapContext(settingThatAllowsMemoryMap, MetaData.EMPTY_META_DATA)));
}
}
public void testMaxMapCountCheckNotEnforcedIfMemoryMapNotAllowed() {
// nothing should happen if current vm.max_map_count is under the limit but mmapfs is not allowed
final Settings settings = Settings.builder().put("node.store.allow_mmapfs", false).build();
final BootstrapContext context = new BootstrapContext(settings, MetaData.EMPTY_META_DATA);
final BootstrapCheck.BootstrapCheckResult result = check.check(context);
assertTrue(result.isSuccess());
}
public void testMaxMapCountCheckAboveLimit() {
// nothing should happen if current vm.max_map_count exceeds the limit
maxMapCount.set(randomIntBetween(Math.toIntExact(BootstrapChecks.MaxMapCountCheck.LIMIT) + 1, Integer.MAX_VALUE));
final BootstrapCheck.BootstrapCheckResult result = check.check(BootstrapChecksTests.defaultContext);
assertTrue(result.isSuccess());
}
public void testMaxMapCountCheckMaxMapCountNotAvailable() {
// nothing should happen if current vm.max_map_count is not available
maxMapCount.set(-1);
final BootstrapCheck.BootstrapCheckResult result = check.check(BootstrapChecksTests.defaultContext);
assertTrue(result.isSuccess());
}
public void testGetMaxMapCountOnLinux() {
if (Constants.LINUX) {
final BootstrapChecks.MaxMapCountCheck check = new BootstrapChecks.MaxMapCountCheck();
@ -142,7 +207,7 @@ public class MaxMapCountCheckTests extends ESTestCase {
}
@Override
public void match(LogEvent event) {
public void match(final LogEvent event) {
if (event.getLevel().equals(level) &&
event.getLoggerName().equals(loggerName) &&
event.getMessage() instanceof ParameterizedMessage) {

17
server/src/test/java/org/elasticsearch/index/IndexModuleTests.java
View File

@ -87,6 +87,8 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Function;
import static java.util.Collections.emptyMap;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.hasToString;
import static org.hamcrest.Matchers.instanceOf;
public class IndexModuleTests extends ESTestCase {
@ -376,6 +378,21 @@ public class IndexModuleTests extends ESTestCase {
indexService.close("simon says", false);
}
public void testMmapfsStoreTypeNotAllowed() {
final Settings settings = Settings.builder()
.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
.put("index.store.type", "mmapfs")
.build();
final Settings nodeSettings = Settings.builder()
.put(IndexModule.NODE_STORE_ALLOW_MMAPFS.getKey(), false)
.build();
final IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(new Index("foo", "_na_"), settings, nodeSettings);
final IndexModule module =
new IndexModule(indexSettings, emptyAnalysisRegistry, new InternalEngineFactory(), Collections.emptyMap());
final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> newIndexService(module));
assertThat(e, hasToString(containsString("store type [mmapfs] is not allowed")));
}
class CustomQueryCache implements QueryCache {
@Override

27
server/src/test/java/org/elasticsearch/plugins/IndexStorePluginTests.java
View File

@ -21,6 +21,7 @@ package org.elasticsearch.plugins;
import org.elasticsearch.bootstrap.JavaVersion;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.IndexModule;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.store.IndexStore;
import org.elasticsearch.node.MockNode;
@ -32,6 +33,7 @@ import java.util.Map;
import java.util.function.Function;
import static org.elasticsearch.test.hamcrest.RegexMatcher.matches;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.hasToString;
public class IndexStorePluginTests extends ESTestCase {
@ -54,7 +56,30 @@ public class IndexStorePluginTests extends ESTestCase {
}
public void testDuplicateIndexStoreProviders() {
public static class ConflictingStorePlugin extends Plugin implements IndexStorePlugin {
public static final String TYPE;
static {
TYPE = randomFrom(Arrays.asList(IndexModule.Type.values())).getSettingsKey();
}
@Override
public Map<String, Function<IndexSettings, IndexStore>> getIndexStoreFactories() {
return Collections.singletonMap(TYPE, IndexStore::new);
}
}
public void testIndexStoreFactoryConflictsWithBuiltInIndexStoreType() {
final Settings settings = Settings.builder().put("path.home", createTempDir()).build();
final IllegalStateException e = expectThrows(
IllegalStateException.class, () -> new MockNode(settings, Collections.singletonList(ConflictingStorePlugin.class)));
assertThat(e, hasToString(containsString(
"registered index store type [" + ConflictingStorePlugin.TYPE + "] conflicts with a built-in type")));
}
public void testDuplicateIndexStoreFactories() {
final Settings settings = Settings.builder().put("path.home", createTempDir()).build();
final IllegalStateException e = expectThrows(
IllegalStateException.class, () -> new MockNode(settings, Arrays.asList(BarStorePlugin.class, FooStorePlugin.class)));