honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add max map count check 

This commit adds a bootstrap check on Linux for the max map count (max
virtual memory areas) available to the Elasticsearch process.

Relates #16944
This commit is contained in:
Jason Tedor 2016-04-14 21:04:03 -04:00
parent f030796b0b
commit f985cfc95a
4 changed files with 195 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

73
core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java
View File

@ -20,6 +20,8 @@
package org.elasticsearch.bootstrap;
import org.apache.lucene.util.Constants;
import org.apache.lucene.util.SuppressForbidden;
import org.elasticsearch.common.io.PathUtils;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.settings.Settings;
@ -30,6 +32,10 @@ import org.elasticsearch.monitor.jvm.JvmInfo;
import org.elasticsearch.monitor.process.ProcessProbe;
import org.elasticsearch.node.Node;
import java.io.BufferedReader;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@ -42,7 +48,6 @@ import java.util.stream.Collectors;
* and all production limit checks must pass. This should be extended as we go to settings like:
* - discovery.zen.ping.unicast.hosts is set if we use zen disco
* - ensure we can write in all data directories
* - fail if vm.max_map_count is under a certain limit (not sure if this works cross platform)
* - fail if the default cluster.name is used, if this is setup on network a real clustername should be used?
*/
final class BootstrapCheck {
@ -121,6 +126,9 @@ final class BootstrapCheck {
checks.add(new MaxSizeVirtualMemoryCheck());
}
checks.add(new MinMasterNodesCheck(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.exists(settings)));
if (Constants.LINUX) {
checks.add(new MaxMapCountCheck());
}
return Collections.unmodifiableList(checks);
}
@ -327,4 +335,67 @@ final class BootstrapCheck {
}
static class MaxMapCountCheck implements Check {
private final long limit = 1 << 18;
@Override
public boolean check() {
return getMaxMapCount() != -1 && getMaxMapCount() < limit;
}
@Override
public String errorMessage() {
return String.format(
Locale.ROOT,
"max virtual memory areas vm.max_map_count [%d] likely too low, increase to at least [%d]",
getMaxMapCount(),
limit);
}
// visible for testing
long getMaxMapCount() {
return getMaxMapCount(Loggers.getLogger(BootstrapCheck.class));
}
// visible for testing
long getMaxMapCount(ESLogger logger) {
final Path path = getProcSysVmMaxMapCountPath();
try (final BufferedReader bufferedReader = getBufferedReader(path)) {
final String rawProcSysVmMaxMapCount = readProcSysVmMaxMapCount(bufferedReader);
if (rawProcSysVmMaxMapCount != null) {
try {
return parseProcSysVmMaxMapCount(rawProcSysVmMaxMapCount);
} catch (final NumberFormatException e) {
logger.warn("unable to parse vm.max_map_count [{}]", e, rawProcSysVmMaxMapCount);
}
}
} catch (final IOException e) {
logger.warn("I/O exception while trying to read [{}]", e, path);
}
return -1;
}
@SuppressForbidden(reason = "access /proc/sys/vm/max_map_count")
private Path getProcSysVmMaxMapCountPath() {
return PathUtils.get("/proc/sys/vm/max_map_count");
}
// visible for testing
BufferedReader getBufferedReader(final Path path) throws IOException {
return Files.newBufferedReader(path);
}
// visible for testing
String readProcSysVmMaxMapCount(final BufferedReader bufferedReader) throws IOException {
return bufferedReader.readLine();
}
// visible for testing
long parseProcSysVmMaxMapCount(final String procSysVmMaxMapCount) throws NumberFormatException {
return Long.parseLong(procSysVmMaxMapCount);
}
}
}

3
core/src/main/resources/org/elasticsearch/bootstrap/security.policy
View File

@ -121,4 +121,7 @@ grant {
// load averages on FreeBSD
permission java.io.FilePermission "/compat/linux/proc/loadavg", "read";
// read max virtual memory areas
permission java.io.FilePermission "/proc/sys/vm/max_map_count", "read";
};

25
core/src/test/java/org/elasticsearch/bootstrap/BootstrapCheckTests.java
View File

@ -322,6 +322,31 @@ public class BootstrapCheckTests extends ESTestCase {
BootstrapCheck.check(true, Collections.singletonList(check), "testMaxSizeVirtualMemory");
}
public void testMaxMapCountCheck() {
final int limit = 1 << 18;
final AtomicLong maxMapCount = new AtomicLong(randomIntBetween(1, limit - 1));
final BootstrapCheck.MaxMapCountCheck check = new BootstrapCheck.MaxMapCountCheck() {
@Override
long getMaxMapCount() {
return maxMapCount.get();
}
};
RuntimeException e = expectThrows(
RuntimeException.class,
() -> BootstrapCheck.check(true, Collections.singletonList(check), "testMaxMapCountCheck"));
assertThat(e.getMessage(), containsString("max virtual memory areas vm.max_map_count"));
maxMapCount.set(randomIntBetween(limit + 1, Integer.MAX_VALUE));
BootstrapCheck.check(true, Collections.singletonList(check), "testMaxMapCountCheck");
// nothing should happen if current vm.max_map_count is not
// available
maxMapCount.set(-1);
BootstrapCheck.check(true, Collections.singletonList(check), "testMaxMapCountCheck");
}
public void testMinMasterNodes() {
boolean isSet = randomBoolean();
BootstrapCheck.Check check = new BootstrapCheck.MinMasterNodesCheck(isSet);

95
core/src/test/java/org/elasticsearch/bootstrap/MaxMapCountCheckTests.java Normal file
View File

@ -0,0 +1,95 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.bootstrap;
import org.apache.lucene.util.Constants;
import org.elasticsearch.common.io.PathUtils;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.test.ESTestCase;
import java.io.BufferedReader;
import java.io.IOException;
import java.nio.file.Path;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.reset;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
public class MaxMapCountCheckTests extends ESTestCase {
public void testGetMaxMapCountOnLinux() {
if (Constants.LINUX) {
final BootstrapCheck.MaxMapCountCheck check = new BootstrapCheck.MaxMapCountCheck();
assertThat(check.getMaxMapCount(), greaterThan(0L));
}
}
public void testGetMaxMapCount() throws IOException {
final long procSysVmMaxMapCount = randomIntBetween(1, Integer.MAX_VALUE);
final BufferedReader reader = mock(BufferedReader.class);
when(reader.readLine()).thenReturn(Long.toString(procSysVmMaxMapCount));
final Path procSysVmMaxMapCountPath = PathUtils.get("/proc/sys/vm/max_map_count");
BootstrapCheck.MaxMapCountCheck check = new BootstrapCheck.MaxMapCountCheck() {
@Override
BufferedReader getBufferedReader(Path path) throws IOException {
assertEquals(path, procSysVmMaxMapCountPath);
return reader;
}
};
assertThat(check.getMaxMapCount(), equalTo(procSysVmMaxMapCount));
verify(reader).close();
reset(reader);
final IOException ioException = new IOException("fatal");
when(reader.readLine()).thenThrow(ioException);
final ESLogger logger = mock(ESLogger.class);
assertThat(check.getMaxMapCount(logger), equalTo(-1L));
verify(logger).warn("I/O exception while trying to read [{}]", ioException, procSysVmMaxMapCountPath);
verify(reader).close();
reset(reader);
reset(logger);
when(reader.readLine()).thenReturn("eof");
assertThat(check.getMaxMapCount(logger), equalTo(-1L));
verify(logger).warn(eq("unable to parse vm.max_map_count [{}]"), any(NumberFormatException.class), eq("eof"));
verify(reader).close();
}
public void testMaxMapCountCheckRead() throws IOException {
final String rawProcSysVmMaxMapCount = Long.toString(randomIntBetween(1, Integer.MAX_VALUE));
final BufferedReader reader = mock(BufferedReader.class);
when(reader.readLine()).thenReturn(rawProcSysVmMaxMapCount);
final BootstrapCheck.MaxMapCountCheck check = new BootstrapCheck.MaxMapCountCheck();
assertThat(check.readProcSysVmMaxMapCount(reader), equalTo(rawProcSysVmMaxMapCount));
}
public void testMaxMapCountCheckParse() {
final long procSysVmMaxMapCount = randomIntBetween(1, Integer.MAX_VALUE);
final BootstrapCheck.MaxMapCountCheck check = new BootstrapCheck.MaxMapCountCheck();
assertThat(check.parseProcSysVmMaxMapCount(Long.toString(procSysVmMaxMapCount)), equalTo(procSysVmMaxMapCount));
}
}