honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #18756 from jasontedor/on-out-of-memory-error 

Bootstrap check for OnOutOfMemoryError and seccomp
This commit is contained in:
Jason Tedor 2016-06-07 09:26:57 -04:00
parent c72ebba5de e94408c0d2
commit 75d3b13790
4 changed files with 280 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

87
core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java
View File

@ -106,7 +106,7 @@ final class BootstrapCheck {
for (final Check check : checks) {
if (check.check()) {
if (!enforceLimits || (check.isSystemCheck() && ignoreSystemChecks)) {
if ((!enforceLimits || (check.isSystemCheck() && ignoreSystemChecks)) && !check.alwaysEnforce()) {
ignoredErrors.add(check.errorMessage());
} else {
errors.add(check.errorMessage());
@ -164,6 +164,8 @@ final class BootstrapCheck {
checks.add(new MaxMapCountCheck());
}
checks.add(new ClientJvmCheck());
checks.add(new OnErrorCheck());
checks.add(new OnOutOfMemoryErrorCheck());
return Collections.unmodifiableList(checks);
}
@ -194,6 +196,10 @@ final class BootstrapCheck {
*/
boolean isSystemCheck();
default boolean alwaysEnforce() {
return false;
}
}
static class HeapSizeCheck implements BootstrapCheck.Check {
@ -245,7 +251,6 @@ final class BootstrapCheck {
}
// visible for testing
static class FileDescriptorCheck implements Check {
private final int limit;
@ -288,7 +293,6 @@ final class BootstrapCheck {
}
// visible for testing
static class MlockallCheck implements Check {
private final boolean mlockallSet;
@ -504,4 +508,81 @@ final class BootstrapCheck {
}
static abstract class MightForkCheck implements BootstrapCheck.Check {
@Override
public boolean check() {
return isSeccompInstalled() && mightFork();
}
// visible for testing
boolean isSeccompInstalled() {
return Natives.isSeccompInstalled();
}
// visible for testing
abstract boolean mightFork();
@Override
public final boolean isSystemCheck() {
return false;
}
@Override
public final boolean alwaysEnforce() {
return true;
}
}
static class OnErrorCheck extends MightForkCheck {
@Override
boolean mightFork() {
final String onError = onError();
return onError != null && !onError.equals("");
}
// visible for testing
String onError() {
return JvmInfo.jvmInfo().onError();
}
@Override
public String errorMessage() {
return String.format(
Locale.ROOT,
"OnError [%s] requires forking but is prevented by system call filters ([%s=true]);" +
" upgrade to at least Java 8u92 and use ExitOnOutOfMemoryError",
onError(),
BootstrapSettings.SECCOMP_SETTING.getKey());
}
}
static class OnOutOfMemoryErrorCheck extends MightForkCheck {
@Override
boolean mightFork() {
final String onOutOfMemoryError = onOutOfMemoryError();
return onOutOfMemoryError != null && !onOutOfMemoryError.equals("");
}
// visible for testing
String onOutOfMemoryError() {
return JvmInfo.jvmInfo().onOutOfMemoryError();
}
@Override
public String errorMessage() {
return String.format(
Locale.ROOT,
"OnOutOfMemoryError [%s] requires forking but is prevented by system call filters ([%s=true]);" +
" upgrade to at least Java 8u92 and use ExitOnOutOfMemoryError",
onOutOfMemoryError(),
BootstrapSettings.SECCOMP_SETTING.getKey());
}
}
}

24
core/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java
View File

@ -115,6 +115,18 @@ public class JvmInfo implements Streamable, ToXContent {
Method vmOptionMethod = clazz.getMethod("getVMOption", String.class);
Method valueMethod = vmOptionClazz.getMethod("getValue");
try {
Object onError = vmOptionMethod.invoke(hotSpotDiagnosticMXBean, "OnError");
info.onError = (String) valueMethod.invoke(onError);
} catch (Exception ignored) {
}
try {
Object onOutOfMemoryError = vmOptionMethod.invoke(hotSpotDiagnosticMXBean, "OnOutOfMemoryError");
info.onOutOfMemoryError = (String) valueMethod.invoke(onOutOfMemoryError);
} catch (Exception ignored) {
}
try {
Object useCompressedOopsVmOption = vmOptionMethod.invoke(hotSpotDiagnosticMXBean, "UseCompressedOops");
info.useCompressedOops = (String) valueMethod.invoke(useCompressedOopsVmOption);
@ -179,6 +191,10 @@ public class JvmInfo implements Streamable, ToXContent {
String[] gcCollectors = Strings.EMPTY_ARRAY;
String[] memoryPools = Strings.EMPTY_ARRAY;
private String onError;
private String onOutOfMemoryError;
private String useCompressedOops = "unknown";
private String useG1GC = "unknown";
@ -314,6 +330,14 @@ public class JvmInfo implements Streamable, ToXContent {
return configuredMaxHeapSize;
}
public String onError() {
return onError;
}
public String onOutOfMemoryError() {
return onOutOfMemoryError;
}
/**
* The value of the JVM flag UseCompressedOops, if available otherwise
* "unknown". The value "unknown" indicates that an attempt was

154
core/src/test/java/org/elasticsearch/bootstrap/BootstrapCheckTests.java
View File

@ -30,8 +30,10 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Consumer;
import static org.hamcrest.CoreMatchers.allOf;
import static org.hamcrest.CoreMatchers.containsString;
@ -383,7 +385,7 @@ public class BootstrapCheckTests extends ESTestCase {
}
};
RuntimeException e = expectThrows(
final RuntimeException e = expectThrows(
RuntimeException.class,
() -> BootstrapCheck.check(true, false, Collections.singletonList(check), "testClientJvmCheck"));
assertThat(
@ -395,8 +397,127 @@ public class BootstrapCheckTests extends ESTestCase {
BootstrapCheck.check(true, false, Collections.singletonList(check), "testClientJvmCheck");
}
public void testMightForkCheck() {
final AtomicBoolean isSeccompInstalled = new AtomicBoolean();
final AtomicBoolean mightFork = new AtomicBoolean();
final BootstrapCheck.MightForkCheck check = new BootstrapCheck.MightForkCheck() {
@Override
boolean isSeccompInstalled() {
return isSeccompInstalled.get();
}
@Override
boolean mightFork() {
return mightFork.get();
}
@Override
public String errorMessage() {
return "error";
}
};
runMightForkTest(
check,
isSeccompInstalled,
() -> mightFork.set(false),
() -> mightFork.set(true),
e -> assertThat(e.getMessage(), containsString("error")));
}
public void testOnErrorCheck() {
final AtomicBoolean isSeccompInstalled = new AtomicBoolean();
final AtomicReference<String> onError = new AtomicReference<>();
final BootstrapCheck.MightForkCheck check = new BootstrapCheck.OnErrorCheck() {
@Override
boolean isSeccompInstalled() {
return isSeccompInstalled.get();
}
@Override
String onError() {
return onError.get();
}
};
final String command = randomAsciiOfLength(16);
runMightForkTest(
check,
isSeccompInstalled,
() -> onError.set(randomBoolean() ? "" : null),
() -> onError.set(command),
e -> assertThat(
e.getMessage(),
containsString(
"OnError [" + command + "] requires forking but is prevented by system call filters ([bootstrap.seccomp=true]);"
+ " upgrade to at least Java 8u92 and use ExitOnOutOfMemoryError")));
}
public void testOnOutOfMemoryErrorCheck() {
final AtomicBoolean isSeccompInstalled = new AtomicBoolean();
final AtomicReference<String> onOutOfMemoryError = new AtomicReference<>();
final BootstrapCheck.MightForkCheck check = new BootstrapCheck.OnOutOfMemoryErrorCheck() {
@Override
boolean isSeccompInstalled() {
return isSeccompInstalled.get();
}
@Override
String onOutOfMemoryError() {
return onOutOfMemoryError.get();
}
};
final String command = randomAsciiOfLength(16);
runMightForkTest(
check,
isSeccompInstalled,
() -> onOutOfMemoryError.set(randomBoolean() ? "" : null),
() -> onOutOfMemoryError.set(command),
e -> assertThat(
e.getMessage(),
containsString(
"OnOutOfMemoryError [" + command + "]"
+ " requires forking but is prevented by system call filters ([bootstrap.seccomp=true]);"
+ " upgrade to at least Java 8u92 and use ExitOnOutOfMemoryError")));
}
private void runMightForkTest(
final BootstrapCheck.MightForkCheck check,
final AtomicBoolean isSeccompInstalled,
final Runnable disableMightFork,
final Runnable enableMightFork,
final Consumer<RuntimeException> consumer) {
// if seccomp is disabled, nothing should happen
isSeccompInstalled.set(false);
if (randomBoolean()) {
disableMightFork.run();
} else {
enableMightFork.run();
}
BootstrapCheck.check(true, randomBoolean(), Collections.singletonList(check), "testMightFork");
// if seccomp is enabled, but we will not fork, nothing should
// happen
isSeccompInstalled.set(true);
disableMightFork.run();
BootstrapCheck.check(true, randomBoolean(), Collections.singletonList(check), "testMightFork");
// if seccomp is enabled, and we might fork, the check should
// be enforced, regardless of bootstrap checks being enabled or
// not
isSeccompInstalled.set(true);
enableMightFork.run();
final RuntimeException e = expectThrows(
RuntimeException.class,
() -> BootstrapCheck.check(randomBoolean(), randomBoolean(), Collections.singletonList(check), "testMightFork"));
consumer.accept(e);
}
public void testIgnoringSystemChecks() {
BootstrapCheck.Check check = new BootstrapCheck.Check() {
final BootstrapCheck.Check check = new BootstrapCheck.Check() {
@Override
public boolean check() {
return true;
@ -430,4 +551,33 @@ public class BootstrapCheckTests extends ESTestCase {
verify(logger).warn("error");
}
public void testAlwaysEnforcedChecks() {
final BootstrapCheck.Check check = new BootstrapCheck.Check() {
@Override
public boolean check() {
return true;
}
@Override
public String errorMessage() {
return "error";
}
@Override
public boolean isSystemCheck() {
return randomBoolean();
}
@Override
public boolean alwaysEnforce() {
return true;
}
};
final RuntimeException alwaysEnforced = expectThrows(
RuntimeException.class,
() -> BootstrapCheck.check(randomBoolean(), randomBoolean(), Collections.singletonList(check), "testAlwaysEnforcedChecks"));
assertThat(alwaysEnforced, hasToString(containsString("error")));
}
}

20
docs/reference/setup/bootstrap-checks.asciidoc
View File

@ -16,6 +16,10 @@ checks that fail appear as warnings in the Elasticsearch log. If
Elasticsearch is in production mode, any bootstrap checks that fail will
cause Elasticsearch to refuse to start.
There are some bootstrap checks that are always enforced to prevent
Elasticsearch from running with incompatible settings. These checks are
documented individually.
[float]
=== Development vs. production mode
@ -152,3 +156,19 @@ JVM check, you must start Elasticsearch with the server VM. On modern
systems and operating systems, the server VM is the
default. Additionally, Elasticsearch is configured by default to force
the server VM.
=== OnError and OnOutOfMemoryError checks
The JVM options `OnError` and `OnOutOfMemoryError` enable executing
arbitrary commands if the JVM encounters a fatal error (`OnError`) or an
`OutOfMemoryError` (`OnOutOfMemoryError`). However, by default,
Elasticsearch system call filters (seccomp) are enabled and these
filters prevent forking. Thus, using `OnError` or `OnOutOfMemoryError`
and system call filters are incompatible. The `OnError` and
`OnOutOfMemoryError` checks prevent Elasticsearch from starting if
either of these JVM options are used and system call filters are
enabled. This check is always enforced. To pass this check do not enable
`OnError` nor `OnOutOfMemoryError`; instead, upgrade to Java 8u92 and
use the JVM flag `ExitOnOutOfMemoryError`. While this does not have the
full capabilities of `OnError` nor `OnOutOfMemoryError`, arbitrary
forking will not be supported with seccomp enabled.