From 9695caa3fb4a07061fcd825b6397e464fce71d37 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Sun, 5 Jun 2016 23:46:59 -0400 Subject: [PATCH 1/2] Bootstrap check for OnOutOfMemoryError and seccomp This commit adds a bootstrap check for the JVM option OnOutOfMemoryError being in use and seccomp being enabled. These two options are incompatible because OnOutOfMemoryError allows the user to specify an arbitrary program to fork when the JVM encounters an OutOfMemoryError, and seccomp enables system call filters that prevents forking. This commit also adds support for bootstrap checks that are always enforced, whether or not Elasticsearch is in production mode. --- .../bootstrap/BootstrapCheck.java | 49 +++++++++++- .../elasticsearch/monitor/jvm/JvmInfo.java | 12 +++ .../bootstrap/BootstrapCheckTests.java | 76 ++++++++++++++++++- .../reference/setup/bootstrap-checks.asciidoc | 16 ++++ 4 files changed, 148 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java index 51ddb752b44..f5b28b19b60 100644 --- a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java +++ b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java @@ -106,7 +106,7 @@ final class BootstrapCheck { for (final Check check : checks) { if (check.check()) { - if (!enforceLimits || (check.isSystemCheck() && ignoreSystemChecks)) { + if ((!enforceLimits || (check.isSystemCheck() && ignoreSystemChecks)) && !check.alwaysEnforce()) { ignoredErrors.add(check.errorMessage()); } else { errors.add(check.errorMessage()); @@ -164,6 +164,7 @@ final class BootstrapCheck { checks.add(new MaxMapCountCheck()); } checks.add(new ClientJvmCheck()); + checks.add(new OnOutOfMemoryErrorCheck()); return Collections.unmodifiableList(checks); } @@ -194,6 +195,10 @@ final class BootstrapCheck { */ boolean isSystemCheck(); + default boolean alwaysEnforce() { + return false; + } + } static class HeapSizeCheck implements BootstrapCheck.Check { @@ -245,7 +250,6 @@ final class BootstrapCheck { } - // visible for testing static class FileDescriptorCheck implements Check { private final int limit; @@ -288,7 +292,6 @@ final class BootstrapCheck { } - // visible for testing static class MlockallCheck implements Check { private final boolean mlockallSet; @@ -504,4 +507,44 @@ final class BootstrapCheck { } + static class OnOutOfMemoryErrorCheck implements BootstrapCheck.Check { + + @Override + public boolean check() { + final String onOutOfMemoryError = onOutOfMemoryError(); + return isSeccompInstalled() && onOutOfMemoryError != null && !onOutOfMemoryError.equals(""); + } + + // visible for testing + boolean isSeccompInstalled() { + return Natives.isSeccompInstalled(); + } + + // visible for testing + String onOutOfMemoryError() { + return JvmInfo.jvmInfo().onOutOfMemoryError(); + } + + @Override + public String errorMessage() { + return String.format( + Locale.ROOT, + "OnOutOfMemoryError [%s] requires forking but is prevented by system call filters ([%s=true]);" + + " upgrade to at least Java 8u92 and use ExitOnOutOfMemoryError", + onOutOfMemoryError(), + BootstrapSettings.SECCOMP_SETTING.getKey()); + } + + @Override + public boolean isSystemCheck() { + return false; + } + + @Override + public boolean alwaysEnforce() { + return true; + } + + } + } diff --git a/core/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java b/core/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java index 3e39edaa47c..b2e1990a680 100644 --- a/core/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java +++ b/core/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java @@ -115,6 +115,12 @@ public class JvmInfo implements Streamable, ToXContent { Method vmOptionMethod = clazz.getMethod("getVMOption", String.class); Method valueMethod = vmOptionClazz.getMethod("getValue"); + try { + Object onOutOfMemoryError = vmOptionMethod.invoke(hotSpotDiagnosticMXBean, "OnOutOfMemoryError"); + info.onOutOfMemoryError = (String) valueMethod.invoke(onOutOfMemoryError); + } catch (Exception ignored) { + } + try { Object useCompressedOopsVmOption = vmOptionMethod.invoke(hotSpotDiagnosticMXBean, "UseCompressedOops"); info.useCompressedOops = (String) valueMethod.invoke(useCompressedOopsVmOption); @@ -179,6 +185,8 @@ public class JvmInfo implements Streamable, ToXContent { String[] gcCollectors = Strings.EMPTY_ARRAY; String[] memoryPools = Strings.EMPTY_ARRAY; + private String onOutOfMemoryError; + private String useCompressedOops = "unknown"; private String useG1GC = "unknown"; @@ -314,6 +322,10 @@ public class JvmInfo implements Streamable, ToXContent { return configuredMaxHeapSize; } + public String onOutOfMemoryError() { + return onOutOfMemoryError; + } + /** * The value of the JVM flag UseCompressedOops, if available otherwise * "unknown". The value "unknown" indicates that an attempt was diff --git a/core/src/test/java/org/elasticsearch/bootstrap/BootstrapCheckTests.java b/core/src/test/java/org/elasticsearch/bootstrap/BootstrapCheckTests.java index d354adc7544..41609f7fc5d 100644 --- a/core/src/test/java/org/elasticsearch/bootstrap/BootstrapCheckTests.java +++ b/core/src/test/java/org/elasticsearch/bootstrap/BootstrapCheckTests.java @@ -30,6 +30,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; @@ -383,7 +384,7 @@ public class BootstrapCheckTests extends ESTestCase { } }; - RuntimeException e = expectThrows( + final RuntimeException e = expectThrows( RuntimeException.class, () -> BootstrapCheck.check(true, false, Collections.singletonList(check), "testClientJvmCheck")); assertThat( @@ -395,8 +396,50 @@ public class BootstrapCheckTests extends ESTestCase { BootstrapCheck.check(true, false, Collections.singletonList(check), "testClientJvmCheck"); } + public void testOnOutOfMemoryErrorCheck() { + final AtomicBoolean isSeccompInstalled = new AtomicBoolean(); + final AtomicReference onOutOfMemoryError = new AtomicReference<>(); + final BootstrapCheck.Check check = new BootstrapCheck.OnOutOfMemoryErrorCheck() { + @Override + boolean isSeccompInstalled() { + return isSeccompInstalled.get(); + } + + @Override + String onOutOfMemoryError() { + return onOutOfMemoryError.get(); + } + }; + + // if seccomp is disabled, nothing should happen + isSeccompInstalled.set(false); + onOutOfMemoryError.set(randomBoolean() ? "" : randomAsciiOfLength(16)); + BootstrapCheck.check(true, false, Collections.singletonList(check), "testOnOutOfMemoryErrorCheck"); + + // if seccomp is enabled, and OnOutOfMemoryError is not, nothing + // should happen + isSeccompInstalled.set(true); + onOutOfMemoryError.set(""); + BootstrapCheck.check(true, false, Collections.singletonList(check), "testOnOutOfMemoryErrorCheck"); + + // if seccomp is enabled, and OnOutOfMemoryError is, the check + // should be enforced, regardless of bootstrap checks being + // enabled or not + isSeccompInstalled.set(true); + final String command = randomAsciiOfLength(16); + onOutOfMemoryError.set(command); + final RuntimeException e = expectThrows( + RuntimeException.class, + () -> BootstrapCheck.check(randomBoolean(), randomBoolean(), Collections.singletonList(check), "testOnOutOfMemoryErrorCheck")); + assertThat( + e.getMessage(), + containsString( + "OnOutOfMemoryError [" + command + "] requires forking but is prevented by system call filters ([bootstrap.seccomp=true]);" + + " upgrade to at least Java 8u92 and use ExitOnOutOfMemoryError")); + } + public void testIgnoringSystemChecks() { - BootstrapCheck.Check check = new BootstrapCheck.Check() { + final BootstrapCheck.Check check = new BootstrapCheck.Check() { @Override public boolean check() { return true; @@ -430,4 +473,33 @@ public class BootstrapCheckTests extends ESTestCase { verify(logger).warn("error"); } + public void testAlwaysEnforcedChecks() { + final BootstrapCheck.Check check = new BootstrapCheck.Check() { + @Override + public boolean check() { + return true; + } + + @Override + public String errorMessage() { + return "error"; + } + + @Override + public boolean isSystemCheck() { + return randomBoolean(); + } + + @Override + public boolean alwaysEnforce() { + return true; + } + }; + + final RuntimeException alwaysEnforced = expectThrows( + RuntimeException.class, + () -> BootstrapCheck.check(randomBoolean(), randomBoolean(), Collections.singletonList(check), "testAlwaysEnforcedChecks")); + assertThat(alwaysEnforced, hasToString(containsString("error"))); + } + } diff --git a/docs/reference/setup/bootstrap-checks.asciidoc b/docs/reference/setup/bootstrap-checks.asciidoc index 2a5c55bf8b3..81d14f24c8f 100644 --- a/docs/reference/setup/bootstrap-checks.asciidoc +++ b/docs/reference/setup/bootstrap-checks.asciidoc @@ -16,6 +16,10 @@ checks that fail appear as warnings in the Elasticsearch log. If Elasticsearch is in production mode, any bootstrap checks that fail will cause Elasticsearch to refuse to start. +There are some bootstrap checks that are always enforced to prevent +Elasticsearch from running with incompatible settings. These checks are +documented individually. + [float] === Development vs. production mode @@ -152,3 +156,15 @@ JVM check, you must start Elasticsearch with the server VM. On modern systems and operating systems, the server VM is the default. Additionally, Elasticsearch is configured by default to force the server VM. + +=== OnOutOfMemoryError check + +The JVM option `OnOutOfMemoryError` enables executing an arbitrary +command if the JVM throws an `OutOfMemoryError`. However, by default, +Elasticsearch system call filters (seccomp) are enabled and these +filters prevent forking. Thus, using `OnOutOfMemoryError` and system +call filters are incompatible. The `OnOutOfMemoryError` check prevents +Elasticsearch from starting if this JVM option is used and system call +filters are enabled. This check is always enforced. To pass this check +do not enable `OnOutOfMemoryError`; instead, upgrade to Java 8u92 and +use the JVM flag `ExitOnOutOfMemoryError`. From e94408c0d2748ce8829fb5096559221763026889 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Mon, 6 Jun 2016 18:28:34 -0400 Subject: [PATCH 2/2] Bootstrap check for OnError and seccomp This commit adds a bootstrap check for the JVM option OnError being in use and seccomp being enabled. These two options are incompatible because OnError allows the user to specify an arbitrary program to fork when the JVM encounters an fatal error, and seccomp enables system call filters that prevents forking. --- .../bootstrap/BootstrapCheck.java | 64 ++++++++-- .../elasticsearch/monitor/jvm/JvmInfo.java | 12 ++ .../bootstrap/BootstrapCheckTests.java | 114 +++++++++++++++--- .../reference/setup/bootstrap-checks.asciidoc | 22 ++-- 4 files changed, 172 insertions(+), 40 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java index f5b28b19b60..fc36e4fb671 100644 --- a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java +++ b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java @@ -164,6 +164,7 @@ final class BootstrapCheck { checks.add(new MaxMapCountCheck()); } checks.add(new ClientJvmCheck()); + checks.add(new OnErrorCheck()); checks.add(new OnOutOfMemoryErrorCheck()); return Collections.unmodifiableList(checks); } @@ -507,12 +508,11 @@ final class BootstrapCheck { } - static class OnOutOfMemoryErrorCheck implements BootstrapCheck.Check { + static abstract class MightForkCheck implements BootstrapCheck.Check { @Override public boolean check() { - final String onOutOfMemoryError = onOutOfMemoryError(); - return isSeccompInstalled() && onOutOfMemoryError != null && !onOutOfMemoryError.equals(""); + return isSeccompInstalled() && mightFork(); } // visible for testing @@ -520,6 +520,54 @@ final class BootstrapCheck { return Natives.isSeccompInstalled(); } + // visible for testing + abstract boolean mightFork(); + + @Override + public final boolean isSystemCheck() { + return false; + } + + @Override + public final boolean alwaysEnforce() { + return true; + } + + } + + static class OnErrorCheck extends MightForkCheck { + + @Override + boolean mightFork() { + final String onError = onError(); + return onError != null && !onError.equals(""); + } + + // visible for testing + String onError() { + return JvmInfo.jvmInfo().onError(); + } + + @Override + public String errorMessage() { + return String.format( + Locale.ROOT, + "OnError [%s] requires forking but is prevented by system call filters ([%s=true]);" + + " upgrade to at least Java 8u92 and use ExitOnOutOfMemoryError", + onError(), + BootstrapSettings.SECCOMP_SETTING.getKey()); + } + + } + + static class OnOutOfMemoryErrorCheck extends MightForkCheck { + + @Override + boolean mightFork() { + final String onOutOfMemoryError = onOutOfMemoryError(); + return onOutOfMemoryError != null && !onOutOfMemoryError.equals(""); + } + // visible for testing String onOutOfMemoryError() { return JvmInfo.jvmInfo().onOutOfMemoryError(); @@ -535,16 +583,6 @@ final class BootstrapCheck { BootstrapSettings.SECCOMP_SETTING.getKey()); } - @Override - public boolean isSystemCheck() { - return false; - } - - @Override - public boolean alwaysEnforce() { - return true; - } - } } diff --git a/core/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java b/core/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java index b2e1990a680..69abd0752f6 100644 --- a/core/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java +++ b/core/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java @@ -115,6 +115,12 @@ public class JvmInfo implements Streamable, ToXContent { Method vmOptionMethod = clazz.getMethod("getVMOption", String.class); Method valueMethod = vmOptionClazz.getMethod("getValue"); + try { + Object onError = vmOptionMethod.invoke(hotSpotDiagnosticMXBean, "OnError"); + info.onError = (String) valueMethod.invoke(onError); + } catch (Exception ignored) { + } + try { Object onOutOfMemoryError = vmOptionMethod.invoke(hotSpotDiagnosticMXBean, "OnOutOfMemoryError"); info.onOutOfMemoryError = (String) valueMethod.invoke(onOutOfMemoryError); @@ -185,6 +191,8 @@ public class JvmInfo implements Streamable, ToXContent { String[] gcCollectors = Strings.EMPTY_ARRAY; String[] memoryPools = Strings.EMPTY_ARRAY; + private String onError; + private String onOutOfMemoryError; private String useCompressedOops = "unknown"; @@ -322,6 +330,10 @@ public class JvmInfo implements Streamable, ToXContent { return configuredMaxHeapSize; } + public String onError() { + return onError; + } + public String onOutOfMemoryError() { return onOutOfMemoryError; } diff --git a/core/src/test/java/org/elasticsearch/bootstrap/BootstrapCheckTests.java b/core/src/test/java/org/elasticsearch/bootstrap/BootstrapCheckTests.java index 41609f7fc5d..ddd4f01ef8b 100644 --- a/core/src/test/java/org/elasticsearch/bootstrap/BootstrapCheckTests.java +++ b/core/src/test/java/org/elasticsearch/bootstrap/BootstrapCheckTests.java @@ -33,6 +33,7 @@ import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; import static org.hamcrest.CoreMatchers.allOf; import static org.hamcrest.CoreMatchers.containsString; @@ -396,10 +397,66 @@ public class BootstrapCheckTests extends ESTestCase { BootstrapCheck.check(true, false, Collections.singletonList(check), "testClientJvmCheck"); } + public void testMightForkCheck() { + final AtomicBoolean isSeccompInstalled = new AtomicBoolean(); + final AtomicBoolean mightFork = new AtomicBoolean(); + final BootstrapCheck.MightForkCheck check = new BootstrapCheck.MightForkCheck() { + @Override + boolean isSeccompInstalled() { + return isSeccompInstalled.get(); + } + + @Override + boolean mightFork() { + return mightFork.get(); + } + + @Override + public String errorMessage() { + return "error"; + } + }; + + runMightForkTest( + check, + isSeccompInstalled, + () -> mightFork.set(false), + () -> mightFork.set(true), + e -> assertThat(e.getMessage(), containsString("error"))); + } + + public void testOnErrorCheck() { + final AtomicBoolean isSeccompInstalled = new AtomicBoolean(); + final AtomicReference onError = new AtomicReference<>(); + final BootstrapCheck.MightForkCheck check = new BootstrapCheck.OnErrorCheck() { + @Override + boolean isSeccompInstalled() { + return isSeccompInstalled.get(); + } + + @Override + String onError() { + return onError.get(); + } + }; + + final String command = randomAsciiOfLength(16); + runMightForkTest( + check, + isSeccompInstalled, + () -> onError.set(randomBoolean() ? "" : null), + () -> onError.set(command), + e -> assertThat( + e.getMessage(), + containsString( + "OnError [" + command + "] requires forking but is prevented by system call filters ([bootstrap.seccomp=true]);" + + " upgrade to at least Java 8u92 and use ExitOnOutOfMemoryError"))); + } + public void testOnOutOfMemoryErrorCheck() { final AtomicBoolean isSeccompInstalled = new AtomicBoolean(); final AtomicReference onOutOfMemoryError = new AtomicReference<>(); - final BootstrapCheck.Check check = new BootstrapCheck.OnOutOfMemoryErrorCheck() { + final BootstrapCheck.MightForkCheck check = new BootstrapCheck.OnOutOfMemoryErrorCheck() { @Override boolean isSeccompInstalled() { return isSeccompInstalled.get(); @@ -411,31 +468,52 @@ public class BootstrapCheckTests extends ESTestCase { } }; + final String command = randomAsciiOfLength(16); + runMightForkTest( + check, + isSeccompInstalled, + () -> onOutOfMemoryError.set(randomBoolean() ? "" : null), + () -> onOutOfMemoryError.set(command), + e -> assertThat( + e.getMessage(), + containsString( + "OnOutOfMemoryError [" + command + "]" + + " requires forking but is prevented by system call filters ([bootstrap.seccomp=true]);" + + " upgrade to at least Java 8u92 and use ExitOnOutOfMemoryError"))); + } + + private void runMightForkTest( + final BootstrapCheck.MightForkCheck check, + final AtomicBoolean isSeccompInstalled, + final Runnable disableMightFork, + final Runnable enableMightFork, + final Consumer consumer) { + // if seccomp is disabled, nothing should happen isSeccompInstalled.set(false); - onOutOfMemoryError.set(randomBoolean() ? "" : randomAsciiOfLength(16)); - BootstrapCheck.check(true, false, Collections.singletonList(check), "testOnOutOfMemoryErrorCheck"); + if (randomBoolean()) { + disableMightFork.run(); + } else { + enableMightFork.run(); + } + BootstrapCheck.check(true, randomBoolean(), Collections.singletonList(check), "testMightFork"); - // if seccomp is enabled, and OnOutOfMemoryError is not, nothing - // should happen + // if seccomp is enabled, but we will not fork, nothing should + // happen isSeccompInstalled.set(true); - onOutOfMemoryError.set(""); - BootstrapCheck.check(true, false, Collections.singletonList(check), "testOnOutOfMemoryErrorCheck"); + disableMightFork.run(); + BootstrapCheck.check(true, randomBoolean(), Collections.singletonList(check), "testMightFork"); - // if seccomp is enabled, and OnOutOfMemoryError is, the check - // should be enforced, regardless of bootstrap checks being - // enabled or not + // if seccomp is enabled, and we might fork, the check should + // be enforced, regardless of bootstrap checks being enabled or + // not isSeccompInstalled.set(true); - final String command = randomAsciiOfLength(16); - onOutOfMemoryError.set(command); + enableMightFork.run(); + final RuntimeException e = expectThrows( RuntimeException.class, - () -> BootstrapCheck.check(randomBoolean(), randomBoolean(), Collections.singletonList(check), "testOnOutOfMemoryErrorCheck")); - assertThat( - e.getMessage(), - containsString( - "OnOutOfMemoryError [" + command + "] requires forking but is prevented by system call filters ([bootstrap.seccomp=true]);" - + " upgrade to at least Java 8u92 and use ExitOnOutOfMemoryError")); + () -> BootstrapCheck.check(randomBoolean(), randomBoolean(), Collections.singletonList(check), "testMightFork")); + consumer.accept(e); } public void testIgnoringSystemChecks() { diff --git a/docs/reference/setup/bootstrap-checks.asciidoc b/docs/reference/setup/bootstrap-checks.asciidoc index 81d14f24c8f..1da61360033 100644 --- a/docs/reference/setup/bootstrap-checks.asciidoc +++ b/docs/reference/setup/bootstrap-checks.asciidoc @@ -157,14 +157,18 @@ systems and operating systems, the server VM is the default. Additionally, Elasticsearch is configured by default to force the server VM. -=== OnOutOfMemoryError check +=== OnError and OnOutOfMemoryError checks -The JVM option `OnOutOfMemoryError` enables executing an arbitrary -command if the JVM throws an `OutOfMemoryError`. However, by default, +The JVM options `OnError` and `OnOutOfMemoryError` enable executing +arbitrary commands if the JVM encounters a fatal error (`OnError`) or an +`OutOfMemoryError` (`OnOutOfMemoryError`). However, by default, Elasticsearch system call filters (seccomp) are enabled and these -filters prevent forking. Thus, using `OnOutOfMemoryError` and system -call filters are incompatible. The `OnOutOfMemoryError` check prevents -Elasticsearch from starting if this JVM option is used and system call -filters are enabled. This check is always enforced. To pass this check -do not enable `OnOutOfMemoryError`; instead, upgrade to Java 8u92 and -use the JVM flag `ExitOnOutOfMemoryError`. +filters prevent forking. Thus, using `OnError` or `OnOutOfMemoryError` +and system call filters are incompatible. The `OnError` and +`OnOutOfMemoryError` checks prevent Elasticsearch from starting if +either of these JVM options are used and system call filters are +enabled. This check is always enforced. To pass this check do not enable +`OnError` nor `OnOutOfMemoryError`; instead, upgrade to Java 8u92 and +use the JVM flag `ExitOnOutOfMemoryError`. While this does not have the +full capabilities of `OnError` nor `OnOutOfMemoryError`, arbitrary +forking will not be supported with seccomp enabled.