From e75a0da4d584779c57eec0f7b852e6e492daaecd Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Wed, 2 Mar 2016 21:55:19 -0500 Subject: [PATCH] Add max number of processes check This commit adds a bootstrap check on Linux for the max number of processes available to the user running the Elasticsearch process. Closes #16919 --- .../elasticsearch/bootstrap/Bootstrap.java | 2 + .../bootstrap/BootstrapCheck.java | 29 +++++++++++ .../elasticsearch/bootstrap/JNANatives.java | 23 +++++++- .../bootstrap/BootstrapCheckTests.java | 27 ++++++++++ .../bootstrap/EvilJNANativesTests.java | 52 +++++++++++++++++++ 5 files changed, 131 insertions(+), 2 deletions(-) create mode 100644 qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilJNANativesTests.java diff --git a/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java b/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java index bf3417c7101..215659054d2 100644 --- a/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java +++ b/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java @@ -134,6 +134,8 @@ final class Bootstrap { // we've already logged this. } + JNANatives.trySetMaxNumberOfThreads(); + // init lucene random seed. it will use /dev/urandom where available: StringHelper.randomId(); } diff --git a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java index 771a7435140..6ac3c477fd7 100644 --- a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java +++ b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java @@ -120,6 +120,9 @@ final class BootstrapCheck { = Constants.MAC_OS_X ? new OsXFileDescriptorCheck() : new FileDescriptorCheck(); checks.add(fileDescriptorCheck); checks.add(new MlockallCheck(BootstrapSettings.MLOCKALL_SETTING.get(settings))); + if (Constants.LINUX) { + checks.add(new MaxNumberOfThreadsCheck()); + } return Collections.unmodifiableList(checks); } @@ -220,4 +223,30 @@ final class BootstrapCheck { } + static class MaxNumberOfThreadsCheck implements Check { + + private final long maxNumberOfThreadsThreshold = 1 << 15; + + @Override + public boolean check() { + return getMaxNumberOfThreads() != -1 && getMaxNumberOfThreads() < maxNumberOfThreadsThreshold; + } + + @Override + public String errorMessage() { + return String.format( + Locale.ROOT, + "max number of threads [%d] for user [%s] likely too low, increase to at least [%d]", + getMaxNumberOfThreads(), + BootstrapInfo.getSystemProperties().get("user.name"), + maxNumberOfThreadsThreshold); + } + + // visible for testing + long getMaxNumberOfThreads() { + return JNANatives.MAX_NUMBER_OF_THREADS; + } + + } + } diff --git a/core/src/main/java/org/elasticsearch/bootstrap/JNANatives.java b/core/src/main/java/org/elasticsearch/bootstrap/JNANatives.java index 64b685c4150..b9d5ce11dbc 100644 --- a/core/src/main/java/org/elasticsearch/bootstrap/JNANatives.java +++ b/core/src/main/java/org/elasticsearch/bootstrap/JNANatives.java @@ -48,6 +48,9 @@ class JNANatives { // Set to true, in case policy can be applied to all threads of the process (even existing ones) // otherwise they are only inherited for new threads (ES app threads) static boolean LOCAL_SECCOMP_ALL = false; + // set to the maximum number of threads that can be created for + // the user ID that owns the running Elasticsearch process + static long MAX_NUMBER_OF_THREADS = -1; static void tryMlockall() { int errno = Integer.MIN_VALUE; @@ -103,13 +106,29 @@ class JNANatives { } } + static void trySetMaxNumberOfThreads() { + if (Constants.LINUX) { + // this is only valid on Linux and the value *is* different on OS X + // see /usr/include/sys/resource.h on OS X + // on Linux the resource RLIMIT_NPROC means *the number of threads* + // this is in opposition to BSD-derived OSes + final int rlimit_nproc = 6; + + final JNACLibrary.Rlimit rlimit = new JNACLibrary.Rlimit(); + if (JNACLibrary.getrlimit(rlimit_nproc, rlimit) == 0) { + MAX_NUMBER_OF_THREADS = rlimit.rlim_cur.longValue(); + } else { + logger.warn("unable to retrieve max number of threads [" + JNACLibrary.strerror(Native.getLastError()) + "]"); + } + } + } + static String rlimitToString(long value) { assert Constants.LINUX || Constants.MAC_OS_X; if (value == JNACLibrary.RLIM_INFINITY) { return "unlimited"; } else { - // TODO, on java 8 use Long.toUnsignedString, since that's what it is. - return Long.toString(value); + return Long.toUnsignedString(value); } } diff --git a/core/src/test/java/org/elasticsearch/bootstrap/BootstrapCheckTests.java b/core/src/test/java/org/elasticsearch/bootstrap/BootstrapCheckTests.java index 38515f2190d..45986eab00e 100644 --- a/core/src/test/java/org/elasticsearch/bootstrap/BootstrapCheckTests.java +++ b/core/src/test/java/org/elasticsearch/bootstrap/BootstrapCheckTests.java @@ -130,6 +130,33 @@ public class BootstrapCheckTests extends ESTestCase { } } + public void testMaxNumberOfThreadsCheck() { + final int limit = 1 << 15; + final AtomicLong maxNumberOfThreads = new AtomicLong(randomIntBetween(1, limit - 1)); + final BootstrapCheck.MaxNumberOfThreadsCheck check = new BootstrapCheck.MaxNumberOfThreadsCheck() { + @Override + long getMaxNumberOfThreads() { + return maxNumberOfThreads.get(); + } + }; + + try { + BootstrapCheck.check(true, Collections.singletonList(check)); + fail("should have failed due to max number of threads too low"); + } catch (final RuntimeException e) { + assertThat(e.getMessage(), containsString("max number of threads")); + } + + maxNumberOfThreads.set(randomIntBetween(limit + 1, Integer.MAX_VALUE)); + + BootstrapCheck.check(true, Collections.singletonList(check)); + + // nothing should happen if current max number of threads is + // not available + maxNumberOfThreads.set(-1); + BootstrapCheck.check(true, Collections.singletonList(check)); + } + public void testEnforceLimits() { final Set enforceSettings = BootstrapCheck.enforceSettings(); final Setting setting = randomFrom(Arrays.asList(enforceSettings.toArray(new Setting[enforceSettings.size()]))); diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilJNANativesTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilJNANativesTests.java new file mode 100644 index 00000000000..080eee2501b --- /dev/null +++ b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilJNANativesTests.java @@ -0,0 +1,52 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.bootstrap; + +import org.apache.lucene.util.Constants; +import org.elasticsearch.common.io.PathUtils; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.nio.file.Files; +import java.util.List; + +import static org.hamcrest.Matchers.equalTo; + +public class EvilJNANativesTests extends ESTestCase { + + public void testSetMaximumNumberOfThreads() throws IOException { + if (Constants.LINUX) { + final List lines = Files.readAllLines(PathUtils.get("/proc/self/limits")); + if (!lines.isEmpty()) { + for (String line : lines) { + if (line != null && line.startsWith("Max processes")) { + final String[] fields = line.split("\\s+"); + final long limit = Long.parseLong(fields[2]); + assertThat(JNANatives.MAX_NUMBER_OF_THREADS, equalTo(limit)); + return; + } + } + } + fail("should have read max processes from /proc/self/limits"); + } else { + assertThat(JNANatives.MAX_NUMBER_OF_THREADS, equalTo(-1L)); + } + } +}