Add max number of processes check

This commit adds a bootstrap check on Linux for the max number of
processes available to the user running the Elasticsearch process.

Closes #16919

core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java

@@ -134,6 +134,8 @@ final class Bootstrap {
             // we've already logged this.
         }
 
+        JNANatives.trySetMaxNumberOfThreads();
+
         // init lucene random seed. it will use /dev/urandom where available:
         StringHelper.randomId();
     }

core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java

@@ -120,6 +120,9 @@ final class BootstrapCheck {
                 = Constants.MAC_OS_X ? new OsXFileDescriptorCheck() : new FileDescriptorCheck();
         checks.add(fileDescriptorCheck);
         checks.add(new MlockallCheck(BootstrapSettings.MLOCKALL_SETTING.get(settings)));
+        if (Constants.LINUX) {
+            checks.add(new MaxNumberOfThreadsCheck());
+        }
         return Collections.unmodifiableList(checks);
     }
 
@@ -220,4 +223,30 @@ final class BootstrapCheck {
 
     }
 
+    static class MaxNumberOfThreadsCheck implements Check {
+
+        private final long maxNumberOfThreadsThreshold = 1 << 15;
+
+        @Override
+        public boolean check() {
+            return getMaxNumberOfThreads() != -1 && getMaxNumberOfThreads() < maxNumberOfThreadsThreshold;
+        }
+
+        @Override
+        public String errorMessage() {
+            return String.format(
+                Locale.ROOT,
+                "max number of threads [%d] for user [%s] likely too low, increase to at least [%d]",
+                getMaxNumberOfThreads(),
+                BootstrapInfo.getSystemProperties().get("user.name"),
+                maxNumberOfThreadsThreshold);
+        }
+
+        // visible for testing
+        long getMaxNumberOfThreads() {
+            return JNANatives.MAX_NUMBER_OF_THREADS;
+        }
+
+    }
+
 }

core/src/main/java/org/elasticsearch/bootstrap/JNANatives.java

@@ -48,6 +48,9 @@ class JNANatives {
     // Set to true, in case policy can be applied to all threads of the process (even existing ones)
     // otherwise they are only inherited for new threads (ES app threads)
     static boolean LOCAL_SECCOMP_ALL = false;
+    // set to the maximum number of threads that can be created for
+    // the user ID that owns the running Elasticsearch process
+    static long MAX_NUMBER_OF_THREADS = -1;
 
     static void tryMlockall() {
         int errno = Integer.MIN_VALUE;
@@ -103,13 +106,29 @@ class JNANatives {
         }
     }
 
+    static void trySetMaxNumberOfThreads() {
+        if (Constants.LINUX) {
+            // this is only valid on Linux and the value *is* different on OS X
+            // see /usr/include/sys/resource.h on OS X
+            // on Linux the resource RLIMIT_NPROC means *the number of threads*
+            // this is in opposition to BSD-derived OSes
+            final int rlimit_nproc = 6;
+
+            final JNACLibrary.Rlimit rlimit = new JNACLibrary.Rlimit();
+            if (JNACLibrary.getrlimit(rlimit_nproc, rlimit) == 0) {
+                MAX_NUMBER_OF_THREADS = rlimit.rlim_cur.longValue();
+            } else {
+                logger.warn("unable to retrieve max number of threads [" + JNACLibrary.strerror(Native.getLastError()) + "]");
+            }
+        }
+    }
+
     static String rlimitToString(long value) {
         assert Constants.LINUX || Constants.MAC_OS_X;
         if (value == JNACLibrary.RLIM_INFINITY) {
             return "unlimited";
         } else {
-            // TODO, on java 8 use Long.toUnsignedString, since that's what it is.
-            return Long.toString(value);
+            return Long.toUnsignedString(value);
         }
     }
 

core/src/test/java/org/elasticsearch/bootstrap/BootstrapCheckTests.java

@@ -130,6 +130,33 @@ public class BootstrapCheckTests extends ESTestCase {
         }
     }
 
+    public void testMaxNumberOfThreadsCheck() {
+        final int limit = 1 << 15;
+        final AtomicLong maxNumberOfThreads = new AtomicLong(randomIntBetween(1, limit - 1));
+        final BootstrapCheck.MaxNumberOfThreadsCheck check = new BootstrapCheck.MaxNumberOfThreadsCheck() {
+            @Override
+            long getMaxNumberOfThreads() {
+                return maxNumberOfThreads.get();
+            }
+        };
+
+        try {
+            BootstrapCheck.check(true, Collections.singletonList(check));
+            fail("should have failed due to max number of threads too low");
+        } catch (final RuntimeException e) {
+            assertThat(e.getMessage(), containsString("max number of threads"));
+        }
+
+        maxNumberOfThreads.set(randomIntBetween(limit + 1, Integer.MAX_VALUE));
+
+        BootstrapCheck.check(true, Collections.singletonList(check));
+
+        // nothing should happen if current max number of threads is
+        // not available
+        maxNumberOfThreads.set(-1);
+        BootstrapCheck.check(true, Collections.singletonList(check));
+    }
+
     public void testEnforceLimits() {
         final Set<Setting> enforceSettings = BootstrapCheck.enforceSettings();
         final Setting setting = randomFrom(Arrays.asList(enforceSettings.toArray(new Setting[enforceSettings.size()])));

qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilJNANativesTests.java

@@ -0,0 +1,52 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.bootstrap;
+
+import org.apache.lucene.util.Constants;
+import org.elasticsearch.common.io.PathUtils;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.List;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class EvilJNANativesTests extends ESTestCase {
+
+    public void testSetMaximumNumberOfThreads() throws IOException {
+        if (Constants.LINUX) {
+            final List<String> lines = Files.readAllLines(PathUtils.get("/proc/self/limits"));
+            if (!lines.isEmpty()) {
+                for (String line : lines) {
+                    if (line != null && line.startsWith("Max processes")) {
+                        final String[] fields = line.split("\\s+");
+                        final long limit = Long.parseLong(fields[2]);
+                        assertThat(JNANatives.MAX_NUMBER_OF_THREADS, equalTo(limit));
+                        return;
+                    }
+                }
+            }
+            fail("should have read max processes from /proc/self/limits");
+        } else {
+            assertThat(JNANatives.MAX_NUMBER_OF_THREADS, equalTo(-1L));
+        }
+    }
+}