From 95b0a6a2cf46de2f900cc08466d9697fbb2d4e5e Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Tue, 8 Mar 2016 08:34:46 -0500 Subject: [PATCH 1/3] Limit generic thread pool The generic thread pool was previously configured to be able to create an unlimited number of threads. The thinking is that tasks that are submitted to its work queue must execute and should not block waiting for a worker. However, in cases of heavy load, this can lead to an explosion in the number of threads; this can even lead to a feedback loop that exacerbates the problem. What is more, this can even bump into OS limits on the number of threads that can be created. This commit limits the number of threads in the generic thread pool to four times the bounded number of processors. Relates #17003 --- core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java b/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java index c7f4392e56a..8b6f1198705 100644 --- a/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java +++ b/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java @@ -222,7 +222,7 @@ public class ThreadPool extends AbstractComponent implements Closeable { int halfProcMaxAt5 = Math.min(((availableProcessors + 1) / 2), 5); int halfProcMaxAt10 = Math.min(((availableProcessors + 1) / 2), 10); Map defaultExecutorTypeSettings = new HashMap<>(); - add(defaultExecutorTypeSettings, new ExecutorSettingsBuilder(Names.GENERIC).keepAlive("30s")); + add(defaultExecutorTypeSettings, new ExecutorSettingsBuilder(Names.GENERIC).size(4 * availableProcessors).keepAlive("30s")); add(defaultExecutorTypeSettings, new ExecutorSettingsBuilder(Names.INDEX).size(availableProcessors).queueSize(200)); add(defaultExecutorTypeSettings, new ExecutorSettingsBuilder(Names.BULK).size(availableProcessors).queueSize(50)); add(defaultExecutorTypeSettings, new ExecutorSettingsBuilder(Names.GET).size(availableProcessors).queueSize(1000)); From 930984eb4fce446c355ba35d17fdf08d054221a6 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Tue, 8 Mar 2016 08:46:11 -0500 Subject: [PATCH 2/3] Reduce maximum number of threads in boostrap check This commit reduces the maximum number of threads required in the bootstrap check. This limit can be reduced since the generic thread pool is no longer unbounded. Relates #17003 --- .../main/java/org/elasticsearch/bootstrap/BootstrapCheck.java | 2 +- .../java/org/elasticsearch/bootstrap/BootstrapCheckTests.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java index 6ac3c477fd7..433dd4498a4 100644 --- a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java +++ b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java @@ -225,7 +225,7 @@ final class BootstrapCheck { static class MaxNumberOfThreadsCheck implements Check { - private final long maxNumberOfThreadsThreshold = 1 << 15; + private final long maxNumberOfThreadsThreshold = 1 << 11; @Override public boolean check() { diff --git a/core/src/test/java/org/elasticsearch/bootstrap/BootstrapCheckTests.java b/core/src/test/java/org/elasticsearch/bootstrap/BootstrapCheckTests.java index 45986eab00e..3c269c39004 100644 --- a/core/src/test/java/org/elasticsearch/bootstrap/BootstrapCheckTests.java +++ b/core/src/test/java/org/elasticsearch/bootstrap/BootstrapCheckTests.java @@ -131,7 +131,7 @@ public class BootstrapCheckTests extends ESTestCase { } public void testMaxNumberOfThreadsCheck() { - final int limit = 1 << 15; + final int limit = 1 << 11; final AtomicLong maxNumberOfThreads = new AtomicLong(randomIntBetween(1, limit - 1)); final BootstrapCheck.MaxNumberOfThreadsCheck check = new BootstrapCheck.MaxNumberOfThreadsCheck() { @Override From c0572c631db574cdcaa17413cb9376dacd1f8210 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Tue, 8 Mar 2016 08:51:21 -0500 Subject: [PATCH 3/3] Note to configuration docs on number of threads This commit adds a note to the configuration docs regarding the number of threads necessary for the Elasticsearch user. Relates #17003 --- docs/reference/setup/configuration.asciidoc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/reference/setup/configuration.asciidoc b/docs/reference/setup/configuration.asciidoc index 03037207fb0..bef563cd965 100644 --- a/docs/reference/setup/configuration.asciidoc +++ b/docs/reference/setup/configuration.asciidoc @@ -43,6 +43,13 @@ using the <> API, with: curl localhost:9200/_nodes/stats/process?pretty -------------------------------------------------- +[float] +[[max-number-of-threads]] +==== Number of threads + +Make sure that the number of threads that the Elasticsearch user can +create is at least 2048. + [float] [[vm-max-map-count]] ==== Virtual memory