From 00cecac86e03195a28f4bec92ccd8cc182a16132 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Tue, 4 Oct 2016 18:15:19 -0400 Subject: [PATCH] Change Watcher thread pool to be scaling Watcher uses a custom thread pool. This is because executing watches can be long-running tasks that often block on I/O and it is best to not consume the core thread pools with these tasks. Today this thread pool is fixed, and sized at five times the bounded number of cores (so 160 on a 32-core box). It makes sense for there to possibly be so many threads, again because these tasks can block on I/O and having excess capacity lets unblocked watches execute. It's the fixed size that can cause problem, all these threads are always consuming resources even when there are no or not that many watches running. This commit changes this thread pool to be a scaling thread pool. Relates elastic/elasticsearch#3660 Original commit: elastic/x-pack-elasticsearch@3cafab6e8360b17ecaf99d37bc1e024a9c7ff256 --- .../org/elasticsearch/xpack/watcher/Watcher.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java index 455dd0a3f35..281605a1fc5 100644 --- a/elasticsearch/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java @@ -25,7 +25,7 @@ import org.elasticsearch.plugins.ScriptPlugin; import org.elasticsearch.rest.RestHandler; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.threadpool.ExecutorBuilder; -import org.elasticsearch.threadpool.FixedExecutorBuilder; +import org.elasticsearch.threadpool.ScalingExecutorBuilder; import org.elasticsearch.xpack.XPackPlugin; import org.elasticsearch.xpack.XPackSettings; import org.elasticsearch.xpack.watcher.actions.WatcherActionModule; @@ -165,12 +165,14 @@ public class Watcher implements ActionPlugin, ScriptPlugin { public List> getExecutorBuilders(final Settings settings) { if (enabled) { - final FixedExecutorBuilder builder = - new FixedExecutorBuilder( - settings, + final ScalingExecutorBuilder builder = + new ScalingExecutorBuilder( InternalWatchExecutor.THREAD_POOL_NAME, + 0, + // watcher threads can block on I/O for a long time, so we let this + // pool be large so that execution of unblocked watches can proceed 5 * EsExecutors.boundedNumberOfProcessors(settings), - 1000, + TimeValue.timeValueMinutes(5), "xpack.watcher.thread_pool"); return Collections.singletonList(builder); }