From 76ad17fb4ca116fdf73c196555590625d47c35a8 Mon Sep 17 00:00:00 2001 From: Atul Mohan Date: Wed, 7 Aug 2024 07:01:10 -0700 Subject: [PATCH] Add config for http client connect timeout (#16831) Adds a configuration clientConnectTimeout to our http client config which controls the connection timeout for our http client requests. It was observed that on busy K8S clusters, the default connect timeout of 500ms is sometimes not enough time to complete syn/acks for a request and in these cases, the requests timeout with the error: exceptionType=java.net.SocketTimeoutException, exceptionMessage=Connect Timeout This behavior was mostly observed on the router while forwarding queries to the broker. Having a slightly higher connect timeout helped resolve these issues. --- docs/configuration/index.md | 4 ++++ .../apache/druid/guice/http/DruidHttpClientConfig.java | 9 +++++++++ .../apache/druid/guice/http/JettyHttpClientModule.java | 5 +---- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/docs/configuration/index.md b/docs/configuration/index.md index 6c09e07634f..8a7e4f19981 100644 --- a/docs/configuration/index.md +++ b/docs/configuration/index.md @@ -840,6 +840,7 @@ All Druid components can communicate with each other over HTTP. |`druid.global.http.readTimeout`|The timeout for data reads.|`PT15M`| |`druid.global.http.unusedConnectionTimeout`|The timeout for idle connections in connection pool. The connection in the pool will be closed after this timeout and a new one will be established. This timeout should be less than `druid.global.http.readTimeout`. Set this timeout = ~90% of `druid.global.http.readTimeout`|`PT4M`| |`druid.global.http.numMaxThreads`|Maximum number of I/O worker threads|`max(10, ((number of cores * 17) / 16 + 2) + 30)`| +|`druid.global.http.clientConnectTimeout`|The timeout (in milliseconds) for establishing client connections.|500| ### Common endpoints configuration @@ -1879,6 +1880,8 @@ client has the following configuration options. |`druid.broker.http.unusedConnectionTimeout`|The timeout for idle connections in connection pool. The connection in the pool will be closed after this timeout and a new one will be established. This timeout should be less than `druid.broker.http.readTimeout`. Set this timeout = ~90% of `druid.broker.http.readTimeout`|`PT4M`| |`druid.broker.http.maxQueuedBytes`|Maximum number of bytes queued per query before exerting [backpressure](../operations/basic-cluster-tuning.md#broker-backpressure) on channels to the data servers.

Similar to `druid.server.http.maxScatterGatherBytes`, except that `maxQueuedBytes` triggers [backpressure](../operations/basic-cluster-tuning.md#broker-backpressure) instead of query failure. Set to zero to disable. You can override this setting by using the [`maxQueuedBytes` query context parameter](../querying/query-context.md). Druid supports [human-readable](human-readable-byte.md) format. |25 MB or 2% of maximum Broker heap size, whichever is greater.| |`druid.broker.http.numMaxThreads`|`Maximum number of I/O worker threads|max(10, ((number of cores * 17) / 16 + 2) + 30)`| +|`druid.broker.http.clientConnectTimeout`|The timeout (in milliseconds) for establishing client connections.|500| + ##### Retry policy @@ -2239,3 +2242,4 @@ Supported query contexts: |`druid.router.http.numMaxThreads`|Maximum number of worker threads to handle HTTP requests and responses|`max(10, ((number of cores * 17) / 16 + 2) + 30)`| |`druid.router.http.numRequestsQueued`|Maximum number of requests that may be queued to a destination|`1024`| |`druid.router.http.requestBuffersize`|Size of the content buffer for receiving requests. These buffers are only used for active connections that have requests with bodies that will not fit within the header buffer|`8 * 1024`| +|`druid.router.http.clientConnectTimeout`|The timeout (in milliseconds) for establishing client connections.|500| diff --git a/server/src/main/java/org/apache/druid/guice/http/DruidHttpClientConfig.java b/server/src/main/java/org/apache/druid/guice/http/DruidHttpClientConfig.java index 70ed44f2eb8..ff5025701bc 100644 --- a/server/src/main/java/org/apache/druid/guice/http/DruidHttpClientConfig.java +++ b/server/src/main/java/org/apache/druid/guice/http/DruidHttpClientConfig.java @@ -27,6 +27,7 @@ import org.joda.time.Duration; import org.joda.time.Period; import javax.validation.constraints.Min; +import java.util.concurrent.TimeUnit; /** * @@ -72,6 +73,9 @@ public class DruidHttpClientConfig @JsonProperty private Boolean eagerInitialization = null; + @JsonProperty + private long clientConnectTimeout = TimeUnit.MILLISECONDS.toMillis(500); + public int getNumConnections() { return numConnections; @@ -129,6 +133,11 @@ public class DruidHttpClientConfig return eagerInitialization; } + public long getClientConnectTimeout() + { + return clientConnectTimeout; + } + private static HumanReadableBytes computeDefaultMaxQueuedBytes() { return HumanReadableBytes.valueOf( diff --git a/server/src/main/java/org/apache/druid/guice/http/JettyHttpClientModule.java b/server/src/main/java/org/apache/druid/guice/http/JettyHttpClientModule.java index 3c2ad854f23..c4eefa02704 100644 --- a/server/src/main/java/org/apache/druid/guice/http/JettyHttpClientModule.java +++ b/server/src/main/java/org/apache/druid/guice/http/JettyHttpClientModule.java @@ -33,15 +33,12 @@ import org.eclipse.jetty.util.thread.QueuedThreadPool; import javax.net.ssl.SSLContext; import java.lang.annotation.Annotation; -import java.util.concurrent.TimeUnit; /** * */ public class JettyHttpClientModule implements Module { - private static final long CLIENT_CONNECT_TIMEOUT_MILLIS = TimeUnit.MILLISECONDS.toMillis(500); - public static JettyHttpClientModule global() { return new JettyHttpClientModule("druid.global.http", Global.class); @@ -91,7 +88,7 @@ public class JettyHttpClientModule implements Module httpClient.setIdleTimeout(config.getReadTimeout().getMillis()); httpClient.setMaxConnectionsPerDestination(config.getNumConnections()); httpClient.setMaxRequestsQueuedPerDestination(config.getNumRequestsQueued()); - httpClient.setConnectTimeout(CLIENT_CONNECT_TIMEOUT_MILLIS); + httpClient.setConnectTimeout(config.getClientConnectTimeout()); httpClient.setRequestBufferSize(config.getRequestBuffersize()); final QueuedThreadPool pool = new QueuedThreadPool(config.getNumMaxThreads()); pool.setName(JettyHttpClientModule.class.getSimpleName() + "-threadPool-" + pool.hashCode());