From 8e7e679984e95465f87dd97dad9022aacd00af04 Mon Sep 17 00:00:00 2001 From: Jian Wang Date: Sun, 7 Nov 2021 03:21:44 -0800 Subject: [PATCH] Add more metrics for Jetty server thread pool usage (#11113) Add more metrics for jetty server thread pool usage so we know if we have allocated enough http threads to handle requests. --- docs/design/extensions-contrib/dropwizard.md | 32 ++++++++ docs/operations/metrics.md | 7 ++ .../src/main/resources/defaultMetrics.json | 7 ++ .../resources/defaultMetricDimensions.json | 11 ++- .../jetty/JettyServerModule.java | 18 ++++ .../jetty/JettyServerModuleTest.java | 82 +++++++++++++++++++ website/.spelling | 5 ++ 7 files changed, 161 insertions(+), 1 deletion(-) create mode 100644 server/src/test/java/org/apache/druid/server/initialization/jetty/JettyServerModuleTest.java diff --git a/docs/design/extensions-contrib/dropwizard.md b/docs/design/extensions-contrib/dropwizard.md index c76afcae228..a2a8c34d6ea 100644 --- a/docs/design/extensions-contrib/dropwizard.md +++ b/docs/design/extensions-contrib/dropwizard.md @@ -629,6 +629,38 @@ Latest default metrics mapping can be found [here] (https://github.com/apache/dr "priority" ], "type": "gauge" + }, + "jetty/numOpenConnections": { + "dimensions": [], + "type": "gauge" + }, + "jetty/threadPool/total": { + "dimensions": [], + "type": "gauge" + }, + "jetty/threadPool/idle": { + "dimensions": [], + "type": "gauge" + }, + "jetty/threadPool/busy": { + "dimensions": [], + "type": "gauge" + }, + "jetty/threadPool/isLowOnThreads": { + "dimensions": [], + "type": "gauge" + }, + "jetty/threadPool/min": { + "dimensions": [], + "type": "gauge" + }, + "jetty/threadPool/max": { + "dimensions": [], + "type": "gauge" + }, + "jetty/threadPool/queueSize": { + "dimensions": [], + "type": "gauge" } } ``` diff --git a/docs/operations/metrics.md b/docs/operations/metrics.md index e987c1c32ef..1c7c736e3c4 100644 --- a/docs/operations/metrics.md +++ b/docs/operations/metrics.md @@ -97,6 +97,13 @@ Available Metrics |Metric|Description|Normal Value| |------|-----------|------------| |`jetty/numOpenConnections`|Number of open jetty connections.|Not much higher than number of jetty threads.| +|`jetty/threadPool/total`|Number of total workable threads allocated.|The number should equal to threadPoolNumIdleThreads + threadPoolNumBusyThreads.| +|`jetty/threadPool/idle`|Number of idle threads.|Less than or equal to threadPoolNumTotalThreads. Non zero number means there is less work to do than configured capacity.| +|`jetty/threadPool/busy`|Number of busy threads that has work to do from the worker queue.|Less than or equal to threadPoolNumTotalThreads.| +|`jetty/threadPool/isLowOnThreads`|A rough indicator of whether number of total workable threads allocated is enough to handle the works in the work queue.|0| +|`jetty/threadPool/min`|Number of minimum threads allocatable.|druid.server.http.numThreads plus a small fixed number of threads allocated for Jetty acceptors and selectors.| +|`jetty/threadPool/max`|Number of maximum threads allocatable.|druid.server.http.numThreads plus a small fixed number of threads allocated for Jetty acceptors and selectors.| +|`jetty/threadPool/queueSize`|Size of the worker queue.|Not much higher than druid.server.http.queueSize| ### Cache diff --git a/extensions-contrib/opentsdb-emitter/src/main/resources/defaultMetrics.json b/extensions-contrib/opentsdb-emitter/src/main/resources/defaultMetrics.json index c2d38455cf9..ae01473a64c 100644 --- a/extensions-contrib/opentsdb-emitter/src/main/resources/defaultMetrics.json +++ b/extensions-contrib/opentsdb-emitter/src/main/resources/defaultMetrics.json @@ -28,6 +28,13 @@ "type" ], "jetty/numOpenConnections": [], + "jetty/threadPool/total": [], + "jetty/threadPool/idle": [], + "jetty/threadPool/busy": [], + "jetty/threadPool/isLowOnThreads": [], + "jetty/threadPool/min": [], + "jetty/threadPool/max": [], + "jetty/threadPool/queueSize": [], "query/cache/delta/numEntries": [], "query/cache/delta/sizeBytes": [], "query/cache/delta/hits": [], diff --git a/extensions-contrib/statsd-emitter/src/main/resources/defaultMetricDimensions.json b/extensions-contrib/statsd-emitter/src/main/resources/defaultMetricDimensions.json index 298b440e520..5ac08862409 100644 --- a/extensions-contrib/statsd-emitter/src/main/resources/defaultMetricDimensions.json +++ b/extensions-contrib/statsd-emitter/src/main/resources/defaultMetricDimensions.json @@ -130,5 +130,14 @@ "sys/cpu" : { "dimensions" : ["cpuName", "cpuTime"], "type" : "gauge"}, "coordinator-segment/count" : { "dimensions" : ["dataSource"], "type" : "gauge" }, - "historical-segment/count" : { "dimensions" : ["dataSource", "tier", "priority"], "type" : "gauge" } + "historical-segment/count" : { "dimensions" : ["dataSource", "tier", "priority"], "type" : "gauge" }, + + "jetty/numOpenConnections": { "dimensions" : [], "type" : "gauge" }, + "jetty/threadPool/total": { "dimensions" : [], "type" : "gauge" }, + "jetty/threadPool/idle": { "dimensions" : [], "type" : "gauge" }, + "jetty/threadPool/busy": { "dimensions" : [], "type" : "gauge" }, + "jetty/threadPool/isLowOnThreads": { "dimensions" : [], "type" : "gauge" }, + "jetty/threadPool/min": { "dimensions" : [], "type" : "gauge" }, + "jetty/threadPool/max": { "dimensions" : [], "type" : "gauge" }, + "jetty/threadPool/queueSize": { "dimensions" : [], "type" : "gauge" } } diff --git a/server/src/main/java/org/apache/druid/server/initialization/jetty/JettyServerModule.java b/server/src/main/java/org/apache/druid/server/initialization/jetty/JettyServerModule.java index 41a00a241ea..b722fb2c325 100644 --- a/server/src/main/java/org/apache/druid/server/initialization/jetty/JettyServerModule.java +++ b/server/src/main/java/org/apache/druid/server/initialization/jetty/JettyServerModule.java @@ -109,6 +109,7 @@ public class JettyServerModule extends JerseyServletModule private static final AtomicInteger ACTIVE_CONNECTIONS = new AtomicInteger(); private static final String HTTP_1_1_STRING = "HTTP/1.1"; + private static QueuedThreadPool jettyServerThreadPool = null; @Override protected void configureServlets() @@ -229,6 +230,7 @@ public class JettyServerModule extends JerseyServletModule } threadPool.setDaemon(true); + jettyServerThreadPool = threadPool; final Server server = new Server(threadPool); @@ -528,6 +530,16 @@ public class JettyServerModule extends JerseyServletModule final ServiceMetricEvent.Builder builder = new ServiceMetricEvent.Builder(); MonitorUtils.addDimensionsToBuilder(builder, dimensions); emitter.emit(builder.build("jetty/numOpenConnections", ACTIVE_CONNECTIONS.get())); + if (jettyServerThreadPool != null) { + emitter.emit(builder.build("jetty/threadPool/total", jettyServerThreadPool.getThreads())); + emitter.emit(builder.build("jetty/threadPool/idle", jettyServerThreadPool.getIdleThreads())); + emitter.emit(builder.build("jetty/threadPool/isLowOnThreads", jettyServerThreadPool.isLowOnThreads() ? 1 : 0)); + emitter.emit(builder.build("jetty/threadPool/min", jettyServerThreadPool.getMinThreads())); + emitter.emit(builder.build("jetty/threadPool/max", jettyServerThreadPool.getMaxThreads())); + emitter.emit(builder.build("jetty/threadPool/queueSize", jettyServerThreadPool.getQueueSize())); + emitter.emit(builder.build("jetty/threadPool/busy", jettyServerThreadPool.getBusyThreads())); + } + return true; } } @@ -578,4 +590,10 @@ public class JettyServerModule extends JerseyServletModule { return ACTIVE_CONNECTIONS.get(); } + + @VisibleForTesting + public static void setJettyServerThreadPool(QueuedThreadPool threadPool) + { + jettyServerThreadPool = threadPool; + } } diff --git a/server/src/test/java/org/apache/druid/server/initialization/jetty/JettyServerModuleTest.java b/server/src/test/java/org/apache/druid/server/initialization/jetty/JettyServerModuleTest.java new file mode 100644 index 00000000000..e4158d7ab2f --- /dev/null +++ b/server/src/test/java/org/apache/druid/server/initialization/jetty/JettyServerModuleTest.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.server.initialization.jetty; + +import org.apache.druid.java.util.common.Pair; +import org.apache.druid.java.util.emitter.core.Emitter; +import org.apache.druid.java.util.emitter.core.Event; +import org.apache.druid.java.util.emitter.service.ServiceEmitter; +import org.apache.druid.java.util.emitter.service.ServiceMetricEvent; +import org.eclipse.jetty.util.thread.QueuedThreadPool; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class JettyServerModuleTest +{ + @Test + public void testJettyServerModule() + { + List events = new ArrayList<>(); + ServiceEmitter serviceEmitter = new ServiceEmitter("service", "host", Mockito.mock(Emitter.class)) + { + @Override + public void emit(Event event) + { + events.add(event); + } + }; + QueuedThreadPool jettyServerThreadPool = Mockito.mock(QueuedThreadPool.class); + JettyServerModule.setJettyServerThreadPool(jettyServerThreadPool); + Mockito.when(jettyServerThreadPool.getThreads()).thenReturn(100); + Mockito.when(jettyServerThreadPool.getIdleThreads()).thenReturn(40); + Mockito.when(jettyServerThreadPool.isLowOnThreads()).thenReturn(true); + Mockito.when(jettyServerThreadPool.getMinThreads()).thenReturn(30); + Mockito.when(jettyServerThreadPool.getMaxThreads()).thenReturn(100); + Mockito.when(jettyServerThreadPool.getQueueSize()).thenReturn(50); + Mockito.when(jettyServerThreadPool.getBusyThreads()).thenReturn(60); + + JettyServerModule.JettyMonitor jettyMonitor = new JettyServerModule.JettyMonitor("ds", "t0"); + jettyMonitor.doMonitor(serviceEmitter); + + Assert.assertEquals(8, events.size()); + List> expectedEvents = Arrays.asList( + new Pair<>("jetty/numOpenConnections", 0), + new Pair<>("jetty/threadPool/total", 100), + new Pair<>("jetty/threadPool/idle", 40), + new Pair<>("jetty/threadPool/isLowOnThreads", 1), + new Pair<>("jetty/threadPool/min", 30), + new Pair<>("jetty/threadPool/max", 100), + new Pair<>("jetty/threadPool/queueSize", 50), + new Pair<>("jetty/threadPool/busy", 60) + ); + + for (int i = 0; i < expectedEvents.size(); i++) { + Pair expected = expectedEvents.get(i); + ServiceMetricEvent actual = (ServiceMetricEvent) (events.get(i)); + Assert.assertEquals(expected.lhs, actual.getMetric()); + Assert.assertEquals(expected.rhs, actual.getValue()); + } + } +} diff --git a/website/.spelling b/website/.spelling index f1324234ee9..c2b9dd99f7d 100644 --- a/website/.spelling +++ b/website/.spelling @@ -1293,6 +1293,8 @@ bufferpoolName cms cpuName cpuTime +druid.server.http.numThreads +druid.server.http.queueSize fsDevName fsDirName fsOptions @@ -1318,6 +1320,9 @@ remoteAddress serviceName taskStatus taskType +threadPoolNumBusyThreads. +threadPoolNumIdleThreads +threadPoolNumTotalThreads. - ../docs/operations/other-hadoop.md CDH Classloader