Add more metrics for Jetty server thread pool usage (#11113)

Add more metrics for jetty server thread pool usage so we know if we have allocated enough http threads to handle requests.

docs/design/extensions-contrib/dropwizard.md

@@ -629,6 +629,38 @@ Latest default metrics mapping can be found [here] (https://github.com/apache/dr
       "priority"
     ],
     "type": "gauge"
+  },
+  "jetty/numOpenConnections": {
+    "dimensions": [],
+    "type": "gauge"
+  },
+  "jetty/threadPool/total": {
+    "dimensions": [],
+    "type": "gauge"
+  },
+  "jetty/threadPool/idle": {
+    "dimensions": [],
+    "type": "gauge"
+  },
+  "jetty/threadPool/busy": {
+    "dimensions": [],
+    "type": "gauge"
+  },
+  "jetty/threadPool/isLowOnThreads": {
+    "dimensions": [],
+    "type": "gauge"
+  },
+  "jetty/threadPool/min": {
+    "dimensions": [],
+    "type": "gauge"
+  },
+  "jetty/threadPool/max": {
+    "dimensions": [],
+    "type": "gauge"
+  },
+  "jetty/threadPool/queueSize": {
+    "dimensions": [],
+    "type": "gauge"
   }
 }
 ```

docs/operations/metrics.md

@@ -97,6 +97,13 @@ Available Metrics
 |Metric|Description|Normal Value|
 |------|-----------|------------|
 |`jetty/numOpenConnections`|Number of open jetty connections.|Not much higher than number of jetty threads.|
+|`jetty/threadPool/total`|Number of total workable threads allocated.|The number should equal to threadPoolNumIdleThreads + threadPoolNumBusyThreads.|
+|`jetty/threadPool/idle`|Number of idle threads.|Less than or equal to threadPoolNumTotalThreads. Non zero number means there is less work to do than configured capacity.|
+|`jetty/threadPool/busy`|Number of busy threads that has work to do from the worker queue.|Less than or equal to threadPoolNumTotalThreads.|
+|`jetty/threadPool/isLowOnThreads`|A rough indicator of whether number of total workable threads allocated is enough to handle the works in the work queue.|0|
+|`jetty/threadPool/min`|Number of minimum threads allocatable.|druid.server.http.numThreads plus a small fixed number of threads allocated for Jetty acceptors and selectors.|
+|`jetty/threadPool/max`|Number of maximum threads allocatable.|druid.server.http.numThreads plus a small fixed number of threads allocated for Jetty acceptors and selectors.|
+|`jetty/threadPool/queueSize`|Size of the worker queue.|Not much higher than druid.server.http.queueSize|
 
 ### Cache
 

extensions-contrib/opentsdb-emitter/src/main/resources/defaultMetrics.json

@@ -28,6 +28,13 @@
     "type"
   ],
   "jetty/numOpenConnections": [],
+  "jetty/threadPool/total": [],
+  "jetty/threadPool/idle": [],
+  "jetty/threadPool/busy": [],
+  "jetty/threadPool/isLowOnThreads": [],
+  "jetty/threadPool/min": [],
+  "jetty/threadPool/max": [],
+  "jetty/threadPool/queueSize": [],
   "query/cache/delta/numEntries": [],
   "query/cache/delta/sizeBytes": [],
   "query/cache/delta/hits": [],

extensions-contrib/statsd-emitter/src/main/resources/defaultMetricDimensions.json

@@ -130,5 +130,14 @@
   "sys/cpu" : { "dimensions" : ["cpuName", "cpuTime"], "type" : "gauge"},
 
   "coordinator-segment/count" : { "dimensions" : ["dataSource"], "type" : "gauge" },
-  "historical-segment/count" : { "dimensions" : ["dataSource", "tier", "priority"], "type" : "gauge" }
+  "historical-segment/count" : { "dimensions" : ["dataSource", "tier", "priority"], "type" : "gauge" },
+
+  "jetty/numOpenConnections": { "dimensions" : [], "type" : "gauge" },
+  "jetty/threadPool/total": { "dimensions" : [], "type" : "gauge" },
+  "jetty/threadPool/idle": { "dimensions" : [], "type" : "gauge" },
+  "jetty/threadPool/busy": { "dimensions" : [], "type" : "gauge" },
+  "jetty/threadPool/isLowOnThreads": { "dimensions" : [], "type" : "gauge" },
+  "jetty/threadPool/min": { "dimensions" : [], "type" : "gauge" },
+  "jetty/threadPool/max": { "dimensions" : [], "type" : "gauge" },
+  "jetty/threadPool/queueSize": { "dimensions" : [], "type" : "gauge" }
 }

server/src/main/java/org/apache/druid/server/initialization/jetty/JettyServerModule.java

@@ -109,6 +109,7 @@ public class JettyServerModule extends JerseyServletModule
 
   private static final AtomicInteger ACTIVE_CONNECTIONS = new AtomicInteger();
   private static final String HTTP_1_1_STRING = "HTTP/1.1";
+  private static QueuedThreadPool jettyServerThreadPool = null;
 
   @Override
   protected void configureServlets()
@@ -229,6 +230,7 @@ public class JettyServerModule extends JerseyServletModule
     }
 
     threadPool.setDaemon(true);
+    jettyServerThreadPool = threadPool;
 
     final Server server = new Server(threadPool);
 
@@ -528,6 +530,16 @@ public class JettyServerModule extends JerseyServletModule
       final ServiceMetricEvent.Builder builder = new ServiceMetricEvent.Builder();
       MonitorUtils.addDimensionsToBuilder(builder, dimensions);
       emitter.emit(builder.build("jetty/numOpenConnections", ACTIVE_CONNECTIONS.get()));
+      if (jettyServerThreadPool != null) {
+        emitter.emit(builder.build("jetty/threadPool/total", jettyServerThreadPool.getThreads()));
+        emitter.emit(builder.build("jetty/threadPool/idle", jettyServerThreadPool.getIdleThreads()));
+        emitter.emit(builder.build("jetty/threadPool/isLowOnThreads", jettyServerThreadPool.isLowOnThreads() ? 1 : 0));
+        emitter.emit(builder.build("jetty/threadPool/min", jettyServerThreadPool.getMinThreads()));
+        emitter.emit(builder.build("jetty/threadPool/max", jettyServerThreadPool.getMaxThreads()));
+        emitter.emit(builder.build("jetty/threadPool/queueSize", jettyServerThreadPool.getQueueSize()));
+        emitter.emit(builder.build("jetty/threadPool/busy", jettyServerThreadPool.getBusyThreads()));
+      }
+
       return true;
     }
   }
@@ -578,4 +590,10 @@ public class JettyServerModule extends JerseyServletModule
   {
     return ACTIVE_CONNECTIONS.get();
   }
+
+  @VisibleForTesting
+  public static void setJettyServerThreadPool(QueuedThreadPool threadPool)
+  {
+    jettyServerThreadPool = threadPool;
+  }
 }

server/src/test/java/org/apache/druid/server/initialization/jetty/JettyServerModuleTest.java

@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.server.initialization.jetty;
+
+import org.apache.druid.java.util.common.Pair;
+import org.apache.druid.java.util.emitter.core.Emitter;
+import org.apache.druid.java.util.emitter.core.Event;
+import org.apache.druid.java.util.emitter.service.ServiceEmitter;
+import org.apache.druid.java.util.emitter.service.ServiceMetricEvent;
+import org.eclipse.jetty.util.thread.QueuedThreadPool;
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+public class JettyServerModuleTest
+{
+  @Test
+  public void testJettyServerModule()
+  {
+    List<Event> events = new ArrayList<>();
+    ServiceEmitter serviceEmitter = new ServiceEmitter("service", "host", Mockito.mock(Emitter.class))
+    {
+      @Override
+      public void emit(Event event)
+      {
+        events.add(event);
+      }
+    };
+    QueuedThreadPool jettyServerThreadPool = Mockito.mock(QueuedThreadPool.class);
+    JettyServerModule.setJettyServerThreadPool(jettyServerThreadPool);
+    Mockito.when(jettyServerThreadPool.getThreads()).thenReturn(100);
+    Mockito.when(jettyServerThreadPool.getIdleThreads()).thenReturn(40);
+    Mockito.when(jettyServerThreadPool.isLowOnThreads()).thenReturn(true);
+    Mockito.when(jettyServerThreadPool.getMinThreads()).thenReturn(30);
+    Mockito.when(jettyServerThreadPool.getMaxThreads()).thenReturn(100);
+    Mockito.when(jettyServerThreadPool.getQueueSize()).thenReturn(50);
+    Mockito.when(jettyServerThreadPool.getBusyThreads()).thenReturn(60);
+
+    JettyServerModule.JettyMonitor jettyMonitor = new JettyServerModule.JettyMonitor("ds", "t0");
+    jettyMonitor.doMonitor(serviceEmitter);
+
+    Assert.assertEquals(8, events.size());
+    List<Pair<String, Number>> expectedEvents = Arrays.asList(
+        new Pair<>("jetty/numOpenConnections", 0),
+        new Pair<>("jetty/threadPool/total", 100),
+        new Pair<>("jetty/threadPool/idle", 40),
+        new Pair<>("jetty/threadPool/isLowOnThreads", 1),
+        new Pair<>("jetty/threadPool/min", 30),
+        new Pair<>("jetty/threadPool/max", 100),
+        new Pair<>("jetty/threadPool/queueSize", 50),
+        new Pair<>("jetty/threadPool/busy", 60)
+    );
+
+    for (int i = 0; i < expectedEvents.size(); i++) {
+      Pair<String, Number> expected = expectedEvents.get(i);
+      ServiceMetricEvent actual = (ServiceMetricEvent) (events.get(i));
+      Assert.assertEquals(expected.lhs, actual.getMetric());
+      Assert.assertEquals(expected.rhs, actual.getValue());
+    }
+  }
+}

website/.spelling

@@ -1293,6 +1293,8 @@ bufferpoolName
 cms
 cpuName
 cpuTime
+druid.server.http.numThreads
+druid.server.http.queueSize
 fsDevName
 fsDirName
 fsOptions
@@ -1318,6 +1320,9 @@ remoteAddress
 serviceName
 taskStatus
 taskType
+threadPoolNumBusyThreads.
+threadPoolNumIdleThreads
+threadPoolNumTotalThreads.
  - ../docs/operations/other-hadoop.md
 CDH
 Classloader