352684 Added a Dumpable to the thread monitor

jetty-monitor/src/main/config/etc/jetty-monitor.xml

@@ -10,13 +10,19 @@
         <Set name="busyThreshold">90</Set>
         <Set name="stackDepth">3</Set>
         <Set name="trailLength">2</Set>
-<!-- To enable logging CPU utilization for threads above specified threshold, -->
-<!-- uncomment the following lines, changing log interval (in milliseconds)  -->
-<!-- and log threshold (in percent) as desired.                              -->
-<!-- 
+        <!-- To enable logging CPU utilization for threads above specified threshold, -->
+        <!-- uncomment the following lines, changing log interval (in milliseconds)  -->
+        <!-- and log threshold (in percent) as desired.                              -->
+        <!-- 
         <Set name="logInterval">10000</Arg>
         <Set name="logThreshold">1</Arg>
--->
+        -->
+        
+        <!-- To enable detail dump of the server whenever a thread is detected as spinning, -->
+        <!-- uncomment the following lines. -->
+        <!--
+        <Set name="dumpable"><Ref id="Server"/></Set>
+        -->
       </New>
     </Arg>
   </Call>

jetty-monitor/src/main/java/org/eclipse/jetty/monitor/ThreadMonitor.java

@@ -24,6 +24,7 @@ import java.util.List;
 import java.util.Map;
 
 import org.eclipse.jetty.util.component.AbstractLifeCycle;
+import org.eclipse.jetty.util.component.Dumpable;
 import org.eclipse.jetty.util.log.Log;
 import org.eclipse.jetty.util.log.Logger;
 
@@ -43,6 +44,7 @@ public class ThreadMonitor extends AbstractLifeCycle implements Runnable
     private Thread _runner;
     private Logger _logger;
     private volatile boolean _done = true;
+    private Dumpable _dumpable;
 
     private Map<Long,ThreadMonitorInfo> _monitorInfo;
     
@@ -264,6 +266,24 @@ public class ThreadMonitor extends AbstractLifeCycle implements Runnable
         setLogThreshold(thresholdPercent);
     }
     
+    /* ------------------------------------------------------------ */
+    /**
+     * @return A {@link Dumpable} that is dumped whenever spinning threads are detected
+     */
+    public Dumpable getDumpable()
+    {
+        return _dumpable;
+    }
+
+    /* ------------------------------------------------------------ */
+    /**
+     * @param dumpable A {@link Dumpable} that is dumped whenever spinning threads are detected
+     */
+    public void setDumpable(Dumpable dumpable)
+    {
+        _dumpable = dumpable;
+    }
+
     /* ------------------------------------------------------------ */
     /**
      * @see org.eclipse.jetty.util.component.AbstractLifeCycle#doStart()
@@ -502,6 +522,7 @@ public class ThreadMonitor extends AbstractLifeCycle implements Runnable
             
             // Log thread information for threads that exceed logging threshold
             // or log spinning threads if their trace count is zero
+            boolean spinning=false;
             for (ThreadMonitorInfo info : all)
             {
                 if (logAll && info.getCpuUtilization() > _logThreshold 
@@ -512,8 +533,15 @@ public class ThreadMonitor extends AbstractLifeCycle implements Runnable
                             info.getThreadState(), info.getCpuUtilization(),
                             info.isSpinning() ? " SPINNING" : "");
                    _logger.info(message);
+                   spinning=true;
                 }
             }
+            
+            // Dump info
+            if (spinning && _dumpable!=null)
+            {
+                System.err.println(_dumpable.dump());
+            }
 
             // Log stack traces for spinning threads with positive trace count
             for (ThreadMonitorInfo info : all)

jetty-monitor/src/test/java/org/eclipse/jetty/monitor/ThreadMonitorTest.java

@@ -16,9 +16,13 @@ package org.eclipse.jetty.monitor;
 
 import static org.junit.Assert.assertTrue;
 
+import java.io.IOException;
 import java.util.Random;
 import java.util.concurrent.atomic.AtomicInteger;
 
+import org.eclipse.jetty.util.component.Dumpable;
+import org.eclipse.jetty.util.log.Log;
+import org.eclipse.jetty.util.log.StdErrLog;
 import org.junit.Test;
 
 
@@ -32,6 +36,9 @@ public class ThreadMonitorTest
     @Test
     public void monitorTest() throws Exception
     {
+        ((StdErrLog)Log.getLogger(ThreadMonitor.class.getName())).setHideStacks(true);
+        ((StdErrLog)Log.getLogger(ThreadMonitor.class.getName())).setSource(false);
+        
         final AtomicInteger countLogs=new AtomicInteger(0);
         final AtomicInteger countSpin=new AtomicInteger(0);
         
@@ -47,6 +54,19 @@ public class ThreadMonitorTest
                 super.logThreadInfo(logAll);
             }
         };
+        monitor.setDumpable(new Dumpable()
+        {
+            public void dump(Appendable out, String indent) throws IOException
+            {
+                out.append(dump());
+            }
+            
+            public String dump()
+            {
+                return "Dump Spinning";
+            }
+        });
+        
         monitor.logCpuUsage(2000,0);
         monitor.start();
         

jetty-util/src/main/java/org/eclipse/jetty/util/log/StdErrLog.java

@@ -14,6 +14,8 @@
 package org.eclipse.jetty.util.log;
 
 import java.security.AccessControlException;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
 
 import org.eclipse.jetty.util.DateCache;
 
@@ -43,6 +45,8 @@ public class StdErrLog implements Logger
             System.getProperty("org.eclipse.jetty.util.log.SOURCE",
                     System.getProperty("org.eclipse.jetty.util.log.stderr.SOURCE", "false")));
 
+    private final static ConcurrentMap<String,StdErrLog> __loggers = new ConcurrentHashMap<String, StdErrLog>();
+    
     static
     {
         try
@@ -315,9 +319,21 @@ public class StdErrLog implements Logger
 
     public Logger getLogger(String name)
     {
-        if ((name == null && this._name == null) || (name != null && name.equals(this._name)))
+        String fullname=_name == null || _name.length() == 0?name:_name + "." + name;
+        
+        if ((name == null && this._name == null) || fullname.equals(_name))
             return this;
-        return new StdErrLog(_name == null || _name.length() == 0?name:_name + "." + name);
+        
+        StdErrLog logger = __loggers.get(name);
+        if (logger==null)
+        {
+            StdErrLog sel=new StdErrLog(fullname);
+            logger=__loggers.putIfAbsent(fullname,sel);
+            if (logger==null)
+                logger=sel;
+        }
+        
+        return logger;
     }
 
     @Override

jetty-util/src/test/java/org/eclipse/jetty/util/log/LogTest.java

@@ -18,6 +18,8 @@ import static org.junit.Assert.assertTrue;
 import java.io.ByteArrayOutputStream;
 import java.io.PrintStream;
 
+import junit.framework.Assert;
+
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -123,8 +125,12 @@ public class LogTest
     public void testStdErrLogName()
     {
         StdErrLog log = new StdErrLog("test");
-            
+        Assert.assertEquals("test",log.getName());
+        
         Logger next=log.getLogger("next");
+        
+        Assert.assertEquals("test.next",next.getName());
+        
         next.info("testing {} {}","next","info");
         logContains(":test.next:testing next info");