Eclipse
/
jetty.project
mirror of https://github.com/jetty/jetty.project.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

352684 Implemented spinning thread analyzer

This commit is contained in:
Michael Gorovoy 2011-07-20 19:30:44 -04:00
parent 3919023be5
commit 0cc914c4bb
10 changed files with 721 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
VERSION.txt
View File

@ -4,6 +4,7 @@ jetty-7.5.0-SNAPSHOT
+ 351576 Do not use deprecated method File.toURL()
+ 352046 Need try/catch around features set in XmlParser
+ 352176 xml parsing on startElement should be more flexible on using qName or localName
+ 352684 Implemented spinning thread analyzer
jetty-7.4.4.v20110707 July 7th 2011
+ 308851 Converted all jetty-client module tests to JUnit 4

19
jetty-distribution/pom.xml
View File

@ -276,6 +276,16 @@
<includes>**</includes>
<outputDirectory>${assembly-directory}</outputDirectory>
</artifactItem>
<artifactItem>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-monitor</artifactId>
<version>${project.version}</version>
<classifier>config</classifier>
<type>jar</type>
<overWrite>true</overWrite>
<includes>**</includes>
<outputDirectory>${assembly-directory}</outputDirectory>
</artifactItem>
<artifactItem>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-overlay-deployer</artifactId>
@ -451,6 +461,15 @@
<includes>**</includes>
<outputDirectory>${assembly-directory}/lib</outputDirectory>
</artifactItem>
<artifactItem>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-monitor</artifactId>
<version>${project.version}</version>
<type>jar</type>
<overWrite>true</overWrite>
<includes>**</includes>
<outputDirectory>${assembly-directory}/lib/monitor</outputDirectory>
</artifactItem>
<artifactItem>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-plus</artifactId>

11
jetty-monitor/README.txt Normal file
View File

@ -0,0 +1,11 @@
The ThreadMonitor is distributed as part of the jetty-monitor module.
In order to start ThreadMonitor when server starts up, the following command line should be used.
java -jar start.jar OPTIONS=monitor jetty-monitor.xml
To run ThreadMonitor on a Jetty installation that doesn't include jetty-monitor module, the jetty-monitor-[version].jar file needs to be copied into ${jetty.home}/lib/ext directory, and jetty-monitor.xml configuration file needs to be copied into ${jetty.home}/etc directory. Subsequently, the following command line should be used.
java -jar start.jar etc/jetty-monitor.xml
If running Jetty on Java VM version 1.5, the -Dcom.sun.management.jmxremote option should be added to the command lines above in order to enable the JMX agent.

83
jetty-monitor/pom.xml Normal file
View File

@ -0,0 +1,83 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<parent>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-project</artifactId>
<version>7.5.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>jetty-monitor</artifactId>
<name>Jetty :: Monitoring</name>
<description>Performance monitoring artifact for jetty.</description>
<properties>
<bundle-symbolic-name>${project.groupId}.jmx</bundle-symbolic-name>
</properties>
<build>
<plugins>
<plugin>
<groupId>org.apache.felix</groupId>
<artifactId>maven-bundle-plugin</artifactId>
<extensions>true</extensions>
<executions>
<execution>
<goals>
<goal>manifest</goal>
</goals>
<configuration>
<instructions>
<Import-Package>javax.management.*,*</Import-Package>
</instructions>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<!--
Required for OSGI
-->
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<configuration>
<archive>
<manifestFile>${project.build.outputDirectory}/META-INF/MANIFEST.MF</manifestFile>
</archive>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
<configuration>
<descriptorRefs>
<descriptorRef>config</descriptorRef>
</descriptorRefs>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>findbugs-maven-plugin</artifactId>
<configuration>
<onlyAnalyze>org.eclipse.jetty.jmx.*</onlyAnalyze>
</configuration>
</plugin>
</plugins>
</build>
<dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-util</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
</project>

12
jetty-monitor/src/main/config/etc/jetty-monitor.xml Normal file
View File

@ -0,0 +1,12 @@
<?xml version="1.0"?>
<!DOCTYPE Configure PUBLIC "-//Jetty//Configure//EN" "http://www.eclipse.org/jetty/configure.dtd">
<Configure id="Server" class="org.eclipse.jetty.server.Server">
<!-- Create Thread Monitor, and add to the Server as a lifecycle -->
<Call name="addBean">
<Arg>
<New class="org.eclipse.jetty.monitor.ThreadMonitor"/>
</Arg>
</Call>
</Configure>

468
jetty-monitor/src/main/java/org/eclipse/jetty/monitor/ThreadMonitor.java Normal file
View File

@ -0,0 +1,468 @@
// ========================================================================
// Copyright (c) Webtide LLC
// ------------------------------------------------------------------------
// All rights reserved. This program and the accompanying materials
// are made available under the terms of the Eclipse Public License v1.0
// and Apache License v2.0 which accompanies this distribution.
// The Eclipse Public License is available at
// http://www.eclipse.org/legal/epl-v10.html
// The Apache License v2.0 is available at
// http://www.opensource.org/licenses/apache2.0.php
// You may elect to redistribute this code under either of these licenses.
// ========================================================================
package org.eclipse.jetty.monitor;
import java.lang.management.ManagementFactory;
import java.lang.management.ThreadInfo;
import java.lang.management.ThreadMXBean;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.eclipse.jetty.util.component.AbstractLifeCycle;
import org.eclipse.jetty.util.log.Log;
import org.eclipse.jetty.util.log.Logger;
/* ------------------------------------------------------------ */
public class ThreadMonitor extends AbstractLifeCycle implements Runnable
{
private int _scanInterval;
private int _busyThreshold;
private int _stackDepth;
private ThreadMXBean _threadBean;
private Method findDeadlockedThreadsMethod;
private Thread _runner;
private boolean _done;
private Logger _logger;
private Map<Long,ExtThreadInfo> _extInfo;
/* ------------------------------------------------------------ */
/**
* Instantiates a new thread monitor.
*
* @throws Exception
*/
public ThreadMonitor() throws Exception
{
this(5, 95, 3);
}
/* ------------------------------------------------------------ */
/**
* Instantiates a new thread monitor.
*
* @param interval scan interval
* @param threshold busy threshold
* @param depth stack compare depth
* @throws Exception
*/
public ThreadMonitor(int interval, int threshold, int depth) throws Exception
{
_scanInterval = interval * 1000;
_busyThreshold = threshold;
_stackDepth = depth;
_logger = Log.getLogger(getClass().getName());
_extInfo = new HashMap<Long, ExtThreadInfo>();
init();
}
/* ------------------------------------------------------------ */
/**
* @see org.eclipse.jetty.util.component.AbstractLifeCycle#doStart()
*/
public void doStart()
{
_done = false;
_runner = new Thread(this);
_runner.start();
Log.info("Thread Monitor started successfully");
}
/* ------------------------------------------------------------ */
/**
* @see org.eclipse.jetty.util.component.AbstractLifeCycle#doStop()
*/
public void doStop()
{
if (_runner != null)
{
_done = true;
try
{
_runner.join();
}
catch (InterruptedException ex) {}
}
}
/* ------------------------------------------------------------ */
/**
* Initialize JMX objects.
*/
protected void init()
{
_threadBean = ManagementFactory.getThreadMXBean();
if (_threadBean.isThreadCpuTimeSupported())
{
_threadBean.setThreadCpuTimeEnabled(true);
}
String versionStr = System.getProperty("java.version");
float version = Float.valueOf(versionStr.substring(0,versionStr.lastIndexOf('.')));
try
{
if (version < 1.6)
{
findDeadlockedThreadsMethod = ThreadMXBean.class.getMethod("findMonitorDeadlockedThreads");
}
else
{
findDeadlockedThreadsMethod = ThreadMXBean.class.getMethod("findDeadlockedThreads");
}
}
catch (Exception ex)
{
Log.debug(ex);
}
}
/* ------------------------------------------------------------ */
/**
* Find deadlocked threads.
*
* @return array of the deadlocked thread ids
* @throws Exception the exception
*/
protected long[] findDeadlockedThreads() throws Exception
{
return (long[]) findDeadlockedThreadsMethod.invoke(_threadBean,(Object[])null);
}
/* ------------------------------------------------------------ */
/**
* Retrieve all avaliable thread ids
*
* @return array of thread ids
*/
protected long[] getAllThreadIds()
{
return _threadBean.getAllThreadIds();
}
/* ------------------------------------------------------------ */
/**
* Retrieve the cpu time for specified thread.
*
* @param id thread id
* @return cpu time of the thread
*/
protected long getThreadCpuTime(long id)
{
return _threadBean.getThreadCpuTime(id);
}
/* ------------------------------------------------------------ */
/**
* Retrieve thread info.
*
* @param id thread id
* @param maxDepth maximum stack depth
* @return thread info
*/
protected ThreadInfo getThreadInfo(long id, int maxDepth)
{
return _threadBean.getThreadInfo(id,maxDepth);
}
/* ------------------------------------------------------------ */
/**
* Output thread info to log.
*
* @param threads thread info list
*/
protected void dump(final List<ThreadInfo> threads)
{
if (threads != null && threads.size() > 0)
{
for (ThreadInfo info : threads)
{
StringBuffer msg = new StringBuffer();
if (info.getLockOwnerId() < 0)
{
msg.append(String.format("Thread %s[%d] is spinning", info.getThreadName(), info.getThreadId()));
}
else
{
msg.append(String.format("Thread %s[%d] is %s", info.getThreadName(), info.getThreadId(), info.getThreadState()));
msg.append(String.format(" on %s owned by %s[%d]", info.getLockName(), info.getLockOwnerName(), info.getLockOwnerId()));
}
_logger.warn(new ThreadMonitorException(msg.toString(), info.getStackTrace()));
}
}
}
/* ------------------------------------------------------------ */
/**
* @see java.lang.Runnable#run()
*/
public void run()
{
long currTime;
long lastTime = 0;
while (!_done)
{
currTime = System.currentTimeMillis();
if (currTime < lastTime + _scanInterval)
{
try
{
Thread.sleep(50);
}
catch (InterruptedException ex)
{
Log.ignore(ex);
}
continue;
}
List<ThreadInfo> threadInfo = new ArrayList<ThreadInfo>();
findSpinningThreads(threadInfo);
findDeadlockedThreads(threadInfo);
lastTime = System.currentTimeMillis();
if (threadInfo.size() > 0)
{
dump(threadInfo);
}
}
}
/* ------------------------------------------------------------ */
/**
* Find spinning threads.
*
* @param threadInfo thread info list to add the results
* @return thread info list
*/
private List<ThreadInfo> findSpinningThreads(final List<ThreadInfo> threadInfo)
{
if (threadInfo != null)
{
try
{
long[] allThreadId = getAllThreadIds();
for (int idx=0; idx < allThreadId.length; idx++)
{
long currId = allThreadId[idx];
if (currId == _runner.getId())
{
continue;
}
long currCpuTime = getThreadCpuTime(currId);
long currNanoTime = System.nanoTime();
ExtThreadInfo currExtInfo = _extInfo.get(Long.valueOf(currId));
if (currExtInfo != null)
{
long elapsedCpuTime = currCpuTime - currExtInfo.getLastCpuTime();
long elapsedNanoTime = currNanoTime - currExtInfo.getLastSampleTime();
if (((elapsedCpuTime * 100.0) / elapsedNanoTime) > _busyThreshold)
{
ThreadInfo currInfo = getThreadInfo(currId, Integer.MAX_VALUE);
if (currInfo != null)
{
StackTraceElement[] lastStackTrace = currExtInfo.getStackTrace();
currExtInfo.setStackTrace(currInfo.getStackTrace());
if (lastStackTrace != null
&& matchStackTraces(lastStackTrace, currInfo.getStackTrace())) {
threadInfo.add(currInfo);
}
}
}
}
else
{
currExtInfo = new ExtThreadInfo(currId);
_extInfo.put(Long.valueOf(currId), currExtInfo);
}
currExtInfo.setLastCpuTime(currCpuTime);
currExtInfo.setLastSampleTime(currNanoTime);
}
}
catch (Exception ex)
{
Log.debug(ex);
}
}
return threadInfo;
}
/* ------------------------------------------------------------ */
/**
* Find deadlocked threads.
*
* @param threadInfo thread info list to add the results
* @return thread info list
*/
private List<ThreadInfo> findDeadlockedThreads(final List<ThreadInfo> threadInfo)
{
if (threadInfo != null)
{
try
{
long[] threads = findDeadlockedThreads();
if (threads != null && threads.length > 0)
{
ThreadInfo currInfo;
for (int idx=0; idx < threads.length; idx++)
{
currInfo = getThreadInfo(threads[idx], Integer.MAX_VALUE);
if (currInfo != null)
{
threadInfo.add(currInfo);
}
}
}
}
catch (Exception ex)
{
Log.debug(ex);
}
}
return threadInfo;
}
/* ------------------------------------------------------------ */
/**
* Match stack traces.
*
* @param lastStackTrace last stack trace
* @param stackTrace current stack trace
* @return true, if successful
*/
private boolean matchStackTraces(StackTraceElement[] lastStackTrace, StackTraceElement[] stackTrace)
{
boolean match = true;
int count = Math.min(_stackDepth, Math.min(lastStackTrace.length, stackTrace.length));
for (int idx=0; idx < count; idx++)
{
if (!stackTrace[idx].equals(lastStackTrace[idx]))
{
match = false;
break;
}
}
return match;
}
/* ------------------------------------------------------------ */
private class ExtThreadInfo
{
private long _threadId;
private long _lastCpuTime;
private long _lastSampleTime;
private StackTraceElement[] _stackTrace;
/* ------------------------------------------------------------ */
public ExtThreadInfo(long threadId)
{
_threadId = threadId;
}
/* ------------------------------------------------------------ */
/**
* @return thread id associated with the instance
*/
public long getThreadId()
{
return _threadId;
}
/* ------------------------------------------------------------ */
/**
* @return the last CPU time of the thread
*/
public long getLastCpuTime()
{
return _lastCpuTime;
}
/* ------------------------------------------------------------ */
/**
* Set the last CPU time.
*
* @param lastCpuTime new last CPU time
*/
public void setLastCpuTime(long lastCpuTime)
{
this._lastCpuTime = lastCpuTime;
}
/* ------------------------------------------------------------ */
/**
* @return the time of last sample
*/
public long getLastSampleTime()
{
return _lastSampleTime;
}
/* ------------------------------------------------------------ */
/**
* Sets the last sample time.
*
* @param lastSampleTime the time of last sample
*/
public void setLastSampleTime(long lastSampleTime)
{
_lastSampleTime = lastSampleTime;
}
/* ------------------------------------------------------------ */
/**
* Gets the stack trace.
*
* @return the stack trace
*/
public StackTraceElement[] getStackTrace()
{
return _stackTrace;
}
/* ------------------------------------------------------------ */
/**
* Sets the stack trace.
*
* @param stackTrace the new stack trace
*/
public void setStackTrace(StackTraceElement[] stackTrace)
{
_stackTrace = stackTrace;
}
}
}

30
jetty-monitor/src/main/java/org/eclipse/jetty/monitor/ThreadMonitorException.java Normal file
View File

@ -0,0 +1,30 @@
// ========================================================================
// Copyright (c) Webtide LLC
// ------------------------------------------------------------------------
// All rights reserved. This program and the accompanying materials
// are made available under the terms of the Eclipse Public License v1.0
// and Apache License v2.0 which accompanies this distribution.
// The Eclipse Public License is available at
// http://www.eclipse.org/legal/epl-v10.html
// The Apache License v2.0 is available at
// http://www.opensource.org/licenses/apache2.0.php
// You may elect to redistribute this code under either of these licenses.
// ========================================================================
package org.eclipse.jetty.monitor;
/* ------------------------------------------------------------ */
/**
*/
public class ThreadMonitorException extends Exception
{
private static final long serialVersionUID = -4345223166315716918L;
public ThreadMonitorException(String message, StackTraceElement[] stackTrace)
{
super(message);
setStackTrace(stackTrace);
}
}

2
jetty-monitor/src/main/resources/org/eclipse/jetty/monitor/jmx/ThreadMonitor-mbean.properties Normal file
View File

@ -0,0 +1,2 @@
ThreadMonitor: Detect and report spinning and deadlocked threads

94
jetty-monitor/src/test/java/org/eclipse/jetty/monitor/ThreadMonitorTest.java Normal file
View File

@ -0,0 +1,94 @@
// ========================================================================
// Copyright (c) Webtide LLC
// ------------------------------------------------------------------------
// All rights reserved. This program and the accompanying materials
// are made available under the terms of the Eclipse Public License v1.0
// and Apache License v2.0 which accompanies this distribution.
// The Eclipse Public License is available at
// http://www.eclipse.org/legal/epl-v10.html
// The Apache License v2.0 is available at
// http://www.opensource.org/licenses/apache2.0.php
// You may elect to redistribute this code under either of these licenses.
// ========================================================================
package org.eclipse.jetty.monitor;
import static org.junit.Assert.assertTrue;
import java.lang.management.ThreadInfo;
import java.util.List;
import org.junit.Test;
/* ------------------------------------------------------------ */
/**
*/
public class ThreadMonitorTest
{
private int count;
@Test
public void monitorTest() throws Exception
{
count = 0;
ThreadMonitor monitor = new ThreadMonitor(1,95,2)
{
@Override
protected void dump(List<ThreadInfo> threads)
{
++count;
super.dump(threads);
}
};
monitor.start();
Spinner spinner = new Spinner();
Thread runner = new Thread(spinner);
runner.start();
Thread.sleep(15000);
spinner.setDone();
monitor.stop();
assertTrue(count > 10);
}
private class Spinner implements Runnable
{
private boolean done = false;
/* ------------------------------------------------------------ */
/**
*/
public void setDone()
{
done = true;
}
/* ------------------------------------------------------------ */
public void run()
{
while (!done)
{
foo();
}
}
private void foo()
{
for (int i=0; i<Integer.MAX_VALUE; i++)
{
long f = 1;
for(int j=1; j<=i; j++)
{
f += j;
}
}
}
}
}

1
pom.xml
View File

@ -315,6 +315,7 @@
<module>jetty-plus</module>
<module>jetty-rewrite</module>
<module>jetty-policy</module>
<module>jetty-monitor</module>
<module>jetty-start</module>
<module>jetty-nested</module>
<module>jetty-overlay-deployer</module>