diff --git a/VERSION.txt b/VERSION.txt index 8a11a4aa404..b3e79a2087b 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -4,6 +4,7 @@ jetty-7.5.0-SNAPSHOT + 351576 Do not use deprecated method File.toURL() + 352046 Need try/catch around features set in XmlParser + 352176 xml parsing on startElement should be more flexible on using qName or localName + + 352684 Implemented spinning thread analyzer jetty-7.4.4.v20110707 July 7th 2011 + 308851 Converted all jetty-client module tests to JUnit 4 diff --git a/jetty-distribution/pom.xml b/jetty-distribution/pom.xml index 1ebfb900e64..aeb2ee1af7e 100644 --- a/jetty-distribution/pom.xml +++ b/jetty-distribution/pom.xml @@ -276,6 +276,16 @@ ** ${assembly-directory} + + org.eclipse.jetty + jetty-monitor + ${project.version} + config + jar + true + ** + ${assembly-directory} + org.eclipse.jetty jetty-overlay-deployer @@ -451,6 +461,15 @@ ** ${assembly-directory}/lib + + org.eclipse.jetty + jetty-monitor + ${project.version} + jar + true + ** + ${assembly-directory}/lib/monitor + org.eclipse.jetty jetty-plus diff --git a/jetty-monitor/README.txt b/jetty-monitor/README.txt new file mode 100644 index 00000000000..920af92e3be --- /dev/null +++ b/jetty-monitor/README.txt @@ -0,0 +1,11 @@ +The ThreadMonitor is distributed as part of the jetty-monitor module. + +In order to start ThreadMonitor when server starts up, the following command line should be used. + + java -jar start.jar OPTIONS=monitor jetty-monitor.xml + +To run ThreadMonitor on a Jetty installation that doesn't include jetty-monitor module, the jetty-monitor-[version].jar file needs to be copied into ${jetty.home}/lib/ext directory, and jetty-monitor.xml configuration file needs to be copied into ${jetty.home}/etc directory. Subsequently, the following command line should be used. + + java -jar start.jar etc/jetty-monitor.xml + +If running Jetty on Java VM version 1.5, the -Dcom.sun.management.jmxremote option should be added to the command lines above in order to enable the JMX agent. diff --git a/jetty-monitor/pom.xml b/jetty-monitor/pom.xml new file mode 100644 index 00000000000..f0da6e9f5b0 --- /dev/null +++ b/jetty-monitor/pom.xml @@ -0,0 +1,83 @@ + + + org.eclipse.jetty + jetty-project + 7.5.0-SNAPSHOT + + 4.0.0 + jetty-monitor + Jetty :: Monitoring + Performance monitoring artifact for jetty. + + ${project.groupId}.jmx + + + + + org.apache.felix + maven-bundle-plugin + true + + + + manifest + + + + javax.management.*,* + + + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + ${project.build.outputDirectory}/META-INF/MANIFEST.MF + + + + + org.apache.maven.plugins + maven-assembly-plugin + + + package + + single + + + + config + + + + + + + org.codehaus.mojo + findbugs-maven-plugin + + org.eclipse.jetty.jmx.* + + + + + + + junit + junit + test + + + org.eclipse.jetty + jetty-util + ${project.version} + + + diff --git a/jetty-monitor/src/main/config/etc/jetty-monitor.xml b/jetty-monitor/src/main/config/etc/jetty-monitor.xml new file mode 100644 index 00000000000..b54243585c9 --- /dev/null +++ b/jetty-monitor/src/main/config/etc/jetty-monitor.xml @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/jetty-monitor/src/main/java/org/eclipse/jetty/monitor/ThreadMonitor.java b/jetty-monitor/src/main/java/org/eclipse/jetty/monitor/ThreadMonitor.java new file mode 100644 index 00000000000..450b61076ed --- /dev/null +++ b/jetty-monitor/src/main/java/org/eclipse/jetty/monitor/ThreadMonitor.java @@ -0,0 +1,468 @@ +// ======================================================================== +// Copyright (c) Webtide LLC +// ------------------------------------------------------------------------ +// All rights reserved. This program and the accompanying materials +// are made available under the terms of the Eclipse Public License v1.0 +// and Apache License v2.0 which accompanies this distribution. +// The Eclipse Public License is available at +// http://www.eclipse.org/legal/epl-v10.html +// The Apache License v2.0 is available at +// http://www.opensource.org/licenses/apache2.0.php +// You may elect to redistribute this code under either of these licenses. +// ======================================================================== + + +package org.eclipse.jetty.monitor; + +import java.lang.management.ManagementFactory; +import java.lang.management.ThreadInfo; +import java.lang.management.ThreadMXBean; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.eclipse.jetty.util.component.AbstractLifeCycle; +import org.eclipse.jetty.util.log.Log; +import org.eclipse.jetty.util.log.Logger; + + +/* ------------------------------------------------------------ */ +public class ThreadMonitor extends AbstractLifeCycle implements Runnable +{ + private int _scanInterval; + private int _busyThreshold; + private int _stackDepth; + + private ThreadMXBean _threadBean; + private Method findDeadlockedThreadsMethod; + + private Thread _runner; + private boolean _done; + private Logger _logger; + + private Map _extInfo; + + /* ------------------------------------------------------------ */ + /** + * Instantiates a new thread monitor. + * + * @throws Exception + */ + public ThreadMonitor() throws Exception + { + this(5, 95, 3); + } + + /* ------------------------------------------------------------ */ + /** + * Instantiates a new thread monitor. + * + * @param interval scan interval + * @param threshold busy threshold + * @param depth stack compare depth + * @throws Exception + */ + public ThreadMonitor(int interval, int threshold, int depth) throws Exception + { + _scanInterval = interval * 1000; + _busyThreshold = threshold; + _stackDepth = depth; + + _logger = Log.getLogger(getClass().getName()); + _extInfo = new HashMap(); + + init(); + } + + /* ------------------------------------------------------------ */ + /** + * @see org.eclipse.jetty.util.component.AbstractLifeCycle#doStart() + */ + public void doStart() + { + _done = false; + + _runner = new Thread(this); + _runner.start(); + + Log.info("Thread Monitor started successfully"); + } + + /* ------------------------------------------------------------ */ + /** + * @see org.eclipse.jetty.util.component.AbstractLifeCycle#doStop() + */ + public void doStop() + { + if (_runner != null) + { + _done = true; + try + { + _runner.join(); + } + catch (InterruptedException ex) {} + } + } + + /* ------------------------------------------------------------ */ + /** + * Initialize JMX objects. + */ + protected void init() + { + _threadBean = ManagementFactory.getThreadMXBean(); + if (_threadBean.isThreadCpuTimeSupported()) + { + _threadBean.setThreadCpuTimeEnabled(true); + } + + String versionStr = System.getProperty("java.version"); + float version = Float.valueOf(versionStr.substring(0,versionStr.lastIndexOf('.'))); + try + { + if (version < 1.6) + { + findDeadlockedThreadsMethod = ThreadMXBean.class.getMethod("findMonitorDeadlockedThreads"); + } + else + { + findDeadlockedThreadsMethod = ThreadMXBean.class.getMethod("findDeadlockedThreads"); + } + } + catch (Exception ex) + { + Log.debug(ex); + } + } + + /* ------------------------------------------------------------ */ + /** + * Find deadlocked threads. + * + * @return array of the deadlocked thread ids + * @throws Exception the exception + */ + protected long[] findDeadlockedThreads() throws Exception + { + return (long[]) findDeadlockedThreadsMethod.invoke(_threadBean,(Object[])null); + } + + /* ------------------------------------------------------------ */ + /** + * Retrieve all avaliable thread ids + * + * @return array of thread ids + */ + protected long[] getAllThreadIds() + { + return _threadBean.getAllThreadIds(); + } + + /* ------------------------------------------------------------ */ + /** + * Retrieve the cpu time for specified thread. + * + * @param id thread id + * @return cpu time of the thread + */ + protected long getThreadCpuTime(long id) + { + return _threadBean.getThreadCpuTime(id); + } + + /* ------------------------------------------------------------ */ + /** + * Retrieve thread info. + * + * @param id thread id + * @param maxDepth maximum stack depth + * @return thread info + */ + protected ThreadInfo getThreadInfo(long id, int maxDepth) + { + return _threadBean.getThreadInfo(id,maxDepth); + } + + /* ------------------------------------------------------------ */ + /** + * Output thread info to log. + * + * @param threads thread info list + */ + protected void dump(final List threads) + { + if (threads != null && threads.size() > 0) + { + for (ThreadInfo info : threads) + { + StringBuffer msg = new StringBuffer(); + if (info.getLockOwnerId() < 0) + { + msg.append(String.format("Thread %s[%d] is spinning", info.getThreadName(), info.getThreadId())); + } + else + { + msg.append(String.format("Thread %s[%d] is %s", info.getThreadName(), info.getThreadId(), info.getThreadState())); + msg.append(String.format(" on %s owned by %s[%d]", info.getLockName(), info.getLockOwnerName(), info.getLockOwnerId())); + } + + _logger.warn(new ThreadMonitorException(msg.toString(), info.getStackTrace())); + } + } + } + + /* ------------------------------------------------------------ */ + /** + * @see java.lang.Runnable#run() + */ + public void run() + { + long currTime; + long lastTime = 0; + while (!_done) + { + currTime = System.currentTimeMillis(); + if (currTime < lastTime + _scanInterval) + { + try + { + Thread.sleep(50); + } + catch (InterruptedException ex) + { + Log.ignore(ex); + } + continue; + } + + List threadInfo = new ArrayList(); + + findSpinningThreads(threadInfo); + findDeadlockedThreads(threadInfo); + + lastTime = System.currentTimeMillis(); + + if (threadInfo.size() > 0) + { + dump(threadInfo); + } + } + + } + + /* ------------------------------------------------------------ */ + /** + * Find spinning threads. + * + * @param threadInfo thread info list to add the results + * @return thread info list + */ + private List findSpinningThreads(final List threadInfo) + { + if (threadInfo != null) + { + try + { + long[] allThreadId = getAllThreadIds(); + for (int idx=0; idx < allThreadId.length; idx++) + { + long currId = allThreadId[idx]; + + if (currId == _runner.getId()) + { + continue; + } + + long currCpuTime = getThreadCpuTime(currId); + long currNanoTime = System.nanoTime(); + + ExtThreadInfo currExtInfo = _extInfo.get(Long.valueOf(currId)); + if (currExtInfo != null) + { + long elapsedCpuTime = currCpuTime - currExtInfo.getLastCpuTime(); + long elapsedNanoTime = currNanoTime - currExtInfo.getLastSampleTime(); + + if (((elapsedCpuTime * 100.0) / elapsedNanoTime) > _busyThreshold) + { + ThreadInfo currInfo = getThreadInfo(currId, Integer.MAX_VALUE); + if (currInfo != null) + { + StackTraceElement[] lastStackTrace = currExtInfo.getStackTrace(); + currExtInfo.setStackTrace(currInfo.getStackTrace()); + + if (lastStackTrace != null + && matchStackTraces(lastStackTrace, currInfo.getStackTrace())) { + threadInfo.add(currInfo); + } + } + } + } + else + { + currExtInfo = new ExtThreadInfo(currId); + _extInfo.put(Long.valueOf(currId), currExtInfo); + } + + currExtInfo.setLastCpuTime(currCpuTime); + currExtInfo.setLastSampleTime(currNanoTime); + } + } + catch (Exception ex) + { + Log.debug(ex); + } + } + + return threadInfo; + } + + /* ------------------------------------------------------------ */ + /** + * Find deadlocked threads. + * + * @param threadInfo thread info list to add the results + * @return thread info list + */ + private List findDeadlockedThreads(final List threadInfo) + { + if (threadInfo != null) + { + try + { + long[] threads = findDeadlockedThreads(); + if (threads != null && threads.length > 0) + { + ThreadInfo currInfo; + for (int idx=0; idx < threads.length; idx++) + { + currInfo = getThreadInfo(threads[idx], Integer.MAX_VALUE); + if (currInfo != null) + { + threadInfo.add(currInfo); + } + } + } + } + catch (Exception ex) + { + Log.debug(ex); + } + } + + return threadInfo; + } + + /* ------------------------------------------------------------ */ + /** + * Match stack traces. + * + * @param lastStackTrace last stack trace + * @param stackTrace current stack trace + * @return true, if successful + */ + private boolean matchStackTraces(StackTraceElement[] lastStackTrace, StackTraceElement[] stackTrace) + { + boolean match = true; + int count = Math.min(_stackDepth, Math.min(lastStackTrace.length, stackTrace.length)); + + for (int idx=0; idx < count; idx++) + { + if (!stackTrace[idx].equals(lastStackTrace[idx])) + { + match = false; + break; + } + } + return match; + } + + /* ------------------------------------------------------------ */ + private class ExtThreadInfo + { + private long _threadId; + + private long _lastCpuTime; + private long _lastSampleTime; + private StackTraceElement[] _stackTrace; + + /* ------------------------------------------------------------ */ + public ExtThreadInfo(long threadId) + { + _threadId = threadId; + } + + /* ------------------------------------------------------------ */ + /** + * @return thread id associated with the instance + */ + public long getThreadId() + { + return _threadId; + } + + /* ------------------------------------------------------------ */ + /** + * @return the last CPU time of the thread + */ + public long getLastCpuTime() + { + return _lastCpuTime; + } + + /* ------------------------------------------------------------ */ + /** + * Set the last CPU time. + * + * @param lastCpuTime new last CPU time + */ + public void setLastCpuTime(long lastCpuTime) + { + this._lastCpuTime = lastCpuTime; + } + + /* ------------------------------------------------------------ */ + /** + * @return the time of last sample + */ + public long getLastSampleTime() + { + return _lastSampleTime; + } + + /* ------------------------------------------------------------ */ + /** + * Sets the last sample time. + * + * @param lastSampleTime the time of last sample + */ + public void setLastSampleTime(long lastSampleTime) + { + _lastSampleTime = lastSampleTime; + } + + /* ------------------------------------------------------------ */ + /** + * Gets the stack trace. + * + * @return the stack trace + */ + public StackTraceElement[] getStackTrace() + { + return _stackTrace; + } + + /* ------------------------------------------------------------ */ + /** + * Sets the stack trace. + * + * @param stackTrace the new stack trace + */ + public void setStackTrace(StackTraceElement[] stackTrace) + { + _stackTrace = stackTrace; + } + } +} diff --git a/jetty-monitor/src/main/java/org/eclipse/jetty/monitor/ThreadMonitorException.java b/jetty-monitor/src/main/java/org/eclipse/jetty/monitor/ThreadMonitorException.java new file mode 100644 index 00000000000..50881116538 --- /dev/null +++ b/jetty-monitor/src/main/java/org/eclipse/jetty/monitor/ThreadMonitorException.java @@ -0,0 +1,30 @@ +// ======================================================================== +// Copyright (c) Webtide LLC +// ------------------------------------------------------------------------ +// All rights reserved. This program and the accompanying materials +// are made available under the terms of the Eclipse Public License v1.0 +// and Apache License v2.0 which accompanies this distribution. +// The Eclipse Public License is available at +// http://www.eclipse.org/legal/epl-v10.html +// The Apache License v2.0 is available at +// http://www.opensource.org/licenses/apache2.0.php +// You may elect to redistribute this code under either of these licenses. +// ======================================================================== + + +package org.eclipse.jetty.monitor; + + +/* ------------------------------------------------------------ */ +/** + */ +public class ThreadMonitorException extends Exception +{ + private static final long serialVersionUID = -4345223166315716918L; + + public ThreadMonitorException(String message, StackTraceElement[] stackTrace) + { + super(message); + setStackTrace(stackTrace); + } +} diff --git a/jetty-monitor/src/main/resources/org/eclipse/jetty/monitor/jmx/ThreadMonitor-mbean.properties b/jetty-monitor/src/main/resources/org/eclipse/jetty/monitor/jmx/ThreadMonitor-mbean.properties new file mode 100644 index 00000000000..4bdb8981735 --- /dev/null +++ b/jetty-monitor/src/main/resources/org/eclipse/jetty/monitor/jmx/ThreadMonitor-mbean.properties @@ -0,0 +1,2 @@ +ThreadMonitor: Detect and report spinning and deadlocked threads + diff --git a/jetty-monitor/src/test/java/org/eclipse/jetty/monitor/ThreadMonitorTest.java b/jetty-monitor/src/test/java/org/eclipse/jetty/monitor/ThreadMonitorTest.java new file mode 100644 index 00000000000..b41efa73ab5 --- /dev/null +++ b/jetty-monitor/src/test/java/org/eclipse/jetty/monitor/ThreadMonitorTest.java @@ -0,0 +1,94 @@ +// ======================================================================== +// Copyright (c) Webtide LLC +// ------------------------------------------------------------------------ +// All rights reserved. This program and the accompanying materials +// are made available under the terms of the Eclipse Public License v1.0 +// and Apache License v2.0 which accompanies this distribution. +// The Eclipse Public License is available at +// http://www.eclipse.org/legal/epl-v10.html +// The Apache License v2.0 is available at +// http://www.opensource.org/licenses/apache2.0.php +// You may elect to redistribute this code under either of these licenses. +// ======================================================================== + + +package org.eclipse.jetty.monitor; + +import static org.junit.Assert.assertTrue; + +import java.lang.management.ThreadInfo; +import java.util.List; + +import org.junit.Test; + + +/* ------------------------------------------------------------ */ +/** + */ +public class ThreadMonitorTest +{ + private int count; + + @Test + public void monitorTest() throws Exception + { + count = 0; + + ThreadMonitor monitor = new ThreadMonitor(1,95,2) + { + @Override + protected void dump(List threads) + { + ++count; + super.dump(threads); + } + }; + monitor.start(); + + Spinner spinner = new Spinner(); + Thread runner = new Thread(spinner); + runner.start(); + + Thread.sleep(15000); + + spinner.setDone(); + monitor.stop(); + + assertTrue(count > 10); + } + + + private class Spinner implements Runnable + { + private boolean done = false; + + /* ------------------------------------------------------------ */ + /** + */ + public void setDone() + { + done = true; + } + + /* ------------------------------------------------------------ */ + public void run() + { + while (!done) + { + foo(); + } + } + + private void foo() + { + for (int i=0; ijetty-plus jetty-rewrite jetty-policy + jetty-monitor jetty-start jetty-nested jetty-overlay-deployer