From 73911b174092cd925ddc168d029830c880cd7b00 Mon Sep 17 00:00:00 2001 From: Zhihong Yu Date: Sun, 4 Nov 2012 00:20:27 +0000 Subject: [PATCH] HBASE-7086 Enhance ResourceChecker to log stack trace for potentially hanging threads git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1405443 13f79535-47bb-0310-9956-ffa450edef68 --- .../apache/hadoop/hbase/ResourceChecker.java | 28 ++++++++--- .../hbase/ResourceCheckerJUnitListener.java | 48 ++++++++++++++++--- .../ServerResourceCheckerJUnitListener.java | 3 +- 3 files changed, 64 insertions(+), 15 deletions(-) diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/ResourceChecker.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/ResourceChecker.java index afe74722af8..e56dea8669d 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/ResourceChecker.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/ResourceChecker.java @@ -24,7 +24,6 @@ import org.apache.commons.logging.LogFactory; import java.util.*; - /** * Utility class to check the resources: * - log them before and after each test method @@ -35,6 +34,10 @@ public class ResourceChecker { private static final Log LOG = LogFactory.getLog(ResourceChecker.class); private String tagLine; + enum Phase { + INITIAL, INTERMEDIATE, END + } + /** * Constructor * @param tagLine - the tagLine is added to the logs. Must be be null. @@ -80,11 +83,16 @@ public class ResourceChecker { /** * The value for the resource. + * @param phase */ - abstract public int getVal(); + abstract public int getVal(Phase phase); + + /* + * Retrieves List of Strings which would be logged in logEndings() + */ + public List getStringsToLog() { return null; } } - private List ras = new ArrayList(); private int[] initialValues; private int[] endingValues; @@ -92,18 +100,18 @@ public class ResourceChecker { private void fillInit() { initialValues = new int[ras.size()]; - fill(initialValues); + fill(Phase.INITIAL, initialValues); } private void fillEndings() { endingValues = new int[ras.size()]; - fill(endingValues); + fill(Phase.END, endingValues); } - private void fill(int[] vals) { + private void fill(Phase phase, int[] vals) { int i = 0; for (ResourceAnalyzer ra : ras) { - vals[i++] = ra.getVal(); + vals[i++] = ra.getVal(phase); } } @@ -151,6 +159,12 @@ public class ResourceChecker { if (sb.length() > 0) sb.append(", "); sb.append(ra.getName()).append("=").append(curN).append(" (was ").append(curP).append(")"); if (curN > curP) { + List strings = ra.getStringsToLog(); + if (strings != null) { + for (String s : strings) { + sb.append(s); + } + } sb.append(" - ").append(ra.getName()).append(" LEAK? -"); } } diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/ResourceCheckerJUnitListener.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/ResourceCheckerJUnitListener.java index 17b3031a3ef..591f364cedf 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/ResourceCheckerJUnitListener.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/ResourceCheckerJUnitListener.java @@ -20,14 +20,20 @@ package org.apache.hadoop.hbase; -import com.sun.management.UnixOperatingSystemMXBean; -import org.junit.runner.notification.RunListener; - import java.lang.management.ManagementFactory; import java.lang.management.OperatingSystemMXBean; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import org.apache.hadoop.hbase.ResourceChecker.Phase; +import org.junit.runner.notification.RunListener; + +import com.sun.management.UnixOperatingSystemMXBean; + /** * Listen to the test progress and check the usage of: * - threads @@ -40,15 +46,43 @@ public class ResourceCheckerJUnitListener extends RunListener { private Map rcs = new ConcurrentHashMap(); static class ThreadResourceAnalyzer extends ResourceChecker.ResourceAnalyzer { + private static Set initialThreadNames = new HashSet(); + private static List stringsToLog = null; + @Override - public int getVal() { - return Thread.getAllStackTraces().size(); + public int getVal(Phase phase) { + Map stackTraces = Thread.getAllStackTraces(); + if (phase == Phase.INITIAL) { + stringsToLog = null; + for (Thread t : stackTraces.keySet()) { + initialThreadNames.add(t.getName()); + } + } else if (phase == Phase.END) { + if (stackTraces.size() > initialThreadNames.size()) { + stringsToLog = new ArrayList(); + for (Thread t : stackTraces.keySet()) { + if (!initialThreadNames.contains(t.getName())) { + stringsToLog.add("\nPotentially hanging thread: " + t.getName() + "\n"); + StackTraceElement[] stackElements = stackTraces.get(t); + for (StackTraceElement ele : stackElements) { + stringsToLog.add("\t" + ele + "\n"); + } + } + } + } + } + return stackTraces.size(); } @Override public int getMax() { return 500; } + + @Override + public List getStringsToLog() { + return stringsToLog; + } } /** @@ -71,7 +105,7 @@ public class ResourceCheckerJUnitListener extends RunListener { static class OpenFileDescriptorResourceAnalyzer extends OSResourceAnalyzer { @Override - public int getVal() { + public int getVal(Phase phase) { if (unixOsStats == null) { return 0; } else { @@ -87,7 +121,7 @@ public class ResourceCheckerJUnitListener extends RunListener { static class MaxFileDescriptorResourceAnalyzer extends OSResourceAnalyzer { @Override - public int getVal() { + public int getVal(Phase phase) { if (unixOsStats == null) { return 0; } else { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/ServerResourceCheckerJUnitListener.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/ServerResourceCheckerJUnitListener.java index f9d891f7594..4e01b5e9f97 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/ServerResourceCheckerJUnitListener.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/ServerResourceCheckerJUnitListener.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hbase; +import org.apache.hadoop.hbase.ResourceChecker.Phase; import org.apache.hadoop.hbase.client.HConnectionTestingUtility; /** @@ -29,7 +30,7 @@ public class ServerResourceCheckerJUnitListener extends ResourceCheckerJUnitList static class ConnectionCountResourceAnalyzer extends ResourceChecker.ResourceAnalyzer { @Override - public int getVal() { + public int getVal(Phase phase) { return HConnectionTestingUtility.getConnectionCount(); } }