HBASE-7086 Enhance ResourceChecker to log stack trace for potentially hanging threads

git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1405443 13f79535-47bb-0310-9956-ffa450edef68

hbase-common/src/test/java/org/apache/hadoop/hbase/ResourceChecker.java

@@ -24,7 +24,6 @@ import org.apache.commons.logging.LogFactory;
 
 import java.util.*;
 
-
 /**
  * Utility class to check the resources:
  *  - log them before and after each test method
@@ -35,6 +34,10 @@ public class ResourceChecker {
   private static final Log LOG = LogFactory.getLog(ResourceChecker.class);
   private String tagLine;
 
+  enum Phase {
+    INITIAL, INTERMEDIATE, END
+  }
+
   /**
    * Constructor
    * @param tagLine - the tagLine is added to the logs. Must be be null.
@@ -80,11 +83,16 @@ public class ResourceChecker {
 
     /**
      * The value for the resource.
+     * @param phase
      */
-    abstract public int getVal();
+    abstract public int getVal(Phase phase);
+    
+    /*
+     * Retrieves List of Strings which would be logged in logEndings()
+     */
+    public List<String> getStringsToLog() { return null; }
   }
 
-
   private List<ResourceAnalyzer> ras = new ArrayList<ResourceAnalyzer>();
   private int[] initialValues;
   private int[] endingValues;
@@ -92,18 +100,18 @@ public class ResourceChecker {
 
   private void fillInit() {
     initialValues = new int[ras.size()];
-    fill(initialValues);
+    fill(Phase.INITIAL, initialValues);
   }
 
   private void fillEndings() {
     endingValues = new int[ras.size()];
-    fill(endingValues);
+    fill(Phase.END, endingValues);
   }
 
-  private void fill(int[] vals) {
+  private void fill(Phase phase, int[] vals) {
     int i = 0;
     for (ResourceAnalyzer ra : ras) {
-      vals[i++] = ra.getVal();
+      vals[i++] = ra.getVal(phase);
     }
   }
 
@@ -151,6 +159,12 @@ public class ResourceChecker {
       if (sb.length() > 0) sb.append(", ");
       sb.append(ra.getName()).append("=").append(curN).append(" (was ").append(curP).append(")");
       if (curN > curP) {
+        List<String> strings = ra.getStringsToLog();
+        if (strings != null) {
+          for (String s : strings) {
+            sb.append(s);
+          }
+        }
         sb.append(" - ").append(ra.getName()).append(" LEAK? -");
       }
     }

hbase-common/src/test/java/org/apache/hadoop/hbase/ResourceCheckerJUnitListener.java

@@ -20,14 +20,20 @@
 package org.apache.hadoop.hbase;
 
 
-import com.sun.management.UnixOperatingSystemMXBean;
-import org.junit.runner.notification.RunListener;
-
 import java.lang.management.ManagementFactory;
 import java.lang.management.OperatingSystemMXBean;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 
+import org.apache.hadoop.hbase.ResourceChecker.Phase;
+import org.junit.runner.notification.RunListener;
+
+import com.sun.management.UnixOperatingSystemMXBean;
+
 /**
  * Listen to the test progress and check the usage of:
  * - threads
@@ -40,15 +46,43 @@ public class ResourceCheckerJUnitListener extends RunListener {
   private Map<String, ResourceChecker> rcs = new ConcurrentHashMap<String, ResourceChecker>();
 
   static class ThreadResourceAnalyzer extends ResourceChecker.ResourceAnalyzer {
+    private static Set<String> initialThreadNames = new HashSet<String>();
+    private static List<String> stringsToLog = null;
+
     @Override
-    public int getVal() {
-      return Thread.getAllStackTraces().size();
+    public int getVal(Phase phase) {
+      Map<Thread, StackTraceElement[]> stackTraces = Thread.getAllStackTraces();
+      if (phase == Phase.INITIAL) {
+        stringsToLog = null;
+        for (Thread t : stackTraces.keySet()) {
+          initialThreadNames.add(t.getName());
+        }
+      } else if (phase == Phase.END) {
+        if (stackTraces.size() > initialThreadNames.size()) {
+          stringsToLog = new ArrayList<String>();
+          for (Thread t : stackTraces.keySet()) {
+            if (!initialThreadNames.contains(t.getName())) {
+              stringsToLog.add("\nPotentially hanging thread: " + t.getName() + "\n");
+              StackTraceElement[] stackElements = stackTraces.get(t);
+              for (StackTraceElement ele : stackElements) {
+                stringsToLog.add("\t" + ele + "\n");
+              }
+            }
+          }
+        }
+      }
+      return stackTraces.size();
     }
 
     @Override
     public int getMax() {
       return 500;
     }
+    
+    @Override
+    public List<String> getStringsToLog() {
+      return stringsToLog;
+    }
   }
 
   /**
@@ -71,7 +105,7 @@ public class ResourceCheckerJUnitListener extends RunListener {
 
   static class OpenFileDescriptorResourceAnalyzer extends OSResourceAnalyzer {
     @Override
-    public int getVal() {
+    public int getVal(Phase phase) {
       if (unixOsStats == null) {
         return 0;
       } else {
@@ -87,7 +121,7 @@ public class ResourceCheckerJUnitListener extends RunListener {
 
   static class MaxFileDescriptorResourceAnalyzer extends OSResourceAnalyzer {
     @Override
-    public int getVal() {
+    public int getVal(Phase phase) {
       if (unixOsStats == null) {
         return 0;
       } else {

hbase-server/src/test/java/org/apache/hadoop/hbase/ServerResourceCheckerJUnitListener.java

@@ -19,6 +19,7 @@
 
 package org.apache.hadoop.hbase;
 
+import org.apache.hadoop.hbase.ResourceChecker.Phase;
 import org.apache.hadoop.hbase.client.HConnectionTestingUtility;
 
 /**
@@ -29,7 +30,7 @@ public class ServerResourceCheckerJUnitListener extends ResourceCheckerJUnitList
 
   static class ConnectionCountResourceAnalyzer extends ResourceChecker.ResourceAnalyzer {
     @Override
-    public int getVal() {
+    public int getVal(Phase phase) {
       return HConnectionTestingUtility.getConnectionCount();
     }
   }