HBASE-14293 TestStochasticBalancerJmxMetrics intermittently fails due to port conflict

hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java

@@ -23,6 +23,7 @@ import java.io.IOException;
 import java.io.OutputStream;
 import java.lang.reflect.Field;
 import java.lang.reflect.Modifier;
+import java.net.DatagramSocket;
 import java.net.InetAddress;
 import java.net.ServerSocket;
 import java.net.Socket;
@@ -222,6 +223,40 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
       Compression.Algorithm.NONE, Compression.Algorithm.GZ
     };
 
+  /**
+   * Checks to see if a specific port is available.
+   *
+   * @param port the port number to check for availability
+   * @return <tt>true</tt> if the port is available, or <tt>false</tt> if not
+   */
+  public static boolean available(int port) {
+    ServerSocket ss = null;
+    DatagramSocket ds = null;
+    try {
+      ss = new ServerSocket(port);
+      ss.setReuseAddress(true);
+      ds = new DatagramSocket(port);
+      ds.setReuseAddress(true);
+      return true;
+    } catch (IOException e) {
+      // Do nothing
+    } finally {
+      if (ds != null) {
+        ds.close();
+      }
+
+      if (ss != null) {
+        try {
+          ss.close();
+        } catch (IOException e) {
+          /* should not be thrown */
+        }
+      }
+    }
+
+    return false;
+  }
+
   /**
    * Create all combinations of Bloom filters and compression algorithms for
    * testing.

hbase-server/src/test/java/org/apache/hadoop/hbase/TestStochasticBalancerJmxMetrics.java

@@ -24,6 +24,7 @@ import java.util.HashSet;
 import java.util.Hashtable;
 import java.util.List;
 import java.util.Map;
+import java.util.Random;
 import java.util.Set;
 
 import javax.management.MBeanAttributeInfo;
@@ -82,15 +83,25 @@ public class TestStochasticBalancerJmxMetrics extends BalancerTestBase {
     conf.setFloat("hbase.master.balancer.stochastic.maxMovePercent", 0.75f);
     conf.setFloat("hbase.regions.slop", 0.0f);
     conf.set(CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY, JMXListener.class.getName());
-    for (int i = 0; i < 5; i++) {
+    Random rand = new Random();
+    for (int i = 0; i < 10; i++) {
+      do {
+        int sign = i % 2 == 0 ? 1 : -1;
+        connectorPort += sign * rand.nextInt(100);
+      } while (!HBaseTestingUtility.available(connectorPort));
       try {
         conf.setInt("regionserver.rmi.registry.port", connectorPort);
 
         UTIL.startMiniCluster();
         break;
       } catch (Exception e) {
-        connectorPort++;
         LOG.debug("Encountered exception when starting cluster. Trying port " + connectorPort, e);
+        try {
+          // this is to avoid "IllegalStateException: A mini-cluster is already running"
+          UTIL.shutdownMiniCluster();
+        } catch (Exception ex) {
+          LOG.debug("Encountered exception shutting down cluster", ex);
+        }
       }
     }
   }