HBASE-10851 Wait for regionservers to join the cluster

git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1583769 13f79535-47bb-0310-9956-ffa450edef68

hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java

@@ -64,7 +64,6 @@ import org.apache.hadoop.hbase.executor.EventType;
 import org.apache.hadoop.hbase.executor.ExecutorService;
 import org.apache.hadoop.hbase.ipc.RpcClient;
 import org.apache.hadoop.hbase.ipc.RpcClient.FailedServerException;
-import org.apache.hadoop.hbase.ipc.RpcClient.FailedServerException;
 import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
 import org.apache.hadoop.hbase.master.RegionState.State;
 import org.apache.hadoop.hbase.master.balancer.FavoredNodeAssignmentHelper;
@@ -2211,11 +2210,9 @@ public class AssignmentManager extends ZooKeeperListener {
       }
       LOG.debug("No previous transition plan found (or ignoring " +
         "an existing plan) for " + region.getRegionNameAsString() +
-        "; generated random plan=" + randomPlan + "; " +
+        "; generated random plan=" + randomPlan + "; " + destServers.size() +
-        serverManager.countOfRegionServers() +
+        " (online=" + serverManager.getOnlineServers().size() +
-               " (online=" + serverManager.getOnlineServers().size() +
+        ") available servers, forceNewPlan=" + forceNewPlan);
-               ", available=" + destServers.size() + ") available servers" +
-               ", forceNewPlan=" + forceNewPlan);
         return randomPlan;
       }
     LOG.debug("Using pre-existing plan for " +

hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java

@@ -532,7 +532,7 @@ public class HMaster extends HRegionServer implements MasterServices, Server {
     this.initializationBeforeMetaAssignment = true;
 
     // Wait for regionserver to finish initialization.
-    while (!isOnline()) {
+    while (!isStopped() && !isOnline()) {
       synchronized (online) {
         online.wait(100);
       }
@@ -543,6 +543,10 @@ public class HMaster extends HRegionServer implements MasterServices, Server {
     this.balancer.setMasterServices(this);
     this.balancer.initialize();
 
+    // Check if master is shutting down because of some issue
+    // in initializing the regionserver or the balancer.
+    if(isStopped()) return;
+
     // Make sure meta assigned before proceeding.
     status.setStatus("Assigning Meta Region");
     assignMeta(status, previouslyFailedMetaRSs);

hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java

@@ -151,6 +151,7 @@ public class HMasterCommandLine extends ServerCommandLine {
       // and regionserver both in the one JVM.
       if (LocalHBaseCluster.isLocal(conf)) {
         DefaultMetricsSystem.setMiniClusterMode(true);
+        conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 1);
         final MiniZooKeeperCluster zooKeeperCluster = new MiniZooKeeperCluster(conf);
         File zkDataPath = new File(conf.get(HConstants.ZOOKEEPER_DATA_DIR));
         int zkClientPort = conf.getInt(HConstants.ZOOKEEPER_CLIENT_PORT, 0);

hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java

@@ -436,10 +436,19 @@ public class ServerManager {
     return averageLoad;
   }
 
-  /** @return the count of active regionservers */
+  /**
-  int countOfRegionServers() {
+   * Get the count of active regionservers that are not backup
+   * masters. This count may not be accurate depending on timing.
+   * @return the count of active regionservers
+   */
+  private int countOfRegionServers() {
     // Presumes onlineServers is a concurrent map
-    return this.onlineServers.size();
+    int count = this.onlineServers.size();
+    if (balancer != null) {
+      count -= balancer.getExcludedServers().size();
+      if (count < 0) count = 0;
+    }
+    return count;
   }
 
   /**
@@ -849,7 +858,7 @@ public class ServerManager {
     final long timeout = this.master.getConfiguration().
       getLong(WAIT_ON_REGIONSERVERS_TIMEOUT, 4500);
     int minToStart = this.master.getConfiguration().
-      getInt(WAIT_ON_REGIONSERVERS_MINTOSTART, 1);
+      getInt(WAIT_ON_REGIONSERVERS_MINTOSTART, 2);
     if (minToStart < 1) {
       LOG.warn(String.format(
         "The value of '%s' (%d) can not be less than 1, ignoring.",

hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java

@@ -496,6 +496,10 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
     if (!usingBackupMasters) excludedServers.add(serverName);
   }
 
+  public Set<ServerName> getExcludedServers() {
+    return excludedServers;
+  }
+
   @Override
   public Configuration getConf() {
     return this.config;