HBASE-10873 Control number of regions assigned to backup masters

git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1590078 13f79535-47bb-0310-9956-ffa450edef68

hbase-common/src/main/resources/hbase-default.xml

@@ -508,10 +508,13 @@ possible configurations would overwhelm and obscure the important.
     <description>Period at which the region balancer runs in the Master.</description>
   </property>
   <property>
-    <name>hbase.balancer.use-backupmaster</name>
+    <name>hbase.balancer.backupMasterWeight</name>
-    <value>true</value>
+    <value>1</value>
-    <description>Whether or not the region balancer uses the backup Masters
+    <description>Used to control how many regions the region balancer can assign to
-    as regionservers, and assigns regions to them.</description>
+    backup Masters, compared to normal region servers. The default value 1 means a
+    backup Master can host as many regions as a normal region server. The bigger the
+    weight, the less the regions a backup Master will host. If the weight is less than 1,
+    the balancer will not assign any region to any backup Master</description>
   </property>
   <property>
     <name>hbase.regions.slop</name>

hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java

@@ -199,9 +199,13 @@ public class ServerManager {
     this.connection = connect ? HConnectionManager.getConnection(c) : null;
 
     // Put this in constructor so we don't cast it every time
+    //
+    // We need to check if a newly added server is a backup master
+    // only if we are configured not to assign any region to it.
     checkingBackupMaster = (master instanceof HMaster)
-      && !c.getBoolean("hbase.balancer.use-backupmaster", true)
+      && ((HMaster)master).balancer instanceof BaseLoadBalancer
-      && ((HMaster)master).balancer instanceof BaseLoadBalancer;
+      && (c.getInt(BaseLoadBalancer.BACKUP_MASTER_WEIGHT_KEY,
+        BaseLoadBalancer.DEFAULT_BACKUP_MASTER_WEIGHT) < 1);
     if (checkingBackupMaster) {
       balancer = (BaseLoadBalancer)((HMaster)master).balancer;
     }

hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java

@@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.master.balancer;
 
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.Deque;
@@ -92,6 +93,8 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
     ArrayList<String> tables;
     HRegionInfo[] regions;
     Deque<RegionLoad>[] regionLoads;
+    boolean[] backupMasterFlags;
+    int activeMasterIndex = -1;
     int[][] regionLocations; //regionIndex -> list of serverIndex sorted by locality
 
     int[][] regionsPerServer;            //serverIndex -> region list
@@ -112,13 +115,15 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
     int numTables;
 
     int numMovedRegions = 0; //num moved regions from the initial configuration
-    int numMovedMetaRegions = 0;       //num of moved regions that are META
+    // num of moved regions away from master that should be on the master
+    int numMovedMasterHostedRegions = 0;
 
     @SuppressWarnings("unchecked")
     protected Cluster(ServerName masterServerName,
         Map<ServerName, List<HRegionInfo>> clusterState,
         Map<String, Deque<RegionLoad>> loads,
-        RegionLocationFinder regionFinder) {
+        RegionLocationFinder regionFinder,
+        Collection<ServerName> backupMasters) {
 
       this.masterServerName = masterServerName;
       serversToIndex = new HashMap<String, Integer>();
@@ -157,6 +162,7 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
       regionLoads = new Deque[numRegions];
       regionLocations = new int[numRegions][];
       serverIndicesSortedByRegionCount = new Integer[numServers];
+      backupMasterFlags = new boolean[numServers];
 
       int tableIndex = 0, regionIndex = 0, regionPerServerIndex = 0;
 
@@ -168,6 +174,8 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
         if (servers[serverIndex] == null ||
             servers[serverIndex].getStartcode() < entry.getKey().getStartcode()) {
           servers[serverIndex] = entry.getKey();
+          backupMasterFlags[serverIndex] = backupMasters != null
+            && backupMasters.contains(servers[serverIndex]);
         }
 
         if (regionsPerServer[serverIndex] != null) {
@@ -180,6 +188,7 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
         serverIndicesSortedByRegionCount[serverIndex] = serverIndex;
 
         if (servers[serverIndex].equals(masterServerName)) {
+          activeMasterIndex = serverIndex;
           for (HRegionInfo hri: entry.getValue()) {
             if (!shouldBeOnMaster(hri)) {
               numUserRegionsOnMaster++;
@@ -294,13 +303,15 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
       regionIndexToServerIndex[regionIndex] = newServerIndex;
       if (initialRegionIndexToServerIndex[regionIndex] == newServerIndex) {
         numMovedRegions--; //region moved back to original location
-        if (regions[regionIndex].isMetaRegion()) {
+        if (shouldBeOnMaster(regions[regionIndex]) && isActiveMaster(newServerIndex)) {
-          numMovedMetaRegions--;
+          // Master hosted region moved back to the active master
+          numMovedMasterHostedRegions--;
         }
       } else if (initialRegionIndexToServerIndex[regionIndex] == oldServerIndex) {
         numMovedRegions++; //region moved from original location
-        if (regions[regionIndex].isMetaRegion()) {
+        if (shouldBeOnMaster(regions[regionIndex]) && isActiveMaster(oldServerIndex)) {
-          numMovedMetaRegions++;
+          // Master hosted region moved away from active the master
+          numMovedMasterHostedRegions++;
         }
       }
       int tableIndex = regionIndexToTableIndex[regionIndex];
@@ -361,6 +372,14 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
       return regionsPerServer[server].length;
     }
 
+    boolean isBackupMaster(int server) {
+      return backupMasterFlags[server];
+    }
+
+    boolean isActiveMaster(int server) {
+      return activeMasterIndex == server;
+    }
+
     private Comparator<Integer> numRegionsComparator = new Comparator<Integer>() {
       @Override
       public int compare(Integer integer, Integer integer2) {
@@ -394,8 +413,8 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
           numTables +
           ", numMovedRegions=" +
           numMovedRegions +
-          ", numMovedMetaRegions=" +
+          ", numMovedMasterHostedRegions=" +
-          numMovedMetaRegions +
+          numMovedMasterHostedRegions +
           '}';
       return desc;
     }
@@ -403,16 +422,42 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
 
   // slop for regions
   protected float slop;
-  private Configuration config;
+  protected Configuration config;
   private static final Random RANDOM = new Random(System.currentTimeMillis());
   private static final Log LOG = LogFactory.getLog(BaseLoadBalancer.class);
 
+  // The weight means that each region on the active/backup master is
+  // equal to that many regions on a normal regionserver, in calculating
+  // the region load by the load balancer. So that the active/backup master
+  // can host less (or equal if weight = 1) regions than normal regionservers.
+  //
+  // The weight can be used to control the number of regions on backup
+  // masters, which shouldn't host as many regions as normal regionservers.
+  // So that we don't need to move around too many regions when a
+  // backup master becomes the active one.
+  //
+  // Currently, the active master weight is used only by StockasticLoadBalancer.
+  // Generally, we don't put any user regions on the active master, which
+  // only hosts regions of tables defined in TABLES_ON_MASTER.
+  // That's why the default activeMasterWeight is high.
+  public static final String BACKUP_MASTER_WEIGHT_KEY =
+    "hbase.balancer.backupMasterWeight";
+  public static final int DEFAULT_BACKUP_MASTER_WEIGHT = 1;
+
+  private static final String ACTIVE_MASTER_WEIGHT_KEY =
+    "hbase.balancer.activeMasterWeight";
+  private static final int DEFAULT_ACTIVE_MASTER_WEIGHT = 200;
+
+  protected int activeMasterWeight;
+  protected int backupMasterWeight;
+
   // a flag to indicate if assigning regions to backup masters
-  protected boolean usingBackupMasters = false;
+  protected boolean usingBackupMasters = true;
   protected final Set<ServerName> excludedServers =
     Collections.synchronizedSet(new HashSet<ServerName>());
 
   protected final MetricsBalancer metricsBalancer = new MetricsBalancer();
+  protected ClusterStatus clusterStatus = null;
   protected ServerName masterServerName;
   protected MasterServices services;
 
@@ -423,7 +468,16 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
     else if (slop > 1) slop = 1;
 
     this.config = conf;
-    usingBackupMasters = conf.getBoolean("hbase.balancer.use-backupmaster", true);
+    activeMasterWeight = conf.getInt(
+      ACTIVE_MASTER_WEIGHT_KEY, DEFAULT_ACTIVE_MASTER_WEIGHT);
+    backupMasterWeight = conf.getInt(
+      BACKUP_MASTER_WEIGHT_KEY, DEFAULT_BACKUP_MASTER_WEIGHT);
+    if (backupMasterWeight < 1) {
+      usingBackupMasters = false;
+      LOG.info("Backup master won't host any region since "
+        + BACKUP_MASTER_WEIGHT_KEY + " is " + backupMasterWeight
+        + "(<1)");
+    }
   }
 
   protected void setSlop(Configuration conf) {
@@ -507,6 +561,7 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
 
   @Override
   public void setClusterStatus(ClusterStatus st) {
+    this.clusterStatus = st;
     if (st == null || usingBackupMasters) return;
 
     // Not assign any region to backup masters.
@@ -523,6 +578,10 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
     this.services = masterServices;
   }
 
+  protected Collection<ServerName> getBackupMasters() {
+    return clusterStatus == null ? null : clusterStatus.getBackupMasters();
+  }
+
   protected boolean needsBalance(ClusterLoadState cs) {
     if (cs.getNumServers() < MIN_SERVER_BALANCE) {
       if (LOG.isDebugEnabled()) {
@@ -541,8 +600,8 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
       if (LOG.isTraceEnabled()) {
         // If nothing to balance, then don't say anything unless trace-level logging.
         LOG.trace("Skipping load balancing because balanced cluster; " +
-          "servers=" + cs.getNumServers() + " " +
+          "servers=" + cs.getNumServers() + "(backupMasters=" + cs.getNumBackupMasters() +
-          "regions=" + cs.getNumRegions() + " average=" + average + " " +
+          ") regions=" + cs.getNumRegions() + " average=" + average + " " +
           "mostloaded=" + serversByLoad.lastKey().getLoad() +
           " leastloaded=" + serversByLoad.firstKey().getLoad());
       }
@@ -572,54 +631,50 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
   public Map<ServerName, List<HRegionInfo>> roundRobinAssignment(List<HRegionInfo> regions,
       List<ServerName> servers) {
     metricsBalancer.incrMiscInvocations();
-
+    if (regions == null || regions.isEmpty()) {
-    if (!excludedServers.isEmpty() && servers != null) {
+      return null;
-      servers.removeAll(excludedServers);
     }
-    if (regions.isEmpty() || servers.isEmpty()) {
+
+    List<ServerName> backupMasters = normalizeServers(servers);
+    int numServers = servers == null ? 0 : servers.size();
+    int numBackupMasters = backupMasters == null ? 0 : backupMasters.size();
+    if (numServers == 0 && numBackupMasters == 0) {
+      LOG.warn("Wanted to do round robin assignment but no servers to assign to");
       return null;
     }
     Map<ServerName, List<HRegionInfo>> assignments = new TreeMap<ServerName, List<HRegionInfo>>();
-    int numServers = servers.size();
+    if (numServers + numBackupMasters == 1) { // Only one server, nothing fancy we can do here
-    if (numServers == 1) { // Only one server, nothing fancy we can do here
+      ServerName server = numServers > 0 ? servers.get(0) : backupMasters.get(0);
-      assignments.put(servers.get(0), new ArrayList<HRegionInfo>(regions));
+      assignments.put(server, new ArrayList<HRegionInfo>(regions));
       return assignments;
     }
-
+    List<HRegionInfo> masterRegions = null;
-    int numRegions = regions.size();
+    if (numServers > 0 && servers.contains(masterServerName)) {
-    // Master regionserver is in the server list.
+      masterRegions = new ArrayList<HRegionInfo>();
-    boolean masterIncluded = servers.contains(masterServerName);
+      if (numServers == 1) {
-    int skipServers = numServers;
+        // The only server in servers is the master,
-    if (masterIncluded) {
+        // Assign all regions to backup masters
-      skipServers--;
+        numServers = 0;
+      }
     }
-    int max = (int) Math.ceil((float) numRegions / skipServers);
+    int total = regions.size();
-    int serverIdx = RANDOM.nextInt(numServers);
+    // Get the number of regions to be assigned
-    int regionIdx = 0;
+    // to backup masters based on the weight
-    for (int j = 0; j < numServers; j++) {
+    int numRegions = total * numBackupMasters
-      ServerName server = servers.get((j + serverIdx) % numServers);
+      / (numServers * backupMasterWeight + numBackupMasters);
-      if (server.equals(masterServerName)) {
+    if (numRegions > 0) {
-        // Don't put non-special region on the master regionserver,
+      // backupMasters can't be null, according to the formula, numBackupMasters != 0
-        // So that it is not overloaded.
+      roundRobinAssignment(regions, 0,
-        continue;
+        numRegions, backupMasters, masterRegions, assignments);
-      }
+    }
-      List<HRegionInfo> serverRegions = new ArrayList<HRegionInfo>(max);
+    int remainder = total - numRegions;
-      for (int i = regionIdx; i < numRegions; i += skipServers) {
+    if (remainder > 0) {
-        HRegionInfo region = regions.get(i % numRegions);
+      // servers can't be null, or contains the master only since numServers != 0
-        if (!(masterIncluded && shouldBeOnMaster(region))) {
+      roundRobinAssignment(regions, numRegions, remainder,
-          serverRegions.add(region);
+        servers, masterRegions, assignments);
-          continue;
+    }
-        }
+    if (masterRegions != null && !masterRegions.isEmpty()) {
-        // Master is in the target list and this is a special region
+      assignments.put(masterServerName, masterRegions);
-        List<HRegionInfo> masterRegions = assignments.get(masterServerName);
-        if (masterRegions == null) {
-          masterRegions = new ArrayList<HRegionInfo>(max);
-          assignments.put(masterServerName, masterRegions);
-        }
-        masterRegions.add(region);
-      }
-      assignments.put(server, serverRegions);
-      regionIdx++;
     }
     return assignments;
   }
@@ -645,10 +700,15 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
   public Map<HRegionInfo, ServerName> immediateAssignment(List<HRegionInfo> regions,
       List<ServerName> servers) {
     metricsBalancer.incrMiscInvocations();
+    if (servers == null || servers.isEmpty()) {
+      LOG.warn("Wanted to do random assignment but no servers to assign to");
+      return null;
+    }
 
     Map<HRegionInfo, ServerName> assignments = new TreeMap<HRegionInfo, ServerName>();
+    List<ServerName> backupMasters = normalizeServers(servers);
     for (HRegionInfo region : regions) {
-      assignments.put(region, randomAssignment(region, servers));
+      assignments.put(region, randomAssignment(region, servers, backupMasters));
     }
     return assignments;
   }
@@ -659,26 +719,12 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
   @Override
   public ServerName randomAssignment(HRegionInfo regionInfo, List<ServerName> servers) {
     metricsBalancer.incrMiscInvocations();
-
-    if (!excludedServers.isEmpty() && servers != null) {
-      servers.removeAll(excludedServers);
-    }
     if (servers == null || servers.isEmpty()) {
       LOG.warn("Wanted to do random assignment but no servers to assign to");
       return null;
     }
-    int numServers = servers.size();
+    return randomAssignment(regionInfo, servers,
-    if (numServers == 1) return servers.get(0);
+      normalizeServers(servers));
-    if (shouldBeOnMaster(regionInfo) && servers.contains(masterServerName)) {
-      return masterServerName;
-    }
-    int i = RANDOM.nextInt(numServers);
-    ServerName sn = servers.get(i);
-    if (sn.equals(masterServerName)) {
-      i = (i == 0 ? 1 : i - 1);
-      sn = servers.get(i);
-    }
-    return sn;
   }
 
   /**
@@ -703,17 +749,21 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
       List<ServerName> servers) {
     // Update metrics
     metricsBalancer.incrMiscInvocations();
-
+    if (regions == null || regions.isEmpty()) {
-    if (!excludedServers.isEmpty() && servers != null) {
+      return null;
-      servers.removeAll(excludedServers);
     }
-    if (regions.isEmpty() || servers.isEmpty()) {
+
+    List<ServerName> backupMasters = normalizeServers(servers);
+    int numServers = servers == null ? 0 : servers.size();
+    int numBackupMasters = backupMasters == null ? 0 : backupMasters.size();
+    if (numServers == 0 && numBackupMasters == 0) {
+      LOG.warn("Wanted to do retain assignment but no servers to assign to");
       return null;
     }
     Map<ServerName, List<HRegionInfo>> assignments = new TreeMap<ServerName, List<HRegionInfo>>();
-    int numServers = servers.size();
+    if (numServers + numBackupMasters == 1) { // Only one server, nothing fancy we can do here
-    if (numServers == 1) { // Only one server, nothing fancy we can do here
+      ServerName server = numServers > 0 ? servers.get(0) : backupMasters.get(0);
-      assignments.put(servers.get(0), new ArrayList<HRegionInfo>(regions.keySet()));
+      assignments.put(server, new ArrayList<HRegionInfo>(regions.keySet()));
       return assignments;
     }
 
@@ -725,13 +775,15 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
     // servers on the same host on different ports.
     ArrayListMultimap<String, ServerName> serversByHostname = ArrayListMultimap.create();
     for (ServerName server : servers) {
+      assignments.put(server, new ArrayList<HRegionInfo>());
       if (!server.equals(masterServerName)) {
         serversByHostname.put(server.getHostname(), server);
       }
     }
-
+    if (numBackupMasters > 0) {
-    for (ServerName server : servers) {
+      for (ServerName server : backupMasters) {
-      assignments.put(server, new ArrayList<HRegionInfo>());
+        assignments.put(server, new ArrayList<HRegionInfo>());
+      }
     }
 
     // Collection of the hostnames that used to have regions
@@ -761,12 +813,7 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
       } else if (localServers.isEmpty()) {
         // No servers on the new cluster match up with this hostname,
         // assign randomly.
-        int i = RANDOM.nextInt(numServers);
+        ServerName randomServer = randomAssignment(region, servers, backupMasters);
-        ServerName randomServer = servers.get(i);
-        if (randomServer.equals(masterServerName)) {
-          i = (i == 0 ? 1 : i - 1);
-          randomServer = servers.get(i);
-        }
         assignments.get(randomServer).add(region);
         numRandomAssignments++;
         if (oldServerName != null) oldHostsNoLongerPresent.add(oldServerName.getHostname());
@@ -821,4 +868,112 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
     LOG.info("Load Balancer stop requested: "+why);
     stopped = true;
   }
+
+  /**
+   * Prepare the list of target regionservers so that it doesn't
+   * contain any excluded server, or backup master. Those backup masters
+   * used to be in the original list are returned.
+   */
+  private List<ServerName> normalizeServers(List<ServerName> servers) {
+    if (servers == null) {
+      return null;
+    }
+    if (!excludedServers.isEmpty()) {
+      servers.removeAll(excludedServers);
+    }
+    Collection<ServerName> allBackupMasters = getBackupMasters();
+    List<ServerName> backupMasters = null;
+    if (allBackupMasters != null && !allBackupMasters.isEmpty()) {
+      for (ServerName server: allBackupMasters) {
+        if (!servers.contains(server)) {
+          // Ignore backup masters not included
+          continue;
+        }
+        servers.remove(server);
+        if (backupMasters == null) {
+          backupMasters = new ArrayList<ServerName>();
+        }
+        backupMasters.add(server);
+      }
+    }
+    return backupMasters;
+  }
+
+  /**
+   * Used to assign a single region to a random server. The input should
+   * have been already normalized: 1) servers doesn't include any exclude sever,
+   * 2) servers doesn't include any backup master, 3) backupMasters contains
+   * only backup masters that are intended to host this region, i.e, it
+   * may not have all the backup masters.
+   */
+  private ServerName randomAssignment(HRegionInfo regionInfo,
+      List<ServerName> servers, List<ServerName> backupMasters) {
+    int numServers = servers == null ? 0 : servers.size();
+    int numBackupMasters = backupMasters == null ? 0 : backupMasters.size();
+    if (numServers == 0 && numBackupMasters == 0) {
+      LOG.warn("Wanted to do random assignment but no servers to assign to");
+      return null;
+    }
+    if (servers != null && shouldBeOnMaster(regionInfo)
+        && servers.contains(masterServerName)) {
+      return masterServerName;
+    }
+    // Generate a random number weighted more towards
+    // regular regionservers instead of backup masters.
+    // This formula is chosen for simplicity.
+    int i = RANDOM.nextInt(
+      numBackupMasters + numServers * backupMasterWeight);
+    if (i < numBackupMasters) {
+      return backupMasters.get(i);
+    }
+    i = (i - numBackupMasters)/backupMasterWeight;
+    ServerName sn = servers.get(i);
+    if (sn.equals(masterServerName)) {
+      // Try to avoid master for a user region
+      if (numServers > 1) {
+        i = (i == 0 ? 1 : i - 1);
+        sn = servers.get(i);
+      } else if (numBackupMasters > 0) {
+        sn = backupMasters.get(0);
+      }
+    }
+    return sn;
+  }
+
+  /**
+   * Round robin a chunk of a list of regions to a list of servers
+   */
+  private void roundRobinAssignment(List<HRegionInfo> regions, int offset,
+      int numRegions, List<ServerName> servers, List<HRegionInfo> masterRegions,
+      Map<ServerName, List<HRegionInfo>> assignments) {
+    boolean masterIncluded = servers.contains(masterServerName);
+    int numServers = servers.size();
+    int skipServers = numServers;
+    if (masterIncluded) {
+      skipServers--;
+    }
+    int max = (int) Math.ceil((float) numRegions / skipServers);
+    int serverIdx = RANDOM.nextInt(numServers);
+    int regionIdx = 0;
+    for (int j = 0; j < numServers; j++) {
+      ServerName server = servers.get((j + serverIdx) % numServers);
+      if (masterIncluded && server.equals(masterServerName)) {
+        // Don't put non-special region on the master regionserver,
+        // So that it is not overloaded.
+        continue;
+      }
+      List<HRegionInfo> serverRegions = new ArrayList<HRegionInfo>(max);
+      for (int i = regionIdx; i < numRegions; i += skipServers) {
+        HRegionInfo region = regions.get(offset + i % numRegions);
+        if (masterRegions == null || !shouldBeOnMaster(region)) {
+          serverRegions.add(region);
+          continue;
+        }
+        // Master is in the list and this is a special region
+        masterRegions.add(region);
+      }
+      assignments.put(server, serverRegions);
+      regionIdx++;
+    }
+  }
 }

hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/ClusterLoadState.java

@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.hbase.master.balancer;
 
+import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 import java.util.NavigableMap;
@@ -34,9 +35,12 @@ public class ClusterLoadState {
   private boolean emptyRegionServerPresent = false;
   private int numRegions = 0;
   private int numServers = 0;
+  private int numBackupMasters = 0;
+  private int backupMasterWeight;
 
-  public ClusterLoadState(ServerName master,
+  public ClusterLoadState(ServerName master, Collection<ServerName> backupMasters,
-      Map<ServerName, List<HRegionInfo>> clusterState) {
+      int backupMasterWeight, Map<ServerName, List<HRegionInfo>> clusterState) {
+    this.backupMasterWeight = backupMasterWeight;
     this.numRegions = 0;
     this.numServers = clusterState.size();
     this.clusterState = clusterState;
@@ -44,20 +48,20 @@ public class ClusterLoadState {
     // Iterate so we can count regions as we build the map
     for (Map.Entry<ServerName, List<HRegionInfo>> server : clusterState.entrySet()) {
       if (master != null && numServers > 1 && master.equals(server.getKey())) {
-        // Don't count the master regionserver since its
+        // Don't count the master since its load is meant to be low.
-        // load is meant to be low.
+        numServers--;
         continue;
       }
       List<HRegionInfo> regions = server.getValue();
       int sz = regions.size();
       if (sz == 0) emptyRegionServerPresent = true;
       numRegions += sz;
+      if (backupMasters != null && backupMasters.contains(server.getKey())) {
+        sz *= backupMasterWeight;
+        numBackupMasters++;
+      }
       serversByLoad.put(new ServerAndLoad(server.getKey(), sz), regions);
     }
-    if (master != null && numServers > 1
-        && clusterState.containsKey(master)) {
-      numServers--;
-    }
   }
 
   Map<ServerName, List<HRegionInfo>> getClusterState() {
@@ -80,8 +84,12 @@ public class ClusterLoadState {
     return numServers;
   }
 
+  int getNumBackupMasters() {
+    return numBackupMasters;
+  }
+
   float getLoadAverage() {
-    return (float) numRegions / numServers;
+    return numRegions / (numServers - numBackupMasters * (1 - 1.0f/backupMasterWeight));
   }
 
   int getMaxLoad() {

hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/FavoredNodeLoadBalancer.java

@@ -59,13 +59,12 @@ public class FavoredNodeLoadBalancer extends BaseLoadBalancer {
 
   private FavoredNodesPlan globalFavoredNodesAssignmentPlan;
   private RackManager rackManager;
-  Configuration conf;
 
   @Override
   public void setConf(Configuration conf) {
     globalFavoredNodesAssignmentPlan = new FavoredNodesPlan();
     this.rackManager = new RackManager(conf);
-    this.conf = conf;
+    super.setConf(conf);
   }
 
   @Override

hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SimpleLoadBalancer.java

@@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.master.balancer;
 
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -188,20 +189,25 @@ public class SimpleLoadBalancer extends BaseLoadBalancer {
     boolean emptyRegionServerPresent = false;
     long startTime = System.currentTimeMillis();
 
-    ClusterLoadState cs = new ClusterLoadState(masterServerName, clusterMap);
+    Collection<ServerName> backupMasters = getBackupMasters();
+    ClusterLoadState cs = new ClusterLoadState(masterServerName,
+      backupMasters, backupMasterWeight, clusterMap);
 
     if (!this.needsBalance(cs)) return null;
     
     int numServers = cs.getNumServers();
     NavigableMap<ServerAndLoad, List<HRegionInfo>> serversByLoad = cs.getServersByLoad();
     int numRegions = cs.getNumRegions();
-    int min = numRegions / numServers;
+    float average = cs.getLoadAverage();
-    int max = numRegions % numServers == 0 ? min : min + 1;
+    int max = (int)Math.ceil(average);
+    int min = (int)average;
 
     // Using to check balance result.
     StringBuilder strBalanceParam = new StringBuilder();
     strBalanceParam.append("Balance parameter: numRegions=").append(numRegions)
-        .append(", numServers=").append(numServers).append(", max=").append(max)
+        .append(", numServers=").append(numServers).append(", numBackupMasters=")
+        .append(cs.getNumBackupMasters()).append(", backupMasterWeight=")
+        .append(backupMasterWeight).append(", max=").append(max)
         .append(", min=").append(min);
     LOG.debug(strBalanceParam.toString());
 
@@ -220,14 +226,18 @@ public class SimpleLoadBalancer extends BaseLoadBalancer {
     for (Map.Entry<ServerAndLoad, List<HRegionInfo>> server:
         serversByLoad.descendingMap().entrySet()) {
       ServerAndLoad sal = server.getKey();
-      int regionCount = sal.getLoad();
+      int load = sal.getLoad();
-      if (regionCount <= max) {
+      if (load <= max) {
         serverBalanceInfo.put(sal.getServerName(), new BalanceInfo(0, 0));
         break;
       }
       serversOverloaded++;
       List<HRegionInfo> regions = server.getValue();
-      int numToOffload = Math.min(regionCount - max, regions.size());
+      int w = 1; // Normal region server has weight 1
+      if (backupMasters != null && backupMasters.contains(sal.getServerName())) {
+        w = backupMasterWeight; // Backup master has heavier weight
+      }
+      int numToOffload = Math.min((load - max) / w, regions.size());
       // account for the out-of-band regions which were assigned to this server
       // after some other region server crashed 
       Collections.sort(regions, riComparator);
@@ -259,16 +269,20 @@ public class SimpleLoadBalancer extends BaseLoadBalancer {
     fetchFromTail = false;
 
     Map<ServerName, Integer> underloadedServers = new HashMap<ServerName, Integer>();
-    float average = (float)numRegions / numServers; // for logging
+    int maxToTake = numRegions - min;
-    int maxToTake = numRegions - (int)average;
     for (Map.Entry<ServerAndLoad, List<HRegionInfo>> server:
         serversByLoad.entrySet()) {
       if (maxToTake == 0) break; // no more to take
-      int regionCount = server.getKey().getLoad();
+      int load = server.getKey().getLoad();
-      if (regionCount >= min && regionCount > 0) {
+      if (load >= min && load > 0) {
         continue; // look for other servers which haven't reached min
       }
-      int regionsToPut = min - regionCount;
+      int w = 1; // Normal region server has weight 1
+      if (backupMasters != null
+          && backupMasters.contains(server.getKey().getServerName())) {
+        w = backupMasterWeight; // Backup master has heavier weight
+      }
+      int regionsToPut = (min - load) / w;
       if (regionsToPut == 0)
       {
         regionsToPut = 1;

hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java

@@ -102,7 +102,6 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
   private static final Log LOG = LogFactory.getLog(StochasticLoadBalancer.class);
 
   private final RegionLocationFinder regionFinder = new RegionLocationFinder();
-  private ClusterStatus clusterStatus = null;
   Map<String, Deque<RegionLoad>> loads = new HashMap<String, Deque<RegionLoad>>();
 
   // values are defaults
@@ -149,8 +148,7 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
     };
 
     costFunctions = new CostFunction[]{
-      new RegionCountSkewCostFunction(conf),
+      new RegionCountSkewCostFunction(conf, activeMasterWeight, backupMasterWeight),
-      new RegionOnMasterCostFunction(conf),
       new MoveCostFunction(conf),
       localityCost,
       new TableSkewCostFunction(conf),
@@ -170,7 +168,6 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
   public void setClusterStatus(ClusterStatus st) {
     super.setClusterStatus(st);
     regionFinder.setClusterStatus(st);
-    this.clusterStatus = st;
     updateRegionLoad();
     for(CostFromRegionLoadFunction cost : regionLoadFunctions) {
       cost.setClusterStatus(st);
@@ -197,14 +194,16 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
       return plans;
     }
     filterExcludedServers(clusterState);
-    if (!needsBalance(new ClusterLoadState(masterServerName, clusterState))) {
+    if (!needsBalance(new ClusterLoadState(masterServerName,
+        getBackupMasters(), backupMasterWeight, clusterState))) {
       return null;
     }
 
     long startTime = EnvironmentEdgeManager.currentTimeMillis();
 
     // Keep track of servers to iterate through them.
-    Cluster cluster = new Cluster(masterServerName, clusterState, loads, regionFinder);
+    Cluster cluster = new Cluster(masterServerName,
+      clusterState, loads, regionFinder, getBackupMasters());
     double currentCost = computeCost(cluster, Double.MAX_VALUE);
 
     double initCost = currentCost;
@@ -631,11 +630,11 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
      * @return The scaled value.
      */
     protected double scale(double min, double max, double value) {
-      if (max == 0 || value == 0) {
+      if (max <= min || value <= min) {
         return 0;
       }
 
-      return Math.max(0d, Math.min(1d, (value - min) / max));
+      return Math.max(0d, Math.min(1d, (value - min) / (max - min)));
     }
   }
 
@@ -650,7 +649,6 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
     private static final float DEFAULT_MOVE_COST = 100;
     private static final int DEFAULT_MAX_MOVES = 600;
     private static final float DEFAULT_MAX_MOVE_PERCENT = 0.25f;
-    private static final int META_MOVE_COST_MULT = 10;
 
     private final float maxMovesPercent;
 
@@ -672,19 +670,15 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
 
       double moveCost = cluster.numMovedRegions;
 
-      // Don't let this single balance move more than the max moves.
+      // Don't let this single balance move more than the max moves,
+      // or move a region that should be on master away from the master.
+      // It is ok to move any master hosted region back to the master.
       // This allows better scaling to accurately represent the actual cost of a move.
-      if (moveCost > maxMoves) {
+      if (moveCost > maxMoves || cluster.numMovedMasterHostedRegions > 0) {
         return 1000000;   // return a number much greater than any of the other cost
       }
 
-      // hbase:meta region is special
+      return scale(0, cluster.numRegions, moveCost);
-      if (cluster.numMovedMetaRegions > 0) {
-        // assume each hbase:meta region move costs 10 times
-        moveCost += META_MOVE_COST_MULT * cluster.numMovedMetaRegions;
-      }
-
-      return scale(0, cluster.numRegions + META_MOVE_COST_MULT, moveCost);
     }
   }
 
@@ -697,12 +691,17 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
         "hbase.master.balancer.stochastic.regionCountCost";
     private static final float DEFAULT_REGION_COUNT_SKEW_COST = 500;
 
+    private double activeMasterWeight;
+    private double backupMasterWeight;
     private double[] stats = null;
 
-    RegionCountSkewCostFunction(Configuration conf) {
+    RegionCountSkewCostFunction(Configuration conf,
+        double activeMasterWeight, double backupMasterWeight) {
       super(conf);
       // Load multiplier should be the greatest as it is the most general way to balance data.
       this.setMultiplier(conf.getFloat(REGION_COUNT_SKEW_COST_KEY, DEFAULT_REGION_COUNT_SKEW_COST));
+      this.activeMasterWeight = activeMasterWeight;
+      this.backupMasterWeight = backupMasterWeight;
     }
 
     @Override
@@ -713,6 +712,13 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
 
       for (int i =0; i < cluster.numServers; i++) {
         stats[i] = cluster.regionsPerServer[i].length;
+        // Use some weight on regions assigned to active/backup masters,
+        // so that they won't carry as many regions as normal regionservers.
+        if (cluster.isActiveMaster(i)) {
+          stats[i] += cluster.numUserRegionsOnMaster * (activeMasterWeight - 1);
+        } else if (cluster.isBackupMaster(i)) {
+          stats[i] *= backupMasterWeight;
+        }
       }
       return costFromArray(stats);
     }
@@ -747,30 +753,6 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
     }
   }
 
-  /**
-   * Compute the cost of a potential cluster configuration based upon if putting
-   * user regions on the master regionserver.
-   */
-  public static class RegionOnMasterCostFunction extends CostFunction {
-
-    private static final String REGION_ON_MASTER_COST_KEY =
-        "hbase.master.balancer.stochastic.regionOnMasterCost";
-    private static final float DEFAULT_REGION_ON_MASTER__COST = 1000;
-
-    RegionOnMasterCostFunction(Configuration conf) {
-      super(conf);
-      this.setMultiplier(conf.getFloat(
-        REGION_ON_MASTER_COST_KEY, DEFAULT_REGION_ON_MASTER__COST));
-    }
-
-    @Override
-    double cost(Cluster cluster) {
-      double max = cluster.numRegions;
-      double value = cluster.numUserRegionsOnMaster;
-      return scale(0, max, value);
-    }
-  }
-
   /**
    * Compute a cost of a potential cluster configuration based upon where
    * {@link org.apache.hadoop.hbase.regionserver.StoreFile}s are located.

hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/BalancerTestBase.java

@@ -164,7 +164,8 @@ public class BalancerTestBase {
   }
 
   protected BaseLoadBalancer.Cluster mockCluster(int[] mockCluster) {
-    return new BaseLoadBalancer.Cluster(null, mockClusterServers(mockCluster, -1), null, null);
+    return new BaseLoadBalancer.Cluster(null,
+      mockClusterServers(mockCluster, -1), null, null, null);
   }
 
   protected Map<ServerName, List<HRegionInfo>> mockClusterServers(int[] mockCluster, int numTables) {

hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestBaseLoadBalancer.java

@@ -289,7 +289,7 @@ public class TestBaseLoadBalancer extends BalancerTestBase {
     assignRegions(regions, oldServers, clusterState);
 
     // should not throw exception:
-    BaseLoadBalancer.Cluster cluster = new Cluster(null, clusterState, null, null);
+    BaseLoadBalancer.Cluster cluster = new Cluster(null, clusterState, null, null, null);
     assertEquals(101 + 9, cluster.numRegions);
     assertEquals(10, cluster.numServers); // only 10 servers because they share the same host + port
   }
@@ -331,7 +331,7 @@ public class TestBaseLoadBalancer extends BalancerTestBase {
     when(locationFinder.getTopBlockLocations(regions.get(43))).thenReturn(
       Lists.newArrayList(ServerName.valueOf("foo", 0, 0))); // this server does not exists in clusterStatus
 
-    BaseLoadBalancer.Cluster cluster = new Cluster(null, clusterState, null, locationFinder);
+    BaseLoadBalancer.Cluster cluster = new Cluster(null, clusterState, null, locationFinder, null);
 
     int r0 = ArrayUtils.indexOf(cluster.regions, regions.get(0)); // this is ok, it is just a test
     int r1 = ArrayUtils.indexOf(cluster.regions, regions.get(1));

hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestDefaultLoadBalancer.java

@@ -17,8 +17,14 @@
  */
 package org.apache.hadoop.hbase.master.balancer;
 
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import java.util.TreeMap;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -27,11 +33,13 @@ import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.MediumTests;
 import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.master.LoadBalancer;
 import org.apache.hadoop.hbase.master.RegionPlan;
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
+import org.mockito.Mockito;
 
 /**
  * Test the load balancer that is created by default.
@@ -124,7 +132,44 @@ public class TestDefaultLoadBalancer extends BalancerTestBase {
         returnServer(entry.getKey());
       }
     }
-
   }
 
+  @Test
+  public void testBalancerClusterWithBackupMaster() throws Exception {
+    SimpleLoadBalancer balancer = Mockito.spy(new SimpleLoadBalancer());
+    balancer.setConf(HBaseConfiguration.create());
+    List<ServerName> backupMasters = new ArrayList<ServerName>();
+    ServerName backupMaster = ServerName.parseServerName("backup:1:1");
+    ServerName rs = ServerName.parseServerName("rs:1:1");
+    backupMasters.add(backupMaster);
+    Mockito.doReturn(backupMasters).when(balancer).getBackupMasters();
+    Map<ServerName, List<HRegionInfo>> servers = new TreeMap<ServerName, List<HRegionInfo>>();
+    List<HRegionInfo> regions = new ArrayList<HRegionInfo>();
+    TableName table = TableName.valueOf("test");
+    regions.add(new HRegionInfo(table));
+    servers.put(backupMaster, regions);
+    regions = new ArrayList<HRegionInfo>();
+    balancer.backupMasterWeight = 4;
+    for (int i=0; i<4; i++) {
+      regions.add(new HRegionInfo(table));
+    }
+    servers.put(rs, regions);
+    List<RegionPlan> plans = balancer.balanceCluster(servers);
+    assertNull(plans);
+
+    // Reset the cluster map
+    regions = new ArrayList<HRegionInfo>();
+    for (int i=0; i<2; i++) {
+      regions.add(new HRegionInfo(table));
+    }
+    servers.put(backupMaster, regions);
+    regions = new ArrayList<HRegionInfo>();
+    for (int i=0; i<3; i++) {
+      regions.add(new HRegionInfo(table));
+    }
+    servers.put(rs, regions);
+    plans = balancer.balanceCluster(servers);
+    assertNotNull(plans);
+    assertEquals(1, plans.size());
+  }
 }

hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java

@@ -185,7 +185,7 @@ public class TestStochasticLoadBalancer extends BalancerTestBase {
   public void testSkewCost() {
     Configuration conf = HBaseConfiguration.create();
     StochasticLoadBalancer.CostFunction
-        costFunction = new StochasticLoadBalancer.RegionCountSkewCostFunction(conf);
+        costFunction = new StochasticLoadBalancer.RegionCountSkewCostFunction(conf, 1, 1);
     for (int[] mockCluster : clusterStateMocks) {
       double cost = costFunction.cost(mockCluster(mockCluster));
       assertTrue(cost >= 0);