HBASE-3681 Check the sloppiness of the region load before balancing (Ted Yu via JD)

git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1085223 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -206,6 +206,7 @@ Release 0.90.2 - Unreleased
    HBASE-3600  Update our jruby to 1.6.0
    HBASE-3640  [replication] Transferring queues shouldn't be done inline with RS startup
    HBASE-3658  Alert when heap is over committed (Subbu M Iyer via Stack)
+   HBASE-3681  Check the sloppiness of the region load before balancing (Ted Yu via JD)
 
 Release 0.90.1 - February 9th, 2011
 

src/main/java/org/apache/hadoop/hbase/master/HMaster.java

@@ -163,7 +163,7 @@ implements HMasterInterface, HMasterRegionInterface, MasterServices, Server {
   // Instance of the hbase executor service.
   ExecutorService executorService;
 
-  private LoadBalancer balancer = new LoadBalancer();
+  private LoadBalancer balancer;
   private Thread balancerChore;
   // If 'true', the balancer is 'on'.  If 'false', the balancer will not run.
   private volatile boolean balanceSwitch = true;
@@ -358,6 +358,7 @@ implements HMasterInterface, HMasterRegionInterface, MasterServices, Server {
 
     this.assignmentManager = new AssignmentManager(this, serverManager,
       this.catalogTracker, this.executorService);
+    this.balancer = new LoadBalancer(conf);
     zooKeeper.registerListenerFirst(assignmentManager);
 
     this.regionServerTracker = new RegionServerTracker(zooKeeper, this,

src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java

@@ -32,6 +32,7 @@ import java.util.TreeSet;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.BlockLocation;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
@@ -59,7 +60,15 @@ import org.apache.hadoop.hbase.HServerInfo;
 public class LoadBalancer {
   private static final Log LOG = LogFactory.getLog(LoadBalancer.class);
   private static final Random RANDOM = new Random(System.currentTimeMillis());
+  // slop for regions
+  private float slop;
 
+  LoadBalancer(Configuration conf) {
+    this.slop = conf.getFloat("hbase.regions.slop", (float) 0.2);
+    if (slop < 0) slop = 0;
+    else if (slop > 1) slop = 1;
+  }
+  
   static class RegionPlanComparator implements Comparator<RegionPlan> {
     @Override
     public int compare(RegionPlan l, RegionPlan r) {
@@ -165,10 +174,11 @@ public class LoadBalancer {
 
     // Check if we even need to do any load balancing
     float average = (float)numRegions / numServers; // for logging
-    int min = numRegions / numServers;
-    int max = numRegions % numServers == 0 ? min : min + 1;
-    if(serversByLoad.lastKey().getLoad().getNumberOfRegions() <= max &&
-       serversByLoad.firstKey().getLoad().getNumberOfRegions() >= min) {
+    // HBASE-3681 check sloppiness first
+    int floor = (int) Math.floor(average * (1 - slop));
+    int ceiling = (int) Math.ceil(average * (1 + slop));
+    if(serversByLoad.lastKey().getLoad().getNumberOfRegions() <= ceiling &&
+       serversByLoad.firstKey().getLoad().getNumberOfRegions() >= floor) {
       // Skipped because no server outside (min,max) range
       LOG.info("Skipping load balancing.  servers=" + numServers + " " +
           "regions=" + numRegions + " average=" + average + " " +
@@ -176,6 +186,8 @@ public class LoadBalancer {
           " leastloaded=" + serversByLoad.firstKey().getLoad().getNumberOfRegions());
       return null;
     }
+    int min = numRegions / numServers;
+    int max = numRegions % numServers == 0 ? min : min + 1;
 
     // Balance the cluster
     // TODO: Look at data block locality or a more complex load to do this

src/main/resources/hbase-default.xml

@@ -281,6 +281,13 @@
     <description>Period at which the region balancer runs in the Master.
     </description>
   </property>
+  <property>
+    <name>hbase.regions.slop</name>
+    <value>0.2</value>
+    <description>Rebalance if any regionserver has average + (average * slop) regions.
+    Default is 20% slop.
+    </description>
+  </property>
   <property>
     <name>hbase.master.logcleaner.ttl</name>
     <value>600000</value>

src/test/java/org/apache/hadoop/hbase/master/TestLoadBalancer.java

@@ -39,6 +39,8 @@ import junit.framework.Assert;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HServerAddress;
 import org.apache.hadoop.hbase.HServerInfo;
@@ -57,7 +59,9 @@ public class TestLoadBalancer {
 
   @BeforeClass
   public static void beforeAllTests() throws Exception {
-    loadBalancer = new LoadBalancer();
+    Configuration conf = HBaseConfiguration.create();
+    conf.set("hbase.regions.slop", "0");
+    loadBalancer = new LoadBalancer(conf);
     rand = new Random();
   }