Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-17706 TableSkewCostFunction improperly computes max skew 

Signed-off-by: tedyu <yuzhihong@gmail.com>
This commit is contained in:
Kahlil Oppenheimer 2017-02-28 00:33:57 -05:00 committed by tedyu
parent e67eb6c424
commit edbd0e494d
3 changed files with 47 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
View File

@ -739,18 +739,15 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
}
numRegionsPerServerPerTable[newServer][tableIndex]++;
//check whether this caused maxRegionsPerTable in the new Server to be updated
if (numRegionsPerServerPerTable[newServer][tableIndex] > numMaxRegionsPerTable[tableIndex]) {
numMaxRegionsPerTable[tableIndex] = numRegionsPerServerPerTable[newServer][tableIndex];
} else if (oldServer >= 0 && (numRegionsPerServerPerTable[oldServer][tableIndex] + 1)
== numMaxRegionsPerTable[tableIndex]) {
//recompute maxRegionsPerTable since the previous value was coming from the old server
for (int serverIndex = 0 ; serverIndex < numRegionsPerServerPerTable.length; serverIndex++) {
if (numRegionsPerServerPerTable[serverIndex][tableIndex] > numMaxRegionsPerTable[tableIndex]) {
numMaxRegionsPerTable[tableIndex] = numRegionsPerServerPerTable[serverIndex][tableIndex];
}
}
// if old server had max num regions, assume (for now)
// max num regions went down since we moved the region
if (oldServer >= 0 &&
(numRegionsPerServerPerTable[oldServer][tableIndex] + 1) == numMaxRegionsPerTable[tableIndex]) {
numMaxRegionsPerTable[tableIndex]--;
}
// Now check if new server sets new max
numMaxRegionsPerTable[tableIndex] =
Math.max(numMaxRegionsPerTable[tableIndex], numRegionsPerServerPerTable[newServer][tableIndex]);
// update for servers
int primary = regionIndexToPrimaryIndex[region];

19
hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
View File

@ -272,14 +272,6 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
@Override
protected boolean needsBalance(Cluster cluster) {
ClusterLoadState cs = new ClusterLoadState(cluster.clusterState);
if (cs.getNumServers() < MIN_SERVER_BALANCE) {
if (LOG.isDebugEnabled()) {
LOG.debug("Not running balancer because only " + cs.getNumServers()
+ " active regionserver(s)");
}
return false;
}
if (areSomeRegionReplicasColocated(cluster)) {
return true;
}
@ -306,6 +298,17 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
+ minCostNeedBalance);
return false;
}
ClusterLoadState cs = new ClusterLoadState(cluster.clusterState);
if (cs.getNumServers() < MIN_SERVER_BALANCE) {
if (LOG.isDebugEnabled()) {
LOG.debug("Not running balancer because only " + cs.getNumServers()
+ " active regionserver(s)");
}
return false;
}
return true;
}

28
hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java
View File

@ -50,6 +50,7 @@ import org.apache.hadoop.hbase.master.RegionPlan;
import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster;
import org.apache.hadoop.hbase.master.balancer.StochasticLoadBalancer.CandidateGenerator;
import org.apache.hadoop.hbase.master.balancer.StochasticLoadBalancer.TableSkewCandidateGenerator;
import org.apache.hadoop.hbase.master.balancer.StochasticLoadBalancer.LoadCandidateGenerator;
import org.apache.hadoop.hbase.testclassification.FlakeyTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.Bytes;
@ -237,6 +238,33 @@ public class TestStochasticLoadBalancer extends BalancerTestBase {
}
}
@Test
public void testTableSkewCostProperlyDecreases() {
int replication = 1;
Configuration conf = HBaseConfiguration.create();
StochasticLoadBalancer.CostFunction
costFunction = new StochasticLoadBalancer.TableSkewCostFunction(conf);
CandidateGenerator generator = new LoadCandidateGenerator();
// Start out with 100 regions on one server and 0 regions on the other
int numNodes = 2;
int numTables = 1;
int numRegions = 100;
int numRegionsPerServer = 0;
Map<ServerName, List<HRegionInfo>> serverMap = createServerMap(numNodes, numRegions, numRegionsPerServer, replication, numTables);
BaseLoadBalancer.Cluster cluster = new Cluster(serverMap, null, null, null);
costFunction.init(cluster);
double cost = costFunction.cost();
assertEquals(1.0, cost, .0001);
for (int i = 0; i < 100; i++) {
Cluster.Action action = generator.generate(cluster);
cluster.doAction(action);
costFunction.postAction(action);
cost = costFunction.cost();
}
assertTrue(cost < 0.5);
}
@Test
public void testRegionLoadCost() {
List<BalancerRegionLoad> regionLoads = new ArrayList<>();