Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-17462 Use sliding window for read/write request costs in StochasticLoadBalancer (Tim Brown)

This commit is contained in:
tedyu 2017-01-22 18:35:38 -08:00
parent f254e278ec
commit 7754a9620e
2 changed files with 58 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
View File

@ -42,7 +42,6 @@ import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.master.RegionPlan;
import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster;
import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster.Action;
import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster.Action.Type;
import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster.AssignRegionAction;
@ -1300,29 +1299,50 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
protected double getRegionLoadCost(Collection<RegionLoad> regionLoadList) {
double cost = 0;
for (RegionLoad rl : regionLoadList) {
double toAdd = getCostFromRl(rl);
if (cost == 0) {
cost = toAdd;
} else {
cost = (.5 * cost) + (.5 * toAdd);
}
cost += getCostFromRl(rl);
}
return cost;
return cost / regionLoadList.size();
}
protected abstract double getCostFromRl(RegionLoad rl);
}
/**
* Class to be used for the subset of RegionLoad costs that should be treated as rates.
* We do not compare about the actual rate in requests per second but rather the rate relative
* to the rest of the regions.
*/
abstract static class CostFromRegionLoadAsRateFunction extends CostFromRegionLoadFunction {
CostFromRegionLoadAsRateFunction(Configuration conf) {
super(conf);
}
@Override
protected double getRegionLoadCost(Collection<RegionLoad> regionLoadList) {
double cost = 0;
double previous = 0;
boolean isFirst = true;
for (RegionLoad rl : regionLoadList) {
double current = getCostFromRl(rl);
if (isFirst) {
isFirst = false;
} else {
cost += current - previous;
}
previous = current;
}
return Math.max(0, cost / (regionLoadList.size() - 1));
}
}
/**
* Compute the cost of total number of read requests The more unbalanced the higher the
* computed cost will be. This uses a rolling average of regionload.
*/
static class ReadRequestCostFunction extends CostFromRegionLoadFunction {
static class ReadRequestCostFunction extends CostFromRegionLoadAsRateFunction {
private static final String READ_REQUEST_COST_KEY =
"hbase.master.balancer.stochastic.readRequestCost";
@ -1333,7 +1353,6 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
this.setMultiplier(conf.getFloat(READ_REQUEST_COST_KEY, DEFAULT_READ_REQUEST_COST));
}
@Override
protected double getCostFromRl(RegionLoad rl) {
return rl.getReadRequestsCount();
@ -1344,7 +1363,7 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
* Compute the cost of total number of write requests. The more unbalanced the higher the
* computed cost will be. This uses a rolling average of regionload.
*/
static class WriteRequestCostFunction extends CostFromRegionLoadFunction {
static class WriteRequestCostFunction extends CostFromRegionLoadAsRateFunction {
private static final String WRITE_REQUEST_COST_KEY =
"hbase.master.balancer.stochastic.writeRequestCost";
@ -1522,7 +1541,7 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
* Compute the cost of total memstore size. The more unbalanced the higher the
* computed cost will be. This uses a rolling average of regionload.
*/
static class MemstoreSizeCostFunction extends CostFromRegionLoadFunction {
static class MemstoreSizeCostFunction extends CostFromRegionLoadAsRateFunction {
private static final String MEMSTORE_SIZE_COST_KEY =
"hbase.master.balancer.stochastic.memstoreSizeCost";

24
hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java
View File

@ -230,6 +230,30 @@ public class TestStochasticLoadBalancer extends BalancerTestBase {
}
}
@Test
public void testRegionLoadCost() {
List<RegionLoad> regionLoads = new ArrayList<>();
for (int i = 1; i < 5; i++) {
RegionLoad regionLoad = mock(RegionLoad.class);
when(regionLoad.getReadRequestsCount()).thenReturn(new Long(i));
when(regionLoad.getStorefileSizeMB()).thenReturn(i);
regionLoads.add(regionLoad);
}
Configuration conf = HBaseConfiguration.create();
StochasticLoadBalancer.ReadRequestCostFunction readCostFunction =
new StochasticLoadBalancer.ReadRequestCostFunction(conf);
double rateResult = readCostFunction.getRegionLoadCost(regionLoads);
// read requests are treated as a rate so the average rate here is simply 1
assertEquals(1, rateResult, 0.01);
StochasticLoadBalancer.StoreFileCostFunction storeFileCostFunction =
new StochasticLoadBalancer.StoreFileCostFunction(conf);
double result = storeFileCostFunction.getRegionLoadCost(regionLoads);
// storefile size cost is simply an average of it's value over time
assertEquals(2.5, result, 0.01);
}
@Test
public void testCostFromArray() {
Configuration conf = HBaseConfiguration.create();