Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-24709 Support MoveCostFunction use a lower multiplier in offpeak hours 

Closes #2099

Signed-off-by: Viraj Jasani <vjasani@apache.org>
Signed-off-by: Mingliang Liu <liuml07@apache.org>
This commit is contained in:
Zheng Wang 2020-07-19 20:05:49 +05:30 committed by Viraj Jasani
parent 552008c577
commit 0a8a7fa2ed
No known key found for this signature in database
GPG Key ID: B3D6C0B41C8ADFD5
2 changed files with 35 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
View File

@ -46,6 +46,7 @@ import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster.AssignRe
import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster.LocalityType; import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster.LocalityType;
import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster.MoveRegionAction; import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster.MoveRegionAction;
import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster.SwapRegionsAction; import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster.SwapRegionsAction;
import org.apache.hadoop.hbase.regionserver.compactions.OffPeakHours;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.ReflectionUtils; import org.apache.hadoop.hbase.util.ReflectionUtils;
import org.apache.yetus.audience.InterfaceAudience; import org.apache.yetus.audience.InterfaceAudience;
@ -826,26 +827,34 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
*/ */
static class MoveCostFunction extends CostFunction { static class MoveCostFunction extends CostFunction {
private static final String MOVE_COST_KEY = "hbase.master.balancer.stochastic.moveCost"; private static final String MOVE_COST_KEY = "hbase.master.balancer.stochastic.moveCost";
private static final String MOVE_COST_OFFPEAK_KEY =
"hbase.master.balancer.stochastic.moveCost.offpeak";
private static final String MAX_MOVES_PERCENT_KEY = private static final String MAX_MOVES_PERCENT_KEY =
"hbase.master.balancer.stochastic.maxMovePercent"; "hbase.master.balancer.stochastic.maxMovePercent";
private static final float DEFAULT_MOVE_COST = 7; static final float DEFAULT_MOVE_COST = 7;
static final float DEFAULT_MOVE_COST_OFFPEAK = 3;
private static final int DEFAULT_MAX_MOVES = 600; private static final int DEFAULT_MAX_MOVES = 600;
private static final float DEFAULT_MAX_MOVE_PERCENT = 0.25f; private static final float DEFAULT_MAX_MOVE_PERCENT = 0.25f;
private final float maxMovesPercent; private final float maxMovesPercent;
private final Configuration conf;
MoveCostFunction(Configuration conf) { MoveCostFunction(Configuration conf) {
super(conf); super(conf);
this.conf = conf;
// Move cost multiplier should be the same cost or higher than the rest of the costs to ensure
// that large benefits are need to overcome the cost of a move.
this.setMultiplier(conf.getFloat(MOVE_COST_KEY, DEFAULT_MOVE_COST));
// What percent of the number of regions a single run of the balancer can move. // What percent of the number of regions a single run of the balancer can move.
maxMovesPercent = conf.getFloat(MAX_MOVES_PERCENT_KEY, DEFAULT_MAX_MOVE_PERCENT); maxMovesPercent = conf.getFloat(MAX_MOVES_PERCENT_KEY, DEFAULT_MAX_MOVE_PERCENT);
} }
@Override @Override
protected double cost() { protected double cost() {
// Move cost multiplier should be the same cost or higher than the rest of the costs to ensure
// that large benefits are need to overcome the cost of a move.
if (OffPeakHours.getInstance(conf).isOffPeakHour()) {
this.setMultiplier(conf.getFloat(MOVE_COST_OFFPEAK_KEY, DEFAULT_MOVE_COST_OFFPEAK));
} else {
this.setMultiplier(conf.getFloat(MOVE_COST_KEY, DEFAULT_MOVE_COST));
}
// Try and size the max number of Moves, but always be prepared to move some. // Try and size the max number of Moves, but always be prepared to move some.
int maxMoves = Math.max((int) (cluster.numRegions * maxMovesPercent), int maxMoves = Math.max((int) (cluster.numRegions * maxMovesPercent),
DEFAULT_MAX_MOVES); DEFAULT_MAX_MOVES);

21
hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java
View File

@ -203,6 +203,27 @@ public class TestStochasticLoadBalancer extends BalancerTestBase {
} }
} }
@Test
public void testMoveCostMultiplier() throws Exception {
Configuration conf = HBaseConfiguration.create();
StochasticLoadBalancer.CostFunction
costFunction = new StochasticLoadBalancer.MoveCostFunction(conf);
BaseLoadBalancer.Cluster cluster = mockCluster(clusterStateMocks[0]);
costFunction.init(cluster);
costFunction.cost();
assertEquals(StochasticLoadBalancer.MoveCostFunction.DEFAULT_MOVE_COST,
costFunction.getMultiplier(), 0.01);
//In offpeak hours, the multiplier of move cost should be lower
conf.setInt("hbase.offpeak.start.hour",0);
conf.setInt("hbase.offpeak.end.hour",23);
costFunction = new StochasticLoadBalancer.MoveCostFunction(conf);
costFunction.init(cluster);
costFunction.cost();
assertEquals(StochasticLoadBalancer.MoveCostFunction.DEFAULT_MOVE_COST_OFFPEAK
, costFunction.getMultiplier(), 0.01);
}
@Test @Test
public void testMoveCost() throws Exception { public void testMoveCost() throws Exception {
Configuration conf = HBaseConfiguration.create(); Configuration conf = HBaseConfiguration.create();