From 0a8a7fa2edca24254d97ee0d4edc520b157fb533 Mon Sep 17 00:00:00 2001 From: Zheng Wang <18031031@qq.com> Date: Sun, 19 Jul 2020 20:05:49 +0530 Subject: [PATCH] HBASE-24709 Support MoveCostFunction use a lower multiplier in offpeak hours Closes #2099 Signed-off-by: Viraj Jasani Signed-off-by: Mingliang Liu --- .../balancer/StochasticLoadBalancer.java | 19 ++++++++++++----- .../balancer/TestStochasticLoadBalancer.java | 21 +++++++++++++++++++ 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java index d3bb9f0a70a..7bbcc9161cb 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java @@ -46,6 +46,7 @@ import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster.AssignRe import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster.LocalityType; import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster.MoveRegionAction; import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster.SwapRegionsAction; +import org.apache.hadoop.hbase.regionserver.compactions.OffPeakHours; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.ReflectionUtils; import org.apache.yetus.audience.InterfaceAudience; @@ -826,26 +827,34 @@ public class StochasticLoadBalancer extends BaseLoadBalancer { */ static class MoveCostFunction extends CostFunction { private static final String MOVE_COST_KEY = "hbase.master.balancer.stochastic.moveCost"; + private static final String MOVE_COST_OFFPEAK_KEY = + "hbase.master.balancer.stochastic.moveCost.offpeak"; private static final String MAX_MOVES_PERCENT_KEY = "hbase.master.balancer.stochastic.maxMovePercent"; - private static final float DEFAULT_MOVE_COST = 7; + static final float DEFAULT_MOVE_COST = 7; + static final float DEFAULT_MOVE_COST_OFFPEAK = 3; private static final int DEFAULT_MAX_MOVES = 600; private static final float DEFAULT_MAX_MOVE_PERCENT = 0.25f; private final float maxMovesPercent; + private final Configuration conf; MoveCostFunction(Configuration conf) { super(conf); - - // Move cost multiplier should be the same cost or higher than the rest of the costs to ensure - // that large benefits are need to overcome the cost of a move. - this.setMultiplier(conf.getFloat(MOVE_COST_KEY, DEFAULT_MOVE_COST)); + this.conf = conf; // What percent of the number of regions a single run of the balancer can move. maxMovesPercent = conf.getFloat(MAX_MOVES_PERCENT_KEY, DEFAULT_MAX_MOVE_PERCENT); } @Override protected double cost() { + // Move cost multiplier should be the same cost or higher than the rest of the costs to ensure + // that large benefits are need to overcome the cost of a move. + if (OffPeakHours.getInstance(conf).isOffPeakHour()) { + this.setMultiplier(conf.getFloat(MOVE_COST_OFFPEAK_KEY, DEFAULT_MOVE_COST_OFFPEAK)); + } else { + this.setMultiplier(conf.getFloat(MOVE_COST_KEY, DEFAULT_MOVE_COST)); + } // Try and size the max number of Moves, but always be prepared to move some. int maxMoves = Math.max((int) (cluster.numRegions * maxMovesPercent), DEFAULT_MAX_MOVES); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java index 0731761caa6..dcacfb56ac6 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java @@ -203,6 +203,27 @@ public class TestStochasticLoadBalancer extends BalancerTestBase { } } + @Test + public void testMoveCostMultiplier() throws Exception { + Configuration conf = HBaseConfiguration.create(); + StochasticLoadBalancer.CostFunction + costFunction = new StochasticLoadBalancer.MoveCostFunction(conf); + BaseLoadBalancer.Cluster cluster = mockCluster(clusterStateMocks[0]); + costFunction.init(cluster); + costFunction.cost(); + assertEquals(StochasticLoadBalancer.MoveCostFunction.DEFAULT_MOVE_COST, + costFunction.getMultiplier(), 0.01); + + //In offpeak hours, the multiplier of move cost should be lower + conf.setInt("hbase.offpeak.start.hour",0); + conf.setInt("hbase.offpeak.end.hour",23); + costFunction = new StochasticLoadBalancer.MoveCostFunction(conf); + costFunction.init(cluster); + costFunction.cost(); + assertEquals(StochasticLoadBalancer.MoveCostFunction.DEFAULT_MOVE_COST_OFFPEAK + , costFunction.getMultiplier(), 0.01); + } + @Test public void testMoveCost() throws Exception { Configuration conf = HBaseConfiguration.create();