From 4a33ed1c61476dfb020062214dc2c076a1f6b4fe Mon Sep 17 00:00:00 2001 From: d-c-manning <67607031+d-c-manning@users.noreply.github.com> Date: Fri, 29 Apr 2022 14:23:32 -0700 Subject: [PATCH] HBASE-26988 dynamic configuration of loadbalance.bytable (#4384) Signed-off-by: Andrew Purtell --- .../master/balancer/BaseLoadBalancer.java | 7 +++-- .../balancer/StochasticLoadBalancer.java | 30 +++++++++++-------- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java index e453d11fd31..43bb7073a30 100644 --- a/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java +++ b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java @@ -71,12 +71,14 @@ public abstract class BaseLoadBalancer implements LoadBalancer { "hbase.master.balancer.rejection.buffer.enabled"; public static final boolean DEFAULT_BALANCER_REJECTION_BUFFER_ENABLED = false; + public static final boolean DEFAULT_HBASE_MASTER_LOADBALANCE_BYTABLE = false; + protected static final int MIN_SERVER_BALANCE = 2; private volatile boolean stopped = false; protected volatile RegionHDFSBlockLocationFinder regionFinder; protected boolean useRegionFinder; - protected boolean isByTable = false; + protected boolean isByTable = DEFAULT_HBASE_MASTER_LOADBALANCE_BYTABLE; // slop for regions protected float slop; @@ -415,7 +417,8 @@ public abstract class BaseLoadBalancer implements LoadBalancer { } else { regionFinder = null; } - this.isByTable = conf.getBoolean(HConstants.HBASE_MASTER_LOADBALANCE_BYTABLE, isByTable); + this.isByTable = conf.getBoolean(HConstants.HBASE_MASTER_LOADBALANCE_BYTABLE, + DEFAULT_HBASE_MASTER_LOADBALANCE_BYTABLE); // Print out base configs. Don't print overallSlop since it for simple balancer exclusively. LOG.info("slop={}", this.slop); } diff --git a/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java index b3754715992..edcbce1bc59 100644 --- a/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java +++ b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java @@ -104,17 +104,23 @@ public class StochasticLoadBalancer extends BaseLoadBalancer { protected static final String STEPS_PER_REGION_KEY = "hbase.master.balancer.stochastic.stepsPerRegion"; + protected static final int DEFAULT_STEPS_PER_REGION = 800; protected static final String MAX_STEPS_KEY = "hbase.master.balancer.stochastic.maxSteps"; + protected static final int DEFAULT_MAX_STEPS = 1000000; protected static final String RUN_MAX_STEPS_KEY = "hbase.master.balancer.stochastic.runMaxSteps"; + protected static final boolean DEFAULT_RUN_MAX_STEPS = false; protected static final String MAX_RUNNING_TIME_KEY = "hbase.master.balancer.stochastic.maxRunningTime"; + protected static final long DEFAULT_MAX_RUNNING_TIME = 30 * 1000; // 30 seconds. protected static final String KEEP_REGION_LOADS = "hbase.master.balancer.stochastic.numRegionLoadsToRemember"; + protected static final int DEFAULT_KEEP_REGION_LOADS = 15; private static final String TABLE_FUNCTION_SEP = "_"; protected static final String MIN_COST_NEED_BALANCE_KEY = "hbase.master.balancer.stochastic.minCostNeedBalance"; + protected static final float DEFAULT_MIN_COST_NEED_BALANCE = 0.025f; protected static final String COST_FUNCTIONS_COST_FUNCTIONS_KEY = "hbase.master.balancer.stochastic.additionalCostFunctions"; public static final String OVERALL_COST_FUNCTION_NAME = "Overall"; @@ -122,12 +128,12 @@ public class StochasticLoadBalancer extends BaseLoadBalancer { Map> loads = new HashMap<>(); // values are defaults - private int maxSteps = 1000000; - private boolean runMaxSteps = false; - private int stepsPerRegion = 800; - private long maxRunningTime = 30 * 1000 * 1; // 30 seconds. - private int numRegionLoadsToRemember = 15; - private float minCostNeedBalance = 0.025f; + private int maxSteps = DEFAULT_MAX_STEPS; + private boolean runMaxSteps = DEFAULT_RUN_MAX_STEPS; + private int stepsPerRegion = DEFAULT_STEPS_PER_REGION; + private long maxRunningTime = DEFAULT_MAX_RUNNING_TIME; + private int numRegionLoadsToRemember = DEFAULT_KEEP_REGION_LOADS; + private float minCostNeedBalance = DEFAULT_MIN_COST_NEED_BALANCE; private List costFunctions; // FindBugs: Wants this protected; IS2_INCONSISTENT_SYNC // To save currently configed sum of multiplier. Defaulted at 1 for cases that carry high cost @@ -216,13 +222,13 @@ public class StochasticLoadBalancer extends BaseLoadBalancer { @Override protected void loadConf(Configuration conf) { super.loadConf(conf); - maxSteps = conf.getInt(MAX_STEPS_KEY, maxSteps); - stepsPerRegion = conf.getInt(STEPS_PER_REGION_KEY, stepsPerRegion); - maxRunningTime = conf.getLong(MAX_RUNNING_TIME_KEY, maxRunningTime); - runMaxSteps = conf.getBoolean(RUN_MAX_STEPS_KEY, runMaxSteps); + maxSteps = conf.getInt(MAX_STEPS_KEY, DEFAULT_MAX_STEPS); + stepsPerRegion = conf.getInt(STEPS_PER_REGION_KEY, DEFAULT_STEPS_PER_REGION); + maxRunningTime = conf.getLong(MAX_RUNNING_TIME_KEY, DEFAULT_MAX_RUNNING_TIME); + runMaxSteps = conf.getBoolean(RUN_MAX_STEPS_KEY, DEFAULT_RUN_MAX_STEPS); - numRegionLoadsToRemember = conf.getInt(KEEP_REGION_LOADS, numRegionLoadsToRemember); - minCostNeedBalance = conf.getFloat(MIN_COST_NEED_BALANCE_KEY, minCostNeedBalance); + numRegionLoadsToRemember = conf.getInt(KEEP_REGION_LOADS, DEFAULT_KEEP_REGION_LOADS); + minCostNeedBalance = conf.getFloat(MIN_COST_NEED_BALANCE_KEY, DEFAULT_MIN_COST_NEED_BALANCE); localityCandidateGenerator = new LocalityBasedCandidateGenerator(); localityCost = new ServerLocalityCostFunction(conf); rackLocalityCost = new RackLocalityCostFunction(conf);