Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-26988 dynamic configuration of loadbalance.bytable (#4384) 

Signed-off-by: Andrew Purtell <apurtell@apache.org>
This commit is contained in:
d-c-manning 2022-04-29 14:23:32 -07:00 committed by GitHub
parent ebfac2164a
commit 4a33ed1c61
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 23 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
View File

@ -71,12 +71,14 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
"hbase.master.balancer.rejection.buffer.enabled"; "hbase.master.balancer.rejection.buffer.enabled";
public static final boolean DEFAULT_BALANCER_REJECTION_BUFFER_ENABLED = false; public static final boolean DEFAULT_BALANCER_REJECTION_BUFFER_ENABLED = false;
public static final boolean DEFAULT_HBASE_MASTER_LOADBALANCE_BYTABLE = false;
protected static final int MIN_SERVER_BALANCE = 2; protected static final int MIN_SERVER_BALANCE = 2;
private volatile boolean stopped = false; private volatile boolean stopped = false;
protected volatile RegionHDFSBlockLocationFinder regionFinder; protected volatile RegionHDFSBlockLocationFinder regionFinder;
protected boolean useRegionFinder; protected boolean useRegionFinder;
protected boolean isByTable = false; protected boolean isByTable = DEFAULT_HBASE_MASTER_LOADBALANCE_BYTABLE;
// slop for regions // slop for regions
protected float slop; protected float slop;
@ -415,7 +417,8 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
} else { } else {
regionFinder = null; regionFinder = null;
} }
this.isByTable = conf.getBoolean(HConstants.HBASE_MASTER_LOADBALANCE_BYTABLE, isByTable); this.isByTable = conf.getBoolean(HConstants.HBASE_MASTER_LOADBALANCE_BYTABLE,
DEFAULT_HBASE_MASTER_LOADBALANCE_BYTABLE);
// Print out base configs. Don't print overallSlop since it for simple balancer exclusively. // Print out base configs. Don't print overallSlop since it for simple balancer exclusively.
LOG.info("slop={}", this.slop); LOG.info("slop={}", this.slop);
} }

30
hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
View File

@ -104,17 +104,23 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
protected static final String STEPS_PER_REGION_KEY = protected static final String STEPS_PER_REGION_KEY =
"hbase.master.balancer.stochastic.stepsPerRegion"; "hbase.master.balancer.stochastic.stepsPerRegion";
protected static final int DEFAULT_STEPS_PER_REGION = 800;
protected static final String MAX_STEPS_KEY = protected static final String MAX_STEPS_KEY =
"hbase.master.balancer.stochastic.maxSteps"; "hbase.master.balancer.stochastic.maxSteps";
protected static final int DEFAULT_MAX_STEPS = 1000000;
protected static final String RUN_MAX_STEPS_KEY = protected static final String RUN_MAX_STEPS_KEY =
"hbase.master.balancer.stochastic.runMaxSteps"; "hbase.master.balancer.stochastic.runMaxSteps";
protected static final boolean DEFAULT_RUN_MAX_STEPS = false;
protected static final String MAX_RUNNING_TIME_KEY = protected static final String MAX_RUNNING_TIME_KEY =
"hbase.master.balancer.stochastic.maxRunningTime"; "hbase.master.balancer.stochastic.maxRunningTime";
protected static final long DEFAULT_MAX_RUNNING_TIME = 30 * 1000; // 30 seconds.
protected static final String KEEP_REGION_LOADS = protected static final String KEEP_REGION_LOADS =
"hbase.master.balancer.stochastic.numRegionLoadsToRemember"; "hbase.master.balancer.stochastic.numRegionLoadsToRemember";
protected static final int DEFAULT_KEEP_REGION_LOADS = 15;
private static final String TABLE_FUNCTION_SEP = "_"; private static final String TABLE_FUNCTION_SEP = "_";
protected static final String MIN_COST_NEED_BALANCE_KEY = protected static final String MIN_COST_NEED_BALANCE_KEY =
"hbase.master.balancer.stochastic.minCostNeedBalance"; "hbase.master.balancer.stochastic.minCostNeedBalance";
protected static final float DEFAULT_MIN_COST_NEED_BALANCE = 0.025f;
protected static final String COST_FUNCTIONS_COST_FUNCTIONS_KEY = protected static final String COST_FUNCTIONS_COST_FUNCTIONS_KEY =
"hbase.master.balancer.stochastic.additionalCostFunctions"; "hbase.master.balancer.stochastic.additionalCostFunctions";
public static final String OVERALL_COST_FUNCTION_NAME = "Overall"; public static final String OVERALL_COST_FUNCTION_NAME = "Overall";
@ -122,12 +128,12 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
Map<String, Deque<BalancerRegionLoad>> loads = new HashMap<>(); Map<String, Deque<BalancerRegionLoad>> loads = new HashMap<>();
// values are defaults // values are defaults
private int maxSteps = 1000000; private int maxSteps = DEFAULT_MAX_STEPS;
private boolean runMaxSteps = false; private boolean runMaxSteps = DEFAULT_RUN_MAX_STEPS;
private int stepsPerRegion = 800; private int stepsPerRegion = DEFAULT_STEPS_PER_REGION;
private long maxRunningTime = 30 * 1000 * 1; // 30 seconds. private long maxRunningTime = DEFAULT_MAX_RUNNING_TIME;
private int numRegionLoadsToRemember = 15; private int numRegionLoadsToRemember = DEFAULT_KEEP_REGION_LOADS;
private float minCostNeedBalance = 0.025f; private float minCostNeedBalance = DEFAULT_MIN_COST_NEED_BALANCE;
private List<CostFunction> costFunctions; // FindBugs: Wants this protected; IS2_INCONSISTENT_SYNC private List<CostFunction> costFunctions; // FindBugs: Wants this protected; IS2_INCONSISTENT_SYNC
// To save currently configed sum of multiplier. Defaulted at 1 for cases that carry high cost // To save currently configed sum of multiplier. Defaulted at 1 for cases that carry high cost
@ -216,13 +222,13 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
@Override @Override
protected void loadConf(Configuration conf) { protected void loadConf(Configuration conf) {
super.loadConf(conf); super.loadConf(conf);
maxSteps = conf.getInt(MAX_STEPS_KEY, maxSteps); maxSteps = conf.getInt(MAX_STEPS_KEY, DEFAULT_MAX_STEPS);
stepsPerRegion = conf.getInt(STEPS_PER_REGION_KEY, stepsPerRegion); stepsPerRegion = conf.getInt(STEPS_PER_REGION_KEY, DEFAULT_STEPS_PER_REGION);
maxRunningTime = conf.getLong(MAX_RUNNING_TIME_KEY, maxRunningTime); maxRunningTime = conf.getLong(MAX_RUNNING_TIME_KEY, DEFAULT_MAX_RUNNING_TIME);
runMaxSteps = conf.getBoolean(RUN_MAX_STEPS_KEY, runMaxSteps); runMaxSteps = conf.getBoolean(RUN_MAX_STEPS_KEY, DEFAULT_RUN_MAX_STEPS);
numRegionLoadsToRemember = conf.getInt(KEEP_REGION_LOADS, numRegionLoadsToRemember); numRegionLoadsToRemember = conf.getInt(KEEP_REGION_LOADS, DEFAULT_KEEP_REGION_LOADS);
minCostNeedBalance = conf.getFloat(MIN_COST_NEED_BALANCE_KEY, minCostNeedBalance); minCostNeedBalance = conf.getFloat(MIN_COST_NEED_BALANCE_KEY, DEFAULT_MIN_COST_NEED_BALANCE);
localityCandidateGenerator = new LocalityBasedCandidateGenerator(); localityCandidateGenerator = new LocalityBasedCandidateGenerator();
localityCost = new ServerLocalityCostFunction(conf); localityCost = new ServerLocalityCostFunction(conf);
rackLocalityCost = new RackLocalityCostFunction(conf); rackLocalityCost = new RackLocalityCostFunction(conf);