Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-25834 Remove balanceTable method from LoadBalancer interface (#3217) 

Signed-off-by: Yulin Niu <niuyulin@apache.org>
This commit is contained in:
Duo Zhang 2021-05-05 15:48:01 +08:00
parent 3f8d3ba6d9
commit afa9836b87
10 changed files with 58 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java
View File

@ -120,8 +120,7 @@ public class RSGroupBasedLoadBalancer implements RSGroupableBalancer {
}
/**
* Override to balance by RSGroup
* not invoke {@link #balanceTable(TableName, Map)}
* Balance by RSGroup.
*/
@Override
public List<RegionPlan> balanceCluster(
@ -427,40 +426,6 @@ public class RSGroupBasedLoadBalancer implements RSGroupableBalancer {
internalBalancer.updateBalancerStatus(status);
}
/**
* can achieve table balanced rather than overall balanced
*/
@Override
public List<RegionPlan> balanceTable(TableName tableName,
Map<ServerName, List<RegionInfo>> loadOfOneTable) {
if (!isOnline()) {
LOG.error(RSGroupInfoManager.class.getSimpleName()
+ " is not online, unable to perform balanceTable");
return null;
}
Map<TableName, Map<ServerName, List<RegionInfo>>> loadOfThisTable = new HashMap<>();
loadOfThisTable.put(tableName, loadOfOneTable);
Pair<Map<TableName, Map<ServerName, List<RegionInfo>>>, List<RegionPlan>>
correctedStateAndRegionPlans;
// Calculate correct assignments and a list of RegionPlan for mis-placed regions
try {
correctedStateAndRegionPlans = correctAssignments(loadOfThisTable);
} catch (IOException e) {
LOG.error("get correct assignments and mis-placed regions error ", e);
return null;
}
Map<TableName, Map<ServerName, List<RegionInfo>>> correctedLoadOfThisTable =
correctedStateAndRegionPlans.getFirst();
List<RegionPlan> regionPlans = correctedStateAndRegionPlans.getSecond();
List<RegionPlan> tablePlans =
this.internalBalancer.balanceTable(tableName, correctedLoadOfThisTable.get(tableName));
if (tablePlans != null) {
regionPlans.addAll(tablePlans);
}
return regionPlans;
}
private List<ServerName> getFallBackCandidates(List<ServerName> servers) {
List<ServerName> serverNames = null;
try {

2
hbase-server/src/main/java/org/apache/hadoop/hbase/favored/FavoredNodeLoadBalancer.java
View File

@ -89,7 +89,7 @@ public class FavoredNodeLoadBalancer extends BaseLoadBalancer implements Favored
}
@Override
public List<RegionPlan> balanceTable(TableName tableName,
protected List<RegionPlan> balanceTable(TableName tableName,
Map<ServerName, List<RegionInfo>> loadOfOneTable) {
// TODO. Look at is whether Stochastic loadbalancer can be integrated with this
List<RegionPlan> plans = new ArrayList<>();

17
hbase-server/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java
View File

@ -86,25 +86,14 @@ public interface LoadBalancer extends Configurable, Stoppable, ConfigurationObse
void setMasterServices(MasterServices masterServices);
/**
* Perform the major balance operation for cluster, will invoke {@link #balanceTable} to do
* actual balance. Normally not need override this method, except SimpleLoadBalancer and
* RSGroupBasedLoadBalancer.
* Perform the major balance operation for cluster.
* @param loadOfAllTable region load of servers for all table
* @return a list of regions to be moved, including source and destination, or null if cluster is
* already balanced
*/
List<RegionPlan> balanceCluster(Map<TableName,
Map<ServerName, List<RegionInfo>>> loadOfAllTable) throws IOException;
List<RegionPlan> balanceCluster(Map<TableName, Map<ServerName, List<RegionInfo>>> loadOfAllTable)
throws IOException;
/**
* Perform the major balance operation for table, all class implement of {@link LoadBalancer}
* should override this method
* @param tableName the table to be balanced
* @param loadOfOneTable region load of servers for the specific one table
* @return List of plans
*/
List<RegionPlan> balanceTable(TableName tableName,
Map<ServerName, List<RegionInfo>> loadOfOneTable);
/**
* Perform a Round Robin assignment of regions.
* @param regions

41
hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
View File

@ -773,13 +773,44 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
return returnMap;
}
@Override
public abstract List<RegionPlan> balanceTable(TableName tableName,
Map<ServerName, List<RegionInfo>> loadOfOneTable);
/**
* Perform the major balance operation for table, all sub classes should override this method.
* <p/>
* Will be invoked by {@link #balanceCluster(Map)}. If
* {@link HConstants#HBASE_MASTER_LOADBALANCE_BYTABLE} is enabled, we will call this method
* multiple times, one table a time, where we will only pass in the regions for a single table
* each time. If not, we will pass in all the regions at once, and the {@code tableName} will be
* {@link HConstants#ENSEMBLE_TABLE_NAME}.
* @param tableName the table to be balanced
* @param loadOfOneTable region load of servers for the specific one table
* @return List of plans
*/
protected abstract List<RegionPlan> balanceTable(TableName tableName,
Map<ServerName, List<RegionInfo>> loadOfOneTable);
/**
* Called before actually executing balanceCluster. The sub classes could override this method to
* do some initialization work.
*/
protected void
preBalanceCluster(Map<TableName, Map<ServerName, List<RegionInfo>>> loadOfAllTable) {
}
/**
* Perform the major balance operation for cluster, will invoke
* {@link #balanceTable(TableName, Map)} to do actual balance.
* <p/>
* THIs method is marked as final which means you should not override this method. See the javadoc
* for {@link #balanceTable(TableName, Map)} for more details.
* @param loadOfAllTable region load of servers for all table
* @return a list of regions to be moved, including source and destination, or null if cluster is
* already balanced
* @see #balanceTable(TableName, Map)
*/
@Override
public List<RegionPlan>
balanceCluster(Map<TableName, Map<ServerName, List<RegionInfo>>> loadOfAllTable) {
public final synchronized List<RegionPlan>
balanceCluster(Map<TableName, Map<ServerName, List<RegionInfo>>> loadOfAllTable) {
preBalanceCluster(loadOfAllTable);
if (isByTable) {
List<RegionPlan> result = new ArrayList<>();
loadOfAllTable.forEach((tableName, loadOfOneTable) -> {

2
hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/FavoredStochasticBalancer.java
View File

@ -700,7 +700,7 @@ public class FavoredStochasticBalancer extends StochasticLoadBalancer implements
* implementation. For the misplaced regions, we assign a bogus server to it and AM takes care.
*/
@Override
public synchronized List<RegionPlan> balanceTable(TableName tableName,
protected List<RegionPlan> balanceTable(TableName tableName,
Map<ServerName, List<RegionInfo>> loadOfOneTable) {
if (this.services != null) {

6
hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/MaintenanceLoadBalancer.java
View File

@ -67,12 +67,6 @@ public class MaintenanceLoadBalancer extends Configured implements LoadBalancer
return Collections.emptyList();
}
@Override
public List<RegionPlan> balanceTable(TableName tableName,
Map<ServerName, List<RegionInfo>> loadOfOneTable) {
return Collections.emptyList();
}
private Map<ServerName, List<RegionInfo>> assign(Collection<RegionInfo> regions,
List<ServerName> servers) {
// should only have 1 region server in maintenance mode

22
hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SimpleLoadBalancer.java
View File

@ -126,6 +126,13 @@ public class SimpleLoadBalancer extends BaseLoadBalancer {
avgLoadOverall = sum / serverLoadList.size();
}
@Override
protected void
preBalanceCluster(Map<TableName, Map<ServerName, List<RegionInfo>>> loadOfAllTable) {
// We need clusterLoad of all regions on every server to achieve overall balanced
setClusterLoad(loadOfAllTable);
}
@Override
public void onConfigurationChange(Configuration conf) {
float originSlop = slop;
@ -251,7 +258,7 @@ public class SimpleLoadBalancer extends BaseLoadBalancer {
* or null if cluster is already balanced
*/
@Override
public List<RegionPlan> balanceTable(TableName tableName,
protected List<RegionPlan> balanceTable(TableName tableName,
Map<ServerName, List<RegionInfo>> loadOfOneTable) {
List<RegionPlan> regionsToReturn = balanceMasterRegions(loadOfOneTable);
if (regionsToReturn != null || loadOfOneTable == null || loadOfOneTable.size() <= 1) {
@ -485,7 +492,7 @@ public class SimpleLoadBalancer extends BaseLoadBalancer {
* max. Together with other regions left to be assigned, we distribute all regionToMove, to the RS
* that have less regions in whole cluster scope.
*/
public void balanceOverall(List<RegionPlan> regionsToReturn,
private void balanceOverall(List<RegionPlan> regionsToReturn,
Map<ServerName, BalanceInfo> serverBalanceInfo, boolean fetchFromTail,
MinMaxPriorityQueue<RegionPlan> regionsToMove, int max, int min) {
// Step 1.
@ -632,15 +639,4 @@ public class SimpleLoadBalancer extends BaseLoadBalancer {
rp.setDestination(sn);
regionsToReturn.add(rp);
}
/**
* Override to invoke {@link #setClusterLoad} before balance, We need clusterLoad of all regions
* on every server to achieve overall balanced
*/
@Override
public synchronized List<RegionPlan>
balanceCluster(Map<TableName, Map<ServerName, List<RegionInfo>>> loadOfAllTable) {
setClusterLoad(loadOfAllTable);
return super.balanceCluster(loadOfAllTable);
}
}

2
hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
View File

@ -377,7 +377,7 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
* should always approach the optimal state given enough steps.
*/
@Override
public synchronized List<RegionPlan> balanceTable(TableName tableName, Map<ServerName,
protected List<RegionPlan> balanceTable(TableName tableName, Map<ServerName,
List<RegionInfo>> loadOfOneTable) {
List<RegionPlan> plans = balanceMasterRegions(loadOfOneTable);
if (plans != null || loadOfOneTable == null || loadOfOneTable.size() <= 1) {

9
hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/LoadBalancerPerformanceEvaluation.java
View File

@ -83,7 +83,7 @@ public class LoadBalancerPerformanceEvaluation extends AbstractHBaseTool {
private List<ServerName> servers;
private List<RegionInfo> regions;
private Map<RegionInfo, ServerName> regionServerMap;
private Map<ServerName, List<RegionInfo>> serverRegionMap;
private Map<TableName, Map<ServerName, List<RegionInfo>>> tableServerRegionMap;
// Non-default configurations.
private void setupConf() {
@ -92,6 +92,7 @@ public class LoadBalancerPerformanceEvaluation extends AbstractHBaseTool {
}
private void generateRegionsAndServers() {
TableName tableName = TableName.valueOf("LoadBalancerPerfTable");
// regions
regions = new ArrayList<>(numRegions);
regionServerMap = new HashMap<>(numRegions);
@ -101,7 +102,6 @@ public class LoadBalancerPerformanceEvaluation extends AbstractHBaseTool {
Bytes.putInt(start, 0, i);
Bytes.putInt(end, 0, i + 1);
TableName tableName = TableName.valueOf("LoadBalancerPerfTable");
RegionInfo hri = RegionInfoBuilder.newBuilder(tableName)
.setStartKey(start)
.setEndKey(end)
@ -114,12 +114,13 @@ public class LoadBalancerPerformanceEvaluation extends AbstractHBaseTool {
// servers
servers = new ArrayList<>(numServers);
serverRegionMap = new HashMap<>(numServers);
Map<ServerName, List<RegionInfo>> serverRegionMap = new HashMap<>(numServers);
for (int i = 0; i < numServers; ++i) {
ServerName sn = ServerName.valueOf("srv" + i, HConstants.DEFAULT_REGIONSERVER_PORT, i);
servers.add(sn);
serverRegionMap.put(sn, i == 0 ? regions : Collections.emptyList());
}
tableServerRegionMap = Collections.singletonMap(tableName, serverRegionMap);
}
@Override
@ -174,7 +175,7 @@ public class LoadBalancerPerformanceEvaluation extends AbstractHBaseTool {
LOG.info("Calling " + methodName);
watch.reset().start();
loadBalancer.balanceTable(HConstants.ENSEMBLE_TABLE_NAME, serverRegionMap);
loadBalancer.balanceCluster(tableServerRegionMap);
System.out.print(formatResults(methodName, watch.elapsed(TimeUnit.MILLISECONDS)));
return EXIT_SUCCESS;

7
hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestBaseLoadBalancer.java
View File

@ -114,14 +114,9 @@ public class TestBaseLoadBalancer extends BalancerTestBase {
}
public static class MockBalancer extends BaseLoadBalancer {
@Override
public List<RegionPlan>
balanceCluster(Map<TableName, Map<ServerName, List<RegionInfo>>> loadOfAllTable) {
return null;
}
@Override
public List<RegionPlan> balanceTable(TableName tableName,
protected List<RegionPlan> balanceTable(TableName tableName,
Map<ServerName, List<RegionInfo>> loadOfOneTable) {
return null;
}