Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-23944 The method setClusterLoad of SimpleLoadBalancer is incorrect when balance by table (#1243) 

Signed-off-by: Guanghao Zhang <zghao@apache.org>
This commit is contained in:
nyl3532016 2020-03-07 16:18:41 +08:00 committed by GitHub
parent 9b0d214b7b
commit 819965a35a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 29 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SimpleLoadBalancer.java
View File

@ -107,16 +107,21 @@ public class SimpleLoadBalancer extends BaseLoadBalancer {
} }
@Override @Override
public void setClusterLoad(Map<TableName, Map<ServerName, List<RegionInfo>>> clusterLoad){ public void setClusterLoad(Map<TableName, Map<ServerName, List<RegionInfo>>> clusterLoad) {
serverLoadList = new ArrayList<>(); serverLoadList = new ArrayList<>();
Map<ServerName, Integer> server2LoadMap = new HashMap<>();
float sum = 0; float sum = 0;
for(Map.Entry<TableName, Map<ServerName, List<RegionInfo>>> clusterEntry : clusterLoad.entrySet()){ for (Map.Entry<TableName, Map<ServerName, List<RegionInfo>>> clusterEntry : clusterLoad.entrySet()) {
for(Map.Entry<ServerName, List<RegionInfo>> entry : clusterEntry.getValue().entrySet()){ for (Map.Entry<ServerName, List<RegionInfo>> entry : clusterEntry.getValue().entrySet()) {
if(entry.getKey().equals(masterServerName)) continue; // we shouldn't include master as potential assignee if (entry.getKey().equals(masterServerName)) continue; // we shouldn't include master as potential assignee
serverLoadList.add(new ServerAndLoad(entry.getKey(), entry.getValue().size())); int regionNum = entry.getValue().size();
sum += entry.getValue().size(); server2LoadMap.compute(entry.getKey(), (k, v) -> v == null ? regionNum : regionNum + v);
sum += regionNum;
} }
} }
server2LoadMap.forEach((k, v) -> {
serverLoadList.add(new ServerAndLoad(k, v));
});
avgLoadOverall = sum / serverLoadList.size(); avgLoadOverall = sum / serverLoadList.size();
} }

23
hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestDefaultLoadBalancer.java
View File

@ -171,20 +171,33 @@ public class TestDefaultLoadBalancer extends BalancerTestBase {
*/ */
@Test @Test
public void testImpactOfBalanceClusterOverall() throws Exception { public void testImpactOfBalanceClusterOverall() throws Exception {
testImpactOfBalanceClusterOverall(false);
}
@Test
public void testImpactOfBalanceClusterOverallWithClusterLoadPerTable() throws Exception {
testImpactOfBalanceClusterOverall(true);
}
private void testImpactOfBalanceClusterOverall(boolean useClusterLoadPerTable) throws Exception {
Map<TableName, Map<ServerName, List<RegionInfo>>> clusterLoad = new TreeMap<>(); Map<TableName, Map<ServerName, List<RegionInfo>>> clusterLoad = new TreeMap<>();
Map<ServerName, List<RegionInfo>> clusterServers = mockUniformClusterServers(mockUniformCluster); Map<ServerName, List<RegionInfo>> clusterServers = mockUniformClusterServers(mockUniformCluster);
List<ServerAndLoad> clusterList = convertToList(clusterServers); List<ServerAndLoad> clusterList = convertToList(clusterServers);
clusterLoad.put(TableName.valueOf(name.getMethodName()), clusterServers); clusterLoad.put(TableName.valueOf(name.getMethodName()), clusterServers);
// use overall can achieve both table and cluster level balance // use overall can achieve both table and cluster level balance
HashMap<TableName, TreeMap<ServerName, List<RegionInfo>>> result1 = mockClusterServersWithTables(clusterServers); HashMap<TableName, TreeMap<ServerName, List<RegionInfo>>> clusterLoadPerTable = mockClusterServersWithTables(clusterServers);
loadBalancer.setClusterLoad(clusterLoad); if (useClusterLoadPerTable) {
loadBalancer.setClusterLoad((Map)clusterLoadPerTable);
} else {
loadBalancer.setClusterLoad(clusterLoad);
}
List<RegionPlan> clusterplans1 = new ArrayList<RegionPlan>(); List<RegionPlan> clusterplans1 = new ArrayList<RegionPlan>();
List<Pair<TableName, Integer>> regionAmountList = new ArrayList<Pair<TableName, Integer>>(); List<Pair<TableName, Integer>> regionAmountList = new ArrayList<Pair<TableName, Integer>>();
for(TreeMap<ServerName, List<RegionInfo>> servers : result1.values()){ for (TreeMap<ServerName, List<RegionInfo>> servers : clusterLoadPerTable.values()) {
List<ServerAndLoad> list = convertToList(servers); List<ServerAndLoad> list = convertToList(servers);
LOG.info("Mock Cluster : " + printMock(list) + " " + printStats(list)); LOG.info("Mock Cluster : " + printMock(list) + " " + printStats(list));
List<RegionPlan> partialplans = loadBalancer.balanceCluster(servers); List<RegionPlan> partialplans = loadBalancer.balanceCluster(servers);
if(partialplans != null) clusterplans1.addAll(partialplans); if (partialplans != null) clusterplans1.addAll(partialplans);
List<ServerAndLoad> balancedClusterPerTable = reconcile(list, partialplans, servers); List<ServerAndLoad> balancedClusterPerTable = reconcile(list, partialplans, servers);
LOG.info("Mock Balance : " + printMock(balancedClusterPerTable)); LOG.info("Mock Balance : " + printMock(balancedClusterPerTable));
assertClusterAsBalanced(balancedClusterPerTable); assertClusterAsBalanced(balancedClusterPerTable);
@ -194,6 +207,6 @@ public class TestDefaultLoadBalancer extends BalancerTestBase {
} }
} }
List<ServerAndLoad> balancedCluster1 = reconcile(clusterList, clusterplans1, clusterServers); List<ServerAndLoad> balancedCluster1 = reconcile(clusterList, clusterplans1, clusterServers);
assertTrue(assertClusterOverallAsBalanced(balancedCluster1, result1.keySet().size())); assertTrue(assertClusterOverallAsBalanced(balancedCluster1, clusterLoadPerTable.keySet().size()));
} }
} }