Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-20133 Calculate correct assignment and build region movement plans for mis-placed regions in one pass 

Signed-off-by: tedyu <yuzhihong@gmail.com>
This commit is contained in:
Xiang LI 2018-03-06 18:24:44 +08:00 committed by tedyu
parent 45bbee4905
commit 19a396b9c2
1 changed files with 28 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java
View File

@ -42,6 +42,7 @@ import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.master.RegionPlan; import org.apache.hadoop.hbase.master.RegionPlan;
import org.apache.hadoop.hbase.master.balancer.StochasticLoadBalancer; import org.apache.hadoop.hbase.master.balancer.StochasticLoadBalancer;
import org.apache.hadoop.hbase.net.Address; import org.apache.hadoop.hbase.net.Address;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.ReflectionUtils;
import org.apache.yetus.audience.InterfaceAudience; import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger; import org.slf4j.Logger;
@ -118,14 +119,15 @@ public class RSGroupBasedLoadBalancer implements RSGroupableBalancer {
" is not online, unable to perform balance"); " is not online, unable to perform balance");
} }
Map<ServerName,List<RegionInfo>> correctedState = correctAssignments(clusterState); // Calculate correct assignments and a list of RegionPlan for mis-placed regions
List<RegionPlan> regionPlans = new ArrayList<>(); Pair<Map<ServerName,List<RegionInfo>>, List<RegionPlan>> correctedStateAndRegionPlans =
correctAssignments(clusterState);
Map<ServerName,List<RegionInfo>> correctedState = correctedStateAndRegionPlans.getFirst();
List<RegionPlan> regionPlans = correctedStateAndRegionPlans.getSecond();
List<RegionInfo> misplacedRegions = correctedState.get(LoadBalancer.BOGUS_SERVER_NAME); // Add RegionPlan
for (RegionInfo regionInfo : misplacedRegions) { // for the regions which have been placed according to the region server group assignment
ServerName serverName = findServerForRegion(clusterState, regionInfo); // into the movement list
regionPlans.add(new RegionPlan(regionInfo, serverName, null));
}
try { try {
List<RSGroupInfo> rsgi = rsGroupInfoManager.listRSGroups(); List<RSGroupInfo> rsgi = rsGroupInfoManager.listRSGroups();
for (RSGroupInfo info: rsgi) { for (RSGroupInfo info: rsgi) {
@ -150,6 +152,8 @@ public class RSGroupBasedLoadBalancer implements RSGroupableBalancer {
LOG.warn("Exception while balancing cluster.", exp); LOG.warn("Exception while balancing cluster.", exp);
regionPlans.clear(); regionPlans.clear();
} }
// Return the whole movement list
return regionPlans; return regionPlans;
} }
@ -334,44 +338,38 @@ public class RSGroupBasedLoadBalancer implements RSGroupableBalancer {
return misplacedRegions; return misplacedRegions;
} }
private ServerName findServerForRegion( private Pair<Map<ServerName, List<RegionInfo>>, List<RegionPlan>> correctAssignments(
Map<ServerName, List<RegionInfo>> existingAssignments, RegionInfo region) {
for (Map.Entry<ServerName, List<RegionInfo>> entry : existingAssignments.entrySet()) {
if (entry.getValue().contains(region)) {
return entry.getKey();
}
}
throw new IllegalStateException("Could not find server for region "
+ region.getShortNameToLog());
}
private Map<ServerName, List<RegionInfo>> correctAssignments(
Map<ServerName, List<RegionInfo>> existingAssignments) Map<ServerName, List<RegionInfo>> existingAssignments)
throws HBaseIOException{ throws HBaseIOException{
// To return
Map<ServerName, List<RegionInfo>> correctAssignments = new TreeMap<>(); Map<ServerName, List<RegionInfo>> correctAssignments = new TreeMap<>();
correctAssignments.put(LoadBalancer.BOGUS_SERVER_NAME, new LinkedList<>()); List<RegionPlan> regionPlansForMisplacedRegions = new ArrayList<>();
for (Map.Entry<ServerName, List<RegionInfo>> assignments : existingAssignments.entrySet()){ for (Map.Entry<ServerName, List<RegionInfo>> assignments : existingAssignments.entrySet()){
ServerName sName = assignments.getKey(); ServerName currentHostServer = assignments.getKey();
correctAssignments.put(sName, new LinkedList<>()); correctAssignments.put(currentHostServer, new LinkedList<>());
List<RegionInfo> regions = assignments.getValue(); List<RegionInfo> regions = assignments.getValue();
for (RegionInfo region : regions) { for (RegionInfo region : regions) {
RSGroupInfo info = null; RSGroupInfo targetRSGInfo = null;
try { try {
info = rsGroupInfoManager.getRSGroup( targetRSGInfo = rsGroupInfoManager.getRSGroup(
rsGroupInfoManager.getRSGroupOfTable(region.getTable())); rsGroupInfoManager.getRSGroupOfTable(region.getTable()));
} catch (IOException exp) { } catch (IOException exp) {
LOG.debug("RSGroup information null for region of table " + region.getTable(), LOG.debug("RSGroup information null for region of table " + region.getTable(),
exp); exp);
} }
if ((info == null) || (!info.containsServer(sName.getAddress()))) { if (targetRSGInfo == null ||
correctAssignments.get(LoadBalancer.BOGUS_SERVER_NAME).add(region); !targetRSGInfo.containsServer(currentHostServer.getAddress())) { // region is mis-placed
} else { regionPlansForMisplacedRegions.add(new RegionPlan(region, currentHostServer, null));
correctAssignments.get(sName).add(region); } else { // region is placed as expected
correctAssignments.get(currentHostServer).add(region);
} }
} }
} }
return correctAssignments;
// Return correct assignments and region movement plan for mis-placed regions together
return new Pair<Map<ServerName, List<RegionInfo>>, List<RegionPlan>>(
correctAssignments, regionPlansForMisplacedRegions);
} }
@Override @Override