Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-20595 Remove the concept of 'special tables' from rsgroups

This commit is contained in:
Andrew Purtell 2018-05-21 13:53:01 -07:00
parent f27a6dc1d2
commit ba803167b3
3 changed files with 41 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java
View File

@ -33,7 +33,9 @@ import java.util.stream.Collectors;
import org.apache.hadoop.hbase.CoprocessorEnvironment; import org.apache.hadoop.hbase.CoprocessorEnvironment;
import org.apache.hadoop.hbase.HBaseIOException; import org.apache.hadoop.hbase.HBaseIOException;
import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.MasterNotRunningException;
import org.apache.hadoop.hbase.NamespaceDescriptor; import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.PleaseHoldException;
import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.RegionInfo;
@ -298,9 +300,9 @@ public class RSGroupAdminEndpoint implements MasterCoprocessor, MasterObserver {
+ hp); + hp);
try { try {
checkPermission("getRSGroupInfoOfServer"); checkPermission("getRSGroupInfoOfServer");
RSGroupInfo RSGroupInfo = groupAdminServer.getRSGroupOfServer(hp); RSGroupInfo info = groupAdminServer.getRSGroupOfServer(hp);
if (RSGroupInfo != null) { if (info != null) {
builder.setRSGroupInfo(RSGroupProtobufUtil.toProtoGroupInfo(RSGroupInfo)); builder.setRSGroupInfo(RSGroupProtobufUtil.toProtoGroupInfo(info));
} }
} catch (IOException e) { } catch (IOException e) {
CoprocessorRpcUtils.setControllerException(controller, e); CoprocessorRpcUtils.setControllerException(controller, e);
@ -354,12 +356,20 @@ public class RSGroupAdminEndpoint implements MasterCoprocessor, MasterObserver {
} }
boolean rsgroupHasServersOnline(TableDescriptor desc) throws IOException { boolean rsgroupHasServersOnline(TableDescriptor desc) throws IOException {
String groupName = String groupName;
try {
groupName =
master.getClusterSchema().getNamespace(desc.getTableName().getNamespaceAsString()) master.getClusterSchema().getNamespace(desc.getTableName().getNamespaceAsString())
.getConfigurationValue(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP); .getConfigurationValue(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP);
if (groupName == null) { if (groupName == null) {
groupName = RSGroupInfo.DEFAULT_GROUP; groupName = RSGroupInfo.DEFAULT_GROUP;
} }
} catch (MasterNotRunningException | PleaseHoldException e) {
LOG.info("Master has not initialized yet; temporarily using default RSGroup '" +
RSGroupInfo.DEFAULT_GROUP + "' for deploy of system table");
groupName = RSGroupInfo.DEFAULT_GROUP;
}
RSGroupInfo rsGroupInfo = groupAdminServer.getRSGroupInfo(groupName); RSGroupInfo rsGroupInfo = groupAdminServer.getRSGroupInfo(groupName);
if (rsGroupInfo == null) { if (rsGroupInfo == null) {
throw new ConstraintException( throw new ConstraintException(

31
hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java
View File

@ -196,6 +196,10 @@ public class RSGroupBasedLoadBalancer implements RSGroupableBalancer {
for (RegionInfo region : regions.keySet()) { for (RegionInfo region : regions.keySet()) {
if (!misplacedRegions.contains(region)) { if (!misplacedRegions.contains(region)) {
String groupName = rsGroupInfoManager.getRSGroupOfTable(region.getTable()); String groupName = rsGroupInfoManager.getRSGroupOfTable(region.getTable());
if (groupName == null) {
LOG.info("Group not found for table " + region.getTable() + ", using default");
groupName = RSGroupInfo.DEFAULT_GROUP;
}
groupToRegion.put(groupName, region); groupToRegion.put(groupName, region);
} }
} }
@ -217,6 +221,10 @@ public class RSGroupBasedLoadBalancer implements RSGroupableBalancer {
for (RegionInfo region : misplacedRegions) { for (RegionInfo region : misplacedRegions) {
String groupName = rsGroupInfoManager.getRSGroupOfTable(region.getTable()); String groupName = rsGroupInfoManager.getRSGroupOfTable(region.getTable());
if (groupName == null) {
LOG.info("Group not found for table " + region.getTable() + ", using default");
groupName = RSGroupInfo.DEFAULT_GROUP;
}
RSGroupInfo info = rsGroupInfoManager.getRSGroup(groupName); RSGroupInfo info = rsGroupInfoManager.getRSGroup(groupName);
List<ServerName> candidateList = filterOfflineServers(info, servers); List<ServerName> candidateList = filterOfflineServers(info, servers);
ServerName server = this.internalBalancer.randomAssignment(region, ServerName server = this.internalBalancer.randomAssignment(region,
@ -259,7 +267,8 @@ public class RSGroupBasedLoadBalancer implements RSGroupableBalancer {
for (RegionInfo region : regions) { for (RegionInfo region : regions) {
String groupName = rsGroupInfoManager.getRSGroupOfTable(region.getTable()); String groupName = rsGroupInfoManager.getRSGroupOfTable(region.getTable());
if (groupName == null) { if (groupName == null) {
LOG.warn("Group for table "+region.getTable()+" is null"); LOG.info("Group not found for table " + region.getTable() + ", using default");
groupName = RSGroupInfo.DEFAULT_GROUP;
} }
regionMap.put(groupName, region); regionMap.put(groupName, region);
} }
@ -318,8 +327,12 @@ public class RSGroupBasedLoadBalancer implements RSGroupableBalancer {
for(Map.Entry<RegionInfo, ServerName> region : regions.entrySet()) { for(Map.Entry<RegionInfo, ServerName> region : regions.entrySet()) {
RegionInfo regionInfo = region.getKey(); RegionInfo regionInfo = region.getKey();
ServerName assignedServer = region.getValue(); ServerName assignedServer = region.getValue();
RSGroupInfo info = rsGroupInfoManager.getRSGroup(rsGroupInfoManager. String groupName = rsGroupInfoManager.getRSGroupOfTable(regionInfo.getTable());
getRSGroupOfTable(regionInfo.getTable())); if (groupName == null) {
LOG.info("Group not found for table " + regionInfo.getTable() + ", using default");
groupName = RSGroupInfo.DEFAULT_GROUP;
}
RSGroupInfo info = rsGroupInfoManager.getRSGroup(groupName);
if (assignedServer == null) { if (assignedServer == null) {
LOG.debug("There is no assigned server for {}", region); LOG.debug("There is no assigned server for {}", region);
continue; continue;
@ -362,15 +375,19 @@ public class RSGroupBasedLoadBalancer implements RSGroupableBalancer {
correctAssignments.put(sName, new LinkedList<>()); correctAssignments.put(sName, new LinkedList<>());
List<RegionInfo> regions = assignments.getValue(); List<RegionInfo> regions = assignments.getValue();
for (RegionInfo region : regions) { for (RegionInfo region : regions) {
RSGroupInfo info = null; RSGroupInfo targetRSGInfo = null;
try { try {
info = rsGroupInfoManager.getRSGroup( String groupName = rsGroupInfoManager.getRSGroupOfTable(region.getTable());
rsGroupInfoManager.getRSGroupOfTable(region.getTable())); if (groupName == null) {
LOG.info("Group not found for table " + region.getTable() + ", using default");
groupName = RSGroupInfo.DEFAULT_GROUP;
}
targetRSGInfo = rsGroupInfoManager.getRSGroup(groupName);
} catch (IOException exp) { } catch (IOException exp) {
LOG.debug("RSGroup information null for region of table " + region.getTable(), LOG.debug("RSGroup information null for region of table " + region.getTable(),
exp); exp);
} }
if ((info == null) || (!info.containsServer(sName.getAddress()))) { if ((targetRSGInfo == null) || (!targetRSGInfo.containsServer(sName.getAddress()))) {
correctAssignments.get(LoadBalancer.BOGUS_SERVER_NAME).add(region); correctAssignments.get(LoadBalancer.BOGUS_SERVER_NAME).add(region);
} else { } else {
correctAssignments.get(sName).add(region); correctAssignments.get(sName).add(region);

11
hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java
View File

@ -23,7 +23,6 @@ import com.google.protobuf.ServiceException;
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
@ -73,7 +72,6 @@ import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos; import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos;
import org.apache.hadoop.hbase.protobuf.generated.RSGroupProtos; import org.apache.hadoop.hbase.protobuf.generated.RSGroupProtos;
import org.apache.hadoop.hbase.regionserver.DisabledRegionSplitPolicy; import org.apache.hadoop.hbase.regionserver.DisabledRegionSplitPolicy;
import org.apache.hadoop.hbase.security.access.AccessControlLists;
import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.zookeeper.ZKUtil; import org.apache.hadoop.hbase.zookeeper.ZKUtil;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher; import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
@ -246,7 +244,7 @@ final class RSGroupInfoManagerImpl implements RSGroupInfoManager {
public synchronized void moveTables(Set<TableName> tableNames, String groupName) public synchronized void moveTables(Set<TableName> tableNames, String groupName)
throws IOException { throws IOException {
if (groupName != null && !rsGroupMap.containsKey(groupName)) { if (groupName != null && !rsGroupMap.containsKey(groupName)) {
throw new DoNotRetryIOException("Group "+groupName+" does not exist or is a special group"); throw new DoNotRetryIOException("Group "+groupName+" does not exist");
} }
Map<String,RSGroupInfo> newGroupMap = Maps.newHashMap(rsGroupMap); Map<String,RSGroupInfo> newGroupMap = Maps.newHashMap(rsGroupMap);
@ -405,13 +403,6 @@ final class RSGroupInfoManagerImpl implements RSGroupInfoManager {
for(String entry: masterServices.getTableDescriptors().getAll().keySet()) { for(String entry: masterServices.getTableDescriptors().getAll().keySet()) {
orphanTables.add(TableName.valueOf(entry)); orphanTables.add(TableName.valueOf(entry));
} }
final List<TableName> specialTables = Arrays.asList(AccessControlLists.ACL_TABLE_NAME,
TableName.META_TABLE_NAME, TableName.NAMESPACE_TABLE_NAME, RSGROUP_TABLE_NAME);
for (TableName table : specialTables) {
orphanTables.add(table);
}
for (RSGroupInfo group: groupList) { for (RSGroupInfo group: groupList) {
if(!group.getName().equals(RSGroupInfo.DEFAULT_GROUP)) { if(!group.getName().equals(RSGroupInfo.DEFAULT_GROUP)) {
orphanTables.removeAll(group.getTables()); orphanTables.removeAll(group.getTables());