From ba803167b31806c70aa9cca792b9a019eab7d51f Mon Sep 17 00:00:00 2001 From: Andrew Purtell Date: Mon, 21 May 2018 13:53:01 -0700 Subject: [PATCH] HBASE-20595 Remove the concept of 'special tables' from rsgroups --- .../hbase/rsgroup/RSGroupAdminEndpoint.java | 22 +++++++++---- .../rsgroup/RSGroupBasedLoadBalancer.java | 31 ++++++++++++++----- .../hbase/rsgroup/RSGroupInfoManagerImpl.java | 11 +------ 3 files changed, 41 insertions(+), 23 deletions(-) diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java index 0006d68d409..9f0a4515919 100644 --- a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java +++ b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java @@ -33,7 +33,9 @@ import java.util.stream.Collectors; import org.apache.hadoop.hbase.CoprocessorEnvironment; import org.apache.hadoop.hbase.HBaseIOException; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.MasterNotRunningException; import org.apache.hadoop.hbase.NamespaceDescriptor; +import org.apache.hadoop.hbase.PleaseHoldException; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.RegionInfo; @@ -298,9 +300,9 @@ public class RSGroupAdminEndpoint implements MasterCoprocessor, MasterObserver { + hp); try { checkPermission("getRSGroupInfoOfServer"); - RSGroupInfo RSGroupInfo = groupAdminServer.getRSGroupOfServer(hp); - if (RSGroupInfo != null) { - builder.setRSGroupInfo(RSGroupProtobufUtil.toProtoGroupInfo(RSGroupInfo)); + RSGroupInfo info = groupAdminServer.getRSGroupOfServer(hp); + if (info != null) { + builder.setRSGroupInfo(RSGroupProtobufUtil.toProtoGroupInfo(info)); } } catch (IOException e) { CoprocessorRpcUtils.setControllerException(controller, e); @@ -354,12 +356,20 @@ public class RSGroupAdminEndpoint implements MasterCoprocessor, MasterObserver { } boolean rsgroupHasServersOnline(TableDescriptor desc) throws IOException { - String groupName = + String groupName; + try { + groupName = master.getClusterSchema().getNamespace(desc.getTableName().getNamespaceAsString()) - .getConfigurationValue(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP); - if (groupName == null) { + .getConfigurationValue(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP); + if (groupName == null) { + groupName = RSGroupInfo.DEFAULT_GROUP; + } + } catch (MasterNotRunningException | PleaseHoldException e) { + LOG.info("Master has not initialized yet; temporarily using default RSGroup '" + + RSGroupInfo.DEFAULT_GROUP + "' for deploy of system table"); groupName = RSGroupInfo.DEFAULT_GROUP; } + RSGroupInfo rsGroupInfo = groupAdminServer.getRSGroupInfo(groupName); if (rsGroupInfo == null) { throw new ConstraintException( diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java index 3182a61a16c..83286a95755 100644 --- a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java +++ b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java @@ -196,6 +196,10 @@ public class RSGroupBasedLoadBalancer implements RSGroupableBalancer { for (RegionInfo region : regions.keySet()) { if (!misplacedRegions.contains(region)) { String groupName = rsGroupInfoManager.getRSGroupOfTable(region.getTable()); + if (groupName == null) { + LOG.info("Group not found for table " + region.getTable() + ", using default"); + groupName = RSGroupInfo.DEFAULT_GROUP; + } groupToRegion.put(groupName, region); } } @@ -217,6 +221,10 @@ public class RSGroupBasedLoadBalancer implements RSGroupableBalancer { for (RegionInfo region : misplacedRegions) { String groupName = rsGroupInfoManager.getRSGroupOfTable(region.getTable()); + if (groupName == null) { + LOG.info("Group not found for table " + region.getTable() + ", using default"); + groupName = RSGroupInfo.DEFAULT_GROUP; + } RSGroupInfo info = rsGroupInfoManager.getRSGroup(groupName); List candidateList = filterOfflineServers(info, servers); ServerName server = this.internalBalancer.randomAssignment(region, @@ -259,7 +267,8 @@ public class RSGroupBasedLoadBalancer implements RSGroupableBalancer { for (RegionInfo region : regions) { String groupName = rsGroupInfoManager.getRSGroupOfTable(region.getTable()); if (groupName == null) { - LOG.warn("Group for table "+region.getTable()+" is null"); + LOG.info("Group not found for table " + region.getTable() + ", using default"); + groupName = RSGroupInfo.DEFAULT_GROUP; } regionMap.put(groupName, region); } @@ -318,8 +327,12 @@ public class RSGroupBasedLoadBalancer implements RSGroupableBalancer { for(Map.Entry region : regions.entrySet()) { RegionInfo regionInfo = region.getKey(); ServerName assignedServer = region.getValue(); - RSGroupInfo info = rsGroupInfoManager.getRSGroup(rsGroupInfoManager. - getRSGroupOfTable(regionInfo.getTable())); + String groupName = rsGroupInfoManager.getRSGroupOfTable(regionInfo.getTable()); + if (groupName == null) { + LOG.info("Group not found for table " + regionInfo.getTable() + ", using default"); + groupName = RSGroupInfo.DEFAULT_GROUP; + } + RSGroupInfo info = rsGroupInfoManager.getRSGroup(groupName); if (assignedServer == null) { LOG.debug("There is no assigned server for {}", region); continue; @@ -362,15 +375,19 @@ public class RSGroupBasedLoadBalancer implements RSGroupableBalancer { correctAssignments.put(sName, new LinkedList<>()); List regions = assignments.getValue(); for (RegionInfo region : regions) { - RSGroupInfo info = null; + RSGroupInfo targetRSGInfo = null; try { - info = rsGroupInfoManager.getRSGroup( - rsGroupInfoManager.getRSGroupOfTable(region.getTable())); + String groupName = rsGroupInfoManager.getRSGroupOfTable(region.getTable()); + if (groupName == null) { + LOG.info("Group not found for table " + region.getTable() + ", using default"); + groupName = RSGroupInfo.DEFAULT_GROUP; + } + targetRSGInfo = rsGroupInfoManager.getRSGroup(groupName); } catch (IOException exp) { LOG.debug("RSGroup information null for region of table " + region.getTable(), exp); } - if ((info == null) || (!info.containsServer(sName.getAddress()))) { + if ((targetRSGInfo == null) || (!targetRSGInfo.containsServer(sName.getAddress()))) { correctAssignments.get(LoadBalancer.BOGUS_SERVER_NAME).add(region); } else { correctAssignments.get(sName).add(region); diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java index d6618bf9ba4..9478401b9d9 100644 --- a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java +++ b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java @@ -23,7 +23,6 @@ import com.google.protobuf.ServiceException; import java.io.ByteArrayInputStream; import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -73,7 +72,6 @@ import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos; import org.apache.hadoop.hbase.protobuf.generated.RSGroupProtos; import org.apache.hadoop.hbase.regionserver.DisabledRegionSplitPolicy; -import org.apache.hadoop.hbase.security.access.AccessControlLists; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.zookeeper.ZKUtil; import org.apache.hadoop.hbase.zookeeper.ZKWatcher; @@ -246,7 +244,7 @@ final class RSGroupInfoManagerImpl implements RSGroupInfoManager { public synchronized void moveTables(Set tableNames, String groupName) throws IOException { if (groupName != null && !rsGroupMap.containsKey(groupName)) { - throw new DoNotRetryIOException("Group "+groupName+" does not exist or is a special group"); + throw new DoNotRetryIOException("Group "+groupName+" does not exist"); } Map newGroupMap = Maps.newHashMap(rsGroupMap); @@ -405,13 +403,6 @@ final class RSGroupInfoManagerImpl implements RSGroupInfoManager { for(String entry: masterServices.getTableDescriptors().getAll().keySet()) { orphanTables.add(TableName.valueOf(entry)); } - - final List specialTables = Arrays.asList(AccessControlLists.ACL_TABLE_NAME, - TableName.META_TABLE_NAME, TableName.NAMESPACE_TABLE_NAME, RSGROUP_TABLE_NAME); - - for (TableName table : specialTables) { - orphanTables.add(table); - } for (RSGroupInfo group: groupList) { if(!group.getName().equals(RSGroupInfo.DEFAULT_GROUP)) { orphanTables.removeAll(group.getTables());