From 304d3e6fa9f44cb25ab03cb6d949f1c9b74779d1 Mon Sep 17 00:00:00 2001 From: Andrew Purtell Date: Mon, 21 May 2018 13:53:01 -0700 Subject: [PATCH] HBASE-20595 Remove the concept of 'special tables' from rsgroups --- .../hbase/rsgroup/RSGroupAdminEndpoint.java | 20 ++++++++++---- .../rsgroup/RSGroupBasedLoadBalancer.java | 27 +++++++++++++++---- .../hbase/rsgroup/RSGroupInfoManagerImpl.java | 18 +------------ 3 files changed, 38 insertions(+), 27 deletions(-) diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java index 624aa446abb..6f3c1d1f15f 100644 --- a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java +++ b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java @@ -33,7 +33,9 @@ import java.util.stream.Collectors; import org.apache.hadoop.hbase.CoprocessorEnvironment; import org.apache.hadoop.hbase.HBaseIOException; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.MasterNotRunningException; import org.apache.hadoop.hbase.NamespaceDescriptor; +import org.apache.hadoop.hbase.PleaseHoldException; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.RegionInfo; @@ -298,9 +300,9 @@ public class RSGroupAdminEndpoint implements MasterCoprocessor, MasterObserver { + hp); try { checkPermission("getRSGroupInfoOfServer"); - RSGroupInfo RSGroupInfo = groupAdminServer.getRSGroupOfServer(hp); - if (RSGroupInfo != null) { - builder.setRSGroupInfo(RSGroupProtobufUtil.toProtoGroupInfo(RSGroupInfo)); + RSGroupInfo info = groupAdminServer.getRSGroupOfServer(hp); + if (info != null) { + builder.setRSGroupInfo(RSGroupProtobufUtil.toProtoGroupInfo(info)); } } catch (IOException e) { CoprocessorRpcUtils.setControllerException(controller, e); @@ -354,12 +356,20 @@ public class RSGroupAdminEndpoint implements MasterCoprocessor, MasterObserver { } boolean rsgroupHasServersOnline(TableDescriptor desc) throws IOException { - String groupName = + String groupName; + try { + groupName = master.getClusterSchema().getNamespace(desc.getTableName().getNamespaceAsString()) .getConfigurationValue(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP); - if (groupName == null) { + if (groupName == null) { + groupName = RSGroupInfo.DEFAULT_GROUP; + } + } catch (MasterNotRunningException | PleaseHoldException e) { + LOG.info("Master has not initialized yet; temporarily using default RSGroup '" + + RSGroupInfo.DEFAULT_GROUP + "' for deploy of system table"); groupName = RSGroupInfo.DEFAULT_GROUP; } + RSGroupInfo rsGroupInfo = groupAdminServer.getRSGroupInfo(groupName); if (rsGroupInfo == null) { throw new ConstraintException( diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java index 9eb048fbf52..69131f98548 100644 --- a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java +++ b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java @@ -200,6 +200,10 @@ public class RSGroupBasedLoadBalancer implements RSGroupableBalancer { for (RegionInfo region : regions.keySet()) { if (!misplacedRegions.contains(region)) { String groupName = rsGroupInfoManager.getRSGroupOfTable(region.getTable()); + if (groupName == null) { + LOG.info("Group not found for table " + region.getTable() + ", using default"); + groupName = RSGroupInfo.DEFAULT_GROUP; + } groupToRegion.put(groupName, region); } } @@ -221,6 +225,10 @@ public class RSGroupBasedLoadBalancer implements RSGroupableBalancer { for (RegionInfo region : misplacedRegions) { String groupName = rsGroupInfoManager.getRSGroupOfTable(region.getTable()); + if (groupName == null) { + LOG.info("Group not found for table " + region.getTable() + ", using default"); + groupName = RSGroupInfo.DEFAULT_GROUP; + } RSGroupInfo info = rsGroupInfoManager.getRSGroup(groupName); List candidateList = filterOfflineServers(info, servers); ServerName server = this.internalBalancer.randomAssignment(region, @@ -263,7 +271,8 @@ public class RSGroupBasedLoadBalancer implements RSGroupableBalancer { for (RegionInfo region : regions) { String groupName = rsGroupInfoManager.getRSGroupOfTable(region.getTable()); if (groupName == null) { - LOG.warn("Group for table "+region.getTable()+" is null"); + LOG.info("Group not found for table " + region.getTable() + ", using default"); + groupName = RSGroupInfo.DEFAULT_GROUP; } regionMap.put(groupName, region); } @@ -322,8 +331,12 @@ public class RSGroupBasedLoadBalancer implements RSGroupableBalancer { for(Map.Entry region : regions.entrySet()) { RegionInfo regionInfo = region.getKey(); ServerName assignedServer = region.getValue(); - RSGroupInfo info = rsGroupInfoManager.getRSGroup(rsGroupInfoManager. - getRSGroupOfTable(regionInfo.getTable())); + String groupName = rsGroupInfoManager.getRSGroupOfTable(regionInfo.getTable()); + if (groupName == null) { + LOG.info("Group not found for table " + regionInfo.getTable() + ", using default"); + groupName = RSGroupInfo.DEFAULT_GROUP; + } + RSGroupInfo info = rsGroupInfoManager.getRSGroup(groupName); if (assignedServer == null) { LOG.debug("There is no assigned server for {}", region); continue; @@ -358,8 +371,12 @@ public class RSGroupBasedLoadBalancer implements RSGroupableBalancer { for (RegionInfo region : regions) { RSGroupInfo targetRSGInfo = null; try { - targetRSGInfo = rsGroupInfoManager.getRSGroup( - rsGroupInfoManager.getRSGroupOfTable(region.getTable())); + String groupName = rsGroupInfoManager.getRSGroupOfTable(region.getTable()); + if (groupName == null) { + LOG.info("Group not found for table " + region.getTable() + ", using default"); + groupName = RSGroupInfo.DEFAULT_GROUP; + } + targetRSGInfo = rsGroupInfoManager.getRSGroup(groupName); } catch (IOException exp) { LOG.debug("RSGroup information null for region of table " + region.getTable(), exp); diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java index 4ae92a615c3..6fe7e39079b 100644 --- a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java +++ b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java @@ -23,7 +23,6 @@ import com.google.protobuf.ServiceException; import java.io.ByteArrayInputStream; import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -72,10 +71,7 @@ import org.apache.hadoop.hbase.protobuf.ProtobufMagic; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos; import org.apache.hadoop.hbase.protobuf.generated.RSGroupProtos; -import org.apache.hadoop.hbase.quotas.QuotaTableUtil; -import org.apache.hadoop.hbase.quotas.QuotaUtil; import org.apache.hadoop.hbase.regionserver.DisabledRegionSplitPolicy; -import org.apache.hadoop.hbase.security.access.AccessControlLists; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.zookeeper.ZKUtil; import org.apache.hadoop.hbase.zookeeper.ZKWatcher; @@ -249,7 +245,7 @@ final class RSGroupInfoManagerImpl implements RSGroupInfoManager { public synchronized void moveTables(Set tableNames, String groupName) throws IOException { if (groupName != null && !rsGroupMap.containsKey(groupName)) { - throw new DoNotRetryIOException("Group "+groupName+" does not exist or is a special group"); + throw new DoNotRetryIOException("Group "+groupName+" does not exist"); } Map newGroupMap = Maps.newHashMap(rsGroupMap); @@ -408,18 +404,6 @@ final class RSGroupInfoManagerImpl implements RSGroupInfoManager { for(String entry: masterServices.getTableDescriptors().getAll().keySet()) { orphanTables.add(TableName.valueOf(entry)); } - - List specialTables = - new ArrayList(Arrays.asList(AccessControlLists.ACL_TABLE_NAME, - TableName.META_TABLE_NAME, TableName.NAMESPACE_TABLE_NAME, RSGROUP_TABLE_NAME)); - // if quota is enabled, add corresponding system table to special tables list - if (QuotaUtil.isQuotaEnabled(conn.getConfiguration())) { - specialTables.add(QuotaTableUtil.QUOTA_TABLE_NAME); - } - - for (TableName table : specialTables) { - orphanTables.add(table); - } for (RSGroupInfo group: groupList) { if(!group.getName().equals(RSGroupInfo.DEFAULT_GROUP)) { orphanTables.removeAll(group.getTables());