Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-20595 Remove the concept of 'special tables' from rsgroups

This commit is contained in:
Andrew Purtell 2018-05-21 12:20:05 -07:00
parent 45d928e2c6
commit 498f3bf953
3 changed files with 52 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java
View File

@ -31,6 +31,8 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.ClusterStatus;
import org.apache.hadoop.hbase.Coprocessor;
import org.apache.hadoop.hbase.CoprocessorEnvironment;
@ -38,8 +40,10 @@ import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.MasterNotRunningException;
import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.ProcedureInfo;
import org.apache.hadoop.hbase.PleaseHoldException;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin.MasterSwitchType;
@ -94,6 +98,8 @@ import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
public class RSGroupAdminEndpoint extends RSGroupAdminService
implements CoprocessorService, Coprocessor, MasterObserver {
private static final Log LOG = LogFactory.getLog(RSGroupAdminEndpoint.class);
private MasterServices master = null;
private static RSGroupInfoManagerImpl groupInfoManager;
@ -179,11 +185,11 @@ public class RSGroupAdminEndpoint extends RSGroupAdminService
GetRSGroupInfoOfTableResponse.newBuilder();
TableName tableName = ProtobufUtil.toTableName(request.getTableName());
checkPermission("getRSGroupInfoOfTable");
RSGroupInfo RSGroupInfo = groupAdminServer.getRSGroupInfoOfTable(tableName);
if (RSGroupInfo == null) {
RSGroupInfo info = groupAdminServer.getRSGroupInfoOfTable(tableName);
if (info == null) {
response = builder.build();
} else {
response = builder.setRSGroupInfo(RSGroupProtobufUtil.toProtoGroupInfo(RSGroupInfo)).build();
response = builder.setRSGroupInfo(RSGroupProtobufUtil.toProtoGroupInfo(info)).build();
}
} catch (IOException e) {
ResponseConverter.setControllerException(controller, e);
@ -361,10 +367,17 @@ public class RSGroupAdminEndpoint extends RSGroupAdminService
}
void assignTableToGroup(HTableDescriptor desc) throws IOException {
String groupName =
String groupName;
try {
groupName =
master.getNamespaceDescriptor(desc.getTableName().getNamespaceAsString())
.getConfigurationValue(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP);
if (groupName == null) {
if (groupName == null) {
groupName = RSGroupInfo.DEFAULT_GROUP;
}
} catch (MasterNotRunningException | PleaseHoldException e) {
LOG.info("Master has not initialized yet; temporarily using default RSGroup '" +
RSGroupInfo.DEFAULT_GROUP + "' for deploy of system table");
groupName = RSGroupInfo.DEFAULT_GROUP;
}
RSGroupInfo rsGroupInfo = groupAdminServer.getRSGroupInfo(groupName);

32
hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java
View File

@ -200,6 +200,10 @@ public class RSGroupBasedLoadBalancer implements RSGroupableBalancer, LoadBalanc
for (HRegionInfo region : regions.keySet()) {
if (!misplacedRegions.contains(region)) {
String groupName = infoManager.getRSGroupOfTable(region.getTable());
if (groupName == null) {
LOG.info("Group not found for table " + region.getTable() + ", using default");
groupName = RSGroupInfo.DEFAULT_GROUP;
}
groupToRegion.put(groupName, region);
}
}
@ -220,8 +224,11 @@ public class RSGroupBasedLoadBalancer implements RSGroupableBalancer, LoadBalanc
}
for (HRegionInfo region : misplacedRegions) {
String groupName = infoManager.getRSGroupOfTable(
region.getTable());
String groupName = infoManager.getRSGroupOfTable(region.getTable());
if (groupName == null) {
LOG.info("Group not found for table " + region.getTable() + ", using default");
groupName = RSGroupInfo.DEFAULT_GROUP;
}
RSGroupInfo info = infoManager.getRSGroup(groupName);
List<ServerName> candidateList = filterOfflineServers(info, servers);
ServerName server = this.internalBalancer.randomAssignment(region,
@ -269,8 +276,9 @@ public class RSGroupBasedLoadBalancer implements RSGroupableBalancer, LoadBalanc
try {
for (HRegionInfo region : regions) {
String groupName = infoManager.getRSGroupOfTable(region.getTable());
if(groupName == null) {
LOG.warn("Group for table "+region.getTable()+" is null");
if (groupName == null) {
LOG.info("Group not found for table " + region.getTable() + ", using default");
groupName = RSGroupInfo.DEFAULT_GROUP;
}
regionMap.put(groupName, region);
}
@ -330,8 +338,12 @@ public class RSGroupBasedLoadBalancer implements RSGroupableBalancer, LoadBalanc
for(Map.Entry<HRegionInfo, ServerName> region : regions.entrySet()) {
HRegionInfo regionInfo = region.getKey();
ServerName assignedServer = region.getValue();
RSGroupInfo info =
infoManager.getRSGroup(infoManager.getRSGroupOfTable(regionInfo.getTable()));
String groupName = infoManager.getRSGroupOfTable(regionInfo.getTable());
if (groupName == null) {
LOG.info("Group not found for table " + regionInfo.getTable() + ", using default");
groupName = RSGroupInfo.DEFAULT_GROUP;
}
RSGroupInfo info = infoManager.getRSGroup(groupName);
if (assignedServer == null) {
LOG.debug("There is no assigned server for " + region);
continue;
@ -364,8 +376,12 @@ public class RSGroupBasedLoadBalancer implements RSGroupableBalancer, LoadBalanc
for (HRegionInfo region : regions) {
RSGroupInfo info = null;
try {
info = infoManager.getRSGroup(
infoManager.getRSGroupOfTable(region.getTable()));
String groupName = infoManager.getRSGroupOfTable(region.getTable());
if (groupName == null) {
LOG.info("Group not found for table " + region.getTable() + ", using default");
groupName = RSGroupInfo.DEFAULT_GROUP;
}
info = infoManager.getRSGroup(groupName);
} catch (IOException exp) {
LOG.debug("Group information null for region of table " + region.getTable(),
exp);

26
hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java
View File

@ -27,7 +27,6 @@ import com.google.common.collect.Sets;
import com.google.protobuf.ServiceException;
import java.io.IOException;
import java.util.Arrays;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
@ -78,7 +77,6 @@ import org.apache.hadoop.hbase.protobuf.generated.RSGroupProtos;
import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos.MutateRowsRequest;
import org.apache.hadoop.hbase.regionserver.DisabledRegionSplitPolicy;
import org.apache.hadoop.hbase.security.access.AccessControlLists;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.ModifyRegionUtils;
import org.apache.hadoop.hbase.util.Threads;
@ -223,12 +221,15 @@ public class RSGroupInfoManagerImpl implements RSGroupInfoManager, ServerListene
*/
@Override
public RSGroupInfo getRSGroup(String groupName) throws IOException {
RSGroupInfo RSGroupInfo = rsGroupMap.get(groupName);
return RSGroupInfo;
return rsGroupMap.get(groupName);
}
/**
* Gets the group name for a given table
*
* @param tableName the table name
* @return group name, or null if not found
*/
@Override
public String getRSGroupOfTable(TableName tableName) throws IOException {
return tableMap.get(tableName);
@ -238,7 +239,7 @@ public class RSGroupInfoManagerImpl implements RSGroupInfoManager, ServerListene
public synchronized void moveTables(
Set<TableName> tableNames, String groupName) throws IOException {
if (groupName != null && !rsGroupMap.containsKey(groupName)) {
throw new DoNotRetryIOException("Group "+groupName+" does not exist or is a special group");
throw new DoNotRetryIOException("Group "+groupName+" does not exist");
}
Map<String,RSGroupInfo> newGroupMap = Maps.newHashMap(rsGroupMap);
@ -302,7 +303,7 @@ public class RSGroupInfoManagerImpl implements RSGroupInfoManager, ServerListene
groupList.addAll(rsGroupSerDe.retrieveGroupList(rsGroupTable));
}
} else {
LOG.debug("Refershing in Offline mode.");
LOG.debug("Refreshing in Offline mode.");
String groupBasePath = ZKUtil.joinZNode(watcher.baseZNode, rsGroupZNode);
groupList.addAll(rsGroupSerDe.retrieveGroupList(watcher, groupBasePath));
}
@ -313,13 +314,7 @@ public class RSGroupInfoManagerImpl implements RSGroupInfoManager, ServerListene
orphanTables.add(TableName.valueOf(entry));
}
final List<TableName> specialTables = Arrays.asList(AccessControlLists.ACL_TABLE_NAME,
TableName.META_TABLE_NAME, TableName.NAMESPACE_TABLE_NAME, RSGROUP_TABLE_NAME);
for(TableName table : specialTables) {
orphanTables.add(table);
}
for(RSGroupInfo group: groupList) {
for (RSGroupInfo group: groupList) {
if(!group.getName().equals(RSGroupInfo.DEFAULT_GROUP)) {
orphanTables.removeAll(group.getTables());
}
@ -332,7 +327,6 @@ public class RSGroupInfoManagerImpl implements RSGroupInfoManager, ServerListene
Sets.newHashSet(getDefaultServers()),
orphanTables));
// populate the data
HashMap<String, RSGroupInfo> newGroupMap = Maps.newHashMap();
HashMap<TableName, String> newTableMap = Maps.newHashMap();