Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-22820 Do not need to persist default rs group now (#482) 

Signed-off-by: Duo Zhang <zhangduo@apache.org>
This commit is contained in:
linkaline 2019-08-16 16:52:41 +08:00 committed by Duo Zhang
parent 44ddb30914
commit 2e0039c2a3
2 changed files with 49 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java
View File

@ -396,12 +396,14 @@ final class RSGroupInfoManagerImpl implements RSGroupInfoManager {
} }
// populate puts // populate puts
for (RSGroupInfo RSGroupInfo : groupMap.values()) { for (RSGroupInfo gi : groupMap.values()) {
RSGroupProtos.RSGroupInfo proto = ProtobufUtil.toProtoGroupInfo(RSGroupInfo); if (!gi.getName().equals(RSGroupInfo.DEFAULT_GROUP)) {
Put p = new Put(Bytes.toBytes(RSGroupInfo.getName())); RSGroupProtos.RSGroupInfo proto = ProtobufUtil.toProtoGroupInfo(gi);
Put p = new Put(Bytes.toBytes(gi.getName()));
p.addColumn(META_FAMILY_BYTES, META_QUALIFIER_BYTES, proto.toByteArray()); p.addColumn(META_FAMILY_BYTES, META_QUALIFIER_BYTES, proto.toByteArray());
mutations.add(p); mutations.add(p);
} }
}
if (mutations.size() > 0) { if (mutations.size() > 0) {
multiMutate(mutations); multiMutate(mutations);
@ -449,7 +451,12 @@ final class RSGroupInfoManagerImpl implements RSGroupInfoManager {
// Make changes visible after having been persisted to the source of truth // Make changes visible after having been persisted to the source of truth
resetRSGroupMap(newGroupMap); resetRSGroupMap(newGroupMap);
saveRSGroupMapToZK(newGroupMap);
updateCacheOfRSGroups(newGroupMap.keySet());
}
private void saveRSGroupMapToZK(Map<String, RSGroupInfo> newGroupMap) throws IOException {
try { try {
String groupBasePath = String groupBasePath =
ZNodePaths.joinZNode(watcher.getZNodePaths().baseZNode, RS_GROUP_ZNODE); ZNodePaths.joinZNode(watcher.getZNodePaths().baseZNode, RS_GROUP_ZNODE);
@ -463,15 +470,17 @@ final class RSGroupInfoManagerImpl implements RSGroupInfoManager {
} }
} }
for (RSGroupInfo RSGroupInfo : newGroupMap.values()) { for (RSGroupInfo gi : newGroupMap.values()) {
String znode = ZNodePaths.joinZNode(groupBasePath, RSGroupInfo.getName()); if (!gi.getName().equals(RSGroupInfo.DEFAULT_GROUP)) {
RSGroupProtos.RSGroupInfo proto = ProtobufUtil.toProtoGroupInfo(RSGroupInfo); String znode = ZNodePaths.joinZNode(groupBasePath, gi.getName());
RSGroupProtos.RSGroupInfo proto = ProtobufUtil.toProtoGroupInfo(gi);
LOG.debug("Updating znode: " + znode); LOG.debug("Updating znode: " + znode);
ZKUtil.createAndFailSilent(watcher, znode); ZKUtil.createAndFailSilent(watcher, znode);
zkOps.add(ZKUtil.ZKUtilOp.deleteNodeFailSilent(znode)); zkOps.add(ZKUtil.ZKUtilOp.deleteNodeFailSilent(znode));
zkOps.add(ZKUtil.ZKUtilOp.createAndFailSilent(znode, zkOps.add(ZKUtil.ZKUtilOp.createAndFailSilent(znode,
ProtobufUtil.prependPBMagic(proto.toByteArray()))); ProtobufUtil.prependPBMagic(proto.toByteArray())));
} }
}
LOG.debug("Writing ZK GroupInfo count: " + zkOps.size()); LOG.debug("Writing ZK GroupInfo count: " + zkOps.size());
ZKUtil.multiOrSequential(watcher, zkOps, false); ZKUtil.multiOrSequential(watcher, zkOps, false);
@ -480,7 +489,6 @@ final class RSGroupInfoManagerImpl implements RSGroupInfoManager {
masterServices.abort("Failed to write to rsGroupZNode", e); masterServices.abort("Failed to write to rsGroupZNode", e);
throw new IOException("Failed to write to rsGroupZNode", e); throw new IOException("Failed to write to rsGroupZNode", e);
} }
updateCacheOfRSGroups(newGroupMap.keySet());
} }
/** /**
@ -540,12 +548,12 @@ final class RSGroupInfoManagerImpl implements RSGroupInfoManager {
// Called by ServerEventsListenerThread. Synchronize on this because redoing // Called by ServerEventsListenerThread. Synchronize on this because redoing
// the rsGroupMap then writing it out. // the rsGroupMap then writing it out.
private synchronized void updateDefaultServers(SortedSet<Address> servers) throws IOException { private synchronized void updateDefaultServers(SortedSet<Address> servers) {
RSGroupInfo info = rsGroupMap.get(RSGroupInfo.DEFAULT_GROUP); RSGroupInfo info = rsGroupMap.get(RSGroupInfo.DEFAULT_GROUP);
RSGroupInfo newInfo = new RSGroupInfo(info.getName(), servers); RSGroupInfo newInfo = new RSGroupInfo(info.getName(), servers);
HashMap<String, RSGroupInfo> newGroupMap = Maps.newHashMap(rsGroupMap); HashMap<String, RSGroupInfo> newGroupMap = Maps.newHashMap(rsGroupMap);
newGroupMap.put(newInfo.getName(), newInfo); newGroupMap.put(newInfo.getName(), newInfo);
flushConfig(newGroupMap); resetRSGroupMap(newGroupMap);
} }
/** /**

25
hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdminClient.java
View File

@ -26,8 +26,11 @@ import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.SortedSet;
import java.util.regex.Pattern;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.NamespaceDescriptor; import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.Connection;
@ -133,6 +136,13 @@ public class VerifyingRSGroupAdminClient extends RSGroupAdminClient {
tds.addAll(admin.listTableDescriptors()); tds.addAll(admin.listTableDescriptors());
tds.addAll(admin.listTableDescriptorsByNamespace(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME)); tds.addAll(admin.listTableDescriptorsByNamespace(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME));
} }
SortedSet<Address> lives = Sets.newTreeSet();
for (ServerName sn : conn.getAdmin().getClusterMetrics().getLiveServerMetrics().keySet()) {
lives.add(sn.getAddress());
}
for (ServerName sn : conn.getAdmin().listDecommissionedRegionServers()) {
lives.remove(sn.getAddress());
}
try (Table table = conn.getTable(RSGroupInfoManagerImpl.RSGROUP_TABLE_NAME); try (Table table = conn.getTable(RSGroupInfoManagerImpl.RSGROUP_TABLE_NAME);
ResultScanner scanner = table.getScanner(new Scan())) { ResultScanner scanner = table.getScanner(new Scan())) {
for (;;) { for (;;) {
@ -144,8 +154,22 @@ public class VerifyingRSGroupAdminClient extends RSGroupAdminClient {
RSGroupInfoManagerImpl.META_FAMILY_BYTES, RSGroupInfoManagerImpl.META_QUALIFIER_BYTES)); RSGroupInfoManagerImpl.META_FAMILY_BYTES, RSGroupInfoManagerImpl.META_QUALIFIER_BYTES));
RSGroupInfo rsGroupInfo = ProtobufUtil.toGroupInfo(proto); RSGroupInfo rsGroupInfo = ProtobufUtil.toGroupInfo(proto);
groupMap.put(proto.getName(), RSGroupUtil.fillTables(rsGroupInfo, tds)); groupMap.put(proto.getName(), RSGroupUtil.fillTables(rsGroupInfo, tds));
for(Address address : rsGroupInfo.getServers()){
lives.remove(address);
} }
} }
}
SortedSet<TableName> tables = Sets.newTreeSet();
for (TableDescriptor td : conn.getAdmin().listTableDescriptors(Pattern.compile(".*"),
true)){
String groupName = td.getRegionServerGroup().orElse(RSGroupInfo.DEFAULT_GROUP);
if (groupName.equals(RSGroupInfo.DEFAULT_GROUP)) {
tables.add(td.getTableName());
}
}
groupMap.put(RSGroupInfo.DEFAULT_GROUP,
new RSGroupInfo(RSGroupInfo.DEFAULT_GROUP, lives, tables));
assertEquals(Sets.newHashSet(groupMap.values()), Sets.newHashSet(wrapped.listRSGroups())); assertEquals(Sets.newHashSet(groupMap.values()), Sets.newHashSet(wrapped.listRSGroups()));
try { try {
String groupBasePath = ZNodePaths.joinZNode(zkw.getZNodePaths().baseZNode, "rsgroup"); String groupBasePath = ZNodePaths.joinZNode(zkw.getZNodePaths().baseZNode, "rsgroup");
@ -160,6 +184,7 @@ public class VerifyingRSGroupAdminClient extends RSGroupAdminClient {
zList.add(RSGroupUtil.fillTables(rsGroupInfo, tds)); zList.add(RSGroupUtil.fillTables(rsGroupInfo, tds));
} }
} }
groupMap.remove(RSGroupInfo.DEFAULT_GROUP);
assertEquals(zList.size(), groupMap.size()); assertEquals(zList.size(), groupMap.size());
for (RSGroupInfo rsGroupInfo : zList) { for (RSGroupInfo rsGroupInfo : zList) {
assertTrue(groupMap.get(rsGroupInfo.getName()).equals(rsGroupInfo)); assertTrue(groupMap.get(rsGroupInfo.getName()).equals(rsGroupInfo));