Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-25103 Remove ZNodePaths.metaReplicaZNodes (#2464) 

Signed-off-by: Huaxiang Sun <huaxiangsun@apache.com>
This commit is contained in:
Duo Zhang 2020-09-29 15:48:57 +08:00 committed by GitHub
parent 5d926627ae
commit b268b1f621
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 38 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hbase-client/src/main/java/org/apache/hadoop/hbase/client/ZKConnectionRegistry.java
View File

@ -141,7 +141,7 @@ class ZKConnectionRegistry implements ConnectionRegistry {
HRegionLocation[] locs = new HRegionLocation[metaReplicaZNodes.size()];
MutableInt remaining = new MutableInt(locs.length);
for (String metaReplicaZNode : metaReplicaZNodes) {
int replicaId = znodePaths.getMetaReplicaIdFromZnode(metaReplicaZNode);
int replicaId = znodePaths.getMetaReplicaIdFromZNode(metaReplicaZNode);
String path = ZNodePaths.joinZNode(znodePaths.baseZNode, metaReplicaZNode);
if (replicaId == DEFAULT_REPLICA_ID) {
addListener(getAndConvert(path, ZKConnectionRegistry::getMetaProto), (proto, error) -> {

68
hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZNodePaths.java
View File

@ -17,22 +17,15 @@
*/
package org.apache.hadoop.hbase.zookeeper;
import static org.apache.hadoop.hbase.HConstants.DEFAULT_META_REPLICA_NUM;
import static org.apache.hadoop.hbase.HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT;
import static org.apache.hadoop.hbase.HConstants.META_REPLICAS_NUM;
import static org.apache.hadoop.hbase.HConstants.SPLIT_LOGDIR_NAME;
import static org.apache.hadoop.hbase.HConstants.ZOOKEEPER_ZNODE_PARENT;
import static org.apache.hadoop.hbase.client.RegionInfo.DEFAULT_REPLICA_ID;
import java.util.Collection;
import java.util.Optional;
import java.util.stream.IntStream;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.RegionReplicaUtil;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap;
/**
* Class that hold all the paths of znode for HBase.
*/
@ -55,11 +48,6 @@ public class ZNodePaths {
*/
private final String metaZNodePrefix;
/**
* znodes containing the locations of the servers hosting the meta replicas
*/
private final ImmutableMap<Integer, String> metaReplicaZNodes;
// znode containing ephemeral nodes of the regionservers
public final String rsZNode;
// znode containing ephemeral nodes of the draining regionservers
@ -104,14 +92,7 @@ public class ZNodePaths {
public ZNodePaths(Configuration conf) {
baseZNode = conf.get(ZOOKEEPER_ZNODE_PARENT, DEFAULT_ZOOKEEPER_ZNODE_PARENT);
ImmutableMap.Builder<Integer, String> builder = ImmutableMap.builder();
metaZNodePrefix = conf.get(META_ZNODE_PREFIX_CONF_KEY, META_ZNODE_PREFIX);
String defaultMetaReplicaZNode = ZNodePaths.joinZNode(baseZNode, metaZNodePrefix);
builder.put(DEFAULT_REPLICA_ID, defaultMetaReplicaZNode);
int numMetaReplicas = conf.getInt(META_REPLICAS_NUM, DEFAULT_META_REPLICA_NUM);
IntStream.range(1, numMetaReplicas)
.forEachOrdered(i -> builder.put(i, defaultMetaReplicaZNode + "-" + i));
metaReplicaZNodes = builder.build();
rsZNode = joinZNode(baseZNode, conf.get("zookeeper.znode.rs", "rs"));
drainingZNode = joinZNode(baseZNode, conf.get("zookeeper.znode.draining.rs", "draining"));
masterAddressZNode = joinZNode(baseZNode, conf.get("zookeeper.znode.master", "master"));
@ -142,7 +123,6 @@ public class ZNodePaths {
public String toString() {
return new StringBuilder()
.append("ZNodePaths [baseZNode=").append(baseZNode)
.append(", metaReplicaZNodes=").append(metaReplicaZNodes)
.append(", rsZNode=").append(rsZNode)
.append(", drainingZNode=").append(drainingZNode)
.append(", masterAddressZNode=").append(masterAddressZNode)
@ -164,29 +144,15 @@ public class ZNodePaths {
.append("]").toString();
}
/**
* @return true if the znode is a meta region replica
*/
public boolean isAnyMetaReplicaZNode(String node) {
return this.metaReplicaZNodes.containsValue(node);
}
/**
* @return Meta Replica ZNodes
*/
public Collection<String> getMetaReplicaZNodes() {
return this.metaReplicaZNodes.values();
}
/**
* @return the znode string corresponding to a replicaId
*/
public String getZNodeForReplica(int replicaId) {
// return a newly created path but don't update the cache of paths
// This is mostly needed for tests that attempt to create meta replicas
// from outside the master
return Optional.ofNullable(metaReplicaZNodes.get(replicaId))
.orElseGet(() -> metaReplicaZNodes.get(DEFAULT_REPLICA_ID) + "-" + replicaId);
if (RegionReplicaUtil.isDefaultReplica(replicaId)) {
return joinZNode(baseZNode, metaZNodePrefix);
} else {
return joinZNode(baseZNode, metaZNodePrefix + "-" + replicaId);
}
}
/**
@ -198,7 +164,7 @@ public class ZNodePaths {
// Extract the znode from path. The prefix is of the following format.
// baseZNode + PATH_SEPARATOR.
int prefixLen = baseZNode.length() + 1;
return getMetaReplicaIdFromZnode(path.substring(prefixLen));
return getMetaReplicaIdFromZNode(path.substring(prefixLen));
}
/**
@ -206,7 +172,7 @@ public class ZNodePaths {
* @param znode the name of the znode, does not include baseZNode
* @return replicaId
*/
public int getMetaReplicaIdFromZnode(String znode) {
public int getMetaReplicaIdFromZNode(String znode) {
return znode.equals(metaZNodePrefix)?
RegionInfo.DEFAULT_REPLICA_ID:
Integer.parseInt(znode.substring(metaZNodePrefix.length() + 1));
@ -220,17 +186,25 @@ public class ZNodePaths {
}
/**
* Returns whether the znode is supposed to be readable by the client and DOES NOT contain
* @return True is the fully qualified path is for meta location
*/
public boolean isMetaZNodePath(String path) {
int prefixLen = baseZNode.length() + 1;
return path.length() > prefixLen && isMetaZNodePrefix(path.substring(prefixLen));
}
/**
* Returns whether the path is supposed to be readable by the client and DOES NOT contain
* sensitive information (world readable).
*/
public boolean isClientReadable(String node) {
public boolean isClientReadable(String path) {
// Developer notice: These znodes are world readable. DO NOT add more znodes here UNLESS
// all clients need to access this data to work. Using zk for sharing data to clients (other
// than service lookup case is not a recommended design pattern.
return node.equals(baseZNode) || isAnyMetaReplicaZNode(node) ||
node.equals(masterAddressZNode) || node.equals(clusterIdZNode) || node.equals(rsZNode) ||
return path.equals(baseZNode) || isMetaZNodePath(path) || path.equals(masterAddressZNode) ||
path.equals(clusterIdZNode) || path.equals(rsZNode) ||
// /hbase/table and /hbase/table/foo is allowed, /hbase/table-lock is not
node.equals(tableZNode) || node.startsWith(tableZNode + "/");
path.equals(tableZNode) || path.startsWith(tableZNode + "/");
}
/**

2
hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterMetaBootstrap.java
View File

@ -92,7 +92,7 @@ class MasterMetaBootstrap {
try {
List<String> metaReplicaZnodes = zooKeeper.getMetaReplicaNodes();
for (String metaReplicaZnode : metaReplicaZnodes) {
int replicaId = zooKeeper.getZNodePaths().getMetaReplicaIdFromZnode(metaReplicaZnode);
int replicaId = zooKeeper.getZNodePaths().getMetaReplicaIdFromZNode(metaReplicaZnode);
if (replicaId >= numMetaReplicasConfigured) {
RegionState r = MetaTableLocator.getMetaRegionState(zooKeeper, replicaId);
LOG.info("Closing excess replica of meta region " + r.getRegion());

6
hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetaRegionLocationCache.java
View File

@ -157,7 +157,7 @@ public class MetaRegionLocationCache extends ZKListener {
}
private void updateMetaLocation(String path, ZNodeOpType opType) {
if (!isValidMetaZNode(path)) {
if (!isValidMetaPath(path)) {
return;
}
LOG.debug("Updating meta znode for path {}: {}", path, opType.name());
@ -220,8 +220,8 @@ public class MetaRegionLocationCache extends ZKListener {
* Helper to check if the given 'path' corresponds to a meta znode. This listener is only
* interested in changes to meta znodes.
*/
private boolean isValidMetaZNode(String path) {
return watcher.getZNodePaths().isAnyMetaReplicaZNode(path);
private boolean isValidMetaPath(String path) {
return watcher.getZNodePaths().isMetaZNodePath(path);
}
@Override

2
hbase-server/src/main/java/org/apache/hadoop/hbase/master/zksyncer/ClientZKSyncer.java
View File

@ -207,7 +207,7 @@ public abstract class ClientZKSyncer extends ZKListener {
/**
* @return the znode(s) to watch
*/
abstract Collection<String> getNodesToWatch();
abstract Collection<String> getNodesToWatch() throws KeeperException;
/**
* Thread to synchronize znode data to client ZK cluster

12
hbase-server/src/main/java/org/apache/hadoop/hbase/master/zksyncer/MetaLocationSyncer.java
View File

@ -19,10 +19,12 @@
package org.apache.hadoop.hbase.master.zksyncer;
import java.util.Collection;
import java.util.stream.Collectors;
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.zookeeper.KeeperException;
/**
* Tracks the meta region locations on server ZK cluster and synchronize them to client ZK cluster
@ -36,11 +38,13 @@ public class MetaLocationSyncer extends ClientZKSyncer {
@Override
boolean validate(String path) {
return watcher.getZNodePaths().isAnyMetaReplicaZNode(path);
return watcher.getZNodePaths().isMetaZNodePath(path);
}
@Override
Collection<String> getNodesToWatch() {
return watcher.getZNodePaths().getMetaReplicaZNodes();
Collection<String> getNodesToWatch() throws KeeperException {
return watcher.getMetaReplicaNodes().stream()
.map(znode -> ZNodePaths.joinZNode(watcher.getZNodePaths().baseZNode, znode))
.collect(Collectors.toList());
}
}

10
hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
View File

@ -37,14 +37,11 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import javax.security.auth.login.AppConfigurationEntry;
import javax.security.auth.login.AppConfigurationEntry.LoginModuleControlFlag;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.AuthUtil;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.exceptions.DeserializationException;
import org.apache.hadoop.hbase.security.Superusers;
@ -78,6 +75,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hbase.thirdparty.com.google.protobuf.InvalidProtocolBufferException;
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos;
@ -1861,9 +1859,7 @@ public final class ZKUtil {
}
sb.append("\nRegion server holding hbase:meta: "
+ MetaTableLocator.getMetaRegionLocation(zkw));
Configuration conf = HBaseConfiguration.create();
int numMetaReplicas = conf.getInt(HConstants.META_REPLICAS_NUM,
HConstants.DEFAULT_META_REPLICA_NUM);
int numMetaReplicas = zkw.getMetaReplicaNodes().size();
for (int i = 1; i < numMetaReplicas; i++) {
sb.append("\nRegion server holding hbase:meta, replicaId " + i + " "
+ MetaTableLocator.getMetaRegionLocation(zkw, i));
@ -2109,7 +2105,7 @@ public final class ZKUtil {
" byte(s) of data from znode " + znode +
(watcherSet? " and set watcher; ": "; data=") +
(data == null? "null": data.length == 0? "empty": (
zkw.getZNodePaths().isMetaZNodePrefix(znode)?
zkw.getZNodePaths().isMetaZNodePath(znode)?
getServerNameOrEmptyString(data):
znode.startsWith(zkw.getZNodePaths().backupMasterAddressesZNode)?
getServerNameOrEmptyString(data):