Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-15263. Fix the logic of scope and excluded scope in Network Topology. Contributed by Ayush Saxena.

This commit is contained in:
Ayush Saxena 2020-04-08 15:26:03 +05:30
parent 2214005c0f
commit 1189af4746
3 changed files with 39 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
View File

@ -487,10 +487,10 @@ public class NetworkTopology {
protected Node chooseRandom(final String scope, String excludedScope,
final Collection<Node> excludedNodes) {
if (excludedScope != null) {
if (scope.startsWith(excludedScope)) {
if (isChildScope(scope, excludedScope)) {
return null;
}
if (!excludedScope.startsWith(scope)) {
if (!isChildScope(excludedScope, scope)) {
excludedScope = null;
}
}
@ -668,8 +668,7 @@ public class NetworkTopology {
if (node == null) {
continue;
}
if ((NodeBase.getPath(node) + NodeBase.PATH_SEPARATOR_STR)
.startsWith(scope + NodeBase.PATH_SEPARATOR_STR)) {
if (isNodeInScope(node, scope)) {
if (node instanceof InnerNode) {
excludedCountInScope += ((InnerNode) node).getNumOfLeaves();
} else {
@ -994,4 +993,33 @@ public class NetworkTopology {
Preconditions.checkState(idx == activeLen,
"Sorted the wrong number of nodes!");
}
/**
* Checks whether one scope is contained in the other scope.
* @param parentScope the parent scope to check
* @param childScope the child scope which needs to be checked.
* @return true if childScope is contained within the parentScope
*/
protected static boolean isChildScope(final String parentScope,
final String childScope) {
String pScope = parentScope.endsWith(NodeBase.PATH_SEPARATOR_STR) ?
parentScope : parentScope + NodeBase.PATH_SEPARATOR_STR;
String cScope = childScope.endsWith(NodeBase.PATH_SEPARATOR_STR) ?
childScope : childScope + NodeBase.PATH_SEPARATOR_STR;
return pScope.startsWith(cScope);
}
/**
* Checks whether a node belongs to the scope.
* @param node the node to check.
* @param scope scope to check.
* @return true if node lies within the scope
*/
protected static boolean isNodeInScope(Node node, String scope) {
if (!scope.endsWith(NodeBase.PATH_SEPARATOR_STR)) {
scope += NodeBase.PATH_SEPARATOR_STR;
}
String nodeLocation = NodeBase.getPath(node) + NodeBase.PATH_SEPARATOR_STR;
return nodeLocation.startsWith(scope);
}
}

5
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestClusterTopology.java
View File

@ -18,6 +18,7 @@
package org.apache.hadoop.net;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Arrays;
@ -193,6 +194,10 @@ public class TestClusterTopology extends Assert {
}
assertEquals("Random is not selecting the nodes it should",
2, histogram.size());
Node val = cluster.chooseRandom("/d1", "/d", Collections.emptyList());
assertNotNull(val);
}
@Test

13
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java
View File

@ -180,10 +180,10 @@ public class DFSNetworkTopology extends NetworkTopology {
String excludedScope, final Collection<Node> excludedNodes,
StorageType type) {
if (excludedScope != null) {
if (scope.startsWith(excludedScope)) {
if (isChildScope(scope, excludedScope)) {
return null;
}
if (!excludedScope.startsWith(scope)) {
if (!isChildScope(excludedScope, scope)) {
excludedScope = null;
}
}
@ -264,15 +264,6 @@ public class DFSNetworkTopology extends NetworkTopology {
return chosen;
}
private boolean isNodeInScope(Node node, String scope) {
if (!scope.endsWith(NodeBase.PATH_SEPARATOR_STR)) {
scope += NodeBase.PATH_SEPARATOR_STR;
}
String nodeLocation =
node.getNetworkLocation() + NodeBase.PATH_SEPARATOR_STR;
return nodeLocation.startsWith(scope);
}
/**
* Choose a random node that has the required storage type, under the given
* root, with an excluded subtree root (could also just be a leaf node).