Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-2825. Add test hook to turn off the writer preferring its local DN. Contributed by Todd Lipcon. 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.23@1235024 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Todd Lipcon 2012-01-23 22:06:53 +00:00
parent 118333dbe0
commit 747551a184
3 changed files with 37 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -110,6 +110,9 @@ Release 0.23.1 - UNRELEASED
for a client on the same node as the block file. (Andrew Purtell, for a client on the same node as the block file. (Andrew Purtell,
Suresh Srinivas and Jitendra Nath Pandey via szetszwo) Suresh Srinivas and Jitendra Nath Pandey via szetszwo)
HDFS-2825. Add test hook to turn off the writer preferring its local
DN. (todd)
BUG FIXES BUG FIXES
HDFS-2541. For a sufficiently large value of blocks, the DN Scanner HDFS-2541. For a sufficiently large value of blocks, the DN Scanner

30
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
View File

@ -38,6 +38,8 @@ import org.apache.hadoop.net.NetworkTopology;
import org.apache.hadoop.net.Node; import org.apache.hadoop.net.Node;
import org.apache.hadoop.net.NodeBase; import org.apache.hadoop.net.NodeBase;
import com.google.common.annotations.VisibleForTesting;
/** The class is responsible for choosing the desired number of targets /** The class is responsible for choosing the desired number of targets
* for placing block replicas. * for placing block replicas.
* The replica placement strategy is that if the writer is on a datanode, * The replica placement strategy is that if the writer is on a datanode,
@ -49,6 +51,7 @@ import org.apache.hadoop.net.NodeBase;
@InterfaceAudience.Private @InterfaceAudience.Private
public class BlockPlacementPolicyDefault extends BlockPlacementPolicy { public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
private boolean considerLoad; private boolean considerLoad;
private boolean preferLocalNode = true;
private NetworkTopology clusterMap; private NetworkTopology clusterMap;
private FSClusterStats stats; private FSClusterStats stats;
static final String enableDebugLogging = "For more information, please enable" static final String enableDebugLogging = "For more information, please enable"
@ -224,17 +227,17 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
if (localMachine == null) if (localMachine == null)
return chooseRandom(NodeBase.ROOT, excludedNodes, return chooseRandom(NodeBase.ROOT, excludedNodes,
blocksize, maxNodesPerRack, results); blocksize, maxNodesPerRack, results);
if (preferLocalNode) {
// otherwise try local machine first // otherwise try local machine first
Node oldNode = excludedNodes.put(localMachine, localMachine); Node oldNode = excludedNodes.put(localMachine, localMachine);
if (oldNode == null) { // was not in the excluded list if (oldNode == null) { // was not in the excluded list
if (isGoodTarget(localMachine, blocksize, if (isGoodTarget(localMachine, blocksize,
maxNodesPerRack, false, results)) { maxNodesPerRack, false, results)) {
results.add(localMachine); results.add(localMachine);
return localMachine; return localMachine;
} }
} }
}
// try a node on local rack // try a node on local rack
return chooseLocalRack(localMachine, excludedNodes, return chooseLocalRack(localMachine, excludedNodes,
blocksize, maxNodesPerRack, results); blocksize, maxNodesPerRack, results);
@ -569,5 +572,10 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
} }
return cur; return cur;
} }
@VisibleForTesting
void setPreferLocalNode(boolean prefer) {
this.preferLocalNode = prefer;
}
} }

15
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java
View File

@ -27,6 +27,8 @@ import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.util.Daemon; import org.apache.hadoop.util.Daemon;
import com.google.common.base.Preconditions;
public class BlockManagerTestUtil { public class BlockManagerTestUtil {
public static void setNodeReplicationLimit(final BlockManager blockManager, public static void setNodeReplicationLimit(final BlockManager blockManager,
final int limit) { final int limit) {
@ -122,4 +124,17 @@ public class BlockManagerTestUtil {
return blockManager.computeDatanodeWork(); return blockManager.computeDatanodeWork();
} }
/**
* Change whether the block placement policy will prefer the writer's
* local Datanode or not.
* @param prefer
*/
public static void setWritingPrefersLocalNode(
BlockManager bm, boolean prefer) {
BlockPlacementPolicy bpp = bm.getBlockPlacementPolicy();
Preconditions.checkState(bpp instanceof BlockPlacementPolicyDefault,
"Must use default policy, got %s", bpp.getClass());
((BlockPlacementPolicyDefault)bpp).setPreferLocalNode(prefer);
}
} }