diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 1d271d6bde8..56eb2c8ce8d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -110,6 +110,9 @@ Release 0.23.1 - UNRELEASED for a client on the same node as the block file. (Andrew Purtell, Suresh Srinivas and Jitendra Nath Pandey via szetszwo) + HDFS-2825. Add test hook to turn off the writer preferring its local + DN. (todd) + BUG FIXES HDFS-2541. For a sufficiently large value of blocks, the DN Scanner diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java index 6455b579a6a..d40aa52e8bc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java @@ -38,6 +38,8 @@ import org.apache.hadoop.net.Node; import org.apache.hadoop.net.NodeBase; +import com.google.common.annotations.VisibleForTesting; + /** The class is responsible for choosing the desired number of targets * for placing block replicas. * The replica placement strategy is that if the writer is on a datanode, @@ -49,6 +51,7 @@ @InterfaceAudience.Private public class BlockPlacementPolicyDefault extends BlockPlacementPolicy { private boolean considerLoad; + private boolean preferLocalNode = true; private NetworkTopology clusterMap; private FSClusterStats stats; static final String enableDebugLogging = "For more information, please enable" @@ -224,17 +227,17 @@ private DatanodeDescriptor chooseLocalNode( if (localMachine == null) return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize, maxNodesPerRack, results); - - // otherwise try local machine first - Node oldNode = excludedNodes.put(localMachine, localMachine); - if (oldNode == null) { // was not in the excluded list - if (isGoodTarget(localMachine, blocksize, - maxNodesPerRack, false, results)) { - results.add(localMachine); - return localMachine; - } - } - + if (preferLocalNode) { + // otherwise try local machine first + Node oldNode = excludedNodes.put(localMachine, localMachine); + if (oldNode == null) { // was not in the excluded list + if (isGoodTarget(localMachine, blocksize, + maxNodesPerRack, false, results)) { + results.add(localMachine); + return localMachine; + } + } + } // try a node on local rack return chooseLocalRack(localMachine, excludedNodes, blocksize, maxNodesPerRack, results); @@ -569,5 +572,10 @@ public DatanodeDescriptor chooseReplicaToDelete(FSInodeInfo inode, } return cur; } + + @VisibleForTesting + void setPreferLocalNode(boolean prefer) { + this.preferLocalNode = prefer; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java index 2d2406b12b9..8c9b4b38200 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java @@ -27,6 +27,8 @@ import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.util.Daemon; +import com.google.common.base.Preconditions; + public class BlockManagerTestUtil { public static void setNodeReplicationLimit(final BlockManager blockManager, final int limit) { @@ -122,4 +124,17 @@ public static int getComputedDatanodeWork(final BlockManager blockManager) throw return blockManager.computeDatanodeWork(); } + + /** + * Change whether the block placement policy will prefer the writer's + * local Datanode or not. + * @param prefer + */ + public static void setWritingPrefersLocalNode( + BlockManager bm, boolean prefer) { + BlockPlacementPolicy bpp = bm.getBlockPlacementPolicy(); + Preconditions.checkState(bpp instanceof BlockPlacementPolicyDefault, + "Must use default policy, got %s", bpp.getClass()); + ((BlockPlacementPolicyDefault)bpp).setPreferLocalNode(prefer); + } }