Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-4946. Allow preferLocalNode in BlockPlacementPolicyDefault to be configurable. Contributed by James Kinley and Nathan Roberts. 

(cherry picked from commit ba6d5ed955)
This commit is contained in:
Kihwal Lee 2016-02-18 13:27:39 -06:00
parent f864faaafa
commit b83348c2e3
4 changed files with 24 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -1890,6 +1890,9 @@ Release 2.7.3 - UNRELEASED
HDFS-9669. TcpPeerServer should respect ipc.server.listen.queue.size HDFS-9669. TcpPeerServer should respect ipc.server.listen.queue.size
(Elliot Clark via cmccabe) (Elliot Clark via cmccabe)
HDFS-4946. Allow preferLocalNode in BlockPlacementPolicyDefault to be
configurable (James Kinley and Nathan Roberts via kihwal)
OPTIMIZATIONS OPTIMIZATIONS
BUG FIXES BUG FIXES

3
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
View File

@ -636,6 +636,9 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
"dfs.namenode.available-space-block-placement-policy.balanced-space-preference-fraction"; "dfs.namenode.available-space-block-placement-policy.balanced-space-preference-fraction";
public static final float DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_DEFAULT = public static final float DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_DEFAULT =
0.6f; 0.6f;
public static final String DFS_NAMENODE_BLOCKPLACEMENTPOLICY_DEFAULT_PREFER_LOCAL_NODE_KEY =
"dfs.namenode.block-placement-policy.default.prefer-local-node";
public static final boolean DFS_NAMENODE_BLOCKPLACEMENTPOLICY_DEFAULT_PREFER_LOCAL_NODE_DEFAULT = true;
public static final String DFS_BLOCK_LOCAL_PATH_ACCESS_USER_KEY = "dfs.block.local-path-access.user"; public static final String DFS_BLOCK_LOCAL_PATH_ACCESS_USER_KEY = "dfs.block.local-path-access.user";
public static final String DFS_DOMAIN_SOCKET_PATH_KEY = public static final String DFS_DOMAIN_SOCKET_PATH_KEY =

7
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
View File

@ -59,7 +59,7 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
protected boolean considerLoad; protected boolean considerLoad;
protected double considerLoadFactor; protected double considerLoadFactor;
private boolean preferLocalNode = true; private boolean preferLocalNode;
protected NetworkTopology clusterMap; protected NetworkTopology clusterMap;
protected Host2NodesMap host2datanodeMap; protected Host2NodesMap host2datanodeMap;
private FSClusterStats stats; private FSClusterStats stats;
@ -95,6 +95,11 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
this.staleInterval = conf.getLong( this.staleInterval = conf.getLong(
DFSConfigKeys.DFS_NAMENODE_STALE_DATANODE_INTERVAL_KEY, DFSConfigKeys.DFS_NAMENODE_STALE_DATANODE_INTERVAL_KEY,
DFSConfigKeys.DFS_NAMENODE_STALE_DATANODE_INTERVAL_DEFAULT); DFSConfigKeys.DFS_NAMENODE_STALE_DATANODE_INTERVAL_DEFAULT);
this.preferLocalNode = conf.getBoolean(
DFSConfigKeys.
DFS_NAMENODE_BLOCKPLACEMENTPOLICY_DEFAULT_PREFER_LOCAL_NODE_KEY,
DFSConfigKeys.
DFS_NAMENODE_BLOCKPLACEMENTPOLICY_DEFAULT_PREFER_LOCAL_NODE_DEFAULT);
} }
@Override @Override

12
hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
View File

@ -921,6 +921,18 @@
</description> </description>
</property> </property>
<property>
<name>dfs.namenode.block-placement-policy.default.prefer-local-node</name>
<value>true</value>
<description>Controls how the default block placement policy places
the first replica of a block. When true, it will prefer the node where
the client is running. When false, it will prefer a node in the same rack
as the client. Setting to false avoids situations where entire copies of
large files end up on a single node, thus creating hotspots.
</description>
</property>
<property> <property>
<name>dfs.stream-buffer-size</name> <name>dfs.stream-buffer-size</name>
<value>4096</value> <value>4096</value>