diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index 6271da26a4e..e0f00ee998f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -1950,6 +1950,7 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory, String dnAddr = dn.getXferAddr(getConf().isConnectToDnViaHostname()); LOG.debug("Connecting to datanode {}", dnAddr); NetUtils.connect(sock, NetUtils.createSocketAddr(dnAddr), timeout); + sock.setTcpNoDelay(dfsClientConf.getDataTransferTcpNoDelay()); sock.setSoTimeout(timeout); OutputStream unbufOut = NetUtils.getOutputStream(sock); diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java index 5daf2ac07f5..9ccb89a0058 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java @@ -139,6 +139,7 @@ class DataStreamer extends Daemon { final int timeout = client.getDatanodeReadTimeout(length); NetUtils.connect(sock, isa, client.getRandomLocalInterfaceAddr(), conf.getSocketTimeout()); + sock.setTcpNoDelay(conf.getDataTransferTcpNoDelay()); sock.setSoTimeout(timeout); sock.setKeepAlive(true); if (conf.getSocketSendBufferSize() > 0) { diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java index 992cf3a4a42..fb823e53acc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java @@ -164,6 +164,10 @@ public interface HdfsClientConfigKeys { String DFS_USER_HOME_DIR_PREFIX_KEY = "dfs.user.home.dir.prefix"; String DFS_USER_HOME_DIR_PREFIX_DEFAULT = "/user"; + String DFS_DATA_TRANSFER_CLIENT_TCPNODELAY_KEY = + "dfs.data.transfer.client.tcpnodelay"; + boolean DFS_DATA_TRANSFER_CLIENT_TCPNODELAY_DEFAULT = true; + /** * These are deprecated config keys to client code. */ diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java index 7f3ae048876..ed6cd23d804 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java @@ -57,6 +57,8 @@ import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_SOCK import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_SOCKET_CACHE_EXPIRY_MSEC_KEY; import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_SOCKET_SEND_BUFFER_SIZE_DEFAULT; import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_SOCKET_SEND_BUFFER_SIZE_KEY; +import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_DATA_TRANSFER_CLIENT_TCPNODELAY_DEFAULT; +import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_DATA_TRANSFER_CLIENT_TCPNODELAY_KEY; import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_SOCKET_TIMEOUT_KEY; import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_USE_DN_HOSTNAME; import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_USE_DN_HOSTNAME_DEFAULT; @@ -135,6 +137,8 @@ public class DfsClientConf { private final List> replicaAccessorBuilderClasses; + private final boolean dataTransferTcpNoDelay; + public DfsClientConf(Configuration conf) { // The hdfsTimeout is currently the same as the ipc timeout hdfsTimeout = Client.getTimeout(conf); @@ -172,6 +176,9 @@ public class DfsClientConf { CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY, CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT); defaultChecksumOpt = getChecksumOptFromConf(conf); + dataTransferTcpNoDelay = conf.getBoolean( + DFS_DATA_TRANSFER_CLIENT_TCPNODELAY_KEY, + DFS_DATA_TRANSFER_CLIENT_TCPNODELAY_DEFAULT); socketTimeout = conf.getInt(DFS_CLIENT_SOCKET_TIMEOUT_KEY, HdfsConstants.READ_TIMEOUT); socketSendBufferSize = conf.getInt(DFS_CLIENT_SOCKET_SEND_BUFFER_SIZE_KEY, @@ -407,6 +414,13 @@ public class DfsClientConf { return writeByteArrayManagerConf; } + /** + * @return whether TCP_NODELAY should be set on client sockets + */ + public boolean getDataTransferTcpNoDelay() { + return dataTransferTcpNoDelay; + } + /** * @return the socketTimeout */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index dba28c3162f..586632f5ce1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -905,6 +905,9 @@ Release 2.8.0 - UNRELEASED HDFS-9777. Fix typos in DFSAdmin command line and documentation. (Wei-Chiu Chuang via umamahesh) + HDFS-9700. DFSClient and DFSOutputStream should set TCP_NODELAY on sockets + for DataTransferProtocol (Gary Helmling via iwasakims) + OPTIMIZATIONS HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than