Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-9700. DFSClient and DFSOutputStream should set TCP_NODELAY on sockets for DataTransferProtocol (Gary Helmling via iwasakims) 

(cherry picked from commit 372d1302c6)

Conflicts:
	hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
This commit is contained in:
Masatake Iwasaki 2016-02-13 03:31:15 +09:00
parent a714d04f5d
commit f71fa500f9
5 changed files with 23 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
View File

@ -1950,6 +1950,7 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
String dnAddr = dn.getXferAddr(getConf().isConnectToDnViaHostname());
LOG.debug("Connecting to datanode {}", dnAddr);
NetUtils.connect(sock, NetUtils.createSocketAddr(dnAddr), timeout);
sock.setTcpNoDelay(dfsClientConf.getDataTransferTcpNoDelay());
sock.setSoTimeout(timeout);
OutputStream unbufOut = NetUtils.getOutputStream(sock);

1
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
View File

@ -139,6 +139,7 @@ class DataStreamer extends Daemon {
final int timeout = client.getDatanodeReadTimeout(length);
NetUtils.connect(sock, isa, client.getRandomLocalInterfaceAddr(),
conf.getSocketTimeout());
sock.setTcpNoDelay(conf.getDataTransferTcpNoDelay());
sock.setSoTimeout(timeout);
sock.setKeepAlive(true);
if (conf.getSocketSendBufferSize() > 0) {

4
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
View File

@ -164,6 +164,10 @@ public interface HdfsClientConfigKeys {
String DFS_USER_HOME_DIR_PREFIX_KEY = "dfs.user.home.dir.prefix";
String DFS_USER_HOME_DIR_PREFIX_DEFAULT = "/user";
String DFS_DATA_TRANSFER_CLIENT_TCPNODELAY_KEY =
"dfs.data.transfer.client.tcpnodelay";
boolean DFS_DATA_TRANSFER_CLIENT_TCPNODELAY_DEFAULT = true;
/**
* These are deprecated config keys to client code.
*/

14
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
View File

@ -57,6 +57,8 @@ import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_SOCK
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_SOCKET_CACHE_EXPIRY_MSEC_KEY;
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_SOCKET_SEND_BUFFER_SIZE_DEFAULT;
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_SOCKET_SEND_BUFFER_SIZE_KEY;
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_DATA_TRANSFER_CLIENT_TCPNODELAY_DEFAULT;
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_DATA_TRANSFER_CLIENT_TCPNODELAY_KEY;
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_SOCKET_TIMEOUT_KEY;
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_USE_DN_HOSTNAME;
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_USE_DN_HOSTNAME_DEFAULT;
@ -135,6 +137,8 @@ public class DfsClientConf {
private final List<Class<? extends ReplicaAccessorBuilder>>
replicaAccessorBuilderClasses;
private final boolean dataTransferTcpNoDelay;
public DfsClientConf(Configuration conf) {
// The hdfsTimeout is currently the same as the ipc timeout
hdfsTimeout = Client.getTimeout(conf);
@ -172,6 +176,9 @@ public class DfsClientConf {
CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY,
CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT);
defaultChecksumOpt = getChecksumOptFromConf(conf);
dataTransferTcpNoDelay = conf.getBoolean(
DFS_DATA_TRANSFER_CLIENT_TCPNODELAY_KEY,
DFS_DATA_TRANSFER_CLIENT_TCPNODELAY_DEFAULT);
socketTimeout = conf.getInt(DFS_CLIENT_SOCKET_TIMEOUT_KEY,
HdfsConstants.READ_TIMEOUT);
socketSendBufferSize = conf.getInt(DFS_CLIENT_SOCKET_SEND_BUFFER_SIZE_KEY,
@ -407,6 +414,13 @@ public class DfsClientConf {
return writeByteArrayManagerConf;
}
/**
* @return whether TCP_NODELAY should be set on client sockets
*/
public boolean getDataTransferTcpNoDelay() {
return dataTransferTcpNoDelay;
}
/**
* @return the socketTimeout
*/

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -983,6 +983,9 @@ Release 2.8.0 - UNRELEASED
HDFS-9777. Fix typos in DFSAdmin command line and documentation.
(Wei-Chiu Chuang via umamahesh)
HDFS-9700. DFSClient and DFSOutputStream should set TCP_NODELAY on sockets
for DataTransferProtocol (Gary Helmling via iwasakims)
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than