HDFS-10326. Disable setting tcp socket send/receive buffers for write pipelines. Contributed by Daryn Sharp.
This commit is contained in:
parent
e0c24145d2
commit
71b8dda4f6
|
@ -48,8 +48,8 @@ public final class HdfsConstants {
|
|||
public static final byte COLD_STORAGE_POLICY_ID = 2;
|
||||
public static final String COLD_STORAGE_POLICY_NAME = "COLD";
|
||||
|
||||
// TODO should be conf injected?
|
||||
public static final int DEFAULT_DATA_SOCKET_SIZE = 128 * 1024;
|
||||
public static final int DEFAULT_DATA_SOCKET_SIZE = 0;
|
||||
|
||||
/**
|
||||
* A special path component contained in the path for a snapshot file/dir
|
||||
*/
|
||||
|
|
|
@ -2545,13 +2545,14 @@
|
|||
|
||||
<property>
|
||||
<name>dfs.client.socket.send.buffer.size</name>
|
||||
<value>131072</value>
|
||||
<value>0</value>
|
||||
<description>
|
||||
Socket send buffer size for a write pipeline in DFSClient side.
|
||||
This may affect TCP connection throughput.
|
||||
If it is set to zero or negative value,
|
||||
no buffer size will be set explicitly,
|
||||
thus enable tcp auto-tuning on some system.
|
||||
The default value is 0.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
|
@ -3025,23 +3026,25 @@
|
|||
|
||||
<property>
|
||||
<name>dfs.datanode.transfer.socket.send.buffer.size</name>
|
||||
<value>131072</value>
|
||||
<value>0</value>
|
||||
<description>
|
||||
Socket send buffer size for DataXceiver (mirroring packets to downstream
|
||||
in pipeline). This may affect TCP connection throughput.
|
||||
If it is set to zero or negative value, no buffer size will be set
|
||||
explicitly, thus enable tcp auto-tuning on some system.
|
||||
The default value is 0.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.datanode.transfer.socket.recv.buffer.size</name>
|
||||
<value>131072</value>
|
||||
<value>0</value>
|
||||
<description>
|
||||
Socket receive buffer size for DataXceiver (receiving packets from client
|
||||
during block writing). This may affect TCP connection throughput.
|
||||
If it is set to zero or negative value, no buffer size will be set
|
||||
explicitly, thus enable tcp auto-tuning on some system.
|
||||
The default value is 0.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
|
|
|
@ -30,7 +30,6 @@ import org.slf4j.LoggerFactory;
|
|||
import java.io.IOException;
|
||||
import java.net.Socket;
|
||||
|
||||
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_SOCKET_SEND_BUFFER_SIZE_DEFAULT;
|
||||
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_SOCKET_SEND_BUFFER_SIZE_KEY;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
|
@ -42,15 +41,16 @@ public class TestDFSClientSocketSize {
|
|||
}
|
||||
|
||||
/**
|
||||
* The setting of socket send buffer size in
|
||||
* {@link java.net.Socket#setSendBufferSize(int)} is only a hint. Actual
|
||||
* value may differ. We just sanity check that it is somewhere close.
|
||||
* Test that the send buffer size default value is 0, in which case the socket
|
||||
* will use a TCP auto-tuned value.
|
||||
*/
|
||||
@Test
|
||||
public void testDefaultSendBufferSize() throws IOException {
|
||||
assertTrue("Send buffer size should be somewhere near default.",
|
||||
getSendBufferSize(new Configuration()) >=
|
||||
DFS_CLIENT_SOCKET_SEND_BUFFER_SIZE_DEFAULT / 2);
|
||||
final int sendBufferSize = getSendBufferSize(new Configuration());
|
||||
LOG.info("If not specified, the auto tuned send buffer size is: {}",
|
||||
sendBufferSize);
|
||||
assertTrue("Send buffer size should be non-negative value which is " +
|
||||
"determined by system (kernel).", sendBufferSize > 0);
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -73,6 +73,10 @@ public class TestDFSClientSocketSize {
|
|||
sendBufferSize1 > sendBufferSize2);
|
||||
}
|
||||
|
||||
/**
|
||||
* Test that if the send buffer size is 0, the socket will use a TCP
|
||||
* auto-tuned value.
|
||||
*/
|
||||
@Test
|
||||
public void testAutoTuningSendBufferSize() throws IOException {
|
||||
final Configuration conf = new Configuration();
|
||||
|
@ -80,7 +84,7 @@ public class TestDFSClientSocketSize {
|
|||
final int sendBufferSize = getSendBufferSize(conf);
|
||||
LOG.info("The auto tuned send buffer size is: {}", sendBufferSize);
|
||||
assertTrue("Send buffer size should be non-negative value which is " +
|
||||
"determined by system (kernel).", sendBufferSize > 0);
|
||||
"determined by system (kernel).", sendBufferSize > 0);
|
||||
}
|
||||
|
||||
private int getSendBufferSize(Configuration conf) throws IOException {
|
||||
|
|
Loading…
Reference in New Issue