Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-14820. The default 8KB buffer of BlockReaderRemote#newBlockReader#BufferedOutputStream is too big. Contributed by Lisheng Sun.

This commit is contained in:
Wei-Chiu Chuang 2020-03-13 19:01:23 -07:00
parent e9955bb8ff
commit 7b637a3341
2 changed files with 3 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
View File

@ -150,7 +150,7 @@ public interface HdfsClientConfigKeys {
TimeUnit.DAYS.toMillis(10); // 10 days TimeUnit.DAYS.toMillis(10); // 10 days
String DFS_CLIENT_BLOCK_READER_REMOTE_BUFFER_SIZE_KEY = String DFS_CLIENT_BLOCK_READER_REMOTE_BUFFER_SIZE_KEY =
"dfs.client.block.reader.remote.buffer.size"; "dfs.client.block.reader.remote.buffer.size";
int DFS_CLIENT_BLOCK_READER_REMOTE_BUFFER_SIZE_DEFAULT = 8192; int DFS_CLIENT_BLOCK_READER_REMOTE_BUFFER_SIZE_DEFAULT = 512;
String DFS_CLIENT_DEAD_NODE_DETECTION_ENABLED_KEY = String DFS_CLIENT_DEAD_NODE_DETECTION_ENABLED_KEY =
"dfs.client.deadnode.detection.enabled"; "dfs.client.deadnode.detection.enabled";

6
hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
View File

@ -4305,13 +4305,11 @@
<property> <property>
<name>dfs.client.block.reader.remote.buffer.size</name> <name>dfs.client.block.reader.remote.buffer.size</name>
<value>8192</value> <value>512</value>
<description> <description>
The output stream buffer size of a DFSClient remote read. The buffer default value is 8KB. The buffer includes The output stream buffer size of a DFSClient remote read. The buffer default value is 512B. The buffer includes
only some request parameters that are: block, blockToken, clientName, startOffset, len, verifyChecksum, only some request parameters that are: block, blockToken, clientName, startOffset, len, verifyChecksum,
cachingStrategy. cachingStrategy.
It is recommended to adjust the value according to the workload, which can reduce unnecessary memory
usage and the frequency of the garbage collection. A value of 512 might be reasonable.
</description> </description>
</property> </property>