Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-9086. Rename dfs.datanode.stripedread.threshold.millis to dfs.datanode.stripedread.timeout.millis. Contributed by Andrew Wang. 

Change-Id: Ice86c5d46d29e94087c0f186b941d5394e7ac0e6
This commit is contained in:
Zhe Zhang 2015-09-18 10:45:29 -07:00
parent 607bda2a21
commit 82a88b92b4
6 changed files with 18 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
View File

@ -432,3 +432,6 @@
HDFS-9097. Erasure coding: update EC command "-s" flag to "-p" when HDFS-9097. Erasure coding: update EC command "-s" flag to "-p" when
specifying policy. (zhz) specifying policy. (zhz)
HDFS-9086. Rename dfs.datanode.stripedread.threshold.millis to
dfs.datanode.stripedread.timeout.millis. (wang via zhz)

4
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
View File

@ -399,8 +399,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final int DFS_DATANODE_STRIPED_READ_THREADS_DEFAULT = 20; public static final int DFS_DATANODE_STRIPED_READ_THREADS_DEFAULT = 20;
public static final String DFS_DATANODE_STRIPED_READ_BUFFER_SIZE_KEY = "dfs.datanode.stripedread.buffer.size"; public static final String DFS_DATANODE_STRIPED_READ_BUFFER_SIZE_KEY = "dfs.datanode.stripedread.buffer.size";
public static final int DFS_DATANODE_STRIPED_READ_BUFFER_SIZE_DEFAULT = 64 * 1024; public static final int DFS_DATANODE_STRIPED_READ_BUFFER_SIZE_DEFAULT = 64 * 1024;
public static final String DFS_DATANODE_STRIPED_READ_THRESHOLD_MILLIS_KEY = "dfs.datanode.stripedread.threshold.millis"; public static final String DFS_DATANODE_STRIPED_READ_TIMEOUT_MILLIS_KEY = "dfs.datanode.stripedread.timeout.millis";
public static final int DFS_DATANODE_STRIPED_READ_THRESHOLD_MILLIS_DEFAULT = 5000; //5s public static final int DFS_DATANODE_STRIPED_READ_TIMEOUT_MILLIS_DEFAULT = 5000; //5s
public static final String DFS_DATANODE_STRIPED_BLK_RECOVERY_THREADS_KEY = "dfs.datanode.striped.blockrecovery.threads.size"; public static final String DFS_DATANODE_STRIPED_BLK_RECOVERY_THREADS_KEY = "dfs.datanode.striped.blockrecovery.threads.size";
public static final int DFS_DATANODE_STRIPED_BLK_RECOVERY_THREADS_DEFAULT = 8; public static final int DFS_DATANODE_STRIPED_BLK_RECOVERY_THREADS_DEFAULT = 8;
public static final String DFS_DATANODE_DNS_INTERFACE_KEY = "dfs.datanode.dns.interface"; public static final String DFS_DATANODE_DNS_INTERFACE_KEY = "dfs.datanode.dns.interface";

10
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
View File

@ -95,16 +95,16 @@ public final class ErasureCodingWorker {
private ThreadPoolExecutor STRIPED_BLK_RECOVERY_THREAD_POOL; private ThreadPoolExecutor STRIPED_BLK_RECOVERY_THREAD_POOL;
private ThreadPoolExecutor STRIPED_READ_THREAD_POOL; private ThreadPoolExecutor STRIPED_READ_THREAD_POOL;
private final int STRIPED_READ_THRESHOLD_MILLIS; private final int STRIPED_READ_TIMEOUT_MILLIS;
private final int STRIPED_READ_BUFFER_SIZE; private final int STRIPED_READ_BUFFER_SIZE;
public ErasureCodingWorker(Configuration conf, DataNode datanode) { public ErasureCodingWorker(Configuration conf, DataNode datanode) {
this.datanode = datanode; this.datanode = datanode;
this.conf = conf; this.conf = conf;
STRIPED_READ_THRESHOLD_MILLIS = conf.getInt( STRIPED_READ_TIMEOUT_MILLIS = conf.getInt(
DFSConfigKeys.DFS_DATANODE_STRIPED_READ_THRESHOLD_MILLIS_KEY, DFSConfigKeys.DFS_DATANODE_STRIPED_READ_TIMEOUT_MILLIS_KEY,
DFSConfigKeys.DFS_DATANODE_STRIPED_READ_THRESHOLD_MILLIS_DEFAULT); DFSConfigKeys.DFS_DATANODE_STRIPED_READ_TIMEOUT_MILLIS_DEFAULT);
initializeStripedReadThreadPool(conf.getInt( initializeStripedReadThreadPool(conf.getInt(
DFSConfigKeys.DFS_DATANODE_STRIPED_READ_THREADS_KEY, DFSConfigKeys.DFS_DATANODE_STRIPED_READ_THREADS_KEY,
DFSConfigKeys.DFS_DATANODE_STRIPED_READ_THREADS_DEFAULT)); DFSConfigKeys.DFS_DATANODE_STRIPED_READ_THREADS_DEFAULT));
@ -556,7 +556,7 @@ public final class ErasureCodingWorker {
try { try {
StripingChunkReadResult result = StripingChunkReadResult result =
StripedBlockUtil.getNextCompletedStripedRead( StripedBlockUtil.getNextCompletedStripedRead(
readService, futures, STRIPED_READ_THRESHOLD_MILLIS); readService, futures, STRIPED_READ_TIMEOUT_MILLIS);
int resultIndex = -1; int resultIndex = -1;
if (result.state == StripingChunkReadResult.SUCCESSFUL) { if (result.state == StripingChunkReadResult.SUCCESSFUL) {
resultIndex = result.index; resultIndex = result.index;

6
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
View File

@ -203,12 +203,12 @@ public class StripedBlockUtil {
*/ */
public static StripingChunkReadResult getNextCompletedStripedRead( public static StripingChunkReadResult getNextCompletedStripedRead(
CompletionService<Void> readService, Map<Future<Void>, Integer> futures, CompletionService<Void> readService, Map<Future<Void>, Integer> futures,
final long threshold) throws InterruptedException { final long timeoutMillis) throws InterruptedException {
Preconditions.checkArgument(!futures.isEmpty()); Preconditions.checkArgument(!futures.isEmpty());
Future<Void> future = null; Future<Void> future = null;
try { try {
if (threshold > 0) { if (timeoutMillis > 0) {
future = readService.poll(threshold, TimeUnit.MILLISECONDS); future = readService.poll(timeoutMillis, TimeUnit.MILLISECONDS);
} else { } else {
future = readService.take(); future = readService.take();
} }

8
hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
View File

@ -2403,23 +2403,23 @@
</property> </property>
<property> <property>
<name>dfs.datanode.stripedread.threshold.millis</name> <name>dfs.datanode.stripedread.timeout.millis</name>
<value>5000</value> <value>5000</value>
<description>datanode striped read threshold in millisecond. <description>Datanode striped read timeout in milliseconds.
</description> </description>
</property> </property>
<property> <property>
<name>dfs.datanode.stripedread.threads</name> <name>dfs.datanode.stripedread.threads</name>
<value>20</value> <value>20</value>
<description>datanode striped read thread pool size. <description>Number of threads used by the Datanode for background recovery work.
</description> </description>
</property> </property>
<property> <property>
<name>dfs.datanode.stripedread.buffer.size</name> <name>dfs.datanode.stripedread.buffer.size</name>
<value>262144</value> <value>262144</value>
<description>datanode striped read buffer size. <description>Datanode striped read buffer size.
</description> </description>
</property> </property>

2
hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
View File

@ -101,7 +101,7 @@ Deployment
Erasure coding background recovery work on the DataNodes can also be tuned via the following configuration parameters: Erasure coding background recovery work on the DataNodes can also be tuned via the following configuration parameters:
1. `dfs.datanode.stripedread.threshold.millis` - Timeout for striped reads. Default value is 5000 ms. 1. `dfs.datanode.stripedread.timeout.millis` - Timeout for striped reads. Default value is 5000 ms.
1. `dfs.datanode.stripedread.threads` - Number of concurrent reader threads. Default value is 20 threads. 1. `dfs.datanode.stripedread.threads` - Number of concurrent reader threads. Default value is 20 threads.
1. `dfs.datanode.stripedread.buffer.size` - Buffer size for reader service. Default value is 256KB. 1. `dfs.datanode.stripedread.buffer.size` - Buffer size for reader service. Default value is 256KB.