From 82a88b92b46911c1b33d11898f8f678a134a9b69 Mon Sep 17 00:00:00 2001 From: Zhe Zhang Date: Fri, 18 Sep 2015 10:45:29 -0700 Subject: [PATCH] HDFS-9086. Rename dfs.datanode.stripedread.threshold.millis to dfs.datanode.stripedread.timeout.millis. Contributed by Andrew Wang. Change-Id: Ice86c5d46d29e94087c0f186b941d5394e7ac0e6 --- .../hadoop-hdfs/CHANGES-HDFS-EC-7285.txt | 3 +++ .../java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 ++-- .../datanode/erasurecode/ErasureCodingWorker.java | 10 +++++----- .../org/apache/hadoop/hdfs/util/StripedBlockUtil.java | 6 +++--- .../hadoop-hdfs/src/main/resources/hdfs-default.xml | 8 ++++---- .../hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md | 2 +- 6 files changed, 18 insertions(+), 15 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt index 3a29db132c1..8ff696baf7e 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt @@ -432,3 +432,6 @@ HDFS-9097. Erasure coding: update EC command "-s" flag to "-p" when specifying policy. (zhz) + + HDFS-9086. Rename dfs.datanode.stripedread.threshold.millis to + dfs.datanode.stripedread.timeout.millis. (wang via zhz) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index f7cda187a62..53c6cdba11d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -399,8 +399,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final int DFS_DATANODE_STRIPED_READ_THREADS_DEFAULT = 20; public static final String DFS_DATANODE_STRIPED_READ_BUFFER_SIZE_KEY = "dfs.datanode.stripedread.buffer.size"; public static final int DFS_DATANODE_STRIPED_READ_BUFFER_SIZE_DEFAULT = 64 * 1024; - public static final String DFS_DATANODE_STRIPED_READ_THRESHOLD_MILLIS_KEY = "dfs.datanode.stripedread.threshold.millis"; - public static final int DFS_DATANODE_STRIPED_READ_THRESHOLD_MILLIS_DEFAULT = 5000; //5s + public static final String DFS_DATANODE_STRIPED_READ_TIMEOUT_MILLIS_KEY = "dfs.datanode.stripedread.timeout.millis"; + public static final int DFS_DATANODE_STRIPED_READ_TIMEOUT_MILLIS_DEFAULT = 5000; //5s public static final String DFS_DATANODE_STRIPED_BLK_RECOVERY_THREADS_KEY = "dfs.datanode.striped.blockrecovery.threads.size"; public static final int DFS_DATANODE_STRIPED_BLK_RECOVERY_THREADS_DEFAULT = 8; public static final String DFS_DATANODE_DNS_INTERFACE_KEY = "dfs.datanode.dns.interface"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java index 56b54f15907..d1fa4788d9a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java @@ -95,16 +95,16 @@ public final class ErasureCodingWorker { private ThreadPoolExecutor STRIPED_BLK_RECOVERY_THREAD_POOL; private ThreadPoolExecutor STRIPED_READ_THREAD_POOL; - private final int STRIPED_READ_THRESHOLD_MILLIS; + private final int STRIPED_READ_TIMEOUT_MILLIS; private final int STRIPED_READ_BUFFER_SIZE; public ErasureCodingWorker(Configuration conf, DataNode datanode) { this.datanode = datanode; this.conf = conf; - STRIPED_READ_THRESHOLD_MILLIS = conf.getInt( - DFSConfigKeys.DFS_DATANODE_STRIPED_READ_THRESHOLD_MILLIS_KEY, - DFSConfigKeys.DFS_DATANODE_STRIPED_READ_THRESHOLD_MILLIS_DEFAULT); + STRIPED_READ_TIMEOUT_MILLIS = conf.getInt( + DFSConfigKeys.DFS_DATANODE_STRIPED_READ_TIMEOUT_MILLIS_KEY, + DFSConfigKeys.DFS_DATANODE_STRIPED_READ_TIMEOUT_MILLIS_DEFAULT); initializeStripedReadThreadPool(conf.getInt( DFSConfigKeys.DFS_DATANODE_STRIPED_READ_THREADS_KEY, DFSConfigKeys.DFS_DATANODE_STRIPED_READ_THREADS_DEFAULT)); @@ -556,7 +556,7 @@ public final class ErasureCodingWorker { try { StripingChunkReadResult result = StripedBlockUtil.getNextCompletedStripedRead( - readService, futures, STRIPED_READ_THRESHOLD_MILLIS); + readService, futures, STRIPED_READ_TIMEOUT_MILLIS); int resultIndex = -1; if (result.state == StripingChunkReadResult.SUCCESSFUL) { resultIndex = result.index; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java index 4fded73c5e2..897b092e813 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java @@ -203,12 +203,12 @@ public class StripedBlockUtil { */ public static StripingChunkReadResult getNextCompletedStripedRead( CompletionService readService, Map, Integer> futures, - final long threshold) throws InterruptedException { + final long timeoutMillis) throws InterruptedException { Preconditions.checkArgument(!futures.isEmpty()); Future future = null; try { - if (threshold > 0) { - future = readService.poll(threshold, TimeUnit.MILLISECONDS); + if (timeoutMillis > 0) { + future = readService.poll(timeoutMillis, TimeUnit.MILLISECONDS); } else { future = readService.take(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 0db56dd3fb5..acbad07108b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -2403,23 +2403,23 @@ - dfs.datanode.stripedread.threshold.millis + dfs.datanode.stripedread.timeout.millis 5000 - datanode striped read threshold in millisecond. + Datanode striped read timeout in milliseconds. dfs.datanode.stripedread.threads 20 - datanode striped read thread pool size. + Number of threads used by the Datanode for background recovery work. dfs.datanode.stripedread.buffer.size 262144 - datanode striped read buffer size. + Datanode striped read buffer size. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md index 2323a6dcf5e..5b3aa34833a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md @@ -101,7 +101,7 @@ Deployment Erasure coding background recovery work on the DataNodes can also be tuned via the following configuration parameters: - 1. `dfs.datanode.stripedread.threshold.millis` - Timeout for striped reads. Default value is 5000 ms. + 1. `dfs.datanode.stripedread.timeout.millis` - Timeout for striped reads. Default value is 5000 ms. 1. `dfs.datanode.stripedread.threads` - Number of concurrent reader threads. Default value is 20 threads. 1. `dfs.datanode.stripedread.buffer.size` - Buffer size for reader service. Default value is 256KB.