diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 512ca207b50..b056e2979d7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -569,8 +569,6 @@ public class DFSConfigKeys extends CommonConfigurationKeys { "dfs.namenode.ec.system.default.policy"; public static final String DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY_DEFAULT = "RS-6-3-1024k"; - public static final String DFS_DN_EC_RECONSTRUCTION_STRIPED_READ_THREADS_KEY = "dfs.datanode.ec.reconstruction.stripedread.threads"; - public static final int DFS_DN_EC_RECONSTRUCTION_STRIPED_READ_THREADS_DEFAULT = 20; public static final String DFS_DN_EC_RECONSTRUCTION_STRIPED_READ_BUFFER_SIZE_KEY = "dfs.datanode.ec.reconstruction.stripedread.buffer.size"; public static final int DFS_DN_EC_RECONSTRUCTION_STRIPED_READ_BUFFER_SIZE_DEFAULT = 64 * 1024; public static final String DFS_DN_EC_RECONSTRUCTION_STRIPED_READ_TIMEOUT_MILLIS_KEY = "dfs.datanode.ec.reconstruction.stripedread.timeout.millis"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java index 70c5378979e..63498bc4dc3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java @@ -55,19 +55,19 @@ public final class ErasureCodingWorker { this.datanode = datanode; this.conf = conf; - initializeStripedReadThreadPool(conf.getInt( - DFSConfigKeys.DFS_DN_EC_RECONSTRUCTION_STRIPED_READ_THREADS_KEY, - DFSConfigKeys.DFS_DN_EC_RECONSTRUCTION_STRIPED_READ_THREADS_DEFAULT)); + initializeStripedReadThreadPool(); initializeStripedBlkReconstructionThreadPool(conf.getInt( DFSConfigKeys.DFS_DN_EC_RECONSTRUCTION_THREADS_KEY, DFSConfigKeys.DFS_DN_EC_RECONSTRUCTION_THREADS_DEFAULT)); } - private void initializeStripedReadThreadPool(int num) { - LOG.debug("Using striped reads; pool threads={}", num); + private void initializeStripedReadThreadPool() { + LOG.debug("Using striped reads"); - stripedReadPool = new ThreadPoolExecutor(1, num, 60, TimeUnit.SECONDS, - new SynchronousQueue(), + // Essentially, this is a cachedThreadPool. + stripedReadPool = new ThreadPoolExecutor(0, Integer.MAX_VALUE, + 60, TimeUnit.SECONDS, + new SynchronousQueue<>(), new Daemon.DaemonFactory() { private final AtomicInteger threadIndex = new AtomicInteger(0); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 726fe8c4643..d778d45e113 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -3056,15 +3056,6 @@ - - dfs.datanode.ec.reconstruction.stripedread.threads - 20 - - Number of threads used by the Datanode to read striped block - during background reconstruction work. - - - dfs.datanode.ec.reconstruction.stripedread.buffer.size 65536 diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md index d66131ef5c9..c8ef6c7e6c8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md @@ -134,7 +134,6 @@ Deployment Erasure coding background recovery work on the DataNodes can also be tuned via the following configuration parameters: 1. `dfs.datanode.ec.reconstruction.stripedread.timeout.millis` - Timeout for striped reads. Default value is 5000 ms. - 1. `dfs.datanode.ec.reconstruction.stripedread.threads` - Number of concurrent reader threads. Default value is 20 threads. 1. `dfs.datanode.ec.reconstruction.stripedread.buffer.size` - Buffer size for reader service. Default value is 64KB. 1. `dfs.datanode.ec.reconstruction.threads` - Number of threads used by the Datanode for background reconstruction work. Default value is 8 threads.