diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index e9d0eec8d2d..7bb30a64942 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -269,6 +269,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys { = "dfs.namenode.storageinfo.defragment.ratio"; public static final double DFS_NAMENODE_STORAGEINFO_DEFRAGMENT_RATIO_DEFAULT = 0.75; + public static final String DFS_NAMENODE_BLOCKREPORT_QUEUE_SIZE_KEY + = "dfs.namenode.blockreport.queue.size"; + public static final int DFS_NAMENODE_BLOCKREPORT_QUEUE_SIZE_DEFAULT + = 1024; public static final String DFS_WEBHDFS_AUTHENTICATION_FILTER_KEY = "dfs.web.authentication.filter"; /* Phrased as below to avoid javac inlining as a constant, to match the behavior when this was AuthFilter.class.getName(). Note that if you change the import for AuthFilter, you diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 9cfa18098e1..bc2141d986a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -317,8 +317,7 @@ public class BlockManager implements BlockStatsMXBean { new Daemon(new StorageInfoDefragmenter()); /** Block report thread for handling async reports. */ - private final BlockReportProcessingThread blockReportThread = - new BlockReportProcessingThread(); + private final BlockReportProcessingThread blockReportThread; /** * Store blocks {@literal ->} datanodedescriptor(s) map of corrupt replicas. @@ -574,6 +573,11 @@ public class BlockManager implements BlockStatsMXBean { bmSafeMode = new BlockManagerSafeMode(this, namesystem, haEnabled, conf); + int queueSize = conf.getInt( + DFSConfigKeys.DFS_NAMENODE_BLOCKREPORT_QUEUE_SIZE_KEY, + DFSConfigKeys.DFS_NAMENODE_BLOCKREPORT_QUEUE_SIZE_DEFAULT); + blockReportThread = new BlockReportProcessingThread(queueSize); + LOG.info("defaultReplication = {}", defaultReplication); LOG.info("maxReplication = {}", maxReplication); LOG.info("minReplication = {}", minReplication); @@ -4966,11 +4970,11 @@ public class BlockManager implements BlockStatsMXBean { private static final long MAX_LOCK_HOLD_MS = 4; private long lastFull = 0; - private final BlockingQueue queue = - new ArrayBlockingQueue(1024); + private final BlockingQueue queue; - BlockReportProcessingThread() { + BlockReportProcessingThread(int size) { super("Block report processor"); + queue = new ArrayBlockingQueue<>(size); setDaemon(true); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 8c575af90c0..76c0660ebde 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -5337,4 +5337,12 @@ inter-DN QOP. + + + dfs.namenode.blockreport.queue.size + 1024 + + The queue size of BlockReportProcessingThread in BlockManager. + +