Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-14553. Make queue size of BlockReportProcessingThread configurable. Contributed by He Xiaoqiao. 

Signed-off-by: Wei-Chiu Chuang <weichiu@apache.org>
(cherry picked from commit bd46bdf9f9)

Conflicts:
	hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
This commit is contained in:
He Xiaoqiao 2019-06-10 17:20:50 -07:00 committed by Wei-Chiu Chuang
parent 210480a23c
commit d7560c866e
3 changed files with 21 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
View File

@ -265,6 +265,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
= "dfs.namenode.storageinfo.defragment.ratio";
public static final double
DFS_NAMENODE_STORAGEINFO_DEFRAGMENT_RATIO_DEFAULT = 0.75;
public static final String DFS_NAMENODE_BLOCKREPORT_QUEUE_SIZE_KEY
= "dfs.namenode.blockreport.queue.size";
public static final int DFS_NAMENODE_BLOCKREPORT_QUEUE_SIZE_DEFAULT
= 1024;
public static final String DFS_WEBHDFS_AUTHENTICATION_FILTER_KEY = "dfs.web.authentication.filter";
/* Phrased as below to avoid javac inlining as a constant, to match the behavior when
this was AuthFilter.class.getName(). Note that if you change the import for AuthFilter, you

14
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
View File

@ -318,8 +318,7 @@ public class BlockManager implements BlockStatsMXBean {
new Daemon(new StorageInfoDefragmenter());
/** Block report thread for handling async reports. */
private final BlockReportProcessingThread blockReportThread =
new BlockReportProcessingThread();
private final BlockReportProcessingThread blockReportThread;
/** Store blocks -> datanodedescriptor(s) map of corrupt replicas */
final CorruptReplicasMap corruptReplicas = new CorruptReplicasMap();
@ -573,6 +572,11 @@ public class BlockManager implements BlockStatsMXBean {
bmSafeMode = new BlockManagerSafeMode(this, namesystem, haEnabled, conf);
int queueSize = conf.getInt(
DFSConfigKeys.DFS_NAMENODE_BLOCKREPORT_QUEUE_SIZE_KEY,
DFSConfigKeys.DFS_NAMENODE_BLOCKREPORT_QUEUE_SIZE_DEFAULT);
blockReportThread = new BlockReportProcessingThread(queueSize);
LOG.info("defaultReplication = {}", defaultReplication);
LOG.info("maxReplication = {}", maxReplication);
LOG.info("minReplication = {}", minReplication);
@ -4931,11 +4935,11 @@ public class BlockManager implements BlockStatsMXBean {
private static final long MAX_LOCK_HOLD_MS = 4;
private long lastFull = 0;
private final BlockingQueue<Runnable> queue =
new ArrayBlockingQueue<Runnable>(1024);
private final BlockingQueue<Runnable> queue;
BlockReportProcessingThread() {
BlockReportProcessingThread(int size) {
super("Block report processor");
queue = new ArrayBlockingQueue<>(size);
setDaemon(true);
}

8
hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
View File

@ -5192,4 +5192,12 @@
ensure that other waiters on the lock can get in.
</description>
</property>
<property>
<name>dfs.namenode.blockreport.queue.size</name>
<value>1024</value>
<description>
The queue size of BlockReportProcessingThread in BlockManager.
</description>
</property>
</configuration>