Apache/hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-15415. Reduce locking in Datanode DirectoryScanner. Contributed by Stephen O'Donnell.

Browse Source 
Signed-off-by: Wei-Chiu Chuang <weichiu@apache.org>
This commit is contained in:
Stephen O'Donnell 2020-09-28 12:43:25 -07:00 committed by Wei-Chiu Chuang
parent 947b0a154a
commit 71d006130e
1 changed files with 68 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
View File

@ -42,7 +42,6 @@
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.AutoCloseableLock;
import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.Block;
@ -406,8 +405,6 @@ private void scan() {
clear(); clear();
Map<String, ScanInfo[]> diskReport = getDiskReport(); Map<String, ScanInfo[]> diskReport = getDiskReport();
// Hold FSDataset lock to prevent further changes to the block map
try(AutoCloseableLock lock = dataset.acquireDatasetLock()) {
for (Entry<String, ScanInfo[]> entry : diskReport.entrySet()) { for (Entry<String, ScanInfo[]> entry : diskReport.entrySet()) {
String bpid = entry.getKey(); String bpid = entry.getKey();
ScanInfo[] blockpoolReport = entry.getValue(); ScanInfo[] blockpoolReport = entry.getValue();
@ -463,8 +460,8 @@ private void scan() {
d++; d++;
if (d < blockpoolReport.length) { if (d < blockpoolReport.length) {
// There may be multiple on-disk records for the same block, don't increment // There may be multiple on-disk records for the same block,
// the memory record pointer if so. // don't increment the memory record pointer if so.
ScanInfo nextInfo = blockpoolReport[d]; ScanInfo nextInfo = blockpoolReport[d];
if (nextInfo.getBlockId() != info.getBlockId()) { if (nextInfo.getBlockId() != info.getBlockId()) {
++m; ++m;
@ -487,7 +484,6 @@ private void scan() {
} }
LOG.info(statsRecord.toString()); LOG.info(statsRecord.toString());
} //end for } //end for
} //end synchronized
} }
/** /**