Apache/hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-15415. Reduce locking in Datanode DirectoryScanner. Contributed by Stephen O'Donnell

Browse Source
This commit is contained in:
S O'Donnell 2020-09-17 14:11:42 +01:00
parent 425f48799c
commit 20a0e6278d
1 changed files with 68 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
View File

@ -46,7 +46,6 @@
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi.ScanInfo; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi.ScanInfo;
import org.apache.hadoop.util.AutoCloseableLock;
import org.apache.hadoop.util.Daemon; import org.apache.hadoop.util.Daemon;
import org.apache.hadoop.util.StopWatch; import org.apache.hadoop.util.StopWatch;
import org.slf4j.Logger; import org.slf4j.Logger;
@ -472,8 +471,6 @@ private void scan() {
// Pre-sort the reports outside of the lock // Pre-sort the reports outside of the lock
blockPoolReport.sortBlocks(); blockPoolReport.sortBlocks();
// Hold FSDataset lock to prevent further changes to the block map
try (AutoCloseableLock lock = dataset.acquireDatasetReadLock()) {
for (final String bpid : blockPoolReport.getBlockPoolIds()) { for (final String bpid : blockPoolReport.getBlockPoolIds()) {
List<ScanInfo> blockpoolReport = blockPoolReport.getScanInfo(bpid); List<ScanInfo> blockpoolReport = blockPoolReport.getScanInfo(bpid);
@ -482,7 +479,8 @@ private void scan() {
Collection<ScanInfo> diffRecord = new ArrayList<>(); Collection<ScanInfo> diffRecord = new ArrayList<>();
statsRecord.totalBlocks = blockpoolReport.size(); statsRecord.totalBlocks = blockpoolReport.size();
final List<ReplicaInfo> bl = dataset.getSortedFinalizedBlocks(bpid); final List<ReplicaInfo> bl;
bl = dataset.getSortedFinalizedBlocks(bpid);
int d = 0; // index for blockpoolReport int d = 0; // index for blockpoolReport
int m = 0; // index for memReprot int m = 0; // index for memReprot
@ -554,7 +552,6 @@ private void scan() {
LOG.info("Scan Results: {}", statsRecord); LOG.info("Scan Results: {}", statsRecord);
} }
} }
}
/** /**
* Add the ScanInfo object to the list of differences and adjust the stats * Add the ScanInfo object to the list of differences and adjust the stats