Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-14751. Synchronize on diffs in DirectoryScanner. Contributed by Lisheng Sun.

This commit is contained in:
Wei-Chiu Chuang 2019-12-06 11:40:03 -08:00
parent 313b76f8e9
commit ecd461f940
1 changed files with 16 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
View File

@ -349,7 +349,9 @@ public class DirectoryScanner implements Runnable {
* Clear the current cache of diffs and statistics.
*/
private void clear() {
diffs.clear();
synchronized (diffs) {
diffs.clear();
}
stats.clear();
}
@ -430,17 +432,19 @@ public class DirectoryScanner implements Runnable {
// HDFS-14476: run checkAndUpadte with batch to avoid holding the lock too
// long
int loopCount = 0;
for (final Map.Entry<String, ScanInfo> entry : diffs.getEntries()) {
dataset.checkAndUpdate(entry.getKey(), entry.getValue());
synchronized (diffs) {
for (final Map.Entry<String, ScanInfo> entry : diffs.getEntries()) {
dataset.checkAndUpdate(entry.getKey(), entry.getValue());
if (loopCount % RECONCILE_BLOCKS_BATCH_SIZE == 0) {
try {
Thread.sleep(2000);
} catch (InterruptedException e) {
// do nothing
if (loopCount % RECONCILE_BLOCKS_BATCH_SIZE == 0) {
try {
Thread.sleep(2000);
} catch (InterruptedException e) {
// do nothing
}
}
loopCount++;
}
loopCount++;
}
if (!retainDiffs) {
@ -545,7 +549,9 @@ public class DirectoryScanner implements Runnable {
}
d++;
}
diffs.addAll(bpid, diffRecord);
synchronized (diffs) {
diffs.addAll(bpid, diffRecord);
}
LOG.info("Scan Results: {}", statsRecord);
}
}