Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-16163. Avoid locking entire blockPinningFailures map (#3296)

This commit is contained in:
Viraj Jasani 2021-08-16 11:10:05 +05:30 committed by GitHub
parent 3aaac8a1f6
commit 919a99bbb4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 14 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java
View File

@ -40,6 +40,7 @@ import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Map.Entry; import java.util.Map.Entry;
import java.util.Set; import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors; import java.util.concurrent.Executors;
@ -405,7 +406,8 @@ public class Dispatcher {
// Pinned block can't be moved. Add this block into failure list. // Pinned block can't be moved. Add this block into failure list.
// Later in the next iteration mover will exclude these blocks from // Later in the next iteration mover will exclude these blocks from
// pending moves. // pending moves.
target.getDDatanode().addBlockPinningFailures(this); target.getDDatanode().addBlockPinningFailures(
this.reportedBlock.getBlock().getBlockId(), this.getSource());
return; return;
} }
@ -643,7 +645,8 @@ public class Dispatcher {
/** blocks being moved but not confirmed yet */ /** blocks being moved but not confirmed yet */
private final List<PendingMove> pendings; private final List<PendingMove> pendings;
private volatile boolean hasFailure = false; private volatile boolean hasFailure = false;
private Map<Long, Set<DatanodeInfo>> blockPinningFailures = new HashMap<>(); private final Map<Long, Set<DatanodeInfo>> blockPinningFailures =
new ConcurrentHashMap<>();
private volatile boolean hasSuccess = false; private volatile boolean hasSuccess = false;
private ExecutorService moveExecutor; private ExecutorService moveExecutor;
@ -729,16 +732,17 @@ public class Dispatcher {
this.hasFailure = true; this.hasFailure = true;
} }
void addBlockPinningFailures(PendingMove pendingBlock) { private void addBlockPinningFailures(long blockId, DatanodeInfo source) {
synchronized (blockPinningFailures) { blockPinningFailures.compute(blockId, (key, pinnedLocations) -> {
long blockId = pendingBlock.reportedBlock.getBlock().getBlockId(); Set<DatanodeInfo> newPinnedLocations;
Set<DatanodeInfo> pinnedLocations = blockPinningFailures.get(blockId);
if (pinnedLocations == null) { if (pinnedLocations == null) {
pinnedLocations = new HashSet<>(); newPinnedLocations = new HashSet<>();
blockPinningFailures.put(blockId, pinnedLocations); } else {
} newPinnedLocations = pinnedLocations;
pinnedLocations.add(pendingBlock.getSource());
} }
newPinnedLocations.add(source);
return newPinnedLocations;
});
} }
Map<Long, Set<DatanodeInfo>> getBlockPinningFailureList() { Map<Long, Set<DatanodeInfo>> getBlockPinningFailureList() {