Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-10477. Stop decommission a rack of DataNodes caused NameNode fail over to standby. Contributed by yunjiong zhao and Wei-Chiu Chuang.

This commit is contained in:
Wei-Chiu Chuang 2019-04-10 14:36:07 -07:00
parent 66ac003128
commit 8af767b069
1 changed files with 35 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
View File

@ -3623,17 +3623,42 @@ public class BlockManager implements BlockStatsMXBean {
if (!isPopulatingReplQueues()) { if (!isPopulatingReplQueues()) {
return; return;
} }
final Iterator<BlockInfo> it = srcNode.getBlockIterator();
int numOverReplicated = 0; int numOverReplicated = 0;
while(it.hasNext()) { for (DatanodeStorageInfo datanodeStorageInfo : srcNode.getStorageInfos()) {
final BlockInfo block = it.next(); // the namesystem lock is released between iterations. Make sure the
short expectedReplication = getExpectedReplicaNum(block); // storage is not removed before continuing.
NumberReplicas num = countNodes(block); if (srcNode.getStorageInfo(datanodeStorageInfo.getStorageID()) == null) {
int numCurrentReplica = num.liveReplicas(); continue;
if (numCurrentReplica > expectedReplication) { }
// over-replicated block final Iterator<BlockInfo> it = datanodeStorageInfo.getBlockIterator();
processOverReplicatedBlock(block, expectedReplication, null, null); while(it.hasNext()) {
numOverReplicated++; final BlockInfo block = it.next();
if (block.isDeleted()) {
//Orphan block, will be handled eventually, skip
continue;
}
short expectedReplication = this.getExpectedReplicaNum(block);
NumberReplicas num = countNodes(block);
int numCurrentReplica = num.liveReplicas();
if (numCurrentReplica > expectedReplication) {
// over-replicated block
processOverReplicatedBlock(block, expectedReplication, null,
null);
numOverReplicated++;
}
}
// When called by tests like TestDefaultBlockPlacementPolicy.
// testPlacementWithLocalRackNodesDecommissioned, it is not protected by
// lock, only when called by DatanodeManager.refreshNodes have writeLock
if (namesystem.hasWriteLock()) {
namesystem.writeUnlock();
try {
Thread.sleep(1);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
namesystem.writeLock();
} }
} }
LOG.info("Invalidated " + numOverReplicated + " over-replicated blocks on " + LOG.info("Invalidated " + numOverReplicated + " over-replicated blocks on " +