Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-11960. Successfully closed files can stay under-replicated. Contributed by Kihwal Lee. 

(cherry picked from commit 8f533ca76b)
This commit is contained in:
Kihwal Lee 2017-06-20 09:20:02 -05:00
parent a526c3b0bf
commit 40e3cd24b0
2 changed files with 48 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
View File

@ -3237,8 +3237,9 @@ public class BlockManager implements BlockStatsMXBean {
// Modify the blocks->datanode map and node's map.
//
BlockInfo storedBlock = getStoredBlock(block);
if (storedBlock != null) {
pendingReplications.decrement(getStoredBlock(block), node);
if (storedBlock != null &&
block.getGenerationStamp() == storedBlock.getGenerationStamp()) {
pendingReplications.decrement(storedBlock, node);
}
processAndHandleReportedBlock(storageInfo, block, ReplicaState.FINALIZED,
delHintNode);

45
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java
View File

@ -209,6 +209,8 @@ public class TestPendingReplication {
// Place into blocksmap with GenerationStamp = 1
blockInfo.setGenerationStamp(1);
blocksMap.addBlockCollection(blockInfo, bc);
//Save it for later.
BlockInfo storedBlock = blockInfo;
assertEquals("Size of pendingReplications ", 1,
pendingReplications.size());
@ -255,6 +257,49 @@ public class TestPendingReplication {
// Verify size of neededReplications is exactly 1.
assertEquals("size of neededReplications is 1 ", 1,
neededReplications.size());
// Verify HDFS-11960
// Stop the replication/redundancy monitor
BlockManagerTestUtil.stopReplicationThread(blkManager);
pendingReplications.clear();
// Pick a real node
DatanodeDescriptor desc[] = { blkManager.getDatanodeManager().
getDatanodes().iterator().next() };
// Add a stored block to the pendingReconstruction.
pendingReplications.increment(storedBlock, desc);
assertEquals("Size of pendingReplications ", 1,
pendingReplications.size());
// A received IBR processing calls addBlock(). If the gen stamp in the
// report is not the same, it should stay in pending.
fsn.writeLock();
try {
// Use a wrong gen stamp.
blkManager.addBlock(desc[0].getStorageInfos()[0],
new Block(1, 1, 0), null);
} finally {
fsn.writeUnlock();
}
// The block should still be pending
assertEquals("Size of pendingReplications ", 1,
pendingReplications.size());
// A block report with the correct gen stamp should remove the record
// from the pending queue.
fsn.writeLock();
try {
blkManager.addBlock(desc[0].getStorageInfos()[0],
new Block(1, 1, 1), null);
} finally {
fsn.writeUnlock();
}
// The pending queue should be empty.
assertEquals("Size of pendingReplications ", 0,
pendingReplications.size());
} finally {
if (cluster != null) {
cluster.shutdown();