Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-11960. Successfully closed files can stay under-replicated. Contributed by Kihwal Lee.

This commit is contained in:
Kihwal Lee 2017-06-20 09:11:23 -05:00
parent 2e9daa2e27
commit 8c0769dee4
2 changed files with 46 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
View File

@ -3749,7 +3749,8 @@ public class BlockManager implements BlockStatsMXBean {
// Modify the blocks->datanode map and node's map.
//
BlockInfo storedBlock = getStoredBlock(block);
if (storedBlock != null) {
if (storedBlock != null &&
block.getGenerationStamp() == storedBlock.getGenerationStamp()) {
pendingReconstruction.decrement(storedBlock, node);
}
processAndHandleReportedBlock(storageInfo, block, ReplicaState.FINALIZED,

44
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReconstruction.java
View File

@ -209,6 +209,8 @@ public class TestPendingReconstruction {
// Place into blocksmap with GenerationStamp = 1
blockInfo.setGenerationStamp(1);
blocksMap.addBlockCollection(blockInfo, bc);
//Save it for later.
BlockInfo storedBlock = blockInfo;
assertEquals("Size of pendingReconstructions ", 1,
pendingReconstruction.size());
@ -255,6 +257,48 @@ public class TestPendingReconstruction {
// Verify size of neededReconstruction is exactly 1.
assertEquals("size of neededReconstruction is 1 ", 1,
neededReconstruction.size());
// Verify HDFS-11960
// Stop the replication/redundancy monitor
BlockManagerTestUtil.stopRedundancyThread(blkManager);
pendingReconstruction.clear();
// Pick a real node
DatanodeDescriptor desc[] = { blkManager.getDatanodeManager().
getDatanodes().iterator().next() };
// Add a stored block to the pendingReconstruction.
pendingReconstruction.increment(storedBlock, desc);
assertEquals("Size of pendingReconstructions ", 1,
pendingReconstruction.size());
// A received IBR processing calls addBlock(). If the gen stamp in the
// report is not the same, it should stay in pending.
fsn.writeLock();
try {
// Use a wrong gen stamp.
blkManager.addBlock(desc[0].getStorageInfos()[0],
new Block(1, 1, 0), null);
} finally {
fsn.writeUnlock();
}
// The block should still be pending
assertEquals("Size of pendingReconstructions ", 1,
pendingReconstruction.size());
// A block report with the correct gen stamp should remove the record
// from the pending queue.
fsn.writeLock();
try {
blkManager.addBlock(desc[0].getStorageInfos()[0],
new Block(1, 1, 1), null);
} finally {
fsn.writeUnlock();
}
// The pending queue should be empty.
assertEquals("Size of pendingReconstructions ", 0,
pendingReconstruction.size());
} finally {
if (cluster != null) {
cluster.shutdown();