From 3a2f3e78fff443033078b17b18c936b91c5ec799 Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Tue, 5 Sep 2017 14:58:08 -0700 Subject: [PATCH] HDFS-12182. BlockManager.metaSave does not distinguish between "under replicated" and "missing" blocks. Contributed by Wellington Chevreuil. --- .../server/blockmanagement/BlockManager.java | 29 ++++++++-- .../blockmanagement/TestBlockManager.java | 55 +++++++++++++++++++ .../hdfs/server/namenode/TestMetaSave.java | 4 +- 3 files changed, 82 insertions(+), 6 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 9bfbca830cc..4f0ec43e231 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -605,17 +605,36 @@ public class BlockManager implements BlockStatsMXBean { datanodeManager.fetchDatanodes(live, dead, false); out.println("Live Datanodes: " + live.size()); out.println("Dead Datanodes: " + dead.size()); + // - // Dump contents of neededReplication + // Need to iterate over all queues from neededReplications + // except for the QUEUE_WITH_CORRUPT_BLOCKS) // synchronized (neededReplications) { - out.println("Metasave: Blocks waiting for replication: " + - neededReplications.size()); - for (Block block : neededReplications) { + out.println("Metasave: Blocks waiting for reconstruction: " + + neededReplications.getUnderReplicatedBlockCount()); + for (int i = 0; i < neededReplications.LEVEL; i++) { + if (i != neededReplications.QUEUE_WITH_CORRUPT_BLOCKS) { + for (Iterator it = neededReplications.iterator(i); + it.hasNext();) { + Block block = it.next(); + dumpBlockMeta(block, out); + } + } + } + // + // Now prints corrupt blocks separately + // + out.println("Metasave: Blocks currently missing: " + + neededReplications.getCorruptBlockSize()); + for (Iterator it = neededReplications. + iterator(neededReplications.QUEUE_WITH_CORRUPT_BLOCKS); + it.hasNext();) { + Block block = it.next(); dumpBlockMeta(block, out); } } - + // Dump any postponed over-replicated blocks out.println("Mis-replicated blocks that have been postponed:"); for (Block block : postponedMisreplicatedBlocks) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java index f4fe38c70fd..ad848052ca5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java @@ -1283,4 +1283,59 @@ public class TestBlockManager { isReplicaCorrupt(Mockito.any(BlockInfo.class), Mockito.any(DatanodeDescriptor.class)); } + + @Test + public void testMetaSaveMissingReplicas() throws Exception { + List origStorages = getStorages(0, 1); + List origNodes = getNodes(origStorages); + BlockInfo block = makeBlockReplicasMissing(0, origNodes); + File file = new File("test.log"); + PrintWriter out = new PrintWriter(file); + bm.metaSave(out); + out.flush(); + FileInputStream fstream = new FileInputStream(file); + DataInputStream in = new DataInputStream(fstream); + BufferedReader reader = new BufferedReader(new InputStreamReader(in)); + StringBuffer buffer = new StringBuffer(); + String line; + try { + while ((line = reader.readLine()) != null) { + buffer.append(line); + } + String output = buffer.toString(); + assertTrue("Metasave output should have reported missing blocks.", + output.contains("Metasave: Blocks currently missing: 1")); + assertTrue("There should be 0 blocks waiting for reconstruction", + output.contains("Metasave: Blocks waiting for reconstruction: 0")); + String blockNameGS = block.getBlockName() + "_" + + block.getGenerationStamp(); + assertTrue("Block " + blockNameGS + " should be MISSING.", + output.contains(blockNameGS + " MISSING")); + } finally { + reader.close(); + file.delete(); + } + } + + private BlockInfo makeBlockReplicasMissing(long blockId, + List nodesList) throws IOException { + long inodeId = ++mockINodeId; + final INodeFile bc = TestINodeFile.createINodeFile(inodeId); + + BlockInfo blockInfo = blockOnNodes(blockId, nodesList); + blockInfo.setReplication((short) 3); + blockInfo.setBlockCollectionId(inodeId); + + Mockito.doReturn(bc).when(fsn).getBlockCollection(inodeId); + bm.blocksMap.addBlockCollection(blockInfo, bc); + bm.markBlockReplicasAsCorrupt(blockInfo, + blockInfo.getGenerationStamp() + 1, + blockInfo.getNumBytes(), + new DatanodeStorageInfo[]{}); + BlockCollection mockedBc = Mockito.mock(BlockCollection.class); + Mockito.when(mockedBc.getBlocks()).thenReturn(new BlockInfo[]{blockInfo}); + bm.checkReplication(mockedBc); + return blockInfo; + } + } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestMetaSave.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestMetaSave.java index 14f9382e906..22691c8989c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestMetaSave.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestMetaSave.java @@ -162,7 +162,9 @@ public class TestMetaSave { line = reader.readLine(); assertTrue(line.equals("Dead Datanodes: 1")); line = reader.readLine(); - assertTrue(line.equals("Metasave: Blocks waiting for replication: 0")); + assertTrue(line.equals("Metasave: Blocks waiting for reconstruction: 0")); + line = reader.readLine(); + assertTrue(line.equals("Metasave: Blocks currently missing: 0")); line = reader.readLine(); assertTrue(line.equals("Mis-replicated blocks that have been postponed:")); line = reader.readLine();