From c87fbeb7a83460fa43437244387249a1c1a7c486 Mon Sep 17 00:00:00 2001 From: Jing Zhao Date: Wed, 26 Mar 2014 18:38:14 +0000 Subject: [PATCH] HDFS-5672. Merge r1581994 from trunk. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1581996 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++ .../server/blockmanagement/BlockManager.java | 17 +++++++------ .../server/namenode/ha/TestHASafeMode.java | 24 ++++++++++++++++--- 3 files changed, 33 insertions(+), 10 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index d5371a4c8c2..0da57dd3f56 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -467,6 +467,8 @@ Release 2.4.0 - UNRELEASED HDFS-6115. Call flush() for every append on block scan verification log. (Vinayakumar B via szetszwo) + HDFS-5672. TestHASafeMode#testSafeBlockTracking fails in trunk. (jing9) + BREAKDOWN OF HDFS-5698 SUBTASKS AND RELATED JIRAS HDFS-5717. Save FSImage header in protobuf. (Haohui Mai via jing9) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 7c763dcc6df..ef250a3a9ff 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -1877,8 +1877,9 @@ public class BlockManager { int headIndex = 0; //currently the delimiter is in the head of the list int curIndex; - if (newReport == null) + if (newReport == null) { newReport = new BlockListAsLongs(); + } // scan the report and process newly reported blocks BlockReportIterator itBR = newReport.getBlockReportIterator(); while(itBR.hasNext()) { @@ -1971,9 +1972,11 @@ public class BlockManager { // Ignore replicas already scheduled to be removed from the DN if(invalidateBlocks.contains(dn.getDatanodeUuid(), block)) { -/* TODO: following assertion is incorrect, see HDFS-2668 -assert storedBlock.findDatanode(dn) < 0 : "Block " + block - + " in recentInvalidatesSet should not appear in DN " + dn; */ + /* + * TODO: following assertion is incorrect, see HDFS-2668 assert + * storedBlock.findDatanode(dn) < 0 : "Block " + block + + * " in recentInvalidatesSet should not appear in DN " + dn; + */ return storedBlock; } @@ -1993,8 +1996,8 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block } if (isBlockUnderConstruction(storedBlock, ucState, reportedState)) { - toUC.add(new StatefulBlockInfo( - (BlockInfoUnderConstruction)storedBlock, block, reportedState)); + toUC.add(new StatefulBlockInfo((BlockInfoUnderConstruction) storedBlock, + new Block(block), reportedState)); return storedBlock; } @@ -2878,7 +2881,7 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block // about new storages from heartbeats but during NN restart we may // receive a block report or incremental report before the heartbeat. // We must handle this for protocol compatibility. This issue was - // uncovered by HDFS-6904. + // uncovered by HDFS-6094. node.updateStorage(srdb.getStorage()); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java index 29b0e34ca95..1e3bc4237a6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java @@ -592,7 +592,16 @@ public class TestHASafeMode { // below 0. assertSafeMode(nn1, 0, 0, 3, 0); } - + + @Test + public void testSafeBlockTracking() throws Exception { + testSafeBlockTracking(false); + } + + @Test + public void testSafeBlockTracking2() throws Exception { + testSafeBlockTracking(true); + } /** * Test that the number of safe blocks is accounted correctly even when @@ -600,9 +609,15 @@ public class TestHASafeMode { * If a FINALIZED report arrives at the SBN before the block is marked * COMPLETE, then when we get the OP_CLOSE we need to count it as "safe" * at that point. This is a regression test for HDFS-2742. + * + * @param noFirstBlockReport If this is set to true, we shutdown NN1 before + * closing the writing streams. In this way, when NN1 restarts, all DNs will + * first send it incremental block report before the first full block report. + * And NN1 will not treat the full block report as the first block report + * in BlockManager#processReport. */ - @Test - public void testSafeBlockTracking() throws Exception { + private void testSafeBlockTracking(boolean noFirstBlockReport) + throws Exception { banner("Starting with NN0 active and NN1 standby, creating some " + "UC blocks plus some other blocks to force safemode"); DFSTestUtil.createFile(fs, new Path("/other-blocks"), 10*BLOCK_SIZE, (short) 3, 1L); @@ -619,6 +634,9 @@ public class TestHASafeMode { // the namespace during startup and enter safemode. nn0.getRpcServer().rollEditLog(); } finally { + if (noFirstBlockReport) { + cluster.shutdownNameNode(1); + } for (FSDataOutputStream stm : stms) { IOUtils.closeStream(stm); }