HDFS-7980. Incremental BlockReport will dramatically slow down namenode startup. Contributed by Walter Su

(cherry picked from commit 4e1f2eb395)

hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt

@@ -153,6 +153,9 @@ Release 2.6.1 - UNRELEASED
     HDFS-8270. create() always retried with hardcoded timeout when file already
     exists with open lease (J.Andreina via vinayakumarb)
 
+    HDFS-7980. Incremental BlockReport will dramatically slow down namenode
+    startup.  (Walter Su via szetszwo)
+
 Release 2.6.0 - 2014-11-18
 
   INCOMPATIBLE CHANGES

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java

@@ -1820,7 +1820,7 @@ public class BlockManager {
         return !node.hasStaleStorages();
       }
 
-      if (storageInfo.numBlocks() == 0) {
+      if (storageInfo.getBlockReportCount() == 0) {
         // The first block report can be processed a lot more efficiently than
         // ordinary block reports.  This shortens restart times.
         processFirstBlockReport(storageInfo, newReport);
@@ -2043,7 +2043,7 @@ public class BlockManager {
       final BlockListAsLongs report) throws IOException {
     if (report == null) return;
     assert (namesystem.hasWriteLock());
-    assert (storageInfo.numBlocks() == 0);
+    assert (storageInfo.getBlockReportCount() == 0);
     BlockReportIterator itBR = report.getBlockReportIterator();
 
     while(itBR.hasNext()) {
@@ -2463,14 +2463,14 @@ public class BlockManager {
     }
 
     // just add it
-    storageInfo.addBlock(storedBlock);
+    boolean result = storageInfo.addBlock(storedBlock);
 
     // Now check for completion of blocks and safe block count
     int numCurrentReplica = countLiveNodes(storedBlock);
     if (storedBlock.getBlockUCState() == BlockUCState.COMMITTED
         && numCurrentReplica >= minReplication) {
       completeBlock(storedBlock.getBlockCollection(), storedBlock, false);
-    } else if (storedBlock.isComplete()) {
+    } else if (storedBlock.isComplete() && result == true) {
       // check whether safe replication is reached for the block
       // only complete blocks are counted towards that.
       // In the case that the block just became complete above, completeBlock()

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java

@@ -47,10 +47,15 @@ import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor.BlockTargetPair;
 import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
+import org.apache.hadoop.hdfs.server.datanode.FinalizedReplica;
+import org.apache.hadoop.hdfs.server.datanode.ReplicaBeingWritten;
+import org.apache.hadoop.hdfs.server.datanode.ReplicaInfo;
 import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
 import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
 import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
 import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
+import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo;
+import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks;
 import org.apache.hadoop.ipc.RemoteException;
 import org.apache.hadoop.net.NetworkTopology;
 import org.junit.Assert;
@@ -653,6 +658,112 @@ public class TestBlockManager {
         new BlockListAsLongs(null, null), null, false);
     assertEquals(1, ds.getBlockReportCount());
   }
+
+  /**
+   * test when NN starts and in same mode, it receives an incremental blockReport
+   * firstly. Then receives first full block report.
+   */
+  @Test
+  public void testSafeModeIBRBeforeFirstFullBR() throws Exception {
+    // pretend to be in safemode
+    doReturn(true).when(fsn).isInStartupSafeMode();
+
+    DatanodeDescriptor node = nodes.get(0);
+    DatanodeStorageInfo ds = node.getStorageInfos()[0];
+    node.isAlive = true;
+    DatanodeRegistration nodeReg =  new DatanodeRegistration(node, null, null, "");
+
+    // register new node
+    bm.getDatanodeManager().registerDatanode(nodeReg);
+    bm.getDatanodeManager().addDatanode(node);
+    assertEquals(node, bm.getDatanodeManager().getDatanode(node));
+    assertEquals(0, ds.getBlockReportCount());
+    // Build a incremental report
+    List<ReceivedDeletedBlockInfo> rdbiList =
+        new ArrayList<ReceivedDeletedBlockInfo>();
+    // Build a full report
+    List<Block> blocks = new ArrayList<Block>();
+
+    // blk_42 is finalized.
+    long receivedBlockId = 42;  // arbitrary
+    BlockInfo receivedBlock = addBlockToBM(receivedBlockId);
+    rdbiList.add(new ReceivedDeletedBlockInfo(new Block(receivedBlock),
+        ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK, null));
+    blocks.add(new FinalizedReplica(receivedBlock, null, null));
+
+    // blk_43 is under construction.
+    long receivingBlockId = 43;
+    BlockInfo receivingBlock = addUcBlockToBM(receivingBlockId);
+    rdbiList.add(new ReceivedDeletedBlockInfo(new Block(receivingBlock),
+        ReceivedDeletedBlockInfo.BlockStatus.RECEIVING_BLOCK, null));
+    blocks.add(new ReplicaBeingWritten(receivingBlock, null, null, null));
+
+    // blk_44 has 2 records in IBR. It's finalized. So full BR has 1 record.
+    long receivingReceivedBlockId = 44;
+    BlockInfo receivingReceivedBlock = addBlockToBM(receivingReceivedBlockId);
+    rdbiList.add(new ReceivedDeletedBlockInfo(new Block(receivingReceivedBlock),
+        ReceivedDeletedBlockInfo.BlockStatus.RECEIVING_BLOCK, null));
+    rdbiList.add(new ReceivedDeletedBlockInfo(new Block(receivingReceivedBlock),
+        ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK, null));
+    blocks.add(new FinalizedReplica(receivingReceivedBlock, null, null));
+
+    // blk_45 is not in full BR, because it's deleted.
+    long ReceivedDeletedBlockId = 45;
+    rdbiList.add(new ReceivedDeletedBlockInfo(
+        new Block(ReceivedDeletedBlockId),
+        ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK, null));
+    rdbiList.add(new ReceivedDeletedBlockInfo(
+        new Block(ReceivedDeletedBlockId),
+        ReceivedDeletedBlockInfo.BlockStatus.DELETED_BLOCK, null));
+
+    // blk_46 exists in DN for a long time, so it's in full BR, but not in IBR.
+    long existedBlockId = 46;
+    BlockInfo existedBlock = addBlockToBM(existedBlockId);
+    blocks.add(new FinalizedReplica(existedBlock, null, null));
+
+    // process IBR and full BR
+    StorageReceivedDeletedBlocks srdb =
+        new StorageReceivedDeletedBlocks(new DatanodeStorage(ds.getStorageID()),
+            rdbiList.toArray(new ReceivedDeletedBlockInfo[rdbiList.size()]));
+    bm.processIncrementalBlockReport(node, srdb);
+    // Make sure it's the first full report
+    assertEquals(0, ds.getBlockReportCount());
+    bm.processReport(node, new DatanodeStorage(ds.getStorageID()),
+      new BlockListAsLongs(blocks,
+        (List<ReplicaInfo>) new ArrayList<ReplicaInfo>()), null, false);
+    assertEquals(1, ds.getBlockReportCount());
+
+    // verify the storage info is correct
+    assertTrue(bm.getStoredBlock(new Block(receivedBlockId)).findStorageInfo
+        (ds) >= 0);
+    assertTrue(((BlockInfoUnderConstruction) bm.
+        getStoredBlock(new Block(receivingBlockId))).getNumExpectedLocations() > 0);
+    assertTrue(bm.getStoredBlock(new Block(receivingReceivedBlockId))
+        .findStorageInfo(ds) >= 0);
+    assertNull(bm.getStoredBlock(new Block(ReceivedDeletedBlockId)));
+    assertTrue(bm.getStoredBlock(new Block(existedBlock)).findStorageInfo
+        (ds) >= 0);
+  }
+
+  private BlockInfo addBlockToBM(long blkId) {
+    Block block = new Block(blkId);
+    BlockInfo blockInfo =
+        new BlockInfo(block, (short) 3);
+    BlockCollection bc = Mockito.mock(BlockCollection.class);
+    Mockito.doReturn((short) 3).when(bc).getBlockReplication();
+    bm.blocksMap.addBlockCollection(blockInfo, bc);
+    return blockInfo;
+  }
+
+  private BlockInfo addUcBlockToBM(long blkId) {
+    Block block = new Block(blkId);
+    BlockInfoUnderConstruction blockInfo =
+        new BlockInfoUnderConstruction(block, (short) 3);
+    BlockCollection bc = Mockito.mock(BlockCollection.class);
+    Mockito.doReturn((short) 3).when(bc).getBlockReplication();
+    bm.blocksMap.addBlockCollection(blockInfo, bc);
+    return blockInfo;
+  }
   
   /**
    * Tests that a namenode doesn't choose a datanode with full disks to