From 09d63d5a192b5d6b172f94ff6c94da348fd49ea6 Mon Sep 17 00:00:00 2001
From: Vinayakumar B <vinayakumarb@apache.org>
Date: Wed, 30 Mar 2016 14:22:00 +0800
Subject: [PATCH] HDFS-5177. blocksScheduled count should be decremented for
 abandoned blocks (Contributed by Vinayakumar B)

---
 .../blockmanagement/DatanodeStorageInfo.java  | 10 ++++
 .../server/namenode/FSDirWriteFileOp.java     |  4 ++
 .../hdfs/TestBlocksScheduledCounter.java      | 51 +++++++++++++++++++
 3 files changed, 65 insertions(+)

diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
index c4729eafecd..843a8d514ba 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
@@ -298,6 +298,16 @@ public static void incrementBlocksScheduled(DatanodeStorageInfo... storages) {
     }
   }
 
+  /**
+   * Decrement the number of blocks scheduled for each given storage. This will
+   * be called during abandon block or delete of UC block.
+   */
+  public static void decrementBlocksScheduled(DatanodeStorageInfo... storages) {
+    for (DatanodeStorageInfo s : storages) {
+      s.getDatanodeDescriptor().decrementBlocksScheduled(s.getStorageType());
+    }
+  }
+
   @Override
   public boolean equals(Object obj) {
     if (this == obj) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
index 41fd8690a18..ab2f0fa32f7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
@@ -82,6 +82,10 @@ static boolean unprotectedRemoveBlock(
     if (uc == null) {
       return false;
     }
+    if (uc.getUnderConstructionFeature() != null) {
+      DatanodeStorageInfo.decrementBlocksScheduled(uc
+          .getUnderConstructionFeature().getExpectedStorageLocations());
+    }
     fsd.getBlockManager().removeBlockFromMap(uc);
 
     if(NameNode.stateChangeLog.isDebugEnabled()) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlocksScheduledCounter.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlocksScheduledCounter.java
index b9432197489..18942780e31 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlocksScheduledCounter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlocksScheduledCounter.java
@@ -78,4 +78,55 @@ public void testBlocksScheduledCounter() throws IOException {
     out.close();   
     assertEquals(0, dn.getBlocksScheduled());
   }
+
+  /**
+   * Abandon block should decrement the scheduledBlocks count for the dataNode.
+   */
+  @Test
+  public void testScheduledBlocksCounterShouldDecrementOnAbandonBlock()
+      throws Exception {
+    cluster = new MiniDFSCluster.Builder(new HdfsConfiguration()).numDataNodes(
+        2).build();
+
+    cluster.waitActive();
+    fs = cluster.getFileSystem();
+
+    DatanodeManager datanodeManager = cluster.getNamesystem().getBlockManager()
+        .getDatanodeManager();
+    ArrayList<DatanodeDescriptor> dnList = new ArrayList<DatanodeDescriptor>();
+    datanodeManager.fetchDatanodes(dnList, dnList, false);
+    for (DatanodeDescriptor descriptor : dnList) {
+      assertEquals("Blocks scheduled should be 0 for " + descriptor.getName(),
+          0, descriptor.getBlocksScheduled());
+    }
+
+    cluster.getDataNodes().get(0).shutdown();
+    // open a file an write a few bytes:
+    FSDataOutputStream out = fs.create(new Path("/testBlockScheduledCounter"),
+        (short) 2);
+    for (int i = 0; i < 1024; i++) {
+      out.write(i);
+    }
+    // flush to make sure a block is allocated.
+    out.hflush();
+
+    DatanodeDescriptor abandonedDn = datanodeManager.getDatanode(cluster
+        .getDataNodes().get(0).getDatanodeId());
+    assertEquals("for the abandoned dn scheduled counts should be 0", 0,
+        abandonedDn.getBlocksScheduled());
+
+    for (DatanodeDescriptor descriptor : dnList) {
+      if (descriptor.equals(abandonedDn)) {
+        continue;
+      }
+      assertEquals("Blocks scheduled should be 1 for " + descriptor.getName(),
+          1, descriptor.getBlocksScheduled());
+    }
+    // close the file and the counter should go to zero.
+    out.close();
+    for (DatanodeDescriptor descriptor : dnList) {
+      assertEquals("Blocks scheduled should be 0 for " + descriptor.getName(),
+          0, descriptor.getBlocksScheduled());
+    }
+  }
 }