HDFS-5475. NN incorrectly tracks more than one replica per DN.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2832@1539890 13f79535-47bb-0310-9956-ffa450edef68

hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt

@@ -84,3 +84,7 @@ IMPROVEMENTS:
 
     HDFS-5472. Fix TestDatanodeManager, TestSafeMode and
     TestNNThroughputBenchmark (Contributed by szetszwo)
+
+    HDFS-5475.  NN incorrectly tracks more than one replica per DN. (Arpit
+    Agarwal)
+

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java

@@ -195,14 +195,24 @@ public class BlockInfo extends Block implements LightWeightGSet.LinkedElement {
    * Add a {@link DatanodeStorageInfo} location for a block
    */
   boolean addStorage(DatanodeStorageInfo storage) {
-    if(findStorageInfo(storage) >= 0) // the node is already there
+    boolean added = true;
-      return false;
+    int idx = findDatanode(storage.getDatanodeDescriptor());
+    if(idx >= 0) {
+      if (getStorageInfo(idx) == storage) { // the storage is already there
+        return false;
+      } else {
+        // The block is on the DN but belongs to a different storage.
+        // Update our state.
+        removeStorage(storage);
+        added = false;      // Just updating storage. Return false.
+      }
+    }
     // find the last null node
     int lastNode = ensureCapacity(1);
     setStorageInfo(lastNode, storage);
     setNext(lastNode, null);
     setPrevious(lastNode, null);
-    return true;
+    return added;
   }
 
   /**

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java

@@ -297,9 +297,22 @@ public class BlockInfoUnderConstruction extends BlockInfo {
   void addReplicaIfNotPresent(DatanodeStorageInfo storage,
                      Block block,
                      ReplicaState rState) {
-    for(ReplicaUnderConstruction r : replicas)
+    Iterator<ReplicaUnderConstruction> it = replicas.iterator();
-      if(r.getExpectedStorageLocation() == storage)
+    while (it.hasNext()) {
+      ReplicaUnderConstruction r = it.next();
+      if(r.getExpectedStorageLocation() == storage) {
         return;
+      } else if (r.getExpectedStorageLocation().getDatanodeDescriptor() ==
+          storage.getDatanodeDescriptor()) {
+
+        // The Datanode reported that the block is on a different storage
+        // than the one chosen by BlockPlacementPolicy. This can occur as
+        // we allow Datanodes to choose the target storage. Update our
+        // state by removing the stale entry and adding a new one.
+        it.remove();
+        break;
+      }
+    }
     replicas.add(new ReplicaUnderConstruction(block, storage, rState));
   }