HDFS-9730. Storage ID update does not happen when there is a layout change. Contributed by Tsz Wo Nicholas Sze.

(cherry picked from commit 496f33de0c)

hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt

@@ -1791,6 +1791,9 @@ Release 2.7.3 - UNRELEASED
     HDFS-9740. Use a reasonable limit in DFSTestUtil.waitForMetric()
     (Chang Li via vinayakumarb)
 
+    HDFS-9730. Storage ID update does not happen when there is a layout change
+    (Tsz Wo Nicholas Sze via kihwal)
+
 Release 2.7.2 - 2016-01-25
 
   INCOMPATIBLE CHANGES

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java

@@ -518,11 +518,7 @@ public class DataStorage extends Storage {
     this.cTime = 0;
     setDatanodeUuid(datanodeUuid);
 
-    if (sd.getStorageUuid() == null) {
+    createStorageID(sd, false);
-      // Assign a new Storage UUID.
-      sd.setStorageUuid(DatanodeStorage.generateUuid());
-    }
-
     writeProperties(sd);
   }
 
@@ -696,7 +692,13 @@ public class DataStorage extends Storage {
 
     // do upgrade
     if (this.layoutVersion > HdfsServerConstants.DATANODE_LAYOUT_VERSION) {
-      doUpgrade(sd, nsInfo, conf);  // upgrade
+      if (federationSupported) {
+        // If the existing on-disk layout version supports federation,
+        // simply update the properties.
+        upgradeProperties(sd);
+      } else {
+        doUpgradePreFederation(sd, nsInfo, conf);
+      }
       return true; // doUgrade already has written properties
     }
     
@@ -710,7 +712,8 @@ public class DataStorage extends Storage {
   }
 
   /**
-   * Upgrade -- Move current storage into a backup directory,
+   * Upgrade from a pre-federation layout.
+   * Move current storage into a backup directory,
    * and hardlink all its blocks into the new current directory.
    * 
    * Upgrade from pre-0.22 to 0.22 or later release e.g. 0.19/0.20/ => 0.22/0.23
@@ -729,25 +732,9 @@ public class DataStorage extends Storage {
    * There should be only ONE namenode in the cluster for first 
    * time upgrade to 0.22
    * @param sd  storage directory
-   * @throws IOException on error
    */
-  void doUpgrade(final StorageDirectory sd, final NamespaceInfo nsInfo,
+  void doUpgradePreFederation(final StorageDirectory sd,
-      final Configuration conf) throws IOException {
+      final NamespaceInfo nsInfo, final Configuration conf) throws IOException {
-    // If the existing on-disk layout version supportes federation, simply
-    // update its layout version.
-    if (DataNodeLayoutVersion.supports(
-        LayoutVersion.Feature.FEDERATION, layoutVersion)) {
-      // The VERSION file is already read in. Override the layoutVersion 
-      // field and overwrite the file. The upgrade work is handled by
-      // {@link BlockPoolSliceStorage#doUpgrade}
-      LOG.info("Updating layout version from " + layoutVersion + " to "
-          + HdfsServerConstants.DATANODE_LAYOUT_VERSION + " for storage "
-          + sd.getRoot());
-      layoutVersion = HdfsServerConstants.DATANODE_LAYOUT_VERSION;
-      writeProperties(sd);
-      return;
-    }
-    
     final int oldLV = getLayoutVersion();
     LOG.info("Upgrading storage directory " + sd.getRoot()
              + ".\n   old LV = " + oldLV
@@ -791,15 +778,21 @@ public class DataStorage extends Storage {
     linkAllBlocks(tmpDir, bbwDir, toDir, oldLV, conf);
 
     // 4. Write version file under <SD>/current
-    layoutVersion = HdfsServerConstants.DATANODE_LAYOUT_VERSION;
     clusterID = nsInfo.getClusterID();
-    writeProperties(sd);
+    upgradeProperties(sd);
     
     // 5. Rename <SD>/previous.tmp to <SD>/previous
     rename(tmpDir, prevDir);
     LOG.info("Upgrade of " + sd.getRoot()+ " is complete");
+  }
 
+  void upgradeProperties(StorageDirectory sd) throws IOException {
     createStorageID(sd, layoutVersion);
+    LOG.info("Updating layout version from " + layoutVersion
+        + " to " + HdfsServerConstants.DATANODE_LAYOUT_VERSION
+        + " for storage " + sd.getRoot());
+    layoutVersion = HdfsServerConstants.DATANODE_LAYOUT_VERSION;
+    writeProperties(sd);
   }
 
   /**