HDFS-15355. Make the default block storage policy ID configurable. Contributed by Yang Yun.

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java

@@ -50,6 +50,52 @@ public final class HdfsConstants {
   public static final byte PROVIDED_STORAGE_POLICY_ID = 1;
   public static final String PROVIDED_STORAGE_POLICY_NAME = "PROVIDED";
 
+  /**
+   * This enum wraps above Storage Policy ID and name.
+   * Recommend to use this enum instead of above static variables.
+   * For example,
+   * StoragePolicy.HOT.value() is equal to HOT_STORAGE_POLICY_ID
+   * StoragePolicy.HOT.name() is equal to HOT_STORAGE_POLICY_NAME
+   */
+  public enum StoragePolicy{
+    PROVIDED(PROVIDED_STORAGE_POLICY_ID),
+    COLD(COLD_STORAGE_POLICY_ID),
+    WARM(WARM_STORAGE_POLICY_ID),
+    HOT(HOT_STORAGE_POLICY_ID),
+    ONE_SSD(ONESSD_STORAGE_POLICY_ID),
+    ALL_SSD(ALLSSD_STORAGE_POLICY_ID),
+    LAZY_PERSIST(MEMORY_STORAGE_POLICY_ID);
+
+    private byte value;
+    StoragePolicy(byte value) {
+      this.value = value;
+    }
+
+    public static StoragePolicy valueOf(int value) {
+      switch (value) {
+      case 1:
+        return PROVIDED;
+      case 2:
+        return COLD;
+      case 5:
+        return WARM;
+      case 7:
+        return HOT;
+      case 10:
+        return ONE_SSD;
+      case 12:
+        return ALL_SSD;
+      case 15:
+        return LAZY_PERSIST;
+      default:
+        return null;
+      }
+    }
+
+    public byte value() {
+      return this.value;
+    }
+  }
 
   public static final int DEFAULT_DATA_SOCKET_SIZE = 0;
 

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java

@@ -754,6 +754,11 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
       "dfs.storage.policy.satisfier.retry.max.attempts";
   public static final int DFS_STORAGE_POLICY_SATISFIER_MAX_RETRY_ATTEMPTS_DEFAULT =
       3;
+  public static final String DFS_STORAGE_DEFAULT_POLICY =
+      "dfs.storage.default.policy";
+  public static final HdfsConstants.StoragePolicy
+      DFS_STORAGE_DEFAULT_POLICY_DEFAULT = HdfsConstants.StoragePolicy.HOT;
+
   public static final String DFS_SPS_MAX_OUTSTANDING_PATHS_KEY =
       "dfs.storage.policy.satisfier.max.outstanding.paths";
   public static final int DFS_SPS_MAX_OUTSTANDING_PATHS_DEFAULT = 10000;

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java

@@ -493,7 +493,7 @@ public class BlockManager implements BlockStatsMXBean {
       conf, datanodeManager.getFSClusterStats(),
       datanodeManager.getNetworkTopology(),
       datanodeManager.getHost2DatanodeMap());
-    storagePolicySuite = BlockStoragePolicySuite.createDefaultSuite();
+    storagePolicySuite = BlockStoragePolicySuite.createDefaultSuite(conf);
     pendingReconstruction = new PendingReconstructionBlocks(conf.getInt(
         DFSConfigKeys.DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_KEY,
         DFSConfigKeys.DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_DEFAULT)

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStoragePolicySuite.java

@@ -21,8 +21,10 @@ import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Lists;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.StorageType;
 import org.apache.hadoop.fs.XAttr;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.XAttrHelper;
 import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
@@ -45,50 +47,75 @@ public class BlockStoragePolicySuite {
 
   @VisibleForTesting
   public static BlockStoragePolicySuite createDefaultSuite() {
+    return createDefaultSuite(null);
+  }
+
+  @VisibleForTesting
+  public static BlockStoragePolicySuite createDefaultSuite(
+      final Configuration conf) {
     final BlockStoragePolicy[] policies =
         new BlockStoragePolicy[1 << ID_BIT_LENGTH];
-    final byte lazyPersistId = HdfsConstants.MEMORY_STORAGE_POLICY_ID;
+    final byte lazyPersistId =
+        HdfsConstants.StoragePolicy.LAZY_PERSIST.value();
     policies[lazyPersistId] = new BlockStoragePolicy(lazyPersistId,
-        HdfsConstants.MEMORY_STORAGE_POLICY_NAME,
+        HdfsConstants.StoragePolicy.LAZY_PERSIST.name(),
         new StorageType[]{StorageType.RAM_DISK, StorageType.DISK},
         new StorageType[]{StorageType.DISK},
         new StorageType[]{StorageType.DISK},
         true);    // Cannot be changed on regular files, but inherited.
-    final byte allssdId = HdfsConstants.ALLSSD_STORAGE_POLICY_ID;
+    final byte allssdId = HdfsConstants.StoragePolicy.ALL_SSD.value();
     policies[allssdId] = new BlockStoragePolicy(allssdId,
-        HdfsConstants.ALLSSD_STORAGE_POLICY_NAME,
+        HdfsConstants.StoragePolicy.ALL_SSD.name(),
         new StorageType[]{StorageType.SSD},
         new StorageType[]{StorageType.DISK},
         new StorageType[]{StorageType.DISK});
-    final byte onessdId = HdfsConstants.ONESSD_STORAGE_POLICY_ID;
+    final byte onessdId = HdfsConstants.StoragePolicy.ONE_SSD.value();
     policies[onessdId] = new BlockStoragePolicy(onessdId,
-        HdfsConstants.ONESSD_STORAGE_POLICY_NAME,
+        HdfsConstants.StoragePolicy.ONE_SSD.name(),
         new StorageType[]{StorageType.SSD, StorageType.DISK},
         new StorageType[]{StorageType.SSD, StorageType.DISK},
         new StorageType[]{StorageType.SSD, StorageType.DISK});
-    final byte hotId = HdfsConstants.HOT_STORAGE_POLICY_ID;
+    final byte hotId = HdfsConstants.StoragePolicy.HOT.value();
     policies[hotId] = new BlockStoragePolicy(hotId,
-        HdfsConstants.HOT_STORAGE_POLICY_NAME,
+        HdfsConstants.StoragePolicy.HOT.name(),
         new StorageType[]{StorageType.DISK}, StorageType.EMPTY_ARRAY,
         new StorageType[]{StorageType.ARCHIVE});
-    final byte warmId = HdfsConstants.WARM_STORAGE_POLICY_ID;
+    final byte warmId = HdfsConstants.StoragePolicy.WARM.value();
     policies[warmId] = new BlockStoragePolicy(warmId,
-        HdfsConstants.WARM_STORAGE_POLICY_NAME,
+        HdfsConstants.StoragePolicy.WARM.name(),
         new StorageType[]{StorageType.DISK, StorageType.ARCHIVE},
         new StorageType[]{StorageType.DISK, StorageType.ARCHIVE},
         new StorageType[]{StorageType.DISK, StorageType.ARCHIVE});
-    final byte coldId = HdfsConstants.COLD_STORAGE_POLICY_ID;
+    final byte coldId = HdfsConstants.StoragePolicy.COLD.value();
     policies[coldId] = new BlockStoragePolicy(coldId,
-        HdfsConstants.COLD_STORAGE_POLICY_NAME,
+        HdfsConstants.StoragePolicy.COLD.name(),
         new StorageType[]{StorageType.ARCHIVE}, StorageType.EMPTY_ARRAY,
         StorageType.EMPTY_ARRAY);
-    final byte providedId = HdfsConstants.PROVIDED_STORAGE_POLICY_ID;
+    final byte providedId = HdfsConstants.StoragePolicy.PROVIDED.value();
     policies[providedId] = new BlockStoragePolicy(providedId,
-      HdfsConstants.PROVIDED_STORAGE_POLICY_NAME,
+      HdfsConstants.StoragePolicy.PROVIDED.name(),
       new StorageType[]{StorageType.PROVIDED, StorageType.DISK},
       new StorageType[]{StorageType.PROVIDED, StorageType.DISK},
       new StorageType[]{StorageType.PROVIDED, StorageType.DISK});
-    return new BlockStoragePolicySuite(hotId, policies);
+
+    return new BlockStoragePolicySuite(getDefaultPolicyID(conf, policies),
+        policies);
+  }
+
+  private static byte getDefaultPolicyID(
+      final Configuration conf, final BlockStoragePolicy[] policies) {
+    if (conf != null) {
+      HdfsConstants.StoragePolicy defaultPolicy = conf.getEnum(
+          DFSConfigKeys.DFS_STORAGE_DEFAULT_POLICY,
+          DFSConfigKeys.DFS_STORAGE_DEFAULT_POLICY_DEFAULT);
+      for (BlockStoragePolicy policy : policies) {
+        if (policy != null &&
+            policy.getName().equalsIgnoreCase(defaultPolicy.name())) {
+          return policy.getId();
+        }
+      }
+    }
+    return DFSConfigKeys.DFS_STORAGE_DEFAULT_POLICY_DEFAULT.value();
   }
 
   private final byte defaultPolicyID;

hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml

@@ -5935,4 +5935,20 @@
       set on fs.protected.directories.
     </description>
   </property>
+
+  <property>
+    <name>dfs.storage.default.policy</name>
+    <value>HOT</value>
+    <description>
+        Set the default Storage Policy name with following value,
+        LAZY_PERSIST: memory storage policy.
+        ALL_SSD     : all SSD storage policy.
+        ONE_SSD     : one SSD_storage policy.
+        HOT         : hot storage policy.
+        WARM        : warm policy.
+        COLD        : cold_storage policy.
+        PROVIDED    : provided storage policy.
+    </description>
+  </property>
+
 </configuration>

hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ArchivalStorage.md

@@ -88,6 +88,7 @@ The effective storage policy can be retrieved by the "[`storagepolicies -getStor
 ### Configuration
 
 * **dfs.storage.policy.enabled** - for enabling/disabling the storage policy feature. The default value is `true`.
+* **dfs.storage.default.policy** - Set the default storage policy with the policy name. The default value is `HOT`.  All possible policies are defined in enum StoragePolicy, including `LAZY_PERSIST` `ALL_SSD` `ONE_SSD` `HOT` `WARM` `COLD` and `PROVIDED`.
 * **dfs.datanode.data.dir** - on each data node, the comma-separated storage locations should be tagged with their storage types. This allows storage policies to place the blocks on different storage types according to policy. For example:
 
     1.  A datanode storage location /grid/dn/disk0 on DISK should be configured with `[DISK]file:///grid/dn/disk0`

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java

@@ -1546,4 +1546,41 @@ public class TestBlockStoragePolicy {
       }
     }
   }
+
+  @Test
+  public void testCreateDefaultPoliciesFromConf() {
+    BlockStoragePolicySuite suite =
+        BlockStoragePolicySuite.createDefaultSuite();
+    Assert.assertEquals(HdfsConstants.StoragePolicy.HOT.value(),
+        suite.getDefaultPolicy().getId());
+
+    Configuration newConf = new Configuration();
+    newConf.setEnum(DFSConfigKeys.DFS_STORAGE_DEFAULT_POLICY,
+        HdfsConstants.StoragePolicy.ONE_SSD);
+    BlockStoragePolicySuite suiteConf =
+        BlockStoragePolicySuite.createDefaultSuite(newConf);
+    Assert.assertEquals(HdfsConstants.StoragePolicy.ONE_SSD.value(),
+        suiteConf.getDefaultPolicy().getId());
+  }
+
+  @Test
+  public void testCreateFileWithConfiguredDefaultPolicies()
+      throws IOException{
+    Configuration newConf = new HdfsConfiguration();
+    newConf.set(DFSConfigKeys.DFS_STORAGE_DEFAULT_POLICY,
+        HdfsConstants.StoragePolicy.WARM.name());
+    MiniDFSCluster cluster = new MiniDFSCluster.Builder(newConf)
+        .numDataNodes(0).build();
+    try {
+      cluster.waitActive();
+      final Path fooFile = new Path("/foo");
+      FileSystem newfs = cluster.getFileSystem();
+      DFSTestUtil.createFile(newfs, fooFile, 0, REPLICATION, 0L);
+
+      String policy = newfs.getStoragePolicy(fooFile).getName();
+      Assert.assertEquals(HdfsConstants.StoragePolicy.WARM.name(), policy);
+    } finally {
+      cluster.shutdown();
+    }
+  }
 }