From 8f00897a4be5af42fff36336e40d2178c06ecccc Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Thu, 21 Aug 2014 00:42:27 +0000 Subject: [PATCH] HDFS-6878. Change MiniDFSCluster to support StorageType configuration for individual directories. (Arpit Agarwal) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1619271 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../apache/hadoop/hdfs/MiniDFSCluster.java | 82 +++++++++++-------- .../hdfs/MiniDFSClusterWithNodeGroup.java | 12 +-- .../server/datanode/TestStorageReport.java | 2 +- 4 files changed, 61 insertions(+), 38 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 6bc8e74a858..0221018c46e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -499,6 +499,9 @@ Release 2.6.0 - UNRELEASED HDFS-6858. Allow dfs.data.transfer.saslproperties.resolver.class default to hadoop.security.saslproperties.resolver.class. (Benoy Antony via cnauroth) + HDFS-6878. Change MiniDFSCluster to support StorageType configuration + for individual directories (Arpit Agarwal) + OPTIMIZATIONS HDFS-6690. Deduplicate xattr names in memory. (wang) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java index fe298d33118..fef5ff85227 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java @@ -144,7 +144,7 @@ public static class Builder { private int nameNodeHttpPort = 0; private final Configuration conf; private int numDataNodes = 1; - private StorageType storageType = StorageType.DEFAULT; + private StorageType[][] storageTypes = null; private boolean format = true; private boolean manageNameDfsDirs = true; private boolean manageNameDfsSharedDirs = true; @@ -193,10 +193,26 @@ public Builder numDataNodes(int val) { } /** - * Default: StorageType.DEFAULT + * Set the same storage type configuration for each datanode. + * If storageTypes is uninitialized or passed null then + * StorageType.DEFAULT is used. */ - public Builder storageType(StorageType type) { - this.storageType = type; + public Builder storageTypes(StorageType[] types) { + assert types.length == DIRS_PER_DATANODE; + this.storageTypes = new StorageType[numDataNodes][types.length]; + for (int i = 0; i < numDataNodes; ++i) { + this.storageTypes[i] = types; + } + return this; + } + + /** + * Set custom storage type configuration for each datanode. + * If storageTypes is uninitialized or passed null then + * StorageType.DEFAULT is used. + */ + public Builder storageTypes(StorageType[][] types) { + this.storageTypes = types; return this; } @@ -369,7 +385,8 @@ protected MiniDFSCluster(Builder builder) throws IOException { builder.nnTopology = MiniDFSNNTopology.simpleSingleNN( builder.nameNodePort, builder.nameNodeHttpPort); } - + assert builder.storageTypes == null || + builder.storageTypes.length == builder.numDataNodes; final int numNameNodes = builder.nnTopology.countNameNodes(); LOG.info("starting cluster: numNameNodes=" + numNameNodes + ", numDataNodes=" + builder.numDataNodes); @@ -377,7 +394,7 @@ protected MiniDFSCluster(Builder builder) throws IOException { initMiniDFSCluster(builder.conf, builder.numDataNodes, - builder.storageType, + builder.storageTypes, builder.format, builder.manageNameDfsDirs, builder.manageNameDfsSharedDirs, @@ -477,8 +494,8 @@ public MiniDFSCluster() { * Servers will be started on free ports. *

* The caller must manage the creation of NameNode and DataNode directories - * and have already set {@link #DFS_NAMENODE_NAME_DIR_KEY} and - * {@link #DFS_DATANODE_DATA_DIR_KEY} in the given conf. + * and have already set {@link DFSConfigKeys#DFS_NAMENODE_NAME_DIR_KEY} and + * {@link DFSConfigKeys#DFS_DATANODE_DATA_DIR_KEY} in the given conf. * * @param conf the base configuration to use in starting the servers. This * will be modified as necessary. @@ -554,8 +571,8 @@ public MiniDFSCluster(Configuration conf, * @param format if true, format the NameNode and DataNodes before starting * up * @param manageDfsDirs if true, the data directories for servers will be - * created and {@link #DFS_NAMENODE_NAME_DIR_KEY} and - * {@link #DFS_DATANODE_DATA_DIR_KEY} will be set in + * created and {@link DFSConfigKeys#DFS_NAMENODE_NAME_DIR_KEY} and + * {@link DFSConfigKeys#DFS_DATANODE_DATA_DIR_KEY} will be set in * the conf * @param operation the operation with which to start the servers. If null * or StartupOption.FORMAT, then StartupOption.REGULAR will be used. @@ -586,8 +603,8 @@ public MiniDFSCluster(int nameNodePort, * @param numDataNodes Number of DataNodes to start; may be zero * @param format if true, format the NameNode and DataNodes before starting up * @param manageDfsDirs if true, the data directories for servers will be - * created and {@link #DFS_NAMENODE_NAME_DIR_KEY} and - * {@link #DFS_DATANODE_DATA_DIR_KEY} will be set in + * created and {@link DFSConfigKeys#DFS_NAMENODE_NAME_DIR_KEY} and + * {@link DFSConfigKeys#DFS_DATANODE_DATA_DIR_KEY} will be set in * the conf * @param operation the operation with which to start the servers. If null * or StartupOption.FORMAT, then StartupOption.REGULAR will be used. @@ -620,11 +637,11 @@ public MiniDFSCluster(int nameNodePort, * @param numDataNodes Number of DataNodes to start; may be zero * @param format if true, format the NameNode and DataNodes before starting up * @param manageNameDfsDirs if true, the data directories for servers will be - * created and {@link #DFS_NAMENODE_NAME_DIR_KEY} and - * {@link #DFS_DATANODE_DATA_DIR_KEY} will be set in + * created and {@link DFSConfigKeys#DFS_NAMENODE_NAME_DIR_KEY} and + * {@link DFSConfigKeys#DFS_DATANODE_DATA_DIR_KEY} will be set in * the conf * @param manageDataDfsDirs if true, the data directories for datanodes will - * be created and {@link #DFS_DATANODE_DATA_DIR_KEY} + * be created and {@link DFSConfigKeys#DFS_DATANODE_DATA_DIR_KEY} * set to same in the conf * @param operation the operation with which to start the servers. If null * or StartupOption.FORMAT, then StartupOption.REGULAR will be used. @@ -643,7 +660,7 @@ public MiniDFSCluster(int nameNodePort, String[] racks, String hosts[], long[] simulatedCapacities) throws IOException { this.nameNodes = new NameNodeInfo[1]; // Single namenode in the cluster - initMiniDFSCluster(conf, numDataNodes, StorageType.DEFAULT, format, + initMiniDFSCluster(conf, numDataNodes, null, format, manageNameDfsDirs, true, manageDataDfsDirs, manageDataDfsDirs, operation, null, racks, hosts, simulatedCapacities, null, true, false, @@ -652,7 +669,7 @@ public MiniDFSCluster(int nameNodePort, private void initMiniDFSCluster( Configuration conf, - int numDataNodes, StorageType storageType, boolean format, boolean manageNameDfsDirs, + int numDataNodes, StorageType[][] storageTypes, boolean format, boolean manageNameDfsDirs, boolean manageNameDfsSharedDirs, boolean enableManagedDfsDirsRedundancy, boolean manageDataDfsDirs, StartupOption startOpt, StartupOption dnStartOpt, String[] racks, @@ -725,7 +742,7 @@ private void initMiniDFSCluster( } // Start the DataNodes - startDataNodes(conf, numDataNodes, storageType, manageDataDfsDirs, + startDataNodes(conf, numDataNodes, storageTypes, manageDataDfsDirs, dnStartOpt != null ? dnStartOpt : startOpt, racks, hosts, simulatedCapacities, setupHostsFile, checkDataNodeAddrConfig, checkDataNodeHostConfig, dnConfOverlays); @@ -1100,15 +1117,18 @@ public void waitClusterUp() throws IOException { } } - String makeDataNodeDirs(int dnIndex, StorageType storageType) throws IOException { + String makeDataNodeDirs(int dnIndex, StorageType[] storageTypes) throws IOException { StringBuilder sb = new StringBuilder(); + assert storageTypes == null || storageTypes.length == DIRS_PER_DATANODE; for (int j = 0; j < DIRS_PER_DATANODE; ++j) { File dir = getInstanceStorageDir(dnIndex, j); dir.mkdirs(); if (!dir.isDirectory()) { throw new IOException("Mkdirs failed to create directory for DataNode " + dir); } - sb.append((j > 0 ? "," : "") + "[" + storageType + "]" + fileAsURI(dir)); + sb.append((j > 0 ? "," : "") + "[" + + (storageTypes == null ? StorageType.DEFAULT : storageTypes[j]) + + "]" + fileAsURI(dir)); } return sb.toString(); } @@ -1127,7 +1147,7 @@ String makeDataNodeDirs(int dnIndex, StorageType storageType) throws IOException * will be modified as necessary. * @param numDataNodes Number of DataNodes to start; may be zero * @param manageDfsDirs if true, the data directories for DataNodes will be - * created and {@link #DFS_DATANODE_DATA_DIR_KEY} will be set + * created and {@link DFSConfigKeys#DFS_DATANODE_DATA_DIR_KEY} will be set * in the conf * @param operation the operation with which to start the DataNodes. If null * or StartupOption.FORMAT, then StartupOption.REGULAR will be used. @@ -1159,7 +1179,7 @@ public synchronized void startDataNodes(Configuration conf, int numDataNodes, * will be modified as necessary. * @param numDataNodes Number of DataNodes to start; may be zero * @param manageDfsDirs if true, the data directories for DataNodes will be - * created and {@link #DFS_DATANODE_DATA_DIR_KEY} will be + * created and {@link DFSConfigKeys#DFS_DATANODE_DATA_DIR_KEY} will be * set in the conf * @param operation the operation with which to start the DataNodes. If null * or StartupOption.FORMAT, then StartupOption.REGULAR will be used. @@ -1175,21 +1195,17 @@ public synchronized void startDataNodes(Configuration conf, int numDataNodes, String[] racks, String[] hosts, long[] simulatedCapacities, boolean setupHostsFile) throws IOException { - startDataNodes(conf, numDataNodes, StorageType.DEFAULT, manageDfsDirs, operation, racks, hosts, + startDataNodes(conf, numDataNodes, null, manageDfsDirs, operation, racks, hosts, simulatedCapacities, setupHostsFile, false, false, null); } - /** - * @see MiniDFSCluster#startDataNodes(Configuration, int, boolean, StartupOption, - * String[], String[], long[], boolean, boolean, boolean) - */ public synchronized void startDataNodes(Configuration conf, int numDataNodes, boolean manageDfsDirs, StartupOption operation, String[] racks, String[] hosts, long[] simulatedCapacities, boolean setupHostsFile, boolean checkDataNodeAddrConfig) throws IOException { - startDataNodes(conf, numDataNodes, StorageType.DEFAULT, manageDfsDirs, operation, racks, hosts, + startDataNodes(conf, numDataNodes, null, manageDfsDirs, operation, racks, hosts, simulatedCapacities, setupHostsFile, checkDataNodeAddrConfig, false, null); } @@ -1207,7 +1223,7 @@ public synchronized void startDataNodes(Configuration conf, int numDataNodes, * will be modified as necessary. * @param numDataNodes Number of DataNodes to start; may be zero * @param manageDfsDirs if true, the data directories for DataNodes will be - * created and {@link #DFS_DATANODE_DATA_DIR_KEY} will be + * created and {@link DFSConfigKeys#DFS_DATANODE_DATA_DIR_KEY} will be * set in the conf * @param operation the operation with which to start the DataNodes. If null * or StartupOption.FORMAT, then StartupOption.REGULAR will be used. @@ -1222,13 +1238,15 @@ public synchronized void startDataNodes(Configuration conf, int numDataNodes, * @throws IllegalStateException if NameNode has been shutdown */ public synchronized void startDataNodes(Configuration conf, int numDataNodes, - StorageType storageType, boolean manageDfsDirs, StartupOption operation, + StorageType[][] storageTypes, boolean manageDfsDirs, StartupOption operation, String[] racks, String[] hosts, long[] simulatedCapacities, boolean setupHostsFile, boolean checkDataNodeAddrConfig, boolean checkDataNodeHostConfig, Configuration[] dnConfOverlays) throws IOException { + assert storageTypes == null || storageTypes.length == numDataNodes; + if (operation == StartupOption.RECOVER) { return; } @@ -1289,7 +1307,7 @@ public synchronized void startDataNodes(Configuration conf, int numDataNodes, // Set up datanode address setupDatanodeAddress(dnConf, setupHostsFile, checkDataNodeAddrConfig); if (manageDfsDirs) { - String dirs = makeDataNodeDirs(i, storageType); + String dirs = makeDataNodeDirs(i, storageTypes == null ? null : storageTypes[i]); dnConf.set(DFS_DATANODE_DATA_DIR_KEY, dirs); conf.set(DFS_DATANODE_DATA_DIR_KEY, dirs); } @@ -2173,7 +2191,7 @@ public void injectBlocks(int dataNodeIndex, } /** - * Multiple-NameNode version of {@link #injectBlocks(Iterable[])}. + * Multiple-NameNode version of injectBlocks. */ public void injectBlocks(int nameNodeIndex, int dataNodeIndex, Iterable blocksToInject) throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSClusterWithNodeGroup.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSClusterWithNodeGroup.java index d06a6a04519..382bf368213 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSClusterWithNodeGroup.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSClusterWithNodeGroup.java @@ -50,12 +50,14 @@ public static void setNodeGroups (String[] nodeGroups) { } public synchronized void startDataNodes(Configuration conf, int numDataNodes, - StorageType storageType, boolean manageDfsDirs, StartupOption operation, + StorageType[][] storageTypes, boolean manageDfsDirs, StartupOption operation, String[] racks, String[] nodeGroups, String[] hosts, long[] simulatedCapacities, boolean setupHostsFile, boolean checkDataNodeAddrConfig, boolean checkDataNodeHostConfig) throws IOException { + assert storageTypes == null || storageTypes.length == numDataNodes; + if (operation == StartupOption.RECOVER) { return; } @@ -112,7 +114,7 @@ public synchronized void startDataNodes(Configuration conf, int numDataNodes, // Set up datanode address setupDatanodeAddress(dnConf, setupHostsFile, checkDataNodeAddrConfig); if (manageDfsDirs) { - String dirs = makeDataNodeDirs(i, storageType); + String dirs = makeDataNodeDirs(i, storageTypes == null ? null : storageTypes[i]); dnConf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, dirs); conf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, dirs); } @@ -190,7 +192,7 @@ public synchronized void startDataNodes(Configuration conf, int numDataNodes, String[] racks, String[] nodeGroups, String[] hosts, long[] simulatedCapacities, boolean setupHostsFile) throws IOException { - startDataNodes(conf, numDataNodes, StorageType.DEFAULT, manageDfsDirs, operation, racks, nodeGroups, + startDataNodes(conf, numDataNodes, null, manageDfsDirs, operation, racks, nodeGroups, hosts, simulatedCapacities, setupHostsFile, false, false); } @@ -205,14 +207,14 @@ public void startDataNodes(Configuration conf, int numDataNodes, // This is for initialize from parent class. @Override public synchronized void startDataNodes(Configuration conf, int numDataNodes, - StorageType storageType, boolean manageDfsDirs, StartupOption operation, + StorageType[][] storageTypes, boolean manageDfsDirs, StartupOption operation, String[] racks, String[] hosts, long[] simulatedCapacities, boolean setupHostsFile, boolean checkDataNodeAddrConfig, boolean checkDataNodeHostConfig, Configuration[] dnConfOverlays) throws IOException { - startDataNodes(conf, numDataNodes, storageType, manageDfsDirs, operation, racks, + startDataNodes(conf, numDataNodes, storageTypes, manageDfsDirs, operation, racks, NODE_GROUPS, hosts, simulatedCapacities, setupHostsFile, checkDataNodeAddrConfig, checkDataNodeHostConfig); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestStorageReport.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestStorageReport.java index 85afa274a88..b6b3fe69b09 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestStorageReport.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestStorageReport.java @@ -58,7 +58,7 @@ public void startUpCluster() throws IOException { conf = new HdfsConfiguration(); cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(REPL_FACTOR) - .storageType(storageType) + .storageTypes(new StorageType[] { storageType, storageType } ) .build(); fs = cluster.getFileSystem(); bpid = cluster.getNamesystem().getBlockPoolId();