diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 7acd64ae118..e4254cef608 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -25,6 +25,9 @@ Release 0.23.2 - UNRELEASED HDFS-2725. hdfs script usage information is missing the information about "dfs" command (Prashant Sharma via stevel) + HDFS-2907. Add a conf property dfs.datanode.fsdataset.factory to make + FSDataset in Datanode pluggable. (szetszwo) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index aef13d4ac34..3245aeac4c4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -167,7 +167,6 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final String DFS_CLIENT_RETRY_WINDOW_BASE= "dfs.client.retry.window.base"; public static final String DFS_METRICS_SESSION_ID_KEY = "dfs.metrics.session-id"; public static final String DFS_DATANODE_HOST_NAME_KEY = "dfs.datanode.hostname"; - public static final String DFS_DATANODE_STORAGEID_KEY = "dfs.datanode.StorageId"; public static final String DFS_NAMENODE_HOSTS_KEY = "dfs.namenode.hosts"; public static final String DFS_NAMENODE_HOSTS_EXCLUDE_KEY = "dfs.namenode.hosts.exclude"; public static final String DFS_CLIENT_SOCKET_TIMEOUT_KEY = "dfs.client.socket-timeout"; @@ -211,10 +210,6 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final int DFS_DATANODE_NUMBLOCKS_DEFAULT = 64; public static final String DFS_DATANODE_SCAN_PERIOD_HOURS_KEY = "dfs.datanode.scan.period.hours"; public static final int DFS_DATANODE_SCAN_PERIOD_HOURS_DEFAULT = 0; - public static final String DFS_DATANODE_SIMULATEDDATASTORAGE_KEY = "dfs.datanode.simulateddatastorage"; - public static final boolean DFS_DATANODE_SIMULATEDDATASTORAGE_DEFAULT = false; - public static final String DFS_DATANODE_SIMULATEDDATASTORAGE_CAPACITY_KEY = "dfs.datanode.simulateddatastorage.capacity"; - public static final long DFS_DATANODE_SIMULATEDDATASTORAGE_CAPACITY_DEFAULT = 2L<<40; public static final String DFS_DATANODE_TRANSFERTO_ALLOWED_KEY = "dfs.datanode.transferTo.allowed"; public static final boolean DFS_DATANODE_TRANSFERTO_ALLOWED_DEFAULT = true; public static final String DFS_DATANODE_BLOCKVOLUMECHOICEPOLICY = "dfs.datanode.block.volume.choice.policy"; @@ -282,6 +277,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys { //Keys with no defaults public static final String DFS_DATANODE_PLUGINS_KEY = "dfs.datanode.plugins"; + public static final String DFS_DATANODE_FSDATASET_FACTORY_KEY = "dfs.datanode.fsdataset.factory"; public static final String DFS_DATANODE_SOCKET_WRITE_TIMEOUT_KEY = "dfs.datanode.socket.write.timeout"; public static final String DFS_DATANODE_STARTUP_KEY = "dfs.datanode.startup"; public static final String DFS_NAMENODE_PLUGINS_KEY = "dfs.namenode.plugins"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HdfsConfiguration.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HdfsConfiguration.java index 44533f13a12..75ce9118a9a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HdfsConfiguration.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HdfsConfiguration.java @@ -88,7 +88,6 @@ private static void addDeprecatedKeys() { deprecate("fs.checkpoint.period", DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_PERIOD_KEY); deprecate("dfs.upgrade.permission", DFSConfigKeys.DFS_NAMENODE_UPGRADE_PERMISSION_KEY); deprecate("heartbeat.recheck.interval", DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY); - deprecate("StorageId", DFSConfigKeys.DFS_DATANODE_STORAGEID_KEY); deprecate("dfs.https.client.keystore.resource", DFSConfigKeys.DFS_CLIENT_HTTPS_KEYSTORE_RESOURCE_KEY); deprecate("dfs.https.need.client.auth", DFSConfigKeys.DFS_CLIENT_HTTPS_NEED_AUTH_KEY); deprecate("slave.host.name", DFSConfigKeys.DFS_DATANODE_HOST_NAME_KEY); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index 8f9cc25de11..888a0363ed1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -43,10 +43,7 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_PLUGINS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SCAN_PERIOD_HOURS_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SCAN_PERIOD_HOURS_KEY; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SIMULATEDDATASTORAGE_DEFAULT; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SIMULATEDDATASTORAGE_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_STARTUP_KEY; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_STORAGEID_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_USER_NAME_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_FEDERATION_NAMESERVICES; @@ -152,13 +149,11 @@ import org.apache.hadoop.util.DiskChecker.DiskErrorException; import org.apache.hadoop.util.DiskChecker.DiskOutOfSpaceException; import org.apache.hadoop.util.GenericOptionsParser; -import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.ServicePlugin; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.VersionInfo; import org.mortbay.util.ajax.JSON; -import com.google.common.base.Preconditions; /********************************************************** @@ -427,13 +422,14 @@ void refreshNamenodes(Configuration conf) } } - private synchronized void setClusterId(String cid) throws IOException { - if(clusterId != null && !clusterId.equals(cid)) { - throw new IOException ("cluster id doesn't match. old cid=" + clusterId - + " new cid="+ cid); + private synchronized void setClusterId(final String nsCid, final String bpid + ) throws IOException { + if(clusterId != null && !clusterId.equals(nsCid)) { + throw new IOException ("Cluster IDs not matched: dn cid=" + clusterId + + " but ns cid="+ nsCid + "; bpid=" + bpid); } // else - clusterId = cid; + clusterId = nsCid; } private static String getHostName(Configuration config) @@ -810,51 +806,22 @@ void shutdownBlockPool(BPOfferService bpos) { */ void initBlockPool(BPOfferService bpos) throws IOException { NamespaceInfo nsInfo = bpos.getNamespaceInfo(); - Preconditions.checkState(nsInfo != null, - "Block pool " + bpos + " should have retrieved " + - "its namespace info before calling initBlockPool."); + if (nsInfo == null) { + throw new IOException("NamespaceInfo not found: Block pool " + bpos + + " should have retrieved namespace info before initBlockPool."); + } - String blockPoolId = nsInfo.getBlockPoolID(); - // Register the new block pool with the BP manager. blockPoolManager.addBlockPool(bpos); - synchronized (this) { - // we do not allow namenode from different cluster to register - if(clusterId != null && !clusterId.equals(nsInfo.clusterID)) { - throw new IOException( - "cannot register with the namenode because clusterid do not match:" - + " nn=" + nsInfo.getBlockPoolID() + "; nn cid=" + nsInfo.clusterID + - ";dn cid=" + clusterId); - } - - setClusterId(nsInfo.clusterID); - } - - StartupOption startOpt = getStartupOption(conf); - assert startOpt != null : "Startup option must be set."; - - boolean simulatedFSDataset = conf.getBoolean( - DFS_DATANODE_SIMULATEDDATASTORAGE_KEY, - DFS_DATANODE_SIMULATEDDATASTORAGE_DEFAULT); - - if (!simulatedFSDataset) { - // read storage info, lock data dirs and transition fs state if necessary - storage.recoverTransitionRead(DataNode.this, blockPoolId, nsInfo, - dataDirs, startOpt); - StorageInfo bpStorage = storage.getBPStorage(blockPoolId); - LOG.info("setting up storage: nsid=" + - bpStorage.getNamespaceID() + ";bpid=" - + blockPoolId + ";lv=" + storage.getLayoutVersion() + - ";nsInfo=" + nsInfo); - } + setClusterId(nsInfo.clusterID, nsInfo.getBlockPoolID()); // In the case that this is the first block pool to connect, initialize // the dataset, block scanners, etc. - initFsDataSet(); + initStorage(nsInfo); initPeriodicScanners(conf); - data.addBlockPool(blockPoolId, conf); + data.addBlockPool(nsInfo.getBlockPoolID(), conf); } /** @@ -881,31 +848,28 @@ int getBpOsCount() { * Initializes the {@link #data}. The initialization is done only once, when * handshake with the the first namenode is completed. */ - private synchronized void initFsDataSet() throws IOException { - if (data != null) { // Already initialized - return; + private void initStorage(final NamespaceInfo nsInfo) throws IOException { + final FSDatasetInterface.Factory factory + = FSDatasetInterface.Factory.getFactory(conf); + + if (!factory.isSimulated()) { + final StartupOption startOpt = getStartupOption(conf); + if (startOpt == null) { + throw new IOException("Startup option not set."); + } + final String bpid = nsInfo.getBlockPoolID(); + //read storage info, lock data dirs and transition fs state if necessary + storage.recoverTransitionRead(this, bpid, nsInfo, dataDirs, startOpt); + final StorageInfo bpStorage = storage.getBPStorage(bpid); + LOG.info("Setting up storage: nsid=" + bpStorage.getNamespaceID() + + ";bpid=" + bpid + ";lv=" + storage.getLayoutVersion() + + ";nsInfo=" + nsInfo); } - // get version and id info from the name-node - boolean simulatedFSDataset = conf.getBoolean( - DFS_DATANODE_SIMULATEDDATASTORAGE_KEY, - DFS_DATANODE_SIMULATEDDATASTORAGE_DEFAULT); - - if (simulatedFSDataset) { - storage.createStorageID(getPort()); - // it would have been better to pass storage as a parameter to - // constructor below - need to augment ReflectionUtils used below. - conf.set(DFS_DATANODE_STORAGEID_KEY, getStorageId()); - try { - data = (FSDatasetInterface) ReflectionUtils.newInstance( - Class.forName( - "org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset"), - conf); - } catch (ClassNotFoundException e) { - throw new IOException(StringUtils.stringifyException(e)); + synchronized(this) { + if (data == null) { + data = factory.createFSDatasetInterface(this, storage, conf); } - } else { - data = new FSDataset(this, storage, conf); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FSDataset.java index 8f649bcef5f..5927217c1d0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FSDataset.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FSDataset.java @@ -76,6 +76,16 @@ ***************************************************/ @InterfaceAudience.Private class FSDataset implements FSDatasetInterface { + /** + * A factory for creating FSDataset objects. + */ + static class Factory extends FSDatasetInterface.Factory { + @Override + public FSDatasetInterface createFSDatasetInterface(DataNode datanode, + DataStorage storage, Configuration conf) throws IOException { + return new FSDataset(datanode, storage, conf); + } + } /** * A node type that can be built into a tree reflecting the @@ -1057,8 +1067,8 @@ public MetaDataInputStream getMetaDataInputStream(ExtendedBlock b) /** * An FSDataset has a directory where it loads its data files. */ - FSDataset(DataNode datanode, DataStorage storage, Configuration conf) - throws IOException { + private FSDataset(DataNode datanode, DataStorage storage, Configuration conf + ) throws IOException { this.datanode = datanode; this.maxBlocksPerDir = conf.getInt(DFSConfigKeys.DFS_DATANODE_NUMBLOCKS_KEY, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FSDatasetInterface.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FSDatasetInterface.java index ff3974f627a..48618cd2611 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FSDatasetInterface.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FSDatasetInterface.java @@ -29,6 +29,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; import org.apache.hadoop.hdfs.protocol.BlockLocalPathInfo; @@ -38,6 +39,7 @@ import org.apache.hadoop.hdfs.server.protocol.ReplicaRecoveryInfo; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.util.DataChecksum; +import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.DiskChecker.DiskErrorException; /** @@ -49,6 +51,30 @@ */ @InterfaceAudience.Private public interface FSDatasetInterface extends FSDatasetMBean { + /** + * A factory for creating FSDatasetInterface objects. + */ + public abstract class Factory { + /** @return the configured factory. */ + public static Factory getFactory(Configuration conf) { + final Class clazz = conf.getClass( + DFSConfigKeys.DFS_DATANODE_FSDATASET_FACTORY_KEY, + FSDataset.Factory.class, + Factory.class); + return ReflectionUtils.newInstance(clazz, conf); + } + + /** Create a FSDatasetInterface object. */ + public abstract FSDatasetInterface createFSDatasetInterface( + DataNode datanode, DataStorage storage, Configuration conf + ) throws IOException; + + /** Does the factory create simulated objects? */ + public boolean isSimulated() { + return false; + } + } + /** * This is an interface for the underlying volume. * @see org.apache.hadoop.hdfs.server.datanode.FSDataset.FSVolume diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DataNodeCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DataNodeCluster.java index f3350b988a7..25198e36925 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DataNodeCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DataNodeCluster.java @@ -25,6 +25,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; +import org.apache.hadoop.hdfs.server.datanode.FSDatasetInterface; import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset; import org.apache.hadoop.hdfs.server.namenode.CreateEditsLog; import org.apache.hadoop.net.DNS; @@ -122,10 +123,9 @@ public static void main(String[] args) { } dataNodeDirs = args[i]; } else if (args[i].equals("-simulated")) { - conf.setBoolean(SimulatedFSDataset.CONFIG_PROPERTY_SIMULATED, true); + SimulatedFSDataset.setFactory(conf); } else if (args[i].equals("-inject")) { - if (!conf.getBoolean(SimulatedFSDataset.CONFIG_PROPERTY_SIMULATED, - false) ) { + if (!FSDatasetInterface.Factory.getFactory(conf).isSimulated()) { System.out.print("-inject is valid only for simulated"); printUsageExit(); } @@ -158,7 +158,7 @@ public static void main(String[] args) { System.exit(-1); } boolean simulated = - conf.getBoolean(SimulatedFSDataset.CONFIG_PROPERTY_SIMULATED, false); + FSDatasetInterface.Factory.getFactory(conf).isSimulated(); System.out.println("Starting " + numDataNodes + (simulated ? " Simulated " : " ") + " Data Nodes that will connect to Name Node at " + nameNodeAdr); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java index d954690b55e..66a4f94d97c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java @@ -875,7 +875,7 @@ public synchronized void startDataNodes(Configuration conf, int numDataNodes, conf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, dirs); } if (simulatedCapacities != null) { - dnConf.setBoolean(SimulatedFSDataset.CONFIG_PROPERTY_SIMULATED, true); + SimulatedFSDataset.setFactory(dnConf); dnConf.setLong(SimulatedFSDataset.CONFIG_PROPERTY_CAPACITY, simulatedCapacities[i-curDatanodesNum]); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend.java index d8b58f7cb6c..20f28376a8e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend.java @@ -107,7 +107,7 @@ private void checkFile(FileSystem fileSys, Path name, int repl) public void testCopyOnWrite() throws IOException { Configuration conf = new HdfsConfiguration(); if (simulatedStorage) { - conf.setBoolean(SimulatedFSDataset.CONFIG_PROPERTY_SIMULATED, true); + SimulatedFSDataset.setFactory(conf); } MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); FileSystem fs = cluster.getFileSystem(); @@ -178,7 +178,7 @@ public void testCopyOnWrite() throws IOException { public void testSimpleFlush() throws IOException { Configuration conf = new HdfsConfiguration(); if (simulatedStorage) { - conf.setBoolean(SimulatedFSDataset.CONFIG_PROPERTY_SIMULATED, true); + SimulatedFSDataset.setFactory(conf); } fileContents = AppendTestUtil.initBuffer(AppendTestUtil.FILE_SIZE); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); @@ -234,7 +234,7 @@ public void testSimpleFlush() throws IOException { public void testComplexFlush() throws IOException { Configuration conf = new HdfsConfiguration(); if (simulatedStorage) { - conf.setBoolean(SimulatedFSDataset.CONFIG_PROPERTY_SIMULATED, true); + SimulatedFSDataset.setFactory(conf); } fileContents = AppendTestUtil.initBuffer(AppendTestUtil.FILE_SIZE); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); @@ -283,7 +283,7 @@ public void testComplexFlush() throws IOException { public void testFileNotFound() throws IOException { Configuration conf = new HdfsConfiguration(); if (simulatedStorage) { - conf.setBoolean(SimulatedFSDataset.CONFIG_PROPERTY_SIMULATED, true); + SimulatedFSDataset.setFactory(conf); } MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); FileSystem fs = cluster.getFileSystem(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend2.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend2.java index c63c4ecc4ce..af27e00820d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend2.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend2.java @@ -82,7 +82,7 @@ public class TestFileAppend2 extends TestCase { public void testSimpleAppend() throws IOException { final Configuration conf = new HdfsConfiguration(); if (simulatedStorage) { - conf.setBoolean(SimulatedFSDataset.CONFIG_PROPERTY_SIMULATED, true); + SimulatedFSDataset.setFactory(conf); } conf.setInt(DFSConfigKeys.DFS_DATANODE_HANDLER_COUNT_KEY, 50); conf.setBoolean("dfs.support.append", true); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend4.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend4.java index 1ba56d3844e..ab60c4c2210 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend4.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend4.java @@ -77,7 +77,7 @@ public class TestFileAppend4 { public void setUp() throws Exception { this.conf = new Configuration(); if (simulatedStorage) { - conf.setBoolean(SimulatedFSDataset.CONFIG_PROPERTY_SIMULATED, true); + SimulatedFSDataset.setFactory(conf); } conf.setBoolean(DFSConfigKeys.DFS_SUPPORT_APPEND_KEY, true); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java index ed1508a2680..9fc7e78dbab 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java @@ -144,7 +144,7 @@ public void testServerDefaults() throws IOException { public void testFileCreation() throws IOException { Configuration conf = new HdfsConfiguration(); if (simulatedStorage) { - conf.setBoolean(SimulatedFSDataset.CONFIG_PROPERTY_SIMULATED, true); + SimulatedFSDataset.setFactory(conf); } MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); FileSystem fs = cluster.getFileSystem(); @@ -223,7 +223,7 @@ public void testFileCreation() throws IOException { public void testDeleteOnExit() throws IOException { Configuration conf = new HdfsConfiguration(); if (simulatedStorage) { - conf.setBoolean(SimulatedFSDataset.CONFIG_PROPERTY_SIMULATED, true); + SimulatedFSDataset.setFactory(conf); } MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); FileSystem fs = cluster.getFileSystem(); @@ -287,7 +287,7 @@ public void testFileCreationError1() throws IOException { conf.setInt(DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 1000); conf.setInt(DFS_HEARTBEAT_INTERVAL_KEY, 1); if (simulatedStorage) { - conf.setBoolean(SimulatedFSDataset.CONFIG_PROPERTY_SIMULATED, true); + SimulatedFSDataset.setFactory(conf); } // create cluster MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); @@ -361,7 +361,7 @@ public void testFileCreationError2() throws IOException { conf.setInt(DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 1000); conf.setInt(DFS_HEARTBEAT_INTERVAL_KEY, 1); if (simulatedStorage) { - conf.setBoolean(SimulatedFSDataset.CONFIG_PROPERTY_SIMULATED, true); + SimulatedFSDataset.setFactory(conf); } // create cluster MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); @@ -460,7 +460,7 @@ public void xxxtestFileCreationNamenodeRestart() throws IOException { conf.setInt(DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 1000); conf.setInt(DFS_HEARTBEAT_INTERVAL_KEY, 1); if (simulatedStorage) { - conf.setBoolean(SimulatedFSDataset.CONFIG_PROPERTY_SIMULATED, true); + SimulatedFSDataset.setFactory(conf); } // create cluster @@ -599,7 +599,7 @@ public void testDFSClientDeath() throws IOException, InterruptedException { Configuration conf = new HdfsConfiguration(); System.out.println("Testing adbornal client death."); if (simulatedStorage) { - conf.setBoolean(SimulatedFSDataset.CONFIG_PROPERTY_SIMULATED, true); + SimulatedFSDataset.setFactory(conf); } MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); FileSystem fs = cluster.getFileSystem(); @@ -634,7 +634,7 @@ public void testDFSClientDeath() throws IOException, InterruptedException { public void testFileCreationNonRecursive() throws IOException { Configuration conf = new HdfsConfiguration(); if (simulatedStorage) { - conf.setBoolean(SimulatedFSDataset.CONFIG_PROPERTY_SIMULATED, true); + SimulatedFSDataset.setFactory(conf); } MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); FileSystem fs = cluster.getFileSystem(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestInjectionForSimulatedStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestInjectionForSimulatedStorage.java index 9491cd13a77..f2de8d805fd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestInjectionForSimulatedStorage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestInjectionForSimulatedStorage.java @@ -136,7 +136,7 @@ public void testInjection() throws IOException { Configuration conf = new HdfsConfiguration(); conf.set(DFSConfigKeys.DFS_REPLICATION_KEY, Integer.toString(numDataNodes)); conf.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, checksumSize); - conf.setBoolean(SimulatedFSDataset.CONFIG_PROPERTY_SIMULATED, true); + SimulatedFSDataset.setFactory(conf); //first time format cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDataNodes).build(); cluster.waitActive(); @@ -159,7 +159,7 @@ public void testInjection() throws IOException { LOG.info("Restarting minicluster"); conf = new HdfsConfiguration(); - conf.setBoolean(SimulatedFSDataset.CONFIG_PROPERTY_SIMULATED, true); + SimulatedFSDataset.setFactory(conf); conf.set(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_THRESHOLD_PCT_KEY, "0.0f"); cluster = new MiniDFSCluster.Builder(conf) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLargeBlock.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLargeBlock.java index 6f2af4043c1..3a6d46cd7a2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLargeBlock.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLargeBlock.java @@ -174,7 +174,7 @@ public void runTest(final long blockSize) throws IOException { Configuration conf = new Configuration(); if (simulatedStorage) { - conf.setBoolean(SimulatedFSDataset.CONFIG_PROPERTY_SIMULATED, true); + SimulatedFSDataset.setFactory(conf); } MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); FileSystem fs = cluster.getFileSystem(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPread.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPread.java index e9de0ad0193..d0ab5afb6cc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPread.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPread.java @@ -206,7 +206,7 @@ private void dfsPreadTest(boolean disableTransferTo) throws IOException { conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 4096); conf.setLong(DFSConfigKeys.DFS_CLIENT_READ_PREFETCH_SIZE_KEY, 4096); if (simulatedStorage) { - conf.setBoolean("dfs.datanode.simulateddatastorage", true); + SimulatedFSDataset.setFactory(conf); } if (disableTransferTo) { conf.setBoolean("dfs.datanode.transferTo.allowed", false); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java index eef83e4174d..066c8c79a53 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java @@ -199,7 +199,7 @@ public void runReplication(boolean simulated) throws IOException { Configuration conf = new HdfsConfiguration(); conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY, false); if (simulated) { - conf.setBoolean(SimulatedFSDataset.CONFIG_PROPERTY_SIMULATED, true); + SimulatedFSDataset.setFactory(conf); } MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(numDatanodes) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSetrepIncreasing.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSetrepIncreasing.java index 0d3a0399e91..29c1aa221e0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSetrepIncreasing.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSetrepIncreasing.java @@ -28,7 +28,7 @@ public class TestSetrepIncreasing extends TestCase { static void setrep(int fromREP, int toREP, boolean simulatedStorage) throws IOException { Configuration conf = new HdfsConfiguration(); if (simulatedStorage) { - conf.setBoolean(SimulatedFSDataset.CONFIG_PROPERTY_SIMULATED, true); + SimulatedFSDataset.setFactory(conf); } conf.set(DFSConfigKeys.DFS_REPLICATION_KEY, "" + fromREP); conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 1000L); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestShortCircuitLocalRead.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestShortCircuitLocalRead.java index 34ed50a9cdd..eb2a1d80ac3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestShortCircuitLocalRead.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestShortCircuitLocalRead.java @@ -124,7 +124,7 @@ public void doTestShortCircuitRead(boolean ignoreChecksum, int size, conf.set(DFSConfigKeys.DFS_BLOCK_LOCAL_PATH_ACCESS_USER_KEY, UserGroupInformation.getCurrentUser().getShortUserName()); if (simulatedStorage) { - conf.setBoolean(SimulatedFSDataset.CONFIG_PROPERTY_SIMULATED, true); + SimulatedFSDataset.setFactory(conf); } MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1) .format(true).build(); @@ -248,7 +248,7 @@ public void testSkipWithVerifyChecksum() throws IOException { conf.set(DFSConfigKeys.DFS_BLOCK_LOCAL_PATH_ACCESS_USER_KEY, UserGroupInformation.getCurrentUser().getShortUserName()); if (simulatedStorage) { - conf.setBoolean(SimulatedFSDataset.CONFIG_PROPERTY_SIMULATED, true); + SimulatedFSDataset.setFactory(conf); } MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1) .format(true).build(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSmallBlock.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSmallBlock.java index 8fb2b7a38cf..efcb74ad1e5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSmallBlock.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSmallBlock.java @@ -92,7 +92,7 @@ private void cleanupFile(FileSystem fileSys, Path name) throws IOException { public void testSmallBlock() throws IOException { Configuration conf = new HdfsConfiguration(); if (simulatedStorage) { - conf.setBoolean("dfs.datanode.simulateddatastorage", true); + SimulatedFSDataset.setFactory(conf); } conf.set(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, "1"); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java index 34cd784bd04..eb567469ab9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java @@ -77,7 +77,7 @@ static void initConf(Configuration conf) { conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, DEFAULT_BLOCK_SIZE); conf.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, DEFAULT_BLOCK_SIZE); conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L); - conf.setBoolean(SimulatedFSDataset.CONFIG_PROPERTY_SIMULATED, true); + SimulatedFSDataset.setFactory(conf); conf.setLong(DFSConfigKeys.DFS_BALANCER_MOVEDWINWIDTH_KEY, 2000L); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java index c9cecd11b9c..3e7c5199636 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java @@ -22,7 +22,6 @@ import java.io.InputStream; import java.io.OutputStream; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -32,7 +31,6 @@ import javax.management.ObjectName; import javax.management.StandardMBean; -import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.protocol.Block; @@ -63,21 +61,33 @@ * * Note the synchronization is coarse grained - it is at each method. */ +public class SimulatedFSDataset implements FSDatasetInterface { + static class Factory extends FSDatasetInterface.Factory { + @Override + public FSDatasetInterface createFSDatasetInterface(DataNode datanode, + DataStorage storage, Configuration conf) throws IOException { + return new SimulatedFSDataset(datanode, storage, conf); + } -public class SimulatedFSDataset implements FSDatasetInterface, Configurable{ + @Override + public boolean isSimulated() { + return true; + } + } + + public static void setFactory(Configuration conf) { + conf.set(DFSConfigKeys.DFS_DATANODE_FSDATASET_FACTORY_KEY, + Factory.class.getName()); + } - public static final String CONFIG_PROPERTY_SIMULATED = - "dfs.datanode.simulateddatastorage"; public static final String CONFIG_PROPERTY_CAPACITY = - "dfs.datanode.simulateddatastorage.capacity"; + "dfs.datanode.simulateddatastorage.capacity"; public static final long DEFAULT_CAPACITY = 2L<<40; // 1 terabyte - public static final byte DEFAULT_DATABYTE = 9; // 1 terabyte - byte simulatedDataByte = DEFAULT_DATABYTE; - Configuration conf = null; + public static final byte DEFAULT_DATABYTE = 9; - static byte[] nullCrcFileData; - { + static final byte[] nullCrcFileData; + static { DataChecksum checksum = DataChecksum.newDataChecksum( DataChecksum. CHECKSUM_NULL, 16*1024 ); byte[] nullCrcHeader = checksum.getHeader(); @@ -360,31 +370,22 @@ private SimulatedBPStorage getBPStorage(String bpid) throws IOException { } } - private Map> blockMap = null; - private SimulatedStorage storage = null; - private String storageId; + private final Map> blockMap + = new HashMap>(); + private final SimulatedStorage storage; + private final String storageId; - public SimulatedFSDataset(Configuration conf) throws IOException { - setConf(conf); - } - - // Constructor used for constructing the object using reflection - @SuppressWarnings("unused") - private SimulatedFSDataset() { // real construction when setConf called.. - } - - public Configuration getConf() { - return conf; - } - - public void setConf(Configuration iconf) { - conf = iconf; - storageId = conf.get(DFSConfigKeys.DFS_DATANODE_STORAGEID_KEY, "unknownStorageId" + - new Random().nextInt()); + public SimulatedFSDataset(DataNode datanode, DataStorage storage, + Configuration conf) { + if (storage != null) { + storage.createStorageID(datanode.getPort()); + this.storageId = storage.getStorageID(); + } else { + this.storageId = "unknownStorageId" + new Random().nextInt(); + } registerMBean(storageId); - storage = new SimulatedStorage( + this.storage = new SimulatedStorage( conf.getLong(CONFIG_PROPERTY_CAPACITY, DEFAULT_CAPACITY)); - blockMap = new HashMap>(); } public synchronized void injectBlocks(String bpid, @@ -441,23 +442,16 @@ public synchronized void unfinalizeBlock(ExtendedBlock b) throws IOException { @Override public synchronized BlockListAsLongs getBlockReport(String bpid) { + final List blocks = new ArrayList(); final Map map = blockMap.get(bpid); - Block[] blockTable = new Block[map.size()]; if (map != null) { - int count = 0; for (BInfo b : map.values()) { if (b.isFinalized()) { - blockTable[count++] = b.theBlock; + blocks.add(b.theBlock); } } - if (count != blockTable.length) { - blockTable = Arrays.copyOf(blockTable, count); - } - } else { - blockTable = new Block[0]; } - return new BlockListAsLongs( - new ArrayList(Arrays.asList(blockTable)), null); + return new BlockListAsLongs(blocks, null); } @Override // FSDatasetMBean diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java index f35e93cb157..bfff3ff19e3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java @@ -34,7 +34,7 @@ public class TestDataNodeMetrics extends TestCase { public void testDataNodeMetrics() throws Exception { Configuration conf = new HdfsConfiguration(); - conf.setBoolean(SimulatedFSDataset.CONFIG_PROPERTY_SIMULATED, true); + SimulatedFSDataset.setFactory(conf); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); try { FileSystem fs = cluster.getFileSystem(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java index 214b4e71a66..6a6c81a6fa2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java @@ -44,8 +44,8 @@ public class TestSimulatedFSDataset extends TestCase { protected void setUp() throws Exception { super.setUp(); - conf = new HdfsConfiguration(); - conf.setBoolean(SimulatedFSDataset.CONFIG_PROPERTY_SIMULATED, true); + conf = new HdfsConfiguration(); + SimulatedFSDataset.setFactory(conf); } protected void tearDown() throws Exception { @@ -86,6 +86,18 @@ int addSomeBlocks(FSDatasetInterface fsdataset, int startingBlockId) int addSomeBlocks(FSDatasetInterface fsdataset ) throws IOException { return addSomeBlocks(fsdataset, 1); } + + public void testFSDatasetFactory() { + final Configuration conf = new Configuration(); + FSDatasetInterface.Factory f = FSDatasetInterface.Factory.getFactory(conf); + assertEquals(FSDataset.Factory.class, f.getClass()); + assertFalse(f.isSimulated()); + + SimulatedFSDataset.setFactory(conf); + FSDatasetInterface.Factory s = FSDatasetInterface.Factory.getFactory(conf); + assertEquals(SimulatedFSDataset.Factory.class, s.getClass()); + assertTrue(s.isSimulated()); + } public void testGetMetaData() throws IOException { FSDatasetInterface fsdataset = getSimulatedFSDataset(); @@ -287,8 +299,8 @@ public void testInvalidate() throws IOException { } } - private SimulatedFSDataset getSimulatedFSDataset() throws IOException { - SimulatedFSDataset fsdataset = new SimulatedFSDataset(conf); + private SimulatedFSDataset getSimulatedFSDataset() { + SimulatedFSDataset fsdataset = new SimulatedFSDataset(null, null, conf); fsdataset.addBlockPool(bpid, conf); return fsdataset; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileLimit.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileLimit.java index 48ab6ce18e1..0bdebec9f45 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileLimit.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileLimit.java @@ -83,7 +83,7 @@ public void testFileLimit() throws IOException { int currentNodes = 0; if (simulatedStorage) { - conf.setBoolean(SimulatedFSDataset.CONFIG_PROPERTY_SIMULATED, true); + SimulatedFSDataset.setFactory(conf); } MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); FileSystem fs = cluster.getFileSystem();