diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index d8fe90c5b17..9f7ac5e5029 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -231,6 +231,9 @@ Release 2.0.0 - UNRELEASED HDFS-3238. ServerCommand and friends don't need to be writables. (eli) + HDFS-3094. add -nonInteractive and -force option to namenode -format + command (Arpit Gupta via todd) + OPTIMIZATIONS HDFS-2477. Optimize computing the diff between a block report and the diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java index 00275c5917c..8e0b9927373 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java @@ -57,13 +57,18 @@ public final class HdfsServerConstants { BOOTSTRAPSTANDBY("-bootstrapStandby"), INITIALIZESHAREDEDITS("-initializeSharedEdits"), RECOVER ("-recover"), - FORCE("-force"); + FORCE("-force"), + NONINTERACTIVE("-nonInteractive"); private String name = null; // Used only with format and upgrade options private String clusterId = null; + // Used only with format option + private boolean isForceFormat = false; + private boolean isInteractiveFormat = true; + // Used only with recovery option private int force = 0; @@ -101,6 +106,22 @@ public final class HdfsServerConstants { public int getForce() { return this.force; } + + public boolean getForceFormat() { + return isForceFormat; + } + + public void setForceFormat(boolean force) { + isForceFormat = force; + } + + public boolean getInteractiveFormat() { + return isInteractiveFormat; + } + + public void setInteractiveFormat(boolean interactive) { + isInteractiveFormat = interactive; + } } // Timeouts for communicating with DataNode for streaming writes/reads diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java index 014c0e43841..9a9f98a39f6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java @@ -215,7 +215,7 @@ public class NameNode { /** Format a new filesystem. Destroys any filesystem that may already * exist at this location. **/ public static void format(Configuration conf) throws IOException { - format(conf, true); + format(conf, true, true); } static NameNodeMetrics metrics; @@ -658,9 +658,8 @@ public class NameNode { * @return true if formatting was aborted, false otherwise * @throws IOException */ - private static boolean format(Configuration conf, - boolean force) - throws IOException { + private static boolean format(Configuration conf, boolean force, + boolean isInteractive) throws IOException { String nsId = DFSUtil.getNamenodeNameServiceId(conf); String namenodeId = HAUtil.getNameNodeId(conf, nsId); initializeGenericKeys(conf, nsId, namenodeId); @@ -669,7 +668,7 @@ public class NameNode { Collection dirsToFormat = FSNamesystem.getNamespaceDirs(conf); List editDirsToFormat = FSNamesystem.getNamespaceEditsDirs(conf); - if (!confirmFormat(dirsToFormat, force, true)) { + if (!confirmFormat(dirsToFormat, force, isInteractive)) { return true; // aborted } @@ -830,8 +829,9 @@ public class NameNode { "Usage: java NameNode [" + StartupOption.BACKUP.getName() + "] | [" + StartupOption.CHECKPOINT.getName() + "] | [" + - StartupOption.FORMAT.getName() + "[" + StartupOption.CLUSTERID.getName() + - " cid ]] | [" + + StartupOption.FORMAT.getName() + " [" + StartupOption.CLUSTERID.getName() + + " cid ] [" + StartupOption.FORCE.getName() + "] [" + + StartupOption.NONINTERACTIVE.getName() + "] ] | [" + StartupOption.UPGRADE.getName() + "] | [" + StartupOption.ROLLBACK.getName() + "] | [" + StartupOption.FINALIZE.getName() + "] | [" + @@ -850,11 +850,35 @@ public class NameNode { String cmd = args[i]; if (StartupOption.FORMAT.getName().equalsIgnoreCase(cmd)) { startOpt = StartupOption.FORMAT; - // might be followed by two args - if (i + 2 < argsLen - && args[i + 1].equalsIgnoreCase(StartupOption.CLUSTERID.getName())) { - i += 2; - startOpt.setClusterId(args[i]); + for (i = i + 1; i < argsLen; i++) { + if (args[i].equalsIgnoreCase(StartupOption.CLUSTERID.getName())) { + i++; + if (i >= argsLen) { + // if no cluster id specified, return null + LOG.fatal("Must specify a valid cluster ID after the " + + StartupOption.CLUSTERID.getName() + " flag"); + return null; + } + String clusterId = args[i]; + // Make sure an id is specified and not another flag + if (clusterId.isEmpty() || + clusterId.equalsIgnoreCase(StartupOption.FORCE.getName()) || + clusterId.equalsIgnoreCase( + StartupOption.NONINTERACTIVE.getName())) { + LOG.fatal("Must specify a valid cluster ID after the " + + StartupOption.CLUSTERID.getName() + " flag"); + return null; + } + startOpt.setClusterId(clusterId); + } + + if (args[i].equalsIgnoreCase(StartupOption.FORCE.getName())) { + startOpt.setForceFormat(true); + } + + if (args[i].equalsIgnoreCase(StartupOption.NONINTERACTIVE.getName())) { + startOpt.setInteractiveFormat(false); + } } } else if (StartupOption.GENCLUSTERID.getName().equalsIgnoreCase(cmd)) { startOpt = StartupOption.GENCLUSTERID; @@ -997,7 +1021,8 @@ public class NameNode { switch (startOpt) { case FORMAT: { - boolean aborted = format(conf, false); + boolean aborted = format(conf, startOpt.getForceFormat(), + startOpt.getInteractiveFormat()); System.exit(aborted ? 1 : 0); return null; // avoid javac warning } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestClusterId.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestClusterId.java index 98c17a7b4d1..9b5a5bd2944 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestClusterId.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestClusterId.java @@ -18,12 +18,19 @@ package org.apache.hadoop.hdfs.server.namenode; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.fail; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; +import java.io.InputStream; +import java.io.PrintStream; import java.net.URI; -import java.util.ArrayList; +import java.security.Permission; import java.util.Collection; import java.util.Iterator; import java.util.List; @@ -40,11 +47,11 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; - public class TestClusterId { private static final Log LOG = LogFactory.getLog(TestClusterId.class); File hdfsDir; - + Configuration config; + private String getClusterId(Configuration config) throws IOException { // see if cluster id not empty. Collection dirsToFormat = FSNamesystem.getNamespaceDirs(config); @@ -59,33 +66,41 @@ public class TestClusterId { LOG.info("successfully formated : sd="+sd.getCurrentDir() + ";cid="+cid); return cid; } - + @Before public void setUp() throws IOException { + System.setSecurityManager(new NoExitSecurityManager()); + String baseDir = System.getProperty("test.build.data", "build/test/data"); - hdfsDir = new File(baseDir, "dfs"); - if ( hdfsDir.exists() && !FileUtil.fullyDelete(hdfsDir) ) { - throw new IOException("Could not delete test directory '" + - hdfsDir + "'"); + hdfsDir = new File(baseDir, "dfs/name"); + if (hdfsDir.exists() && !FileUtil.fullyDelete(hdfsDir)) { + throw new IOException("Could not delete test directory '" + hdfsDir + "'"); } LOG.info("hdfsdir is " + hdfsDir.getAbsolutePath()); + + // as some tests might change these values we reset them to defaults before + // every test + StartupOption.FORMAT.setForceFormat(false); + StartupOption.FORMAT.setInteractiveFormat(true); + + config = new Configuration(); + config.set(DFS_NAMENODE_NAME_DIR_KEY, hdfsDir.getPath()); } - + @After public void tearDown() throws IOException { - if ( hdfsDir.exists() && !FileUtil.fullyDelete(hdfsDir) ) { - throw new IOException("Could not tearDown test directory '" + - hdfsDir + "'"); + System.setSecurityManager(null); + + if (hdfsDir.exists() && !FileUtil.fullyDelete(hdfsDir)) { + throw new IOException("Could not tearDown test directory '" + hdfsDir + + "'"); } } - + @Test public void testFormatClusterIdOption() throws IOException { - Configuration config = new Configuration(); - config.set(DFS_NAMENODE_NAME_DIR_KEY, new File(hdfsDir, "name").getPath()); - // 1. should format without cluster id //StartupOption.FORMAT.setClusterId(""); NameNode.format(config); @@ -107,4 +122,356 @@ public class TestClusterId { String newCid = getClusterId(config); assertFalse("ClusterId should not be the same", newCid.equals(cid)); } -} + + /** + * Test namenode format with -format option. Format should succeed. + * + * @throws IOException + */ + @Test + public void testFormat() throws IOException { + String[] argv = { "-format" }; + try { + NameNode.createNameNode(argv, config); + fail("createNameNode() did not call System.exit()"); + } catch (ExitException e) { + assertEquals("Format should have succeeded", 0, e.status); + } + + String cid = getClusterId(config); + assertTrue("Didn't get new ClusterId", (cid != null && !cid.equals(""))); + } + + /** + * Test namenode format with -format option when an empty name directory + * exists. Format should succeed. + * + * @throws IOException + */ + @Test + public void testFormatWithEmptyDir() throws IOException { + + if (!hdfsDir.mkdirs()) { + fail("Failed to create dir " + hdfsDir.getPath()); + } + + String[] argv = { "-format" }; + try { + NameNode.createNameNode(argv, config); + fail("createNameNode() did not call System.exit()"); + } catch (ExitException e) { + assertEquals("Format should have succeeded", 0, e.status); + } + + String cid = getClusterId(config); + assertTrue("Didn't get new ClusterId", (cid != null && !cid.equals(""))); + } + + /** + * Test namenode format with -format -force options when name directory + * exists. Format should succeed. + * + * @throws IOException + */ + @Test + public void testFormatWithForce() throws IOException { + + if (!hdfsDir.mkdirs()) { + fail("Failed to create dir " + hdfsDir.getPath()); + } + + String[] argv = { "-format", "-force" }; + try { + NameNode.createNameNode(argv, config); + fail("createNameNode() did not call System.exit()"); + } catch (ExitException e) { + assertEquals("Format should have succeeded", 0, e.status); + } + + String cid = getClusterId(config); + assertTrue("Didn't get new ClusterId", (cid != null && !cid.equals(""))); + } + + /** + * Test namenode format with -format -force -clusterid option when name + * directory exists. Format should succeed. + * + * @throws IOException + */ + @Test + public void testFormatWithForceAndClusterId() throws IOException { + + if (!hdfsDir.mkdirs()) { + fail("Failed to create dir " + hdfsDir.getPath()); + } + + String myId = "testFormatWithForceAndClusterId"; + String[] argv = { "-format", "-force", "-clusterid", myId }; + try { + NameNode.createNameNode(argv, config); + fail("createNameNode() did not call System.exit()"); + } catch (ExitException e) { + assertEquals("Format should have succeeded", 0, e.status); + } + + String cId = getClusterId(config); + assertEquals("ClusterIds do not match", myId, cId); + } + + /** + * Test namenode format with -clusterid -force option. Format command should + * fail as no cluster id was provided. + * + * @throws IOException + */ + @Test + public void testFormatWithInvalidClusterIdOption() throws IOException { + + String[] argv = { "-format", "-clusterid", "-force" }; + PrintStream origErr = System.err; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintStream stdErr = new PrintStream(baos); + System.setErr(stdErr); + + NameNode.createNameNode(argv, config); + + // Check if usage is printed + assertTrue(baos.toString("UTF-8").contains("Usage: java NameNode")); + System.setErr(origErr); + + // check if the version file does not exists. + File version = new File(hdfsDir, "current/VERSION"); + assertFalse("Check version should not exist", version.exists()); + } + + /** + * Test namenode format with -format -clusterid options. Format should fail + * was no clusterid was sent. + * + * @throws IOException + */ + @Test + public void testFormatWithNoClusterIdOption() throws IOException { + + String[] argv = { "-format", "-clusterid" }; + PrintStream origErr = System.err; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintStream stdErr = new PrintStream(baos); + System.setErr(stdErr); + + NameNode.createNameNode(argv, config); + + // Check if usage is printed + assertTrue(baos.toString("UTF-8").contains("Usage: java NameNode")); + System.setErr(origErr); + + // check if the version file does not exists. + File version = new File(hdfsDir, "current/VERSION"); + assertFalse("Check version should not exist", version.exists()); + } + + /** + * Test namenode format with -format -clusterid and empty clusterid. Format + * should fail as no valid if was provided. + * + * @throws IOException + */ + @Test + public void testFormatWithEmptyClusterIdOption() throws IOException { + + String[] argv = { "-format", "-clusterid", "" }; + + PrintStream origErr = System.err; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintStream stdErr = new PrintStream(baos); + System.setErr(stdErr); + + NameNode.createNameNode(argv, config); + + // Check if usage is printed + assertTrue(baos.toString("UTF-8").contains("Usage: java NameNode")); + System.setErr(origErr); + + // check if the version file does not exists. + File version = new File(hdfsDir, "current/VERSION"); + assertFalse("Check version should not exist", version.exists()); + } + + /** + * Test namenode format with -format -nonInteractive options when a non empty + * name directory exists. Format should not succeed. + * + * @throws IOException + */ + @Test + public void testFormatWithNonInteractive() throws IOException { + + // we check for a non empty dir, so create a child path + File data = new File(hdfsDir, "file"); + if (!data.mkdirs()) { + fail("Failed to create dir " + data.getPath()); + } + + String[] argv = { "-format", "-nonInteractive" }; + try { + NameNode.createNameNode(argv, config); + fail("createNameNode() did not call System.exit()"); + } catch (ExitException e) { + assertEquals("Format should have been aborted with exit code 1", 1, + e.status); + } + + // check if the version file does not exists. + File version = new File(hdfsDir, "current/VERSION"); + assertFalse("Check version should not exist", version.exists()); + } + + /** + * Test namenode format with -format -nonInteractive options when name + * directory does not exist. Format should succeed. + * + * @throws IOException + */ + @Test + public void testFormatWithNonInteractiveNameDirDoesNotExit() + throws IOException { + + String[] argv = { "-format", "-nonInteractive" }; + try { + NameNode.createNameNode(argv, config); + fail("createNameNode() did not call System.exit()"); + } catch (ExitException e) { + assertEquals("Format should have succeeded", 0, e.status); + } + + String cid = getClusterId(config); + assertTrue("Didn't get new ClusterId", (cid != null && !cid.equals(""))); + } + + /** + * Test namenode format with -force -nonInteractive -force option. Format + * should succeed. + * + * @throws IOException + */ + @Test + public void testFormatWithNonInteractiveAndForce() throws IOException { + + if (!hdfsDir.mkdirs()) { + fail("Failed to create dir " + hdfsDir.getPath()); + } + + String[] argv = { "-format", "-nonInteractive", "-force" }; + try { + NameNode.createNameNode(argv, config); + fail("createNameNode() did not call System.exit()"); + } catch (ExitException e) { + assertEquals("Format should have succeeded", 0, e.status); + } + + String cid = getClusterId(config); + assertTrue("Didn't get new ClusterId", (cid != null && !cid.equals(""))); + } + + /** + * Test namenode format with -format option when a non empty name directory + * exists. Enter Y when prompted and the format should succeed. + * + * @throws IOException + * @throws InterruptedException + */ + @Test + public void testFormatWithoutForceEnterYes() throws IOException, + InterruptedException { + + // we check for a non empty dir, so create a child path + File data = new File(hdfsDir, "file"); + if (!data.mkdirs()) { + fail("Failed to create dir " + data.getPath()); + } + + // capture the input stream + InputStream origIn = System.in; + ByteArrayInputStream bins = new ByteArrayInputStream("Y\n".getBytes()); + System.setIn(bins); + + String[] argv = { "-format" }; + + try { + NameNode.createNameNode(argv, config); + fail("createNameNode() did not call System.exit()"); + } catch (ExitException e) { + assertEquals("Format should have succeeded", 0, e.status); + } + + System.setIn(origIn); + + String cid = getClusterId(config); + assertTrue("Didn't get new ClusterId", (cid != null && !cid.equals(""))); + } + + /** + * Test namenode format with -format option when a non empty name directory + * exists. Enter N when prompted and format should be aborted. + * + * @throws IOException + * @throws InterruptedException + */ + @Test + public void testFormatWithoutForceEnterNo() throws IOException, + InterruptedException { + + // we check for a non empty dir, so create a child path + File data = new File(hdfsDir, "file"); + if (!data.mkdirs()) { + fail("Failed to create dir " + data.getPath()); + } + + // capture the input stream + InputStream origIn = System.in; + ByteArrayInputStream bins = new ByteArrayInputStream("N\n".getBytes()); + System.setIn(bins); + + String[] argv = { "-format" }; + try { + NameNode.createNameNode(argv, config); + fail("createNameNode() did not call System.exit()"); + } catch (ExitException e) { + assertEquals("Format should not have succeeded", 1, e.status); + } + + System.setIn(origIn); + + // check if the version file does not exists. + File version = new File(hdfsDir, "current/VERSION"); + assertFalse("Check version should not exist", version.exists()); + } + + private static class ExitException extends SecurityException { + private static final long serialVersionUID = 1L; + public final int status; + + public ExitException(int status) { + super("There is no escape!"); + this.status = status; + } + } + + private static class NoExitSecurityManager extends SecurityManager { + @Override + public void checkPermission(Permission perm) { + // allow anything. + } + + @Override + public void checkPermission(Permission perm, Object context) { + // allow anything. + } + + @Override + public void checkExit(int status) { + super.checkExit(status); + throw new ExitException(status); + } + } +} \ No newline at end of file