From b2f524582cfcf6f34623ca7134f0c5188516df38 Mon Sep 17 00:00:00 2001 From: Uma Maheswara Rao G Date: Tue, 1 May 2012 05:55:16 +0000 Subject: [PATCH] merge HDFS-3275 git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1332528 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../hadoop/hdfs/server/namenode/NameNode.java | 24 ++++++++++--- .../hdfs/server/namenode/TestAllowFormat.java | 36 +++++++++++++++++++ .../hdfs/server/namenode/ha/HATestUtil.java | 9 +++++ 4 files changed, 68 insertions(+), 4 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 584f5948dba..060f101a068 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -451,6 +451,9 @@ Release 2.0.0 - UNRELEASED HDFS-3326. Append enabled log message uses the wrong variable. (Matthew Jacobs via eli) + HDFS-3275. Skip format for non-file based directories. + (Amith D K via umamahesh) + BREAKDOWN OF HDFS-1623 SUBTASKS HDFS-2179. Add fencing framework and mechanisms for NameNode HA. (todd) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java index d7a2fb42499..5cfb9c78968 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java @@ -25,6 +25,7 @@ import java.io.InputStream; import java.io.OutputStream; import java.net.InetSocketAddress; import java.net.URI; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Iterator; @@ -674,10 +675,14 @@ public class NameNode { initializeGenericKeys(conf, nsId, namenodeId); checkAllowFormat(conf); - Collection dirsToFormat = FSNamesystem.getNamespaceDirs(conf); + Collection nameDirsToFormat = FSNamesystem.getNamespaceDirs(conf); + List sharedDirs = FSNamesystem.getSharedEditsDirs(conf); + List dirsToPrompt = new ArrayList(); + dirsToPrompt.addAll(nameDirsToFormat); + dirsToPrompt.addAll(sharedDirs); List editDirsToFormat = FSNamesystem.getNamespaceEditsDirs(conf); - if (!confirmFormat(dirsToFormat, force, isInteractive)) { + if (!confirmFormat(dirsToPrompt, force, isInteractive)) { return true; // aborted } @@ -689,7 +694,7 @@ public class NameNode { } System.out.println("Formatting using clusterid: " + clusterId); - FSImage fsImage = new FSImage(conf, dirsToFormat, editDirsToFormat); + FSImage fsImage = new FSImage(conf, nameDirsToFormat, editDirsToFormat); FSNamesystem fsn = new FSNamesystem(conf, fsImage); fsImage.format(fsn, clusterId); return false; @@ -711,7 +716,18 @@ public class NameNode { boolean force, boolean interactive) throws IOException { for(Iterator it = dirsToFormat.iterator(); it.hasNext();) { - File curDir = new File(it.next().getPath()); + URI dirUri = it.next(); + if (!dirUri.getScheme().equals(NNStorage.LOCAL_URI_SCHEME)) { + System.err.println("Skipping format for directory \"" + dirUri + + "\". Can only format local directories with scheme \"" + + NNStorage.LOCAL_URI_SCHEME + "\"."); + continue; + } + // To validate only file based schemes are formatted + assert dirUri.getScheme().equals(NNStorage.LOCAL_URI_SCHEME) : + "formatting is not supported for " + dirUri; + + File curDir = new File(dirUri.getPath()); // Its alright for a dir not to exist, or to exist (properly accessible) // and be completely empty. if (!curDir.exists() || diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAllowFormat.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAllowFormat.java index eb8758f0c6e..d18b343c749 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAllowFormat.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAllowFormat.java @@ -27,13 +27,19 @@ import static org.junit.Assert.fail; import java.io.File; import java.io.IOException; +import java.net.InetSocketAddress; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DFSUtil; +import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.server.namenode.TestGenericJournalConf.DummyJournalManager; +import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil; import org.apache.hadoop.util.StringUtils; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -144,4 +150,34 @@ public class TestAllowFormat { NameNode.format(config); LOG.info("Done verifying format will succeed with allowformat true"); } + + /** + * Test to skip format for non file scheme directory configured + * + * @throws Exception + */ + @Test + public void testFormatShouldBeIgnoredForNonFileBasedDirs() throws Exception { + Configuration conf = new HdfsConfiguration(); + String logicalName = "mycluster"; + + // DFS_NAMENODE_RPC_ADDRESS_KEY are required to identify the NameNode + // is configured in HA, then only DFS_NAMENODE_SHARED_EDITS_DIR_KEY + // is considered. + String localhost = "127.0.0.1"; + InetSocketAddress nnAddr1 = new InetSocketAddress(localhost, 8020); + InetSocketAddress nnAddr2 = new InetSocketAddress(localhost, 9020); + HATestUtil.setFailoverConfigurations(conf, logicalName, nnAddr1, nnAddr2); + + conf.setBoolean(DFS_NAMENODE_SUPPORT_ALLOW_FORMAT_KEY, true); + conf.set(DFSUtil.addKeySuffixes( + DFSConfigKeys.DFS_NAMENODE_EDITS_PLUGIN_PREFIX, "dummy"), + DummyJournalManager.class.getName()); + conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY, "dummy://" + + localhost + ":2181/ledgers"); + conf.set(DFSConfigKeys.DFS_HA_NAMENODE_ID_KEY, "nn1"); + + // An internal assert is added to verify the working of test + NameNode.format(conf); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/HATestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/HATestUtil.java index 7224b430d12..e7101c64d86 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/HATestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/HATestUtil.java @@ -167,6 +167,15 @@ public abstract class HATestUtil { Configuration conf, String logicalName, int nsIndex) { InetSocketAddress nnAddr1 = cluster.getNameNode(2 * nsIndex).getNameNodeAddress(); InetSocketAddress nnAddr2 = cluster.getNameNode(2 * nsIndex + 1).getNameNodeAddress(); + setFailoverConfigurations(conf, logicalName, nnAddr1, nnAddr2); + } + + /** + * Sets the required configurations for performing failover + */ + public static void setFailoverConfigurations(Configuration conf, + String logicalName, InetSocketAddress nnAddr1, + InetSocketAddress nnAddr2) { String nameNodeId1 = "nn1"; String nameNodeId2 = "nn2"; String address1 = "hdfs://" + nnAddr1.getHostName() + ":" + nnAddr1.getPort();