From 5f5072b4081e429939613bcdebe03a136a1220cf Mon Sep 17 00:00:00 2001 From: Eli Collins Date: Thu, 31 May 2012 21:06:19 +0000 Subject: [PATCH] HDFS-3484. hdfs fsck doesn't work if NN HTTP address is set to 0.0.0.0 even if NN RPC address is configured. Contributed by Aaron T. Myers git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1344909 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../java/org/apache/hadoop/hdfs/DFSUtil.java | 17 ++++++++++++----- .../hdfs/server/namenode/FSNamesystem.java | 8 ++++++-- .../server/namenode/ha/StandbyCheckpointer.java | 8 +++++--- .../org/apache/hadoop/hdfs/TestDFSUtil.java | 8 +++++++- .../server/namenode/ha/TestHAConfiguration.java | 2 +- 6 files changed, 34 insertions(+), 12 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index dec3fecbdd1..da9dd850072 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -135,6 +135,9 @@ Release 2.0.1-alpha - UNRELEASED HDFS-3441. Race condition between rolling logs at active NN and purging at standby. (Rakesh R via umamahesh) + HDFS-3484. hdfs fsck doesn't work if NN HTTP address is set to + 0.0.0.0 even if NN RPC address is configured. (atm via eli) + Release 2.0.0-alpha - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java index b04ad041063..03ae1607a37 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java @@ -712,9 +712,10 @@ public class DFSUtil { * @param httpsAddress -If true, and if security is enabled, returns server * https address. If false, returns server http address. * @return server http or https address + * @throws IOException */ - public static String getInfoServer( - InetSocketAddress namenodeAddr, Configuration conf, boolean httpsAddress) { + public static String getInfoServer(InetSocketAddress namenodeAddr, + Configuration conf, boolean httpsAddress) throws IOException { boolean securityOn = UserGroupInformation.isSecurityEnabled(); String httpAddressKey = (securityOn && httpsAddress) ? DFS_NAMENODE_HTTPS_ADDRESS_KEY : DFS_NAMENODE_HTTP_ADDRESS_KEY; @@ -731,8 +732,14 @@ public class DFSUtil { } else { suffixes = new String[2]; } - - return getSuffixedConf(conf, httpAddressKey, httpAddressDefault, suffixes); + String configuredInfoAddr = getSuffixedConf(conf, httpAddressKey, + httpAddressDefault, suffixes); + if (namenodeAddr != null) { + return substituteForWildcardAddress(configuredInfoAddr, + namenodeAddr.getHostName()); + } else { + return configuredInfoAddr; + } } @@ -757,7 +764,7 @@ public class DFSUtil { if (UserGroupInformation.isSecurityEnabled() && defaultSockAddr.getAddress().isAnyLocalAddress()) { throw new IOException("Cannot use a wildcard address with security. " + - "Must explicitly set bind address for Kerberos"); + "Must explicitly set bind address for Kerberos"); } return defaultHost + ":" + sockAddr.getPort(); } else { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 4b576232bf3..0c65c2b90ec 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -664,8 +664,12 @@ public class FSNamesystem implements Namesystem, FSClusterStats, } } - /** Start services required in standby state */ - void startStandbyServices(final Configuration conf) { + /** + * Start services required in standby state + * + * @throws IOException + */ + void startStandbyServices(final Configuration conf) throws IOException { LOG.info("Starting services required for standby state"); if (!dir.fsImage.editLog.isOpenForRead()) { // During startup, we're already open for read. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/StandbyCheckpointer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/StandbyCheckpointer.java index d575dd23167..c3ee0d069ac 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/StandbyCheckpointer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/StandbyCheckpointer.java @@ -67,7 +67,8 @@ public class StandbyCheckpointer { // This is for use in tests. private static int canceledCount = 0; - public StandbyCheckpointer(Configuration conf, FSNamesystem ns) { + public StandbyCheckpointer(Configuration conf, FSNamesystem ns) + throws IOException { this.namesystem = ns; this.checkpointConf = new CheckpointConf(conf); this.thread = new CheckpointerThread(); @@ -78,8 +79,9 @@ public class StandbyCheckpointer { /** * Determine the address of the NN we are checkpointing * as well as our own HTTP address from the configuration. + * @throws IOException */ - private void setNameNodeAddresses(Configuration conf) { + private void setNameNodeAddresses(Configuration conf) throws IOException { // Look up our own address. String myAddrString = getHttpAddress(conf); @@ -95,7 +97,7 @@ public class StandbyCheckpointer { myNNAddress = NetUtils.createSocketAddr(myAddrString); } - private String getHttpAddress(Configuration conf) { + private String getHttpAddress(Configuration conf) throws IOException { String configuredAddr = DFSUtil.getInfoServer(null, conf, false); // Use the hostname from the RPC address as a default, in case diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java index 783528a3d97..286bbc75820 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java @@ -409,14 +409,20 @@ public class TestDFSUtil { } @Test - public void testGetServerInfo() { + public void testGetInfoServer() throws IOException { HdfsConfiguration conf = new HdfsConfiguration(); conf.set(HADOOP_SECURITY_AUTHENTICATION, "kerberos"); UserGroupInformation.setConfiguration(conf); + String httpsport = DFSUtil.getInfoServer(null, conf, true); assertEquals("0.0.0.0:"+DFS_NAMENODE_HTTPS_PORT_DEFAULT, httpsport); + String httpport = DFSUtil.getInfoServer(null, conf, false); assertEquals("0.0.0.0:"+DFS_NAMENODE_HTTP_PORT_DEFAULT, httpport); + + String httpAddress = DFSUtil.getInfoServer(new InetSocketAddress( + "localhost", 8020), conf, false); + assertEquals("localhost:" + DFS_NAMENODE_HTTP_PORT_DEFAULT, httpAddress); } @Test diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAConfiguration.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAConfiguration.java index 563ddff69c1..abd7c72f17d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAConfiguration.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAConfiguration.java @@ -72,7 +72,7 @@ public class TestHAConfiguration { } @Test - public void testGetOtherNNHttpAddress() { + public void testGetOtherNNHttpAddress() throws IOException { // Use non-local addresses to avoid host address matching Configuration conf = getHAConf("ns1", "1.2.3.1", "1.2.3.2"); conf.set(DFSConfigKeys.DFS_NAMESERVICE_ID, "ns1");