From cf4bc7fdd49974324b177c99b820587cc5854adb Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Thu, 24 Apr 2014 17:40:30 +0000 Subject: [PATCH] HDFS-6273. Config options to allow wildcard endpoints for namenode HTTP and HTTPS servers. (Contributed by Arpit Agarwal) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1589803 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 2 + .../hadoop/hdfs/server/namenode/NameNode.java | 27 +++++++++++- .../server/namenode/NameNodeHttpServer.java | 10 +++++ .../server/namenode/NameNodeRpcServer.java | 6 +++ .../src/main/resources/hdfs-default.xml | 44 ++++++++++++++----- 6 files changed, 81 insertions(+), 11 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 539a032bf68..6d70d5cd105 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -314,6 +314,9 @@ Release 2.5.0 - UNRELEASED HDFS-5693. Few NN metrics data points were collected via JMX when NN is under heavy load. (Ming Ma via jing9) + HDFS-6273. Config options to allow wildcard endpoints for namenode HTTP + and HTTPS servers. (Arpit Agarwal) + OPTIMIZATIONS HDFS-6214. Webhdfs has poor throughput for files >2GB (daryn) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 134c27c0006..97b569f25c7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -126,6 +126,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final int DFS_NAMENODE_HTTP_PORT_DEFAULT = 50070; public static final String DFS_NAMENODE_HTTP_ADDRESS_KEY = "dfs.namenode.http-address"; public static final String DFS_NAMENODE_HTTP_ADDRESS_DEFAULT = "0.0.0.0:" + DFS_NAMENODE_HTTP_PORT_DEFAULT; + public static final String DFS_NAMENODE_HTTP_BIND_HOST_KEY = "dfs.namenode.http-bind-host"; public static final String DFS_NAMENODE_RPC_ADDRESS_KEY = "dfs.namenode.rpc-address"; public static final String DFS_NAMENODE_RPC_BIND_HOST_KEY = "dfs.namenode.rpc-bind-host"; public static final String DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY = "dfs.namenode.servicerpc-address"; @@ -295,6 +296,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final String DFS_DATANODE_DATA_DIR_KEY = "dfs.datanode.data.dir"; public static final int DFS_NAMENODE_HTTPS_PORT_DEFAULT = 50470; public static final String DFS_NAMENODE_HTTPS_ADDRESS_KEY = "dfs.namenode.https-address"; + public static final String DFS_NAMENODE_HTTPS_BIND_HOST_KEY = "dfs.namenode.https-bind-host"; public static final String DFS_NAMENODE_HTTPS_ADDRESS_DEFAULT = "0.0.0.0:" + DFS_NAMENODE_HTTPS_PORT_DEFAULT; public static final String DFS_NAMENODE_NAME_DIR_KEY = "dfs.namenode.name.dir"; public static final String DFS_NAMENODE_EDITS_DIR_KEY = "dfs.namenode.edits.dir"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java index 5984e5b6d90..445866b3f76 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java @@ -175,6 +175,8 @@ public class NameNode implements NameNodeStatusMXBean { DFS_NAMENODE_SERVICE_RPC_BIND_HOST_KEY, DFS_NAMENODE_HTTP_ADDRESS_KEY, DFS_NAMENODE_HTTPS_ADDRESS_KEY, + DFS_NAMENODE_HTTP_BIND_HOST_KEY, + DFS_NAMENODE_HTTPS_BIND_HOST_KEY, DFS_NAMENODE_KEYTAB_FILE_KEY, DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY, DFS_NAMENODE_SECONDARY_HTTPS_ADDRESS_KEY, @@ -444,6 +446,29 @@ public class NameNode implements NameNodeStatusMXBean { return getHttpAddress(conf); } + /** + * HTTP server address for binding the endpoint. This method is + * for use by the NameNode and its derivatives. It may return + * a different address than the one that should be used by clients to + * connect to the NameNode. See + * {@link DFSConfigKeys#DFS_NAMENODE_HTTP_BIND_HOST_KEY} + * + * @param conf + * @return + */ + protected InetSocketAddress getHttpServerBindAddress(Configuration conf) { + InetSocketAddress bindAddress = getHttpServerAddress(conf); + + // If DFS_NAMENODE_HTTP_BIND_HOST_KEY exists then it overrides the + // host name portion of DFS_NAMENODE_HTTP_ADDRESS_KEY. + final String bindHost = conf.getTrimmed(DFS_NAMENODE_HTTP_BIND_HOST_KEY); + if (bindHost != null && !bindHost.isEmpty()) { + bindAddress = new InetSocketAddress(bindHost, bindAddress.getPort()); + } + + return bindAddress; + } + /** @return the NameNode HTTP address. */ public static InetSocketAddress getHttpAddress(Configuration conf) { return NetUtils.createSocketAddr( @@ -608,7 +633,7 @@ public class NameNode implements NameNodeStatusMXBean { } private void startHttpServer(final Configuration conf) throws IOException { - httpServer = new NameNodeHttpServer(conf, this, getHttpServerAddress(conf)); + httpServer = new NameNodeHttpServer(conf, this, getHttpServerBindAddress(conf)); httpServer.start(); httpServer.setStartupProgress(startupProgress); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java index 59bd62c7a78..f9fbdc204c8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java @@ -108,6 +108,16 @@ public class NameNodeHttpServer { DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_DEFAULT); InetSocketAddress httpsAddr = NetUtils.createSocketAddr(httpsAddrString); + if (httpsAddr != null) { + // If DFS_NAMENODE_HTTPS_BIND_HOST_KEY exists then it overrides the + // host name portion of DFS_NAMENODE_HTTPS_ADDRESS_KEY. + final String bindHost = + conf.getTrimmed(DFSConfigKeys.DFS_NAMENODE_HTTPS_BIND_HOST_KEY); + if (bindHost != null && !bindHost.isEmpty()) { + httpsAddr = new InetSocketAddress(bindHost, httpsAddr.getPort()); + } + } + HttpServer2.Builder builder = DFSUtil.httpServerTemplateForNNAndJN(conf, httpAddr, httpsAddr, "hdfs", DFSConfigKeys.DFS_NAMENODE_KERBEROS_INTERNAL_SPNEGO_PRINCIPAL_KEY, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java index 566a0573ad3..a023b3af863 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java @@ -370,6 +370,12 @@ class NameNodeRpcServer implements NamenodeProtocols { return clientRpcServer; } + /** Allow access to the service RPC server for testing */ + @VisibleForTesting + RPC.Server getServiceRpcServer() { + return serviceRpcServer; + } + /** * Start client and service RPC servers. */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 6ef6c2d39c0..e1ae7deb894 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -55,11 +55,11 @@ dfs.namenode.rpc-bind-host - The actual address the server will bind to. If this optional address is - set, the RPC server will bind to this address and the port specified in - dfs.namenode.rpc-address for the RPC server. It can also be specified - per name node or name service for HA/Federation. This is most useful for - making name node listen to all interfaces by setting to 0.0.0.0. + The actual address the RPC server will bind to. If this optional address is + set, it overrides only the hostname portion of dfs.namenode.rpc-address. + It can also be specified per name node or name service for HA/Federation. + This is useful for making the name node listen on all interfaces by + setting it to 0.0.0.0. @@ -80,11 +80,11 @@ dfs.namenode.servicerpc-bind-host - The actual address the server will bind to. If this optional address is - set, the service RPC server will bind to this address and the port - specified in dfs.namenode.servicerpc-address. It can also be specified - per name node or name service for HA/Federation. This is most useful for - making name node listen to all interfaces by setting to 0.0.0.0. + The actual address the service RPC server will bind to. If this optional address is + set, it overrides only the hostname portion of dfs.namenode.servicerpc-address. + It can also be specified per name node or name service for HA/Federation. + This is useful for making the name node listen on all interfaces by + setting it to 0.0.0.0. @@ -142,6 +142,18 @@ + + dfs.namenode.http-bind-host + + + The actual adress the HTTP server will bind to. If this optional address + is set, it overrides only the hostname portion of dfs.namenode.http-address. + It can also be specified per name node or name service for HA/Federation. + This is useful for making the name node HTTP server listen on all + interfaces by setting it to 0.0.0.0. + + + dfs.https.enable false @@ -207,6 +219,18 @@ The namenode secure http server address and port. + + dfs.namenode.https-bind-host + + + The actual adress the HTTPS server will bind to. If this optional address + is set, it overrides only the hostname portion of dfs.namenode.https-address. + It can also be specified per name node or name service for HA/Federation. + This is useful for making the name node HTTPS server listen on all + interfaces by setting it to 0.0.0.0. + + + dfs.datanode.dns.interface default