From 24436924340428f91b28906457e2a14be1f3722c Mon Sep 17 00:00:00 2001 From: Vinayakumar B Date: Wed, 17 Jan 2018 14:16:48 +0530 Subject: [PATCH] HDFS-9049. Make Datanode Netty reverse proxy port to be configurable. Contributed by Vinayakumar B. (cherry picked from commit 09efdfe9e13c9695867ce4034aa6ec970c2032f1) --- .../main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 2 ++ .../hdfs/server/datanode/web/DatanodeHttpServer.java | 5 ++++- .../hadoop-hdfs/src/main/resources/hdfs-default.xml | 9 +++++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 1058b4ab898..9c51f256025 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -617,6 +617,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final String DFS_DATANODE_HTTP_ADDRESS_KEY = "dfs.datanode.http.address"; public static final int DFS_DATANODE_HTTP_DEFAULT_PORT = 9864; public static final String DFS_DATANODE_HTTP_ADDRESS_DEFAULT = "0.0.0.0:" + DFS_DATANODE_HTTP_DEFAULT_PORT; + public static final String DFS_DATANODE_HTTP_INTERNAL_PROXY_PORT = + "dfs.datanode.http.internal-proxy.port"; public static final String DFS_DATANODE_MAX_RECEIVER_THREADS_KEY = HdfsClientConfigKeys.DeprecatedKeys.DFS_DATANODE_MAX_RECEIVER_THREADS_KEY; public static final int DFS_DATANODE_MAX_RECEIVER_THREADS_DEFAULT = 4096; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java index b51b1fc60e8..2e46b284ac6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java @@ -72,6 +72,7 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ADMIN; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HTTPS_ADDRESS_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HTTPS_ADDRESS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HTTP_ADDRESS_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HTTP_INTERNAL_PROXY_PORT; public class DatanodeHttpServer implements Closeable { private final HttpServer2 infoServer; @@ -97,12 +98,14 @@ public class DatanodeHttpServer implements Closeable { Configuration confForInfoServer = new Configuration(conf); confForInfoServer.setInt(HttpServer2.HTTP_MAX_THREADS_KEY, 10); + int proxyPort = + confForInfoServer.getInt(DFS_DATANODE_HTTP_INTERNAL_PROXY_PORT, 0); HttpServer2.Builder builder = new HttpServer2.Builder() .setName("datanode") .setConf(confForInfoServer) .setACL(new AccessControlList(conf.get(DFS_ADMIN, " "))) .hostName(getHostnameForSpnegoPrincipal(confForInfoServer)) - .addEndpoint(URI.create("http://localhost:0")) + .addEndpoint(URI.create("http://localhost:" + proxyPort)) .setFindPort(true); final boolean xFrameEnabled = conf.getBoolean( diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 190a06d5cdf..6be11b9654e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -145,6 +145,15 @@ + + dfs.datanode.http.internal-proxy.port + 0 + + The datanode's internal web proxy port. + By default it selects a random port available in runtime. + + + dfs.datanode.handler.count 10