From 08aae6818263368ce601c4915fc2f54dbe1c0dbc Mon Sep 17 00:00:00 2001 From: Haohui Mai Date: Fri, 18 Jul 2014 20:35:02 +0000 Subject: [PATCH] HDFS-6632. Reintroduce dfs.http.port / dfs.https.port in branch-2. Contributed by Yongjun Zhang. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1611794 13f79535-47bb-0310-9956-ffa450edef68 --- .../fs/http/client/HttpFSFileSystem.java | 3 +- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 2 + .../hadoop/hdfs/web/SWebHdfsFileSystem.java | 8 +- .../hadoop/hdfs/web/WebHdfsFileSystem.java | 6 +- .../hadoop/hdfs/web/TestHttpFSPorts.java | 82 +++++++++++++++++++ 6 files changed, 99 insertions(+), 5 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestHttpFSPorts.java diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java index e637972cb1f..6869b5de9a4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java @@ -384,7 +384,8 @@ public class HttpFSFileSystem extends FileSystem */ @Override protected int getDefaultPort() { - return DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_DEFAULT; + return getConf().getInt(DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_KEY, + DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_DEFAULT); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 18f1fefa58b..3460d502232 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -92,6 +92,9 @@ Release 2.6.0 - UNRELEASED HDFS-6667. In HDFS HA mode, Distcp/SLive with webhdfs on secure cluster fails with Client cannot authenticate via:[TOKEN, KERBEROS] error. (jing9) + HDFS-6632. Reintroduce dfs.http.port / dfs.https.port in branch-2. + (Yongjun Zhang via wheat9) + Release 2.5.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index aea6b72acef..2ad20251a0e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -127,6 +127,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys { "dfs.namenode.path.based.cache.block.map.allocation.percent"; public static final float DFS_NAMENODE_PATH_BASED_CACHE_BLOCK_MAP_ALLOCATION_PERCENT_DEFAULT = 0.25f; + public static final String DFS_NAMENODE_HTTP_PORT_KEY = "dfs.http.port"; public static final int DFS_NAMENODE_HTTP_PORT_DEFAULT = 50070; public static final String DFS_NAMENODE_HTTP_ADDRESS_KEY = "dfs.namenode.http-address"; public static final String DFS_NAMENODE_HTTP_ADDRESS_DEFAULT = "0.0.0.0:" + DFS_NAMENODE_HTTP_PORT_DEFAULT; @@ -309,6 +310,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys { //Following keys have no defaults public static final String DFS_DATANODE_DATA_DIR_KEY = "dfs.datanode.data.dir"; + public static final String DFS_NAMENODE_HTTPS_PORT_KEY = "dfs.https.port"; public static final int DFS_NAMENODE_HTTPS_PORT_DEFAULT = 50470; public static final String DFS_NAMENODE_HTTPS_ADDRESS_KEY = "dfs.namenode.https-address"; public static final String DFS_NAMENODE_HTTPS_BIND_HOST_KEY = "dfs.namenode.https-bind-host"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/SWebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/SWebHdfsFileSystem.java index fa89ec3551b..7c78d525361 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/SWebHdfsFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/SWebHdfsFileSystem.java @@ -20,6 +20,8 @@ package org.apache.hadoop.hdfs.web; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.io.Text; +import com.google.common.annotations.VisibleForTesting; + public class SWebHdfsFileSystem extends WebHdfsFileSystem { public static final Text TOKEN_KIND = new Text("SWEBHDFS delegation"); @@ -41,7 +43,9 @@ public class SWebHdfsFileSystem extends WebHdfsFileSystem { } @Override - protected int getDefaultPort() { - return DFSConfigKeys.DFS_NAMENODE_HTTPS_PORT_DEFAULT; + @VisibleForTesting + public int getDefaultPort() { + return getConf().getInt(DFSConfigKeys.DFS_NAMENODE_HTTPS_PORT_KEY, + DFSConfigKeys.DFS_NAMENODE_HTTPS_PORT_DEFAULT); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java index b5fccd1b557..15e906e3478 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java @@ -248,8 +248,10 @@ public class WebHdfsFileSystem extends FileSystem } @Override - protected int getDefaultPort() { - return DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_DEFAULT; + @VisibleForTesting + public int getDefaultPort() { + return getConf().getInt(DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_KEY, + DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_DEFAULT); } @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestHttpFSPorts.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestHttpFSPorts.java new file mode 100644 index 00000000000..bb60d4782fe --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestHttpFSPorts.java @@ -0,0 +1,82 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdfs.web; + +import static org.junit.Assert.assertEquals; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.web.SWebHdfsFileSystem; +import org.apache.hadoop.hdfs.web.WebHdfsFileSystem; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.net.URI; + +public class TestHttpFSPorts { + private static final Configuration conf = new Configuration(); + + @Before + public void setupConfig() { + conf.setInt(DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_KEY, 123); + conf.setInt(DFSConfigKeys.DFS_NAMENODE_HTTPS_PORT_KEY, 456); + } + + @Test + public void testWebHdfsCustomDefaultPorts() throws IOException { + URI uri = URI.create("webhdfs://localhost"); + WebHdfsFileSystem fs = (WebHdfsFileSystem) FileSystem.get(uri, conf); + + assertEquals(123, fs.getDefaultPort()); + assertEquals(uri, fs.getUri()); + assertEquals("127.0.0.1:123", fs.getCanonicalServiceName()); + } + + @Test + public void testWebHdfsCustomUriPortWithCustomDefaultPorts() throws IOException { + URI uri = URI.create("webhdfs://localhost:789"); + WebHdfsFileSystem fs = (WebHdfsFileSystem) FileSystem.get(uri, conf); + + assertEquals(123, fs.getDefaultPort()); + assertEquals(uri, fs.getUri()); + assertEquals("127.0.0.1:789", fs.getCanonicalServiceName()); + } + + @Test + public void testSWebHdfsCustomDefaultPorts() throws IOException { + URI uri = URI.create("swebhdfs://localhost"); + SWebHdfsFileSystem fs = (SWebHdfsFileSystem) FileSystem.get(uri, conf); + + assertEquals(456, fs.getDefaultPort()); + assertEquals(uri, fs.getUri()); + assertEquals("127.0.0.1:456", fs.getCanonicalServiceName()); + } + + @Test + public void testSwebHdfsCustomUriPortWithCustomDefaultPorts() throws IOException { + URI uri = URI.create("swebhdfs://localhost:789"); + SWebHdfsFileSystem fs = (SWebHdfsFileSystem) FileSystem.get(uri, conf); + + assertEquals(456, fs.getDefaultPort()); + assertEquals(uri, fs.getUri()); + assertEquals("127.0.0.1:789", fs.getCanonicalServiceName()); + } +}