From a428d4f50e8caf553fb19a26200ec58f0b83da5d Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Wed, 10 Aug 2016 22:17:11 -0700 Subject: [PATCH] HDFS-8897. Balancer should handle fs.defaultFS trailing slash in HA. Contributed by John Zhuge. --- .../java/org/apache/hadoop/hdfs/DFSUtil.java | 60 +++++++++++-------- .../org/apache/hadoop/hdfs/TestDFSUtil.java | 54 +++++++++++------ 2 files changed, 71 insertions(+), 43 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java index 1ff39e12299..83870cfd2d2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java @@ -771,12 +771,7 @@ public class DFSUtil { Set nonPreferredUris = new HashSet(); for (String nsId : nameServices) { - URI nsUri; - try { - nsUri = new URI(HdfsConstants.HDFS_URI_SCHEME + "://" + nsId); - } catch (URISyntaxException ue) { - throw new IllegalArgumentException(ue); - } + URI nsUri = createUri(HdfsConstants.HDFS_URI_SCHEME, nsId, -1); /** * Determine whether the logical URI of the name service can be resolved * by the configured failover proxy provider. If not, we should try to @@ -816,7 +811,8 @@ public class DFSUtil { for (String key : keys) { String addr = conf.get(key); if (addr != null) { - URI uri = createUri("hdfs", NetUtils.createSocketAddr(addr)); + URI uri = createUri(HdfsConstants.HDFS_URI_SCHEME, + NetUtils.createSocketAddr(addr)); if (!uriFound) { uriFound = true; ret.add(uri); @@ -834,19 +830,21 @@ public class DFSUtil { // nor the rpc-address (which overrides defaultFS) is given. if (!uriFound) { URI defaultUri = FileSystem.getDefaultUri(conf); + if (defaultUri != null) { + // checks if defaultUri is ip:port format + // and convert it to hostname:port format + if (defaultUri.getPort() != -1) { + defaultUri = createUri(defaultUri.getScheme(), + NetUtils.createSocketAddr(defaultUri.getHost(), + defaultUri.getPort())); + } - // checks if defaultUri is ip:port format - // and convert it to hostname:port format - if (defaultUri != null && (defaultUri.getPort() != -1)) { - defaultUri = createUri(defaultUri.getScheme(), - NetUtils.createSocketAddr(defaultUri.getHost(), - defaultUri.getPort())); - } + defaultUri = trimUri(defaultUri); - if (defaultUri != null && - HdfsConstants.HDFS_URI_SCHEME.equals(defaultUri.getScheme()) && - !nonPreferredUris.contains(defaultUri)) { - ret.add(defaultUri); + if (HdfsConstants.HDFS_URI_SCHEME.equals(defaultUri.getScheme()) && + !nonPreferredUris.contains(defaultUri)) { + ret.add(defaultUri); + } } } @@ -1175,16 +1173,30 @@ public class DFSUtil { public boolean match(InetSocketAddress s); } - /** Create a URI from the scheme and address */ + /** Create an URI from scheme and address. */ public static URI createUri(String scheme, InetSocketAddress address) { + return createUri(scheme, address.getHostName(), address.getPort()); + } + + /** Create an URI from scheme, host, and port. */ + public static URI createUri(String scheme, String host, int port) { try { - return new URI(scheme, null, address.getHostName(), address.getPort(), - null, null, null); - } catch (URISyntaxException ue) { - throw new IllegalArgumentException(ue); + return new URI(scheme, null, host, port, null, null, null); + } catch (URISyntaxException x) { + throw new IllegalArgumentException(x.getMessage(), x); } } - + + /** Remove unnecessary path from HDFS URI. */ + static URI trimUri(URI uri) { + String path = uri.getPath(); + if (HdfsConstants.HDFS_URI_SCHEME.equals(uri.getScheme()) && + path != null && !path.isEmpty()) { + uri = createUri(uri.getScheme(), uri.getHost(), uri.getPort()); + } + return uri; + } + /** * Add protobuf based protocol to the {@link org.apache.hadoop.ipc.RPC.Server} * @param conf configuration diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java index bef5e330276..b7db60148fd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java @@ -81,7 +81,11 @@ import org.junit.Test; import com.google.common.collect.Sets; public class TestDFSUtil { - + + static final String NS1_NN_ADDR = "ns1-nn.example.com:9820"; + static final String NS1_NN1_ADDR = "ns1-nn1.example.com:9820"; + static final String NS1_NN2_ADDR = "ns1-nn2.example.com:9820"; + /** * Reset to default UGI settings since some tests change them. */ @@ -587,8 +591,6 @@ public class TestDFSUtil { @Test public void testGetHaNnHttpAddresses() throws IOException { final String LOGICAL_HOST_NAME = "ns1"; - final String NS1_NN1_ADDR = "ns1-nn1.example.com:9820"; - final String NS1_NN2_ADDR = "ns1-nn2.example.com:9820"; Configuration conf = createWebHDFSHAConfiguration(LOGICAL_HOST_NAME, NS1_NN1_ADDR, NS1_NN2_ADDR); @@ -637,9 +639,6 @@ public class TestDFSUtil { public void testGetNNUris() throws Exception { HdfsConfiguration conf = new HdfsConfiguration(); - final String NS1_NN_ADDR = "ns1-nn.example.com:9820"; - final String NS1_NN1_ADDR = "ns1-nn1.example.com:9820"; - final String NS1_NN2_ADDR = "ns1-nn2.example.com:9820"; final String NS2_NN_ADDR = "ns2-nn.example.com:9820"; final String NN1_ADDR = "nn.example.com:9820"; final String NN1_SRVC_ADDR = "nn.example.com:9821"; @@ -648,12 +647,10 @@ public class TestDFSUtil { conf.set(DFS_NAMESERVICES, "ns1"); conf.set(DFSUtil.addKeySuffixes( DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, "ns1"), NS1_NN1_ADDR); - conf.set(DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, "hdfs://" + NN2_ADDR); conf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, "hdfs://" + NN1_ADDR); Collection uris = DFSUtil.getInternalNsRpcUris(conf); - assertEquals("Incorrect number of URIs returned", 2, uris.size()); assertTrue("Missing URI for name service ns1", uris.contains(new URI("hdfs://" + NS1_NN1_ADDR))); @@ -668,15 +665,11 @@ public class TestDFSUtil { DFS_NAMENODE_RPC_ADDRESS_KEY, "ns1", "nn1"), NS1_NN1_ADDR); conf.set(DFSUtil.addKeySuffixes( DFS_NAMENODE_RPC_ADDRESS_KEY, "ns1", "nn2"), NS1_NN2_ADDR); - conf.set(DFSUtil.addKeySuffixes( DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, "ns1"), NS1_NN_ADDR); - conf.set(DFSUtil.addKeySuffixes( DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, "ns2"), NS2_NN_ADDR); - conf.set(DFS_NAMENODE_RPC_ADDRESS_KEY, "hdfs://" + NN1_ADDR); - conf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, "hdfs://" + NN2_ADDR); /** @@ -714,7 +707,6 @@ public class TestDFSUtil { + "ConfiguredFailoverProxyProvider"); uris = DFSUtil.getInternalNsRpcUris(conf); - assertEquals("Incorrect number of URIs returned", 3, uris.size()); assertTrue("Missing URI for name service ns1", uris.contains(new URI("hdfs://ns1"))); @@ -728,7 +720,6 @@ public class TestDFSUtil { "viewfs://vfs-name.example.com"); uris = DFSUtil.getInternalNsRpcUris(conf); - assertEquals("Incorrect number of URIs returned", 3, uris.size()); assertTrue("Missing URI for name service ns1", uris.contains(new URI("hdfs://ns1"))); @@ -742,7 +733,6 @@ public class TestDFSUtil { conf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, "hdfs://ns1"); uris = DFSUtil.getInternalNsRpcUris(conf); - assertEquals("Incorrect number of URIs returned", 3, uris.size()); assertTrue("Missing URI for name service ns1", uris.contains(new URI("hdfs://ns1"))); @@ -756,7 +746,6 @@ public class TestDFSUtil { conf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, "hdfs://" + NN1_ADDR); uris = DFSUtil.getInternalNsRpcUris(conf); - assertEquals("Incorrect number of URIs returned", 1, uris.size()); assertTrue("Missing URI for RPC address (defaultFS)", uris.contains(new URI("hdfs://" + NN1_ADDR))); @@ -766,7 +755,6 @@ public class TestDFSUtil { conf.set(DFS_NAMENODE_RPC_ADDRESS_KEY, NN2_ADDR); uris = DFSUtil.getInternalNsRpcUris(conf); - assertEquals("Incorrect number of URIs returned", 1, uris.size()); assertTrue("Missing URI for RPC address", uris.contains(new URI("hdfs://" + NN2_ADDR))); @@ -778,7 +766,6 @@ public class TestDFSUtil { conf.set(DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, NN1_ADDR); uris = DFSUtil.getInternalNsRpcUris(conf); - assertEquals("Incorrect number of URIs returned", 1, uris.size()); assertTrue("Missing URI for service ns1", uris.contains(new URI("hdfs://" + NN1_ADDR))); @@ -790,12 +777,41 @@ public class TestDFSUtil { conf.set(DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, NN1_SRVC_ADDR); uris = DFSUtil.getInternalNsRpcUris(conf); - assertEquals("Incorrect number of URIs returned", 1, uris.size()); assertTrue("Missing URI for service address", uris.contains(new URI("hdfs://" + NN1_SRVC_ADDR))); } + @Test + public void testGetNNUris2() throws Exception { + // Make sure that an HA URI plus a slash being the default URI doesn't + // result in multiple entries being returned. + HdfsConfiguration conf = new HdfsConfiguration(); + conf.set(DFS_NAMESERVICES, "ns1"); + conf.set(DFSUtil.addKeySuffixes(DFS_HA_NAMENODES_KEY_PREFIX, "ns1"), + "nn1,nn2"); + conf.set(DFSUtil.addKeySuffixes( + DFS_NAMENODE_RPC_ADDRESS_KEY, "ns1", "nn1"), NS1_NN1_ADDR); + conf.set(DFSUtil.addKeySuffixes( + DFS_NAMENODE_RPC_ADDRESS_KEY, "ns1", "nn2"), NS1_NN2_ADDR); + + conf.set(DFSUtil.addKeySuffixes( + DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, "ns1"), NS1_NN_ADDR); + + String proxyProviderKey = HdfsClientConfigKeys.Failover. + PROXY_PROVIDER_KEY_PREFIX + ".ns1"; + conf.set(proxyProviderKey, "org.apache.hadoop.hdfs.server.namenode.ha." + + "ConfiguredFailoverProxyProvider"); + + conf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, "hdfs://ns1/"); + + Collection uris = DFSUtil.getInternalNsRpcUris(conf); + + assertEquals("Incorrect number of URIs returned", 1, uris.size()); + assertTrue("Missing URI for name service ns1", + uris.contains(new URI("hdfs://ns1"))); + } + @Test (timeout=15000) public void testLocalhostReverseLookup() { // 127.0.0.1 -> localhost reverse resolution does not happen on Windows.