From 9684b659ffee6664d8059149c0c204ca92f71d4b Mon Sep 17 00:00:00 2001 From: Zhe Zhang Date: Tue, 12 Jul 2016 23:18:37 -0700 Subject: [PATCH] HDFS-10544. Balancer doesn't work with IPFailoverProxyProvider. (cherry picked from commit 087290e6b1cb1082646d966b65494082712ebe3e) (cherry picked from commit 5b4e916b3cae2540267d48716fd9240dfc055288) (cherry picked from commit af94dfaa53532d7c650478050621d216360e04b5) (cherry picked from commit c7e1e92c1afea72b37fc4788956caa20fea7e5c6) --- .../java/org/apache/hadoop/hdfs/DFSUtil.java | 46 ++++++++----- .../org/apache/hadoop/hdfs/TestDFSUtil.java | 67 ++++++++++++++++--- 2 files changed, 88 insertions(+), 25 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java index 151028f080a..031896ab545 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java @@ -955,11 +955,12 @@ public String toString() { } /** - * Get a URI for each configured nameservice. If a nameservice is - * HA-enabled, then the logical URI of the nameservice is returned. If the - * nameservice is not HA-enabled, then a URI corresponding to an RPC address - * of the single NN for that nameservice is returned, preferring the service - * RPC address over the client RPC address. + * Get a URI for each internal nameservice. If a nameservice is + * HA-enabled, and the configured failover proxy provider supports logical + * URIs, then the logical URI of the nameservice is returned. + * Otherwise, a URI corresponding to an RPC address of the single NN for that + * nameservice is returned, preferring the service RPC address over the + * client RPC address. * * @param conf configuration * @return a collection of all configured NN URIs, preferring service @@ -973,9 +974,10 @@ public static Collection getNsServiceRpcUris(Configuration conf) { /** * Get a URI for each configured nameservice. If a nameservice is - * HA-enabled, then the logical URI of the nameservice is returned. If the - * nameservice is not HA-enabled, then a URI corresponding to the address of - * the single NN for that nameservice is returned. + * HA-enabled, and the configured failover proxy provider supports logical + * URIs, then the logical URI of the nameservice is returned. + * Otherwise, a URI corresponding to the address of the single NN for that + * nameservice is returned. * * @param conf configuration * @param keys configuration keys to try in order to get the URI for non-HA @@ -992,15 +994,29 @@ public static Collection getNameServiceUris(Configuration conf, // URI for a config key for which we've already found a preferred entry, we // keep track of non-preferred keys here. Set nonPreferredUris = new HashSet(); - + for (String nsId : getNameServiceIds(conf)) { - if (HAUtil.isHAEnabled(conf, nsId)) { + URI nsUri; + try { + nsUri = new URI(HdfsConstants.HDFS_URI_SCHEME + "://" + nsId); + } catch (URISyntaxException ue) { + throw new IllegalArgumentException(ue); + } + /** + * Determine whether the logical URI of the name service can be resolved + * by the configured failover proxy provider. If not, we should try to + * resolve the URI here + */ + boolean useLogicalUri = false; + try { + useLogicalUri = HAUtil.useLogicalUri(conf, nsUri); + } catch (IOException e){ + LOG.warn("Getting exception while trying to determine if nameservice " + + nsId + " can use logical URI: " + e); + } + if (HAUtil.isHAEnabled(conf, nsId) && useLogicalUri) { // Add the logical URI of the nameservice. - try { - ret.add(new URI(HdfsConstants.HDFS_URI_SCHEME + "://" + nsId)); - } catch (URISyntaxException ue) { - throw new IllegalArgumentException(ue); - } + ret.add(nsUri); } else { // Add the URI corresponding to the address of the NN. boolean uriFound = false; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java index 046265f5520..2a43db2f24a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java @@ -62,6 +62,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; +import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.server.namenode.NameNode; @@ -527,7 +528,11 @@ public void testHANameNodesWithFederation() throws URISyntaxException { // Ditto for nameservice IDs, if multiple are defined assertEquals(null, DFSUtil.getNamenodeNameServiceId(conf)); assertEquals(null, DFSUtil.getSecondaryNameServiceId(conf)); - + + String proxyProviderKey = DFS_CLIENT_FAILOVER_PROXY_PROVIDER_KEY_PREFIX + + ".ns2"; + conf.set(proxyProviderKey, "org.apache.hadoop.hdfs.server.namenode.ha." + + "ConfiguredFailoverProxyProvider"); Collection uris = DFSUtil.getNameServiceUris(conf, DFS_NAMENODE_RPC_ADDRESS_KEY); assertEquals(2, uris.size()); assertTrue(uris.contains(new URI("hdfs://ns1"))); @@ -615,7 +620,8 @@ public void testSubstituteForWildcardAddress() throws IOException { @Test public void testGetNNUris() throws Exception { HdfsConfiguration conf = new HdfsConfiguration(); - + + final String NS1_NN_ADDR = "ns1-nn.example.com:8020"; final String NS1_NN1_ADDR = "ns1-nn1.example.com:8020"; final String NS1_NN2_ADDR = "ns1-nn2.example.com:8020"; final String NS2_NN_ADDR = "ns2-nn.example.com:8020"; @@ -629,23 +635,64 @@ public void testGetNNUris() throws Exception { DFS_NAMENODE_RPC_ADDRESS_KEY, "ns1", "nn1"), NS1_NN1_ADDR); conf.set(DFSUtil.addKeySuffixes( DFS_NAMENODE_RPC_ADDRESS_KEY, "ns1", "nn2"), NS1_NN2_ADDR); - + + conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, "ns1"), + NS1_NN_ADDR); conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, "ns2"), NS2_NN_ADDR); conf.set(DFS_NAMENODE_RPC_ADDRESS_KEY, "hdfs://" + NN1_ADDR); conf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, "hdfs://" + NN2_ADDR); - + + /** + * {@link DFSUtil#getInternalNsRpcUris} decides whether to resolve a logical + * URI based on whether the failover proxy provider supports logical URIs. + * We will test both cases. + * + * First configure ns1 to use {@link IPFailoverProxyProvider} which doesn't + * support logical Uris. So {@link DFSUtil#getInternalNsRpcUris} will + * resolve the logical URI of ns1 based on the configured value at + * dfs.namenode.servicerpc-address.ns1, which is {@link NS1_NN_ADDR} + */ + String proxyProviderKey = DFS_CLIENT_FAILOVER_PROXY_PROVIDER_KEY_PREFIX + + ".ns1"; + conf.set(proxyProviderKey, "org.apache.hadoop.hdfs.server.namenode.ha." + + "IPFailoverProxyProvider"); Collection uris = DFSUtil.getNameServiceUris(conf, DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, DFS_NAMENODE_RPC_ADDRESS_KEY); - assertEquals(4, uris.size()); - assertTrue(uris.contains(new URI("hdfs://ns1"))); - assertTrue(uris.contains(new URI("hdfs://" + NS2_NN_ADDR))); - assertTrue(uris.contains(new URI("hdfs://" + NN1_ADDR))); - assertTrue(uris.contains(new URI("hdfs://" + NN2_ADDR))); - + assertEquals("Incorrect number of URIs returned", 4, uris.size()); + assertTrue("Missing URI for name service ns1", + uris.contains(new URI("hdfs://" + NS1_NN_ADDR))); + assertTrue("Missing URI for name service ns2", + uris.contains(new URI("hdfs://" + NS2_NN_ADDR))); + assertTrue("Missing URI for RPC address", + uris.contains(new URI("hdfs://" + NN1_ADDR))); + assertTrue("Missing FS default URI", + uris.contains(new URI("hdfs://" + NN2_ADDR))); + + /** + * Second, test ns1 with {@link ConfiguredFailoverProxyProvider} which does + * support logical URIs. So instead of {@link NS1_NN_ADDR}, the logical URI + * of ns1, hdfs://ns1, will be returned. + */ + conf.set(proxyProviderKey, "org.apache.hadoop.hdfs.server.namenode.ha." + + "ConfiguredFailoverProxyProvider"); + + uris = DFSUtil.getNameServiceUris(conf, + DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, DFS_NAMENODE_RPC_ADDRESS_KEY); + + assertEquals("Incorrect number of URIs returned", 4, uris.size()); + assertTrue("Missing URI for name service ns1", + uris.contains(new URI("hdfs://ns1"))); + assertTrue("Missing URI for name service ns2", + uris.contains(new URI("hdfs://" + NS2_NN_ADDR))); + assertTrue("Missing URI for RPC address", + uris.contains(new URI("hdfs://" + NN1_ADDR))); + assertTrue("Missing FS default URI", + uris.contains(new URI("hdfs://" + NN2_ADDR))); + // Make sure that non-HDFS URIs in fs.defaultFS don't get included. conf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, "viewfs://vfs-name.example.com");