diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java index c49706d66f2..45eb42f3c03 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java @@ -46,6 +46,7 @@ import java.util.concurrent.ConcurrentHashMap; import javax.net.SocketFactory; +import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.thirdparty.com.google.common.cache.Cache; import org.apache.hadoop.thirdparty.com.google.common.cache.CacheBuilder; @@ -229,6 +230,23 @@ public class NetUtils { } target = target.trim(); boolean hasScheme = target.contains("://"); + if (NetUtils.isIPV6Address(target)) { + // if scheme exists in the target, for example: + // https://ffff:ffff:ffff:ffff::1:8088 will be formed like + // https://[ffff:ffff:ffff:ffff::1]:8088 + if (hasScheme) { + int i = target.lastIndexOf("/"); + String scheme = target.substring(0, i + 1); + String ipAddrWithPort = target.substring(i + 1); + target = scheme + normalizeV6Address(ipAddrWithPort); + } else { + // if scheme does not exists in the target + // for example : ffff:ffff:ffff:ffff::1:8088 will be formed like + // [ffff:ffff:ffff:ffff::1]:8088 + target = normalizeV6Address(target); + } + } + URI uri = createURI(target, hasScheme, helpText, useCacheIfPresent); String host = uri.getHost(); @@ -247,6 +265,28 @@ public class NetUtils { return createSocketAddrForHost(host, port); } + public static String normalizeV6Address(final String target) { + String normalizedAddr = target; + if (!target.startsWith("[")) { + if (target.contains("%")) { + int i = target.lastIndexOf('%'); + String port = target.trim().substring(target.lastIndexOf(":") + 1); + String addr = target.trim().substring(0, i); + normalizedAddr = "[" + addr + "]" + ":" + port; + } else { + int i = target.lastIndexOf(':'); + String port = target.substring(target.lastIndexOf(":") + 1); + String addr = target.substring(0, i); + normalizedAddr = "[" + addr + "]" + ":" + port; + } + } + return normalizedAddr; + } + + public static boolean isIPV6Address(String addr) { + return StringUtils.countMatches(addr, ":") > 2; + } + private static final long URI_CACHE_SIZE_DEFAULT = 1000; private static final long URI_CACHE_EXPIRE_TIME_DEFAULT = 12; private static final Cache URI_CACHE = CacheBuilder.newBuilder() diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java index 4b18d74d9b7..0e6d71944a9 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java @@ -798,4 +798,74 @@ public class TestNetUtils { String gotStr = StringUtils.join(got, ", "); assertEquals(expectStr, gotStr); } + + @Test + public void testIPV6Address() { + final String IPV6_SAMPLE_ADDRESS1 = "FFFF:FFFF:FFFF:FFFF::1:12345"; + final String IPV6_SAMPLE_ADDRESS2 = "FFFF:FFFF:FFFF::1:FFFF:12345"; + final String IPV6_SAMPLE_ADDRESS3 = "FFFF:FFFF::1:FFFF:FFFF:12345"; + final String IPV4_SAMPLE_ADDRESS = "192.168.0.1:12345"; + assertTrue(NetUtils.isIPV6Address(IPV6_SAMPLE_ADDRESS1)); + assertTrue(NetUtils.isIPV6Address(IPV6_SAMPLE_ADDRESS2)); + assertTrue(NetUtils.isIPV6Address(IPV6_SAMPLE_ADDRESS3)); + assertFalse(NetUtils.isIPV6Address(IPV4_SAMPLE_ADDRESS)); + } + + @Test + public void testNormalizeIPV6Address() { + final String IPV6_SAMPLE_ADDRESS1 = "FFFF:FFFF:FFFF:FFFF::1"; + final String IPV6_SAMPLE_ADDRESS2 = "FFFF:FFFF:FFFF::1:FFFF"; + final String IPV6_SAMPLE_ADDRESS3 = "FFFF:FFFF::1:FFFF:FFFF"; + final String PORT_SUFFIX = ":12345"; + String IPV6_SAMPLE_ADDRESS1_WITH_PORT = IPV6_SAMPLE_ADDRESS1 + PORT_SUFFIX; + String IPV6_SAMPLE_ADDRESS2_WITH_PORT = IPV6_SAMPLE_ADDRESS2 + PORT_SUFFIX; + String IPV6_SAMPLE_ADDRESS3_WITH_PORT = IPV6_SAMPLE_ADDRESS3 + PORT_SUFFIX; + assertEquals("[" + IPV6_SAMPLE_ADDRESS1 + "]" + PORT_SUFFIX , NetUtils.normalizeV6Address(IPV6_SAMPLE_ADDRESS1_WITH_PORT)); + assertEquals("[" + IPV6_SAMPLE_ADDRESS2 + "]" + PORT_SUFFIX , NetUtils.normalizeV6Address(IPV6_SAMPLE_ADDRESS2_WITH_PORT)); + assertEquals("[" + IPV6_SAMPLE_ADDRESS3 + "]" + PORT_SUFFIX , NetUtils.normalizeV6Address(IPV6_SAMPLE_ADDRESS3_WITH_PORT)); + } + + @Test + public void testCreateSocketAddressWithIPV6() throws Throwable { + final String IPV6_SAMPLE_ADDRESS1 = "FFFF:FFFF:FFFF:FFFF::1"; + final String IPV6_SAMPLE_ADDRESS2 = "FFFF:FFFF:FFFF::1:FFFF"; + final String IPV6_SAMPLE_ADDRESS3 = "FFFF:FFFF::1:FFFF:FFFF"; + final String IPV4_SAMPLE_ADDRESS = "192.168.0.1"; + final String PORT_SUFFIX = ":12345"; + final int DEFAULT_PORT = 10000; + + String IPV6_SAMPLE_ADDRESS1_WITH_PORT = IPV6_SAMPLE_ADDRESS1 + PORT_SUFFIX; + String IPV6_SAMPLE_ADDRESS2_WITH_PORT = IPV6_SAMPLE_ADDRESS2 + PORT_SUFFIX; + String IPV6_SAMPLE_ADDRESS3_WITH_PORT = IPV6_SAMPLE_ADDRESS3 + PORT_SUFFIX; + String IPV4_SAMPLE_ADDRESS_WITH_PORT = IPV4_SAMPLE_ADDRESS + PORT_SUFFIX; + + InetSocketAddress addr1 = NetUtils.createSocketAddr(IPV6_SAMPLE_ADDRESS1_WITH_PORT, DEFAULT_PORT, "myconfig1"); + InetSocketAddress addr2 = NetUtils.createSocketAddr(IPV6_SAMPLE_ADDRESS2_WITH_PORT, DEFAULT_PORT, "myconfig2"); + InetSocketAddress addr3 = NetUtils.createSocketAddr(IPV6_SAMPLE_ADDRESS3_WITH_PORT, DEFAULT_PORT, "myconfig3"); + InetSocketAddress addr4 = NetUtils.createSocketAddr(IPV4_SAMPLE_ADDRESS_WITH_PORT, DEFAULT_PORT, "myconfig4"); + assertEquals("[" + IPV6_SAMPLE_ADDRESS1 + "]", addr1.getHostName()); + assertEquals("[" + IPV6_SAMPLE_ADDRESS2 + "]", addr2.getHostName()); + assertEquals("[" + IPV6_SAMPLE_ADDRESS3 + "]", addr3.getHostName()); + assertEquals("[" + IPV4_SAMPLE_ADDRESS + "]", addr4.getHostName()); + assertEquals(12345, addr1.getPort()); + assertEquals(12345, addr2.getPort()); + assertEquals(12345, addr3.getPort()); + assertEquals(12345, addr4.getPort()); + + final String IPV6_SAMPLE_ADDRESS1_WITHSCOPE = IPV6_SAMPLE_ADDRESS1 + "%2"; + final String IPV6_SAMPLE_ADDRESS2_WITHSCOPE = IPV6_SAMPLE_ADDRESS2 + "%2"; + final String IPV6_SAMPLE_ADDRESS3_WITHSCOPE = IPV6_SAMPLE_ADDRESS3 + "%2"; + IPV6_SAMPLE_ADDRESS1_WITH_PORT = IPV6_SAMPLE_ADDRESS1_WITHSCOPE + PORT_SUFFIX; + IPV6_SAMPLE_ADDRESS2_WITH_PORT = IPV6_SAMPLE_ADDRESS2_WITHSCOPE + PORT_SUFFIX; + IPV6_SAMPLE_ADDRESS3_WITH_PORT = IPV6_SAMPLE_ADDRESS3_WITHSCOPE + PORT_SUFFIX; + addr1 = NetUtils.createSocketAddr(IPV6_SAMPLE_ADDRESS1_WITH_PORT, DEFAULT_PORT, "myconfig1"); + addr2 = NetUtils.createSocketAddr(IPV6_SAMPLE_ADDRESS2_WITH_PORT, DEFAULT_PORT, "myconfig2"); + addr3 = NetUtils.createSocketAddr(IPV6_SAMPLE_ADDRESS3_WITH_PORT, DEFAULT_PORT, "myconfig3"); + assertEquals("[" + IPV6_SAMPLE_ADDRESS1 + "]", addr1.getHostName()); + assertEquals("[" + IPV6_SAMPLE_ADDRESS2 + "]", addr2.getHostName()); + assertEquals("[" + IPV6_SAMPLE_ADDRESS3 + "]", addr3.getHostName()); + assertEquals(12345, addr1.getPort()); + assertEquals(12345, addr2.getPort()); + assertEquals(12345, addr3.getPort()); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineConnector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineConnector.java index 5a216d20d35..c307972f3b5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineConnector.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineConnector.java @@ -35,6 +35,8 @@ import javax.net.ssl.HostnameVerifier; import javax.net.ssl.HttpsURLConnection; import javax.net.ssl.SSLSocketFactory; +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler; import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler; import org.slf4j.Logger; @@ -197,6 +199,12 @@ public class TimelineConnector extends AbstractService { public static URI constructResURI(Configuration conf, String address, String uri) { + if (NetUtils.isIPV6Address(address)) { + // normalize the ip: port with ipv6 scheme + // For an address like ffff:ffff:ffff::0001:8080 will be transformed into + // [ffff:ffff:ffff::0001]:8080 + address = NetUtils.normalizeV6Address(address); + } return URI.create( JOINER.join(YarnConfiguration.useHttps(conf) ? "https://" : "http://", address, uri)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java index 02fec115059..3f932176b2d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java @@ -30,6 +30,7 @@ import java.util.Arrays; import java.util.Iterator; import java.util.List; +import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.conf.Configuration; @@ -428,6 +429,12 @@ public class WebAppUtils { user == null || user.isEmpty()) { return null; } + String[] ipAdds = serverHttpAddress.split("//"); + if (ipAdds.length >= 2 && NetUtils.isIPV6Address(ipAdds[1])) { + ipAdds[1] = NetUtils.normalizeV6Address(ipAdds[1]); + serverHttpAddress = ipAdds[0] + "//" + ipAdds[1]; + } + return PATH_JOINER.join(serverHttpAddress, "applicationhistory", "logs", allocatedNode, containerId, entity, user); }