Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

MAPREDUCE-7429 Application log link is not accessible via TimelineServer WebUI in IPV6 scenario

This commit is contained in:
Daniel-009497 2022-12-17 17:14:07 +08:00
parent ca3526da92
commit 85aa87771d
4 changed files with 125 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
View File

@ -46,6 +46,7 @@ import java.util.concurrent.ConcurrentHashMap;
import javax.net.SocketFactory;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.thirdparty.com.google.common.cache.Cache;
import org.apache.hadoop.thirdparty.com.google.common.cache.CacheBuilder;
@ -229,6 +230,23 @@ public class NetUtils {
}
target = target.trim();
boolean hasScheme = target.contains("://");
if (NetUtils.isIPV6Address(target)) {
// if scheme exists in the target, for example:
// https://ffff:ffff:ffff:ffff::1:8088 will be formed like
// https://[ffff:ffff:ffff:ffff::1]:8088
if (hasScheme) {
int i = target.lastIndexOf("/");
String scheme = target.substring(0, i + 1);
String ipAddrWithPort = target.substring(i + 1);
target = scheme + normalizeV6Address(ipAddrWithPort);
} else {
// if scheme does not exists in the target
// for example : ffff:ffff:ffff:ffff::1:8088 will be formed like
// [ffff:ffff:ffff:ffff::1]:8088
target = normalizeV6Address(target);
}
}
URI uri = createURI(target, hasScheme, helpText, useCacheIfPresent);
String host = uri.getHost();
@ -247,6 +265,28 @@ public class NetUtils {
return createSocketAddrForHost(host, port);
}
public static String normalizeV6Address(final String target) {
String normalizedAddr = target;
if (!target.startsWith("[")) {
if (target.contains("%")) {
int i = target.lastIndexOf('%');
String port = target.trim().substring(target.lastIndexOf(":") + 1);
String addr = target.trim().substring(0, i);
normalizedAddr = "[" + addr + "]" + ":" + port;
} else {
int i = target.lastIndexOf(':');
String port = target.substring(target.lastIndexOf(":") + 1);
String addr = target.substring(0, i);
normalizedAddr = "[" + addr + "]" + ":" + port;
}
}
return normalizedAddr;
}
public static boolean isIPV6Address(String addr) {
return StringUtils.countMatches(addr, ":") > 2;
}
private static final long URI_CACHE_SIZE_DEFAULT = 1000;
private static final long URI_CACHE_EXPIRE_TIME_DEFAULT = 12;
private static final Cache<String, URI> URI_CACHE = CacheBuilder.newBuilder()

70
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java
View File

@ -798,4 +798,74 @@ public class TestNetUtils {
String gotStr = StringUtils.join(got, ", ");
assertEquals(expectStr, gotStr);
}
@Test
public void testIPV6Address() {
final String IPV6_SAMPLE_ADDRESS1 = "FFFF:FFFF:FFFF:FFFF::1:12345";
final String IPV6_SAMPLE_ADDRESS2 = "FFFF:FFFF:FFFF::1:FFFF:12345";
final String IPV6_SAMPLE_ADDRESS3 = "FFFF:FFFF::1:FFFF:FFFF:12345";
final String IPV4_SAMPLE_ADDRESS = "192.168.0.1:12345";
assertTrue(NetUtils.isIPV6Address(IPV6_SAMPLE_ADDRESS1));
assertTrue(NetUtils.isIPV6Address(IPV6_SAMPLE_ADDRESS2));
assertTrue(NetUtils.isIPV6Address(IPV6_SAMPLE_ADDRESS3));
assertFalse(NetUtils.isIPV6Address(IPV4_SAMPLE_ADDRESS));
}
@Test
public void testNormalizeIPV6Address() {
final String IPV6_SAMPLE_ADDRESS1 = "FFFF:FFFF:FFFF:FFFF::1";
final String IPV6_SAMPLE_ADDRESS2 = "FFFF:FFFF:FFFF::1:FFFF";
final String IPV6_SAMPLE_ADDRESS3 = "FFFF:FFFF::1:FFFF:FFFF";
final String PORT_SUFFIX = ":12345";
String IPV6_SAMPLE_ADDRESS1_WITH_PORT = IPV6_SAMPLE_ADDRESS1 + PORT_SUFFIX;
String IPV6_SAMPLE_ADDRESS2_WITH_PORT = IPV6_SAMPLE_ADDRESS2 + PORT_SUFFIX;
String IPV6_SAMPLE_ADDRESS3_WITH_PORT = IPV6_SAMPLE_ADDRESS3 + PORT_SUFFIX;
assertEquals("[" + IPV6_SAMPLE_ADDRESS1 + "]" + PORT_SUFFIX , NetUtils.normalizeV6Address(IPV6_SAMPLE_ADDRESS1_WITH_PORT));
assertEquals("[" + IPV6_SAMPLE_ADDRESS2 + "]" + PORT_SUFFIX , NetUtils.normalizeV6Address(IPV6_SAMPLE_ADDRESS2_WITH_PORT));
assertEquals("[" + IPV6_SAMPLE_ADDRESS3 + "]" + PORT_SUFFIX , NetUtils.normalizeV6Address(IPV6_SAMPLE_ADDRESS3_WITH_PORT));
}
@Test
public void testCreateSocketAddressWithIPV6() throws Throwable {
final String IPV6_SAMPLE_ADDRESS1 = "FFFF:FFFF:FFFF:FFFF::1";
final String IPV6_SAMPLE_ADDRESS2 = "FFFF:FFFF:FFFF::1:FFFF";
final String IPV6_SAMPLE_ADDRESS3 = "FFFF:FFFF::1:FFFF:FFFF";
final String IPV4_SAMPLE_ADDRESS = "192.168.0.1";
final String PORT_SUFFIX = ":12345";
final int DEFAULT_PORT = 10000;
String IPV6_SAMPLE_ADDRESS1_WITH_PORT = IPV6_SAMPLE_ADDRESS1 + PORT_SUFFIX;
String IPV6_SAMPLE_ADDRESS2_WITH_PORT = IPV6_SAMPLE_ADDRESS2 + PORT_SUFFIX;
String IPV6_SAMPLE_ADDRESS3_WITH_PORT = IPV6_SAMPLE_ADDRESS3 + PORT_SUFFIX;
String IPV4_SAMPLE_ADDRESS_WITH_PORT = IPV4_SAMPLE_ADDRESS + PORT_SUFFIX;
InetSocketAddress addr1 = NetUtils.createSocketAddr(IPV6_SAMPLE_ADDRESS1_WITH_PORT, DEFAULT_PORT, "myconfig1");
InetSocketAddress addr2 = NetUtils.createSocketAddr(IPV6_SAMPLE_ADDRESS2_WITH_PORT, DEFAULT_PORT, "myconfig2");
InetSocketAddress addr3 = NetUtils.createSocketAddr(IPV6_SAMPLE_ADDRESS3_WITH_PORT, DEFAULT_PORT, "myconfig3");
InetSocketAddress addr4 = NetUtils.createSocketAddr(IPV4_SAMPLE_ADDRESS_WITH_PORT, DEFAULT_PORT, "myconfig4");
assertEquals("[" + IPV6_SAMPLE_ADDRESS1 + "]", addr1.getHostName());
assertEquals("[" + IPV6_SAMPLE_ADDRESS2 + "]", addr2.getHostName());
assertEquals("[" + IPV6_SAMPLE_ADDRESS3 + "]", addr3.getHostName());
assertEquals("[" + IPV4_SAMPLE_ADDRESS + "]", addr4.getHostName());
assertEquals(12345, addr1.getPort());
assertEquals(12345, addr2.getPort());
assertEquals(12345, addr3.getPort());
assertEquals(12345, addr4.getPort());
final String IPV6_SAMPLE_ADDRESS1_WITHSCOPE = IPV6_SAMPLE_ADDRESS1 + "%2";
final String IPV6_SAMPLE_ADDRESS2_WITHSCOPE = IPV6_SAMPLE_ADDRESS2 + "%2";
final String IPV6_SAMPLE_ADDRESS3_WITHSCOPE = IPV6_SAMPLE_ADDRESS3 + "%2";
IPV6_SAMPLE_ADDRESS1_WITH_PORT = IPV6_SAMPLE_ADDRESS1_WITHSCOPE + PORT_SUFFIX;
IPV6_SAMPLE_ADDRESS2_WITH_PORT = IPV6_SAMPLE_ADDRESS2_WITHSCOPE + PORT_SUFFIX;
IPV6_SAMPLE_ADDRESS3_WITH_PORT = IPV6_SAMPLE_ADDRESS3_WITHSCOPE + PORT_SUFFIX;
addr1 = NetUtils.createSocketAddr(IPV6_SAMPLE_ADDRESS1_WITH_PORT, DEFAULT_PORT, "myconfig1");
addr2 = NetUtils.createSocketAddr(IPV6_SAMPLE_ADDRESS2_WITH_PORT, DEFAULT_PORT, "myconfig2");
addr3 = NetUtils.createSocketAddr(IPV6_SAMPLE_ADDRESS3_WITH_PORT, DEFAULT_PORT, "myconfig3");
assertEquals("[" + IPV6_SAMPLE_ADDRESS1 + "]", addr1.getHostName());
assertEquals("[" + IPV6_SAMPLE_ADDRESS2 + "]", addr2.getHostName());
assertEquals("[" + IPV6_SAMPLE_ADDRESS3 + "]", addr3.getHostName());
assertEquals(12345, addr1.getPort());
assertEquals(12345, addr2.getPort());
assertEquals(12345, addr3.getPort());
}
}

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineConnector.java
View File

@ -35,6 +35,8 @@ import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLSocketFactory;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler;
import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler;
import org.slf4j.Logger;
@ -197,6 +199,12 @@ public class TimelineConnector extends AbstractService {
public static URI constructResURI(Configuration conf, String address,
String uri) {
if (NetUtils.isIPV6Address(address)) {
// normalize the ip: port with ipv6 scheme
// For an address like ffff:ffff:ffff::0001:8080 will be transformed into
// [ffff:ffff:ffff::0001]:8080
address = NetUtils.normalizeV6Address(address);
}
return URI.create(
JOINER.join(YarnConfiguration.useHttps(conf) ? "https://" : "http://",
address, uri));

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
View File

@ -30,6 +30,7 @@ import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.conf.Configuration;
@ -428,6 +429,12 @@ public class WebAppUtils {
user == null || user.isEmpty()) {
return null;
}
String[] ipAdds = serverHttpAddress.split("//");
if (ipAdds.length >= 2 && NetUtils.isIPV6Address(ipAdds[1])) {
ipAdds[1] = NetUtils.normalizeV6Address(ipAdds[1]);
serverHttpAddress = ipAdds[0] + "//" + ipAdds[1];
}
return PATH_JOINER.join(serverHttpAddress, "applicationhistory", "logs",
allocatedNode, containerId, entity, user);
}