diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeId.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeId.java index a0b87a7be62..9080905bb48 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeId.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeId.java @@ -23,6 +23,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.classification.InterfaceStability.Stable; import org.apache.hadoop.yarn.util.Records; +import com.google.common.net.HostAndPort; /** *

NodeId is the unique identifier for a node.

@@ -116,17 +117,18 @@ public abstract class NodeId implements Comparable { @Public @Stable public static NodeId fromString(String nodeIdStr) { - String[] parts = nodeIdStr.split(":"); - if (parts.length != 2) { - throw new IllegalArgumentException("Invalid NodeId [" + nodeIdStr - + "]. Expected host:port"); + HostAndPort hp = HostAndPort.fromString(nodeIdStr); + if (!hp.hasPort()) { + throw new IllegalArgumentException( + "Invalid NodeId [" + nodeIdStr + "]. Expected host:port"); } try { - NodeId nodeId = - NodeId.newInstance(parts[0].trim(), Integer.parseInt(parts[1])); + String hostPortStr = hp.toString(); + String host = hostPortStr.substring(0, hostPortStr.lastIndexOf(":")); + NodeId nodeId = NodeId.newInstance(host, hp.getPort()); return nodeId; } catch (NumberFormatException e) { - throw new IllegalArgumentException("Invalid port: " + parts[1], e); + throw new IllegalArgumentException("Invalid port: " + hp.getPort(), e); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java index 67bc2b74fd1..4ef8510767a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java @@ -36,7 +36,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.URL; import org.apache.hadoop.yarn.factories.RecordFactory; - +import com.google.common.net.HostAndPort; /** * This class contains a set of utilities which help converting data structures @@ -114,11 +114,11 @@ public class ConverterUtils { @Private @InterfaceStability.Unstable - public static NodeId toNodeIdWithDefaultPort(String nodeIdStr) { - if (nodeIdStr.indexOf(":") < 0) { - return NodeId.fromString(nodeIdStr + ":0"); - } - return NodeId.fromString(nodeIdStr); + public static NodeId toNodeIdWithDefaultPort( + String nodeIdStr) { + HostAndPort hp = HostAndPort.fromString(nodeIdStr); + hp = hp.withDefaultPort(0); + return toNodeId(hp.toString()); } /* diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java index 5b578197100..6c28197dc3b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java @@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.webapp.BadRequestException; import org.apache.hadoop.yarn.webapp.NotFoundException; import org.apache.http.NameValuePair; import org.apache.http.client.utils.URLEncodedUtils; +import com.google.common.net.HostAndPort; import javax.servlet.http.HttpServletRequest; @@ -64,15 +65,13 @@ public class WebAppUtils { public static void setRMWebAppPort(Configuration conf, int port) { String hostname = getRMWebAppURLWithoutScheme(conf); - hostname = - (hostname.contains(":")) ? hostname.substring(0, hostname.indexOf(":")) - : hostname; - setRMWebAppHostnameAndPort(conf, hostname, port); + HostAndPort hp = HostAndPort.fromString(hostname); + setRMWebAppHostnameAndPort(conf, hp.getHost(), port); } public static void setRMWebAppHostnameAndPort(Configuration conf, String hostname, int port) { - String resolvedAddress = hostname + ":" + port; + String resolvedAddress = HostAndPort.fromParts(hostname, port).toString(); if (YarnConfiguration.useHttps(conf)) { conf.set(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS, resolvedAddress); } else { @@ -82,12 +81,11 @@ public class WebAppUtils { public static void setNMWebAppHostNameAndPort(Configuration conf, String hostName, int port) { + String hostPortString = HostAndPort.fromParts(hostName, port).toString(); if (YarnConfiguration.useHttps(conf)) { - conf.set(YarnConfiguration.NM_WEBAPP_HTTPS_ADDRESS, - hostName + ":" + port); + conf.set(YarnConfiguration.NM_WEBAPP_HTTPS_ADDRESS, hostPortString); } else { - conf.set(YarnConfiguration.NM_WEBAPP_ADDRESS, - hostName + ":" + port); + conf.set(YarnConfiguration.NM_WEBAPP_ADDRESS, hostPortString); } } @@ -325,7 +323,8 @@ public class WebAppUtils { String host = conf.getTrimmed(hostProperty); if (host != null && !host.isEmpty()) { if (webAppURLWithoutScheme.contains(":")) { - webAppURLWithoutScheme = host + ":" + webAppURLWithoutScheme.split(":")[1]; + String[] splits = webAppURLWithoutScheme.split(":"); + webAppURLWithoutScheme = host + ":" + splits[splits.length - 1]; } else { throw new YarnRuntimeException("webAppURLWithoutScheme must include port specification but doesn't: " + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java index 212e09c02e9..502414bb53e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java @@ -18,6 +18,8 @@ package org.apache.hadoop.yarn.conf; +import com.google.common.net.HostAndPort; +import org.apache.hadoop.net.NetUtils; import org.junit.Assert; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; @@ -74,7 +76,7 @@ public class TestYarnConfiguration { conf.set(YarnConfiguration.RM_ADDRESS, "rmtesting:9999"); String rmWebUrl = WebAppUtils.getRMWebAppURLWithScheme(conf); String[] parts = rmWebUrl.split(":"); - Assert.assertEquals("RM Web URL Port is incrrect", 24543, + Assert.assertEquals("RM Web URL Port is incorrect", 24543, Integer.parseInt(parts[parts.length - 1])); Assert.assertNotSame( "RM Web Url not resolved correctly. Should not be rmtesting", @@ -112,10 +114,9 @@ public class TestYarnConfiguration { YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS, YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT); - assertEquals( - new InetSocketAddress( - YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS.split(":")[0], - YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT), + assertEquals(new InetSocketAddress( + YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS.split(":")[0], + YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT), resourceTrackerAddress); //with address @@ -125,10 +126,8 @@ public class TestYarnConfiguration { YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS, YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT); - assertEquals( - new InetSocketAddress( - "10.0.0.1", - YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT), + assertEquals(new InetSocketAddress("10.0.0.1", + YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT), resourceTrackerAddress); //address and socket @@ -139,9 +138,23 @@ public class TestYarnConfiguration { YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS, YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT); assertEquals( - new InetSocketAddress( - "10.0.0.2", - 5001), + new InetSocketAddress( + "10.0.0.2", + 5001), + resourceTrackerAddress); + + // IPv6 address and socket + conf.set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, + "[2401:db00:20:a01e:face:0:5:0]:5001"); + resourceTrackerAddress = conf.getSocketAddr( + YarnConfiguration.RM_BIND_HOST, + YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, + YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS, + YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT); + assertEquals( + new InetSocketAddress( + "2401:db00:20:a01e:face:0:5:0", + 5001), resourceTrackerAddress); //bind host only @@ -152,10 +165,8 @@ public class TestYarnConfiguration { YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS, YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT); - assertEquals( - new InetSocketAddress( - "10.0.0.3", - YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT), + assertEquals(new InetSocketAddress("10.0.0.3", + YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT), resourceTrackerAddress); //bind host and address no port @@ -166,10 +177,8 @@ public class TestYarnConfiguration { YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS, YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT); - assertEquals( - new InetSocketAddress( - "0.0.0.0", - YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT), + assertEquals(new InetSocketAddress("0.0.0.0", + YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT), resourceTrackerAddress); //bind host and address with port @@ -180,10 +189,7 @@ public class TestYarnConfiguration { YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS, YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT); - assertEquals( - new InetSocketAddress( - "0.0.0.0", - 5003), + assertEquals(new InetSocketAddress("0.0.0.0", 5003), resourceTrackerAddress); } @@ -197,9 +203,8 @@ public class TestYarnConfiguration { //no override, old behavior. Won't work on a host named "yo.yo.yo" conf = new YarnConfiguration(); conf.set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, "yo.yo.yo"); - serverAddress = new InetSocketAddress( - YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS.split(":")[0], - Integer.parseInt(YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS.split(":")[1])); + serverAddress = newInetSocketAddressFromHostPort( + YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS); resourceTrackerConnectAddress = conf.updateConnectAddr( YarnConfiguration.RM_BIND_HOST, @@ -207,15 +212,15 @@ public class TestYarnConfiguration { YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS, serverAddress); - assertFalse(resourceTrackerConnectAddress.toString().startsWith("yo.yo.yo")); + assertFalse(NetUtils.getSocketAddressString(resourceTrackerConnectAddress) + .startsWith("yo.yo.yo")); //cause override with address conf = new YarnConfiguration(); conf.set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, "yo.yo.yo"); conf.set(YarnConfiguration.RM_BIND_HOST, "0.0.0.0"); - serverAddress = new InetSocketAddress( - YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS.split(":")[0], - Integer.parseInt(YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS.split(":")[1])); + serverAddress = newInetSocketAddressFromHostPort( + YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS); resourceTrackerConnectAddress = conf.updateConnectAddr( YarnConfiguration.RM_BIND_HOST, @@ -223,7 +228,8 @@ public class TestYarnConfiguration { YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS, serverAddress); - assertTrue(resourceTrackerConnectAddress.toString().startsWith("yo.yo.yo")); + assertTrue(NetUtils.getSocketAddressString(resourceTrackerConnectAddress) + .startsWith("yo.yo.yo")); //tests updateConnectAddr won't add suffix to NM service address configurations conf = new YarnConfiguration(); @@ -232,9 +238,8 @@ public class TestYarnConfiguration { conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true); conf.set(YarnConfiguration.RM_HA_ID, "rm1"); - serverAddress = new InetSocketAddress( - YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS.split(":")[0], - Integer.parseInt(YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS.split(":")[1])); + serverAddress = newInetSocketAddressFromHostPort( + YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS); InetSocketAddress localizerAddress = conf.updateConnectAddr( YarnConfiguration.NM_BIND_HOST, @@ -242,8 +247,15 @@ public class TestYarnConfiguration { YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS, serverAddress); - assertTrue(localizerAddress.toString().startsWith("yo.yo.yo")); + assertTrue(NetUtils.getSocketAddressString(localizerAddress) + .startsWith("yo.yo.yo")); assertNull(conf.get( HAUtil.addSuffix(YarnConfiguration.NM_LOCALIZER_ADDRESS, "rm1"))); } + + private InetSocketAddress newInetSocketAddressFromHostPort( + String hostPort) { + HostAndPort hp = HostAndPort.fromString(hostPort); + return new InetSocketAddress(hp.getHost(), hp.getPort()); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestConverterUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestConverterUtils.java index 57542218aa2..0256de2120e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestConverterUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestConverterUtils.java @@ -31,7 +31,7 @@ import org.apache.hadoop.yarn.api.records.NodeId; import org.junit.Test; public class TestConverterUtils { - + @Test public void testConvertUrlWithNoPort() throws URISyntaxException { Path expectedPath = new Path("hdfs://foo.com"); @@ -92,14 +92,24 @@ public class TestConverterUtils { @Test public void testNodeIdWithDefaultPort() throws URISyntaxException { NodeId nid; - + nid = ConverterUtils.toNodeIdWithDefaultPort("node:10"); assertThat(nid.getPort()).isEqualTo(10); assertThat(nid.getHost()).isEqualTo("node"); - + nid = ConverterUtils.toNodeIdWithDefaultPort("node"); assertThat(nid.getPort()).isEqualTo(0); assertThat(nid.getHost()).isEqualTo("node"); + + nid = ConverterUtils + .toNodeIdWithDefaultPort("[2401:db00:20:a01e:face:0:5:0]:10"); + assertEquals(nid.getPort(), 10); + assertEquals(nid.getHost(), "[2401:db00:20:a01e:face:0:5:0]"); + + nid = ConverterUtils + .toNodeIdWithDefaultPort("[2401:db00:20:a01e:face:0:5:0]"); + assertEquals(nid.getPort(), 0); + assertEquals(nid.getHost(), "[2401:db00:20:a01e:face:0:5:0]"); } @Test(expected = IllegalArgumentException.class) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/lib/TestZKClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/lib/TestZKClient.java index c2a86fcaa28..fb7a481e0ec 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/lib/TestZKClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/lib/TestZKClient.java @@ -27,6 +27,7 @@ import java.net.InetSocketAddress; import java.net.Socket; import org.apache.hadoop.net.ServerSocketUtil; +import com.google.common.net.HostAndPort; import org.junit.Assert; import org.apache.hadoop.yarn.lib.ZKClient; @@ -86,8 +87,9 @@ public class TestZKClient { long start = System.currentTimeMillis(); while (true) { try { - String host = hp.split(":")[0]; - int port = Integer.parseInt(hp.split(":")[1]); + HostAndPort hap = HostAndPort.fromString(hp); + String host = hap.getHost(); + int port = hap.getPort(); send4LetterWord(host, port, "stat"); } catch (IOException e) { return true; @@ -110,8 +112,9 @@ public class TestZKClient { long start = System.currentTimeMillis(); while (true) { try { - String host = hp.split(":")[0]; - int port = Integer.parseInt(hp.split(":")[1]); + HostAndPort hap = HostAndPort.fromString(hp); + String host = hap.getHost(); + int port = hap.getPort(); // if there are multiple hostports, just take the first one String result = send4LetterWord(host, port, "stat"); if (result.startsWith("Zookeeper version:")) { @@ -151,14 +154,15 @@ public class TestZKClient { } File dataDir = createTmpDir(BASETEST); zks = new ZooKeeperServer(dataDir, dataDir, 3000); - final int PORT = Integer.parseInt(hostPort.split(":")[1]); + HostAndPort hp = HostAndPort.fromString(hostPort); + final int port = hp.getPort(); if (factory == null) { factory = new NIOServerCnxnFactory(); - factory.configure(new InetSocketAddress(PORT), maxCnxns); + factory.configure(new InetSocketAddress(port), maxCnxns); } factory.startup(zks); Assert.assertTrue("waiting for server up", - waitForServerUp("127.0.0.1:" + PORT, + waitForServerUp("127.0.0.1:" + port, CONNECTION_TIMEOUT)); } @@ -172,10 +176,11 @@ public class TestZKClient { zkDb.close(); } catch (IOException ie) { } - final int PORT = Integer.parseInt(hostPort.split(":")[1]); + HostAndPort hp = HostAndPort.fromString(hostPort); + final int port = hp.getPort(); Assert.assertTrue("waiting for server down", - waitForServerDown("127.0.0.1:" + PORT, + waitForServerDown("127.0.0.1:" + port, CONNECTION_TIMEOUT)); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java index ddfcdd82026..5dfc91e3992 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java @@ -185,6 +185,7 @@ import java.util.Set; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock; import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock; +import com.google.common.net.HostAndPort; import static org.apache.hadoop.service.Service.STATE.STARTED; @@ -645,7 +646,7 @@ public class ContainerManagerImpl extends CompositeService implements //hostname found when querying for our hostname with the specified //address, combine the specified address with the actual port listened //on by the server - hostOverride = nmAddress.split(":")[0]; + hostOverride = HostAndPort.fromString(nmAddress).getHost(); } // setup node ID diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index a813a8524e6..7ede0332610 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -137,6 +137,7 @@ import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.apache.zookeeper.server.auth.DigestAuthenticationProvider; import org.eclipse.jetty.webapp.WebAppContext; +import com.google.common.net.HostAndPort; import java.io.IOException; import java.io.InputStream; import java.io.PrintStream; @@ -1399,8 +1400,8 @@ public class ResourceManager extends CompositeService builder.withAttribute(WebAppProxy.PROXY_CA, rmContext.getProxyCAManager().getProxyCA()); builder.withAttribute(WebAppProxy.FETCHER_ATTRIBUTE, fetcher); - String[] proxyParts = proxyHostAndPort.split(":"); - builder.withAttribute(WebAppProxy.PROXY_HOST_ATTRIBUTE, proxyParts[0]); + builder.withAttribute(WebAppProxy.PROXY_HOST_ATTRIBUTE, + HostAndPort.fromString(proxyHostAndPort).getHost()); } WebAppContext uiWebAppContext = null; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java index d433753701b..7f18c31da58 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java @@ -30,6 +30,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import com.google.common.net.HostAndPort; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Container; @@ -105,8 +106,10 @@ public class MockNM { this.capability = capability; this.resourceTracker = resourceTracker; this.version = version; - String[] splits = nodeIdStr.split(":"); - nodeId = BuilderUtils.newNodeId(splits[0], Integer.parseInt(splits[1])); + HostAndPort hostAndPort = HostAndPort.fromString(nodeIdStr); + String hostPortStr = hostAndPort.toString(); + String host = hostPortStr.substring(0, hostPortStr.lastIndexOf(":")); + nodeId = BuilderUtils.newNodeId(host, hostAndPort.getPort()); } public MockNM(String nodeIdStr, Resource capability, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java index f373e94b8a1..eaf93605b2c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java @@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.webproxy; import java.io.IOException; import java.net.URI; +import com.google.common.net.HostAndPort; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.http.HttpServer2; import org.apache.hadoop.security.authorize.AccessControlList; @@ -57,7 +58,8 @@ public class WebAppProxy extends AbstractService { @Override protected void serviceInit(Configuration conf) throws Exception { - String auth = conf.get(CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION); + String auth = conf.get( + CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION); if (auth == null || "simple".equals(auth)) { isSecurityEnabled = false; } else if ("kerberos".equals(auth)) { @@ -68,8 +70,7 @@ public class WebAppProxy extends AbstractService { " of " + auth); } String proxy = WebAppUtils.getProxyHostAndPort(conf); - String[] proxyParts = proxy.split(":"); - proxyHost = proxyParts[0]; + proxyHost = HostAndPort.fromString(proxy).getHost(); fetcher = new AppReportFetcher(conf); bindAddress = conf.get(YarnConfiguration.PROXY_ADDRESS); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java index c023aaebeca..1ac7cab689c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.webproxy.amfilter; import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import com.google.common.net.HostAndPort; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.http.FilterContainer; import org.apache.hadoop.http.FilterInitializer; @@ -38,14 +39,15 @@ public class AmFilterInitializer extends FilterInitializer { private static final String FILTER_NAME = "AM_PROXY_FILTER"; private static final String FILTER_CLASS = AmIpFilter.class.getCanonicalName(); public static final String RM_HA_URLS = "RM_HA_URLS"; - + @Override public void initFilter(FilterContainer container, Configuration conf) { Map params = new HashMap<>(); List proxies = WebAppUtils.getProxyHostsAndPortsForAmFilter(conf); StringBuilder sb = new StringBuilder(); for (String proxy : proxies) { - sb.append(proxy.split(":")[0]).append(AmIpFilter.PROXY_HOSTS_DELIMITER); + sb.append(HostAndPort.fromString(proxy).getHost()) + .append(AmIpFilter.PROXY_HOSTS_DELIMITER); } sb.setLength(sb.length() - 1); params.put(AmIpFilter.PROXY_HOSTS, sb.toString()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java index 6c8993f6e80..65121a62dec 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java @@ -50,6 +50,7 @@ import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import com.google.common.net.HostAndPort; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.http.HttpServer2; @@ -589,8 +590,7 @@ public class TestWebAppProxyServlet { proxyServer.setAttribute(IS_SECURITY_ENABLED_ATTRIBUTE, Boolean.TRUE); String proxy = WebAppUtils.getProxyHostAndPort(conf); - String[] proxyParts = proxy.split(":"); - String proxyHost = proxyParts[0]; + String proxyHost = HostAndPort.fromString(proxy).getHost(); proxyServer.setAttribute(PROXY_HOST_ATTRIBUTE, proxyHost); proxyServer.start();