YARN-4283. Avoid unsafe split and append on fields that might be IPv6 literals. Contributed by Nemanja Matkovic and Hemanth Boyina
This commit is contained in:
parent
3133386ac4
commit
c0c70e0833
|
@ -23,6 +23,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Public;
|
||||||
import org.apache.hadoop.classification.InterfaceStability.Unstable;
|
import org.apache.hadoop.classification.InterfaceStability.Unstable;
|
||||||
import org.apache.hadoop.classification.InterfaceStability.Stable;
|
import org.apache.hadoop.classification.InterfaceStability.Stable;
|
||||||
import org.apache.hadoop.yarn.util.Records;
|
import org.apache.hadoop.yarn.util.Records;
|
||||||
|
import com.google.common.net.HostAndPort;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* <p><code>NodeId</code> is the unique identifier for a node.</p>
|
* <p><code>NodeId</code> is the unique identifier for a node.</p>
|
||||||
|
@ -116,17 +117,18 @@ public abstract class NodeId implements Comparable<NodeId> {
|
||||||
@Public
|
@Public
|
||||||
@Stable
|
@Stable
|
||||||
public static NodeId fromString(String nodeIdStr) {
|
public static NodeId fromString(String nodeIdStr) {
|
||||||
String[] parts = nodeIdStr.split(":");
|
HostAndPort hp = HostAndPort.fromString(nodeIdStr);
|
||||||
if (parts.length != 2) {
|
if (!hp.hasPort()) {
|
||||||
throw new IllegalArgumentException("Invalid NodeId [" + nodeIdStr
|
throw new IllegalArgumentException(
|
||||||
+ "]. Expected host:port");
|
"Invalid NodeId [" + nodeIdStr + "]. Expected host:port");
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
NodeId nodeId =
|
String hostPortStr = hp.toString();
|
||||||
NodeId.newInstance(parts[0].trim(), Integer.parseInt(parts[1]));
|
String host = hostPortStr.substring(0, hostPortStr.lastIndexOf(":"));
|
||||||
|
NodeId nodeId = NodeId.newInstance(host, hp.getPort());
|
||||||
return nodeId;
|
return nodeId;
|
||||||
} catch (NumberFormatException e) {
|
} catch (NumberFormatException e) {
|
||||||
throw new IllegalArgumentException("Invalid port: " + parts[1], e);
|
throw new IllegalArgumentException("Invalid port: " + hp.getPort(), e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -36,7 +36,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
|
||||||
import org.apache.hadoop.yarn.api.records.NodeId;
|
import org.apache.hadoop.yarn.api.records.NodeId;
|
||||||
import org.apache.hadoop.yarn.api.records.URL;
|
import org.apache.hadoop.yarn.api.records.URL;
|
||||||
import org.apache.hadoop.yarn.factories.RecordFactory;
|
import org.apache.hadoop.yarn.factories.RecordFactory;
|
||||||
|
import com.google.common.net.HostAndPort;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* This class contains a set of utilities which help converting data structures
|
* This class contains a set of utilities which help converting data structures
|
||||||
|
@ -114,11 +114,11 @@ public class ConverterUtils {
|
||||||
|
|
||||||
@Private
|
@Private
|
||||||
@InterfaceStability.Unstable
|
@InterfaceStability.Unstable
|
||||||
public static NodeId toNodeIdWithDefaultPort(String nodeIdStr) {
|
public static NodeId toNodeIdWithDefaultPort(
|
||||||
if (nodeIdStr.indexOf(":") < 0) {
|
String nodeIdStr) {
|
||||||
return NodeId.fromString(nodeIdStr + ":0");
|
HostAndPort hp = HostAndPort.fromString(nodeIdStr);
|
||||||
}
|
hp = hp.withDefaultPort(0);
|
||||||
return NodeId.fromString(nodeIdStr);
|
return toNodeId(hp.toString());
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
|
|
@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.webapp.BadRequestException;
|
||||||
import org.apache.hadoop.yarn.webapp.NotFoundException;
|
import org.apache.hadoop.yarn.webapp.NotFoundException;
|
||||||
import org.apache.http.NameValuePair;
|
import org.apache.http.NameValuePair;
|
||||||
import org.apache.http.client.utils.URLEncodedUtils;
|
import org.apache.http.client.utils.URLEncodedUtils;
|
||||||
|
import com.google.common.net.HostAndPort;
|
||||||
|
|
||||||
import javax.servlet.http.HttpServletRequest;
|
import javax.servlet.http.HttpServletRequest;
|
||||||
|
|
||||||
|
@ -64,15 +65,13 @@ public class WebAppUtils {
|
||||||
|
|
||||||
public static void setRMWebAppPort(Configuration conf, int port) {
|
public static void setRMWebAppPort(Configuration conf, int port) {
|
||||||
String hostname = getRMWebAppURLWithoutScheme(conf);
|
String hostname = getRMWebAppURLWithoutScheme(conf);
|
||||||
hostname =
|
HostAndPort hp = HostAndPort.fromString(hostname);
|
||||||
(hostname.contains(":")) ? hostname.substring(0, hostname.indexOf(":"))
|
setRMWebAppHostnameAndPort(conf, hp.getHost(), port);
|
||||||
: hostname;
|
|
||||||
setRMWebAppHostnameAndPort(conf, hostname, port);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public static void setRMWebAppHostnameAndPort(Configuration conf,
|
public static void setRMWebAppHostnameAndPort(Configuration conf,
|
||||||
String hostname, int port) {
|
String hostname, int port) {
|
||||||
String resolvedAddress = hostname + ":" + port;
|
String resolvedAddress = HostAndPort.fromParts(hostname, port).toString();
|
||||||
if (YarnConfiguration.useHttps(conf)) {
|
if (YarnConfiguration.useHttps(conf)) {
|
||||||
conf.set(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS, resolvedAddress);
|
conf.set(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS, resolvedAddress);
|
||||||
} else {
|
} else {
|
||||||
|
@ -82,12 +81,11 @@ public class WebAppUtils {
|
||||||
|
|
||||||
public static void setNMWebAppHostNameAndPort(Configuration conf,
|
public static void setNMWebAppHostNameAndPort(Configuration conf,
|
||||||
String hostName, int port) {
|
String hostName, int port) {
|
||||||
|
String hostPortString = HostAndPort.fromParts(hostName, port).toString();
|
||||||
if (YarnConfiguration.useHttps(conf)) {
|
if (YarnConfiguration.useHttps(conf)) {
|
||||||
conf.set(YarnConfiguration.NM_WEBAPP_HTTPS_ADDRESS,
|
conf.set(YarnConfiguration.NM_WEBAPP_HTTPS_ADDRESS, hostPortString);
|
||||||
hostName + ":" + port);
|
|
||||||
} else {
|
} else {
|
||||||
conf.set(YarnConfiguration.NM_WEBAPP_ADDRESS,
|
conf.set(YarnConfiguration.NM_WEBAPP_ADDRESS, hostPortString);
|
||||||
hostName + ":" + port);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -325,7 +323,8 @@ public class WebAppUtils {
|
||||||
String host = conf.getTrimmed(hostProperty);
|
String host = conf.getTrimmed(hostProperty);
|
||||||
if (host != null && !host.isEmpty()) {
|
if (host != null && !host.isEmpty()) {
|
||||||
if (webAppURLWithoutScheme.contains(":")) {
|
if (webAppURLWithoutScheme.contains(":")) {
|
||||||
webAppURLWithoutScheme = host + ":" + webAppURLWithoutScheme.split(":")[1];
|
String[] splits = webAppURLWithoutScheme.split(":");
|
||||||
|
webAppURLWithoutScheme = host + ":" + splits[splits.length - 1];
|
||||||
}
|
}
|
||||||
else {
|
else {
|
||||||
throw new YarnRuntimeException("webAppURLWithoutScheme must include port specification but doesn't: " +
|
throw new YarnRuntimeException("webAppURLWithoutScheme must include port specification but doesn't: " +
|
||||||
|
|
|
@ -18,6 +18,8 @@
|
||||||
|
|
||||||
package org.apache.hadoop.yarn.conf;
|
package org.apache.hadoop.yarn.conf;
|
||||||
|
|
||||||
|
import com.google.common.net.HostAndPort;
|
||||||
|
import org.apache.hadoop.net.NetUtils;
|
||||||
import org.junit.Assert;
|
import org.junit.Assert;
|
||||||
|
|
||||||
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
|
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
|
||||||
|
@ -74,7 +76,7 @@ public class TestYarnConfiguration {
|
||||||
conf.set(YarnConfiguration.RM_ADDRESS, "rmtesting:9999");
|
conf.set(YarnConfiguration.RM_ADDRESS, "rmtesting:9999");
|
||||||
String rmWebUrl = WebAppUtils.getRMWebAppURLWithScheme(conf);
|
String rmWebUrl = WebAppUtils.getRMWebAppURLWithScheme(conf);
|
||||||
String[] parts = rmWebUrl.split(":");
|
String[] parts = rmWebUrl.split(":");
|
||||||
Assert.assertEquals("RM Web URL Port is incrrect", 24543,
|
Assert.assertEquals("RM Web URL Port is incorrect", 24543,
|
||||||
Integer.parseInt(parts[parts.length - 1]));
|
Integer.parseInt(parts[parts.length - 1]));
|
||||||
Assert.assertNotSame(
|
Assert.assertNotSame(
|
||||||
"RM Web Url not resolved correctly. Should not be rmtesting",
|
"RM Web Url not resolved correctly. Should not be rmtesting",
|
||||||
|
@ -112,10 +114,9 @@ public class TestYarnConfiguration {
|
||||||
YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
|
YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
|
||||||
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
|
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
|
||||||
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
|
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
|
||||||
assertEquals(
|
assertEquals(new InetSocketAddress(
|
||||||
new InetSocketAddress(
|
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS.split(":")[0],
|
||||||
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS.split(":")[0],
|
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT),
|
||||||
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT),
|
|
||||||
resourceTrackerAddress);
|
resourceTrackerAddress);
|
||||||
|
|
||||||
//with address
|
//with address
|
||||||
|
@ -125,10 +126,8 @@ public class TestYarnConfiguration {
|
||||||
YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
|
YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
|
||||||
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
|
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
|
||||||
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
|
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
|
||||||
assertEquals(
|
assertEquals(new InetSocketAddress("10.0.0.1",
|
||||||
new InetSocketAddress(
|
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT),
|
||||||
"10.0.0.1",
|
|
||||||
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT),
|
|
||||||
resourceTrackerAddress);
|
resourceTrackerAddress);
|
||||||
|
|
||||||
//address and socket
|
//address and socket
|
||||||
|
@ -139,9 +138,23 @@ public class TestYarnConfiguration {
|
||||||
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
|
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
|
||||||
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
|
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
|
||||||
assertEquals(
|
assertEquals(
|
||||||
new InetSocketAddress(
|
new InetSocketAddress(
|
||||||
"10.0.0.2",
|
"10.0.0.2",
|
||||||
5001),
|
5001),
|
||||||
|
resourceTrackerAddress);
|
||||||
|
|
||||||
|
// IPv6 address and socket
|
||||||
|
conf.set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
|
||||||
|
"[2401:db00:20:a01e:face:0:5:0]:5001");
|
||||||
|
resourceTrackerAddress = conf.getSocketAddr(
|
||||||
|
YarnConfiguration.RM_BIND_HOST,
|
||||||
|
YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
|
||||||
|
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
|
||||||
|
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
|
||||||
|
assertEquals(
|
||||||
|
new InetSocketAddress(
|
||||||
|
"2401:db00:20:a01e:face:0:5:0",
|
||||||
|
5001),
|
||||||
resourceTrackerAddress);
|
resourceTrackerAddress);
|
||||||
|
|
||||||
//bind host only
|
//bind host only
|
||||||
|
@ -152,10 +165,8 @@ public class TestYarnConfiguration {
|
||||||
YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
|
YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
|
||||||
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
|
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
|
||||||
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
|
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
|
||||||
assertEquals(
|
assertEquals(new InetSocketAddress("10.0.0.3",
|
||||||
new InetSocketAddress(
|
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT),
|
||||||
"10.0.0.3",
|
|
||||||
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT),
|
|
||||||
resourceTrackerAddress);
|
resourceTrackerAddress);
|
||||||
|
|
||||||
//bind host and address no port
|
//bind host and address no port
|
||||||
|
@ -166,10 +177,8 @@ public class TestYarnConfiguration {
|
||||||
YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
|
YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
|
||||||
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
|
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
|
||||||
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
|
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
|
||||||
assertEquals(
|
assertEquals(new InetSocketAddress("0.0.0.0",
|
||||||
new InetSocketAddress(
|
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT),
|
||||||
"0.0.0.0",
|
|
||||||
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT),
|
|
||||||
resourceTrackerAddress);
|
resourceTrackerAddress);
|
||||||
|
|
||||||
//bind host and address with port
|
//bind host and address with port
|
||||||
|
@ -180,10 +189,7 @@ public class TestYarnConfiguration {
|
||||||
YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
|
YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
|
||||||
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
|
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
|
||||||
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
|
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
|
||||||
assertEquals(
|
assertEquals(new InetSocketAddress("0.0.0.0", 5003),
|
||||||
new InetSocketAddress(
|
|
||||||
"0.0.0.0",
|
|
||||||
5003),
|
|
||||||
resourceTrackerAddress);
|
resourceTrackerAddress);
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -197,9 +203,8 @@ public class TestYarnConfiguration {
|
||||||
//no override, old behavior. Won't work on a host named "yo.yo.yo"
|
//no override, old behavior. Won't work on a host named "yo.yo.yo"
|
||||||
conf = new YarnConfiguration();
|
conf = new YarnConfiguration();
|
||||||
conf.set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, "yo.yo.yo");
|
conf.set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, "yo.yo.yo");
|
||||||
serverAddress = new InetSocketAddress(
|
serverAddress = newInetSocketAddressFromHostPort(
|
||||||
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS.split(":")[0],
|
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS);
|
||||||
Integer.parseInt(YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS.split(":")[1]));
|
|
||||||
|
|
||||||
resourceTrackerConnectAddress = conf.updateConnectAddr(
|
resourceTrackerConnectAddress = conf.updateConnectAddr(
|
||||||
YarnConfiguration.RM_BIND_HOST,
|
YarnConfiguration.RM_BIND_HOST,
|
||||||
|
@ -207,15 +212,15 @@ public class TestYarnConfiguration {
|
||||||
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
|
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
|
||||||
serverAddress);
|
serverAddress);
|
||||||
|
|
||||||
assertFalse(resourceTrackerConnectAddress.toString().startsWith("yo.yo.yo"));
|
assertFalse(NetUtils.getSocketAddressString(resourceTrackerConnectAddress)
|
||||||
|
.startsWith("yo.yo.yo"));
|
||||||
|
|
||||||
//cause override with address
|
//cause override with address
|
||||||
conf = new YarnConfiguration();
|
conf = new YarnConfiguration();
|
||||||
conf.set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, "yo.yo.yo");
|
conf.set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, "yo.yo.yo");
|
||||||
conf.set(YarnConfiguration.RM_BIND_HOST, "0.0.0.0");
|
conf.set(YarnConfiguration.RM_BIND_HOST, "0.0.0.0");
|
||||||
serverAddress = new InetSocketAddress(
|
serverAddress = newInetSocketAddressFromHostPort(
|
||||||
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS.split(":")[0],
|
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS);
|
||||||
Integer.parseInt(YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS.split(":")[1]));
|
|
||||||
|
|
||||||
resourceTrackerConnectAddress = conf.updateConnectAddr(
|
resourceTrackerConnectAddress = conf.updateConnectAddr(
|
||||||
YarnConfiguration.RM_BIND_HOST,
|
YarnConfiguration.RM_BIND_HOST,
|
||||||
|
@ -223,7 +228,8 @@ public class TestYarnConfiguration {
|
||||||
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
|
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
|
||||||
serverAddress);
|
serverAddress);
|
||||||
|
|
||||||
assertTrue(resourceTrackerConnectAddress.toString().startsWith("yo.yo.yo"));
|
assertTrue(NetUtils.getSocketAddressString(resourceTrackerConnectAddress)
|
||||||
|
.startsWith("yo.yo.yo"));
|
||||||
|
|
||||||
//tests updateConnectAddr won't add suffix to NM service address configurations
|
//tests updateConnectAddr won't add suffix to NM service address configurations
|
||||||
conf = new YarnConfiguration();
|
conf = new YarnConfiguration();
|
||||||
|
@ -232,9 +238,8 @@ public class TestYarnConfiguration {
|
||||||
conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
|
conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
|
||||||
conf.set(YarnConfiguration.RM_HA_ID, "rm1");
|
conf.set(YarnConfiguration.RM_HA_ID, "rm1");
|
||||||
|
|
||||||
serverAddress = new InetSocketAddress(
|
serverAddress = newInetSocketAddressFromHostPort(
|
||||||
YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS.split(":")[0],
|
YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS);
|
||||||
Integer.parseInt(YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS.split(":")[1]));
|
|
||||||
|
|
||||||
InetSocketAddress localizerAddress = conf.updateConnectAddr(
|
InetSocketAddress localizerAddress = conf.updateConnectAddr(
|
||||||
YarnConfiguration.NM_BIND_HOST,
|
YarnConfiguration.NM_BIND_HOST,
|
||||||
|
@ -242,8 +247,15 @@ public class TestYarnConfiguration {
|
||||||
YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS,
|
YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS,
|
||||||
serverAddress);
|
serverAddress);
|
||||||
|
|
||||||
assertTrue(localizerAddress.toString().startsWith("yo.yo.yo"));
|
assertTrue(NetUtils.getSocketAddressString(localizerAddress)
|
||||||
|
.startsWith("yo.yo.yo"));
|
||||||
assertNull(conf.get(
|
assertNull(conf.get(
|
||||||
HAUtil.addSuffix(YarnConfiguration.NM_LOCALIZER_ADDRESS, "rm1")));
|
HAUtil.addSuffix(YarnConfiguration.NM_LOCALIZER_ADDRESS, "rm1")));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private InetSocketAddress newInetSocketAddressFromHostPort(
|
||||||
|
String hostPort) {
|
||||||
|
HostAndPort hp = HostAndPort.fromString(hostPort);
|
||||||
|
return new InetSocketAddress(hp.getHost(), hp.getPort());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -100,6 +100,16 @@ public class TestConverterUtils {
|
||||||
nid = ConverterUtils.toNodeIdWithDefaultPort("node");
|
nid = ConverterUtils.toNodeIdWithDefaultPort("node");
|
||||||
assertThat(nid.getPort()).isEqualTo(0);
|
assertThat(nid.getPort()).isEqualTo(0);
|
||||||
assertThat(nid.getHost()).isEqualTo("node");
|
assertThat(nid.getHost()).isEqualTo("node");
|
||||||
|
|
||||||
|
nid = ConverterUtils
|
||||||
|
.toNodeIdWithDefaultPort("[2401:db00:20:a01e:face:0:5:0]:10");
|
||||||
|
assertEquals(nid.getPort(), 10);
|
||||||
|
assertEquals(nid.getHost(), "[2401:db00:20:a01e:face:0:5:0]");
|
||||||
|
|
||||||
|
nid = ConverterUtils
|
||||||
|
.toNodeIdWithDefaultPort("[2401:db00:20:a01e:face:0:5:0]");
|
||||||
|
assertEquals(nid.getPort(), 0);
|
||||||
|
assertEquals(nid.getHost(), "[2401:db00:20:a01e:face:0:5:0]");
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test(expected = IllegalArgumentException.class)
|
@Test(expected = IllegalArgumentException.class)
|
||||||
|
|
|
@ -27,6 +27,7 @@ import java.net.InetSocketAddress;
|
||||||
import java.net.Socket;
|
import java.net.Socket;
|
||||||
|
|
||||||
import org.apache.hadoop.net.ServerSocketUtil;
|
import org.apache.hadoop.net.ServerSocketUtil;
|
||||||
|
import com.google.common.net.HostAndPort;
|
||||||
import org.junit.Assert;
|
import org.junit.Assert;
|
||||||
|
|
||||||
import org.apache.hadoop.yarn.lib.ZKClient;
|
import org.apache.hadoop.yarn.lib.ZKClient;
|
||||||
|
@ -86,8 +87,9 @@ public class TestZKClient {
|
||||||
long start = System.currentTimeMillis();
|
long start = System.currentTimeMillis();
|
||||||
while (true) {
|
while (true) {
|
||||||
try {
|
try {
|
||||||
String host = hp.split(":")[0];
|
HostAndPort hap = HostAndPort.fromString(hp);
|
||||||
int port = Integer.parseInt(hp.split(":")[1]);
|
String host = hap.getHost();
|
||||||
|
int port = hap.getPort();
|
||||||
send4LetterWord(host, port, "stat");
|
send4LetterWord(host, port, "stat");
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
return true;
|
return true;
|
||||||
|
@ -110,8 +112,9 @@ public class TestZKClient {
|
||||||
long start = System.currentTimeMillis();
|
long start = System.currentTimeMillis();
|
||||||
while (true) {
|
while (true) {
|
||||||
try {
|
try {
|
||||||
String host = hp.split(":")[0];
|
HostAndPort hap = HostAndPort.fromString(hp);
|
||||||
int port = Integer.parseInt(hp.split(":")[1]);
|
String host = hap.getHost();
|
||||||
|
int port = hap.getPort();
|
||||||
// if there are multiple hostports, just take the first one
|
// if there are multiple hostports, just take the first one
|
||||||
String result = send4LetterWord(host, port, "stat");
|
String result = send4LetterWord(host, port, "stat");
|
||||||
if (result.startsWith("Zookeeper version:")) {
|
if (result.startsWith("Zookeeper version:")) {
|
||||||
|
@ -151,14 +154,15 @@ public class TestZKClient {
|
||||||
}
|
}
|
||||||
File dataDir = createTmpDir(BASETEST);
|
File dataDir = createTmpDir(BASETEST);
|
||||||
zks = new ZooKeeperServer(dataDir, dataDir, 3000);
|
zks = new ZooKeeperServer(dataDir, dataDir, 3000);
|
||||||
final int PORT = Integer.parseInt(hostPort.split(":")[1]);
|
HostAndPort hp = HostAndPort.fromString(hostPort);
|
||||||
|
final int port = hp.getPort();
|
||||||
if (factory == null) {
|
if (factory == null) {
|
||||||
factory = new NIOServerCnxnFactory();
|
factory = new NIOServerCnxnFactory();
|
||||||
factory.configure(new InetSocketAddress(PORT), maxCnxns);
|
factory.configure(new InetSocketAddress(port), maxCnxns);
|
||||||
}
|
}
|
||||||
factory.startup(zks);
|
factory.startup(zks);
|
||||||
Assert.assertTrue("waiting for server up",
|
Assert.assertTrue("waiting for server up",
|
||||||
waitForServerUp("127.0.0.1:" + PORT,
|
waitForServerUp("127.0.0.1:" + port,
|
||||||
CONNECTION_TIMEOUT));
|
CONNECTION_TIMEOUT));
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -172,10 +176,11 @@ public class TestZKClient {
|
||||||
zkDb.close();
|
zkDb.close();
|
||||||
} catch (IOException ie) {
|
} catch (IOException ie) {
|
||||||
}
|
}
|
||||||
final int PORT = Integer.parseInt(hostPort.split(":")[1]);
|
HostAndPort hp = HostAndPort.fromString(hostPort);
|
||||||
|
final int port = hp.getPort();
|
||||||
|
|
||||||
Assert.assertTrue("waiting for server down",
|
Assert.assertTrue("waiting for server down",
|
||||||
waitForServerDown("127.0.0.1:" + PORT,
|
waitForServerDown("127.0.0.1:" + port,
|
||||||
CONNECTION_TIMEOUT));
|
CONNECTION_TIMEOUT));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -185,6 +185,7 @@ import java.util.Set;
|
||||||
import java.util.concurrent.locks.ReentrantReadWriteLock;
|
import java.util.concurrent.locks.ReentrantReadWriteLock;
|
||||||
import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
|
import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
|
||||||
import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
|
import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
|
||||||
|
import com.google.common.net.HostAndPort;
|
||||||
|
|
||||||
import static org.apache.hadoop.service.Service.STATE.STARTED;
|
import static org.apache.hadoop.service.Service.STATE.STARTED;
|
||||||
|
|
||||||
|
@ -645,7 +646,7 @@ public class ContainerManagerImpl extends CompositeService implements
|
||||||
//hostname found when querying for our hostname with the specified
|
//hostname found when querying for our hostname with the specified
|
||||||
//address, combine the specified address with the actual port listened
|
//address, combine the specified address with the actual port listened
|
||||||
//on by the server
|
//on by the server
|
||||||
hostOverride = nmAddress.split(":")[0];
|
hostOverride = HostAndPort.fromString(nmAddress).getHost();
|
||||||
}
|
}
|
||||||
|
|
||||||
// setup node ID
|
// setup node ID
|
||||||
|
|
|
@ -137,6 +137,7 @@ import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
|
||||||
import org.apache.zookeeper.server.auth.DigestAuthenticationProvider;
|
import org.apache.zookeeper.server.auth.DigestAuthenticationProvider;
|
||||||
import org.eclipse.jetty.webapp.WebAppContext;
|
import org.eclipse.jetty.webapp.WebAppContext;
|
||||||
|
|
||||||
|
import com.google.common.net.HostAndPort;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.io.InputStream;
|
import java.io.InputStream;
|
||||||
import java.io.PrintStream;
|
import java.io.PrintStream;
|
||||||
|
@ -1399,8 +1400,8 @@ public class ResourceManager extends CompositeService
|
||||||
builder.withAttribute(WebAppProxy.PROXY_CA,
|
builder.withAttribute(WebAppProxy.PROXY_CA,
|
||||||
rmContext.getProxyCAManager().getProxyCA());
|
rmContext.getProxyCAManager().getProxyCA());
|
||||||
builder.withAttribute(WebAppProxy.FETCHER_ATTRIBUTE, fetcher);
|
builder.withAttribute(WebAppProxy.FETCHER_ATTRIBUTE, fetcher);
|
||||||
String[] proxyParts = proxyHostAndPort.split(":");
|
builder.withAttribute(WebAppProxy.PROXY_HOST_ATTRIBUTE,
|
||||||
builder.withAttribute(WebAppProxy.PROXY_HOST_ATTRIBUTE, proxyParts[0]);
|
HostAndPort.fromString(proxyHostAndPort).getHost());
|
||||||
}
|
}
|
||||||
|
|
||||||
WebAppContext uiWebAppContext = null;
|
WebAppContext uiWebAppContext = null;
|
||||||
|
|
|
@ -30,6 +30,7 @@ import java.util.Map;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
import java.util.concurrent.ConcurrentHashMap;
|
import java.util.concurrent.ConcurrentHashMap;
|
||||||
|
|
||||||
|
import com.google.common.net.HostAndPort;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
|
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||||
import org.apache.hadoop.yarn.api.records.Container;
|
import org.apache.hadoop.yarn.api.records.Container;
|
||||||
|
@ -105,8 +106,10 @@ public class MockNM {
|
||||||
this.capability = capability;
|
this.capability = capability;
|
||||||
this.resourceTracker = resourceTracker;
|
this.resourceTracker = resourceTracker;
|
||||||
this.version = version;
|
this.version = version;
|
||||||
String[] splits = nodeIdStr.split(":");
|
HostAndPort hostAndPort = HostAndPort.fromString(nodeIdStr);
|
||||||
nodeId = BuilderUtils.newNodeId(splits[0], Integer.parseInt(splits[1]));
|
String hostPortStr = hostAndPort.toString();
|
||||||
|
String host = hostPortStr.substring(0, hostPortStr.lastIndexOf(":"));
|
||||||
|
nodeId = BuilderUtils.newNodeId(host, hostAndPort.getPort());
|
||||||
}
|
}
|
||||||
|
|
||||||
public MockNM(String nodeIdStr, Resource capability,
|
public MockNM(String nodeIdStr, Resource capability,
|
||||||
|
|
|
@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.webproxy;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.net.URI;
|
import java.net.URI;
|
||||||
|
|
||||||
|
import com.google.common.net.HostAndPort;
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.http.HttpServer2;
|
import org.apache.hadoop.http.HttpServer2;
|
||||||
import org.apache.hadoop.security.authorize.AccessControlList;
|
import org.apache.hadoop.security.authorize.AccessControlList;
|
||||||
|
@ -57,7 +58,8 @@ public class WebAppProxy extends AbstractService {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected void serviceInit(Configuration conf) throws Exception {
|
protected void serviceInit(Configuration conf) throws Exception {
|
||||||
String auth = conf.get(CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION);
|
String auth = conf.get(
|
||||||
|
CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION);
|
||||||
if (auth == null || "simple".equals(auth)) {
|
if (auth == null || "simple".equals(auth)) {
|
||||||
isSecurityEnabled = false;
|
isSecurityEnabled = false;
|
||||||
} else if ("kerberos".equals(auth)) {
|
} else if ("kerberos".equals(auth)) {
|
||||||
|
@ -68,8 +70,7 @@ public class WebAppProxy extends AbstractService {
|
||||||
" of " + auth);
|
" of " + auth);
|
||||||
}
|
}
|
||||||
String proxy = WebAppUtils.getProxyHostAndPort(conf);
|
String proxy = WebAppUtils.getProxyHostAndPort(conf);
|
||||||
String[] proxyParts = proxy.split(":");
|
proxyHost = HostAndPort.fromString(proxy).getHost();
|
||||||
proxyHost = proxyParts[0];
|
|
||||||
|
|
||||||
fetcher = new AppReportFetcher(conf);
|
fetcher = new AppReportFetcher(conf);
|
||||||
bindAddress = conf.get(YarnConfiguration.PROXY_ADDRESS);
|
bindAddress = conf.get(YarnConfiguration.PROXY_ADDRESS);
|
||||||
|
|
|
@ -19,6 +19,7 @@
|
||||||
package org.apache.hadoop.yarn.server.webproxy.amfilter;
|
package org.apache.hadoop.yarn.server.webproxy.amfilter;
|
||||||
|
|
||||||
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
|
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
|
||||||
|
import com.google.common.net.HostAndPort;
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.http.FilterContainer;
|
import org.apache.hadoop.http.FilterContainer;
|
||||||
import org.apache.hadoop.http.FilterInitializer;
|
import org.apache.hadoop.http.FilterInitializer;
|
||||||
|
@ -45,7 +46,8 @@ public class AmFilterInitializer extends FilterInitializer {
|
||||||
List<String> proxies = WebAppUtils.getProxyHostsAndPortsForAmFilter(conf);
|
List<String> proxies = WebAppUtils.getProxyHostsAndPortsForAmFilter(conf);
|
||||||
StringBuilder sb = new StringBuilder();
|
StringBuilder sb = new StringBuilder();
|
||||||
for (String proxy : proxies) {
|
for (String proxy : proxies) {
|
||||||
sb.append(proxy.split(":")[0]).append(AmIpFilter.PROXY_HOSTS_DELIMITER);
|
sb.append(HostAndPort.fromString(proxy).getHost())
|
||||||
|
.append(AmIpFilter.PROXY_HOSTS_DELIMITER);
|
||||||
}
|
}
|
||||||
sb.setLength(sb.length() - 1);
|
sb.setLength(sb.length() - 1);
|
||||||
params.put(AmIpFilter.PROXY_HOSTS, sb.toString());
|
params.put(AmIpFilter.PROXY_HOSTS, sb.toString());
|
||||||
|
|
|
@ -50,6 +50,7 @@ import javax.servlet.http.HttpServlet;
|
||||||
import javax.servlet.http.HttpServletRequest;
|
import javax.servlet.http.HttpServletRequest;
|
||||||
import javax.servlet.http.HttpServletResponse;
|
import javax.servlet.http.HttpServletResponse;
|
||||||
|
|
||||||
|
import com.google.common.net.HostAndPort;
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.fs.CommonConfigurationKeys;
|
import org.apache.hadoop.fs.CommonConfigurationKeys;
|
||||||
import org.apache.hadoop.http.HttpServer2;
|
import org.apache.hadoop.http.HttpServer2;
|
||||||
|
@ -589,8 +590,7 @@ public class TestWebAppProxyServlet {
|
||||||
proxyServer.setAttribute(IS_SECURITY_ENABLED_ATTRIBUTE, Boolean.TRUE);
|
proxyServer.setAttribute(IS_SECURITY_ENABLED_ATTRIBUTE, Boolean.TRUE);
|
||||||
|
|
||||||
String proxy = WebAppUtils.getProxyHostAndPort(conf);
|
String proxy = WebAppUtils.getProxyHostAndPort(conf);
|
||||||
String[] proxyParts = proxy.split(":");
|
String proxyHost = HostAndPort.fromString(proxy).getHost();
|
||||||
String proxyHost = proxyParts[0];
|
|
||||||
|
|
||||||
proxyServer.setAttribute(PROXY_HOST_ATTRIBUTE, proxyHost);
|
proxyServer.setAttribute(PROXY_HOST_ATTRIBUTE, proxyHost);
|
||||||
proxyServer.start();
|
proxyServer.start();
|
||||||
|
|
Loading…
Reference in New Issue