Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-4283. Avoid unsafe split and append on fields that might be IPv6 literals. Contributed by Nemanja Matkovic and Hemanth Boyina

This commit is contained in:
Brahma Reddy Battula 2021-08-04 17:49:52 +05:30
parent 3133386ac4
commit c0c70e0833
12 changed files with 119 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeId.java
View File

@ -23,6 +23,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.classification.InterfaceStability.Stable;
import org.apache.hadoop.yarn.util.Records;
import com.google.common.net.HostAndPort;
/**
* <p><code>NodeId</code> is the unique identifier for a node.</p>
@ -116,17 +117,18 @@ public abstract class NodeId implements Comparable<NodeId> {
@Public
@Stable
public static NodeId fromString(String nodeIdStr) {
String[] parts = nodeIdStr.split(":");
if (parts.length != 2) {
throw new IllegalArgumentException("Invalid NodeId [" + nodeIdStr
+ "]. Expected host:port");
HostAndPort hp = HostAndPort.fromString(nodeIdStr);
if (!hp.hasPort()) {
throw new IllegalArgumentException(
"Invalid NodeId [" + nodeIdStr + "]. Expected host:port");
}
try {
NodeId nodeId =
NodeId.newInstance(parts[0].trim(), Integer.parseInt(parts[1]));
String hostPortStr = hp.toString();
String host = hostPortStr.substring(0, hostPortStr.lastIndexOf(":"));
NodeId nodeId = NodeId.newInstance(host, hp.getPort());
return nodeId;
} catch (NumberFormatException e) {
throw new IllegalArgumentException("Invalid port: " + parts[1], e);
throw new IllegalArgumentException("Invalid port: " + hp.getPort(), e);
}
}

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java
View File

@ -36,7 +36,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.factories.RecordFactory;
import com.google.common.net.HostAndPort;
/**
* This class contains a set of utilities which help converting data structures
@ -114,11 +114,11 @@ public class ConverterUtils {
@Private
@InterfaceStability.Unstable
public static NodeId toNodeIdWithDefaultPort(String nodeIdStr) {
if (nodeIdStr.indexOf(":") < 0) {
return NodeId.fromString(nodeIdStr + ":0");
}
return NodeId.fromString(nodeIdStr);
public static NodeId toNodeIdWithDefaultPort(
String nodeIdStr) {
HostAndPort hp = HostAndPort.fromString(nodeIdStr);
hp = hp.withDefaultPort(0);
return toNodeId(hp.toString());
}
/*

19
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
View File

@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.webapp.BadRequestException;
import org.apache.hadoop.yarn.webapp.NotFoundException;
import org.apache.http.NameValuePair;
import org.apache.http.client.utils.URLEncodedUtils;
import com.google.common.net.HostAndPort;
import javax.servlet.http.HttpServletRequest;
@ -64,15 +65,13 @@ public class WebAppUtils {
public static void setRMWebAppPort(Configuration conf, int port) {
String hostname = getRMWebAppURLWithoutScheme(conf);
hostname =
(hostname.contains(":")) ? hostname.substring(0, hostname.indexOf(":"))
: hostname;
setRMWebAppHostnameAndPort(conf, hostname, port);
HostAndPort hp = HostAndPort.fromString(hostname);
setRMWebAppHostnameAndPort(conf, hp.getHost(), port);
}
public static void setRMWebAppHostnameAndPort(Configuration conf,
String hostname, int port) {
String resolvedAddress = hostname + ":" + port;
String resolvedAddress = HostAndPort.fromParts(hostname, port).toString();
if (YarnConfiguration.useHttps(conf)) {
conf.set(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS, resolvedAddress);
} else {
@ -82,12 +81,11 @@ public class WebAppUtils {
public static void setNMWebAppHostNameAndPort(Configuration conf,
String hostName, int port) {
String hostPortString = HostAndPort.fromParts(hostName, port).toString();
if (YarnConfiguration.useHttps(conf)) {
conf.set(YarnConfiguration.NM_WEBAPP_HTTPS_ADDRESS,
hostName + ":" + port);
conf.set(YarnConfiguration.NM_WEBAPP_HTTPS_ADDRESS, hostPortString);
} else {
conf.set(YarnConfiguration.NM_WEBAPP_ADDRESS,
hostName + ":" + port);
conf.set(YarnConfiguration.NM_WEBAPP_ADDRESS, hostPortString);
}
}
@ -325,7 +323,8 @@ public class WebAppUtils {
String host = conf.getTrimmed(hostProperty);
if (host != null && !host.isEmpty()) {
if (webAppURLWithoutScheme.contains(":")) {
webAppURLWithoutScheme = host + ":" + webAppURLWithoutScheme.split(":")[1];
String[] splits = webAppURLWithoutScheme.split(":");
webAppURLWithoutScheme = host + ":" + splits[splits.length - 1];
}
else {
throw new YarnRuntimeException("webAppURLWithoutScheme must include port specification but doesn't: " +

84
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java
View File

@ -18,6 +18,8 @@
package org.apache.hadoop.yarn.conf;
import com.google.common.net.HostAndPort;
import org.apache.hadoop.net.NetUtils;
import org.junit.Assert;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
@ -74,7 +76,7 @@ public class TestYarnConfiguration {
conf.set(YarnConfiguration.RM_ADDRESS, "rmtesting:9999");
String rmWebUrl = WebAppUtils.getRMWebAppURLWithScheme(conf);
String[] parts = rmWebUrl.split(":");
Assert.assertEquals("RM Web URL Port is incrrect", 24543,
Assert.assertEquals("RM Web URL Port is incorrect", 24543,
Integer.parseInt(parts[parts.length - 1]));
Assert.assertNotSame(
"RM Web Url not resolved correctly. Should not be rmtesting",
@ -112,10 +114,9 @@ public class TestYarnConfiguration {
YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
assertEquals(
new InetSocketAddress(
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS.split(":")[0],
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT),
assertEquals(new InetSocketAddress(
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS.split(":")[0],
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT),
resourceTrackerAddress);
//with address
@ -125,10 +126,8 @@ public class TestYarnConfiguration {
YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
assertEquals(
new InetSocketAddress(
"10.0.0.1",
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT),
assertEquals(new InetSocketAddress("10.0.0.1",
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT),
resourceTrackerAddress);
//address and socket
@ -139,9 +138,23 @@ public class TestYarnConfiguration {
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
assertEquals(
new InetSocketAddress(
"10.0.0.2",
5001),
new InetSocketAddress(
"10.0.0.2",
5001),
resourceTrackerAddress);
// IPv6 address and socket
conf.set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
"[2401:db00:20:a01e:face:0:5:0]:5001");
resourceTrackerAddress = conf.getSocketAddr(
YarnConfiguration.RM_BIND_HOST,
YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
assertEquals(
new InetSocketAddress(
"2401:db00:20:a01e:face:0:5:0",
5001),
resourceTrackerAddress);
//bind host only
@ -152,10 +165,8 @@ public class TestYarnConfiguration {
YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
assertEquals(
new InetSocketAddress(
"10.0.0.3",
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT),
assertEquals(new InetSocketAddress("10.0.0.3",
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT),
resourceTrackerAddress);
//bind host and address no port
@ -166,10 +177,8 @@ public class TestYarnConfiguration {
YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
assertEquals(
new InetSocketAddress(
"0.0.0.0",
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT),
assertEquals(new InetSocketAddress("0.0.0.0",
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT),
resourceTrackerAddress);
//bind host and address with port
@ -180,10 +189,7 @@ public class TestYarnConfiguration {
YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
assertEquals(
new InetSocketAddress(
"0.0.0.0",
5003),
assertEquals(new InetSocketAddress("0.0.0.0", 5003),
resourceTrackerAddress);
}
@ -197,9 +203,8 @@ public class TestYarnConfiguration {
//no override, old behavior. Won't work on a host named "yo.yo.yo"
conf = new YarnConfiguration();
conf.set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, "yo.yo.yo");
serverAddress = new InetSocketAddress(
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS.split(":")[0],
Integer.parseInt(YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS.split(":")[1]));
serverAddress = newInetSocketAddressFromHostPort(
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS);
resourceTrackerConnectAddress = conf.updateConnectAddr(
YarnConfiguration.RM_BIND_HOST,
@ -207,15 +212,15 @@ public class TestYarnConfiguration {
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
serverAddress);
assertFalse(resourceTrackerConnectAddress.toString().startsWith("yo.yo.yo"));
assertFalse(NetUtils.getSocketAddressString(resourceTrackerConnectAddress)
.startsWith("yo.yo.yo"));
//cause override with address
conf = new YarnConfiguration();
conf.set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, "yo.yo.yo");
conf.set(YarnConfiguration.RM_BIND_HOST, "0.0.0.0");
serverAddress = new InetSocketAddress(
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS.split(":")[0],
Integer.parseInt(YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS.split(":")[1]));
serverAddress = newInetSocketAddressFromHostPort(
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS);
resourceTrackerConnectAddress = conf.updateConnectAddr(
YarnConfiguration.RM_BIND_HOST,
@ -223,7 +228,8 @@ public class TestYarnConfiguration {
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
serverAddress);
assertTrue(resourceTrackerConnectAddress.toString().startsWith("yo.yo.yo"));
assertTrue(NetUtils.getSocketAddressString(resourceTrackerConnectAddress)
.startsWith("yo.yo.yo"));
//tests updateConnectAddr won't add suffix to NM service address configurations
conf = new YarnConfiguration();
@ -232,9 +238,8 @@ public class TestYarnConfiguration {
conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
conf.set(YarnConfiguration.RM_HA_ID, "rm1");
serverAddress = new InetSocketAddress(
YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS.split(":")[0],
Integer.parseInt(YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS.split(":")[1]));
serverAddress = newInetSocketAddressFromHostPort(
YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS);
InetSocketAddress localizerAddress = conf.updateConnectAddr(
YarnConfiguration.NM_BIND_HOST,
@ -242,8 +247,15 @@ public class TestYarnConfiguration {
YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS,
serverAddress);
assertTrue(localizerAddress.toString().startsWith("yo.yo.yo"));
assertTrue(NetUtils.getSocketAddressString(localizerAddress)
.startsWith("yo.yo.yo"));
assertNull(conf.get(
HAUtil.addSuffix(YarnConfiguration.NM_LOCALIZER_ADDRESS, "rm1")));
}
private InetSocketAddress newInetSocketAddressFromHostPort(
String hostPort) {
HostAndPort hp = HostAndPort.fromString(hostPort);
return new InetSocketAddress(hp.getHost(), hp.getPort());
}
}

16
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestConverterUtils.java
View File

@ -31,7 +31,7 @@ import org.apache.hadoop.yarn.api.records.NodeId;
import org.junit.Test;
public class TestConverterUtils {
@Test
public void testConvertUrlWithNoPort() throws URISyntaxException {
Path expectedPath = new Path("hdfs://foo.com");
@ -92,14 +92,24 @@ public class TestConverterUtils {
@Test
public void testNodeIdWithDefaultPort() throws URISyntaxException {
NodeId nid;
nid = ConverterUtils.toNodeIdWithDefaultPort("node:10");
assertThat(nid.getPort()).isEqualTo(10);
assertThat(nid.getHost()).isEqualTo("node");
nid = ConverterUtils.toNodeIdWithDefaultPort("node");
assertThat(nid.getPort()).isEqualTo(0);
assertThat(nid.getHost()).isEqualTo("node");
nid = ConverterUtils
.toNodeIdWithDefaultPort("[2401:db00:20:a01e:face:0:5:0]:10");
assertEquals(nid.getPort(), 10);
assertEquals(nid.getHost(), "[2401:db00:20:a01e:face:0:5:0]");
nid = ConverterUtils
.toNodeIdWithDefaultPort("[2401:db00:20:a01e:face:0:5:0]");
assertEquals(nid.getPort(), 0);
assertEquals(nid.getHost(), "[2401:db00:20:a01e:face:0:5:0]");
}
@Test(expected = IllegalArgumentException.class)

23
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/lib/TestZKClient.java
View File

@ -27,6 +27,7 @@ import java.net.InetSocketAddress;
import java.net.Socket;
import org.apache.hadoop.net.ServerSocketUtil;
import com.google.common.net.HostAndPort;
import org.junit.Assert;
import org.apache.hadoop.yarn.lib.ZKClient;
@ -86,8 +87,9 @@ public class TestZKClient {
long start = System.currentTimeMillis();
while (true) {
try {
String host = hp.split(":")[0];
int port = Integer.parseInt(hp.split(":")[1]);
HostAndPort hap = HostAndPort.fromString(hp);
String host = hap.getHost();
int port = hap.getPort();
send4LetterWord(host, port, "stat");
} catch (IOException e) {
return true;
@ -110,8 +112,9 @@ public class TestZKClient {
long start = System.currentTimeMillis();
while (true) {
try {
String host = hp.split(":")[0];
int port = Integer.parseInt(hp.split(":")[1]);
HostAndPort hap = HostAndPort.fromString(hp);
String host = hap.getHost();
int port = hap.getPort();
// if there are multiple hostports, just take the first one
String result = send4LetterWord(host, port, "stat");
if (result.startsWith("Zookeeper version:")) {
@ -151,14 +154,15 @@ public class TestZKClient {
}
File dataDir = createTmpDir(BASETEST);
zks = new ZooKeeperServer(dataDir, dataDir, 3000);
final int PORT = Integer.parseInt(hostPort.split(":")[1]);
HostAndPort hp = HostAndPort.fromString(hostPort);
final int port = hp.getPort();
if (factory == null) {
factory = new NIOServerCnxnFactory();
factory.configure(new InetSocketAddress(PORT), maxCnxns);
factory.configure(new InetSocketAddress(port), maxCnxns);
}
factory.startup(zks);
Assert.assertTrue("waiting for server up",
waitForServerUp("127.0.0.1:" + PORT,
waitForServerUp("127.0.0.1:" + port,
CONNECTION_TIMEOUT));
}
@ -172,10 +176,11 @@ public class TestZKClient {
zkDb.close();
} catch (IOException ie) {
}
final int PORT = Integer.parseInt(hostPort.split(":")[1]);
HostAndPort hp = HostAndPort.fromString(hostPort);
final int port = hp.getPort();
Assert.assertTrue("waiting for server down",
waitForServerDown("127.0.0.1:" + PORT,
waitForServerDown("127.0.0.1:" + port,
CONNECTION_TIMEOUT));
}

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
View File

@ -185,6 +185,7 @@ import java.util.Set;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
import com.google.common.net.HostAndPort;
import static org.apache.hadoop.service.Service.STATE.STARTED;
@ -645,7 +646,7 @@ public class ContainerManagerImpl extends CompositeService implements
//hostname found when querying for our hostname with the specified
//address, combine the specified address with the actual port listened
//on by the server
hostOverride = nmAddress.split(":")[0];
hostOverride = HostAndPort.fromString(nmAddress).getHost();
}
// setup node ID

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
View File

@ -137,6 +137,7 @@ import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import org.apache.zookeeper.server.auth.DigestAuthenticationProvider;
import org.eclipse.jetty.webapp.WebAppContext;
import com.google.common.net.HostAndPort;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintStream;
@ -1399,8 +1400,8 @@ public class ResourceManager extends CompositeService
builder.withAttribute(WebAppProxy.PROXY_CA,
rmContext.getProxyCAManager().getProxyCA());
builder.withAttribute(WebAppProxy.FETCHER_ATTRIBUTE, fetcher);
String[] proxyParts = proxyHostAndPort.split(":");
builder.withAttribute(WebAppProxy.PROXY_HOST_ATTRIBUTE, proxyParts[0]);
builder.withAttribute(WebAppProxy.PROXY_HOST_ATTRIBUTE,
HostAndPort.fromString(proxyHostAndPort).getHost());
}
WebAppContext uiWebAppContext = null;

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
View File

@ -30,6 +30,7 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import com.google.common.net.HostAndPort;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
@ -105,8 +106,10 @@ public class MockNM {
this.capability = capability;
this.resourceTracker = resourceTracker;
this.version = version;
String[] splits = nodeIdStr.split(":");
nodeId = BuilderUtils.newNodeId(splits[0], Integer.parseInt(splits[1]));
HostAndPort hostAndPort = HostAndPort.fromString(nodeIdStr);
String hostPortStr = hostAndPort.toString();
String host = hostPortStr.substring(0, hostPortStr.lastIndexOf(":"));
nodeId = BuilderUtils.newNodeId(host, hostAndPort.getPort());
}
public MockNM(String nodeIdStr, Resource capability,

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java
View File

@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.webproxy;
import java.io.IOException;
import java.net.URI;
import com.google.common.net.HostAndPort;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.security.authorize.AccessControlList;
@ -57,7 +58,8 @@ public class WebAppProxy extends AbstractService {
@Override
protected void serviceInit(Configuration conf) throws Exception {
String auth = conf.get(CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION);
String auth = conf.get(
CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION);
if (auth == null || "simple".equals(auth)) {
isSecurityEnabled = false;
} else if ("kerberos".equals(auth)) {
@ -68,8 +70,7 @@ public class WebAppProxy extends AbstractService {
" of " + auth);
}
String proxy = WebAppUtils.getProxyHostAndPort(conf);
String[] proxyParts = proxy.split(":");
proxyHost = proxyParts[0];
proxyHost = HostAndPort.fromString(proxy).getHost();
fetcher = new AppReportFetcher(conf);
bindAddress = conf.get(YarnConfiguration.PROXY_ADDRESS);

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java
View File

@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.webproxy.amfilter;
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
import com.google.common.net.HostAndPort;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.http.FilterContainer;
import org.apache.hadoop.http.FilterInitializer;
@ -38,14 +39,15 @@ public class AmFilterInitializer extends FilterInitializer {
private static final String FILTER_NAME = "AM_PROXY_FILTER";
private static final String FILTER_CLASS = AmIpFilter.class.getCanonicalName();
public static final String RM_HA_URLS = "RM_HA_URLS";
@Override
public void initFilter(FilterContainer container, Configuration conf) {
Map<String, String> params = new HashMap<>();
List<String> proxies = WebAppUtils.getProxyHostsAndPortsForAmFilter(conf);
StringBuilder sb = new StringBuilder();
for (String proxy : proxies) {
sb.append(proxy.split(":")[0]).append(AmIpFilter.PROXY_HOSTS_DELIMITER);
sb.append(HostAndPort.fromString(proxy).getHost())
.append(AmIpFilter.PROXY_HOSTS_DELIMITER);
}
sb.setLength(sb.length() - 1);
params.put(AmIpFilter.PROXY_HOSTS, sb.toString());

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java
View File

@ -50,6 +50,7 @@ import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import com.google.common.net.HostAndPort;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.http.HttpServer2;
@ -589,8 +590,7 @@ public class TestWebAppProxyServlet {
proxyServer.setAttribute(IS_SECURITY_ENABLED_ATTRIBUTE, Boolean.TRUE);
String proxy = WebAppUtils.getProxyHostAndPort(conf);
String[] proxyParts = proxy.split(":");
String proxyHost = proxyParts[0];
String proxyHost = HostAndPort.fromString(proxy).getHost();
proxyServer.setAttribute(PROXY_HOST_ATTRIBUTE, proxyHost);
proxyServer.start();