Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-3714. AM proxy filter can not get RM webapp address from 

yarn.resourcemanager.hostname.rm-id. Contributed by Masatake Iwasaki
This commit is contained in:
Xuan 2015-06-16 14:05:09 -07:00
parent 6a76250b39
commit e27d5a13b0
3 changed files with 70 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/CHANGES.txt
View File

@ -527,6 +527,9 @@ Release 2.8.0 - UNRELEASED
YARN-3794. TestRMEmbeddedElector fails because of ambiguous LOG reference.
(Chengbing Liu via devaraj)
YARN-3714. AM proxy filter can not get RM webapp address from
yarn.resourcemanager.hostname.rm-id. (Masatake Iwasaki via xgong)
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES

33
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/RMHAUtils.java
View File

@ -28,6 +28,7 @@ import org.apache.hadoop.ha.HAServiceProtocol;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.ha.HAServiceTarget;
import org.apache.hadoop.yarn.client.RMHAServiceTarget;
import org.apache.hadoop.yarn.conf.HAUtil;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@Private
@ -71,25 +72,31 @@ public class RMHAUtils {
public static List<String> getRMHAWebappAddresses(
final YarnConfiguration conf) {
String prefix;
String defaultPort;
if (YarnConfiguration.useHttps(conf)) {
prefix = YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS;
defaultPort = ":" + YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_PORT;
} else {
prefix =YarnConfiguration.RM_WEBAPP_ADDRESS;
defaultPort = ":" + YarnConfiguration.DEFAULT_RM_WEBAPP_PORT;
}
Collection<String> rmIds =
conf.getStringCollection(YarnConfiguration.RM_HA_IDS);
List<String> addrs = new ArrayList<String>();
if (YarnConfiguration.useHttps(conf)) {
for (String id : rmIds) {
String addr = conf.get(
YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS + "." + id);
if (addr != null) {
addrs.add(addr);
for (String id : rmIds) {
String addr = conf.get(HAUtil.addSuffix(prefix, id));
if (addr == null) {
String hostname =
conf.get(HAUtil.addSuffix(YarnConfiguration.RM_HOSTNAME, id));
if (hostname != null) {
addr = hostname + defaultPort;
}
}
} else {
for (String id : rmIds) {
String addr = conf.get(YarnConfiguration.RM_WEBAPP_ADDRESS + "." + id);
if (addr != null) {
addrs.add(addr);
}
if (addr != null) {
addrs.add(addr);
}
}
return addrs;
}
}
}

47
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilterInitializer.java
View File

@ -135,6 +135,15 @@ public class TestAmFilterInitializer extends TestCase {
assertEquals(WebAppUtils.getResolvedRMWebAppURLWithoutScheme(conf),
proxyHosts.get(0));
// Check conf in which only RM hostname is set
conf = new Configuration(false);
conf.set(YarnConfiguration.RM_WEBAPP_ADDRESS,
"${yarn.resourcemanager.hostname}:8088"); // default in yarn-default.xml
conf.set(YarnConfiguration.RM_HOSTNAME, "host1");
proxyHosts = WebAppUtils.getProxyHostsAndPortsForAmFilter(conf);
assertEquals(1, proxyHosts.size());
assertEquals("host1:8088", proxyHosts.get(0));
// Check PROXY_ADDRESS has priority
conf = new Configuration(false);
conf.set(YarnConfiguration.PROXY_ADDRESS, "host1:1000");
@ -188,6 +197,44 @@ public class TestAmFilterInitializer extends TestCase {
Collections.sort(proxyHosts);
assertEquals("host5:5000", proxyHosts.get(0));
assertEquals("host6:6000", proxyHosts.get(1));
// Check config without explicit RM_WEBAPP_ADDRESS settings (RM HA)
conf = new Configuration(false);
conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
conf.set(YarnConfiguration.RM_HA_IDS, "rm1,rm2,rm3");
conf.set(YarnConfiguration.RM_HOSTNAME + ".rm1", "host2");
conf.set(YarnConfiguration.RM_HOSTNAME + ".rm2", "host3");
conf.set(YarnConfiguration.RM_HOSTNAME + ".rm3", "host4");
conf.set(YarnConfiguration.RM_HOSTNAME + ".rm4", "dummy");
proxyHosts = WebAppUtils.getProxyHostsAndPortsForAmFilter(conf);
assertEquals(3, proxyHosts.size());
Collections.sort(proxyHosts);
assertEquals("host2:" + YarnConfiguration.DEFAULT_RM_WEBAPP_PORT,
proxyHosts.get(0));
assertEquals("host3:" + YarnConfiguration.DEFAULT_RM_WEBAPP_PORT,
proxyHosts.get(1));
assertEquals("host4:" + YarnConfiguration.DEFAULT_RM_WEBAPP_PORT,
proxyHosts.get(2));
// Check config without explicit RM_WEBAPP_HTTPS_ADDRESS settings (RM HA)
conf = new Configuration(false);
conf.set(YarnConfiguration.YARN_HTTP_POLICY_KEY,
HttpConfig.Policy.HTTPS_ONLY.toString());
conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
conf.set(YarnConfiguration.RM_HA_IDS, "rm1,rm2,rm3");
conf.set(YarnConfiguration.RM_HOSTNAME + ".rm1", "host2");
conf.set(YarnConfiguration.RM_HOSTNAME + ".rm2", "host3");
conf.set(YarnConfiguration.RM_HOSTNAME + ".rm3", "host4");
conf.set(YarnConfiguration.RM_HOSTNAME + ".rm4", "dummy");
proxyHosts = WebAppUtils.getProxyHostsAndPortsForAmFilter(conf);
assertEquals(3, proxyHosts.size());
Collections.sort(proxyHosts);
assertEquals("host2:" + YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_PORT,
proxyHosts.get(0));
assertEquals("host3:" + YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_PORT,
proxyHosts.get(1));
assertEquals("host4:" + YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_PORT,
proxyHosts.get(2));
}
class MockAmFilterInitializer extends AmFilterInitializer {