YARN-1553. Modified YARN and MR to stop using HttpConfig.isSecure() and
instead rely on the http policy framework. And also fix some bugs related to https handling in YARN web-apps. Contributed by Haohui Mai. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1568501 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
72f63c8957
commit
990cffdcfa
|
@ -279,7 +279,10 @@ public class CommonConfigurationKeysPublic {
|
|||
60;
|
||||
|
||||
// HTTP policies to be used in configuration
|
||||
// Use HttpPolicy.name() instead
|
||||
@Deprecated
|
||||
public static final String HTTP_POLICY_HTTP_ONLY = "HTTP_ONLY";
|
||||
@Deprecated
|
||||
public static final String HTTP_POLICY_HTTPS_ONLY = "HTTPS_ONLY";
|
||||
}
|
||||
|
||||
|
|
|
@ -28,7 +28,6 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
|
|||
@InterfaceAudience.Private
|
||||
@InterfaceStability.Unstable
|
||||
public class HttpConfig {
|
||||
private static Policy policy;
|
||||
public enum Policy {
|
||||
HTTP_ONLY,
|
||||
HTTPS_ONLY,
|
||||
|
@ -52,28 +51,4 @@ public class HttpConfig {
|
|||
return this == HTTPS_ONLY || this == HTTP_AND_HTTPS;
|
||||
}
|
||||
}
|
||||
|
||||
static {
|
||||
Configuration conf = new Configuration();
|
||||
boolean sslEnabled = conf.getBoolean(
|
||||
CommonConfigurationKeysPublic.HADOOP_SSL_ENABLED_KEY,
|
||||
CommonConfigurationKeysPublic.HADOOP_SSL_ENABLED_DEFAULT);
|
||||
policy = sslEnabled ? Policy.HTTPS_ONLY : Policy.HTTP_ONLY;
|
||||
}
|
||||
|
||||
public static void setPolicy(Policy policy) {
|
||||
HttpConfig.policy = policy;
|
||||
}
|
||||
|
||||
public static boolean isSecure() {
|
||||
return policy == Policy.HTTPS_ONLY;
|
||||
}
|
||||
|
||||
public static String getSchemePrefix() {
|
||||
return (isSecure()) ? "https://" : "http://";
|
||||
}
|
||||
|
||||
public static String getScheme(Policy policy) {
|
||||
return policy == Policy.HTTPS_ONLY ? "https://" : "http://";
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1387,7 +1387,8 @@ public class MRAppMaster extends CompositeService {
|
|||
// RM/NM to issue SSL certificates but definitely not MR-AM as it is
|
||||
// running in user-land.
|
||||
MRWebAppUtil.initialize(conf);
|
||||
HttpConfig.setPolicy(HttpConfig.Policy.HTTP_ONLY);
|
||||
conf.set(YarnConfiguration.YARN_HTTP_POLICY_KEY,
|
||||
HttpConfig.Policy.HTTP_ONLY.name());
|
||||
// log the system properties
|
||||
String systemPropsToLog = MRApps.getSystemPropertiesToLog(conf);
|
||||
if (systemPropsToLog != null) {
|
||||
|
|
|
@ -21,6 +21,7 @@ package org.apache.hadoop.mapreduce.v2.jobhistory;
|
|||
import org.apache.hadoop.classification.InterfaceAudience;
|
||||
import org.apache.hadoop.classification.InterfaceStability;
|
||||
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
|
||||
import org.apache.hadoop.http.HttpConfig;
|
||||
|
||||
/**
|
||||
* Stores Job History configuration keys that can be set by administrators of
|
||||
|
@ -135,7 +136,7 @@ public class JHAdminConfig {
|
|||
public static final String MR_HS_HTTP_POLICY = MR_HISTORY_PREFIX
|
||||
+ "http.policy";
|
||||
public static String DEFAULT_MR_HS_HTTP_POLICY =
|
||||
CommonConfigurationKeysPublic.HTTP_POLICY_HTTP_ONLY;
|
||||
HttpConfig.Policy.HTTP_ONLY.name();
|
||||
|
||||
/**The address the history server webapp is on.*/
|
||||
public static final String MR_HISTORY_WEBAPP_ADDRESS =
|
||||
|
|
|
@ -71,11 +71,13 @@ public class MRWebAppUtil {
|
|||
}
|
||||
|
||||
public static String getYARNWebappScheme() {
|
||||
return HttpConfig.getScheme(httpPolicyInYarn);
|
||||
return httpPolicyInYarn == HttpConfig.Policy.HTTPS_ONLY ? "https://"
|
||||
: "http://";
|
||||
}
|
||||
|
||||
|
||||
public static String getJHSWebappScheme() {
|
||||
return HttpConfig.getScheme(httpPolicyInJHS);
|
||||
return httpPolicyInJHS == HttpConfig.Policy.HTTPS_ONLY ? "https://"
|
||||
: "http://";
|
||||
}
|
||||
|
||||
public static void setJHSWebappURLWithoutScheme(Configuration conf,
|
||||
|
|
|
@ -45,6 +45,7 @@ import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo;
|
|||
import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskInfo;
|
||||
import org.apache.hadoop.mapreduce.util.HostUtil;
|
||||
import org.apache.hadoop.util.StringUtils;
|
||||
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
|
||||
|
||||
/**
|
||||
* HistoryViewer is used to parse and view the JobHistory files
|
||||
|
@ -231,7 +232,8 @@ public class HistoryViewer {
|
|||
taskList.append("\t");
|
||||
taskList.append(attempt.getHostname()).append("\t");
|
||||
taskList.append(attempt.getError());
|
||||
String taskLogsUrl = getTaskLogsUrl(attempt);
|
||||
String taskLogsUrl = getTaskLogsUrl(
|
||||
WebAppUtils.getHttpSchemePrefix(fs.getConf()), attempt);
|
||||
taskList.append(taskLogsUrl != null ? taskLogsUrl : "n/a");
|
||||
System.out.println(taskList.toString());
|
||||
}
|
||||
|
@ -446,7 +448,7 @@ public class HistoryViewer {
|
|||
* @return the taskLogsUrl. null if http-port or tracker-name or
|
||||
* task-attempt-id are unavailable.
|
||||
*/
|
||||
public static String getTaskLogsUrl(
|
||||
public static String getTaskLogsUrl(String scheme,
|
||||
JobHistoryParser.TaskAttemptInfo attempt) {
|
||||
if (attempt.getHttpPort() == -1
|
||||
|| attempt.getTrackerName().equals("")
|
||||
|
@ -457,7 +459,7 @@ public class HistoryViewer {
|
|||
String taskTrackerName =
|
||||
HostUtil.convertTrackerNameToHostName(
|
||||
attempt.getTrackerName());
|
||||
return HostUtil.getTaskLogUrl(taskTrackerName,
|
||||
return HostUtil.getTaskLogUrl(scheme, taskTrackerName,
|
||||
Integer.toString(attempt.getHttpPort()),
|
||||
attempt.getAttemptId().toString());
|
||||
}
|
||||
|
|
|
@ -20,7 +20,6 @@ package org.apache.hadoop.mapreduce.util;
|
|||
|
||||
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
||||
import org.apache.hadoop.classification.InterfaceStability.Unstable;
|
||||
import org.apache.hadoop.http.HttpConfig;
|
||||
|
||||
@Private
|
||||
@Unstable
|
||||
|
@ -33,9 +32,9 @@ public class HostUtil {
|
|||
* @param taskAttemptID
|
||||
* @return the taskLogUrl
|
||||
*/
|
||||
public static String getTaskLogUrl(String taskTrackerHostName,
|
||||
public static String getTaskLogUrl(String scheme, String taskTrackerHostName,
|
||||
String httpPort, String taskAttemptID) {
|
||||
return (HttpConfig.getSchemePrefix() + taskTrackerHostName + ":" +
|
||||
return (scheme + taskTrackerHostName + ":" +
|
||||
httpPort + "/tasklog?attemptid=" + taskAttemptID);
|
||||
}
|
||||
|
||||
|
|
|
@ -24,7 +24,6 @@ import org.apache.commons.logging.Log;
|
|||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.http.HttpConfig;
|
||||
import org.apache.hadoop.mapred.JobConf;
|
||||
import org.apache.hadoop.mapreduce.MRConfig;
|
||||
import org.apache.hadoop.mapreduce.v2.hs.HistoryServerStateStoreService.HistoryServerState;
|
||||
|
@ -121,7 +120,6 @@ public class JobHistoryServer extends CompositeService {
|
|||
|
||||
// This is required for WebApps to use https if enabled.
|
||||
MRWebAppUtil.initialize(getConfig());
|
||||
HttpConfig.setPolicy(MRWebAppUtil.getJHSHttpPolicy());
|
||||
try {
|
||||
doSecureLogin(conf);
|
||||
} catch(IOException ie) {
|
||||
|
|
|
@ -230,9 +230,9 @@ public class MiniMRYarnCluster extends MiniYARNCluster {
|
|||
WebAppUtils.getRMWebAppURLWithoutScheme(getConfig()));
|
||||
LOG.info("MiniMRYARN HistoryServer address: " +
|
||||
getConfig().get(JHAdminConfig.MR_HISTORY_ADDRESS));
|
||||
LOG.info("MiniMRYARN HistoryServer web address: " +
|
||||
getResolvedMRHistoryWebAppURLWithoutScheme(getConfig(),
|
||||
HttpConfig.isSecure()));
|
||||
LOG.info("MiniMRYARN HistoryServer web address: "
|
||||
+ getResolvedMRHistoryWebAppURLWithoutScheme(getConfig(),
|
||||
MRWebAppUtil.getJHSHttpPolicy() == HttpConfig.Policy.HTTPS_ONLY));
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
@ -273,6 +273,10 @@ Release 2.4.0 - UNRELEASED
|
|||
at allocation time so as to prevent RM from shelling out containers with
|
||||
expired tokens. (Omkar Vinit Joshi and Jian He via vinodkv)
|
||||
|
||||
YARN-1553. Modified YARN and MR to stop using HttpConfig.isSecure() and
|
||||
instead rely on the http policy framework. And also fix some bugs related
|
||||
to https handling in YARN web-apps. (Haohui Mai via vinodkv)
|
||||
|
||||
Release 2.3.1 - UNRELEASED
|
||||
|
||||
INCOMPATIBLE CHANGES
|
||||
|
|
|
@ -100,7 +100,7 @@ public class HAUtil {
|
|||
StringBuilder setValue = new StringBuilder();
|
||||
for (String id: ids) {
|
||||
// verify the RM service addresses configurations for every RMIds
|
||||
for (String prefix : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) {
|
||||
for (String prefix : YarnConfiguration.getServiceAddressConfKeys(conf)) {
|
||||
checkAndSetRMRPCAddress(prefix, id, conf);
|
||||
}
|
||||
setValue.append(id);
|
||||
|
@ -158,7 +158,7 @@ public class HAUtil {
|
|||
}
|
||||
|
||||
public static void verifyAndSetAllServiceAddresses(Configuration conf) {
|
||||
for (String confKey : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) {
|
||||
for (String confKey : YarnConfiguration.getServiceAddressConfKeys(conf)) {
|
||||
verifyAndSetConfValue(confKey, conf);
|
||||
}
|
||||
}
|
||||
|
@ -236,7 +236,7 @@ public class HAUtil {
|
|||
@InterfaceAudience.Private
|
||||
@VisibleForTesting
|
||||
static String getConfKeyForRMInstance(String prefix, Configuration conf) {
|
||||
if (!YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS.contains(prefix)) {
|
||||
if (!YarnConfiguration.getServiceAddressConfKeys(conf).contains(prefix)) {
|
||||
return prefix;
|
||||
} else {
|
||||
String RMId = getRMHAId(conf);
|
||||
|
@ -289,7 +289,7 @@ public class HAUtil {
|
|||
hostNameConfKey + " or " + addSuffix(prefix, RMId)));
|
||||
} else {
|
||||
conf.set(addSuffix(prefix, RMId), confVal + ":"
|
||||
+ YarnConfiguration.getRMDefaultPortNumber(prefix));
|
||||
+ YarnConfiguration.getRMDefaultPortNumber(prefix, conf));
|
||||
}
|
||||
}
|
||||
} catch (IllegalArgumentException iae) {
|
||||
|
|
|
@ -26,10 +26,8 @@ import java.util.List;
|
|||
import org.apache.hadoop.HadoopIllegalArgumentException;
|
||||
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
||||
import org.apache.hadoop.classification.InterfaceAudience.Public;
|
||||
import org.apache.hadoop.classification.InterfaceStability;
|
||||
import org.apache.hadoop.classification.InterfaceStability.Evolving;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
|
||||
import org.apache.hadoop.http.HttpConfig;
|
||||
import org.apache.hadoop.net.NetUtils;
|
||||
import org.apache.hadoop.util.StringUtils;
|
||||
|
@ -187,6 +185,8 @@ public class YarnConfiguration extends Configuration {
|
|||
/** The https address of the RM web application.*/
|
||||
public static final String RM_WEBAPP_HTTPS_ADDRESS =
|
||||
RM_PREFIX + "webapp.https.address";
|
||||
public static final boolean YARN_SSL_CLIENT_HTTPS_NEED_AUTH_DEFAULT = false;
|
||||
public static final String YARN_SSL_SERVER_RESOURCE_DEFAULT = "ssl-server.xml";
|
||||
|
||||
public static final int DEFAULT_RM_WEBAPP_HTTPS_PORT = 8090;
|
||||
public static final String DEFAULT_RM_WEBAPP_HTTPS_ADDRESS = "0.0.0.0:"
|
||||
|
@ -361,15 +361,21 @@ public class YarnConfiguration extends Configuration {
|
|||
public static final String DEFAULT_RM_CONFIGURATION_PROVIDER_CLASS =
|
||||
"org.apache.hadoop.yarn.LocalConfigurationProvider";
|
||||
|
||||
@Private
|
||||
public static final List<String> RM_SERVICES_ADDRESS_CONF_KEYS =
|
||||
private static final List<String> RM_SERVICES_ADDRESS_CONF_KEYS_HTTP =
|
||||
Collections.unmodifiableList(Arrays.asList(
|
||||
RM_ADDRESS,
|
||||
RM_SCHEDULER_ADDRESS,
|
||||
RM_ADMIN_ADDRESS,
|
||||
RM_RESOURCE_TRACKER_ADDRESS,
|
||||
HttpConfig.isSecure() ? RM_WEBAPP_HTTPS_ADDRESS
|
||||
: RM_WEBAPP_ADDRESS));
|
||||
RM_WEBAPP_ADDRESS));
|
||||
|
||||
private static final List<String> RM_SERVICES_ADDRESS_CONF_KEYS_HTTPS =
|
||||
Collections.unmodifiableList(Arrays.asList(
|
||||
RM_ADDRESS,
|
||||
RM_SCHEDULER_ADDRESS,
|
||||
RM_ADMIN_ADDRESS,
|
||||
RM_RESOURCE_TRACKER_ADDRESS,
|
||||
RM_WEBAPP_HTTPS_ADDRESS));
|
||||
|
||||
public static final String AUTO_FAILOVER_PREFIX =
|
||||
RM_HA_PREFIX + "automatic-failover.";
|
||||
|
@ -1102,10 +1108,9 @@ public class YarnConfiguration extends Configuration {
|
|||
YARN_PREFIX + "client.max-nodemanagers-proxies";
|
||||
public static final int DEFAULT_NM_CLIENT_MAX_NM_PROXIES = 500;
|
||||
|
||||
public static final String YARN_HTTP_POLICY_KEY =
|
||||
YARN_PREFIX + "http.policy";
|
||||
public static final String YARN_HTTP_POLICY_DEFAULT =
|
||||
CommonConfigurationKeysPublic.HTTP_POLICY_HTTP_ONLY;
|
||||
public static final String YARN_HTTP_POLICY_KEY = YARN_PREFIX + "http.policy";
|
||||
public static final String YARN_HTTP_POLICY_DEFAULT = HttpConfig.Policy.HTTP_ONLY
|
||||
.name();
|
||||
|
||||
public YarnConfiguration() {
|
||||
super();
|
||||
|
@ -1118,6 +1123,12 @@ public class YarnConfiguration extends Configuration {
|
|||
}
|
||||
}
|
||||
|
||||
@Private
|
||||
public static List<String> getServiceAddressConfKeys(Configuration conf) {
|
||||
return useHttps(conf) ? RM_SERVICES_ADDRESS_CONF_KEYS_HTTPS
|
||||
: RM_SERVICES_ADDRESS_CONF_KEYS_HTTP;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the socket address for <code>name</code> property as a
|
||||
* <code>InetSocketAddress</code>.
|
||||
|
@ -1130,7 +1141,7 @@ public class YarnConfiguration extends Configuration {
|
|||
public InetSocketAddress getSocketAddr(
|
||||
String name, String defaultAddress, int defaultPort) {
|
||||
String address;
|
||||
if (HAUtil.isHAEnabled(this) && RM_SERVICES_ADDRESS_CONF_KEYS.contains(name)) {
|
||||
if (HAUtil.isHAEnabled(this) && getServiceAddressConfKeys(this).contains(name)) {
|
||||
address = HAUtil.getConfValueForRMInstance(name, defaultAddress, this);
|
||||
} else {
|
||||
address = get(name, defaultAddress);
|
||||
|
@ -1149,7 +1160,8 @@ public class YarnConfiguration extends Configuration {
|
|||
}
|
||||
|
||||
@Private
|
||||
public static int getRMDefaultPortNumber(String addressPrefix) {
|
||||
public static int getRMDefaultPortNumber(String addressPrefix,
|
||||
Configuration conf) {
|
||||
if (addressPrefix.equals(YarnConfiguration.RM_ADDRESS)) {
|
||||
return YarnConfiguration.DEFAULT_RM_PORT;
|
||||
} else if (addressPrefix.equals(YarnConfiguration.RM_SCHEDULER_ADDRESS)) {
|
||||
|
@ -1167,7 +1179,13 @@ public class YarnConfiguration extends Configuration {
|
|||
throw new HadoopIllegalArgumentException(
|
||||
"Invalid RM RPC address Prefix: " + addressPrefix
|
||||
+ ". The valid value should be one of "
|
||||
+ YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS);
|
||||
+ getServiceAddressConfKeys(conf));
|
||||
}
|
||||
}
|
||||
|
||||
public static boolean useHttps(Configuration conf) {
|
||||
return HttpConfig.Policy.HTTPS_ONLY == HttpConfig.Policy.fromString(conf
|
||||
.get(YARN_HTTP_POLICY_KEY,
|
||||
YARN_HTTP_POLICY_DEFAULT));
|
||||
}
|
||||
}
|
||||
|
|
|
@ -29,7 +29,6 @@ import org.apache.commons.logging.LogFactory;
|
|||
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
||||
import org.apache.hadoop.classification.InterfaceStability.Unstable;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.http.HttpConfig;
|
||||
import org.apache.hadoop.yarn.api.records.apptimeline.ATSEntities;
|
||||
import org.apache.hadoop.yarn.api.records.apptimeline.ATSEntity;
|
||||
import org.apache.hadoop.yarn.api.records.apptimeline.ATSPutErrors;
|
||||
|
@ -65,12 +64,17 @@ public class TimelineClientImpl extends TimelineClient {
|
|||
}
|
||||
|
||||
protected void serviceInit(Configuration conf) throws Exception {
|
||||
resURI = new URI(JOINER.join(HttpConfig.getSchemePrefix(),
|
||||
HttpConfig.isSecure() ? conf.get(
|
||||
YarnConfiguration.AHS_WEBAPP_HTTPS_ADDRESS,
|
||||
YarnConfiguration.DEFAULT_AHS_WEBAPP_HTTPS_ADDRESS) : conf.get(
|
||||
YarnConfiguration.AHS_WEBAPP_ADDRESS,
|
||||
YarnConfiguration.DEFAULT_AHS_WEBAPP_ADDRESS), RESOURCE_URI_STR));
|
||||
if (YarnConfiguration.useHttps(conf)) {
|
||||
resURI = URI
|
||||
.create(JOINER.join("https://", conf.get(
|
||||
YarnConfiguration.AHS_WEBAPP_HTTPS_ADDRESS,
|
||||
YarnConfiguration.DEFAULT_AHS_WEBAPP_HTTPS_ADDRESS),
|
||||
RESOURCE_URI_STR));
|
||||
} else {
|
||||
resURI = URI.create(JOINER.join("http://", conf.get(
|
||||
YarnConfiguration.AHS_WEBAPP_ADDRESS,
|
||||
YarnConfiguration.DEFAULT_AHS_WEBAPP_ADDRESS), RESOURCE_URI_STR));
|
||||
}
|
||||
super.serviceInit(conf);
|
||||
}
|
||||
|
||||
|
|
|
@ -37,7 +37,9 @@ import org.apache.hadoop.classification.InterfaceAudience;
|
|||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.http.HttpServer2;
|
||||
import org.apache.hadoop.security.UserGroupInformation;
|
||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||
import org.apache.hadoop.yarn.security.AdminACLsManager;
|
||||
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
|
@ -216,9 +218,11 @@ public class WebApps {
|
|||
System.exit(1);
|
||||
}
|
||||
}
|
||||
HttpServer2.Builder builder = new HttpServer2.Builder().setName(name)
|
||||
.addEndpoint(URI.create("http://" + bindAddress + ":" + port))
|
||||
.setConf(conf).setFindPort(findPort)
|
||||
HttpServer2.Builder builder = new HttpServer2.Builder()
|
||||
.setName(name)
|
||||
.addEndpoint(
|
||||
URI.create(WebAppUtils.getHttpSchemePrefix(conf) + bindAddress
|
||||
+ ":" + port)).setConf(conf).setFindPort(findPort)
|
||||
.setACL(new AdminACLsManager(conf).getAdminAcl())
|
||||
.setPathSpec(pathList.toArray(new String[0]));
|
||||
|
||||
|
@ -231,6 +235,11 @@ public class WebApps {
|
|||
.setKeytabConfKey(spnegoKeytabKey)
|
||||
.setSecurityEnabled(UserGroupInformation.isSecurityEnabled());
|
||||
}
|
||||
|
||||
if (YarnConfiguration.useHttps(conf)) {
|
||||
WebAppUtils.loadSslConfiguration(builder);
|
||||
}
|
||||
|
||||
HttpServer2 server = builder.build();
|
||||
|
||||
for(ServletStruct struct: servlets) {
|
||||
|
|
|
@ -26,20 +26,16 @@ import java.net.UnknownHostException;
|
|||
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
||||
import org.apache.hadoop.classification.InterfaceStability.Evolving;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.http.HttpConfig;
|
||||
import org.apache.hadoop.http.HttpConfig.Policy;
|
||||
import org.apache.hadoop.http.HttpServer2;
|
||||
import org.apache.hadoop.net.NetUtils;
|
||||
import org.apache.hadoop.yarn.api.records.ContainerId;
|
||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||
import org.apache.hadoop.yarn.util.ConverterUtils;
|
||||
|
||||
import com.google.common.base.Joiner;
|
||||
|
||||
@Private
|
||||
@Evolving
|
||||
public class WebAppUtils {
|
||||
private static final Joiner JOINER = Joiner.on("");
|
||||
|
||||
public static void setRMWebAppPort(Configuration conf, int port) {
|
||||
String hostname = getRMWebAppURLWithoutScheme(conf);
|
||||
hostname =
|
||||
|
@ -51,7 +47,7 @@ public class WebAppUtils {
|
|||
public static void setRMWebAppHostnameAndPort(Configuration conf,
|
||||
String hostname, int port) {
|
||||
String resolvedAddress = hostname + ":" + port;
|
||||
if (HttpConfig.isSecure()) {
|
||||
if (YarnConfiguration.useHttps(conf)) {
|
||||
conf.set(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS, resolvedAddress);
|
||||
} else {
|
||||
conf.set(YarnConfiguration.RM_WEBAPP_ADDRESS, resolvedAddress);
|
||||
|
@ -60,7 +56,7 @@ public class WebAppUtils {
|
|||
|
||||
public static void setNMWebAppHostNameAndPort(Configuration conf,
|
||||
String hostName, int port) {
|
||||
if (HttpConfig.isSecure()) {
|
||||
if (YarnConfiguration.useHttps(conf)) {
|
||||
conf.set(YarnConfiguration.NM_WEBAPP_HTTPS_ADDRESS,
|
||||
hostName + ":" + port);
|
||||
} else {
|
||||
|
@ -70,16 +66,11 @@ public class WebAppUtils {
|
|||
}
|
||||
|
||||
public static String getRMWebAppURLWithScheme(Configuration conf) {
|
||||
return JOINER.join(HttpConfig.getSchemePrefix(),
|
||||
HttpConfig.isSecure() ? conf.get(
|
||||
YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS,
|
||||
YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_ADDRESS) : conf.get(
|
||||
YarnConfiguration.RM_WEBAPP_ADDRESS,
|
||||
YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS));
|
||||
return getHttpSchemePrefix(conf) + getRMWebAppURLWithoutScheme(conf);
|
||||
}
|
||||
|
||||
public static String getRMWebAppURLWithoutScheme(Configuration conf) {
|
||||
if (HttpConfig.isSecure()) {
|
||||
if (YarnConfiguration.useHttps(conf)) {
|
||||
return conf.get(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS,
|
||||
YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_ADDRESS);
|
||||
}else {
|
||||
|
@ -97,13 +88,13 @@ public class WebAppUtils {
|
|||
}
|
||||
|
||||
public static String getResolvedRMWebAppURLWithScheme(Configuration conf) {
|
||||
return HttpConfig.getSchemePrefix()
|
||||
return getHttpSchemePrefix(conf)
|
||||
+ getResolvedRMWebAppURLWithoutScheme(conf);
|
||||
}
|
||||
|
||||
public static String getResolvedRMWebAppURLWithoutScheme(Configuration conf) {
|
||||
return getResolvedRMWebAppURLWithoutScheme(conf,
|
||||
HttpConfig.isSecure() ? Policy.HTTPS_ONLY : Policy.HTTP_ONLY);
|
||||
YarnConfiguration.useHttps(conf) ? Policy.HTTPS_ONLY : Policy.HTTP_ONLY);
|
||||
}
|
||||
|
||||
public static String getResolvedRMWebAppURLWithoutScheme(Configuration conf,
|
||||
|
@ -140,7 +131,7 @@ public class WebAppUtils {
|
|||
}
|
||||
|
||||
public static String getNMWebAppURLWithoutScheme(Configuration conf) {
|
||||
if (HttpConfig.isSecure()) {
|
||||
if (YarnConfiguration.useHttps(conf)) {
|
||||
return conf.get(YarnConfiguration.NM_WEBAPP_HTTPS_ADDRESS,
|
||||
YarnConfiguration.DEFAULT_NM_WEBAPP_HTTPS_ADDRESS);
|
||||
} else {
|
||||
|
@ -150,7 +141,7 @@ public class WebAppUtils {
|
|||
}
|
||||
|
||||
public static String getAHSWebAppURLWithoutScheme(Configuration conf) {
|
||||
if (HttpConfig.isSecure()) {
|
||||
if (YarnConfiguration.useHttps(conf)) {
|
||||
return conf.get(YarnConfiguration.AHS_WEBAPP_HTTPS_ADDRESS,
|
||||
YarnConfiguration.DEFAULT_AHS_WEBAPP_HTTPS_ADDRESS);
|
||||
} else {
|
||||
|
@ -177,8 +168,38 @@ public class WebAppUtils {
|
|||
|
||||
public static String getLogUrl(String nodeHttpAddress, String allocatedNode,
|
||||
ContainerId containerId, String user) {
|
||||
return join(HttpConfig.getSchemePrefix(), nodeHttpAddress, "/logs", "/",
|
||||
return join("//", nodeHttpAddress, "/logs", "/",
|
||||
allocatedNode, "/", ConverterUtils.toString(containerId), "/",
|
||||
ConverterUtils.toString(containerId), "/", user);
|
||||
}
|
||||
|
||||
/**
|
||||
* Choose which scheme (HTTP or HTTPS) to use when generating a URL based on
|
||||
* the configuration.
|
||||
*
|
||||
* @return the schmeme (HTTP / HTTPS)
|
||||
*/
|
||||
public static String getHttpSchemePrefix(Configuration conf) {
|
||||
return YarnConfiguration.useHttps(conf) ? "https://" : "http://";
|
||||
}
|
||||
|
||||
/**
|
||||
* Load the SSL keystore / truststore into the HttpServer builder.
|
||||
*/
|
||||
public static HttpServer2.Builder loadSslConfiguration(
|
||||
HttpServer2.Builder builder) {
|
||||
Configuration sslConf = new Configuration(false);
|
||||
boolean needsClientAuth = YarnConfiguration.YARN_SSL_CLIENT_HTTPS_NEED_AUTH_DEFAULT;
|
||||
sslConf.addResource(YarnConfiguration.YARN_SSL_SERVER_RESOURCE_DEFAULT);
|
||||
|
||||
return builder
|
||||
.needsClientAuth(needsClientAuth)
|
||||
.keyPassword(sslConf.get("ssl.server.keystore.keypassword"))
|
||||
.keyStore(sslConf.get("ssl.server.keystore.location"),
|
||||
sslConf.get("ssl.server.keystore.password"),
|
||||
sslConf.get("ssl.server.keystore.type", "jks"))
|
||||
.trustStore(sslConf.get("ssl.server.truststore.location"),
|
||||
sslConf.get("ssl.server.truststore.password"),
|
||||
sslConf.get("ssl.server.truststore.type", "jks"));
|
||||
}
|
||||
}
|
||||
|
|
|
@ -54,7 +54,7 @@ public class TestHAUtil {
|
|||
conf.set(YarnConfiguration.RM_HA_IDS, RM_NODE_IDS_UNTRIMMED);
|
||||
conf.set(YarnConfiguration.RM_HA_ID, RM1_NODE_ID_UNTRIMMED);
|
||||
|
||||
for (String confKey : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) {
|
||||
for (String confKey : YarnConfiguration.getServiceAddressConfKeys(conf)) {
|
||||
// configuration key itself cannot contains space/tab/return chars.
|
||||
conf.set(HAUtil.addSuffix(confKey, RM1_NODE_ID), RM1_ADDRESS_UNTRIMMED);
|
||||
conf.set(HAUtil.addSuffix(confKey, RM2_NODE_ID), RM2_ADDRESS);
|
||||
|
@ -95,7 +95,7 @@ public class TestHAUtil {
|
|||
StringUtils.getStringCollection(RM_NODE_IDS), HAUtil.getRMHAIds(conf));
|
||||
assertEquals("Should be saved as Trimmed string",
|
||||
RM1_NODE_ID, HAUtil.getRMHAId(conf));
|
||||
for (String confKey : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) {
|
||||
for (String confKey : YarnConfiguration.getServiceAddressConfKeys(conf)) {
|
||||
assertEquals("RPC address not set for " + confKey,
|
||||
RM1_ADDRESS, conf.get(confKey));
|
||||
}
|
||||
|
@ -117,7 +117,7 @@ public class TestHAUtil {
|
|||
// simulate the case YarnConfiguration.RM_HA_ID is not set
|
||||
conf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + ","
|
||||
+ RM2_NODE_ID);
|
||||
for (String confKey : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) {
|
||||
for (String confKey : YarnConfiguration.getServiceAddressConfKeys(conf)) {
|
||||
conf.set(HAUtil.addSuffix(confKey, RM1_NODE_ID), RM1_ADDRESS);
|
||||
conf.set(HAUtil.addSuffix(confKey, RM2_NODE_ID), RM2_ADDRESS);
|
||||
}
|
||||
|
@ -134,7 +134,7 @@ public class TestHAUtil {
|
|||
conf.set(YarnConfiguration.RM_HA_ID, RM_INVALID_NODE_ID);
|
||||
conf.set(YarnConfiguration.RM_HA_IDS, RM_INVALID_NODE_ID + ","
|
||||
+ RM1_NODE_ID);
|
||||
for (String confKey : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) {
|
||||
for (String confKey : YarnConfiguration.getServiceAddressConfKeys(conf)) {
|
||||
// simulate xml with invalid node id
|
||||
conf.set(confKey + RM_INVALID_NODE_ID, RM_INVALID_NODE_ID);
|
||||
}
|
||||
|
@ -169,7 +169,7 @@ public class TestHAUtil {
|
|||
conf.clear();
|
||||
conf.set(YarnConfiguration.RM_HA_IDS, RM2_NODE_ID + "," + RM3_NODE_ID);
|
||||
conf.set(YarnConfiguration.RM_HA_ID, RM1_NODE_ID_UNTRIMMED);
|
||||
for (String confKey : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) {
|
||||
for (String confKey : YarnConfiguration.getServiceAddressConfKeys(conf)) {
|
||||
conf.set(HAUtil.addSuffix(confKey, RM1_NODE_ID), RM1_ADDRESS_UNTRIMMED);
|
||||
conf.set(HAUtil.addSuffix(confKey, RM2_NODE_ID), RM2_ADDRESS);
|
||||
conf.set(HAUtil.addSuffix(confKey, RM3_NODE_ID), RM3_ADDRESS);
|
||||
|
|
|
@ -163,7 +163,7 @@ public class AppBlock extends HtmlBlock {
|
|||
.append(startTime)
|
||||
.append("\",\"<a href='")
|
||||
.append(
|
||||
nodeLink == null ? "#" : url(HttpConfig.getSchemePrefix(), nodeLink))
|
||||
nodeLink == null ? "#" : url("//", nodeLink))
|
||||
.append("'>")
|
||||
.append(
|
||||
nodeLink == null ? "N/A" : StringEscapeUtils
|
||||
|
|
|
@ -28,8 +28,6 @@ import org.apache.commons.logging.Log;
|
|||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.http.HttpConfig;
|
||||
import org.apache.hadoop.http.HttpConfig.Policy;
|
||||
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
|
||||
import org.apache.hadoop.security.SecurityUtil;
|
||||
import org.apache.hadoop.service.CompositeService;
|
||||
|
@ -397,15 +395,8 @@ public class NodeManager extends CompositeService
|
|||
StringUtils.startupShutdownMessage(NodeManager.class, args, LOG);
|
||||
NodeManager nodeManager = new NodeManager();
|
||||
Configuration conf = new YarnConfiguration();
|
||||
setHttpPolicy(conf);
|
||||
nodeManager.initAndStartNodeManager(conf, false);
|
||||
}
|
||||
|
||||
private static void setHttpPolicy(Configuration conf) {
|
||||
HttpConfig.setPolicy(Policy.fromString(conf.get(
|
||||
YarnConfiguration.YARN_HTTP_POLICY_KEY,
|
||||
YarnConfiguration.YARN_HTTP_POLICY_DEFAULT)));
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
@Private
|
||||
|
|
|
@ -19,7 +19,6 @@
|
|||
package org.apache.hadoop.yarn.server.nodemanager.webapp;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.http.HttpConfig;
|
||||
import org.apache.hadoop.yarn.webapp.YarnWebParams;
|
||||
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
|
||||
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
|
||||
|
|
|
@ -1015,7 +1015,6 @@ public class ResourceManager extends CompositeService implements Recoverable {
|
|||
ShutdownHookManager.get().addShutdownHook(
|
||||
new CompositeServiceShutdownHook(resourceManager),
|
||||
SHUTDOWN_HOOK_PRIORITY);
|
||||
setHttpPolicy(conf);
|
||||
resourceManager.init(conf);
|
||||
resourceManager.start();
|
||||
} catch (Throwable t) {
|
||||
|
@ -1023,12 +1022,6 @@ public class ResourceManager extends CompositeService implements Recoverable {
|
|||
System.exit(-1);
|
||||
}
|
||||
}
|
||||
|
||||
private static void setHttpPolicy(Configuration conf) {
|
||||
HttpConfig.setPolicy(Policy.fromString(conf.get(
|
||||
YarnConfiguration.YARN_HTTP_POLICY_KEY,
|
||||
YarnConfiguration.YARN_HTTP_POLICY_DEFAULT)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Register the handlers for alwaysOn services
|
||||
|
|
|
@ -503,10 +503,11 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
|
|||
final String trackingUriWithoutScheme) {
|
||||
this.readLock.lock();
|
||||
try {
|
||||
final String scheme = WebAppUtils.getHttpSchemePrefix(conf);
|
||||
URI trackingUri = StringUtils.isEmpty(trackingUriWithoutScheme) ? null :
|
||||
ProxyUriUtils.getUriFromAMUrl(trackingUriWithoutScheme);
|
||||
ProxyUriUtils.getUriFromAMUrl(scheme, trackingUriWithoutScheme);
|
||||
String proxy = WebAppUtils.getProxyHostAndPort(conf);
|
||||
URI proxyUri = ProxyUriUtils.getUriFromAMUrl(proxy);
|
||||
URI proxyUri = ProxyUriUtils.getUriFromAMUrl(scheme, proxy);
|
||||
URI result = ProxyUriUtils.getProxyUri(trackingUri, proxyUri,
|
||||
applicationAttemptId.getApplicationId());
|
||||
return result.toASCIIString();
|
||||
|
|
|
@ -27,7 +27,7 @@ import static org.apache.hadoop.yarn.webapp.view.JQueryUI._TH;
|
|||
|
||||
import java.util.Collection;
|
||||
|
||||
import org.apache.hadoop.http.HttpConfig;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.security.UserGroupInformation;
|
||||
import org.apache.hadoop.util.StringUtils;
|
||||
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
|
||||
|
@ -46,6 +46,7 @@ import org.apache.hadoop.yarn.util.Times;
|
|||
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
|
||||
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV;
|
||||
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
|
||||
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
|
||||
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
|
||||
import org.apache.hadoop.yarn.webapp.view.InfoBlock;
|
||||
|
||||
|
@ -55,13 +56,16 @@ public class AppBlock extends HtmlBlock {
|
|||
|
||||
private ApplicationACLsManager aclsManager;
|
||||
private QueueACLsManager queueACLsManager;
|
||||
private final Configuration conf;
|
||||
|
||||
@Inject
|
||||
AppBlock(ResourceManager rm, ViewContext ctx,
|
||||
ApplicationACLsManager aclsManager, QueueACLsManager queueACLsManager) {
|
||||
ApplicationACLsManager aclsManager, QueueACLsManager queueACLsManager,
|
||||
Configuration conf) {
|
||||
super(ctx);
|
||||
this.aclsManager = aclsManager;
|
||||
this.queueACLsManager = queueACLsManager;
|
||||
this.conf = conf;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -86,7 +90,7 @@ public class AppBlock extends HtmlBlock {
|
|||
puts("Application not found: "+ aid);
|
||||
return;
|
||||
}
|
||||
AppInfo app = new AppInfo(rmApp, true);
|
||||
AppInfo app = new AppInfo(rmApp, true, WebAppUtils.getHttpSchemePrefix(conf));
|
||||
|
||||
// Check for the authorization.
|
||||
String remoteUser = request().getRemoteUser();
|
||||
|
@ -146,7 +150,7 @@ public class AppBlock extends HtmlBlock {
|
|||
table.tr((odd = !odd) ? _ODD : _EVEN).
|
||||
td(String.valueOf(attemptInfo.getAttemptId())).
|
||||
td(Times.format(attemptInfo.getStartTime())).
|
||||
td().a(".nodelink", url(HttpConfig.getSchemePrefix(),
|
||||
td().a(".nodelink", url("//",
|
||||
attemptInfo.getNodeHttpAddress()),
|
||||
attemptInfo.getNodeHttpAddress())._().
|
||||
td().a(".logslink", url(attemptInfo.getLogsLink()), "logs")._().
|
||||
|
|
|
@ -28,6 +28,7 @@ import java.util.HashSet;
|
|||
import java.util.concurrent.ConcurrentMap;
|
||||
|
||||
import org.apache.commons.lang.StringEscapeUtils;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
|
||||
|
@ -36,16 +37,19 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo;
|
|||
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
|
||||
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
|
||||
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY;
|
||||
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
|
||||
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
|
||||
|
||||
import com.google.inject.Inject;
|
||||
|
||||
class AppsBlock extends HtmlBlock {
|
||||
final ConcurrentMap<ApplicationId, RMApp> apps;
|
||||
private final Configuration conf;
|
||||
|
||||
@Inject AppsBlock(RMContext rmContext, ViewContext ctx) {
|
||||
@Inject AppsBlock(RMContext rmContext, ViewContext ctx, Configuration conf) {
|
||||
super(ctx);
|
||||
apps = rmContext.getRMApps();
|
||||
this.conf = conf;
|
||||
}
|
||||
|
||||
@Override public void render(Block html) {
|
||||
|
@ -79,7 +83,7 @@ class AppsBlock extends HtmlBlock {
|
|||
if (reqAppStates != null && !reqAppStates.contains(app.createApplicationState())) {
|
||||
continue;
|
||||
}
|
||||
AppInfo appInfo = new AppInfo(app, true);
|
||||
AppInfo appInfo = new AppInfo(app, true, WebAppUtils.getHttpSchemePrefix(conf));
|
||||
String percent = String.format("%.1f", appInfo.getProgress());
|
||||
//AppID numerical value parsed by parseHadoopID in yarn.dt.plugins.js
|
||||
appsTableData.append("[\"<a href='")
|
||||
|
|
|
@ -28,6 +28,7 @@ import java.util.HashSet;
|
|||
import java.util.concurrent.ConcurrentMap;
|
||||
|
||||
import org.apache.commons.lang.StringEscapeUtils;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
|
||||
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
|
||||
|
@ -40,6 +41,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FairSchedulerInf
|
|||
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
|
||||
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
|
||||
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY;
|
||||
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
|
||||
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
|
||||
|
||||
import com.google.inject.Inject;
|
||||
|
@ -51,13 +53,15 @@ import com.google.inject.Inject;
|
|||
public class FairSchedulerAppsBlock extends HtmlBlock {
|
||||
final ConcurrentMap<ApplicationId, RMApp> apps;
|
||||
final FairSchedulerInfo fsinfo;
|
||||
final Configuration conf;
|
||||
|
||||
@Inject public FairSchedulerAppsBlock(RMContext rmContext,
|
||||
ResourceManager rm, ViewContext ctx) {
|
||||
ResourceManager rm, ViewContext ctx, Configuration conf) {
|
||||
super(ctx);
|
||||
FairScheduler scheduler = (FairScheduler) rm.getResourceScheduler();
|
||||
fsinfo = new FairSchedulerInfo(scheduler);
|
||||
apps = rmContext.getRMApps();
|
||||
this.conf = conf;
|
||||
}
|
||||
|
||||
@Override public void render(Block html) {
|
||||
|
@ -91,7 +95,7 @@ public class FairSchedulerAppsBlock extends HtmlBlock {
|
|||
if (reqAppStates != null && !reqAppStates.contains(app.getState())) {
|
||||
continue;
|
||||
}
|
||||
AppInfo appInfo = new AppInfo(app, true);
|
||||
AppInfo appInfo = new AppInfo(app, true, WebAppUtils.getHttpSchemePrefix(conf));
|
||||
String percent = String.format("%.1f", appInfo.getProgress());
|
||||
ApplicationAttemptId attemptId = app.getCurrentAppAttempt().getAppAttemptId();
|
||||
int fairShare = fsinfo.getAppFairShare(attemptId);
|
||||
|
|
|
@ -26,7 +26,6 @@ import static org.apache.hadoop.yarn.webapp.view.JQueryUI.tableInit;
|
|||
|
||||
import java.util.Collection;
|
||||
|
||||
import org.apache.hadoop.http.HttpConfig;
|
||||
import org.apache.hadoop.util.StringUtils;
|
||||
import org.apache.hadoop.yarn.api.records.NodeState;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
|
||||
|
@ -119,7 +118,7 @@ class NodesPage extends RmView {
|
|||
row.td()._("N/A")._();
|
||||
} else {
|
||||
String httpAddress = info.getNodeHTTPAddress();
|
||||
row.td().a(HttpConfig.getSchemePrefix() + httpAddress,
|
||||
row.td().a("//" + httpAddress,
|
||||
httpAddress)._();
|
||||
}
|
||||
row.td().br().$title(String.valueOf(info.getLastHealthUpdate()))._().
|
||||
|
|
|
@ -41,6 +41,7 @@ import javax.ws.rs.core.MediaType;
|
|||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.security.UserGroupInformation;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
|
||||
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
|
||||
|
@ -85,6 +86,7 @@ import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
|
|||
import org.apache.hadoop.yarn.util.ConverterUtils;
|
||||
import org.apache.hadoop.yarn.webapp.BadRequestException;
|
||||
import org.apache.hadoop.yarn.webapp.NotFoundException;
|
||||
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
|
||||
|
||||
import com.google.inject.Inject;
|
||||
import com.google.inject.Singleton;
|
||||
|
@ -101,16 +103,18 @@ public class RMWebServices {
|
|||
.getRecordFactory(null);
|
||||
private final ApplicationACLsManager aclsManager;
|
||||
private final QueueACLsManager queueACLsManager;
|
||||
|
||||
private final Configuration conf;
|
||||
private @Context HttpServletResponse response;
|
||||
|
||||
@Inject
|
||||
public RMWebServices(final ResourceManager rm,
|
||||
final ApplicationACLsManager aclsManager,
|
||||
final QueueACLsManager queueACLsManager) {
|
||||
final QueueACLsManager queueACLsManager,
|
||||
Configuration conf) {
|
||||
this.rm = rm;
|
||||
this.aclsManager = aclsManager;
|
||||
this.queueACLsManager = queueACLsManager;
|
||||
this.conf = conf;
|
||||
}
|
||||
|
||||
protected Boolean hasAccess(RMApp app, HttpServletRequest hsr) {
|
||||
|
@ -415,7 +419,8 @@ public class RMWebServices {
|
|||
}
|
||||
}
|
||||
|
||||
AppInfo app = new AppInfo(rmapp, hasAccess(rmapp, hsr));
|
||||
AppInfo app = new AppInfo(rmapp, hasAccess(rmapp, hsr),
|
||||
WebAppUtils.getHttpSchemePrefix(conf));
|
||||
allApps.add(app);
|
||||
}
|
||||
return allApps;
|
||||
|
@ -555,7 +560,7 @@ public class RMWebServices {
|
|||
if (app == null) {
|
||||
throw new NotFoundException("app with id: " + appId + " not found");
|
||||
}
|
||||
return new AppInfo(app, hasAccess(app, hsr));
|
||||
return new AppInfo(app, hasAccess(app, hsr), hsr.getScheme() + "://");
|
||||
}
|
||||
|
||||
@GET
|
||||
|
|
|
@ -23,7 +23,6 @@ import javax.xml.bind.annotation.XmlAccessType;
|
|||
import javax.xml.bind.annotation.XmlAccessorType;
|
||||
import javax.xml.bind.annotation.XmlRootElement;
|
||||
|
||||
import org.apache.hadoop.http.HttpConfig;
|
||||
import org.apache.hadoop.yarn.api.records.Container;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
|
||||
import org.apache.hadoop.yarn.util.ConverterUtils;
|
||||
|
@ -56,7 +55,7 @@ public class AppAttemptInfo {
|
|||
this.containerId = masterContainer.getId().toString();
|
||||
this.nodeHttpAddress = masterContainer.getNodeHttpAddress();
|
||||
this.nodeId = masterContainer.getNodeId().toString();
|
||||
this.logsLink = join(HttpConfig.getSchemePrefix(),
|
||||
this.logsLink = join("//",
|
||||
masterContainer.getNodeHttpAddress(),
|
||||
"/node", "/containerlogs/",
|
||||
ConverterUtils.toString(masterContainer.getId()), "/", user);
|
||||
|
|
|
@ -25,7 +25,6 @@ import javax.xml.bind.annotation.XmlRootElement;
|
|||
import javax.xml.bind.annotation.XmlTransient;
|
||||
|
||||
import com.google.common.base.Joiner;
|
||||
import org.apache.hadoop.http.HttpConfig;
|
||||
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||
import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
|
||||
import org.apache.hadoop.yarn.api.records.Container;
|
||||
|
@ -53,6 +52,8 @@ public class AppInfo {
|
|||
protected boolean amContainerLogsExist = false;
|
||||
@XmlTransient
|
||||
protected ApplicationId applicationId;
|
||||
@XmlTransient
|
||||
private String schemePrefix;
|
||||
|
||||
// these are ok for any user to see
|
||||
protected String id;
|
||||
|
@ -82,12 +83,8 @@ public class AppInfo {
|
|||
public AppInfo() {
|
||||
} // JAXB needs this
|
||||
|
||||
public AppInfo(RMApp app, Boolean hasAccess, String host) {
|
||||
this(app, hasAccess);
|
||||
}
|
||||
|
||||
public AppInfo(RMApp app, Boolean hasAccess) {
|
||||
|
||||
public AppInfo(RMApp app, Boolean hasAccess, String schemePrefix) {
|
||||
this.schemePrefix = schemePrefix;
|
||||
if (app != null) {
|
||||
String trackingUrl = app.getTrackingUrl();
|
||||
this.state = app.createApplicationState();
|
||||
|
@ -100,7 +97,7 @@ public class AppInfo {
|
|||
.getFinishTime() == 0 ? "ApplicationMaster" : "History");
|
||||
if (!trackingUrlIsNotReady) {
|
||||
this.trackingUrl =
|
||||
WebAppUtils.getURLWithScheme(HttpConfig.getSchemePrefix(),
|
||||
WebAppUtils.getURLWithScheme(schemePrefix,
|
||||
trackingUrl);
|
||||
this.trackingUrlPretty = this.trackingUrl;
|
||||
} else {
|
||||
|
@ -134,7 +131,7 @@ public class AppInfo {
|
|||
Container masterContainer = attempt.getMasterContainer();
|
||||
if (masterContainer != null) {
|
||||
this.amContainerLogsExist = true;
|
||||
String url = join(HttpConfig.getSchemePrefix(),
|
||||
String url = join(schemePrefix,
|
||||
masterContainer.getNodeHttpAddress(),
|
||||
"/node", "/containerlogs/",
|
||||
ConverterUtils.toString(masterContainer.getId()),
|
||||
|
|
|
@ -63,8 +63,10 @@ public class TestRMHA {
|
|||
@Before
|
||||
public void setUp() throws Exception {
|
||||
configuration.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
|
||||
configuration.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + RM2_NODE_ID);
|
||||
for (String confKey : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) {
|
||||
configuration.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + ","
|
||||
+ RM2_NODE_ID);
|
||||
for (String confKey : YarnConfiguration
|
||||
.getServiceAddressConfKeys(configuration)) {
|
||||
configuration.set(HAUtil.addSuffix(confKey, RM1_NODE_ID), RM1_ADDRESS);
|
||||
configuration.set(HAUtil.addSuffix(confKey, RM2_NODE_ID), RM2_ADDRESS);
|
||||
configuration.set(HAUtil.addSuffix(confKey, RM3_NODE_ID), RM3_ADDRESS);
|
||||
|
@ -329,7 +331,7 @@ public class TestRMHA {
|
|||
Configuration conf = new YarnConfiguration(configuration);
|
||||
rm = new MockRM(conf);
|
||||
rm.init(conf);
|
||||
for (String confKey : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) {
|
||||
for (String confKey : YarnConfiguration.getServiceAddressConfKeys(conf)) {
|
||||
assertEquals("RPC address not set for " + confKey,
|
||||
RM1_ADDRESS, conf.get(HAUtil.addSuffix(confKey, RM1_NODE_ID)));
|
||||
assertEquals("RPC address not set for " + confKey,
|
||||
|
|
|
@ -134,7 +134,9 @@ public class TestZKRMStateStore extends RMStateStoreTestBase {
|
|||
conf.set(YarnConfiguration.RM_ZK_ADDRESS, hostPort);
|
||||
conf.setInt(YarnConfiguration.RM_ZK_TIMEOUT_MS, ZK_TIMEOUT_MS);
|
||||
conf.set(YarnConfiguration.RM_HA_ID, rmId);
|
||||
for (String rpcAddress : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) {
|
||||
conf.set(YarnConfiguration.RM_WEBAPP_ADDRESS, "localhost:0");
|
||||
|
||||
for (String rpcAddress : YarnConfiguration.getServiceAddressConfKeys(conf)) {
|
||||
for (String id : HAUtil.getRMHAIds(conf)) {
|
||||
conf.set(HAUtil.addSuffix(rpcAddress, id), "localhost:0");
|
||||
}
|
||||
|
|
|
@ -285,13 +285,14 @@ public class TestRMAppAttemptTransitions {
|
|||
|
||||
private String getProxyUrl(RMAppAttempt appAttempt) {
|
||||
String url = null;
|
||||
final String scheme = WebAppUtils.getHttpSchemePrefix(conf);
|
||||
try {
|
||||
URI trackingUri =
|
||||
StringUtils.isEmpty(appAttempt.getOriginalTrackingUrl()) ? null :
|
||||
ProxyUriUtils
|
||||
.getUriFromAMUrl(appAttempt.getOriginalTrackingUrl());
|
||||
.getUriFromAMUrl(scheme, appAttempt.getOriginalTrackingUrl());
|
||||
String proxy = WebAppUtils.getProxyHostAndPort(conf);
|
||||
URI proxyUri = ProxyUriUtils.getUriFromAMUrl(proxy);
|
||||
URI proxyUri = ProxyUriUtils.getUriFromAMUrl(scheme, proxy);
|
||||
URI result = ProxyUriUtils.getProxyUri(trackingUri, proxyUri,
|
||||
appAttempt.getAppAttemptId().getApplicationId());
|
||||
url = result.toASCIIString();
|
||||
|
|
|
@ -105,7 +105,7 @@ public class TestRMContainerImpl {
|
|||
drainDispatcher.await();
|
||||
assertEquals(RMContainerState.RUNNING, rmContainer.getState());
|
||||
assertEquals(
|
||||
"http://host:3465/logs/host:3425/container_1_0001_01_000001/container_1_0001_01_000001/user",
|
||||
"//host:3465/logs/host:3425/container_1_0001_01_000001/container_1_0001_01_000001/user",
|
||||
rmContainer.getLogURL());
|
||||
|
||||
// In RUNNING state. Verify RELEASED and associated actions.
|
||||
|
@ -192,7 +192,7 @@ public class TestRMContainerImpl {
|
|||
drainDispatcher.await();
|
||||
assertEquals(RMContainerState.RUNNING, rmContainer.getState());
|
||||
assertEquals(
|
||||
"http://host:3465/logs/host:3425/container_1_0001_01_000001/container_1_0001_01_000001/user",
|
||||
"//host:3465/logs/host:3425/container_1_0001_01_000001/container_1_0001_01_000001/user",
|
||||
rmContainer.getLogURL());
|
||||
|
||||
// In RUNNING state. Verify EXPIRE and associated actions.
|
||||
|
|
|
@ -1606,8 +1606,7 @@ public class TestRMWebServicesApps extends JerseyTest {
|
|||
.getMasterContainer().getNodeHttpAddress(), nodeHttpAddress);
|
||||
WebServicesTestUtils.checkStringMatch("nodeId", appAttempt
|
||||
.getMasterContainer().getNodeId().toString(), nodeId);
|
||||
assertTrue("logsLink doesn't match",
|
||||
logsLink.startsWith("http://"));
|
||||
assertTrue("logsLink doesn't match", logsLink.startsWith("//"));
|
||||
assertTrue(
|
||||
"logsLink doesn't contain user info", logsLink.endsWith("/"
|
||||
+ user));
|
||||
|
|
|
@ -253,7 +253,7 @@ public class MiniYARNCluster extends CompositeService {
|
|||
|
||||
private void setHARMConfiguration(final int index, Configuration conf) {
|
||||
String hostname = MiniYARNCluster.getHostname();
|
||||
for (String confKey : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) {
|
||||
for (String confKey : YarnConfiguration.getServiceAddressConfKeys(conf)) {
|
||||
conf.set(HAUtil.addSuffix(confKey, rmIds[index]), hostname + ":0");
|
||||
}
|
||||
}
|
||||
|
|
|
@ -18,23 +18,17 @@
|
|||
|
||||
package org.apache.hadoop.yarn.server;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.ha.HAServiceProtocol;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest;
|
||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.AdminService;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
import static org.junit.Assert.assertFalse;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertFalse;
|
||||
import static org.junit.Assert.assertNotSame;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
import static org.junit.Assert.fail;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.ha.HAServiceProtocol;
|
||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
|
||||
public class TestMiniYARNClusterForHA {
|
||||
MiniYARNCluster cluster;
|
||||
|
@ -43,6 +37,8 @@ public class TestMiniYARNClusterForHA {
|
|||
public void setup() throws IOException, InterruptedException {
|
||||
Configuration conf = new YarnConfiguration();
|
||||
conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
|
||||
conf.set(YarnConfiguration.RM_WEBAPP_ADDRESS, "localhost:0");
|
||||
|
||||
cluster = new MiniYARNCluster(TestMiniYARNClusterForHA.class.getName(),
|
||||
2, 1, 1, 1);
|
||||
cluster.init(conf);
|
||||
|
|
|
@ -135,27 +135,6 @@ public class ProxyUriUtils {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a URI form a no scheme Url, such as is returned by the AM.
|
||||
* @param url the URL format returned by an AM. This may or may not contain
|
||||
* scheme.
|
||||
* @return a URI with an http scheme
|
||||
* @throws URISyntaxException if the url is not formatted correctly.
|
||||
*/
|
||||
public static URI getUriFromAMUrl(String url)
|
||||
throws URISyntaxException {
|
||||
if (getSchemeFromUrl(url).isEmpty()) {
|
||||
/*
|
||||
* check is made to make sure if AM reports with scheme then it will be
|
||||
* used by default otherwise it will default to the one configured using
|
||||
* "yarn.http.policy".
|
||||
*/
|
||||
return new URI(HttpConfig.getSchemePrefix() + url);
|
||||
} else {
|
||||
return new URI(url);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a URI form a no scheme Url, such as is returned by the AM.
|
||||
* @param noSchemeUrl the URL formate returned by an AM
|
||||
|
@ -170,7 +149,7 @@ public class ProxyUriUtils {
|
|||
* used by default otherwise it will default to the one configured using
|
||||
* "yarn.http.policy".
|
||||
*/
|
||||
return new URI(scheme + "://" + noSchemeUrl);
|
||||
return new URI(scheme + noSchemeUrl);
|
||||
} else {
|
||||
return new URI(noSchemeUrl);
|
||||
}
|
||||
|
|
|
@ -90,14 +90,22 @@ public class WebAppProxy extends AbstractService {
|
|||
@Override
|
||||
protected void serviceStart() throws Exception {
|
||||
try {
|
||||
proxyServer = new HttpServer2.Builder().setName("proxy")
|
||||
.addEndpoint(URI.create("http://" + bindAddress + ":" + port))
|
||||
.setFindPort(port == 0)
|
||||
.setConf(getConfig()).setACL(acl).build();
|
||||
proxyServer.addServlet(ProxyUriUtils.PROXY_SERVLET_NAME,
|
||||
Configuration conf = getConfig();
|
||||
HttpServer2.Builder b = new HttpServer2.Builder()
|
||||
.setName("proxy")
|
||||
.addEndpoint(
|
||||
URI.create(WebAppUtils.getHttpSchemePrefix(conf) + bindAddress
|
||||
+ ":" + port)).setFindPort(port == 0).setConf(getConfig())
|
||||
.setACL(acl);
|
||||
if (YarnConfiguration.useHttps(conf)) {
|
||||
WebAppUtils.loadSslConfiguration(b);
|
||||
}
|
||||
proxyServer = b.build();
|
||||
proxyServer.addServlet(ProxyUriUtils.PROXY_SERVLET_NAME,
|
||||
ProxyUriUtils.PROXY_PATH_SPEC, WebAppProxyServlet.class);
|
||||
proxyServer.setAttribute(FETCHER_ATTRIBUTE, fetcher);
|
||||
proxyServer.setAttribute(IS_SECURITY_ENABLED_ATTRIBUTE, isSecurityEnabled);
|
||||
proxyServer
|
||||
.setAttribute(IS_SECURITY_ENABLED_ATTRIBUTE, isSecurityEnabled);
|
||||
proxyServer.setAttribute(PROXY_HOST_ATTRIBUTE, proxyHost);
|
||||
proxyServer.start();
|
||||
} catch (IOException e) {
|
||||
|
|
|
@ -69,6 +69,7 @@ public class WebAppProxyServlet extends HttpServlet {
|
|||
|
||||
private final List<TrackingUriPlugin> trackingUriPlugins;
|
||||
private final String rmAppPageUrlBase;
|
||||
private final transient YarnConfiguration conf;
|
||||
|
||||
private static class _ implements Hamlet._ {
|
||||
//Empty
|
||||
|
@ -90,7 +91,7 @@ public class WebAppProxyServlet extends HttpServlet {
|
|||
public WebAppProxyServlet()
|
||||
{
|
||||
super();
|
||||
YarnConfiguration conf = new YarnConfiguration();
|
||||
conf = new YarnConfiguration();
|
||||
this.trackingUriPlugins =
|
||||
conf.getInstances(YarnConfiguration.YARN_TRACKING_URL_GENERATOR,
|
||||
TrackingUriPlugin.class);
|
||||
|
@ -300,7 +301,8 @@ public class WebAppProxyServlet extends HttpServlet {
|
|||
return;
|
||||
} else {
|
||||
if (ProxyUriUtils.getSchemeFromUrl(original).isEmpty()) {
|
||||
trackingUri = ProxyUriUtils.getUriFromAMUrl("http", original);
|
||||
trackingUri = ProxyUriUtils.getUriFromAMUrl(
|
||||
WebAppUtils.getHttpSchemePrefix(conf), original);
|
||||
} else {
|
||||
trackingUri = new URI(original);
|
||||
}
|
||||
|
|
|
@ -24,7 +24,6 @@ import java.util.Map;
|
|||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.http.FilterContainer;
|
||||
import org.apache.hadoop.http.FilterInitializer;
|
||||
import org.apache.hadoop.http.HttpConfig;
|
||||
import org.apache.hadoop.yarn.api.ApplicationConstants;
|
||||
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
|
||||
|
||||
|
@ -39,7 +38,7 @@ public class AmFilterInitializer extends FilterInitializer {
|
|||
String[] parts = proxy.split(":");
|
||||
params.put(AmIpFilter.PROXY_HOST, parts[0]);
|
||||
params.put(AmIpFilter.PROXY_URI_BASE,
|
||||
HttpConfig.getSchemePrefix() + proxy +
|
||||
WebAppUtils.getHttpSchemePrefix(conf) + proxy +
|
||||
System.getenv(ApplicationConstants.APPLICATION_WEB_PROXY_BASE_ENV));
|
||||
container.addFilter(FILTER_NAME, FILTER_CLASS, params);
|
||||
}
|
||||
|
|
|
@ -288,8 +288,9 @@ public class TestWebAppProxyServlet {
|
|||
YarnConfiguration.DEFAULT_YARN_ADMIN_ACL));
|
||||
proxyServer = new HttpServer2.Builder()
|
||||
.setName("proxy")
|
||||
.addEndpoint(URI.create("http://" + bindAddress + ":0"))
|
||||
.setFindPort(true)
|
||||
.addEndpoint(
|
||||
URI.create(WebAppUtils.getHttpSchemePrefix(conf) + bindAddress
|
||||
+ ":0")).setFindPort(true)
|
||||
.setConf(conf)
|
||||
.setACL(acl)
|
||||
.build();
|
||||
|
|
Loading…
Reference in New Issue