From 990cffdcfa9349fff0cee144b1d0e5267c40f63d Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Fri, 14 Feb 2014 20:01:02 +0000 Subject: [PATCH] YARN-1553. Modified YARN and MR to stop using HttpConfig.isSecure() and instead rely on the http policy framework. And also fix some bugs related to https handling in YARN web-apps. Contributed by Haohui Mai. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1568501 13f79535-47bb-0310-9956-ffa450edef68 --- .../fs/CommonConfigurationKeysPublic.java | 3 + .../org/apache/hadoop/http/HttpConfig.java | 25 -------- .../hadoop/mapreduce/v2/app/MRAppMaster.java | 3 +- .../v2/jobhistory/JHAdminConfig.java | 3 +- .../mapreduce/v2/util/MRWebAppUtil.java | 8 ++- .../mapreduce/jobhistory/HistoryViewer.java | 8 ++- .../hadoop/mapreduce/util/HostUtil.java | 5 +- .../mapreduce/v2/hs/JobHistoryServer.java | 2 - .../mapreduce/v2/MiniMRYarnCluster.java | 6 +- hadoop-yarn-project/CHANGES.txt | 4 ++ .../org/apache/hadoop/yarn/conf/HAUtil.java | 8 +-- .../hadoop/yarn/conf/YarnConfiguration.java | 44 ++++++++++---- .../client/api/impl/TimelineClientImpl.java | 18 +++--- .../apache/hadoop/yarn/webapp/WebApps.java | 15 ++++- .../hadoop/yarn/webapp/util/WebAppUtils.java | 59 +++++++++++++------ .../apache/hadoop/yarn/conf/TestHAUtil.java | 10 ++-- .../hadoop/yarn/server/webapp/AppBlock.java | 2 +- .../yarn/server/nodemanager/NodeManager.java | 9 --- .../server/nodemanager/webapp/NavBlock.java | 1 - .../resourcemanager/ResourceManager.java | 7 --- .../rmapp/attempt/RMAppAttemptImpl.java | 5 +- .../resourcemanager/webapp/AppBlock.java | 12 ++-- .../resourcemanager/webapp/AppsBlock.java | 8 ++- .../webapp/FairSchedulerAppsBlock.java | 8 ++- .../resourcemanager/webapp/NodesPage.java | 3 +- .../resourcemanager/webapp/RMWebServices.java | 13 ++-- .../webapp/dao/AppAttemptInfo.java | 3 +- .../resourcemanager/webapp/dao/AppInfo.java | 15 ++--- .../yarn/server/resourcemanager/TestRMHA.java | 8 ++- .../recovery/TestZKRMStateStore.java | 4 +- .../attempt/TestRMAppAttemptTransitions.java | 5 +- .../rmcontainer/TestRMContainerImpl.java | 4 +- .../webapp/TestRMWebServicesApps.java | 3 +- .../hadoop/yarn/server/MiniYARNCluster.java | 2 +- .../yarn/server/TestMiniYARNClusterForHA.java | 24 ++++---- .../yarn/server/webproxy/ProxyUriUtils.java | 23 +------- .../yarn/server/webproxy/WebAppProxy.java | 20 +++++-- .../server/webproxy/WebAppProxyServlet.java | 6 +- .../amfilter/AmFilterInitializer.java | 3 +- .../webproxy/TestWebAppProxyServlet.java | 5 +- 40 files changed, 218 insertions(+), 196 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java index 87746bbe396..f5c814d7c1d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java @@ -279,7 +279,10 @@ public class CommonConfigurationKeysPublic { 60; // HTTP policies to be used in configuration + // Use HttpPolicy.name() instead + @Deprecated public static final String HTTP_POLICY_HTTP_ONLY = "HTTP_ONLY"; + @Deprecated public static final String HTTP_POLICY_HTTPS_ONLY = "HTTPS_ONLY"; } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpConfig.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpConfig.java index d323f764359..15008addc8f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpConfig.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpConfig.java @@ -28,7 +28,6 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic; @InterfaceAudience.Private @InterfaceStability.Unstable public class HttpConfig { - private static Policy policy; public enum Policy { HTTP_ONLY, HTTPS_ONLY, @@ -52,28 +51,4 @@ public class HttpConfig { return this == HTTPS_ONLY || this == HTTP_AND_HTTPS; } } - - static { - Configuration conf = new Configuration(); - boolean sslEnabled = conf.getBoolean( - CommonConfigurationKeysPublic.HADOOP_SSL_ENABLED_KEY, - CommonConfigurationKeysPublic.HADOOP_SSL_ENABLED_DEFAULT); - policy = sslEnabled ? Policy.HTTPS_ONLY : Policy.HTTP_ONLY; - } - - public static void setPolicy(Policy policy) { - HttpConfig.policy = policy; - } - - public static boolean isSecure() { - return policy == Policy.HTTPS_ONLY; - } - - public static String getSchemePrefix() { - return (isSecure()) ? "https://" : "http://"; - } - - public static String getScheme(Policy policy) { - return policy == Policy.HTTPS_ONLY ? "https://" : "http://"; - } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java index 969e2fa7a13..ce1e4487d0e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java @@ -1387,7 +1387,8 @@ public class MRAppMaster extends CompositeService { // RM/NM to issue SSL certificates but definitely not MR-AM as it is // running in user-land. MRWebAppUtil.initialize(conf); - HttpConfig.setPolicy(HttpConfig.Policy.HTTP_ONLY); + conf.set(YarnConfiguration.YARN_HTTP_POLICY_KEY, + HttpConfig.Policy.HTTP_ONLY.name()); // log the system properties String systemPropsToLog = MRApps.getSystemPropertiesToLog(conf); if (systemPropsToLog != null) { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java index 2cc233688b8..2e1a22e4310 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java @@ -21,6 +21,7 @@ package org.apache.hadoop.mapreduce.v2.jobhistory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.http.HttpConfig; /** * Stores Job History configuration keys that can be set by administrators of @@ -135,7 +136,7 @@ public class JHAdminConfig { public static final String MR_HS_HTTP_POLICY = MR_HISTORY_PREFIX + "http.policy"; public static String DEFAULT_MR_HS_HTTP_POLICY = - CommonConfigurationKeysPublic.HTTP_POLICY_HTTP_ONLY; + HttpConfig.Policy.HTTP_ONLY.name(); /**The address the history server webapp is on.*/ public static final String MR_HISTORY_WEBAPP_ADDRESS = diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRWebAppUtil.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRWebAppUtil.java index 49a0407d0eb..2d453f1d308 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRWebAppUtil.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRWebAppUtil.java @@ -71,11 +71,13 @@ public class MRWebAppUtil { } public static String getYARNWebappScheme() { - return HttpConfig.getScheme(httpPolicyInYarn); + return httpPolicyInYarn == HttpConfig.Policy.HTTPS_ONLY ? "https://" + : "http://"; } - + public static String getJHSWebappScheme() { - return HttpConfig.getScheme(httpPolicyInJHS); + return httpPolicyInJHS == HttpConfig.Policy.HTTPS_ONLY ? "https://" + : "http://"; } public static void setJHSWebappURLWithoutScheme(Configuration conf, diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java index 92490a59e0a..eaeadea6ff4 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java @@ -45,6 +45,7 @@ import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo; import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskInfo; import org.apache.hadoop.mapreduce.util.HostUtil; import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.yarn.webapp.util.WebAppUtils; /** * HistoryViewer is used to parse and view the JobHistory files @@ -231,7 +232,8 @@ public class HistoryViewer { taskList.append("\t"); taskList.append(attempt.getHostname()).append("\t"); taskList.append(attempt.getError()); - String taskLogsUrl = getTaskLogsUrl(attempt); + String taskLogsUrl = getTaskLogsUrl( + WebAppUtils.getHttpSchemePrefix(fs.getConf()), attempt); taskList.append(taskLogsUrl != null ? taskLogsUrl : "n/a"); System.out.println(taskList.toString()); } @@ -446,7 +448,7 @@ public class HistoryViewer { * @return the taskLogsUrl. null if http-port or tracker-name or * task-attempt-id are unavailable. */ - public static String getTaskLogsUrl( + public static String getTaskLogsUrl(String scheme, JobHistoryParser.TaskAttemptInfo attempt) { if (attempt.getHttpPort() == -1 || attempt.getTrackerName().equals("") @@ -457,7 +459,7 @@ public class HistoryViewer { String taskTrackerName = HostUtil.convertTrackerNameToHostName( attempt.getTrackerName()); - return HostUtil.getTaskLogUrl(taskTrackerName, + return HostUtil.getTaskLogUrl(scheme, taskTrackerName, Integer.toString(attempt.getHttpPort()), attempt.getAttemptId().toString()); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/HostUtil.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/HostUtil.java index 0a42bb73a20..e131fc8933a 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/HostUtil.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/HostUtil.java @@ -20,7 +20,6 @@ package org.apache.hadoop.mapreduce.util; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.http.HttpConfig; @Private @Unstable @@ -33,9 +32,9 @@ public class HostUtil { * @param taskAttemptID * @return the taskLogUrl */ - public static String getTaskLogUrl(String taskTrackerHostName, + public static String getTaskLogUrl(String scheme, String taskTrackerHostName, String httpPort, String taskAttemptID) { - return (HttpConfig.getSchemePrefix() + taskTrackerHostName + ":" + + return (scheme + taskTrackerHostName + ":" + httpPort + "/tasklog?attemptid=" + taskAttemptID); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java index 9c92bed6acb..1373f3cdc23 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java @@ -24,7 +24,6 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapreduce.MRConfig; import org.apache.hadoop.mapreduce.v2.hs.HistoryServerStateStoreService.HistoryServerState; @@ -121,7 +120,6 @@ public class JobHistoryServer extends CompositeService { // This is required for WebApps to use https if enabled. MRWebAppUtil.initialize(getConfig()); - HttpConfig.setPolicy(MRWebAppUtil.getJHSHttpPolicy()); try { doSecureLogin(conf); } catch(IOException ie) { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java index 01047043380..00109c5d8e2 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java @@ -230,9 +230,9 @@ public class MiniMRYarnCluster extends MiniYARNCluster { WebAppUtils.getRMWebAppURLWithoutScheme(getConfig())); LOG.info("MiniMRYARN HistoryServer address: " + getConfig().get(JHAdminConfig.MR_HISTORY_ADDRESS)); - LOG.info("MiniMRYARN HistoryServer web address: " + - getResolvedMRHistoryWebAppURLWithoutScheme(getConfig(), - HttpConfig.isSecure())); + LOG.info("MiniMRYARN HistoryServer web address: " + + getResolvedMRHistoryWebAppURLWithoutScheme(getConfig(), + MRWebAppUtil.getJHSHttpPolicy() == HttpConfig.Policy.HTTPS_ONLY)); } @Override diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 8b880d33f91..071fce490c8 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -273,6 +273,10 @@ Release 2.4.0 - UNRELEASED at allocation time so as to prevent RM from shelling out containers with expired tokens. (Omkar Vinit Joshi and Jian He via vinodkv) + YARN-1553. Modified YARN and MR to stop using HttpConfig.isSecure() and + instead rely on the http policy framework. And also fix some bugs related + to https handling in YARN web-apps. (Haohui Mai via vinodkv) + Release 2.3.1 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java index b5a0b1a2077..e3114ceae1e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java @@ -100,7 +100,7 @@ public class HAUtil { StringBuilder setValue = new StringBuilder(); for (String id: ids) { // verify the RM service addresses configurations for every RMIds - for (String prefix : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) { + for (String prefix : YarnConfiguration.getServiceAddressConfKeys(conf)) { checkAndSetRMRPCAddress(prefix, id, conf); } setValue.append(id); @@ -158,7 +158,7 @@ public class HAUtil { } public static void verifyAndSetAllServiceAddresses(Configuration conf) { - for (String confKey : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) { + for (String confKey : YarnConfiguration.getServiceAddressConfKeys(conf)) { verifyAndSetConfValue(confKey, conf); } } @@ -236,7 +236,7 @@ public class HAUtil { @InterfaceAudience.Private @VisibleForTesting static String getConfKeyForRMInstance(String prefix, Configuration conf) { - if (!YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS.contains(prefix)) { + if (!YarnConfiguration.getServiceAddressConfKeys(conf).contains(prefix)) { return prefix; } else { String RMId = getRMHAId(conf); @@ -289,7 +289,7 @@ public class HAUtil { hostNameConfKey + " or " + addSuffix(prefix, RMId))); } else { conf.set(addSuffix(prefix, RMId), confVal + ":" - + YarnConfiguration.getRMDefaultPortNumber(prefix)); + + YarnConfiguration.getRMDefaultPortNumber(prefix, conf)); } } } catch (IllegalArgumentException iae) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 8c8ad16e8e4..9612cac15e5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -26,10 +26,8 @@ import java.util.List; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Public; -import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.util.StringUtils; @@ -187,6 +185,8 @@ public class YarnConfiguration extends Configuration { /** The https address of the RM web application.*/ public static final String RM_WEBAPP_HTTPS_ADDRESS = RM_PREFIX + "webapp.https.address"; + public static final boolean YARN_SSL_CLIENT_HTTPS_NEED_AUTH_DEFAULT = false; + public static final String YARN_SSL_SERVER_RESOURCE_DEFAULT = "ssl-server.xml"; public static final int DEFAULT_RM_WEBAPP_HTTPS_PORT = 8090; public static final String DEFAULT_RM_WEBAPP_HTTPS_ADDRESS = "0.0.0.0:" @@ -361,15 +361,21 @@ public class YarnConfiguration extends Configuration { public static final String DEFAULT_RM_CONFIGURATION_PROVIDER_CLASS = "org.apache.hadoop.yarn.LocalConfigurationProvider"; - @Private - public static final List RM_SERVICES_ADDRESS_CONF_KEYS = + private static final List RM_SERVICES_ADDRESS_CONF_KEYS_HTTP = Collections.unmodifiableList(Arrays.asList( RM_ADDRESS, RM_SCHEDULER_ADDRESS, RM_ADMIN_ADDRESS, RM_RESOURCE_TRACKER_ADDRESS, - HttpConfig.isSecure() ? RM_WEBAPP_HTTPS_ADDRESS - : RM_WEBAPP_ADDRESS)); + RM_WEBAPP_ADDRESS)); + + private static final List RM_SERVICES_ADDRESS_CONF_KEYS_HTTPS = + Collections.unmodifiableList(Arrays.asList( + RM_ADDRESS, + RM_SCHEDULER_ADDRESS, + RM_ADMIN_ADDRESS, + RM_RESOURCE_TRACKER_ADDRESS, + RM_WEBAPP_HTTPS_ADDRESS)); public static final String AUTO_FAILOVER_PREFIX = RM_HA_PREFIX + "automatic-failover."; @@ -1102,10 +1108,9 @@ public class YarnConfiguration extends Configuration { YARN_PREFIX + "client.max-nodemanagers-proxies"; public static final int DEFAULT_NM_CLIENT_MAX_NM_PROXIES = 500; - public static final String YARN_HTTP_POLICY_KEY = - YARN_PREFIX + "http.policy"; - public static final String YARN_HTTP_POLICY_DEFAULT = - CommonConfigurationKeysPublic.HTTP_POLICY_HTTP_ONLY; + public static final String YARN_HTTP_POLICY_KEY = YARN_PREFIX + "http.policy"; + public static final String YARN_HTTP_POLICY_DEFAULT = HttpConfig.Policy.HTTP_ONLY + .name(); public YarnConfiguration() { super(); @@ -1118,6 +1123,12 @@ public class YarnConfiguration extends Configuration { } } + @Private + public static List getServiceAddressConfKeys(Configuration conf) { + return useHttps(conf) ? RM_SERVICES_ADDRESS_CONF_KEYS_HTTPS + : RM_SERVICES_ADDRESS_CONF_KEYS_HTTP; + } + /** * Get the socket address for name property as a * InetSocketAddress. @@ -1130,7 +1141,7 @@ public class YarnConfiguration extends Configuration { public InetSocketAddress getSocketAddr( String name, String defaultAddress, int defaultPort) { String address; - if (HAUtil.isHAEnabled(this) && RM_SERVICES_ADDRESS_CONF_KEYS.contains(name)) { + if (HAUtil.isHAEnabled(this) && getServiceAddressConfKeys(this).contains(name)) { address = HAUtil.getConfValueForRMInstance(name, defaultAddress, this); } else { address = get(name, defaultAddress); @@ -1149,7 +1160,8 @@ public class YarnConfiguration extends Configuration { } @Private - public static int getRMDefaultPortNumber(String addressPrefix) { + public static int getRMDefaultPortNumber(String addressPrefix, + Configuration conf) { if (addressPrefix.equals(YarnConfiguration.RM_ADDRESS)) { return YarnConfiguration.DEFAULT_RM_PORT; } else if (addressPrefix.equals(YarnConfiguration.RM_SCHEDULER_ADDRESS)) { @@ -1167,7 +1179,13 @@ public class YarnConfiguration extends Configuration { throw new HadoopIllegalArgumentException( "Invalid RM RPC address Prefix: " + addressPrefix + ". The valid value should be one of " - + YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS); + + getServiceAddressConfKeys(conf)); } } + + public static boolean useHttps(Configuration conf) { + return HttpConfig.Policy.HTTPS_ONLY == HttpConfig.Policy.fromString(conf + .get(YARN_HTTP_POLICY_KEY, + YARN_HTTP_POLICY_DEFAULT)); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java index 9fcc2bd6e3d..3269b8b0131 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java @@ -29,7 +29,6 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.yarn.api.records.apptimeline.ATSEntities; import org.apache.hadoop.yarn.api.records.apptimeline.ATSEntity; import org.apache.hadoop.yarn.api.records.apptimeline.ATSPutErrors; @@ -65,12 +64,17 @@ public class TimelineClientImpl extends TimelineClient { } protected void serviceInit(Configuration conf) throws Exception { - resURI = new URI(JOINER.join(HttpConfig.getSchemePrefix(), - HttpConfig.isSecure() ? conf.get( - YarnConfiguration.AHS_WEBAPP_HTTPS_ADDRESS, - YarnConfiguration.DEFAULT_AHS_WEBAPP_HTTPS_ADDRESS) : conf.get( - YarnConfiguration.AHS_WEBAPP_ADDRESS, - YarnConfiguration.DEFAULT_AHS_WEBAPP_ADDRESS), RESOURCE_URI_STR)); + if (YarnConfiguration.useHttps(conf)) { + resURI = URI + .create(JOINER.join("https://", conf.get( + YarnConfiguration.AHS_WEBAPP_HTTPS_ADDRESS, + YarnConfiguration.DEFAULT_AHS_WEBAPP_HTTPS_ADDRESS), + RESOURCE_URI_STR)); + } else { + resURI = URI.create(JOINER.join("http://", conf.get( + YarnConfiguration.AHS_WEBAPP_ADDRESS, + YarnConfiguration.DEFAULT_AHS_WEBAPP_ADDRESS), RESOURCE_URI_STR)); + } super.serviceInit(conf); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java index 317baec4a31..590cd962f12 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java @@ -37,7 +37,9 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.http.HttpServer2; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.security.AdminACLsManager; +import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -216,9 +218,11 @@ public class WebApps { System.exit(1); } } - HttpServer2.Builder builder = new HttpServer2.Builder().setName(name) - .addEndpoint(URI.create("http://" + bindAddress + ":" + port)) - .setConf(conf).setFindPort(findPort) + HttpServer2.Builder builder = new HttpServer2.Builder() + .setName(name) + .addEndpoint( + URI.create(WebAppUtils.getHttpSchemePrefix(conf) + bindAddress + + ":" + port)).setConf(conf).setFindPort(findPort) .setACL(new AdminACLsManager(conf).getAdminAcl()) .setPathSpec(pathList.toArray(new String[0])); @@ -231,6 +235,11 @@ public class WebApps { .setKeytabConfKey(spnegoKeytabKey) .setSecurityEnabled(UserGroupInformation.isSecurityEnabled()); } + + if (YarnConfiguration.useHttps(conf)) { + WebAppUtils.loadSslConfiguration(builder); + } + HttpServer2 server = builder.build(); for(ServletStruct struct: servlets) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java index 4a288c44af9..a7dce0732c0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java @@ -26,20 +26,16 @@ import java.net.UnknownHostException; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.http.HttpConfig.Policy; +import org.apache.hadoop.http.HttpServer2; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.util.ConverterUtils; -import com.google.common.base.Joiner; - @Private @Evolving public class WebAppUtils { - private static final Joiner JOINER = Joiner.on(""); - public static void setRMWebAppPort(Configuration conf, int port) { String hostname = getRMWebAppURLWithoutScheme(conf); hostname = @@ -51,7 +47,7 @@ public class WebAppUtils { public static void setRMWebAppHostnameAndPort(Configuration conf, String hostname, int port) { String resolvedAddress = hostname + ":" + port; - if (HttpConfig.isSecure()) { + if (YarnConfiguration.useHttps(conf)) { conf.set(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS, resolvedAddress); } else { conf.set(YarnConfiguration.RM_WEBAPP_ADDRESS, resolvedAddress); @@ -60,7 +56,7 @@ public class WebAppUtils { public static void setNMWebAppHostNameAndPort(Configuration conf, String hostName, int port) { - if (HttpConfig.isSecure()) { + if (YarnConfiguration.useHttps(conf)) { conf.set(YarnConfiguration.NM_WEBAPP_HTTPS_ADDRESS, hostName + ":" + port); } else { @@ -70,16 +66,11 @@ public class WebAppUtils { } public static String getRMWebAppURLWithScheme(Configuration conf) { - return JOINER.join(HttpConfig.getSchemePrefix(), - HttpConfig.isSecure() ? conf.get( - YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS, - YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_ADDRESS) : conf.get( - YarnConfiguration.RM_WEBAPP_ADDRESS, - YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS)); + return getHttpSchemePrefix(conf) + getRMWebAppURLWithoutScheme(conf); } public static String getRMWebAppURLWithoutScheme(Configuration conf) { - if (HttpConfig.isSecure()) { + if (YarnConfiguration.useHttps(conf)) { return conf.get(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS, YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_ADDRESS); }else { @@ -97,13 +88,13 @@ public class WebAppUtils { } public static String getResolvedRMWebAppURLWithScheme(Configuration conf) { - return HttpConfig.getSchemePrefix() + return getHttpSchemePrefix(conf) + getResolvedRMWebAppURLWithoutScheme(conf); } public static String getResolvedRMWebAppURLWithoutScheme(Configuration conf) { return getResolvedRMWebAppURLWithoutScheme(conf, - HttpConfig.isSecure() ? Policy.HTTPS_ONLY : Policy.HTTP_ONLY); + YarnConfiguration.useHttps(conf) ? Policy.HTTPS_ONLY : Policy.HTTP_ONLY); } public static String getResolvedRMWebAppURLWithoutScheme(Configuration conf, @@ -140,7 +131,7 @@ public class WebAppUtils { } public static String getNMWebAppURLWithoutScheme(Configuration conf) { - if (HttpConfig.isSecure()) { + if (YarnConfiguration.useHttps(conf)) { return conf.get(YarnConfiguration.NM_WEBAPP_HTTPS_ADDRESS, YarnConfiguration.DEFAULT_NM_WEBAPP_HTTPS_ADDRESS); } else { @@ -150,7 +141,7 @@ public class WebAppUtils { } public static String getAHSWebAppURLWithoutScheme(Configuration conf) { - if (HttpConfig.isSecure()) { + if (YarnConfiguration.useHttps(conf)) { return conf.get(YarnConfiguration.AHS_WEBAPP_HTTPS_ADDRESS, YarnConfiguration.DEFAULT_AHS_WEBAPP_HTTPS_ADDRESS); } else { @@ -177,8 +168,38 @@ public class WebAppUtils { public static String getLogUrl(String nodeHttpAddress, String allocatedNode, ContainerId containerId, String user) { - return join(HttpConfig.getSchemePrefix(), nodeHttpAddress, "/logs", "/", + return join("//", nodeHttpAddress, "/logs", "/", allocatedNode, "/", ConverterUtils.toString(containerId), "/", ConverterUtils.toString(containerId), "/", user); } + + /** + * Choose which scheme (HTTP or HTTPS) to use when generating a URL based on + * the configuration. + * + * @return the schmeme (HTTP / HTTPS) + */ + public static String getHttpSchemePrefix(Configuration conf) { + return YarnConfiguration.useHttps(conf) ? "https://" : "http://"; + } + + /** + * Load the SSL keystore / truststore into the HttpServer builder. + */ + public static HttpServer2.Builder loadSslConfiguration( + HttpServer2.Builder builder) { + Configuration sslConf = new Configuration(false); + boolean needsClientAuth = YarnConfiguration.YARN_SSL_CLIENT_HTTPS_NEED_AUTH_DEFAULT; + sslConf.addResource(YarnConfiguration.YARN_SSL_SERVER_RESOURCE_DEFAULT); + + return builder + .needsClientAuth(needsClientAuth) + .keyPassword(sslConf.get("ssl.server.keystore.keypassword")) + .keyStore(sslConf.get("ssl.server.keystore.location"), + sslConf.get("ssl.server.keystore.password"), + sslConf.get("ssl.server.keystore.type", "jks")) + .trustStore(sslConf.get("ssl.server.truststore.location"), + sslConf.get("ssl.server.truststore.password"), + sslConf.get("ssl.server.truststore.type", "jks")); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java index 891b434262f..6ced5f26326 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java @@ -54,7 +54,7 @@ public class TestHAUtil { conf.set(YarnConfiguration.RM_HA_IDS, RM_NODE_IDS_UNTRIMMED); conf.set(YarnConfiguration.RM_HA_ID, RM1_NODE_ID_UNTRIMMED); - for (String confKey : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) { + for (String confKey : YarnConfiguration.getServiceAddressConfKeys(conf)) { // configuration key itself cannot contains space/tab/return chars. conf.set(HAUtil.addSuffix(confKey, RM1_NODE_ID), RM1_ADDRESS_UNTRIMMED); conf.set(HAUtil.addSuffix(confKey, RM2_NODE_ID), RM2_ADDRESS); @@ -95,7 +95,7 @@ public class TestHAUtil { StringUtils.getStringCollection(RM_NODE_IDS), HAUtil.getRMHAIds(conf)); assertEquals("Should be saved as Trimmed string", RM1_NODE_ID, HAUtil.getRMHAId(conf)); - for (String confKey : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) { + for (String confKey : YarnConfiguration.getServiceAddressConfKeys(conf)) { assertEquals("RPC address not set for " + confKey, RM1_ADDRESS, conf.get(confKey)); } @@ -117,7 +117,7 @@ public class TestHAUtil { // simulate the case YarnConfiguration.RM_HA_ID is not set conf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + RM2_NODE_ID); - for (String confKey : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) { + for (String confKey : YarnConfiguration.getServiceAddressConfKeys(conf)) { conf.set(HAUtil.addSuffix(confKey, RM1_NODE_ID), RM1_ADDRESS); conf.set(HAUtil.addSuffix(confKey, RM2_NODE_ID), RM2_ADDRESS); } @@ -134,7 +134,7 @@ public class TestHAUtil { conf.set(YarnConfiguration.RM_HA_ID, RM_INVALID_NODE_ID); conf.set(YarnConfiguration.RM_HA_IDS, RM_INVALID_NODE_ID + "," + RM1_NODE_ID); - for (String confKey : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) { + for (String confKey : YarnConfiguration.getServiceAddressConfKeys(conf)) { // simulate xml with invalid node id conf.set(confKey + RM_INVALID_NODE_ID, RM_INVALID_NODE_ID); } @@ -169,7 +169,7 @@ public class TestHAUtil { conf.clear(); conf.set(YarnConfiguration.RM_HA_IDS, RM2_NODE_ID + "," + RM3_NODE_ID); conf.set(YarnConfiguration.RM_HA_ID, RM1_NODE_ID_UNTRIMMED); - for (String confKey : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) { + for (String confKey : YarnConfiguration.getServiceAddressConfKeys(conf)) { conf.set(HAUtil.addSuffix(confKey, RM1_NODE_ID), RM1_ADDRESS_UNTRIMMED); conf.set(HAUtil.addSuffix(confKey, RM2_NODE_ID), RM2_ADDRESS); conf.set(HAUtil.addSuffix(confKey, RM3_NODE_ID), RM3_ADDRESS); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java index 9a15fe78369..09bb95816ff 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java @@ -163,7 +163,7 @@ public class AppBlock extends HtmlBlock { .append(startTime) .append("\",\"") .append( nodeLink == null ? "N/A" : StringEscapeUtils diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java index a169c125a38..d9a540815b2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java @@ -28,8 +28,6 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.http.HttpConfig; -import org.apache.hadoop.http.HttpConfig.Policy; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.service.CompositeService; @@ -397,15 +395,8 @@ public class NodeManager extends CompositeService StringUtils.startupShutdownMessage(NodeManager.class, args, LOG); NodeManager nodeManager = new NodeManager(); Configuration conf = new YarnConfiguration(); - setHttpPolicy(conf); nodeManager.initAndStartNodeManager(conf, false); } - - private static void setHttpPolicy(Configuration conf) { - HttpConfig.setPolicy(Policy.fromString(conf.get( - YarnConfiguration.YARN_HTTP_POLICY_KEY, - YarnConfiguration.YARN_HTTP_POLICY_DEFAULT))); - } @VisibleForTesting @Private diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NavBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NavBlock.java index 424da04d2be..c198ae6b0fe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NavBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NavBlock.java @@ -19,7 +19,6 @@ package org.apache.hadoop.yarn.server.nodemanager.webapp; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.yarn.webapp.YarnWebParams; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.apache.hadoop.yarn.webapp.view.HtmlBlock; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index 5ef58a74d8e..3aa11c55296 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -1015,7 +1015,6 @@ public class ResourceManager extends CompositeService implements Recoverable { ShutdownHookManager.get().addShutdownHook( new CompositeServiceShutdownHook(resourceManager), SHUTDOWN_HOOK_PRIORITY); - setHttpPolicy(conf); resourceManager.init(conf); resourceManager.start(); } catch (Throwable t) { @@ -1023,12 +1022,6 @@ public class ResourceManager extends CompositeService implements Recoverable { System.exit(-1); } } - - private static void setHttpPolicy(Configuration conf) { - HttpConfig.setPolicy(Policy.fromString(conf.get( - YarnConfiguration.YARN_HTTP_POLICY_KEY, - YarnConfiguration.YARN_HTTP_POLICY_DEFAULT))); - } /** * Register the handlers for alwaysOn services diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 4ca8c28243a..88c9ba5591c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -503,10 +503,11 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { final String trackingUriWithoutScheme) { this.readLock.lock(); try { + final String scheme = WebAppUtils.getHttpSchemePrefix(conf); URI trackingUri = StringUtils.isEmpty(trackingUriWithoutScheme) ? null : - ProxyUriUtils.getUriFromAMUrl(trackingUriWithoutScheme); + ProxyUriUtils.getUriFromAMUrl(scheme, trackingUriWithoutScheme); String proxy = WebAppUtils.getProxyHostAndPort(conf); - URI proxyUri = ProxyUriUtils.getUriFromAMUrl(proxy); + URI proxyUri = ProxyUriUtils.getUriFromAMUrl(scheme, proxy); URI result = ProxyUriUtils.getProxyUri(trackingUri, proxyUri, applicationAttemptId.getApplicationId()); return result.toASCIIString(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java index 445a5a2ca99..ac8578ec65c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java @@ -27,7 +27,7 @@ import static org.apache.hadoop.yarn.webapp.view.JQueryUI._TH; import java.util.Collection; -import org.apache.hadoop.http.HttpConfig; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; @@ -46,6 +46,7 @@ import org.apache.hadoop.yarn.util.Times; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE; +import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.apache.hadoop.yarn.webapp.view.HtmlBlock; import org.apache.hadoop.yarn.webapp.view.InfoBlock; @@ -55,13 +56,16 @@ public class AppBlock extends HtmlBlock { private ApplicationACLsManager aclsManager; private QueueACLsManager queueACLsManager; + private final Configuration conf; @Inject AppBlock(ResourceManager rm, ViewContext ctx, - ApplicationACLsManager aclsManager, QueueACLsManager queueACLsManager) { + ApplicationACLsManager aclsManager, QueueACLsManager queueACLsManager, + Configuration conf) { super(ctx); this.aclsManager = aclsManager; this.queueACLsManager = queueACLsManager; + this.conf = conf; } @Override @@ -86,7 +90,7 @@ public class AppBlock extends HtmlBlock { puts("Application not found: "+ aid); return; } - AppInfo app = new AppInfo(rmApp, true); + AppInfo app = new AppInfo(rmApp, true, WebAppUtils.getHttpSchemePrefix(conf)); // Check for the authorization. String remoteUser = request().getRemoteUser(); @@ -146,7 +150,7 @@ public class AppBlock extends HtmlBlock { table.tr((odd = !odd) ? _ODD : _EVEN). td(String.valueOf(attemptInfo.getAttemptId())). td(Times.format(attemptInfo.getStartTime())). - td().a(".nodelink", url(HttpConfig.getSchemePrefix(), + td().a(".nodelink", url("//", attemptInfo.getNodeHttpAddress()), attemptInfo.getNodeHttpAddress())._(). td().a(".logslink", url(attemptInfo.getLogsLink()), "logs")._(). diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlock.java index cec95ac2288..4f644d1a39d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlock.java @@ -28,6 +28,7 @@ import java.util.HashSet; import java.util.concurrent.ConcurrentMap; import org.apache.commons.lang.StringEscapeUtils; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; @@ -36,16 +37,19 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY; +import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.apache.hadoop.yarn.webapp.view.HtmlBlock; import com.google.inject.Inject; class AppsBlock extends HtmlBlock { final ConcurrentMap apps; + private final Configuration conf; -@Inject AppsBlock(RMContext rmContext, ViewContext ctx) { +@Inject AppsBlock(RMContext rmContext, ViewContext ctx, Configuration conf) { super(ctx); apps = rmContext.getRMApps(); + this.conf = conf; } @Override public void render(Block html) { @@ -79,7 +83,7 @@ class AppsBlock extends HtmlBlock { if (reqAppStates != null && !reqAppStates.contains(app.createApplicationState())) { continue; } - AppInfo appInfo = new AppInfo(app, true); + AppInfo appInfo = new AppInfo(app, true, WebAppUtils.getHttpSchemePrefix(conf)); String percent = String.format("%.1f", appInfo.getProgress()); //AppID numerical value parsed by parseHadoopID in yarn.dt.plugins.js appsTableData.append("[\" trackingUriPlugins; private final String rmAppPageUrlBase; + private final transient YarnConfiguration conf; private static class _ implements Hamlet._ { //Empty @@ -90,7 +91,7 @@ public class WebAppProxyServlet extends HttpServlet { public WebAppProxyServlet() { super(); - YarnConfiguration conf = new YarnConfiguration(); + conf = new YarnConfiguration(); this.trackingUriPlugins = conf.getInstances(YarnConfiguration.YARN_TRACKING_URL_GENERATOR, TrackingUriPlugin.class); @@ -300,7 +301,8 @@ public class WebAppProxyServlet extends HttpServlet { return; } else { if (ProxyUriUtils.getSchemeFromUrl(original).isEmpty()) { - trackingUri = ProxyUriUtils.getUriFromAMUrl("http", original); + trackingUri = ProxyUriUtils.getUriFromAMUrl( + WebAppUtils.getHttpSchemePrefix(conf), original); } else { trackingUri = new URI(original); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java index 7f81f9ba785..e35ed8410c8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java @@ -24,7 +24,6 @@ import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.http.FilterContainer; import org.apache.hadoop.http.FilterInitializer; -import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; @@ -39,7 +38,7 @@ public class AmFilterInitializer extends FilterInitializer { String[] parts = proxy.split(":"); params.put(AmIpFilter.PROXY_HOST, parts[0]); params.put(AmIpFilter.PROXY_URI_BASE, - HttpConfig.getSchemePrefix() + proxy + + WebAppUtils.getHttpSchemePrefix(conf) + proxy + System.getenv(ApplicationConstants.APPLICATION_WEB_PROXY_BASE_ENV)); container.addFilter(FILTER_NAME, FILTER_CLASS, params); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java index f39ab3ecd10..c53d098f4b7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java @@ -288,8 +288,9 @@ public class TestWebAppProxyServlet { YarnConfiguration.DEFAULT_YARN_ADMIN_ACL)); proxyServer = new HttpServer2.Builder() .setName("proxy") - .addEndpoint(URI.create("http://" + bindAddress + ":0")) - .setFindPort(true) + .addEndpoint( + URI.create(WebAppUtils.getHttpSchemePrefix(conf) + bindAddress + + ":0")).setFindPort(true) .setConf(conf) .setACL(acl) .build();