From 3de0da2a7659db268d630cb8c4ad1d1c4b8398a2 Mon Sep 17 00:00:00 2001 From: Wangda Tan Date: Wed, 12 Oct 2016 13:22:20 -0700 Subject: [PATCH] YARN-5698. [YARN-3368] Launch new YARN UI under hadoop web app port. (Sunil G via wangda) --- .../hadoop/yarn/conf/YarnConfiguration.java | 21 ++---- .../apache/hadoop/yarn/webapp/WebApps.java | 8 +++ .../src/main/resources/yarn-default.xml | 20 ++---- .../resourcemanager/ResourceManager.java | 70 ++++++++++--------- .../src/main/webapp/config/default-config.js | 4 +- 5 files changed, 56 insertions(+), 67 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 0134465c189..c16e1ea47f4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -266,25 +266,12 @@ public class YarnConfiguration extends Configuration { /** * Enable YARN WebApp V2. */ - public static final String RM_WEBAPP_UI2_ENABLE = RM_PREFIX + public static final String YARN_WEBAPP_UI2_ENABLE = "yarn." + "webapp.ui2.enable"; - public static final boolean DEFAULT_RM_WEBAPP_UI2_ENABLE = false; + public static final boolean DEFAULT_YARN_WEBAPP_UI2_ENABLE = false; - /** The address of the RM web ui2 application. */ - public static final String RM_WEBAPP_UI2_ADDRESS = RM_PREFIX - + "webapp.ui2.address"; - - public static final int DEFAULT_RM_WEBAPP_UI2_PORT = 8288; - public static final String DEFAULT_RM_WEBAPP_UI2_ADDRESS = "0.0.0.0:" + - DEFAULT_RM_WEBAPP_UI2_PORT; - - /** The https address of the RM web ui2 application.*/ - public static final String RM_WEBAPP_UI2_HTTPS_ADDRESS = - RM_PREFIX + "webapp.ui2.https.address"; - - public static final int DEFAULT_RM_WEBAPP_UI2_HTTPS_PORT = 8290; - public static final String DEFAULT_RM_WEBAPP_UI2_HTTPS_ADDRESS = "0.0.0.0:" - + DEFAULT_RM_WEBAPP_UI2_HTTPS_PORT; + public static final String YARN_WEBAPP_UI2_WARFILE_PATH = "yarn." + + "webapp.ui2.war-file-path"; public static final String RM_RESOURCE_TRACKER_ADDRESS = RM_PREFIX + "resource-tracker.address"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java index 53cb3eea1eb..d3b37d97693 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java @@ -43,6 +43,7 @@ import org.apache.hadoop.security.http.RestCsrfPreventionFilter; import org.apache.hadoop.security.http.XFrameOptionsFilter; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; +import org.mortbay.jetty.webapp.WebAppContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -369,8 +370,15 @@ public class WebApps { } public WebApp start(WebApp webapp) { + return start(webapp, null); + } + + public WebApp start(WebApp webapp, WebAppContext ui2Context) { WebApp webApp = build(webapp); HttpServer2 httpServer = webApp.httpServer(); + if (ui2Context != null) { + httpServer.addContext(ui2Context, true); + } try { httpServer.start(); LOG.info("Web app " + name + " started at " diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 0e46dbdcbbd..e890b403d35 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -181,28 +181,16 @@ To enable RM web ui2 application. - yarn.resourcemanager.webapp.ui2.enable + yarn.webapp.ui2.enable false - The http address of the RM web ui2 application. - If only a host is provided as the value, - the webapp will be served on a random port. + Explicitly provide WAR file path for ui2 if needed. - yarn.resourcemanager.webapp.ui2.address - ${yarn.resourcemanager.hostname}:8288 - - - - - The https address of the RM web ui2 application. - If only a host is provided as the value, - the webapp will be served on a random port. - - yarn.resourcemanager.webapp.ui2.https.address - ${yarn.resourcemanager.hostname}:8290 + yarn.webapp.ui2.war-file-path + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index f739e31d371..4c0a3bbd997 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -49,6 +49,7 @@ import org.apache.hadoop.util.JvmPauseMonitor; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.ShutdownHookManager; import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.util.VersionInfo; import org.apache.hadoop.util.ZKUtil; import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; @@ -114,6 +115,7 @@ import org.apache.hadoop.yarn.webapp.WebApps; import org.apache.hadoop.yarn.webapp.WebApps.Builder; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.apache.zookeeper.server.auth.DigestAuthenticationProvider; +import org.mortbay.jetty.webapp.WebAppContext; import com.google.common.annotations.VisibleForTesting; import java.io.IOException; @@ -121,6 +123,8 @@ import java.io.InputStream; import java.io.PrintStream; import java.net.InetSocketAddress; import java.net.URI; +import java.net.URL; +import java.net.URLClassLoader; import java.nio.charset.Charset; import java.security.PrivilegedExceptionAction; import java.security.SecureRandom; @@ -148,6 +152,11 @@ public class ResourceManager extends CompositeService implements Recoverable { private static final Log LOG = LogFactory.getLog(ResourceManager.class); private static long clusterTimeStamp = System.currentTimeMillis(); + /* + * UI2 webapp name + */ + public static final String UI2_WEBAPP_NAME = "/ui2"; + /** * "Always On" services. Services that need to run always irrespective of * the HA state of the RM. @@ -940,26 +949,6 @@ public class ResourceManager extends CompositeService implements Recoverable { return builder; } - protected void startWebAppV2() throws IOException { - Configuration config = getConfig(); - final InetSocketAddress httpAddr = config.getSocketAddr( - YarnConfiguration.RM_WEBAPP_UI2_ADDRESS, - YarnConfiguration.DEFAULT_RM_WEBAPP_UI2_ADDRESS, - YarnConfiguration.DEFAULT_RM_WEBAPP_UI2_PORT); - final InetSocketAddress httpsAddr = config.getSocketAddr( - YarnConfiguration.RM_WEBAPP_UI2_HTTPS_ADDRESS, - YarnConfiguration.DEFAULT_RM_WEBAPP_UI2_HTTPS_ADDRESS, - YarnConfiguration.DEFAULT_RM_WEBAPP_UI2_HTTPS_PORT); - - HttpServer2.Builder builder = httpServerTemplateForRM(config, httpAddr, - httpsAddr, "rm"); - - HttpServer2 infoServer = builder.build(); - infoServer.start(); - - LOG.info("Web server init done"); - } - protected void startWepApp() { // Use the customized yarn filter instead of the standard kerberos filter to @@ -1074,9 +1063,36 @@ public class ResourceManager extends CompositeService implements Recoverable { builder.withAttribute(WebAppProxy.FETCHER_ATTRIBUTE, fetcher); String[] proxyParts = proxyHostAndPort.split(":"); builder.withAttribute(WebAppProxy.PROXY_HOST_ATTRIBUTE, proxyParts[0]); - } - webApp = builder.start(new RMWebApp(this)); + + WebAppContext uiWebAppContext = null; + if (getConfig().getBoolean(YarnConfiguration.YARN_WEBAPP_UI2_ENABLE, + YarnConfiguration.DEFAULT_YARN_WEBAPP_UI2_ENABLE)) { + String webPath = UI2_WEBAPP_NAME; + String onDiskPath = getConfig() + .get(YarnConfiguration.YARN_WEBAPP_UI2_WARFILE_PATH); + + if (null == onDiskPath) { + String war = "hadoop-yarn-ui-" + VersionInfo.getVersion() + ".war"; + URLClassLoader cl = (URLClassLoader) ClassLoader.getSystemClassLoader(); + URL url = cl.findResource(war); + + if (null == url) { + onDiskPath = ""; + } else { + onDiskPath = url.getFile(); + } + + LOG.info( + "New web UI war file name:" + war + ", and path:" + onDiskPath); + } + + uiWebAppContext = new WebAppContext(); + uiWebAppContext.setContextPath(webPath); + uiWebAppContext.setWar(onDiskPath); + } + + webApp = builder.start(new RMWebApp(this), uiWebAppContext); } /** @@ -1178,16 +1194,6 @@ public class ResourceManager extends CompositeService implements Recoverable { transitionToActive(); } - if (getConfig().getBoolean(YarnConfiguration.RM_WEBAPP_UI2_ENABLE, - YarnConfiguration.DEFAULT_RM_WEBAPP_UI2_ENABLE)) { - try { - startWebAppV2(); - LOG.info("Yarn WebApp UI 2 is started"); - } catch (Exception e) { - LOG.error("Failed to start Yarn web app v2:" + e.getMessage()); - } - } - startWepApp(); if (getConfig().getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/config/default-config.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/config/default-config.js index 92d07216567..c5dbbea0c72 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/config/default-config.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/config/default-config.js @@ -18,7 +18,7 @@ module.exports = { // Yarn UI App configurations hosts: { - localBaseAddress: "localhost:1337", + localBaseAddress: "", timelineWebAddress: "localhost:8188", rmWebAddress: "localhost:8088", protocolScheme: "http:" @@ -29,4 +29,4 @@ module.exports = { // Yarn UI App configurations metrics: 'ws/v1/cluster/metrics', node: 'ws/v1/node' }, -}; \ No newline at end of file +};