YARN-2513. Host framework UIs in YARN for use with the ATS (jeagles)

(cherry picked from commit 25f8f801d1)

hadoop-yarn-project/CHANGES.txt

@@ -17,6 +17,8 @@ Release 2.7.2 - UNRELEASED
     YARN-2801. Add documentation for node labels feature. (Wangda Tan and Naganarasimha 
     G R  via ozawa)
 
+    YARN-2513. Host framework UIs in YARN for use with the ATS (jeagles)
+
   OPTIMIZATIONS
 
   BUG FIXES

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java

@@ -1307,6 +1307,23 @@ public class YarnConfiguration extends Configuration {
   public static final String TIMELINE_SERVICE_PREFIX =
       YARN_PREFIX + "timeline-service.";
 
+  /**
+   * Comma seperated list of names for UIs hosted in the timeline server
+   * (For pluggable UIs).
+   */
+  public static final String TIMELINE_SERVICE_UI_NAMES =
+      TIMELINE_SERVICE_PREFIX + "ui-names";
+
+  /** Relative web path that will serve up this UI (For pluggable UIs). */
+  public static final String TIMELINE_SERVICE_UI_WEB_PATH_PREFIX =
+      TIMELINE_SERVICE_PREFIX + "ui-web-path.";
+
+  /**
+   * Path to war file or static content directory for this UI
+   * (For pluggable UIs).
+   */
+  public static final String TIMELINE_SERVICE_UI_ON_DISK_PATH_PREFIX =
+      TIMELINE_SERVICE_PREFIX + "ui-on-disk-path.";
 
   // mark app-history related configs @Private as application history is going
   // to be integrated into the timeline service

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java

@@ -166,7 +166,7 @@ public class WebApps {
       return this;
     }
 
-    public WebApp start(WebApp webapp) {
+    public WebApp build(WebApp webapp) {
       if (webapp == null) {
         webapp = new WebApp() {
           @Override
@@ -271,8 +271,7 @@ public class WebApps {
 
         webapp.setConf(conf);
         webapp.setHttpServer(server);
-        server.start();
+
-        LOG.info("Web app /"+ name +" started at "+ server.getConnectorAddress(0).getPort());
       } catch (ClassNotFoundException e) {
         throw new WebAppException("Error starting http server", e);
       } catch (IOException e) {
@@ -300,6 +299,19 @@ public class WebApps {
       return start(null);
     }
 
+    public WebApp start(WebApp webapp) {
+      WebApp webApp = build(webapp);
+      HttpServer2 httpServer = webApp.httpServer();
+      try {
+        httpServer.start();
+        LOG.info("Web app " + name + " started at "
+            + httpServer.getConnectorAddress(0).getPort());
+      } catch (IOException e) {
+        throw new WebAppException("Error starting http server", e);
+      }
+      return webApp;
+    }
+
     private String inferHostClass() {
       String thisClass = this.getClass().getName();
       Throwable t = new Throwable();

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml

@@ -1448,6 +1448,12 @@
     <value>/etc/krb5.keytab</value>
   </property>
 
+  <property>
+    <description>Comma separated list of UIs that will be hosted</description>
+    <name>yarn.timeline-service.ui-names</name>
+    <value></value>
+  </property>
+
   <property>
     <description>
     Default maximum number of retires for timeline servive client

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java

@@ -26,6 +26,7 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.http.HttpServer2;
 import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
 import org.apache.hadoop.metrics2.source.JvmMetrics;
 import org.apache.hadoop.security.AuthenticationFilterInitializer;
@@ -53,6 +54,8 @@ import org.apache.hadoop.yarn.server.timeline.webapp.CrossOriginFilterInitialize
 import org.apache.hadoop.yarn.webapp.WebApp;
 import org.apache.hadoop.yarn.webapp.WebApps;
 import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
+import org.mortbay.jetty.servlet.FilterHolder;
+import org.mortbay.jetty.webapp.WebAppContext;
 
 import com.google.common.annotations.VisibleForTesting;
 
@@ -127,6 +130,12 @@ public class ApplicationHistoryServer extends CompositeService {
     return this.ahsClientService;
   }
 
+  @Private
+  @VisibleForTesting
+  int getPort() {
+    return this.webApp.httpServer().getConnectorAddress(0).getPort();
+  }
+
   /**
    * @return ApplicationTimelineStore
    */
@@ -213,6 +222,7 @@ public class ApplicationHistoryServer extends CompositeService {
         timelineStore, new TimelineACLsManager(conf));
   }
 
+  @SuppressWarnings("unchecked")
   private void startWebApp() {
     Configuration conf = getConfig();
     TimelineAuthenticationFilter.setTimelineDelegationTokenSecretManager(
@@ -268,14 +278,42 @@ public class ApplicationHistoryServer extends CompositeService {
     String bindAddress = WebAppUtils.getWebAppBindURL(conf,
                           YarnConfiguration.TIMELINE_SERVICE_BIND_HOST,
                           WebAppUtils.getAHSWebAppURLWithoutScheme(conf));
-    LOG.info("Instantiating AHSWebApp at " + bindAddress);
     try {
+      AHSWebApp ahsWebApp = new AHSWebApp(timelineDataManager, ahsClientService);
       webApp =
           WebApps
             .$for("applicationhistory", ApplicationHistoryClientService.class,
                 ahsClientService, "ws")
-            .with(conf).at(bindAddress).start(
+             .with(conf).withAttribute(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS,
-                new AHSWebApp(timelineDataManager, ahsClientService));
+                 conf.get(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS)).at(bindAddress).build(ahsWebApp);
+       HttpServer2 httpServer = webApp.httpServer();
+
+       String[] names = conf.getTrimmedStrings(YarnConfiguration.TIMELINE_SERVICE_UI_NAMES);
+       WebAppContext webAppContext = httpServer.getWebAppContext();
+
+       for (String name : names) {
+         String webPath = conf.get(
+             YarnConfiguration.TIMELINE_SERVICE_UI_WEB_PATH_PREFIX + name);
+         String onDiskPath = conf.get(
+             YarnConfiguration.TIMELINE_SERVICE_UI_ON_DISK_PATH_PREFIX + name);
+         WebAppContext uiWebAppContext = new WebAppContext();
+         uiWebAppContext.setContextPath(webPath);
+         uiWebAppContext.setWar(onDiskPath);
+         final String[] ALL_URLS = { "/*" };
+         FilterHolder[] filterHolders =
+           webAppContext.getServletHandler().getFilters();
+         for (FilterHolder filterHolder: filterHolders) {
+           if (!"guice".equals(filterHolder.getName())) {
+             HttpServer2.defineFilter(uiWebAppContext, filterHolder.getName(),
+                 filterHolder.getClassName(), filterHolder.getInitParameters(),
+                 ALL_URLS);
+           }
+         }
+         LOG.info("Hosting " + name + " from " + onDiskPath + " at " + webPath);
+         httpServer.addContext(uiWebAppContext, true);
+       }
+       httpServer.start();
+       LOG.info("Instantiating AHSWebApp at " + getPort());
     } catch (Exception e) {
       String msg = "AHSWebApp failed to start.";
       LOG.error(msg, e);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java

@@ -28,16 +28,20 @@ import org.apache.hadoop.security.AuthenticationFilterInitializer;
 import org.apache.hadoop.service.Service.STATE;
 import org.apache.hadoop.util.ExitUtil;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp;
 import org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore;
 import org.apache.hadoop.yarn.server.timeline.TimelineStore;
 import org.apache.hadoop.yarn.server.timeline.recovery.MemoryTimelineStateStore;
 import org.apache.hadoop.yarn.server.timeline.recovery.TimelineStateStore;
 import org.apache.hadoop.yarn.server.timeline.security.TimelineAuthenticationFilterInitializer;
-import org.junit.After;
 import org.junit.Assert;
 import org.junit.Test;
 
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.net.HttpURLConnection;
+import java.net.URL;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -173,4 +177,49 @@ public class TestApplicationHistoryServer {
     }
   }
 
+  @Test(timeout = 240000)
+  public void testHostedUIs() throws Exception {
+
+    ApplicationHistoryServer historyServer = new ApplicationHistoryServer();
+    Configuration config = new YarnConfiguration();
+    config.setClass(YarnConfiguration.TIMELINE_SERVICE_STORE,
+        MemoryTimelineStore.class, TimelineStore.class);
+    config.setClass(YarnConfiguration.TIMELINE_SERVICE_STATE_STORE_CLASS,
+        MemoryTimelineStateStore.class, TimelineStateStore.class);
+    config.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS,
+        "localhost:0");
+    final String UI1 = "UI1";
+    String connFileStr = "";
+
+    File diskFile = new File("./pom.xml");
+    String diskFileStr = readInputStream(new FileInputStream(diskFile));
+    try {
+      config.set(YarnConfiguration.TIMELINE_SERVICE_UI_NAMES, UI1);
+      config.set(YarnConfiguration.TIMELINE_SERVICE_UI_WEB_PATH_PREFIX + UI1,
+          "/" + UI1);
+      config.set(YarnConfiguration.TIMELINE_SERVICE_UI_ON_DISK_PATH_PREFIX
+          + UI1, "./");
+      historyServer.init(config);
+      historyServer.start();
+      URL url = new URL("http://localhost:" + historyServer.getPort() + "/"
+          + UI1 + "/pom.xml");
+      HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+      conn.connect();
+      assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode());
+      connFileStr = readInputStream(conn.getInputStream());
+    } finally {
+      historyServer.stop();
+    }
+    assertEquals("Web file contents should be the same as on disk contents",
+        diskFileStr, connFileStr);
+  }
+  private String readInputStream(InputStream input) throws Exception {
+    ByteArrayOutputStream data = new ByteArrayOutputStream();
+    byte[] buffer = new byte[512];
+    int read;
+    while ((read = input.read(buffer)) >= 0) {
+      data.write(buffer, 0, read);
+    }
+    return new String(data.toByteArray(), "UTF-8");
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md

@@ -185,6 +185,14 @@ selected if this policy is either of `HTTPS_ONLY` or `HTTP_AND_HTTPS`.
 | `yarn.timeline-service.client.retry-interval-ms` | The interval in milliseconds between retries for the timeline service client. Defaults to `1000`. |
 | `yarn.timeline-service.generic-application-history.max-applications` | The max number of applications could be fetched by using REST API or application history protocol and shown in timeline server web ui. Defaults to `10000`. |
 
+#### UI Hosting Configuration
+
+The timeline service can host multiple UIs if enabled. The service can support both static web sites hosted in a directory or war files bundled. The web UI is then hosted on the timeline service HTTP port under the path configured.
+| Configuration Property | Description |
+|:---- |:---- |
+| `yarn.timeline-service.ui-names` | Comma separated list of UIs that will be hosted. Defaults to `none`. |
+| `yarn.timeline-service.ui-on-disk-path.$name` | For each of the ui-names, an on disk path should be specified to the directory service static content or the location of a web archive (war file). |
+| `yarn.timeline-service.ui-web-path.$name` | For each of the ui-names, the web path should be specified relative to the Timeline server root. Paths should begin with a starting slash. |
 
 
 #### Security Configuration