YARN-2513. Host framework UIs in YARN for use with the ATS (jeagles)
This commit is contained in:
parent
d759b4bd74
commit
25f8f801d1
|
@ -989,6 +989,8 @@ Release 2.7.2 - UNRELEASED
|
|||
YARN-2801. Add documentation for node labels feature. (Wangda Tan and Naganarasimha
|
||||
G R via ozawa)
|
||||
|
||||
YARN-2513. Host framework UIs in YARN for use with the ATS (jeagles)
|
||||
|
||||
OPTIMIZATIONS
|
||||
|
||||
BUG FIXES
|
||||
|
|
|
@ -1524,6 +1524,23 @@ public class YarnConfiguration extends Configuration {
|
|||
public static final String TIMELINE_SERVICE_PREFIX =
|
||||
YARN_PREFIX + "timeline-service.";
|
||||
|
||||
/**
|
||||
* Comma seperated list of names for UIs hosted in the timeline server
|
||||
* (For pluggable UIs).
|
||||
*/
|
||||
public static final String TIMELINE_SERVICE_UI_NAMES =
|
||||
TIMELINE_SERVICE_PREFIX + "ui-names";
|
||||
|
||||
/** Relative web path that will serve up this UI (For pluggable UIs). */
|
||||
public static final String TIMELINE_SERVICE_UI_WEB_PATH_PREFIX =
|
||||
TIMELINE_SERVICE_PREFIX + "ui-web-path.";
|
||||
|
||||
/**
|
||||
* Path to war file or static content directory for this UI
|
||||
* (For pluggable UIs).
|
||||
*/
|
||||
public static final String TIMELINE_SERVICE_UI_ON_DISK_PATH_PREFIX =
|
||||
TIMELINE_SERVICE_PREFIX + "ui-on-disk-path.";
|
||||
|
||||
// mark app-history related configs @Private as application history is going
|
||||
// to be integrated into the timeline service
|
||||
|
|
|
@ -166,7 +166,7 @@ public class WebApps {
|
|||
return this;
|
||||
}
|
||||
|
||||
public WebApp start(WebApp webapp) {
|
||||
public WebApp build(WebApp webapp) {
|
||||
if (webapp == null) {
|
||||
webapp = new WebApp() {
|
||||
@Override
|
||||
|
@ -271,8 +271,7 @@ public class WebApps {
|
|||
|
||||
webapp.setConf(conf);
|
||||
webapp.setHttpServer(server);
|
||||
server.start();
|
||||
LOG.info("Web app /"+ name +" started at "+ server.getConnectorAddress(0).getPort());
|
||||
|
||||
} catch (ClassNotFoundException e) {
|
||||
throw new WebAppException("Error starting http server", e);
|
||||
} catch (IOException e) {
|
||||
|
@ -300,6 +299,19 @@ public class WebApps {
|
|||
return start(null);
|
||||
}
|
||||
|
||||
public WebApp start(WebApp webapp) {
|
||||
WebApp webApp = build(webapp);
|
||||
HttpServer2 httpServer = webApp.httpServer();
|
||||
try {
|
||||
httpServer.start();
|
||||
LOG.info("Web app " + name + " started at "
|
||||
+ httpServer.getConnectorAddress(0).getPort());
|
||||
} catch (IOException e) {
|
||||
throw new WebAppException("Error starting http server", e);
|
||||
}
|
||||
return webApp;
|
||||
}
|
||||
|
||||
private String inferHostClass() {
|
||||
String thisClass = this.getClass().getName();
|
||||
Throwable t = new Throwable();
|
||||
|
|
|
@ -1868,6 +1868,12 @@
|
|||
<value>/etc/krb5.keytab</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<description>Comma separated list of UIs that will be hosted</description>
|
||||
<name>yarn.timeline-service.ui-names</name>
|
||||
<value></value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<description>
|
||||
Default maximum number of retries for timeline service client
|
||||
|
|
|
@ -26,6 +26,7 @@ import org.apache.commons.logging.Log;
|
|||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.http.HttpServer2;
|
||||
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
|
||||
import org.apache.hadoop.metrics2.source.JvmMetrics;
|
||||
import org.apache.hadoop.security.AuthenticationFilterInitializer;
|
||||
|
@ -54,6 +55,8 @@ import org.apache.hadoop.yarn.server.timeline.webapp.CrossOriginFilterInitialize
|
|||
import org.apache.hadoop.yarn.webapp.WebApp;
|
||||
import org.apache.hadoop.yarn.webapp.WebApps;
|
||||
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
|
||||
import org.mortbay.jetty.servlet.FilterHolder;
|
||||
import org.mortbay.jetty.webapp.WebAppContext;
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
|
||||
|
@ -137,6 +140,12 @@ public class ApplicationHistoryServer extends CompositeService {
|
|||
return this.ahsClientService;
|
||||
}
|
||||
|
||||
@Private
|
||||
@VisibleForTesting
|
||||
int getPort() {
|
||||
return this.webApp.httpServer().getConnectorAddress(0).getPort();
|
||||
}
|
||||
|
||||
/**
|
||||
* @return ApplicationTimelineStore
|
||||
*/
|
||||
|
@ -223,6 +232,7 @@ public class ApplicationHistoryServer extends CompositeService {
|
|||
timelineStore, new TimelineACLsManager(conf));
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private void startWebApp() {
|
||||
Configuration conf = getConfig();
|
||||
TimelineAuthenticationFilter.setTimelineDelegationTokenSecretManager(
|
||||
|
@ -278,14 +288,42 @@ public class ApplicationHistoryServer extends CompositeService {
|
|||
String bindAddress = WebAppUtils.getWebAppBindURL(conf,
|
||||
YarnConfiguration.TIMELINE_SERVICE_BIND_HOST,
|
||||
WebAppUtils.getAHSWebAppURLWithoutScheme(conf));
|
||||
LOG.info("Instantiating AHSWebApp at " + bindAddress);
|
||||
try {
|
||||
AHSWebApp ahsWebApp = new AHSWebApp(timelineDataManager, ahsClientService);
|
||||
webApp =
|
||||
WebApps
|
||||
.$for("applicationhistory", ApplicationHistoryClientService.class,
|
||||
ahsClientService, "ws")
|
||||
.with(conf).at(bindAddress).start(
|
||||
new AHSWebApp(timelineDataManager, ahsClientService));
|
||||
.with(conf).withAttribute(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS,
|
||||
conf.get(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS)).at(bindAddress).build(ahsWebApp);
|
||||
HttpServer2 httpServer = webApp.httpServer();
|
||||
|
||||
String[] names = conf.getTrimmedStrings(YarnConfiguration.TIMELINE_SERVICE_UI_NAMES);
|
||||
WebAppContext webAppContext = httpServer.getWebAppContext();
|
||||
|
||||
for (String name : names) {
|
||||
String webPath = conf.get(
|
||||
YarnConfiguration.TIMELINE_SERVICE_UI_WEB_PATH_PREFIX + name);
|
||||
String onDiskPath = conf.get(
|
||||
YarnConfiguration.TIMELINE_SERVICE_UI_ON_DISK_PATH_PREFIX + name);
|
||||
WebAppContext uiWebAppContext = new WebAppContext();
|
||||
uiWebAppContext.setContextPath(webPath);
|
||||
uiWebAppContext.setWar(onDiskPath);
|
||||
final String[] ALL_URLS = { "/*" };
|
||||
FilterHolder[] filterHolders =
|
||||
webAppContext.getServletHandler().getFilters();
|
||||
for (FilterHolder filterHolder: filterHolders) {
|
||||
if (!"guice".equals(filterHolder.getName())) {
|
||||
HttpServer2.defineFilter(uiWebAppContext, filterHolder.getName(),
|
||||
filterHolder.getClassName(), filterHolder.getInitParameters(),
|
||||
ALL_URLS);
|
||||
}
|
||||
}
|
||||
LOG.info("Hosting " + name + " from " + onDiskPath + " at " + webPath);
|
||||
httpServer.addContext(uiWebAppContext, true);
|
||||
}
|
||||
httpServer.start();
|
||||
LOG.info("Instantiating AHSWebApp at " + getPort());
|
||||
} catch (Exception e) {
|
||||
String msg = "AHSWebApp failed to start.";
|
||||
LOG.error(msg, e);
|
||||
|
|
|
@ -28,16 +28,20 @@ import org.apache.hadoop.security.AuthenticationFilterInitializer;
|
|||
import org.apache.hadoop.service.Service.STATE;
|
||||
import org.apache.hadoop.util.ExitUtil;
|
||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||
import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp;
|
||||
import org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore;
|
||||
import org.apache.hadoop.yarn.server.timeline.TimelineStore;
|
||||
import org.apache.hadoop.yarn.server.timeline.recovery.MemoryTimelineStateStore;
|
||||
import org.apache.hadoop.yarn.server.timeline.recovery.TimelineStateStore;
|
||||
import org.apache.hadoop.yarn.server.timeline.security.TimelineAuthenticationFilterInitializer;
|
||||
import org.junit.After;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.InputStream;
|
||||
import java.net.HttpURLConnection;
|
||||
import java.net.URL;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
|
@ -173,4 +177,49 @@ public class TestApplicationHistoryServer {
|
|||
}
|
||||
}
|
||||
|
||||
@Test(timeout = 240000)
|
||||
public void testHostedUIs() throws Exception {
|
||||
|
||||
ApplicationHistoryServer historyServer = new ApplicationHistoryServer();
|
||||
Configuration config = new YarnConfiguration();
|
||||
config.setClass(YarnConfiguration.TIMELINE_SERVICE_STORE,
|
||||
MemoryTimelineStore.class, TimelineStore.class);
|
||||
config.setClass(YarnConfiguration.TIMELINE_SERVICE_STATE_STORE_CLASS,
|
||||
MemoryTimelineStateStore.class, TimelineStateStore.class);
|
||||
config.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS,
|
||||
"localhost:0");
|
||||
final String UI1 = "UI1";
|
||||
String connFileStr = "";
|
||||
|
||||
File diskFile = new File("./pom.xml");
|
||||
String diskFileStr = readInputStream(new FileInputStream(diskFile));
|
||||
try {
|
||||
config.set(YarnConfiguration.TIMELINE_SERVICE_UI_NAMES, UI1);
|
||||
config.set(YarnConfiguration.TIMELINE_SERVICE_UI_WEB_PATH_PREFIX + UI1,
|
||||
"/" + UI1);
|
||||
config.set(YarnConfiguration.TIMELINE_SERVICE_UI_ON_DISK_PATH_PREFIX
|
||||
+ UI1, "./");
|
||||
historyServer.init(config);
|
||||
historyServer.start();
|
||||
URL url = new URL("http://localhost:" + historyServer.getPort() + "/"
|
||||
+ UI1 + "/pom.xml");
|
||||
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
|
||||
conn.connect();
|
||||
assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode());
|
||||
connFileStr = readInputStream(conn.getInputStream());
|
||||
} finally {
|
||||
historyServer.stop();
|
||||
}
|
||||
assertEquals("Web file contents should be the same as on disk contents",
|
||||
diskFileStr, connFileStr);
|
||||
}
|
||||
private String readInputStream(InputStream input) throws Exception {
|
||||
ByteArrayOutputStream data = new ByteArrayOutputStream();
|
||||
byte[] buffer = new byte[512];
|
||||
int read;
|
||||
while ((read = input.read(buffer)) >= 0) {
|
||||
data.write(buffer, 0, read);
|
||||
}
|
||||
return new String(data.toByteArray(), "UTF-8");
|
||||
}
|
||||
}
|
||||
|
|
|
@ -185,6 +185,14 @@ selected if this policy is either of `HTTPS_ONLY` or `HTTP_AND_HTTPS`.
|
|||
| `yarn.timeline-service.client.retry-interval-ms` | The interval in milliseconds between retries for the timeline service client. Defaults to `1000`. |
|
||||
| `yarn.timeline-service.generic-application-history.max-applications` | The max number of applications could be fetched by using REST API or application history protocol and shown in timeline server web ui. Defaults to `10000`. |
|
||||
|
||||
#### UI Hosting Configuration
|
||||
|
||||
The timeline service can host multiple UIs if enabled. The service can support both static web sites hosted in a directory or war files bundled. The web UI is then hosted on the timeline service HTTP port under the path configured.
|
||||
| Configuration Property | Description |
|
||||
|:---- |:---- |
|
||||
| `yarn.timeline-service.ui-names` | Comma separated list of UIs that will be hosted. Defaults to `none`. |
|
||||
| `yarn.timeline-service.ui-on-disk-path.$name` | For each of the ui-names, an on disk path should be specified to the directory service static content or the location of a web archive (war file). |
|
||||
| `yarn.timeline-service.ui-web-path.$name` | For each of the ui-names, the web path should be specified relative to the Timeline server root. Paths should begin with a starting slash. |
|
||||
|
||||
|
||||
#### Security Configuration
|
||||
|
|
Loading…
Reference in New Issue