YARN-285. Added a temporary plugin interface for RM to be able to redirect to JobHistory server for apps that it no longer tracks. Contributed by Derek Dagit.

svn merge --ignore-ancestry -c 1425210 ../../trunk/


git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1425211 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Vinod Kumar Vavilapalli 2012-12-22 00:28:02 +00:00
parent 41ad080fd9
commit 674d6ba079
6 changed files with 144 additions and 2 deletions

View File

@ -11,6 +11,10 @@ Release 2.0.3-alpha - Unreleased
YARN-230. RM Restart phase 1 - includes support for saving/restarting all
applications on an RM bounce. (Bikas Saha via acmurthy)
YARN-285. Added a temporary plugin interface for RM to be able to redirect
to JobHistory server for apps that it no longer tracks. (Derek Dagit via
vinodkv)
IMPROVEMENTS
YARN-223. Update process tree instead of getting new process trees.

View File

@ -400,6 +400,9 @@ public class YarnConfiguration extends Configuration {
public static final String YARN_LOG_SERVER_URL =
YARN_PREFIX + "log.server.url";
public static final String YARN_TRACKING_URL_GENERATOR =
YARN_PREFIX + "tracking.url.generator";
/** Amount of memory in GB that can be allocated for containers.*/
public static final String NM_PMEM_MB = NM_PREFIX + "resource.memory-mb";

View File

@ -0,0 +1,46 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.util;
import java.net.URI;
import java.net.URISyntaxException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.yarn.api.records.ApplicationId;
/**
* Plugin to derive a tracking URL from a Yarn Application ID
*
*/
@InterfaceAudience.LimitedPrivate({"MapReduce"})
@InterfaceStability.Unstable
public abstract class TrackingUriPlugin extends Configured {
/**
* Given an application ID, return a tracking URI.
* @param id the ID for which a URI is returned
* @return the tracking URI
* @throws URISyntaxException
*/
public abstract URI getTrackingUri(ApplicationId id)
throws URISyntaxException;
}

View File

@ -24,11 +24,13 @@ import java.io.UnsupportedEncodingException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URLEncoder;
import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.util.TrackingUriPlugin;
public class ProxyUriUtils {
@SuppressWarnings("unused")
@ -143,4 +145,28 @@ public class ProxyUriUtils {
throws URISyntaxException {
return new URI(HttpConfig.getSchemePrefix() + noSchemeUrl);
}
/**
* Returns the first valid tracking link, if any, from the given id from the
* given list of plug-ins, if any.
*
* @param id the id of the application for which the tracking link is desired
* @param trackingUriPlugins list of plugins from which to get the tracking link
* @return the desired link if possible, otherwise null
* @throws URISyntaxException
*/
public static URI getUriFromTrackingPlugins(ApplicationId id,
List<TrackingUriPlugin> trackingUriPlugins)
throws URISyntaxException {
URI toRet = null;
for(TrackingUriPlugin plugin : trackingUriPlugins)
{
toRet = plugin.getTrackingUri(id);
if (toRet != null)
{
return toRet;
}
}
return null;
}
}

View File

@ -30,6 +30,7 @@ import java.util.Arrays;
import java.util.EnumSet;
import java.util.Enumeration;
import java.util.HashSet;
import java.util.List;
import javax.servlet.http.Cookie;
import javax.servlet.http.HttpServlet;
@ -48,8 +49,10 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.util.Apps;
import org.apache.hadoop.yarn.util.StringHelper;
import org.apache.hadoop.yarn.util.TrackingUriPlugin;
import org.apache.hadoop.yarn.webapp.MimeType;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
@ -61,8 +64,9 @@ public class WebAppProxyServlet extends HttpServlet {
"Accept-Language", "Accept-Charset"));
public static final String PROXY_USER_COOKIE_NAME = "proxy-user";
private final List<TrackingUriPlugin> trackingUriPlugins;
private static class _ implements Hamlet._ {
//Empty
}
@ -77,6 +81,18 @@ public class WebAppProxyServlet extends HttpServlet {
}
}
/**
* Default constructor
*/
public WebAppProxyServlet()
{
super();
YarnConfiguration conf = new YarnConfiguration();
this.trackingUriPlugins =
conf.getInstances(YarnConfiguration.YARN_TRACKING_URL_GENERATOR,
TrackingUriPlugin.class);
}
/**
* Output 404 with appropriate message.
* @param resp the http response.
@ -253,6 +269,16 @@ public class WebAppProxyServlet extends HttpServlet {
if(applicationReport == null) {
LOG.warn(req.getRemoteUser()+" Attempting to access "+id+
" that was not found");
URI toFetch =
ProxyUriUtils
.getUriFromTrackingPlugins(id, this.trackingUriPlugins);
if (toFetch != null)
{
resp.sendRedirect(resp.encodeRedirectURL(toFetch.toString()));
return;
}
notFound(resp, "Application "+appId+" could not be found, " +
"please try the history server");
return;

View File

@ -21,11 +21,16 @@ package org.apache.hadoop.yarn.server.webproxy;
import static org.junit.Assert.*;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.List;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.hadoop.yarn.util.TrackingUriPlugin;
import org.junit.Test;
import com.google.common.collect.Lists;
public class TestProxyUriUtils {
@Test
public void testGetPathApplicationId() {
@ -83,4 +88,36 @@ public class TestProxyUriUtils {
URI result = ProxyUriUtils.getProxyUri(originalUri, proxyUri, id);
assertEquals(expected, result);
}
@Test
public void testGetProxyUriFromPluginsReturnsNullIfNoPlugins()
throws URISyntaxException {
ApplicationId id = BuilderUtils.newApplicationId(6384623l, 5);
List<TrackingUriPlugin> list =
Lists.newArrayListWithExpectedSize(0);
assertNull(ProxyUriUtils.getUriFromTrackingPlugins(id, list));
}
@Test
public void testGetProxyUriFromPluginsReturnsValidUriWhenAble()
throws URISyntaxException {
ApplicationId id = BuilderUtils.newApplicationId(6384623l, 5);
List<TrackingUriPlugin> list =
Lists.newArrayListWithExpectedSize(2);
// Insert a plugin that returns null.
list.add(new TrackingUriPlugin() {
public URI getTrackingUri(ApplicationId id) throws URISyntaxException {
return null;
}
});
// Insert a plugin that returns a valid URI.
list.add(new TrackingUriPlugin() {
public URI getTrackingUri(ApplicationId id) throws URISyntaxException {
return new URI("http://history.server.net/");
}
});
URI result = ProxyUriUtils.getUriFromTrackingPlugins(id, list);
assertNotNull(result);
}
}