YARN-3147. Clean up RM web proxy code. Contributed by Steve Loughran

This commit is contained in:
Xuan 2015-02-12 10:57:55 -08:00
parent 9e33c9944c
commit 83be450acc
10 changed files with 324 additions and 154 deletions

View File

@ -272,6 +272,8 @@ Release 2.7.0 - UNRELEASED
YARN-3157. Refactor the exception handling in ConverterUtils#to*Id. YARN-3157. Refactor the exception handling in ConverterUtils#to*Id.
(Bibin A Chundatt via ozawa) (Bibin A Chundatt via ozawa)
YARN-3147. Clean up RM web proxy code. (Steve Loughran via xgong)
OPTIMIZATIONS OPTIMIZATIONS
YARN-2990. FairScheduler's delay-scheduling always waits for node-local and YARN-2990. FairScheduler's delay-scheduling always waits for node-local and

View File

@ -18,11 +18,10 @@
package org.apache.hadoop.yarn.server.webproxy; package org.apache.hadoop.yarn.server.webproxy;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.util.TrackingUriPlugin; import org.apache.hadoop.yarn.util.TrackingUriPlugin;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.UnsupportedEncodingException; import java.io.UnsupportedEncodingException;
import java.net.URI; import java.net.URI;
@ -34,8 +33,9 @@
public class ProxyUriUtils { public class ProxyUriUtils {
@SuppressWarnings("unused") @SuppressWarnings("unused")
private static final Log LOG = LogFactory.getLog(ProxyUriUtils.class); private static final Logger LOG = LoggerFactory.getLogger(
ProxyUriUtils.class);
/**Name of the servlet to use when registering the proxy servlet. */ /**Name of the servlet to use when registering the proxy servlet. */
public static final String PROXY_SERVLET_NAME = "proxy"; public static final String PROXY_SERVLET_NAME = "proxy";
/**Base path where the proxy servlet will handle requests.*/ /**Base path where the proxy servlet will handle requests.*/
@ -194,4 +194,5 @@ public static String getSchemeFromUrl(String url) {
return ""; return "";
} }
} }
} }

View File

@ -0,0 +1,127 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.webproxy;
import org.apache.hadoop.yarn.webapp.MimeType;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.servlet.ServletException;
import javax.servlet.ServletRequest;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.EnumSet;
/**
* Class containing general purpose proxy utilities
*/
public class ProxyUtils {
private static final Logger LOG = LoggerFactory.getLogger(
ProxyUtils.class);
public static final String E_HTTP_HTTPS_ONLY =
"This filter only works for HTTP/HTTPS";
public static final String LOCATION = "Location";
public static class _ implements Hamlet._ {
//Empty
}
public static class Page extends Hamlet {
Page(PrintWriter out) {
super(out, 0, false);
}
public HTML<ProxyUtils._> html() {
return new HTML<>("html", null, EnumSet.of(EOpt.ENDTAG));
}
}
/**
* Handle redirects with a status code that can in future support verbs other
* than GET, thus supporting full REST functionality.
* <p>
* The target URL is included in the redirect text returned
* <p>
* At the end of this method, the output stream is closed.
*
* @param request request (hence: the verb and any other information
* relevant to a redirect)
* @param response the response
* @param target the target URL -unencoded
*
*/
public static void sendRedirect(HttpServletRequest request,
HttpServletResponse response,
String target)
throws IOException {
if (LOG.isDebugEnabled()) {
LOG.debug("Redirecting {} {} to {}",
request.getMethod(),
request.getRequestURI(),
target);
}
String location = response.encodeRedirectURL(target);
response.setStatus(HttpServletResponse.SC_FOUND);
response.setHeader(LOCATION, location);
response.setContentType(MimeType.HTML);
PrintWriter writer = response.getWriter();
Page p = new Page(writer);
p.html()
.head().title("Moved")._()
.body()
.h1("Moved")
.div()
._("Content has moved ")
.a(location, "here")._()
._()._();
writer.close();
}
/**
* Output 404 with appropriate message.
* @param resp the http response.
* @param message the message to include on the page.
* @throws IOException on any error.
*/
public static void notFound(HttpServletResponse resp, String message)
throws IOException {
resp.setStatus(HttpServletResponse.SC_NOT_FOUND);
resp.setContentType(MimeType.HTML);
Page p = new Page(resp.getWriter());
p.html().
h1(message).
_();
}
/**
* Reject any request that isn't from an HTTP servlet
* @param req request
* @throws ServletException if the request is of the wrong type
*/
public static void rejectNonHttpRequests(ServletRequest req) throws
ServletException {
if (!(req instanceof HttpServletRequest)) {
throw new ServletException(E_HTTP_HTTPS_ONLY);
}
}
}

View File

@ -21,8 +21,6 @@
import java.io.IOException; import java.io.IOException;
import java.net.URI; import java.net.URI;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.http.HttpServer2; import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.security.authorize.AccessControlList;
@ -34,12 +32,15 @@
import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.CommonConfigurationKeys;
import com.google.common.annotations.VisibleForTesting; import com.google.common.annotations.VisibleForTesting;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class WebAppProxy extends AbstractService { public class WebAppProxy extends AbstractService {
public static final String FETCHER_ATTRIBUTE= "AppUrlFetcher"; public static final String FETCHER_ATTRIBUTE= "AppUrlFetcher";
public static final String IS_SECURITY_ENABLED_ATTRIBUTE = "IsSecurityEnabled"; public static final String IS_SECURITY_ENABLED_ATTRIBUTE = "IsSecurityEnabled";
public static final String PROXY_HOST_ATTRIBUTE = "proxyHost"; public static final String PROXY_HOST_ATTRIBUTE = "proxyHost";
private static final Log LOG = LogFactory.getLog(WebAppProxy.class); private static final Logger LOG = LoggerFactory.getLogger(
WebAppProxy.class);
private HttpServer2 proxyServer = null; private HttpServer2 proxyServer = null;
private String bindAddress = null; private String bindAddress = null;
@ -109,8 +110,8 @@ protected void serviceStart() throws Exception {
proxyServer.setAttribute(PROXY_HOST_ATTRIBUTE, proxyHost); proxyServer.setAttribute(PROXY_HOST_ATTRIBUTE, proxyHost);
proxyServer.start(); proxyServer.start();
} catch (IOException e) { } catch (IOException e) {
LOG.fatal("Could not start proxy web server",e); LOG.error("Could not start proxy web server",e);
throw new YarnRuntimeException("Could not start proxy web server",e); throw e;
} }
super.serviceStart(); super.serviceStart();
} }
@ -121,7 +122,7 @@ protected void serviceStop() throws Exception {
try { try {
proxyServer.stop(); proxyServer.stop();
} catch (Exception e) { } catch (Exception e) {
LOG.fatal("Error stopping proxy web server", e); LOG.error("Error stopping proxy web server", e);
throw new YarnRuntimeException("Error stopping proxy web server",e); throw new YarnRuntimeException("Error stopping proxy web server",e);
} }
} }
@ -136,6 +137,7 @@ public void join() {
try { try {
proxyServer.join(); proxyServer.join();
} catch (InterruptedException e) { } catch (InterruptedException e) {
// ignored
} }
} }
} }

View File

@ -21,16 +21,16 @@
import java.io.IOException; import java.io.IOException;
import java.net.InetSocketAddress; import java.net.InetSocketAddress;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.service.CompositeService; import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.util.ShutdownHookManager; import org.apache.hadoop.util.ShutdownHookManager;
import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler; import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/** /**
* ProxyServer will sit in between the end user and AppMaster * ProxyServer will sit in between the end user and AppMaster
@ -43,8 +43,9 @@ public class WebAppProxyServer extends CompositeService {
*/ */
public static final int SHUTDOWN_HOOK_PRIORITY = 30; public static final int SHUTDOWN_HOOK_PRIORITY = 30;
private static final Log LOG = LogFactory.getLog(WebAppProxyServer.class); private static final Logger LOG = LoggerFactory.getLogger(
WebAppProxyServer.class);
private WebAppProxy proxy = null; private WebAppProxy proxy = null;
public WebAppProxyServer() { public WebAppProxyServer() {
@ -54,11 +55,7 @@ public WebAppProxyServer() {
@Override @Override
protected void serviceInit(Configuration conf) throws Exception { protected void serviceInit(Configuration conf) throws Exception {
Configuration config = new YarnConfiguration(conf); Configuration config = new YarnConfiguration(conf);
try { doSecureLogin(conf);
doSecureLogin(conf);
} catch(IOException ie) {
throw new YarnRuntimeException("Proxy Server Failed to login", ie);
}
proxy = new WebAppProxy(); proxy = new WebAppProxy();
addService(proxy); addService(proxy);
super.serviceInit(config); super.serviceInit(config);
@ -95,8 +92,7 @@ public static void main(String[] args) {
WebAppProxyServer proxyServer = startServer(configuration); WebAppProxyServer proxyServer = startServer(configuration);
proxyServer.proxy.join(); proxyServer.proxy.join();
} catch (Throwable t) { } catch (Throwable t) {
LOG.fatal("Error starting Proxy server", t); ExitUtil.terminate(-1, t);
System.exit(-1);
} }
} }

View File

@ -32,6 +32,7 @@
import java.util.Enumeration; import java.util.Enumeration;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Set;
import javax.servlet.http.Cookie; import javax.servlet.http.Cookie;
import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServlet;
@ -45,8 +46,6 @@
import org.apache.commons.httpclient.cookie.CookiePolicy; import org.apache.commons.httpclient.cookie.CookiePolicy;
import org.apache.commons.httpclient.methods.GetMethod; import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.params.HttpClientParams; import org.apache.commons.httpclient.params.HttpClientParams;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationReport;
@ -59,13 +58,20 @@
import org.apache.hadoop.yarn.webapp.MimeType; import org.apache.hadoop.yarn.webapp.MimeType;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class WebAppProxyServlet extends HttpServlet { public class WebAppProxyServlet extends HttpServlet {
private static final long serialVersionUID = 1L; private static final long serialVersionUID = 1L;
private static final Log LOG = LogFactory.getLog(WebAppProxyServlet.class); private static final Logger LOG = LoggerFactory.getLogger(
private static final HashSet<String> passThroughHeaders = WebAppProxyServlet.class);
new HashSet<String>(Arrays.asList("User-Agent", "Accept", "Accept-Encoding", private static final Set<String> passThroughHeaders =
"Accept-Language", "Accept-Charset")); new HashSet<>(Arrays.asList(
"User-Agent",
"Accept",
"Accept-Encoding",
"Accept-Language",
"Accept-Charset"));
public static final String PROXY_USER_COOKIE_NAME = "proxy-user"; public static final String PROXY_USER_COOKIE_NAME = "proxy-user";
@ -83,15 +89,14 @@ private static class Page extends Hamlet {
} }
public HTML<WebAppProxyServlet._> html() { public HTML<WebAppProxyServlet._> html() {
return new HTML<WebAppProxyServlet._>("html", null, EnumSet.of(EOpt.ENDTAG)); return new HTML<>("html", null, EnumSet.of(EOpt.ENDTAG));
} }
} }
/** /**
* Default constructor * Default constructor
*/ */
public WebAppProxyServlet() public WebAppProxyServlet() {
{
super(); super();
conf = new YarnConfiguration(); conf = new YarnConfiguration();
this.trackingUriPlugins = this.trackingUriPlugins =
@ -109,12 +114,7 @@ public WebAppProxyServlet()
*/ */
private static void notFound(HttpServletResponse resp, String message) private static void notFound(HttpServletResponse resp, String message)
throws IOException { throws IOException {
resp.setStatus(HttpServletResponse.SC_NOT_FOUND); ProxyUtils.notFound(resp, message);
resp.setContentType(MimeType.HTML);
Page p = new Page(resp.getWriter());
p.html().
h1(message).
_();
} }
/** /**
@ -133,7 +133,8 @@ private static void warnUserPage(HttpServletResponse resp, String link,
resp.setContentType(MimeType.HTML); resp.setContentType(MimeType.HTML);
Page p = new Page(resp.getWriter()); Page p = new Page(resp.getWriter());
p.html(). p.html().
h1("WARNING: The following page may not be safe!").h3(). h1("WARNING: The following page may not be safe!").
h3().
_("click ").a(link, "here"). _("click ").a(link, "here").
_(" to continue to an Application Master web interface owned by ", user). _(" to continue to an Application Master web interface owned by ", user).
_(). _().
@ -163,7 +164,7 @@ private static void proxyLink(HttpServletRequest req,
HostConfiguration config = new HostConfiguration(); HostConfiguration config = new HostConfiguration();
InetAddress localAddress = InetAddress.getByName(proxyHost); InetAddress localAddress = InetAddress.getByName(proxyHost);
if (LOG.isDebugEnabled()) { if (LOG.isDebugEnabled()) {
LOG.debug("local InetAddress for proxy host: " + localAddress.toString()); LOG.debug("local InetAddress for proxy host: {}", localAddress);
} }
config.setLocalAddress(localAddress); config.setLocalAddress(localAddress);
HttpMethod method = new GetMethod(uri.getEscapedURI()); HttpMethod method = new GetMethod(uri.getEscapedURI());
@ -174,15 +175,17 @@ private static void proxyLink(HttpServletRequest req,
String name = names.nextElement(); String name = names.nextElement();
if(passThroughHeaders.contains(name)) { if(passThroughHeaders.contains(name)) {
String value = req.getHeader(name); String value = req.getHeader(name);
LOG.debug("REQ HEADER: "+name+" : "+value); if (LOG.isDebugEnabled()) {
LOG.debug("REQ HEADER: {} : {}", name, value);
}
method.setRequestHeader(name, value); method.setRequestHeader(name, value);
} }
} }
String user = req.getRemoteUser(); String user = req.getRemoteUser();
if(user != null && !user.isEmpty()) { if (user != null && !user.isEmpty()) {
method.setRequestHeader("Cookie",PROXY_USER_COOKIE_NAME+"="+ method.setRequestHeader("Cookie",
URLEncoder.encode(user, "ASCII")); PROXY_USER_COOKIE_NAME + "=" + URLEncoder.encode(user, "ASCII"));
} }
OutputStream out = resp.getOutputStream(); OutputStream out = resp.getOutputStream();
try { try {
@ -190,11 +193,11 @@ private static void proxyLink(HttpServletRequest req,
for(Header header : method.getResponseHeaders()) { for(Header header : method.getResponseHeaders()) {
resp.setHeader(header.getName(), header.getValue()); resp.setHeader(header.getName(), header.getValue());
} }
if(c != null) { if (c != null) {
resp.addCookie(c); resp.addCookie(c);
} }
InputStream in = method.getResponseBodyAsStream(); InputStream in = method.getResponseBodyAsStream();
if(in != null) { if (in != null) {
IOUtils.copyBytes(in, out, 4096, true); IOUtils.copyBytes(in, out, 4096, true);
} }
} finally { } finally {
@ -216,8 +219,7 @@ private static Cookie makeCheckCookie(ApplicationId id, boolean isSet) {
private boolean isSecurityEnabled() { private boolean isSecurityEnabled() {
Boolean b = (Boolean) getServletContext() Boolean b = (Boolean) getServletContext()
.getAttribute(WebAppProxy.IS_SECURITY_ENABLED_ATTRIBUTE); .getAttribute(WebAppProxy.IS_SECURITY_ENABLED_ATTRIBUTE);
if(b != null) return b; return b != null ? b : false;
return false;
} }
private ApplicationReport getApplicationReport(ApplicationId id) private ApplicationReport getApplicationReport(ApplicationId id)
@ -238,15 +240,14 @@ protected void doGet(HttpServletRequest req, HttpServletResponse resp)
String userApprovedParamS = String userApprovedParamS =
req.getParameter(ProxyUriUtils.PROXY_APPROVAL_PARAM); req.getParameter(ProxyUriUtils.PROXY_APPROVAL_PARAM);
boolean userWasWarned = false; boolean userWasWarned = false;
boolean userApproved = boolean userApproved = Boolean.valueOf(userApprovedParamS);
(userApprovedParamS != null && Boolean.valueOf(userApprovedParamS));
boolean securityEnabled = isSecurityEnabled(); boolean securityEnabled = isSecurityEnabled();
final String remoteUser = req.getRemoteUser(); final String remoteUser = req.getRemoteUser();
final String pathInfo = req.getPathInfo(); final String pathInfo = req.getPathInfo();
String parts[] = pathInfo.split("/", 3); String[] parts = pathInfo.split("/", 3);
if(parts.length < 2) { if(parts.length < 2) {
LOG.warn(remoteUser+" Gave an invalid proxy path "+pathInfo); LOG.warn("{} gave an invalid proxy path {}", remoteUser, pathInfo);
notFound(resp, "Your path appears to be formatted incorrectly."); notFound(resp, "Your path appears to be formatted incorrectly.");
return; return;
} }
@ -255,9 +256,9 @@ protected void doGet(HttpServletRequest req, HttpServletResponse resp)
String rest = parts.length > 2 ? parts[2] : ""; String rest = parts.length > 2 ? parts[2] : "";
ApplicationId id = Apps.toAppID(appId); ApplicationId id = Apps.toAppID(appId);
if(id == null) { if(id == null) {
LOG.warn(req.getRemoteUser()+" Attempting to access "+appId+ LOG.warn("{} attempting to access {} that is invalid",
" that is invalid"); remoteUser, appId);
notFound(resp, appId+" appears to be formatted incorrectly."); notFound(resp, appId + " appears to be formatted incorrectly.");
return; return;
} }
@ -277,35 +278,34 @@ protected void doGet(HttpServletRequest req, HttpServletResponse resp)
boolean checkUser = securityEnabled && (!userWasWarned || !userApproved); boolean checkUser = securityEnabled && (!userWasWarned || !userApproved);
ApplicationReport applicationReport = null; ApplicationReport applicationReport;
try { try {
applicationReport = getApplicationReport(id); applicationReport = getApplicationReport(id);
} catch (ApplicationNotFoundException e) { } catch (ApplicationNotFoundException e) {
applicationReport = null; applicationReport = null;
} }
if(applicationReport == null) { if(applicationReport == null) {
LOG.warn(req.getRemoteUser()+" Attempting to access "+id+ LOG.warn("{} attempting to access {} that was not found",
" that was not found"); remoteUser, id);
URI toFetch = URI toFetch =
ProxyUriUtils ProxyUriUtils
.getUriFromTrackingPlugins(id, this.trackingUriPlugins); .getUriFromTrackingPlugins(id, this.trackingUriPlugins);
if (toFetch != null) if (toFetch != null) {
{ ProxyUtils.sendRedirect(req, resp, toFetch.toString());
resp.sendRedirect(resp.encodeRedirectURL(toFetch.toString()));
return; return;
} }
notFound(resp, "Application "+appId+" could not be found, " + notFound(resp, "Application " + appId + " could not be found, " +
"please try the history server"); "please try the history server");
return; return;
} }
String original = applicationReport.getOriginalTrackingUrl(); String original = applicationReport.getOriginalTrackingUrl();
URI trackingUri = null; URI trackingUri;
// fallback to ResourceManager's app page if no tracking URI provided // fallback to ResourceManager's app page if no tracking URI provided
if(original == null || original.equals("N/A")) { if(original == null || original.equals("N/A")) {
resp.sendRedirect(resp.encodeRedirectURL( ProxyUtils.sendRedirect(req, resp,
StringHelper.pjoin(rmAppPageUrlBase, id.toString()))); StringHelper.pjoin(rmAppPageUrlBase, id.toString()));
return; return;
} else { } else {
if (ProxyUriUtils.getSchemeFromUrl(original).isEmpty()) { if (ProxyUriUtils.getSchemeFromUrl(original).isEmpty()) {
@ -318,8 +318,9 @@ protected void doGet(HttpServletRequest req, HttpServletResponse resp)
String runningUser = applicationReport.getUser(); String runningUser = applicationReport.getUser();
if(checkUser && !runningUser.equals(remoteUser)) { if(checkUser && !runningUser.equals(remoteUser)) {
LOG.info("Asking "+remoteUser+" if they want to connect to the " + LOG.info("Asking {} if they want to connect to the "
"app master GUI of "+appId+" owned by "+runningUser); + "app master GUI of {} owned by {}",
remoteUser, appId, runningUser);
warnUserPage(resp, ProxyUriUtils.getPathAndQuery(id, rest, warnUserPage(resp, ProxyUriUtils.getPathAndQuery(id, rest,
req.getQueryString(), true), runningUser, id); req.getQueryString(), true), runningUser, id);
return; return;
@ -329,29 +330,45 @@ protected void doGet(HttpServletRequest req, HttpServletResponse resp)
StringHelper.ujoin(trackingUri.getPath(), rest), req.getQueryString(), StringHelper.ujoin(trackingUri.getPath(), rest), req.getQueryString(),
null); null);
LOG.info(req.getRemoteUser()+" is accessing unchecked "+toFetch+ LOG.info("{} is accessing unchecked {}"
" which is the app master GUI of "+appId+" owned by "+runningUser); + " which is the app master GUI of {} owned by {}",
remoteUser, toFetch, appId, runningUser);
switch(applicationReport.getYarnApplicationState()) { switch (applicationReport.getYarnApplicationState()) {
case KILLED: case KILLED:
case FINISHED: case FINISHED:
case FAILED: case FAILED:
resp.sendRedirect(resp.encodeRedirectURL(toFetch.toString())); ProxyUtils.sendRedirect(req, resp, toFetch.toString());
return; return;
default:
// fall out of the switch
} }
Cookie c = null; Cookie c = null;
if(userWasWarned && userApproved) { if (userWasWarned && userApproved) {
c = makeCheckCookie(id, true); c = makeCheckCookie(id, true);
} }
proxyLink(req, resp, toFetch, c, getProxyHost()); proxyLink(req, resp, toFetch, c, getProxyHost());
} catch(URISyntaxException e) { } catch(URISyntaxException | YarnException e) {
throw new IOException(e); throw new IOException(e);
} catch (YarnException e) {
throw new IOException(e);
} }
} }
/**
* This method is used by Java object deserialization, to fill in the
* transient {@link #trackingUriPlugins} field.
* See {@link ObjectInputStream#defaultReadObject()}
* <p>
* <I>Do not remove</I>
* <p>
* Yarn isn't currently serializing this class, but findbugs
* complains in its absence.
*
*
* @param input source
* @throws IOException IO failure
* @throws ClassNotFoundException classloader fun
*/
private void readObject(ObjectInputStream input) private void readObject(ObjectInputStream input)
throws IOException, ClassNotFoundException { throws IOException, ClassNotFoundException {
input.defaultReadObject(); input.defaultReadObject();

View File

@ -36,7 +36,7 @@ public class AmFilterInitializer extends FilterInitializer {
@Override @Override
public void initFilter(FilterContainer container, Configuration conf) { public void initFilter(FilterContainer container, Configuration conf) {
Map<String, String> params = new HashMap<String, String>(); Map<String, String> params = new HashMap<>();
List<String> proxies = WebAppUtils.getProxyHostsAndPortsForAmFilter(conf); List<String> proxies = WebAppUtils.getProxyHostsAndPortsForAmFilter(conf);
StringBuilder sb = new StringBuilder(); StringBuilder sb = new StringBuilder();
for (String proxy : proxies) { for (String proxy : proxies) {

View File

@ -38,26 +38,27 @@
import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse; import javax.servlet.http.HttpServletResponse;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.yarn.conf.HAUtil; import org.apache.hadoop.yarn.conf.HAUtil;
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.webproxy.ProxyUtils;
import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServlet; import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServlet;
import org.apache.hadoop.yarn.util.RMHAUtils; import org.apache.hadoop.yarn.util.RMHAUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@Public @Public
public class AmIpFilter implements Filter { public class AmIpFilter implements Filter {
private static final Log LOG = LogFactory.getLog(AmIpFilter.class); private static final Logger LOG = LoggerFactory.getLogger(AmIpFilter.class);
@Deprecated @Deprecated
public static final String PROXY_HOST = "PROXY_HOST"; public static final String PROXY_HOST = "PROXY_HOST";
@Deprecated @Deprecated
public static final String PROXY_URI_BASE = "PROXY_URI_BASE"; public static final String PROXY_URI_BASE = "PROXY_URI_BASE";
static final String PROXY_HOSTS = "PROXY_HOSTS"; public static final String PROXY_HOSTS = "PROXY_HOSTS";
static final String PROXY_HOSTS_DELIMITER = ","; public static final String PROXY_HOSTS_DELIMITER = ",";
static final String PROXY_URI_BASES = "PROXY_URI_BASES"; public static final String PROXY_URI_BASES = "PROXY_URI_BASES";
static final String PROXY_URI_BASES_DELIMITER = ","; public static final String PROXY_URI_BASES_DELIMITER = ",";
//update the proxy IP list about every 5 min //update the proxy IP list about every 5 min
private static final long updateInterval = 5 * 60 * 1000; private static final long updateInterval = 5 * 60 * 1000;
@ -72,7 +73,7 @@ public void init(FilterConfig conf) throws ServletException {
if (conf.getInitParameter(PROXY_HOST) != null if (conf.getInitParameter(PROXY_HOST) != null
&& conf.getInitParameter(PROXY_URI_BASE) != null) { && conf.getInitParameter(PROXY_URI_BASE) != null) {
proxyHosts = new String[]{conf.getInitParameter(PROXY_HOST)}; proxyHosts = new String[]{conf.getInitParameter(PROXY_HOST)};
proxyUriBases = new HashMap<String, String>(1); proxyUriBases = new HashMap<>(1);
proxyUriBases.put("dummy", conf.getInitParameter(PROXY_URI_BASE)); proxyUriBases.put("dummy", conf.getInitParameter(PROXY_URI_BASE));
} else { } else {
proxyHosts = conf.getInitParameter(PROXY_HOSTS) proxyHosts = conf.getInitParameter(PROXY_HOSTS)
@ -80,13 +81,13 @@ public void init(FilterConfig conf) throws ServletException {
String[] proxyUriBasesArr = conf.getInitParameter(PROXY_URI_BASES) String[] proxyUriBasesArr = conf.getInitParameter(PROXY_URI_BASES)
.split(PROXY_URI_BASES_DELIMITER); .split(PROXY_URI_BASES_DELIMITER);
proxyUriBases = new HashMap<String, String>(); proxyUriBases = new HashMap<>(proxyUriBasesArr.length);
for (String proxyUriBase : proxyUriBasesArr) { for (String proxyUriBase : proxyUriBasesArr) {
try { try {
URL url = new URL(proxyUriBase); URL url = new URL(proxyUriBase);
proxyUriBases.put(url.getHost() + ":" + url.getPort(), proxyUriBase); proxyUriBases.put(url.getHost() + ":" + url.getPort(), proxyUriBase);
} catch(MalformedURLException e) { } catch(MalformedURLException e) {
LOG.warn(proxyUriBase + " does not appear to be a valid URL", e); LOG.warn("{} does not appear to be a valid URL", proxyUriBase, e);
} }
} }
} }
@ -96,18 +97,18 @@ protected Set<String> getProxyAddresses() throws ServletException {
long now = System.currentTimeMillis(); long now = System.currentTimeMillis();
synchronized(this) { synchronized(this) {
if(proxyAddresses == null || (lastUpdate + updateInterval) >= now) { if(proxyAddresses == null || (lastUpdate + updateInterval) >= now) {
proxyAddresses = new HashSet<String>(); proxyAddresses = new HashSet<>();
for (String proxyHost : proxyHosts) { for (String proxyHost : proxyHosts) {
try { try {
for(InetAddress add : InetAddress.getAllByName(proxyHost)) { for(InetAddress add : InetAddress.getAllByName(proxyHost)) {
if (LOG.isDebugEnabled()) { if (LOG.isDebugEnabled()) {
LOG.debug("proxy address is: " + add.getHostAddress()); LOG.debug("proxy address is: {}", add.getHostAddress());
} }
proxyAddresses.add(add.getHostAddress()); proxyAddresses.add(add.getHostAddress());
} }
lastUpdate = now; lastUpdate = now;
} catch (UnknownHostException e) { } catch (UnknownHostException e) {
LOG.warn("Could not locate " + proxyHost + " - skipping", e); LOG.warn("Could not locate {} - skipping", proxyHost, e);
} }
} }
if (proxyAddresses.isEmpty()) { if (proxyAddresses.isEmpty()) {
@ -126,20 +127,17 @@ public void destroy() {
@Override @Override
public void doFilter(ServletRequest req, ServletResponse resp, public void doFilter(ServletRequest req, ServletResponse resp,
FilterChain chain) throws IOException, ServletException { FilterChain chain) throws IOException, ServletException {
if(!(req instanceof HttpServletRequest)) { ProxyUtils.rejectNonHttpRequests(req);
throw new ServletException("This filter only works for HTTP/HTTPS");
}
HttpServletRequest httpReq = (HttpServletRequest)req; HttpServletRequest httpReq = (HttpServletRequest)req;
HttpServletResponse httpResp = (HttpServletResponse)resp; HttpServletResponse httpResp = (HttpServletResponse)resp;
if (LOG.isDebugEnabled()) { if (LOG.isDebugEnabled()) {
LOG.debug("Remote address for request is: " + httpReq.getRemoteAddr()); LOG.debug("Remote address for request is: {}", httpReq.getRemoteAddr());
} }
if(!getProxyAddresses().contains(httpReq.getRemoteAddr())) { if (!getProxyAddresses().contains(httpReq.getRemoteAddr())) {
String redirectUrl = findRedirectUrl(); String redirectUrl = findRedirectUrl();
redirectUrl = httpResp.encodeRedirectURL(redirectUrl + String target = redirectUrl + httpReq.getRequestURI();
httpReq.getRequestURI()); ProxyUtils.sendRedirect(httpReq, httpResp, target);
httpResp.sendRedirect(redirectUrl);
return; return;
} }
@ -153,9 +151,9 @@ public void doFilter(ServletRequest req, ServletResponse resp,
} }
} }
} }
if(user == null) { if (user == null) {
LOG.warn("Could not find "+WebAppProxyServlet.PROXY_USER_COOKIE_NAME LOG.warn("Could not find " + WebAppProxyServlet.PROXY_USER_COOKIE_NAME
+" cookie, so user will not be set"); + " cookie, so user will not be set");
chain.doFilter(req, resp); chain.doFilter(req, resp);
} else { } else {
final AmIpPrincipal principal = new AmIpPrincipal(user); final AmIpPrincipal principal = new AmIpPrincipal(user);

View File

@ -39,8 +39,6 @@
import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse; import javax.servlet.http.HttpServletResponse;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.http.HttpServer2; import org.apache.hadoop.http.HttpServer2;
@ -54,7 +52,6 @@
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import org.junit.AfterClass; import org.junit.AfterClass;
import org.junit.BeforeClass; import org.junit.BeforeClass;
@ -62,15 +59,16 @@
import org.mortbay.jetty.Server; import org.mortbay.jetty.Server;
import org.mortbay.jetty.servlet.Context; import org.mortbay.jetty.servlet.Context;
import org.mortbay.jetty.servlet.ServletHolder; import org.mortbay.jetty.servlet.ServletHolder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/** /**
* Test the WebAppProxyServlet and WebAppProxy. For back end use simple web * Test the WebAppProxyServlet and WebAppProxy. For back end use simple web
* server. * server.
*/ */
public class TestWebAppProxyServlet { public class TestWebAppProxyServlet {
private static final Logger LOG = LoggerFactory.getLogger(
private static final Log LOG = LogFactory TestWebAppProxyServlet.class);
.getLog(TestWebAppProxyServlet.class);
private static Server server; private static Server server;
private static int originalPort = 0; private static int originalPort = 0;
@ -273,11 +271,10 @@ public WebAppProxyServerForTest() {
} }
@Override @Override
public synchronized void init(Configuration conf) { public synchronized void serviceInit(Configuration conf) throws Exception {
Configuration config = new YarnConfiguration(conf);
proxy = new WebAppProxyForTest(); proxy = new WebAppProxyForTest();
addService(proxy); addService(proxy);
super.init(config); super.serviceInit(conf);
} }
} }
@ -286,44 +283,39 @@ private class WebAppProxyForTest extends WebAppProxy {
HttpServer2 proxyServer; HttpServer2 proxyServer;
AppReportFetcherForTest appReportFetcher; AppReportFetcherForTest appReportFetcher;
@Override
public void start() {
try {
Configuration conf = getConfig();
String bindAddress = conf.get(YarnConfiguration.PROXY_ADDRESS);
bindAddress = StringUtils.split(bindAddress, ':')[0];
AccessControlList acl = new AccessControlList(
conf.get(YarnConfiguration.YARN_ADMIN_ACL,
YarnConfiguration.DEFAULT_YARN_ADMIN_ACL));
proxyServer = new HttpServer2.Builder()
.setName("proxy")
.addEndpoint(
URI.create(WebAppUtils.getHttpSchemePrefix(conf) + bindAddress
+ ":0")).setFindPort(true)
.setConf(conf)
.setACL(acl)
.build();
proxyServer.addServlet(ProxyUriUtils.PROXY_SERVLET_NAME,
ProxyUriUtils.PROXY_PATH_SPEC, WebAppProxyServlet.class);
appReportFetcher = new AppReportFetcherForTest(conf); @Override
proxyServer.setAttribute(FETCHER_ATTRIBUTE, protected void serviceStart() throws Exception {
appReportFetcher ); Configuration conf = getConfig();
proxyServer.setAttribute(IS_SECURITY_ENABLED_ATTRIBUTE, Boolean.TRUE); String bindAddress = conf.get(YarnConfiguration.PROXY_ADDRESS);
bindAddress = StringUtils.split(bindAddress, ':')[0];
String proxy = WebAppUtils.getProxyHostAndPort(conf); AccessControlList acl = new AccessControlList(
String[] proxyParts = proxy.split(":"); conf.get(YarnConfiguration.YARN_ADMIN_ACL,
String proxyHost = proxyParts[0]; YarnConfiguration.DEFAULT_YARN_ADMIN_ACL));
proxyServer = new HttpServer2.Builder()
proxyServer.setAttribute(PROXY_HOST_ATTRIBUTE, proxyHost); .setName("proxy")
proxyServer.start(); .addEndpoint(
System.out.println("Proxy server is started at port " + URI.create(WebAppUtils.getHttpSchemePrefix(conf) + bindAddress
proxyServer.getConnectorAddress(0).getPort()); + ":0")).setFindPort(true)
} catch (Exception e) { .setConf(conf)
LOG.fatal("Could not start proxy web server", e); .setACL(acl)
throw new YarnRuntimeException("Could not start proxy web server", e); .build();
} proxyServer.addServlet(ProxyUriUtils.PROXY_SERVLET_NAME,
ProxyUriUtils.PROXY_PATH_SPEC, WebAppProxyServlet.class);
appReportFetcher = new AppReportFetcherForTest(conf);
proxyServer.setAttribute(FETCHER_ATTRIBUTE,
appReportFetcher );
proxyServer.setAttribute(IS_SECURITY_ENABLED_ATTRIBUTE, Boolean.TRUE);
String proxy = WebAppUtils.getProxyHostAndPort(conf);
String[] proxyParts = proxy.split(":");
String proxyHost = proxyParts[0];
proxyServer.setAttribute(PROXY_HOST_ATTRIBUTE, proxyHost);
proxyServer.start();
LOG.info("Proxy server is started at port {}",
proxyServer.getConnectorAddress(0).getPort());
} }
} }

View File

@ -19,6 +19,8 @@
package org.apache.hadoop.yarn.server.webproxy.amfilter; package org.apache.hadoop.yarn.server.webproxy.amfilter;
import java.io.IOException; import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.util.*; import java.util.*;
import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicBoolean;
@ -29,6 +31,7 @@
import static org.junit.Assert.*; import static org.junit.Assert.*;
import org.apache.hadoop.yarn.server.webproxy.ProxyUtils;
import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServlet; import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServlet;
import org.glassfish.grizzly.servlet.HttpServletResponseImpl; import org.glassfish.grizzly.servlet.HttpServletResponseImpl;
import org.junit.Test; import org.junit.Test;
@ -151,7 +154,7 @@ public void doFilter(ServletRequest servletRequest,
testFilter.doFilter(failRequest, response, chain); testFilter.doFilter(failRequest, response, chain);
fail(); fail();
} catch (ServletException e) { } catch (ServletException e) {
assertEquals("This filter only works for HTTP/HTTPS", e.getMessage()); assertEquals(ProxyUtils.E_HTTP_HTTPS_ONLY, e.getMessage());
} }
// request with HttpServletRequest // request with HttpServletRequest
@ -160,7 +163,9 @@ public void doFilter(ServletRequest servletRequest,
Mockito.when(request.getRequestURI()).thenReturn("/redirect"); Mockito.when(request.getRequestURI()).thenReturn("/redirect");
testFilter.doFilter(request, response, chain); testFilter.doFilter(request, response, chain);
// address "redirect" is not in host list // address "redirect" is not in host list
assertEquals("http://bogus/redirect", response.getRedirect()); assertEquals(302, response.status);
String redirect = response.getHeader(ProxyUtils.LOCATION);
assertEquals("http://bogus/redirect", redirect);
// "127.0.0.1" contains in host list. Without cookie // "127.0.0.1" contains in host list. Without cookie
Mockito.when(request.getRemoteAddr()).thenReturn("127.0.0.1"); Mockito.when(request.getRemoteAddr()).thenReturn("127.0.0.1");
testFilter.doFilter(request, response, chain); testFilter.doFilter(request, response, chain);
@ -186,6 +191,11 @@ public void doFilter(ServletRequest servletRequest,
private class HttpServletResponseForTest extends HttpServletResponseImpl { private class HttpServletResponseForTest extends HttpServletResponseImpl {
String redirectLocation = ""; String redirectLocation = "";
int status;
private String contentType;
private final Map<String, String> headers = new HashMap<>(1);
private StringWriter body;
public String getRedirect() { public String getRedirect() {
return redirectLocation; return redirectLocation;
@ -201,6 +211,31 @@ public String encodeRedirectURL(String url) {
return url; return url;
} }
@Override
public void setStatus(int status) {
this.status = status;
}
@Override
public void setContentType(String type) {
this.contentType = type;
}
@Override
public void setHeader(String name, String value) {
headers.put(name, value);
}
public String getHeader(String name) {
return headers.get(name);
}
@Override
public PrintWriter getWriter() throws IOException {
body = new StringWriter();
return new PrintWriter(body);
}
} }
} }