MAPREDUCE-3960. Fix web-proxy to forward request to AM with configured hostname or IP. Contributed by Thomas Graves.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1296878 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Arun Murthy 2012-03-04 21:14:53 +00:00
parent 9f16abfd63
commit d41cb76b56
5 changed files with 39 additions and 3 deletions

View File

@ -250,6 +250,9 @@ Release 0.23.2 - UNRELEASED
MAPREDUCE-3929. Fixed output of 'bin/mapred queue -showacl' command to MAPREDUCE-3929. Fixed output of 'bin/mapred queue -showacl' command to
clarify ACLs for users. (John George via acmurthy) clarify ACLs for users. (John George via acmurthy)
MAPREDUCE-3960. Fix web-proxy to forward request to AM with configured
hostname or IP. (tgraves via acmurthy)
Release 0.23.1 - 2012-02-17 Release 0.23.1 - 2012-02-17
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

View File

@ -432,6 +432,10 @@ public class ResourceManager extends CompositeService implements Recoverable {
builder.withServlet(ProxyUriUtils.PROXY_SERVLET_NAME, builder.withServlet(ProxyUriUtils.PROXY_SERVLET_NAME,
ProxyUriUtils.PROXY_PATH_SPEC, WebAppProxyServlet.class); ProxyUriUtils.PROXY_PATH_SPEC, WebAppProxyServlet.class);
builder.withAttribute(WebAppProxy.FETCHER_ATTRIBUTE, fetcher); builder.withAttribute(WebAppProxy.FETCHER_ATTRIBUTE, fetcher);
String proxy = YarnConfiguration.getProxyHostAndPort(conf);
String[] proxyParts = proxy.split(":");
builder.withAttribute(WebAppProxy.PROXY_HOST_ATTRIBUTE, proxyParts[0]);
} }
webApp = builder.start(new RMWebApp(this)); webApp = builder.start(new RMWebApp(this));
} }

View File

@ -35,6 +35,7 @@ import org.apache.hadoop.fs.CommonConfigurationKeys;
public class WebAppProxy extends AbstractService { public class WebAppProxy extends AbstractService {
public static final String FETCHER_ATTRIBUTE= "AppUrlFetcher"; public static final String FETCHER_ATTRIBUTE= "AppUrlFetcher";
public static final String IS_SECURITY_ENABLED_ATTRIBUTE = "IsSecurityEnabled"; public static final String IS_SECURITY_ENABLED_ATTRIBUTE = "IsSecurityEnabled";
public static final String PROXY_HOST_ATTRIBUTE = "proxyHost";
private static final Log LOG = LogFactory.getLog(WebAppProxy.class); private static final Log LOG = LogFactory.getLog(WebAppProxy.class);
private HttpServer proxyServer = null; private HttpServer proxyServer = null;
@ -43,6 +44,7 @@ public class WebAppProxy extends AbstractService {
private AccessControlList acl = null; private AccessControlList acl = null;
private AppReportFetcher fetcher = null; private AppReportFetcher fetcher = null;
private boolean isSecurityEnabled = false; private boolean isSecurityEnabled = false;
private String proxyHost = null;
public WebAppProxy() { public WebAppProxy() {
super(WebAppProxy.class.getName()); super(WebAppProxy.class.getName());
@ -60,6 +62,9 @@ public class WebAppProxy extends AbstractService {
CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION + CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION +
" of " + auth); " of " + auth);
} }
String proxy = YarnConfiguration.getProxyHostAndPort(conf);
String[] proxyParts = proxy.split(":");
proxyHost = proxyParts[0];
fetcher = new AppReportFetcher(conf); fetcher = new AppReportFetcher(conf);
bindAddress = conf.get(YarnConfiguration.PROXY_ADDRESS); bindAddress = conf.get(YarnConfiguration.PROXY_ADDRESS);
@ -88,6 +93,7 @@ public class WebAppProxy extends AbstractService {
ProxyUriUtils.PROXY_PATH_SPEC, WebAppProxyServlet.class); ProxyUriUtils.PROXY_PATH_SPEC, WebAppProxyServlet.class);
proxyServer.setAttribute(FETCHER_ATTRIBUTE, fetcher); proxyServer.setAttribute(FETCHER_ATTRIBUTE, fetcher);
proxyServer.setAttribute(IS_SECURITY_ENABLED_ATTRIBUTE, isSecurityEnabled); proxyServer.setAttribute(IS_SECURITY_ENABLED_ATTRIBUTE, isSecurityEnabled);
proxyServer.setAttribute(PROXY_HOST_ATTRIBUTE, proxyHost);
proxyServer.start(); proxyServer.start();
} catch (IOException e) { } catch (IOException e) {
LOG.fatal("Could not start proxy web server",e); LOG.fatal("Could not start proxy web server",e);

View File

@ -22,6 +22,7 @@ import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.OutputStream; import java.io.OutputStream;
import java.io.PrintWriter; import java.io.PrintWriter;
import java.net.InetAddress;
import java.net.URI; import java.net.URI;
import java.net.URISyntaxException; import java.net.URISyntaxException;
import java.net.URLEncoder; import java.net.URLEncoder;
@ -38,6 +39,7 @@ import javax.servlet.http.HttpServletResponse;
import org.apache.commons.httpclient.Header; import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.HttpClient; import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpMethod; import org.apache.commons.httpclient.HttpMethod;
import org.apache.commons.httpclient.HostConfiguration;
import org.apache.commons.httpclient.cookie.CookiePolicy; import org.apache.commons.httpclient.cookie.CookiePolicy;
import org.apache.commons.httpclient.methods.GetMethod; import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.params.HttpClientParams; import org.apache.commons.httpclient.params.HttpClientParams;
@ -123,13 +125,23 @@ public class WebAppProxyServlet extends HttpServlet {
* @throws IOException on any error. * @throws IOException on any error.
*/ */
private static void proxyLink(HttpServletRequest req, private static void proxyLink(HttpServletRequest req,
HttpServletResponse resp, URI link,Cookie c) throws IOException { HttpServletResponse resp, URI link, Cookie c, String proxyHost)
throws IOException {
org.apache.commons.httpclient.URI uri = org.apache.commons.httpclient.URI uri =
new org.apache.commons.httpclient.URI(link.toString(), false); new org.apache.commons.httpclient.URI(link.toString(), false);
HttpClientParams params = new HttpClientParams(); HttpClientParams params = new HttpClientParams();
params.setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY); params.setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY);
params.setBooleanParameter(HttpClientParams.ALLOW_CIRCULAR_REDIRECTS, true); params.setBooleanParameter(HttpClientParams.ALLOW_CIRCULAR_REDIRECTS, true);
HttpClient client = new HttpClient(params); HttpClient client = new HttpClient(params);
// Make sure we send the request from the proxy address in the config
// since that is what the AM filter checks against. IP aliasing or
// similar could cause issues otherwise.
HostConfiguration config = new HostConfiguration();
InetAddress localAddress = InetAddress.getByName(proxyHost);
if (LOG.isDebugEnabled()) {
LOG.debug("local InetAddress for proxy host: " + localAddress.toString());
}
config.setLocalAddress(localAddress);
HttpMethod method = new GetMethod(uri.getEscapedURI()); HttpMethod method = new GetMethod(uri.getEscapedURI());
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
@ -150,7 +162,7 @@ public class WebAppProxyServlet extends HttpServlet {
} }
OutputStream out = resp.getOutputStream(); OutputStream out = resp.getOutputStream();
try { try {
resp.setStatus(client.executeMethod(method)); resp.setStatus(client.executeMethod(config, method));
for(Header header : method.getResponseHeaders()) { for(Header header : method.getResponseHeaders()) {
resp.setHeader(header.getName(), header.getValue()); resp.setHeader(header.getName(), header.getValue());
} }
@ -189,6 +201,11 @@ public class WebAppProxyServlet extends HttpServlet {
.getAttribute(WebAppProxy.FETCHER_ATTRIBUTE)).getApplicationReport(id); .getAttribute(WebAppProxy.FETCHER_ATTRIBUTE)).getApplicationReport(id);
} }
private String getProxyHost() throws IOException {
return ((String) getServletContext()
.getAttribute(WebAppProxy.PROXY_HOST_ATTRIBUTE));
}
@Override @Override
protected void doGet(HttpServletRequest req, HttpServletResponse resp) protected void doGet(HttpServletRequest req, HttpServletResponse resp)
throws IOException{ throws IOException{
@ -275,7 +292,7 @@ public class WebAppProxyServlet extends HttpServlet {
if(userWasWarned && userApproved) { if(userWasWarned && userApproved) {
c = makeCheckCookie(id, true); c = makeCheckCookie(id, true);
} }
proxyLink(req, resp, toFetch, c); proxyLink(req, resp, toFetch, c, getProxyHost());
} catch(URISyntaxException e) { } catch(URISyntaxException e) {
throw new IOException(e); throw new IOException(e);

View File

@ -64,6 +64,9 @@ public class AmIpFilter implements Filter {
try { try {
proxyAddresses = new HashSet<String>(); proxyAddresses = new HashSet<String>();
for(InetAddress add : InetAddress.getAllByName(proxyHost)) { for(InetAddress add : InetAddress.getAllByName(proxyHost)) {
if (LOG.isDebugEnabled()) {
LOG.debug("proxy address is: " + add.getHostAddress());
}
proxyAddresses.add(add.getHostAddress()); proxyAddresses.add(add.getHostAddress());
} }
lastUpdate = now; lastUpdate = now;
@ -89,6 +92,9 @@ public class AmIpFilter implements Filter {
HttpServletRequest httpReq = (HttpServletRequest)req; HttpServletRequest httpReq = (HttpServletRequest)req;
HttpServletResponse httpResp = (HttpServletResponse)resp; HttpServletResponse httpResp = (HttpServletResponse)resp;
if (LOG.isDebugEnabled()) {
LOG.debug("Remote address for request is: " + httpReq.getRemoteAddr());
}
if(!getProxyAddresses().contains(httpReq.getRemoteAddr())) { if(!getProxyAddresses().contains(httpReq.getRemoteAddr())) {
String redirectUrl = httpResp.encodeRedirectURL(proxyUriBase + String redirectUrl = httpResp.encodeRedirectURL(proxyUriBase +
httpReq.getRequestURI()); httpReq.getRequestURI());