From d41cb76b56e19f6eb411b3dacaf28a490d079c68 Mon Sep 17 00:00:00 2001 From: Arun Murthy Date: Sun, 4 Mar 2012 21:14:53 +0000 Subject: [PATCH] MAPREDUCE-3960. Fix web-proxy to forward request to AM with configured hostname or IP. Contributed by Thomas Graves. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1296878 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../resourcemanager/ResourceManager.java | 4 ++++ .../yarn/server/webproxy/WebAppProxy.java | 6 +++++ .../server/webproxy/WebAppProxyServlet.java | 23 ++++++++++++++++--- .../server/webproxy/amfilter/AmIpFilter.java | 6 +++++ 5 files changed, 39 insertions(+), 3 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index bd8bf406679..1a89d94ac4d 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -250,6 +250,9 @@ Release 0.23.2 - UNRELEASED MAPREDUCE-3929. Fixed output of 'bin/mapred queue -showacl' command to clarify ACLs for users. (John George via acmurthy) + MAPREDUCE-3960. Fix web-proxy to forward request to AM with configured + hostname or IP. (tgraves via acmurthy) + Release 0.23.1 - 2012-02-17 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index bf01cef0058..c5357b11cc5 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -432,6 +432,10 @@ public class ResourceManager extends CompositeService implements Recoverable { builder.withServlet(ProxyUriUtils.PROXY_SERVLET_NAME, ProxyUriUtils.PROXY_PATH_SPEC, WebAppProxyServlet.class); builder.withAttribute(WebAppProxy.FETCHER_ATTRIBUTE, fetcher); + String proxy = YarnConfiguration.getProxyHostAndPort(conf); + String[] proxyParts = proxy.split(":"); + builder.withAttribute(WebAppProxy.PROXY_HOST_ATTRIBUTE, proxyParts[0]); + } webApp = builder.start(new RMWebApp(this)); } diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java index 0fd8e29c802..6a2bbb7a359 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java @@ -35,6 +35,7 @@ import org.apache.hadoop.fs.CommonConfigurationKeys; public class WebAppProxy extends AbstractService { public static final String FETCHER_ATTRIBUTE= "AppUrlFetcher"; public static final String IS_SECURITY_ENABLED_ATTRIBUTE = "IsSecurityEnabled"; + public static final String PROXY_HOST_ATTRIBUTE = "proxyHost"; private static final Log LOG = LogFactory.getLog(WebAppProxy.class); private HttpServer proxyServer = null; @@ -43,6 +44,7 @@ public class WebAppProxy extends AbstractService { private AccessControlList acl = null; private AppReportFetcher fetcher = null; private boolean isSecurityEnabled = false; + private String proxyHost = null; public WebAppProxy() { super(WebAppProxy.class.getName()); @@ -60,6 +62,9 @@ public class WebAppProxy extends AbstractService { CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION + " of " + auth); } + String proxy = YarnConfiguration.getProxyHostAndPort(conf); + String[] proxyParts = proxy.split(":"); + proxyHost = proxyParts[0]; fetcher = new AppReportFetcher(conf); bindAddress = conf.get(YarnConfiguration.PROXY_ADDRESS); @@ -88,6 +93,7 @@ public class WebAppProxy extends AbstractService { ProxyUriUtils.PROXY_PATH_SPEC, WebAppProxyServlet.class); proxyServer.setAttribute(FETCHER_ATTRIBUTE, fetcher); proxyServer.setAttribute(IS_SECURITY_ENABLED_ATTRIBUTE, isSecurityEnabled); + proxyServer.setAttribute(PROXY_HOST_ATTRIBUTE, proxyHost); proxyServer.start(); } catch (IOException e) { LOG.fatal("Could not start proxy web server",e); diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java index d5b99583c81..657b466c299 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.io.PrintWriter; +import java.net.InetAddress; import java.net.URI; import java.net.URISyntaxException; import java.net.URLEncoder; @@ -38,6 +39,7 @@ import javax.servlet.http.HttpServletResponse; import org.apache.commons.httpclient.Header; import org.apache.commons.httpclient.HttpClient; import org.apache.commons.httpclient.HttpMethod; +import org.apache.commons.httpclient.HostConfiguration; import org.apache.commons.httpclient.cookie.CookiePolicy; import org.apache.commons.httpclient.methods.GetMethod; import org.apache.commons.httpclient.params.HttpClientParams; @@ -123,13 +125,23 @@ public class WebAppProxyServlet extends HttpServlet { * @throws IOException on any error. */ private static void proxyLink(HttpServletRequest req, - HttpServletResponse resp, URI link,Cookie c) throws IOException { + HttpServletResponse resp, URI link, Cookie c, String proxyHost) + throws IOException { org.apache.commons.httpclient.URI uri = new org.apache.commons.httpclient.URI(link.toString(), false); HttpClientParams params = new HttpClientParams(); params.setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY); params.setBooleanParameter(HttpClientParams.ALLOW_CIRCULAR_REDIRECTS, true); HttpClient client = new HttpClient(params); + // Make sure we send the request from the proxy address in the config + // since that is what the AM filter checks against. IP aliasing or + // similar could cause issues otherwise. + HostConfiguration config = new HostConfiguration(); + InetAddress localAddress = InetAddress.getByName(proxyHost); + if (LOG.isDebugEnabled()) { + LOG.debug("local InetAddress for proxy host: " + localAddress.toString()); + } + config.setLocalAddress(localAddress); HttpMethod method = new GetMethod(uri.getEscapedURI()); @SuppressWarnings("unchecked") @@ -150,7 +162,7 @@ public class WebAppProxyServlet extends HttpServlet { } OutputStream out = resp.getOutputStream(); try { - resp.setStatus(client.executeMethod(method)); + resp.setStatus(client.executeMethod(config, method)); for(Header header : method.getResponseHeaders()) { resp.setHeader(header.getName(), header.getValue()); } @@ -189,6 +201,11 @@ public class WebAppProxyServlet extends HttpServlet { .getAttribute(WebAppProxy.FETCHER_ATTRIBUTE)).getApplicationReport(id); } + private String getProxyHost() throws IOException { + return ((String) getServletContext() + .getAttribute(WebAppProxy.PROXY_HOST_ATTRIBUTE)); + } + @Override protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException{ @@ -275,7 +292,7 @@ public class WebAppProxyServlet extends HttpServlet { if(userWasWarned && userApproved) { c = makeCheckCookie(id, true); } - proxyLink(req, resp, toFetch, c); + proxyLink(req, resp, toFetch, c, getProxyHost()); } catch(URISyntaxException e) { throw new IOException(e); diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java index 1fe45720f85..fbf87bdc85b 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java @@ -64,6 +64,9 @@ public class AmIpFilter implements Filter { try { proxyAddresses = new HashSet(); for(InetAddress add : InetAddress.getAllByName(proxyHost)) { + if (LOG.isDebugEnabled()) { + LOG.debug("proxy address is: " + add.getHostAddress()); + } proxyAddresses.add(add.getHostAddress()); } lastUpdate = now; @@ -89,6 +92,9 @@ public class AmIpFilter implements Filter { HttpServletRequest httpReq = (HttpServletRequest)req; HttpServletResponse httpResp = (HttpServletResponse)resp; + if (LOG.isDebugEnabled()) { + LOG.debug("Remote address for request is: " + httpReq.getRemoteAddr()); + } if(!getProxyAddresses().contains(httpReq.getRemoteAddr())) { String redirectUrl = httpResp.encodeRedirectURL(proxyUriBase + httpReq.getRequestURI());