Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-4767. Network issues can cause persistent RM UI outage. (Daniel Templeton via kasha) 

(cherry picked from commit 736d33cddd)
This commit is contained in:
Karthik Kambatla 2016-10-03 14:35:57 -07:00
parent 6f3744195d
commit 1578781fe9
10 changed files with 454 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
View File

@ -41,4 +41,5 @@ public interface YarnWebParams {
String NODE_LABEL = "node.label";
String WEB_UI_TYPE = "web.ui.type";
String NEXT_REFRESH_INTERVAL = "next.refresh.interval";
String ERROR_MESSAGE = "error.message";
}

39
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/ErrorBlock.java Normal file
View File

@ -0,0 +1,39 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
import com.google.inject.Inject;
import static org.apache.hadoop.yarn.webapp.YarnWebParams.ERROR_MESSAGE;
/**
* This class is used to display an error message to the user in the UI.
*/
public class ErrorBlock extends HtmlBlock {
@Inject
ErrorBlock(ViewContext ctx) {
super(ctx);
}
@Override
protected void render(Block html) {
html.p()._($(ERROR_MESSAGE))._();
}
}

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java
View File

@ -71,6 +71,7 @@ public class RMWebApp extends WebApp implements YarnWebParams {
route("/errors-and-warnings", RmController.class, "errorsAndWarnings");
route(pajoin("/logaggregationstatus", APPLICATION_ID),
RmController.class, "logaggregationstatus");
route(pajoin("/failure", APPLICATION_ID), RmController.class, "failure");
}
@Override

47
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RedirectionErrorPage.java Normal file
View File

@ -0,0 +1,47 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
import org.apache.hadoop.yarn.webapp.SubView;
import org.apache.hadoop.yarn.webapp.YarnWebParams;
/**
* This class is used to display a message that the proxy request failed
* because of a redirection issue.
*/
public class RedirectionErrorPage extends RmView {
@Override protected void preHead(Page.HTML<_> html) {
String aid = $(YarnWebParams.APPLICATION_ID);
commonPreHead(html);
set(YarnWebParams.ERROR_MESSAGE,
"The application master for " + aid + " redirected the "
+ "resource manager's web proxy's request back to the web proxy, "
+ "which means your request to view the application master's web UI "
+ "cannot be fulfilled. The typical cause for this error is a "
+ "network misconfiguration that causes the resource manager's web "
+ "proxy host to resolve to an unexpected IP address on the "
+ "application master host. Please contact your cluster "
+ "administrator to resolve the issue.");
}
@Override protected Class<? extends SubView> content() {
return ErrorBlock.class;
}
}

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java
View File

@ -62,6 +62,10 @@ public class RmController extends Controller {
render(ContainerPage.class);
}
public void failure() {
render(RedirectionErrorPage.class);
}
public void nodes() {
render(NodesPage.class);
}

68
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRedirectionErrorPage.java Normal file
View File

@ -0,0 +1,68 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
import java.io.IOException;
import org.apache.hadoop.yarn.api.ApplicationBaseProtocol;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.webapp.YarnWebParams;
import org.apache.hadoop.yarn.webapp.test.WebAppTests;
import org.junit.Test;
import com.google.inject.Binder;
import com.google.inject.Injector;
import com.google.inject.Module;
/**
* This class tests the RedirectionErrorPage.
*/
public class TestRedirectionErrorPage {
@Test
public void testAppBlockRenderWithNullCurrentAppAttempt() throws Exception {
ApplicationId appId = ApplicationId.newInstance(1234L, 0);
Injector injector;
// initialize RM Context, and create RMApp, without creating RMAppAttempt
final RMContext rmContext = TestRMWebApp.mockRMContext(15, 1, 2, 8);
injector = WebAppTests.createMockInjector(RMContext.class, rmContext,
new Module() {
@Override
public void configure(Binder binder) {
try {
ResourceManager rm = TestRMWebApp.mockRm(rmContext);
binder.bind(ResourceManager.class).toInstance(rm);
binder.bind(ApplicationBaseProtocol.class).toInstance(
rm.getClientRMService());
} catch (IOException e) {
throw new IllegalStateException(e);
}
}
});
ErrorBlock instance = injector.getInstance(ErrorBlock.class);
instance.set(YarnWebParams.APPLICATION_ID, appId.toString());
instance.set(YarnWebParams.ERROR_MESSAGE, "This is an error");
instance.render();
}
}

274
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
View File

@ -26,6 +26,7 @@ import java.io.ObjectInputStream;
import java.io.OutputStream;
import java.io.PrintWriter;
import java.net.InetAddress;
import java.net.SocketException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URLEncoder;
@ -42,8 +43,10 @@ import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.core.UriBuilder;
import javax.ws.rs.core.UriBuilderException;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@ -76,7 +79,8 @@ public class WebAppProxyServlet extends HttpServlet {
private static final long serialVersionUID = 1L;
private static final Logger LOG = LoggerFactory.getLogger(
WebAppProxyServlet.class);
private static final Set<String> passThroughHeaders =
private static final String REDIRECT = "/redirect";
private static final Set<String> PASS_THROUGH_HEADERS =
new HashSet<>(Arrays.asList(
"User-Agent",
"Accept",
@ -93,6 +97,7 @@ public class WebAppProxyServlet extends HttpServlet {
private transient List<TrackingUriPlugin> trackingUriPlugins;
private final String rmAppPageUrlBase;
private final String ahsAppPageUrlBase;
private final String failurePageUrlBase;
private transient YarnConfiguration conf;
/**
@ -126,11 +131,16 @@ public class WebAppProxyServlet extends HttpServlet {
this.trackingUriPlugins =
conf.getInstances(YarnConfiguration.YARN_TRACKING_URL_GENERATOR,
TrackingUriPlugin.class);
this.rmAppPageUrlBase = StringHelper.pjoin(
WebAppUtils.getResolvedRMWebAppURLWithScheme(conf), "cluster", "app");
this.ahsAppPageUrlBase = StringHelper.pjoin(
WebAppUtils.getHttpSchemePrefix(conf) + WebAppUtils
.getAHSWebAppURLWithoutScheme(conf), "applicationhistory", "app");
this.rmAppPageUrlBase =
StringHelper.pjoin(WebAppUtils.getResolvedRMWebAppURLWithScheme(conf),
"cluster", "app");
this.failurePageUrlBase =
StringHelper.pjoin(WebAppUtils.getResolvedRMWebAppURLWithScheme(conf),
"cluster", "failure");
this.ahsAppPageUrlBase =
StringHelper.pjoin(WebAppUtils.getHttpSchemePrefix(conf)
+ WebAppUtils.getAHSWebAppURLWithoutScheme(conf),
"applicationhistory", "app");
}
/**
@ -220,9 +230,9 @@ public class WebAppProxyServlet extends HttpServlet {
@SuppressWarnings("unchecked")
Enumeration<String> names = req.getHeaderNames();
while(names.hasMoreElements()) {
while (names.hasMoreElements()) {
String name = names.nextElement();
if(passThroughHeaders.contains(name)) {
if (PASS_THROUGH_HEADERS.contains(name)) {
String value = req.getHeader(name);
if (LOG.isDebugEnabled()) {
LOG.debug("REQ HEADER: {} : {}", name, value);
@ -312,30 +322,49 @@ public class WebAppProxyServlet extends HttpServlet {
boolean userWasWarned = false;
boolean userApproved = Boolean.parseBoolean(userApprovedParamS);
boolean securityEnabled = isSecurityEnabled();
boolean isRedirect = false;
String pathInfo = req.getPathInfo();
final String remoteUser = req.getRemoteUser();
final String pathInfo = req.getPathInfo();
String[] parts = null;
if (pathInfo != null) {
// If there's a redirect, strip the redirect so that the path can be
// parsed
if (pathInfo.startsWith(REDIRECT)) {
pathInfo = pathInfo.substring(REDIRECT.length());
isRedirect = true;
}
parts = pathInfo.split("/", 3);
}
if(parts == null || parts.length < 2) {
if ((parts == null) || (parts.length < 2)) {
LOG.warn("{} gave an invalid proxy path {}", remoteUser, pathInfo);
notFound(resp, "Your path appears to be formatted incorrectly.");
return;
}
//parts[0] is empty because path info always starts with a /
String appId = parts[1];
String rest = parts.length > 2 ? parts[2] : "";
ApplicationId id = Apps.toAppID(appId);
if(id == null) {
if (id == null) {
LOG.warn("{} attempting to access {} that is invalid",
remoteUser, appId);
notFound(resp, appId + " appears to be formatted incorrectly.");
return;
}
if(securityEnabled) {
// If this call is from an AM redirect, we need to be careful about how
// we handle it. If this method returns true, it means the method
// already redirected the response, so we can just return.
if (isRedirect && handleRedirect(appId, req, resp)) {
return;
}
if (securityEnabled) {
String cookieName = getCheckCookieName(id);
Cookie[] cookies = req.getCookies();
if (cookies != null) {
@ -351,22 +380,21 @@ public class WebAppProxyServlet extends HttpServlet {
boolean checkUser = securityEnabled && (!userWasWarned || !userApproved);
FetchedAppReport fetchedAppReport = null;
ApplicationReport applicationReport = null;
FetchedAppReport fetchedAppReport;
try {
fetchedAppReport = getApplicationReport(id);
if (fetchedAppReport != null) {
if (fetchedAppReport.getAppReportSource() != AppReportSource.RM &&
fetchedAppReport.getAppReportSource() != AppReportSource.AHS) {
throw new UnsupportedOperationException("Application report not "
+ "fetched from RM or history server.");
}
applicationReport = fetchedAppReport.getApplicationReport();
}
fetchedAppReport = getFetchedAppReport(id);
} catch (ApplicationNotFoundException e) {
applicationReport = null;
fetchedAppReport = null;
}
if(applicationReport == null) {
ApplicationReport applicationReport = null;
if (fetchedAppReport != null) {
applicationReport = fetchedAppReport.getApplicationReport();
}
if (applicationReport == null) {
LOG.warn("{} attempting to access {} that was not found",
remoteUser, id);
@ -382,57 +410,31 @@ public class WebAppProxyServlet extends HttpServlet {
"in RM or history server");
return;
}
String original = applicationReport.getOriginalTrackingUrl();
URI trackingUri;
if (original == null || original.equals("N/A") || original.equals("")) {
if (fetchedAppReport.getAppReportSource() == AppReportSource.RM) {
// fallback to ResourceManager's app page if no tracking URI provided
// and Application Report was fetched from RM
LOG.debug("Original tracking url is '{}'. Redirecting to RM app page",
original == null? "NULL" : original);
ProxyUtils.sendRedirect(req, resp,
StringHelper.pjoin(rmAppPageUrlBase, id.toString()));
} else if (fetchedAppReport.getAppReportSource()
== AppReportSource.AHS) {
// fallback to Application History Server app page if the application
// report was fetched from AHS
LOG.debug("Original tracking url is '{}'. Redirecting to AHS app page"
, original == null? "NULL" : original);
ProxyUtils.sendRedirect(req, resp,
StringHelper.pjoin(ahsAppPageUrlBase, id.toString()));
}
URI trackingUri = getTrackingUri(req, resp, id,
applicationReport.getOriginalTrackingUrl(),
fetchedAppReport.getAppReportSource());
// If the tracking URI is null, there was a redirect, so just return.
if (trackingUri == null) {
return;
} else {
if (ProxyUriUtils.getSchemeFromUrl(original).isEmpty()) {
trackingUri = ProxyUriUtils.getUriFromAMUrl(
WebAppUtils.getHttpSchemePrefix(conf), original);
} else {
trackingUri = new URI(original);
}
}
String runningUser = applicationReport.getUser();
if(checkUser && !runningUser.equals(remoteUser)) {
if (checkUser && !runningUser.equals(remoteUser)) {
LOG.info("Asking {} if they want to connect to the "
+ "app master GUI of {} owned by {}",
remoteUser, appId, runningUser);
warnUserPage(resp, ProxyUriUtils.getPathAndQuery(id, rest,
req.getQueryString(), true), runningUser, id);
return;
}
// Append the user-provided path and query parameter to the original
// tracking url.
UriBuilder builder = UriBuilder.fromUri(trackingUri);
String queryString = req.getQueryString();
if (queryString != null) {
List<NameValuePair> queryPairs =
URLEncodedUtils.parse(queryString, null);
for (NameValuePair pair : queryPairs) {
builder.queryParam(pair.getName(), pair.getValue());
}
}
URI toFetch = builder.path(rest).build();
URI toFetch = buildTrackingUrl(trackingUri, req, rest);
LOG.info("{} is accessing unchecked {}"
+ " which is the app master GUI of {} owned by {}",
@ -458,6 +460,152 @@ public class WebAppProxyServlet extends HttpServlet {
}
}
/**
* Return a URL based on the {@code trackingUri} that includes the
* user-provided path and query parameters.
*
* @param trackingUri the base tracking URI
* @param req the service request
* @param rest the user-provided path
* @return the new tracking URI
* @throws UriBuilderException if there's an error building the URL
*/
private URI buildTrackingUrl(URI trackingUri, final HttpServletRequest req,
String rest) throws UriBuilderException {
UriBuilder builder = UriBuilder.fromUri(trackingUri);
String queryString = req.getQueryString();
if (queryString != null) {
List<NameValuePair> queryPairs = URLEncodedUtils.parse(queryString, null);
for (NameValuePair pair : queryPairs) {
builder.queryParam(pair.getName(), pair.getValue());
}
}
return builder.path(rest).build();
}
/**
* Locate the tracking URI for the application based on the reported tracking
* URI. If the reported URI is invalid, redirect to the history server or RM
* app page. If the URI is valid, covert it into a usable URI object with a
* schema. If the returned URI is null, that means there was a redirect.
*
* @param req the servlet request for redirects
* @param resp the servlet response for redirects
* @param id the application ID
* @param originalUri the reported tracking URI
* @param appReportSource the source of the application report
* @return a valid tracking URI or null if redirected instead
* @throws IOException thrown if the redirect fails
* @throws URISyntaxException if the tracking URI is invalid
*/
private URI getTrackingUri(HttpServletRequest req, HttpServletResponse resp,
ApplicationId id, String originalUri, AppReportSource appReportSource)
throws IOException, URISyntaxException {
URI trackingUri = null;
if ((originalUri == null) ||
originalUri.equals("N/A") ||
originalUri.equals("")) {
if (appReportSource == AppReportSource.RM) {
// fallback to ResourceManager's app page if no tracking URI provided
// and Application Report was fetched from RM
LOG.debug("Original tracking url is '{}'. Redirecting to RM app page",
originalUri == null ? "NULL" : originalUri);
ProxyUtils.sendRedirect(req, resp,
StringHelper.pjoin(rmAppPageUrlBase, id.toString()));
} else if (appReportSource == AppReportSource.AHS) {
// fallback to Application History Server app page if the application
// report was fetched from AHS
LOG.debug("Original tracking url is '{}'. Redirecting to AHS app page",
originalUri == null ? "NULL" : originalUri);
ProxyUtils.sendRedirect(req, resp,
StringHelper.pjoin(ahsAppPageUrlBase, id.toString()));
}
} else if (ProxyUriUtils.getSchemeFromUrl(originalUri).isEmpty()) {
trackingUri =
ProxyUriUtils.getUriFromAMUrl(WebAppUtils.getHttpSchemePrefix(conf),
originalUri);
} else {
trackingUri = new URI(originalUri);
}
return trackingUri;
}
/**
* Fetch the application report from the RM.
*
* @param id the app ID
* @return the application report
* @throws IOException if the request to the RM fails
* @throws YarnException if the request to the RM fails
*/
private FetchedAppReport getFetchedAppReport(ApplicationId id)
throws IOException, YarnException {
FetchedAppReport fetchedAppReport = getApplicationReport(id);
if (fetchedAppReport != null) {
if ((fetchedAppReport.getAppReportSource() != AppReportSource.RM) &&
(fetchedAppReport.getAppReportSource() != AppReportSource.AHS)) {
throw new UnsupportedOperationException("Application report not "
+ "fetched from RM or history server.");
}
}
return fetchedAppReport;
}
/**
* Check whether the request is a redirect from the AM and handle it
* appropriately. This check exists to prevent the AM from forwarding back to
* the web proxy, which would contact the AM again, which would forward
* again... If this method returns true, there was a redirect, and
* it was handled by redirecting the current request to an error page.
*
* @param path the part of the request path after the app id
* @param id the app id
* @param req the request object
* @param resp the response object
* @return whether there was a redirect
* @throws IOException if a redirect fails
*/
private boolean handleRedirect(String id, HttpServletRequest req,
HttpServletResponse resp) throws IOException {
// If this isn't a redirect, we don't care.
boolean badRedirect = false;
// If this is a redirect, check if we're calling ourselves.
try {
badRedirect = NetUtils.getLocalInetAddress(req.getRemoteHost()) != null;
} catch (SocketException ex) {
// This exception means we can't determine the calling host. Odds are
// that means it's not us. Let it go and hope it works out better next
// time.
}
// If the proxy tries to call itself, it gets into an endless
// loop and consumes all available handler threads until the
// application completes. Redirect to the app page with a flag
// that tells it to print an appropriate error message.
if (badRedirect) {
LOG.error("The AM's web app redirected the RM web proxy's request back "
+ "to the web proxy. The typical cause is that the AM is resolving "
+ "the RM's address as something other than what it expects. Check "
+ "your network configuration and the value of the "
+ "yarn.web-proxy.address property. Once the host resolution issue "
+ "has been resolved, you will likely need to delete the "
+ "misbehaving application, " + id);
String redirect = StringHelper.pjoin(failurePageUrlBase, id);
LOG.error("REDIRECT: sending redirect to " + redirect);
ProxyUtils.sendRedirect(req, resp, redirect);
}
return badRedirect;
}
/**
* This method is used by Java object deserialization, to fill in the
* transient {@link #trackingUriPlugins} field.

64
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java
View File

@ -59,8 +59,9 @@ public class AmIpFilter implements Filter {
public static final String PROXY_HOSTS_DELIMITER = ",";
public static final String PROXY_URI_BASES = "PROXY_URI_BASES";
public static final String PROXY_URI_BASES_DELIMITER = ",";
private static final String PROXY_PATH = "/proxy";
//update the proxy IP list about every 5 min
private static final long updateInterval = 5 * 60 * 1000;
private static final long UPDATE_INTERVAL = 5 * 60 * 1000;
private String[] proxyHosts;
private Set<String> proxyAddresses = null;
@ -96,7 +97,7 @@ public class AmIpFilter implements Filter {
protected Set<String> getProxyAddresses() throws ServletException {
long now = System.currentTimeMillis();
synchronized(this) {
if(proxyAddresses == null || (lastUpdate + updateInterval) >= now) {
if (proxyAddresses == null || (lastUpdate + UPDATE_INTERVAL) >= now) {
proxyAddresses = new HashSet<>();
for (String proxyHost : proxyHosts) {
try {
@ -131,37 +132,52 @@ public class AmIpFilter implements Filter {
HttpServletRequest httpReq = (HttpServletRequest)req;
HttpServletResponse httpResp = (HttpServletResponse)resp;
if (LOG.isDebugEnabled()) {
LOG.debug("Remote address for request is: {}", httpReq.getRemoteAddr());
}
if (!getProxyAddresses().contains(httpReq.getRemoteAddr())) {
String redirectUrl = findRedirectUrl();
String target = redirectUrl + httpReq.getRequestURI();
ProxyUtils.sendRedirect(httpReq, httpResp, target);
return;
}
StringBuilder redirect = new StringBuilder(findRedirectUrl());
String user = null;
redirect.append(httpReq.getRequestURI());
if (httpReq.getCookies() != null) {
for(Cookie c: httpReq.getCookies()) {
if(WebAppProxyServlet.PROXY_USER_COOKIE_NAME.equals(c.getName())){
user = c.getValue();
break;
int insertPoint = redirect.indexOf(PROXY_PATH);
if (insertPoint >= 0) {
// Add /redirect as the second component of the path so that the RM web
// proxy knows that this request was a redirect.
insertPoint += PROXY_PATH.length();
redirect.insert(insertPoint, "/redirect");
}
ProxyUtils.sendRedirect(httpReq, httpResp, redirect.toString());
} else {
String user = null;
if (httpReq.getCookies() != null) {
for(Cookie c: httpReq.getCookies()) {
if(WebAppProxyServlet.PROXY_USER_COOKIE_NAME.equals(c.getName())){
user = c.getValue();
break;
}
}
}
}
if (user == null) {
if (LOG.isDebugEnabled()) {
LOG.debug("Could not find " + WebAppProxyServlet.PROXY_USER_COOKIE_NAME
+ " cookie, so user will not be set");
if (user == null) {
if (LOG.isDebugEnabled()) {
LOG.debug("Could not find "
+ WebAppProxyServlet.PROXY_USER_COOKIE_NAME
+ " cookie, so user will not be set");
}
chain.doFilter(req, resp);
} else {
AmIpPrincipal principal = new AmIpPrincipal(user);
ServletRequest requestWrapper = new AmIpServletRequestWrapper(httpReq,
principal);
chain.doFilter(requestWrapper, resp);
}
chain.doFilter(req, resp);
} else {
final AmIpPrincipal principal = new AmIpPrincipal(user);
ServletRequest requestWrapper = new AmIpServletRequestWrapper(httpReq,
principal);
chain.doFilter(requestWrapper, resp);
}
}

24
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java
View File

@ -155,7 +155,7 @@ public class TestWebAppProxyServlet {
URL emptyUrl = new URL("http://localhost:" + proxyPort + "/proxy");
HttpURLConnection emptyProxyConn = (HttpURLConnection) emptyUrl
.openConnection();
emptyProxyConn.connect();;
emptyProxyConn.connect();
assertEquals(HttpURLConnection.HTTP_NOT_FOUND, emptyProxyConn.getResponseCode());
// wrong url. Set wrong app ID
@ -176,6 +176,25 @@ public class TestWebAppProxyServlet {
assertEquals(HttpURLConnection.HTTP_OK, proxyConn.getResponseCode());
assertTrue(isResponseCookiePresent(
proxyConn, "checked_application_0_0000", "true"));
// test that redirection is squashed correctly
URL redirectUrl = new URL("http://localhost:" + proxyPort
+ "/proxy/redirect/application_00_0");
proxyConn = (HttpURLConnection) redirectUrl.openConnection();
proxyConn.setInstanceFollowRedirects(false);
proxyConn.connect();
assertEquals("The proxy returned an unexpected status code rather than"
+ "redirecting the connection (302)",
HttpURLConnection.HTTP_MOVED_TEMP, proxyConn.getResponseCode());
String expected =
WebAppUtils.getResolvedRMWebAppURLWithScheme(configuration)
+ "/cluster/failure/application_00_0";
String redirect = proxyConn.getHeaderField(ProxyUtils.LOCATION);
assertEquals("The proxy did not redirect the connection to the failure "
+ "page of the RM", expected, redirect);
// cannot found application 1: null
appReportFetcher.answer = 1;
proxyConn = (HttpURLConnection) url.openConnection();
@ -185,6 +204,7 @@ public class TestWebAppProxyServlet {
proxyConn.getResponseCode());
assertFalse(isResponseCookiePresent(
proxyConn, "checked_application_0_0000", "true"));
// cannot found application 2: ApplicationNotFoundException
appReportFetcher.answer = 4;
proxyConn = (HttpURLConnection) url.openConnection();
@ -194,6 +214,7 @@ public class TestWebAppProxyServlet {
proxyConn.getResponseCode());
assertFalse(isResponseCookiePresent(
proxyConn, "checked_application_0_0000", "true"));
// wrong user
appReportFetcher.answer = 2;
proxyConn = (HttpURLConnection) url.openConnection();
@ -203,6 +224,7 @@ public class TestWebAppProxyServlet {
assertTrue(s
.contains("to continue to an Application Master web interface owned by"));
assertTrue(s.contains("WARNING: The following page may not be safe!"));
//case if task has a not running status
appReportFetcher.answer = 3;
proxyConn = (HttpURLConnection) url.openConnection();

29
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilter.java
View File

@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.webproxy.amfilter;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.net.HttpURLConnection;
import java.util.*;
import java.util.concurrent.atomic.AtomicBoolean;
@ -147,8 +148,8 @@ public class TestAmFilter {
testFilter.init(config);
HttpServletResponseForTest response = new HttpServletResponseForTest();
// Test request should implements HttpServletRequest
// Test request should implements HttpServletRequest
ServletRequest failRequest = Mockito.mock(ServletRequest.class);
try {
testFilter.doFilter(failRequest, response, chain);
@ -159,22 +160,32 @@ public class TestAmFilter {
// request with HttpServletRequest
HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
Mockito.when(request.getRemoteAddr()).thenReturn("redirect");
Mockito.when(request.getRequestURI()).thenReturn("/redirect");
Mockito.when(request.getRemoteAddr()).thenReturn("nowhere");
Mockito.when(request.getRequestURI()).thenReturn("/app/application_00_0");
// address "redirect" is not in host list for non-proxy connection
testFilter.doFilter(request, response, chain);
// address "redirect" is not in host list
assertEquals(302, response.status);
assertEquals(HttpURLConnection.HTTP_MOVED_TEMP, response.status);
String redirect = response.getHeader(ProxyUtils.LOCATION);
assertEquals("http://bogus/redirect", redirect);
assertEquals("http://bogus/app/application_00_0", redirect);
// address "redirect" is not in host list for proxy connection
Mockito.when(request.getRequestURI()).thenReturn("/proxy/application_00_0");
testFilter.doFilter(request, response, chain);
assertEquals(HttpURLConnection.HTTP_MOVED_TEMP, response.status);
redirect = response.getHeader(ProxyUtils.LOCATION);
assertEquals("http://bogus/proxy/redirect/application_00_0", redirect);
// "127.0.0.1" contains in host list. Without cookie
Mockito.when(request.getRemoteAddr()).thenReturn("127.0.0.1");
testFilter.doFilter(request, response, chain);
assertTrue(doFilterRequest
.contains("javax.servlet.http.HttpServletRequest"));
// cookie added
Cookie[] cookies = new Cookie[1];
cookies[0] = new Cookie(WebAppProxyServlet.PROXY_USER_COOKIE_NAME, "user");
Cookie[] cookies = new Cookie[] {
new Cookie(WebAppProxyServlet.PROXY_USER_COOKIE_NAME, "user")
};
Mockito.when(request.getCookies()).thenReturn(cookies);
testFilter.doFilter(request, response, chain);