Apache/hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-4092. Fixed UI redirection to print useful messages when both RMs are in standby mode. Contributed by Xuan Gong

Browse Source 
(cherry picked from commit a3fd2ccc869dfc1f04d1cf0a8678d4d90a43a80f)
(cherry picked from commit 48f5161cd5d4c2f4e385b253a5bea066b2e23b9e)
This commit is contained in:
Jian He 2015-08-31 17:33:24 -07:00
parent dce4ef20f8
commit f44ed4f4b0
4 changed files with 117 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/CHANGES.txt
View File

@ -17,6 +17,9 @@ Release 2.7.2 - UNRELEASED
YARN-3978. Configurably turn off the saving of container info in Generic AHS
(Eric Payne via jeagles)
YARN-4092. Fixed UI redirection to print useful messages when both RMs are
in standby mode. (Xuan Gong via jianhe)
OPTIMIZATIONS
BUG FIXES

27
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java
View File

@ -27,6 +27,7 @@
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URL;
import javax.servlet.http.HttpServletResponse;
import org.apache.commons.logging.Log;
@ -45,6 +46,7 @@
import org.apache.hadoop.yarn.server.resourcemanager.AdminService;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServer;
import org.apache.hadoop.yarn.webapp.YarnWebParams;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@ -285,6 +287,7 @@ public void testRMWebAppRedirect() throws YarnException,
getAdminService(0).transitionToActive(req);
String rm1Url = "http://0.0.0.0:18088";
String rm2Url = "http://0.0.0.0:28088";
String redirectURL = getRedirectURL(rm2Url);
// if uri is null, RMWebAppFilter will append a slash at the trail of the redirection url
assertEquals(redirectURL,rm1Url+"/");
@ -324,6 +327,17 @@ public void testRMWebAppRedirect() throws YarnException,
redirectURL = getRedirectURL(rm2Url + "/proxy/" + fakeAppId);
assertNull(redirectURL);
// transit the active RM to standby
// Both of RMs are in standby mode
getAdminService(0).transitionToStandby(req);
// RM2 is expected to send the httpRequest to itself.
// The Header Field: Refresh is expected to be set.
redirectURL = getRefreshURL(rm2Url);
assertTrue(redirectURL != null
&& redirectURL.contains(YarnWebParams.NEXT_REFRESH_INTERVAL)
&& redirectURL.contains(rm2Url));
}
// set up http connection with the given url and get the redirection url from the response
@ -343,4 +357,17 @@ static String getRedirectURL(String url) {
return redirectUrl;
}
static String getRefreshURL(String url) {
String redirectUrl = null;
try {
HttpURLConnection conn = (HttpURLConnection) new URL(url).openConnection();
// do not automatically follow the redirection
// otherwise we get too many redirections exception
conn.setInstanceFollowRedirects(false);
redirectUrl = conn.getHeaderField("Refresh");
} catch (Exception e) {
// throw new RuntimeException(e);
}
return redirectUrl;
}
}

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
View File

@ -37,4 +37,5 @@ public interface YarnWebParams {
String NODE_STATE = "node.state";
String NODE_LABEL = "node.label";
String WEB_UI_TYPE = "web.ui.type";
String NEXT_REFRESH_INTERVAL = "next.fresh.interval";
}

90
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java
View File

@ -20,6 +20,10 @@
import java.io.IOException;
import java.io.PrintWriter;
import java.net.InetSocketAddress;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Random;
import java.util.Set;
import javax.inject.Inject;
@ -29,8 +33,11 @@
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.http.HtmlQuoting;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils;
import org.apache.hadoop.yarn.webapp.YarnWebParams;
import com.google.common.collect.Sets;
import com.google.inject.Injector;
@ -48,11 +55,26 @@ public class RMWebAppFilter extends GuiceContainer {
// define a set of URIs which do not need to do redirection
private static final Set<String> NON_REDIRECTED_URIS = Sets.newHashSet(
"/conf", "/stacks", "/logLevel", "/logs");
private String path;
private static final int BASIC_SLEEP_TIME = 5;
private static final int MAX_SLEEP_TIME = 5 * 60;
@Inject
public RMWebAppFilter(Injector injector) {
public RMWebAppFilter(Injector injector, Configuration conf) {
super(injector);
this.injector=injector;
InetSocketAddress sock = YarnConfiguration.useHttps(conf)
? conf.getSocketAddr(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS,
YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_ADDRESS,
YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_PORT)
: conf.getSocketAddr(YarnConfiguration.RM_WEBAPP_ADDRESS,
YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS,
YarnConfiguration.DEFAULT_RM_WEBAPP_PORT);
path = sock.getHostName() + ":" + Integer.toString(sock.getPort());
path = YarnConfiguration.useHttps(conf)
? "https://" + path
: "http://" + path;
}
@Override
@ -69,9 +91,11 @@ public void doFilter(HttpServletRequest request,
rmWebApp.checkIfStandbyRM();
if (rmWebApp.isStandby()
&& shouldRedirect(rmWebApp, uri)) {
String redirectPath = rmWebApp.getRedirectPath() + uri;
String redirectPath = rmWebApp.getRedirectPath();
if (redirectPath != null && !redirectPath.isEmpty()) {
redirectPath += uri;
String redirectMsg =
"This is standby RM. The redirect url is: " + redirectPath;
PrintWriter out = response.getWriter();
@ -79,11 +103,40 @@ && shouldRedirect(rmWebApp, uri)) {
response.setHeader("Location", redirectPath);
response.setStatus(HttpServletResponse.SC_TEMPORARY_REDIRECT);
return;
} else {
boolean doRetry = true;
String retryIntervalStr =
request.getParameter(YarnWebParams.NEXT_REFRESH_INTERVAL);
int retryInterval = 0;
if (retryIntervalStr != null) {
try {
retryInterval = Integer.parseInt(retryIntervalStr.trim());
} catch (NumberFormatException ex) {
doRetry = false;
}
}
int next = calculateExponentialTime(retryInterval);
String redirectUrl =
appendOrReplaceParamter(path + uri,
YarnWebParams.NEXT_REFRESH_INTERVAL + "=" + (retryInterval + 1));
if (redirectUrl == null || next > MAX_SLEEP_TIME) {
doRetry = false;
}
String redirectMsg =
doRetry ? "Can not find any active RM. Will retry in next " + next
+ " seconds." : "There is no active RM right now.";
PrintWriter out = response.getWriter();
out.println(redirectMsg);
if (doRetry) {
response.setHeader("Refresh", next + ";url=" + redirectUrl);
response.setStatus(HttpServletResponse.SC_TEMPORARY_REDIRECT);
}
}
return;
}
super.doFilter(request, response, chain);
}
private boolean shouldRedirect(RMWebApp rmWebApp, String uri) {
@ -92,4 +145,33 @@ private boolean shouldRedirect(RMWebApp rmWebApp, String uri) {
&& !uri.startsWith(ProxyUriUtils.PROXY_BASE)
&& !NON_REDIRECTED_URIS.contains(uri);
}
}
private String appendOrReplaceParamter(String uri, String newQuery) {
if (uri.contains(YarnWebParams.NEXT_REFRESH_INTERVAL + "=")) {
return uri.replaceAll(YarnWebParams.NEXT_REFRESH_INTERVAL + "=[^&]+",
newQuery);
}
try {
URI oldUri = new URI(uri);
String appendQuery = oldUri.getQuery();
if (appendQuery == null) {
appendQuery = newQuery;
} else {
appendQuery += "&" + newQuery;
}
URI newUri =
new URI(oldUri.getScheme(), oldUri.getAuthority(), oldUri.getPath(),
appendQuery, oldUri.getFragment());
return newUri.toString();
} catch (URISyntaxException e) {
return null;
}
}
private static int calculateExponentialTime(int retries) {
long baseTime = BASIC_SLEEP_TIME * (1L << retries);
return (int) (baseTime * ((new Random()).nextDouble() + 0.5));
}
}