From 3ac3e5f89748d4ac19631784716918ea88215c54 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Fri, 14 Mar 2014 02:40:28 +0000 Subject: [PATCH] YARN-1658. Modified web-app framework to let standby RMs redirect web-service calls to the active RM. Contributed by Cindy Li. svn merge --ignore-ancestry -c 1577408 ../../trunk/ git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1577409 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 ++ .../hadoop/yarn/client/TestRMFailover.java | 7 +++ .../apache/hadoop/yarn/webapp/Dispatcher.java | 4 +- .../org/apache/hadoop/yarn/webapp/Router.java | 2 +- .../org/apache/hadoop/yarn/webapp/WebApp.java | 16 +++++-- .../resourcemanager/webapp/RMWebApp.java | 18 +++----- ...{RMDispatcher.java => RMWebAppFilter.java} | 44 ++++++++++--------- 7 files changed, 55 insertions(+), 39 deletions(-) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/{RMDispatcher.java => RMWebAppFilter.java} (66%) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 5cb3a763666..19699ae5ec3 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -276,6 +276,9 @@ Release 2.4.0 - UNRELEASED YARN-1771. Reduce the number of NameNode operations during localization of public resources using a cache. (Sangjin Lee via cdouglas) + YARN-1658. Modified web-app framework to let standby RMs redirect + web-service calls to the active RM. (Cindy Li via vinodkv) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java index 3515a53fe00..1269f89ea87 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java @@ -271,6 +271,12 @@ public class TestRMFailover extends ClientBaseWithFixes { header = getHeader("Refresh", rm2Url + "/cluster/cluster"); assertEquals(null, header); + header = getHeader("Refresh", rm2Url + "/ws/v1/cluster/info"); + assertEquals(null, header); + + header = getHeader("Refresh", rm2Url + "/ws/v1/cluster/apps"); + assertTrue(header.contains("; url=" + rm1Url)); + // Due to the limitation of MiniYARNCluster and dispatcher is a singleton, // we couldn't add the test case after explicitFailover(); } @@ -286,4 +292,5 @@ public class TestRMFailover extends ClientBaseWithFixes { } return fieldHeader; } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/Dispatcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/Dispatcher.java index a0500036952..66dd21bbacc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/Dispatcher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/Dispatcher.java @@ -57,11 +57,11 @@ public class Dispatcher extends HttpServlet { private transient final Injector injector; private transient final Router router; - protected transient final WebApp webApp; + private transient final WebApp webApp; private volatile boolean devMode = false; @Inject - protected Dispatcher(WebApp webApp, Injector injector, Router router) { + Dispatcher(WebApp webApp, Injector injector, Router router) { this.webApp = webApp; this.injector = injector; this.router = router; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/Router.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/Router.java index 9d546593bec..cf78818ede5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/Router.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/Router.java @@ -44,7 +44,7 @@ import com.google.common.collect.Maps; * Manages path info to controller#action routing. */ @InterfaceAudience.LimitedPrivate({"YARN", "MapReduce"}) -public class Router { +class Router { static final Logger LOG = LoggerFactory.getLogger(Router.class); static final ImmutableList EMPTY_LIST = ImmutableList.of(); static final CharMatcher SLASH = CharMatcher.is('/'); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java index 171b36a3494..2c21d1b3129 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java @@ -122,6 +122,10 @@ public abstract class WebApp extends ServletModule { public String name() { return this.name; } + public String wsName() { + return this.wsName; + } + void addServePathSpec(String path) { this.servePathSpecs.add(path); } public String[] getServePathSpecs() { @@ -134,7 +138,7 @@ public abstract class WebApp extends ServletModule { * more easily differentiate the different webapps. * @param path the path to redirect to */ - protected void setRedirectPath(String path) { + void setRedirectPath(String path) { this.redirectPath = path; } @@ -160,10 +164,10 @@ public abstract class WebApp extends ServletModule { serve(path).with(Dispatcher.class); } - configureRSServlets(); + configureWebAppServlets(); } - protected void configureRSServlets() { + protected void configureWebAppServlets() { // Add in the web services filters/serves if app has them. // Using Jersey/guice integration module. If user has web services // they must have also bound a default one in their webapp code. @@ -182,9 +186,12 @@ public abstract class WebApp extends ServletModule { params.put(FeaturesAndProperties.FEATURE_XMLROOTELEMENT_PROCESSING, "true"); params.put(ResourceConfig.PROPERTY_CONTAINER_REQUEST_FILTERS, GZIPContentEncodingFilter.class.getName()); params.put(ResourceConfig.PROPERTY_CONTAINER_RESPONSE_FILTERS, GZIPContentEncodingFilter.class.getName()); - filter("/*").through(GuiceContainer.class, params); + filter("/*").through(getWebAppFilterClass(), params); } + } + protected Class getWebAppFilterClass() { + return GuiceContainer.class; } /** @@ -274,4 +281,5 @@ public abstract class WebApp extends ServletModule { } public abstract void setup(); + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java index fe4a5923614..1264cd299fa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java @@ -29,11 +29,12 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMHAUtils; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; -import org.apache.hadoop.yarn.webapp.Dispatcher; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.WebApp; import org.apache.hadoop.yarn.webapp.YarnWebParams; +import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; + /** * The RM webapp */ @@ -51,6 +52,8 @@ public class RMWebApp extends WebApp implements YarnWebParams { bind(JAXBContextResolver.class); bind(RMWebServices.class); bind(GenericExceptionHandler.class); + bind(RMWebApp.class).toInstance(this); + if (rm != null) { bind(ResourceManager.class).toInstance(rm); bind(RMContext.class).toInstance(rm.getRMContext()); @@ -68,17 +71,8 @@ public class RMWebApp extends WebApp implements YarnWebParams { } @Override - public void configureServlets() { - setup(); - - serve("/").with(RMDispatcher.class); - serve("/__stop").with(Dispatcher.class); - - for (String path : super.getServePathSpecs()) { - serve(path).with(RMDispatcher.class); - } - - configureRSServlets(); + protected Class getWebAppFilterClass() { + return RMWebAppFilter.class; } public void checkIfStandbyRM() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMDispatcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java similarity index 66% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMDispatcher.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java index 5691a11970e..fd472dd3906 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMDispatcher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java @@ -21,59 +21,63 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp; import java.io.IOException; import java.io.PrintWriter; +import javax.inject.Inject; +import javax.inject.Singleton; +import javax.servlet.FilterChain; import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.http.HtmlQuoting; -import org.apache.hadoop.yarn.webapp.Dispatcher; -import org.apache.hadoop.yarn.webapp.Router; -import org.apache.hadoop.yarn.webapp.WebApp; -import com.google.inject.Inject; import com.google.inject.Injector; -import com.google.inject.Singleton; +import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; -@InterfaceAudience.LimitedPrivate({ "YARN", "MapReduce" }) @Singleton -public class RMDispatcher extends Dispatcher { +public class RMWebAppFilter extends GuiceContainer { + private Injector injector; /** - * + * */ private static final long serialVersionUID = 1L; @Inject - RMDispatcher(WebApp webApp, Injector injector, Router router) { - super(webApp, injector, router); + public RMWebAppFilter(Injector injector) { + super(injector); + this.injector=injector; } @Override - public void service(HttpServletRequest req, HttpServletResponse res) - throws ServletException, IOException { - res.setCharacterEncoding("UTF-8"); - String uri = HtmlQuoting.quoteHtmlChars(req.getRequestURI()); + public void doFilter(HttpServletRequest request, + HttpServletResponse response, FilterChain chain) throws IOException, + ServletException { + response.setCharacterEncoding("UTF-8"); + String uri = HtmlQuoting.quoteHtmlChars(request.getRequestURI()); if (uri == null) { uri = "/"; } - - RMWebApp rmWebApp = (RMWebApp) webApp; + RMWebApp rmWebApp = injector.getInstance(RMWebApp.class); rmWebApp.checkIfStandbyRM(); if (rmWebApp.isStandby() + && !uri.equals("/" + rmWebApp.wsName() + "/v1/cluster/info") && !uri.equals("/" + rmWebApp.name() + "/cluster")) { String redirectPath = rmWebApp.getRedirectPath() + uri; + if (redirectPath != null && !redirectPath.isEmpty()) { String redirectMsg = "This is standby RM. Redirecting to the current active RM: " + redirectPath; - res.addHeader("Refresh", "3; url=" + redirectPath); - PrintWriter out = res.getWriter(); + response.addHeader("Refresh", "3; url=" + redirectPath); + PrintWriter out = response.getWriter(); out.println(redirectMsg); return; } } - super.service(req, res); + + super.doFilter(request, response, chain); + } + }