From 1f0a71a92bedc7668984e80a3195efe892b5d0ca Mon Sep 17 00:00:00 2001 From: slfan1989 <55643692+slfan1989@users.noreply.github.com> Date: Wed, 3 Aug 2022 05:07:09 +0800 Subject: [PATCH] YARN-8973. [Router] Add missing methods in RMWebProtocol. (#4664) --- .../webapp/RMWebServiceProtocol.java | 28 +++++++++++++++++++ .../webapp/DefaultRequestInterceptorREST.java | 20 +++++++++++++ .../webapp/FederationInterceptorREST.java | 14 ++++++++++ .../router/webapp/RouterWebServices.java | 28 +++++++++++++++++++ .../webapp/BaseRouterWebServicesTest.java | 12 ++++++++ .../webapp/MockRESTRequestInterceptor.java | 13 +++++++++ .../PassThroughRESTRequestInterceptor.java | 14 ++++++++++ .../router/webapp/TestRouterWebServices.java | 6 ++++ 8 files changed, 135 insertions(+) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java index 41fc4ea8709..6c0309969ee 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java @@ -62,6 +62,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceOptionInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.BulkActivitiesInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo; +import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; /** *

@@ -745,4 +746,31 @@ public interface RMWebServiceProtocol { */ Response signalToContainer(String containerId, String command, HttpServletRequest req) throws AuthorizationException; + + /** + * This method updates the Scheduler configuration, and it is reachable by + * using {@link RMWSConsts#SCHEDULER_CONF}. + * + * @param mutationInfo th information for making scheduler configuration + * changes (supports adding, removing, or updating a queue, as well + * as global scheduler conf changes) + * @param hsr the servlet request + * @return Response containing the status code + * @throws AuthorizationException if the user is not authorized to invoke this + * method + * @throws InterruptedException if interrupted + */ + Response updateSchedulerConfiguration(SchedConfUpdateInfo mutationInfo, + HttpServletRequest hsr) throws AuthorizationException, InterruptedException; + + /** + * This method retrieves all the Scheduler configuration, and it is reachable + * by using {@link RMWSConsts#SCHEDULER_CONF}. + * + * @param hsr the servlet request + * @return Response containing the status code + * @throws AuthorizationException if the user is not authorized to invoke this + * method. + */ + Response getSchedulerConfiguration(HttpServletRequest hsr) throws AuthorizationException; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/DefaultRequestInterceptorREST.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/DefaultRequestInterceptorREST.java index 21fd2be8546..c07056ce8a1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/DefaultRequestInterceptorREST.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/DefaultRequestInterceptorREST.java @@ -66,6 +66,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInf import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo; import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo; import org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo; +import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; /** @@ -565,6 +566,25 @@ public class DefaultRequestInterceptorREST null, null, getConf(), client); } + @Override + public Response updateSchedulerConfiguration(SchedConfUpdateInfo mutationInfo, + HttpServletRequest req) + throws AuthorizationException, InterruptedException { + return RouterWebServiceUtil.genericForward(webAppAddress, req, + Response.class, HTTPMethods.PUT, + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.SCHEDULER_CONF, + mutationInfo, null, getConf(), client); + } + + @Override + public Response getSchedulerConfiguration(HttpServletRequest req) + throws AuthorizationException { + return RouterWebServiceUtil.genericForward(webAppAddress, req, + Response.class, HTTPMethods.GET, + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.SCHEDULER_CONF, + null, null, getConf(), client); + } + @Override public void setNextInterceptor(RESTRequestInterceptor next) { throw new YarnRuntimeException("setNextInterceptor is being called on " diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java index bd5286021e1..2bc6d60b4e8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java @@ -101,6 +101,7 @@ import org.apache.hadoop.yarn.server.router.clientrm.ClientMethod; import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo; import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo; import org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo; +import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; import org.apache.hadoop.yarn.util.Clock; import org.apache.hadoop.yarn.util.MonotonicClock; import org.apache.hadoop.yarn.webapp.NotFoundException; @@ -1417,6 +1418,19 @@ public class FederationInterceptorREST extends AbstractRESTRequestInterceptor { throw new NotImplementedException("Code is not implemented"); } + @Override + public Response updateSchedulerConfiguration(SchedConfUpdateInfo mutationInfo, + HttpServletRequest hsr) + throws AuthorizationException, InterruptedException { + throw new NotImplementedException("Code is not implemented"); + } + + @Override + public Response getSchedulerConfiguration(HttpServletRequest hsr) + throws AuthorizationException { + throw new NotImplementedException("Code is not implemented"); + } + @Override public void setNextInterceptor(RESTRequestInterceptor next) { throw new YarnRuntimeException("setNextInterceptor is being called on " diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServices.java index 6beafc8183f..8faf0619f3f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServices.java @@ -89,6 +89,7 @@ import org.apache.hadoop.yarn.server.router.Router; import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo; import org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo; import org.apache.hadoop.yarn.util.LRUCacheHashMap; +import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -956,6 +957,33 @@ public class RouterWebServices implements RMWebServiceProtocol { appAttemptId, containerId); } + @PUT + @Path(RMWSConsts.SCHEDULER_CONF) + @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, + MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 }) + @Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + @Override + public Response updateSchedulerConfiguration(SchedConfUpdateInfo mutationInfo, + HttpServletRequest hsr) + throws AuthorizationException, InterruptedException { + init(); + RequestInterceptorChainWrapper pipeline = getInterceptorChain(hsr); + return pipeline.getRootInterceptor() + .updateSchedulerConfiguration(mutationInfo, hsr); + } + + @GET + @Path(RMWSConsts.SCHEDULER_CONF) + @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, + MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 }) + @Override + public Response getSchedulerConfiguration(HttpServletRequest hsr) + throws AuthorizationException { + init(); + RequestInterceptorChainWrapper pipeline = getInterceptorChain(hsr); + return pipeline.getRootInterceptor().getSchedulerConfiguration(hsr); + } + @VisibleForTesting protected void setResponse(HttpServletResponse response) { this.response = response; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/BaseRouterWebServicesTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/BaseRouterWebServicesTest.java index 025f44d4bd2..a4294bc3610 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/BaseRouterWebServicesTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/BaseRouterWebServicesTest.java @@ -395,4 +395,16 @@ public abstract class BaseRouterWebServicesTest { when(request.getRemoteUser()).thenReturn(user); return request; } + + protected Response updateSchedulerConfiguration(String user) + throws IOException, InterruptedException { + return routerWebService.updateSchedulerConfiguration(null, + createHttpServletRequest(user)); + } + + protected Response getSchedulerConfiguration(String user) + throws IOException, InterruptedException { + return routerWebService. + getSchedulerConfiguration(createHttpServletRequest(user)); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockRESTRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockRESTRequestInterceptor.java index 68b8db6f334..5951676a6d8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockRESTRequestInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockRESTRequestInterceptor.java @@ -62,6 +62,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInf import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo; import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo; import org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo; +import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; /** * This class mocks the RESTRequestInterceptor. @@ -373,4 +374,16 @@ public class MockRESTRequestInterceptor extends AbstractRESTRequestInterceptor { HttpServletRequest req) { return Response.status(Status.OK).build(); } + + @Override + public Response updateSchedulerConfiguration(SchedConfUpdateInfo mutationInfo, + HttpServletRequest hsr) throws AuthorizationException, InterruptedException { + return Response.status(Status.OK).build(); + } + + @Override + public Response getSchedulerConfiguration(HttpServletRequest hsr) + throws AuthorizationException { + return Response.status(Status.OK).build(); + } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/PassThroughRESTRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/PassThroughRESTRequestInterceptor.java index 219020f2bbd..84a6de3205f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/PassThroughRESTRequestInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/PassThroughRESTRequestInterceptor.java @@ -60,6 +60,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInf import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo; import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo; import org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo; +import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; /** * Mock interceptor that does not do anything other than forwarding it to the @@ -379,4 +380,17 @@ public class PassThroughRESTRequestInterceptor String command, HttpServletRequest req) throws AuthorizationException { return getNextInterceptor().signalToContainer(containerId, command, req); } + + @Override + public Response updateSchedulerConfiguration(SchedConfUpdateInfo mutationInfo, + HttpServletRequest hsr) + throws AuthorizationException, InterruptedException { + return getNextInterceptor().updateSchedulerConfiguration(mutationInfo, hsr); + } + + @Override + public Response getSchedulerConfiguration(HttpServletRequest hsr) + throws AuthorizationException { + return getNextInterceptor().getSchedulerConfiguration(hsr); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServices.java index 7491cbc2a9f..5bf8db04b06 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServices.java @@ -200,6 +200,12 @@ public class TestRouterWebServices extends BaseRouterWebServicesTest { ContainerInfo containerInfo = getContainer(user); Assert.assertNotNull(containerInfo); + + Response response19 = updateSchedulerConfiguration(user); + Assert.assertNotNull(response19); + + Response response20 = getSchedulerConfiguration(user); + Assert.assertNotNull(response20); } /**