diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DeSelectFields.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DeSelectFields.java new file mode 100644 index 00000000000..258bbfa38dd --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DeSelectFields.java @@ -0,0 +1,127 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements.See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership.The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License.You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.webapp; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.yarn.webapp.BadRequestException; + +/** + * DeSelectFields make the /apps api more flexible. + * It can be used to strip off more fields if there's such use case in future. + * You can simply extend it via two steps: + *
1. add a DeSelectType enum with a string literals + *
2. write your logical based on + * the return of method contains(DeSelectType) + */ +public class DeSelectFields { + private static final Log LOG = + LogFactory.getLog(DeSelectFields.class.getName()); + + private final Set types; + + public DeSelectFields() { + this.types = new HashSet(); + } + + /** + * Initial DeSelectFields with unselected fields. + * @param unselectedFields a set of unselected field. + */ + public void initFields(Set unselectedFields) { + if (unselectedFields == null) { + return; + } + for (String field : unselectedFields) { + if (!field.trim().isEmpty()) { + String[] literalsArray = field.split(","); + for (String literals : literalsArray) { + if (literals != null && !literals.trim().isEmpty()) { + DeSelectType type = DeSelectType.obtainType(literals); + if (type == null) { + LOG.warn("Invalid deSelects string " + literals.trim()); + DeSelectType[] typeArray = DeSelectType.values(); + String allSuppportLiterals = Arrays.toString(typeArray); + throw new BadRequestException("Invalid deSelects string " + + literals.trim() + " specified. It should be one of " + + allSuppportLiterals); + } else { + this.types.add(type); + } + } + } + } + } + } + + /** + * Determine the deselect type should be handled or not. + * @param type deselected type + * @return true if the deselect type should be handled + */ + public boolean contains(DeSelectType type) { + return types.contains(type); + } + + /** + * Deselect field type, can be boost in future. + */ + public enum DeSelectType { + + /** + * RESOURCE_REQUESTS is the first + * supported type from YARN-6280. + */ + RESOURCE_REQUESTS("resourceRequests"); + + private final String literals; + + DeSelectType(String literals) { + this.literals = literals; + } + + /** + * use literals as toString. + * @return the literals of this type. + */ + @Override + public String toString() { + return literals; + } + + /** + * Obtain the DeSelectType by the literals given behind + * deSelects in URL. + *
e.g: deSelects="resourceRequests" + * @param literals e.g: resourceRequests + * @return DeSelectType e.g: DeSelectType.RESOURCE_REQUESTS + */ + public static DeSelectType obtainType(String literals) { + for (DeSelectType type : values()) { + if (type.literals.equalsIgnoreCase(literals)) { + return type; + } + } + return null; + } + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index 8227d982965..cbab0369754 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -426,7 +426,8 @@ public AppsInfo getApps(@Context HttpServletRequest hsr, @QueryParam("finishedTimeBegin") String finishBegin, @QueryParam("finishedTimeEnd") String finishEnd, @QueryParam("applicationTypes") Set applicationTypes, - @QueryParam("applicationTags") Set applicationTags) { + @QueryParam("applicationTags") Set applicationTags, + @QueryParam("deSelects") Set unselectedFields) { boolean checkCount = false; boolean checkStart = false; boolean checkEnd = false; @@ -582,8 +583,11 @@ public AppsInfo getApps(@Context HttpServletRequest hsr, } } - AppInfo app = new AppInfo(rm, rmapp, - hasAccess(rmapp, hsr), WebAppUtils.getHttpSchemePrefix(conf)); + DeSelectFields deSelectFields = new DeSelectFields(); + deSelectFields.initFields(unselectedFields); + + AppInfo app = new AppInfo(rm, rmapp, hasAccess(rmapp, hsr), + WebAppUtils.getHttpSchemePrefix(conf), deSelectFields); allApps.add(app); } return allApps; @@ -798,14 +802,20 @@ private static void countApp( @Path("/apps/{appid}") @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) public AppInfo getApp(@Context HttpServletRequest hsr, - @PathParam("appid") String appId) { + @PathParam("appid") String appId, + @QueryParam("deSelects") Set unselectedFields) { init(); ApplicationId id = WebAppUtils.parseApplicationId(recordFactory, appId); RMApp app = rm.getRMContext().getRMApps().get(id); if (app == null) { throw new NotFoundException("app with id: " + appId + " not found"); } - return new AppInfo(rm, app, hasAccess(app, hsr), hsr.getScheme() + "://"); + + DeSelectFields deSelectFields = new DeSelectFields(); + deSelectFields.initFields(unselectedFields); + + return new AppInfo(rm, app, hasAccess(app, hsr), hsr.getScheme() + "://", + deSelectFields); } @GET @@ -1473,7 +1483,7 @@ private boolean isStaticUser(UserGroupInformation callerUGI) { } /** - * Generates a new ApplicationId which is then sent to the client + * Generates a new ApplicationId which is then sent to the client. * * @param hsr * the servlet request @@ -1508,7 +1518,7 @@ public Response createNewApplication(@Context HttpServletRequest hsr) // get the new app id and submit app // set location header with new app location /** - * Function to submit an app to the RM + * Function to submit an app to the RM. * * @param newApp * structure containing information to construct the @@ -1763,8 +1773,7 @@ private UserGroupInformation createKerberosUserGroupInformation( + KerberosAuthenticationHandler.TYPE + ", got type " + authType; throw new YarnException(msg); } - if (hsr - .getAttribute(DelegationTokenAuthenticationHandler.DELEGATION_TOKEN_UGI_ATTRIBUTE) != null) { + if (hsr.getAttribute(DelegationTokenAuthenticationHandler.DELEGATION_TOKEN_UGI_ATTRIBUTE) != null) { String msg = "Delegation token operations cannot be carried out using delegation" + " token authentication."; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java index b27f23e432e..6f7763e81e8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java @@ -44,6 +44,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.DeSelectFields; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.DeSelectFields.DeSelectType; import org.apache.hadoop.yarn.util.Times; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; @@ -123,9 +125,13 @@ public class AppInfo { public AppInfo() { } // JAXB needs this - @SuppressWarnings({ "rawtypes", "unchecked" }) public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess, String schemePrefix) { + this(rm, app, hasAccess, schemePrefix, new DeSelectFields()); + } + + public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess, + String schemePrefix, DeSelectFields deSelects) { this.schemePrefix = schemePrefix; if (app != null) { String trackingUrl = app.getTrackingUrl(); @@ -201,13 +207,18 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess, clusterUsagePercentage = resourceReport.getClusterUsagePercentage(); } - List resourceRequestsRaw = rm.getRMContext() + /* When the deSelects parameter contains "resourceRequests", + it skips returning massive ResourceRequest objects and vice versa. + Default behavior is no skipping. (YARN-6280) + */ + if (!deSelects.contains(DeSelectType.RESOURCE_REQUESTS)) { + List resourceRequestsRaw = rm.getRMContext() .getScheduler() .getPendingResourceRequestsForAttempt(attempt.getAppAttemptId()); - - if (resourceRequestsRaw != null) { - for (ResourceRequest req : resourceRequestsRaw) { - resourceRequests.add(new ResourceRequestInfo(req)); + if (resourceRequestsRaw != null) { + for (ResourceRequest req : resourceRequestsRaw) { + resourceRequests.add(new ResourceRequestInfo(req)); + } } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java index 50a4849e921..7ec62a02769 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java @@ -443,7 +443,7 @@ public void verifyClusterMetricsJSON(JSONObject json) throws JSONException, clusterinfo.getInt("totalMB"), clusterinfo.getInt("totalNodes"), clusterinfo.getInt("lostNodes"), clusterinfo.getInt("unhealthyNodes"), clusterinfo.getInt("decommissionedNodes"), - clusterinfo.getInt("rebootedNodes"),clusterinfo.getInt("activeNodes"), + clusterinfo.getInt("rebootedNodes"), clusterinfo.getInt("activeNodes"), clusterinfo.getInt("shutdownNodes")); } @@ -652,12 +652,12 @@ public void testAppsRace() throws Exception { // verify we don't get any apps when querying HttpServletRequest mockHsr = mock(HttpServletRequest.class); AppsInfo appsInfo = webSvc.getApps(mockHsr, null, emptySet, null, - null, null, null, null, null, null, null, emptySet, emptySet); + null, null, null, null, null, null, null, emptySet, emptySet, null); assertTrue(appsInfo.getApps().isEmpty()); // verify we don't get an NPE when specifying a final status query appsInfo = webSvc.getApps(mockHsr, null, emptySet, "FAILED", - null, null, null, null, null, null, null, emptySet, emptySet); + null, null, null, null, null, null, null, emptySet, emptySet, null); assertTrue(appsInfo.getApps().isEmpty()); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index e63494c3b2f..ba9a653e03a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -1015,6 +1015,68 @@ public void testAppsQueryAppTypes() throws JSONException, Exception { rm.stop(); } + @Test + public void testAppsQueryWithInvaildDeselects() + throws JSONException, Exception { + try { + rm.start(); + MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); + rm.submitApp(CONTAINER_MB); + amNodeManager.nodeHeartbeat(true); + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("apps").queryParam("deSelects", "INVALIED_deSelectsParam") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject msg = response.getEntity(JSONObject.class); + JSONObject exception = msg.getJSONObject("RemoteException"); + assertEquals("incorrect number of elements", 3, exception.length()); + String message = exception.getString("message"); + String type = exception.getString("exception"); + String classname = exception.getString("javaClassName"); + WebServicesTestUtils.checkStringContains("exception message", + "java.lang.Exception: Invalid deSelects string" + + " INVALIED_deSelectsParam " + "specified. It should be one of", + message); + WebServicesTestUtils.checkStringEqual("exception type", + "BadRequestException", type); + WebServicesTestUtils.checkStringEqual("exception classname", + "org.apache.hadoop.yarn.webapp.BadRequestException", classname); + } finally { + rm.stop(); + } + } + + @Test + public void testAppsQueryWithDeselects() + throws JSONException, Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); + rm.submitApp(CONTAINER_MB); + amNodeManager.nodeHeartbeat(true); + WebResource r = resource(); + + MultivaluedMapImpl params = new MultivaluedMapImpl(); + params.add("deSelects", + DeSelectFields.DeSelectType.RESOURCE_REQUESTS.toString()); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("apps").queryParams(params) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + + JSONObject json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + JSONObject apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + JSONArray array = apps.getJSONArray("app"); + assertEquals("incorrect number of elements", 1, array.length()); + JSONObject app = array.getJSONObject(0); + assertTrue("resource requests shouldn't exits", + !app.has("resourceRequests")); + rm.stop(); + } + @Test public void testAppStatistics() throws JSONException, Exception { try { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md index c119651c741..7c1f4188651 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md @@ -1330,6 +1330,7 @@ Multiple parameters can be specified for GET operations. The started and finishe * finishedTimeEnd - applications with finish time ending with this time, specified in ms since epoch * applicationTypes - applications matching the given application types, specified as a comma-separated list. * applicationTags - applications matching any of the given application tags, specified as a comma-separated list. + * deSelects - a generic fields which will be skipped in the result. ### Elements of the *apps* (Applications) object @@ -1339,6 +1340,21 @@ When you make a request for the list of applications, the information will be re |:---- |:---- |:---- | | app | array of app objects(JSON)/zero or more application objects(XML) | The collection of application objects | +###Elements of the *deSelects* parameter + +Help requesters who don't need certain information to reduce the overhead. + +Current supported items: + +| Item | Data Type | Description | +|:---- |:---- |:---- | +| resouceRequests | comma separated string | Skip resource requests of application in return | + +e.g: + + * http:///ws/v1/cluster/apps?deSelects=resouceRequests + + ### Response Examples **JSON response** @@ -1396,7 +1412,47 @@ Response Body: "logAggregationStatus": "DISABLED", "unmanagedApplication": false, "appNodeLabelExpression": "", - "amNodeLabelExpression": "" + "amNodeLabelExpression": "", + "resourceRequests": [ + { + "capability": { + "memory": 4096, + "virtualCores": 1 + }, + "nodeLabelExpression": "", + "numContainers": 0, + "priority": { + "priority": 0 + }, + "relaxLocality": true, + "resourceName": "*" + }, + { + "capability": { + "memory": 4096, + "virtualCores": 1 + }, + "nodeLabelExpression": "", + "numContainers": 0, + "priority": { + "priority": 20 + }, + "relaxLocality": true, + "resourceName": "host1.domain.com" + }, + { + "capability": { + "memory": 4096, + "virtualCores": 1 + }, + "nodeLabelExpression": "", + "numContainers": 0, + "priority": { + "priority": 20 + }, + "relaxLocality": true, + "resourceName": "host2.domain.com" + }] }, { "id": "application_1476912658570_0001", @@ -1432,7 +1488,47 @@ Response Body: "logAggregationStatus": "DISABLED", "unmanagedApplication": false, "appNodeLabelExpression": "", - "amNodeLabelExpression": "" + "amNodeLabelExpression": "", + "resourceRequests": [ + { + "capability": { + "memory": 4096, + "virtualCores": 1 + }, + "nodeLabelExpression": "", + "numContainers": 0, + "priority": { + "priority": 0 + }, + "relaxLocality": true, + "resourceName": "*" + }, + { + "capability": { + "memory": 4096, + "virtualCores": 1 + }, + "nodeLabelExpression": "", + "numContainers": 0, + "priority": { + "priority": 20 + }, + "relaxLocality": true, + "resourceName": "host3.domain.com" + }, + { + "capability": { + "memory": 4096, + "virtualCores": 1 + }, + "nodeLabelExpression": "", + "numContainers": 0, + "priority": { + "priority": 20 + }, + "relaxLocality": true, + "resourceName": "host4.domain.com" + }] } ] } @@ -1493,6 +1589,45 @@ Response Body: false + + + 4096 + 1 + + + 0 + + 0 + + true + * + + + + 4096 + 1 + + + 0 + + 20 + + true + host1.domain.com + + + + 4096 + 1 + + + 0 + + 20 + + true + host2.domain.com + application_1476912658570_0001 @@ -1529,6 +1664,45 @@ Response Body: false + + + 4096 + 1 + + + 0 + + 0 + + true + * + + + + 4096 + 1 + + + 0 + + 20 + + true + host1.domain.com + + + + 4096 + 1 + + + 0 + + 20 + + true + host2.domain.com + ```