From 1c133ffb72094a832b1d6c4b641821c6e2a7f9b2 Mon Sep 17 00:00:00 2001 From: Hitesh Shah Date: Wed, 17 Jul 2013 21:42:24 +0000 Subject: [PATCH] YARN-865. RM webservices can't query based on application Types. Contributed by Xuan Gong. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1504288 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../resourcemanager/webapp/RMWebServices.java | 31 ++- .../webapp/TestRMWebServicesApps.java | 185 ++++++++++++++++++ .../src/site/apt/ResourceManagerRest.apt.vm | 1 + 4 files changed, 219 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 014456659ad..5a052d4e42c 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -503,6 +503,9 @@ Release 2.1.0-beta - 2013-07-02 YARN-922. Change FileSystemRMStateStore to use directories (Jian He via bikas) + YARN-865. RM webservices can't query based on application Types. (Xuan Gong + via hitesh) + OPTIMIZATIONS YARN-512. Log aggregation root directory check is more expensive than it diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index d17c8804356..12213a3e721 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -21,6 +21,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp; import java.io.IOException; import java.util.Collection; import java.util.EnumSet; +import java.util.HashSet; +import java.util.Set; import java.util.concurrent.ConcurrentMap; import javax.servlet.http.HttpServletRequest; @@ -231,11 +233,13 @@ public class RMWebServices { @QueryParam("startedTimeBegin") String startedBegin, @QueryParam("startedTimeEnd") String startedEnd, @QueryParam("finishedTimeBegin") String finishBegin, - @QueryParam("finishedTimeEnd") String finishEnd) { + @QueryParam("finishedTimeEnd") String finishEnd, + @QueryParam("applicationTypes") Set applicationTypes) { long num = 0; boolean checkCount = false; boolean checkStart = false; boolean checkEnd = false; + boolean checkAppTypes = false; long countNum = 0; // set values suitable in case both of begin/end not specified @@ -291,6 +295,27 @@ public class RMWebServices { "finishTimeEnd must be greater than finishTimeBegin"); } + Set appTypes = new HashSet(); + if (!applicationTypes.isEmpty()) { + for (String applicationType : applicationTypes) { + if (applicationType != null && !applicationType.trim().isEmpty()) { + if (applicationType.indexOf(",") == -1) { + appTypes.add(applicationType.trim()); + } else { + String[] types = applicationType.split(","); + for (String type : types) { + if (!type.trim().isEmpty()) { + appTypes.add(type.trim()); + } + } + } + } + } + } + if (!appTypes.isEmpty()) { + checkAppTypes = true; + } + final ConcurrentMap apps = rm.getRMContext() .getRMApps(); AppsInfo allApps = new AppsInfo(); @@ -332,6 +357,10 @@ public class RMWebServices { continue; } } + if (checkAppTypes + && !appTypes.contains(rmapp.getApplicationType())) { + continue; + } if (checkStart && (rmapp.getStartTime() < sBegin || rmapp.getStartTime() > sEnd)) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index deb53eb7a66..52f72d8692e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -637,6 +637,191 @@ public class TestRMWebServicesApps extends JerseyTest { rm.stop(); } + @Test + public void testAppsQueryAppTypes() throws JSONException, Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); + Thread.sleep(1); + RMApp app1 = rm.submitApp(1024); + amNodeManager.nodeHeartbeat(true); + // finish App + MockAM am = rm + .sendAMLaunched(app1.getCurrentAppAttempt().getAppAttemptId()); + am.registerAppAttempt(); + am.unregisterAppAttempt(); + amNodeManager.nodeHeartbeat(app1.getCurrentAppAttempt().getAppAttemptId(), + 1, ContainerState.COMPLETE); + + rm.submitApp(1024, "", UserGroupInformation.getCurrentUser() + .getShortUserName(), null, false, null, 2, null, "MAPREDUCE"); + rm.submitApp(1024, "", UserGroupInformation.getCurrentUser() + .getShortUserName(), null, false, null, 2, null, "NON-YARN"); + + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("apps").queryParam("applicationTypes", "MAPREDUCE") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + JSONObject apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + JSONArray array = apps.getJSONArray("app"); + assertEquals("incorrect number of elements", 1, array.length()); + assertEquals("MAPREDUCE", + array.getJSONObject(0).getString("applicationType")); + + r = resource(); + response = + r.path("ws").path("v1").path("cluster").path("apps") + .queryParam("applicationTypes", "YARN") + .queryParam("applicationTypes", "MAPREDUCE") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + array = apps.getJSONArray("app"); + assertEquals("incorrect number of elements", 2, array.length()); + assertTrue((array.getJSONObject(0).getString("applicationType") + .equals("YARN") && array.getJSONObject(1).getString("applicationType") + .equals("MAPREDUCE")) || + (array.getJSONObject(1).getString("applicationType").equals("YARN") + && array.getJSONObject(0).getString("applicationType") + .equals("MAPREDUCE"))); + + r = resource(); + response = + r.path("ws").path("v1").path("cluster").path("apps") + .queryParam("applicationTypes", "YARN,NON-YARN") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + array = apps.getJSONArray("app"); + assertEquals("incorrect number of elements", 2, array.length()); + assertTrue((array.getJSONObject(0).getString("applicationType") + .equals("YARN") && array.getJSONObject(1).getString("applicationType") + .equals("NON-YARN")) || + (array.getJSONObject(1).getString("applicationType").equals("YARN") + && array.getJSONObject(0).getString("applicationType") + .equals("NON-YARN"))); + + r = resource(); + response = r.path("ws").path("v1").path("cluster") + .path("apps").queryParam("applicationTypes", "") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + array = apps.getJSONArray("app"); + assertEquals("incorrect number of elements", 3, array.length()); + + r = resource(); + response = + r.path("ws").path("v1").path("cluster").path("apps") + .queryParam("applicationTypes", "YARN,NON-YARN") + .queryParam("applicationTypes", "MAPREDUCE") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + array = apps.getJSONArray("app"); + assertEquals("incorrect number of elements", 3, array.length()); + + r = resource(); + response = + r.path("ws").path("v1").path("cluster").path("apps") + .queryParam("applicationTypes", "YARN") + .queryParam("applicationTypes", "") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + array = apps.getJSONArray("app"); + assertEquals("incorrect number of elements", 1, array.length()); + assertEquals("YARN", + array.getJSONObject(0).getString("applicationType")); + + r = resource(); + response = + r.path("ws").path("v1").path("cluster").path("apps") + .queryParam("applicationTypes", ",,, ,, YARN ,, ,") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + array = apps.getJSONArray("app"); + assertEquals("incorrect number of elements", 1, array.length()); + assertEquals("YARN", + array.getJSONObject(0).getString("applicationType")); + + r = resource(); + response = + r.path("ws").path("v1").path("cluster").path("apps") + .queryParam("applicationTypes", ",,, ,, ,, ,") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + array = apps.getJSONArray("app"); + assertEquals("incorrect number of elements", 3, array.length()); + + r = resource(); + response = + r.path("ws").path("v1").path("cluster").path("apps") + .queryParam("applicationTypes", "YARN, ,NON-YARN, ,,") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + array = apps.getJSONArray("app"); + assertEquals("incorrect number of elements", 2, array.length()); + assertTrue((array.getJSONObject(0).getString("applicationType") + .equals("YARN") && array.getJSONObject(1).getString("applicationType") + .equals("NON-YARN")) || + (array.getJSONObject(1).getString("applicationType").equals("YARN") + && array.getJSONObject(0).getString("applicationType") + .equals("NON-YARN"))); + + r = resource(); + response = + r.path("ws").path("v1").path("cluster").path("apps") + .queryParam("applicationTypes", " YARN, , ,,,") + .queryParam("applicationTypes", "MAPREDUCE , ,, ,") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + array = apps.getJSONArray("app"); + assertEquals("incorrect number of elements", 2, array.length()); + assertTrue((array.getJSONObject(0).getString("applicationType") + .equals("YARN") && array.getJSONObject(1).getString("applicationType") + .equals("MAPREDUCE")) || + (array.getJSONObject(1).getString("applicationType").equals("YARN") + && array.getJSONObject(0).getString("applicationType") + .equals("MAPREDUCE"))); + + rm.stop(); + } + @Test public void testSingleApp() throws JSONException, Exception { rm.start(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm index c1c88894544..5f0580128df 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm @@ -1119,6 +1119,7 @@ ResourceManager REST API's. * startedTimeEnd - applications with start time ending with this time, specified in ms since epoch * finishedTimeBegin - applications with finish time beginning with this time, specified in ms since epoch * finishedTimeEnd - applications with finish time ending with this time, specified in ms since epoch + * applicationTypes - applications matching the given application types, specified as a comma-separated list. ------ ** Elements of the (Applications) object