Merge r1504288 from trunk to branch-2 for YARN-865. RM webservices can't query based on application Types. Contributed by Xuan Gong.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1504290 13f79535-47bb-0310-9956-ffa450edef68

hadoop-yarn-project/CHANGES.txt

@@ -486,6 +486,9 @@ Release 2.1.0-beta - 2013-07-02
     YARN-922. Change FileSystemRMStateStore to use directories (Jian He via
     bikas)
 
+    YARN-865. RM webservices can't query based on application Types. (Xuan Gong
+    via hitesh)
+
   OPTIMIZATIONS
 
     YARN-512. Log aggregation root directory check is more expensive than it

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java

@@ -21,6 +21,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp;
 import java.io.IOException;
 import java.util.Collection;
 import java.util.EnumSet;
+import java.util.HashSet;
+import java.util.Set;
 import java.util.concurrent.ConcurrentMap;
 
 import javax.servlet.http.HttpServletRequest;
@@ -231,11 +233,13 @@ public class RMWebServices {
       @QueryParam("startedTimeBegin") String startedBegin,
       @QueryParam("startedTimeEnd") String startedEnd,
       @QueryParam("finishedTimeBegin") String finishBegin,
-      @QueryParam("finishedTimeEnd") String finishEnd) {
+      @QueryParam("finishedTimeEnd") String finishEnd,
+      @QueryParam("applicationTypes") Set<String> applicationTypes) {
     long num = 0;
     boolean checkCount = false;
     boolean checkStart = false;
     boolean checkEnd = false;
+    boolean checkAppTypes = false;
     long countNum = 0;
 
     // set values suitable in case both of begin/end not specified
@@ -291,6 +295,27 @@ public class RMWebServices {
           "finishTimeEnd must be greater than finishTimeBegin");
     }
 
+    Set<String> appTypes = new HashSet<String>();
+    if (!applicationTypes.isEmpty()) {
+      for (String applicationType : applicationTypes) {
+        if (applicationType != null && !applicationType.trim().isEmpty()) {
+          if (applicationType.indexOf(",") == -1) {
+            appTypes.add(applicationType.trim());
+          } else {
+            String[] types = applicationType.split(",");
+            for (String type : types) {
+              if (!type.trim().isEmpty()) {
+                appTypes.add(type.trim());
+              }
+            }
+          }
+        }
+      }
+    }
+    if (!appTypes.isEmpty()) {
+      checkAppTypes = true;
+    }
+
     final ConcurrentMap<ApplicationId, RMApp> apps = rm.getRMContext()
         .getRMApps();
     AppsInfo allApps = new AppsInfo();
@@ -332,6 +357,10 @@ public class RMWebServices {
           continue;
         }
       }
+      if (checkAppTypes
+          && !appTypes.contains(rmapp.getApplicationType())) {
+        continue;
+      }
 
       if (checkStart
           && (rmapp.getStartTime() < sBegin || rmapp.getStartTime() > sEnd)) {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java

@@ -637,6 +637,191 @@ public class TestRMWebServicesApps extends JerseyTest {
     rm.stop();
   }
 
+  @Test
+  public void testAppsQueryAppTypes() throws JSONException, Exception {
+    rm.start();
+    MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
+    Thread.sleep(1);
+    RMApp app1 = rm.submitApp(1024);
+    amNodeManager.nodeHeartbeat(true);
+    // finish App
+    MockAM am = rm
+        .sendAMLaunched(app1.getCurrentAppAttempt().getAppAttemptId());
+    am.registerAppAttempt();
+    am.unregisterAppAttempt();
+    amNodeManager.nodeHeartbeat(app1.getCurrentAppAttempt().getAppAttemptId(),
+        1, ContainerState.COMPLETE);
+
+    rm.submitApp(1024, "", UserGroupInformation.getCurrentUser()
+        .getShortUserName(), null, false, null, 2, null, "MAPREDUCE");
+    rm.submitApp(1024, "", UserGroupInformation.getCurrentUser()
+        .getShortUserName(), null, false, null, 2, null, "NON-YARN");
+
+    WebResource r = resource();
+    ClientResponse response = r.path("ws").path("v1").path("cluster")
+        .path("apps").queryParam("applicationTypes", "MAPREDUCE")
+        .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+    JSONObject json = response.getEntity(JSONObject.class);
+    assertEquals("incorrect number of elements", 1, json.length());
+    JSONObject apps = json.getJSONObject("apps");
+    assertEquals("incorrect number of elements", 1, apps.length());
+    JSONArray array = apps.getJSONArray("app");
+    assertEquals("incorrect number of elements", 1, array.length());
+    assertEquals("MAPREDUCE",
+        array.getJSONObject(0).getString("applicationType"));
+
+    r = resource();
+    response =
+        r.path("ws").path("v1").path("cluster").path("apps")
+            .queryParam("applicationTypes", "YARN")
+            .queryParam("applicationTypes", "MAPREDUCE")
+            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+    json = response.getEntity(JSONObject.class);
+    assertEquals("incorrect number of elements", 1, json.length());
+    apps = json.getJSONObject("apps");
+    assertEquals("incorrect number of elements", 1, apps.length());
+    array = apps.getJSONArray("app");
+    assertEquals("incorrect number of elements", 2, array.length());
+    assertTrue((array.getJSONObject(0).getString("applicationType")
+        .equals("YARN") && array.getJSONObject(1).getString("applicationType")
+        .equals("MAPREDUCE")) ||
+        (array.getJSONObject(1).getString("applicationType").equals("YARN")
+            && array.getJSONObject(0).getString("applicationType")
+            .equals("MAPREDUCE")));
+
+    r = resource();
+    response =
+        r.path("ws").path("v1").path("cluster").path("apps")
+            .queryParam("applicationTypes", "YARN,NON-YARN")
+            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+    json = response.getEntity(JSONObject.class);
+    assertEquals("incorrect number of elements", 1, json.length());
+    apps = json.getJSONObject("apps");
+    assertEquals("incorrect number of elements", 1, apps.length());
+    array = apps.getJSONArray("app");
+    assertEquals("incorrect number of elements", 2, array.length());
+    assertTrue((array.getJSONObject(0).getString("applicationType")
+        .equals("YARN") && array.getJSONObject(1).getString("applicationType")
+        .equals("NON-YARN")) ||
+        (array.getJSONObject(1).getString("applicationType").equals("YARN")
+            && array.getJSONObject(0).getString("applicationType")
+            .equals("NON-YARN")));
+
+    r = resource();
+    response = r.path("ws").path("v1").path("cluster")
+        .path("apps").queryParam("applicationTypes", "")
+        .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+    json = response.getEntity(JSONObject.class);
+    assertEquals("incorrect number of elements", 1, json.length());
+    apps = json.getJSONObject("apps");
+    assertEquals("incorrect number of elements", 1, apps.length());
+    array = apps.getJSONArray("app");
+    assertEquals("incorrect number of elements", 3, array.length());
+
+    r = resource();
+    response =
+        r.path("ws").path("v1").path("cluster").path("apps")
+            .queryParam("applicationTypes", "YARN,NON-YARN")
+            .queryParam("applicationTypes", "MAPREDUCE")
+            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+    json = response.getEntity(JSONObject.class);
+    assertEquals("incorrect number of elements", 1, json.length());
+    apps = json.getJSONObject("apps");
+    assertEquals("incorrect number of elements", 1, apps.length());
+    array = apps.getJSONArray("app");
+    assertEquals("incorrect number of elements", 3, array.length());
+
+    r = resource();
+    response =
+        r.path("ws").path("v1").path("cluster").path("apps")
+            .queryParam("applicationTypes", "YARN")
+            .queryParam("applicationTypes", "")
+            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+    json = response.getEntity(JSONObject.class);
+    assertEquals("incorrect number of elements", 1, json.length());
+    apps = json.getJSONObject("apps");
+    assertEquals("incorrect number of elements", 1, apps.length());
+    array = apps.getJSONArray("app");
+    assertEquals("incorrect number of elements", 1, array.length());
+    assertEquals("YARN",
+        array.getJSONObject(0).getString("applicationType"));
+
+    r = resource();
+    response =
+        r.path("ws").path("v1").path("cluster").path("apps")
+            .queryParam("applicationTypes", ",,, ,, YARN ,, ,")
+            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+    json = response.getEntity(JSONObject.class);
+    assertEquals("incorrect number of elements", 1, json.length());
+    apps = json.getJSONObject("apps");
+    assertEquals("incorrect number of elements", 1, apps.length());
+    array = apps.getJSONArray("app");
+    assertEquals("incorrect number of elements", 1, array.length());
+    assertEquals("YARN",
+        array.getJSONObject(0).getString("applicationType"));
+
+    r = resource();
+    response =
+        r.path("ws").path("v1").path("cluster").path("apps")
+            .queryParam("applicationTypes", ",,, ,,  ,, ,")
+            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+    json = response.getEntity(JSONObject.class);
+    assertEquals("incorrect number of elements", 1, json.length());
+    apps = json.getJSONObject("apps");
+    assertEquals("incorrect number of elements", 1, apps.length());
+    array = apps.getJSONArray("app");
+    assertEquals("incorrect number of elements", 3, array.length());
+
+    r = resource();
+    response =
+        r.path("ws").path("v1").path("cluster").path("apps")
+            .queryParam("applicationTypes", "YARN, ,NON-YARN, ,,")
+            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+    json = response.getEntity(JSONObject.class);
+    assertEquals("incorrect number of elements", 1, json.length());
+    apps = json.getJSONObject("apps");
+    assertEquals("incorrect number of elements", 1, apps.length());
+    array = apps.getJSONArray("app");
+    assertEquals("incorrect number of elements", 2, array.length());
+    assertTrue((array.getJSONObject(0).getString("applicationType")
+        .equals("YARN") && array.getJSONObject(1).getString("applicationType")
+        .equals("NON-YARN")) ||
+        (array.getJSONObject(1).getString("applicationType").equals("YARN")
+            && array.getJSONObject(0).getString("applicationType")
+            .equals("NON-YARN")));
+
+    r = resource();
+    response =
+        r.path("ws").path("v1").path("cluster").path("apps")
+            .queryParam("applicationTypes", " YARN, ,  ,,,")
+            .queryParam("applicationTypes", "MAPREDUCE , ,, ,")
+            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+    json = response.getEntity(JSONObject.class);
+    assertEquals("incorrect number of elements", 1, json.length());
+    apps = json.getJSONObject("apps");
+    assertEquals("incorrect number of elements", 1, apps.length());
+    array = apps.getJSONArray("app");
+    assertEquals("incorrect number of elements", 2, array.length());
+    assertTrue((array.getJSONObject(0).getString("applicationType")
+        .equals("YARN") && array.getJSONObject(1).getString("applicationType")
+        .equals("MAPREDUCE")) ||
+        (array.getJSONObject(1).getString("applicationType").equals("YARN")
+            && array.getJSONObject(0).getString("applicationType")
+            .equals("MAPREDUCE")));
+
+    rm.stop();
+  }
+
   @Test
   public void testSingleApp() throws JSONException, Exception {
     rm.start();

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm

@@ -1119,6 +1119,7 @@ ResourceManager REST API's.
   * startedTimeEnd - applications with start time ending with this time, specified in ms since epoch
   * finishedTimeBegin - applications with finish time beginning with this time, specified in ms since epoch
   * finishedTimeEnd - applications with finish time ending with this time, specified in ms since epoch
+  * applicationTypes - applications matching the given application types, specified as a comma-separated list.
 ------
 
 ** Elements of the <apps> (Applications) object