Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge r1504288 from trunk to branch-2 for YARN-865. RM webservices can't query based on application Types. Contributed by Xuan Gong. 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1504290 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Hitesh Shah 2013-07-17 21:43:51 +00:00
parent 859b53435f
commit db854fd104
4 changed files with 219 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/CHANGES.txt
View File

@ -486,6 +486,9 @@ Release 2.1.0-beta - 2013-07-02
YARN-922. Change FileSystemRMStateStore to use directories (Jian He via
bikas)
YARN-865. RM webservices can't query based on application Types. (Xuan Gong
via hitesh)
OPTIMIZATIONS
YARN-512. Log aggregation root directory check is more expensive than it

31
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
View File

@ -21,6 +21,8 @@
import java.io.IOException;
import java.util.Collection;
import java.util.EnumSet;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.ConcurrentMap;
import javax.servlet.http.HttpServletRequest;
@ -231,11 +233,13 @@ public AppsInfo getApps(@Context HttpServletRequest hsr,
@QueryParam("startedTimeBegin") String startedBegin,
@QueryParam("startedTimeEnd") String startedEnd,
@QueryParam("finishedTimeBegin") String finishBegin,
@QueryParam("finishedTimeEnd") String finishEnd) {
@QueryParam("finishedTimeEnd") String finishEnd,
@QueryParam("applicationTypes") Set<String> applicationTypes) {
long num = 0;
boolean checkCount = false;
boolean checkStart = false;
boolean checkEnd = false;
boolean checkAppTypes = false;
long countNum = 0;
// set values suitable in case both of begin/end not specified
@ -291,6 +295,27 @@ public AppsInfo getApps(@Context HttpServletRequest hsr,
"finishTimeEnd must be greater than finishTimeBegin");
}
Set<String> appTypes = new HashSet<String>();
if (!applicationTypes.isEmpty()) {
for (String applicationType : applicationTypes) {
if (applicationType != null && !applicationType.trim().isEmpty()) {
if (applicationType.indexOf(",") == -1) {
appTypes.add(applicationType.trim());
} else {
String[] types = applicationType.split(",");
for (String type : types) {
if (!type.trim().isEmpty()) {
appTypes.add(type.trim());
}
}
}
}
}
}
if (!appTypes.isEmpty()) {
checkAppTypes = true;
}
final ConcurrentMap<ApplicationId, RMApp> apps = rm.getRMContext()
.getRMApps();
AppsInfo allApps = new AppsInfo();
@ -332,6 +357,10 @@ public AppsInfo getApps(@Context HttpServletRequest hsr,
continue;
}
}
if (checkAppTypes
&& !appTypes.contains(rmapp.getApplicationType())) {
continue;
}
if (checkStart
&& (rmapp.getStartTime() < sBegin || rmapp.getStartTime() > sEnd)) {

185
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
View File

@ -637,6 +637,191 @@ public void testAppsQueryFinishBeginEnd() throws JSONException, Exception {
rm.stop();
}
@Test
public void testAppsQueryAppTypes() throws JSONException, Exception {
rm.start();
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
Thread.sleep(1);
RMApp app1 = rm.submitApp(1024);
amNodeManager.nodeHeartbeat(true);
// finish App
MockAM am = rm
.sendAMLaunched(app1.getCurrentAppAttempt().getAppAttemptId());
am.registerAppAttempt();
am.unregisterAppAttempt();
amNodeManager.nodeHeartbeat(app1.getCurrentAppAttempt().getAppAttemptId(),
1, ContainerState.COMPLETE);
rm.submitApp(1024, "", UserGroupInformation.getCurrentUser()
.getShortUserName(), null, false, null, 2, null, "MAPREDUCE");
rm.submitApp(1024, "", UserGroupInformation.getCurrentUser()
.getShortUserName(), null, false, null, 2, null, "NON-YARN");
WebResource r = resource();
ClientResponse response = r.path("ws").path("v1").path("cluster")
.path("apps").queryParam("applicationTypes", "MAPREDUCE")
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
JSONObject json = response.getEntity(JSONObject.class);
assertEquals("incorrect number of elements", 1, json.length());
JSONObject apps = json.getJSONObject("apps");
assertEquals("incorrect number of elements", 1, apps.length());
JSONArray array = apps.getJSONArray("app");
assertEquals("incorrect number of elements", 1, array.length());
assertEquals("MAPREDUCE",
array.getJSONObject(0).getString("applicationType"));
r = resource();
response =
r.path("ws").path("v1").path("cluster").path("apps")
.queryParam("applicationTypes", "YARN")
.queryParam("applicationTypes", "MAPREDUCE")
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
json = response.getEntity(JSONObject.class);
assertEquals("incorrect number of elements", 1, json.length());
apps = json.getJSONObject("apps");
assertEquals("incorrect number of elements", 1, apps.length());
array = apps.getJSONArray("app");
assertEquals("incorrect number of elements", 2, array.length());
assertTrue((array.getJSONObject(0).getString("applicationType")
.equals("YARN") && array.getJSONObject(1).getString("applicationType")
.equals("MAPREDUCE")) ||
(array.getJSONObject(1).getString("applicationType").equals("YARN")
&& array.getJSONObject(0).getString("applicationType")
.equals("MAPREDUCE")));
r = resource();
response =
r.path("ws").path("v1").path("cluster").path("apps")
.queryParam("applicationTypes", "YARN,NON-YARN")
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
json = response.getEntity(JSONObject.class);
assertEquals("incorrect number of elements", 1, json.length());
apps = json.getJSONObject("apps");
assertEquals("incorrect number of elements", 1, apps.length());
array = apps.getJSONArray("app");
assertEquals("incorrect number of elements", 2, array.length());
assertTrue((array.getJSONObject(0).getString("applicationType")
.equals("YARN") && array.getJSONObject(1).getString("applicationType")
.equals("NON-YARN")) ||
(array.getJSONObject(1).getString("applicationType").equals("YARN")
&& array.getJSONObject(0).getString("applicationType")
.equals("NON-YARN")));
r = resource();
response = r.path("ws").path("v1").path("cluster")
.path("apps").queryParam("applicationTypes", "")
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
json = response.getEntity(JSONObject.class);
assertEquals("incorrect number of elements", 1, json.length());
apps = json.getJSONObject("apps");
assertEquals("incorrect number of elements", 1, apps.length());
array = apps.getJSONArray("app");
assertEquals("incorrect number of elements", 3, array.length());
r = resource();
response =
r.path("ws").path("v1").path("cluster").path("apps")
.queryParam("applicationTypes", "YARN,NON-YARN")
.queryParam("applicationTypes", "MAPREDUCE")
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
json = response.getEntity(JSONObject.class);
assertEquals("incorrect number of elements", 1, json.length());
apps = json.getJSONObject("apps");
assertEquals("incorrect number of elements", 1, apps.length());
array = apps.getJSONArray("app");
assertEquals("incorrect number of elements", 3, array.length());
r = resource();
response =
r.path("ws").path("v1").path("cluster").path("apps")
.queryParam("applicationTypes", "YARN")
.queryParam("applicationTypes", "")
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
json = response.getEntity(JSONObject.class);
assertEquals("incorrect number of elements", 1, json.length());
apps = json.getJSONObject("apps");
assertEquals("incorrect number of elements", 1, apps.length());
array = apps.getJSONArray("app");
assertEquals("incorrect number of elements", 1, array.length());
assertEquals("YARN",
array.getJSONObject(0).getString("applicationType"));
r = resource();
response =
r.path("ws").path("v1").path("cluster").path("apps")
.queryParam("applicationTypes", ",,, ,, YARN ,, ,")
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
json = response.getEntity(JSONObject.class);
assertEquals("incorrect number of elements", 1, json.length());
apps = json.getJSONObject("apps");
assertEquals("incorrect number of elements", 1, apps.length());
array = apps.getJSONArray("app");
assertEquals("incorrect number of elements", 1, array.length());
assertEquals("YARN",
array.getJSONObject(0).getString("applicationType"));
r = resource();
response =
r.path("ws").path("v1").path("cluster").path("apps")
.queryParam("applicationTypes", ",,, ,, ,, ,")
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
json = response.getEntity(JSONObject.class);
assertEquals("incorrect number of elements", 1, json.length());
apps = json.getJSONObject("apps");
assertEquals("incorrect number of elements", 1, apps.length());
array = apps.getJSONArray("app");
assertEquals("incorrect number of elements", 3, array.length());
r = resource();
response =
r.path("ws").path("v1").path("cluster").path("apps")
.queryParam("applicationTypes", "YARN, ,NON-YARN, ,,")
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
json = response.getEntity(JSONObject.class);
assertEquals("incorrect number of elements", 1, json.length());
apps = json.getJSONObject("apps");
assertEquals("incorrect number of elements", 1, apps.length());
array = apps.getJSONArray("app");
assertEquals("incorrect number of elements", 2, array.length());
assertTrue((array.getJSONObject(0).getString("applicationType")
.equals("YARN") && array.getJSONObject(1).getString("applicationType")
.equals("NON-YARN")) ||
(array.getJSONObject(1).getString("applicationType").equals("YARN")
&& array.getJSONObject(0).getString("applicationType")
.equals("NON-YARN")));
r = resource();
response =
r.path("ws").path("v1").path("cluster").path("apps")
.queryParam("applicationTypes", " YARN, , ,,,")
.queryParam("applicationTypes", "MAPREDUCE , ,, ,")
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
json = response.getEntity(JSONObject.class);
assertEquals("incorrect number of elements", 1, json.length());
apps = json.getJSONObject("apps");
assertEquals("incorrect number of elements", 1, apps.length());
array = apps.getJSONArray("app");
assertEquals("incorrect number of elements", 2, array.length());
assertTrue((array.getJSONObject(0).getString("applicationType")
.equals("YARN") && array.getJSONObject(1).getString("applicationType")
.equals("MAPREDUCE")) ||
(array.getJSONObject(1).getString("applicationType").equals("YARN")
&& array.getJSONObject(0).getString("applicationType")
.equals("MAPREDUCE")));
rm.stop();
}
@Test
public void testSingleApp() throws JSONException, Exception {
rm.start();

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
View File

@ -1119,6 +1119,7 @@ ResourceManager REST API's.
* startedTimeEnd - applications with start time ending with this time, specified in ms since epoch
* finishedTimeBegin - applications with finish time beginning with this time, specified in ms since epoch
* finishedTimeEnd - applications with finish time ending with this time, specified in ms since epoch
* applicationTypes - applications matching the given application types, specified as a comma-separated list.
------
** Elements of the <apps> (Applications) object