diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 209aab5cf33..1d3fbbc0c2d 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -116,6 +116,9 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-4089. Hung Tasks never time out. (Robert Evans via tgraves) + MAPREDUCE-4024. RM webservices can't query on finalStatus (Tom Graves + via bobby) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java index 944a28ee8d3..404cfbb22cb 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java @@ -31,6 +31,7 @@ import javax.ws.rs.core.UriInfo; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapreduce.v2.api.records.AMInfo; +import org.apache.hadoop.mapreduce.v2.api.records.JobState; import org.apache.hadoop.mapreduce.v2.api.records.JobReport; import org.apache.hadoop.mapreduce.v2.api.records.TaskId; import org.apache.hadoop.mapreduce.v2.api.records.TaskType; @@ -96,6 +97,7 @@ public class HsWebServices { @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) public JobsInfo getJobs(@QueryParam("user") String userQuery, @QueryParam("limit") String count, + @QueryParam("state") String stateQuery, @QueryParam("queue") String queueQuery, @QueryParam("startedTimeBegin") String startedBegin, @QueryParam("startedTimeEnd") String startedEnd, @@ -185,6 +187,13 @@ public class HsWebServices { break; } + if (stateQuery != null && !stateQuery.isEmpty()) { + JobState.valueOf(stateQuery); + if (!job.getState().toString().equalsIgnoreCase(stateQuery)) { + continue; + } + } + // can't really validate queue is a valid one since queues could change if (queueQuery != null && !queueQuery.isEmpty()) { if (!job.getQueueName().equals(queueQuery)) { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobsQuery.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobsQuery.java index 0ee92f2d3a3..c0110dcd087 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobsQuery.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobsQuery.java @@ -32,6 +32,7 @@ import javax.ws.rs.core.MediaType; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapreduce.v2.api.records.JobId; +import org.apache.hadoop.mapreduce.v2.api.records.JobState; import org.apache.hadoop.mapreduce.v2.app.AppContext; import org.apache.hadoop.mapreduce.v2.app.MockJobs; import org.apache.hadoop.mapreduce.v2.app.job.Job; @@ -120,7 +121,7 @@ public class TestHsWebServicesJobsQuery extends JerseyTest { public Job getPartialJob(JobId jobID) { return partialJobs.get(jobID); } - + @Override public Map getAllJobs() { return partialJobs; // OK @@ -195,6 +196,72 @@ public class TestHsWebServicesJobsQuery extends JerseyTest { .contextPath("jersey-guice-filter").servletPath("/").build()); } + @Test + public void testJobsQueryStateNone() throws JSONException, Exception { + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("history") + .path("mapreduce").path("jobs").queryParam("state", JobState.KILL_WAIT.toString()) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + assertEquals("jobs is not null", JSONObject.NULL, json.get("jobs")); + } + + @Test + public void testJobsQueryState() throws JSONException, Exception { + WebResource r = resource(); + // we only create 3 jobs and it cycles through states so we should have 3 unique states + Map jobsMap = appContext.getAllJobs(); + String queryState = "BOGUS"; + JobId jid = null; + for (Map.Entry entry : jobsMap.entrySet()) { + jid = entry.getValue().getID(); + queryState = entry.getValue().getState().toString(); + break; + } + ClientResponse response = r.path("ws").path("v1").path("history") + .path("mapreduce").path("jobs").queryParam("state", queryState) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + JSONObject jobs = json.getJSONObject("jobs"); + JSONArray arr = jobs.getJSONArray("job"); + assertEquals("incorrect number of elements", 1, arr.length()); + JSONObject info = arr.getJSONObject(0); + Job job = appContext.getPartialJob(jid); + VerifyJobsUtils.verifyHsJobPartial(info, job); + } + + @Test + public void testJobsQueryStateInvalid() throws JSONException, Exception { + WebResource r = resource(); + + ClientResponse response = r.path("ws").path("v1").path("history") + .path("mapreduce").path("jobs").queryParam("state", "InvalidState") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + + assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject msg = response.getEntity(JSONObject.class); + JSONObject exception = msg.getJSONObject("RemoteException"); + assertEquals("incorrect number of elements", 3, exception.length()); + String message = exception.getString("message"); + String type = exception.getString("exception"); + String classname = exception.getString("javaClassName"); + WebServicesTestUtils + .checkStringMatch( + "exception message", + "No enum const class org.apache.hadoop.mapreduce.v2.api.records.JobState.InvalidState", + message); + WebServicesTestUtils.checkStringMatch("exception type", + "IllegalArgumentException", type); + WebServicesTestUtils.checkStringMatch("exception classname", + "java.lang.IllegalArgumentException", classname); + } + + @Test public void testJobsQueryUserNone() throws JSONException, Exception { WebResource r = resource(); @@ -215,6 +282,8 @@ public class TestHsWebServicesJobsQuery extends JerseyTest { .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); JSONObject json = response.getEntity(JSONObject.class); + System.out.println(json.toString()); + assertEquals("incorrect number of elements", 1, json.length()); JSONObject jobs = json.getJSONObject("jobs"); JSONArray arr = jobs.getJSONArray("job"); diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index d745446b46c..eafe94b1800 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -36,6 +36,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; @@ -225,6 +226,7 @@ public class RMWebServices { @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) public AppsInfo getApps(@Context HttpServletRequest hsr, @QueryParam("state") String stateQuery, + @QueryParam("finalStatus") String finalStatusQuery, @QueryParam("user") String userQuery, @QueryParam("queue") String queueQuery, @QueryParam("limit") String count, @@ -294,19 +296,25 @@ public class RMWebServices { .getRMApps(); AppsInfo allApps = new AppsInfo(); for (RMApp rmapp : apps.values()) { + if (checkCount && num == countNum) { break; } - AppInfo app = new AppInfo(rmapp, hasAccess(rmapp, hsr)); - if (stateQuery != null && !stateQuery.isEmpty()) { RMAppState.valueOf(stateQuery); - if (!app.getState().equalsIgnoreCase(stateQuery)) { + if (!rmapp.getState().toString().equalsIgnoreCase(stateQuery)) { + continue; + } + } + if (finalStatusQuery != null && !finalStatusQuery.isEmpty()) { + FinalApplicationStatus.valueOf(finalStatusQuery); + if (!rmapp.getFinalApplicationStatus().toString() + .equalsIgnoreCase(finalStatusQuery)) { continue; } } if (userQuery != null && !userQuery.isEmpty()) { - if (!app.getUser().equals(userQuery)) { + if (!rmapp.getUser().equals(userQuery)) { continue; } } @@ -321,19 +329,20 @@ public class RMWebServices { throw new BadRequestException(e.getMessage()); } } - if (!app.getQueue().equals(queueQuery)) { + if (!rmapp.getQueue().equals(queueQuery)) { continue; } } if (checkStart - && (app.getStartTime() < sBegin || app.getStartTime() > sEnd)) { + && (rmapp.getStartTime() < sBegin || rmapp.getStartTime() > sEnd)) { continue; } if (checkEnd - && (app.getFinishTime() < fBegin || app.getFinishTime() > fEnd)) { + && (rmapp.getFinishTime() < fBegin || rmapp.getFinishTime() > fEnd)) { continue; } + AppInfo app = new AppInfo(rmapp, hasAccess(rmapp, hsr)); allApps.add(app); num++; diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index 2fbf3fdd864..3a30b928873 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -30,6 +30,7 @@ import javax.xml.parsers.DocumentBuilderFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; @@ -280,6 +281,85 @@ public class TestRMWebServicesApps extends JerseyTest { } } + @Test + public void testAppsQueryFinalStatus() throws JSONException, Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); + RMApp app1 = rm.submitApp(1024); + amNodeManager.nodeHeartbeat(true); + WebResource r = resource(); + + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("apps").queryParam("finalStatus", FinalApplicationStatus.UNDEFINED.toString()) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + System.out.println(json.toString()); + JSONObject apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + JSONArray array = apps.getJSONArray("app"); + assertEquals("incorrect number of elements", 1, array.length()); + verifyAppInfo(array.getJSONObject(0), app1); + rm.stop(); + } + + @Test + public void testAppsQueryFinalStatusNone() throws JSONException, Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); + rm.submitApp(1024); + amNodeManager.nodeHeartbeat(true); + WebResource r = resource(); + + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("apps").queryParam("finalStatus", FinalApplicationStatus.KILLED.toString()) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + assertEquals("apps is not null", JSONObject.NULL, json.get("apps")); + rm.stop(); + } + + @Test + public void testAppsQueryFinalStatusInvalid() throws JSONException, Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); + rm.submitApp(1024); + amNodeManager.nodeHeartbeat(true); + WebResource r = resource(); + + try { + r.path("ws").path("v1").path("cluster").path("apps") + .queryParam("finalStatus", "INVALID_test") + .accept(MediaType.APPLICATION_JSON).get(JSONObject.class); + fail("should have thrown exception on invalid state query"); + } catch (UniformInterfaceException ue) { + ClientResponse response = ue.getResponse(); + assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject msg = response.getEntity(JSONObject.class); + JSONObject exception = msg.getJSONObject("RemoteException"); + assertEquals("incorrect number of elements", 3, exception.length()); + String message = exception.getString("message"); + String type = exception.getString("exception"); + String classname = exception.getString("javaClassName"); + WebServicesTestUtils + .checkStringMatch( + "exception message", + "No enum const class org.apache.hadoop.yarn.api.records.FinalApplicationStatus.INVALID_test", + message); + WebServicesTestUtils.checkStringMatch("exception type", + "IllegalArgumentException", type); + WebServicesTestUtils.checkStringMatch("exception classname", + "java.lang.IllegalArgumentException", classname); + + } finally { + rm.stop(); + } + } + @Test public void testAppsQueryUser() throws JSONException, Exception { rm.start(); diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm index 558b2c35ad9..de51c8ad3fb 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm @@ -149,6 +149,7 @@ History Server REST API's. ------ * user - user name + * state - the job state * queue - queue name * limit - total number of app objects to be returned * startedTimeBegin - jobs with start time beginning with this time, specified in ms since epoch diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm index ce489692fe0..eaaae564f51 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm @@ -890,6 +890,7 @@ ResourceManager REST API's. ------ * state - state of the application + * finalStatus - the final status of the application - reported by the application itself * user - user name * queue - queue name * limit - total number of app objects to be returned