From efdbd0a006e35a131c215a046146254281d619fc Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Sat, 8 Jun 2013 00:55:25 +0000 Subject: [PATCH] YARN-642. Removed health parameter from ResourceManager /nodes web-service and cleaned the behaviour of the status parameter. Contributed by Sandy Ryza. svn merge --ignore-ancestry -c 1490890 ../../trunk/ git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1490892 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../resourcemanager/webapp/RMWebServices.java | 87 ++++++------- .../webapp/TestRMWebServicesNodes.java | 121 +++++------------- 3 files changed, 74 insertions(+), 137 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index c03061d191b..682aba337ee 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -109,6 +109,9 @@ Release 2.1.0-beta - UNRELEASED YARN-777. Removed unreferenced objects from .proto files. (Jian He via vinodkv) + YARN-642. Removed health parameter from ResourceManager /nodes web-service + and cleaned the behaviour of the status parameter. (Sandy Ryza vid vinodkv) + NEW FEATURES YARN-482. FS: Extend SchedulingMode to intermediate queues. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index b769c2169cd..abb3cfc9259 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -19,7 +19,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp; import java.io.IOException; -import java.util.Collection; import java.util.concurrent.ConcurrentMap; import javax.servlet.http.HttpServletRequest; @@ -32,8 +31,6 @@ import javax.ws.rs.QueryParam; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -75,7 +72,6 @@ import com.google.inject.Singleton; @Path("/ws/v1/cluster") public class RMWebServices { private static final String EMPTY = ""; - private static final Log LOG = LogFactory.getLog(RMWebServices.class); private final ResourceManager rm; private static RecordFactory recordFactory = RecordFactoryProvider .getRecordFactory(null); @@ -152,60 +148,57 @@ public class RMWebServices { return new SchedulerTypeInfo(sinfo); } + /** + * If no params are given, returns all active nodes, which includes + * nodes in the NEW and RUNNING states. If state param is "all", returns all + * nodes in all states. Otherwise, if the state param is set to a state name, + * returns all nodes that are in that state. + */ @GET @Path("/nodes") @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) - public NodesInfo getNodes(@QueryParam("state") String filterState, - @QueryParam("healthy") String healthState) { + public NodesInfo getNodes(@QueryParam("state") String state) { init(); ResourceScheduler sched = this.rm.getResourceScheduler(); if (sched == null) { throw new NotFoundException("Null ResourceScheduler instance"); } - Collection rmNodes = this.rm.getRMContext().getRMNodes().values(); - boolean isInactive = false; - if (filterState != null && !filterState.isEmpty()) { - NodeState nodeState = NodeState.valueOf(filterState.toUpperCase()); - switch (nodeState) { - case DECOMMISSIONED: - case LOST: - case REBOOTED: - rmNodes = this.rm.getRMContext().getInactiveRMNodes().values(); - isInactive = true; - break; - } - } - NodesInfo allNodes = new NodesInfo(); - for (RMNode ni : rmNodes) { - NodeInfo nodeInfo = new NodeInfo(ni, sched); - if (filterState != null) { - if (!(nodeInfo.getState().equalsIgnoreCase(filterState))) { - continue; - } + + NodeState acceptedState = null; + boolean all = false; + + if (state != null && !state.isEmpty()) { + if (state.equalsIgnoreCase("all")) { + all = true; } else { - // No filter. User is asking for all nodes. Make sure you skip the - // unhealthy nodes. - if (ni.getState() == NodeState.UNHEALTHY) { - continue; - } + acceptedState = NodeState.valueOf(state.toUpperCase()); } - if ((healthState != null) && (!healthState.isEmpty())) { - LOG.info("heatlh state is : " + healthState); - if (!healthState.equalsIgnoreCase("true") - && !healthState.equalsIgnoreCase("false")) { - String msg = "Error: You must specify either true or false to query on health"; - throw new BadRequestException(msg); - } - if ((ni.getState() != NodeState.UNHEALTHY) - != Boolean.parseBoolean(healthState)) { - continue; - } - } - if (isInactive) { - nodeInfo.setNodeHTTPAddress(EMPTY); - } - allNodes.add(nodeInfo); } + + // getRMNodes() contains nodes that are NEW, RUNNING OR UNHEALTHY + NodesInfo allNodes = new NodesInfo(); + for (RMNode ni : this.rm.getRMContext().getRMNodes().values()) { + if (all || (acceptedState == null && ni.getState() != NodeState.UNHEALTHY) + || acceptedState == ni.getState()) { + NodeInfo nodeInfo = new NodeInfo(ni, sched); + allNodes.add(nodeInfo); + } + } + + // getInactiveNodes() contains nodes that are DECOMMISSIONED, LOST, OR REBOOTED + if (all || (acceptedState != null && + (acceptedState == NodeState.DECOMMISSIONED || + acceptedState == NodeState.LOST || + acceptedState == NodeState.REBOOTED))) { + for (RMNode ni : this.rm.getRMContext().getInactiveRMNodes().values()) { + if (all || acceptedState == ni.getState()) { + NodeInfo nodeInfo = new NodeInfo(ni, sched); + nodeInfo.setNodeHTTPAddress(EMPTY); + allNodes.add(nodeInfo); + } + } + } + return allNodes; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java index 44041389454..7a185e4c7e9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java @@ -163,7 +163,7 @@ public class TestRMWebServicesNodes extends JerseyTest { } @Test - public void testNodesQueryState() throws JSONException, Exception { + public void testNodesQueryNew() throws JSONException, Exception { WebResource r = resource(); MockNM nm1 = rm.registerNode("h1:1234", 5120); MockNM nm2 = rm.registerNode("h2:1235", 5121); @@ -172,7 +172,7 @@ public class TestRMWebServicesNodes extends JerseyTest { rm.NMwaitForState(nm2.getNodeId(), NodeState.NEW); ClientResponse response = r.path("ws").path("v1").path("cluster") - .path("nodes").queryParam("state", NodeState.RUNNING.toString()) + .path("nodes").queryParam("state", NodeState.NEW.toString()) .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); @@ -184,7 +184,7 @@ public class TestRMWebServicesNodes extends JerseyTest { assertEquals("incorrect number of elements", 1, nodeArray.length()); JSONObject info = nodeArray.getJSONObject(0); - verifyNodeInfo(info, nm1); + verifyNodeInfo(info, nm2); } @Test @@ -306,7 +306,7 @@ public class TestRMWebServicesNodes extends JerseyTest { } @Test - public void testNodesQueryHealthy() throws JSONException, Exception { + public void testNodesQueryRunning() throws JSONException, Exception { WebResource r = resource(); MockNM nm1 = rm.registerNode("h1:1234", 5120); MockNM nm2 = rm.registerNode("h2:1235", 5121); @@ -314,7 +314,7 @@ public class TestRMWebServicesNodes extends JerseyTest { rm.NMwaitForState(nm1.getNodeId(), NodeState.RUNNING); rm.NMwaitForState(nm2.getNodeId(), NodeState.NEW); ClientResponse response = r.path("ws").path("v1").path("cluster") - .path("nodes").queryParam("healthy", "true") + .path("nodes").queryParam("state", "running") .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); JSONObject json = response.getEntity(JSONObject.class); @@ -322,54 +322,7 @@ public class TestRMWebServicesNodes extends JerseyTest { JSONObject nodes = json.getJSONObject("nodes"); assertEquals("incorrect number of elements", 1, nodes.length()); JSONArray nodeArray = nodes.getJSONArray("node"); - assertEquals("incorrect number of elements", 2, nodeArray.length()); - } - - @Test - public void testNodesQueryHealthyCase() throws JSONException, Exception { - WebResource r = resource(); - MockNM nm1 = rm.registerNode("h1:1234", 5120); - MockNM nm2 = rm.registerNode("h2:1235", 5121); - rm.sendNodeStarted(nm1); - rm.NMwaitForState(nm1.getNodeId(), NodeState.RUNNING); - rm.NMwaitForState(nm2.getNodeId(), NodeState.NEW); - ClientResponse response = r.path("ws").path("v1").path("cluster") - .path("nodes").queryParam("healthy", "TRUe") - .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); - assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); - JSONObject json = response.getEntity(JSONObject.class); - assertEquals("incorrect number of elements", 1, json.length()); - JSONObject nodes = json.getJSONObject("nodes"); - assertEquals("incorrect number of elements", 1, nodes.length()); - JSONArray nodeArray = nodes.getJSONArray("node"); - assertEquals("incorrect number of elements", 2, nodeArray.length()); - - } - - @Test - public void testNodesQueryHealthyAndState() throws JSONException, Exception { - WebResource r = resource(); - MockNM nm1 = rm.registerNode("h1:1234", 5120); - MockNM nm2 = rm.registerNode("h2:1235", 5121); - rm.sendNodeStarted(nm1); - rm.NMwaitForState(nm2.getNodeId(), NodeState.NEW); - rm.NMwaitForState(nm1.getNodeId(), NodeState.RUNNING); - RMNodeImpl node = (RMNodeImpl) rm.getRMContext().getRMNodes() - .get(nm1.getNodeId()); - NodeHealthStatus nodeHealth = NodeHealthStatus.newInstance(false, - "test health report", System.currentTimeMillis()); - node.handle(new RMNodeStatusEvent(nm1.getNodeId(), nodeHealth, - new ArrayList(), null, null)); - rm.NMwaitForState(nm1.getNodeId(), NodeState.UNHEALTHY); - - ClientResponse response = r.path("ws").path("v1").path("cluster") - .path("nodes").queryParam("healthy", "true") - .queryParam("state", NodeState.RUNNING.toString()) - .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); - assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); - JSONObject json = response.getEntity(JSONObject.class); - assertEquals("incorrect number of elements", 1, json.length()); - assertEquals("nodes is not null", JSONObject.NULL, json.get("nodes")); + assertEquals("incorrect number of elements", 1, nodeArray.length()); } @Test @@ -381,7 +334,7 @@ public class TestRMWebServicesNodes extends JerseyTest { rm.NMwaitForState(nm1.getNodeId(), NodeState.RUNNING); rm.NMwaitForState(nm2.getNodeId(), NodeState.NEW); ClientResponse response = r.path("ws").path("v1").path("cluster") - .path("nodes").queryParam("healthy", "false") + .path("nodes").queryParam("state", "UNHEALTHY") .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); JSONObject json = response.getEntity(JSONObject.class); @@ -389,42 +342,6 @@ public class TestRMWebServicesNodes extends JerseyTest { assertEquals("nodes is not null", JSONObject.NULL, json.get("nodes")); } - @Test - public void testNodesQueryHealthyInvalid() throws JSONException, Exception { - WebResource r = resource(); - rm.registerNode("h1:1234", 5120); - rm.registerNode("h2:1235", 5121); - - try { - r.path("ws").path("v1").path("cluster").path("nodes") - .queryParam("healthy", "tr").accept(MediaType.APPLICATION_JSON) - .get(JSONObject.class); - fail("should have thrown exception querying invalid healthy string"); - } catch (UniformInterfaceException ue) { - ClientResponse response = ue.getResponse(); - assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); - assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); - JSONObject msg = response.getEntity(JSONObject.class); - JSONObject exception = msg.getJSONObject("RemoteException"); - assertEquals("incorrect number of elements", 3, exception.length()); - String message = exception.getString("message"); - String type = exception.getString("exception"); - String classname = exception.getString("javaClassName"); - WebServicesTestUtils - .checkStringMatch( - "exception message", - "java.lang.Exception: Error: You must specify either true or false to query on health", - message); - WebServicesTestUtils.checkStringMatch("exception type", - "BadRequestException", type); - WebServicesTestUtils.checkStringMatch("exception classname", - "org.apache.hadoop.yarn.webapp.BadRequestException", classname); - - } finally { - rm.stop(); - } - } - public void testNodesHelper(String path, String media) throws JSONException, Exception { WebResource r = resource(); @@ -689,6 +606,30 @@ public class TestRMWebServicesNodes extends JerseyTest { assertEquals("incorrect number of elements", 2, nodes.getLength()); rm.stop(); } + + @Test + public void testQueryAll() throws Exception { + WebResource r = resource(); + MockNM nm1 = rm.registerNode("h1:1234", 5120); + MockNM nm2 = rm.registerNode("h2:1235", 5121); + MockNM nm3 = rm.registerNode("h3:1236", 5122); + rm.sendNodeStarted(nm1); + rm.sendNodeStarted(nm3); + rm.NMwaitForState(nm1.getNodeId(), NodeState.RUNNING); + rm.NMwaitForState(nm2.getNodeId(), NodeState.NEW); + rm.sendNodeLost(nm3); + + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("nodes").queryParam("state", "aLl") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + JSONObject nodes = json.getJSONObject("nodes"); + assertEquals("incorrect number of elements", 1, nodes.length()); + JSONArray nodeArray = nodes.getJSONArray("node"); + assertEquals("incorrect number of elements", 3, nodeArray.length()); + } public void verifyNodesXML(NodeList nodes, MockNM nm) throws JSONException, Exception {