Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-642. Removed health parameter from ResourceManager /nodes web-service and cleaned the behaviour of the status parameter. Contributed by Sandy Ryza. 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1490890 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Vinod Kumar Vavilapalli 2013-06-08 00:54:22 +00:00
parent 176315d697
commit 0b0be8d4dd
3 changed files with 74 additions and 137 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/CHANGES.txt
View File

@ -129,6 +129,9 @@ Release 2.1.0-beta - UNRELEASED
YARN-777. Removed unreferenced objects from .proto files. (Jian He via YARN-777. Removed unreferenced objects from .proto files. (Jian He via
vinodkv) vinodkv)
YARN-642. Removed health parameter from ResourceManager /nodes web-service
and cleaned the behaviour of the status parameter. (Sandy Ryza vid vinodkv)
NEW FEATURES NEW FEATURES
YARN-482. FS: Extend SchedulingMode to intermediate queues. YARN-482. FS: Extend SchedulingMode to intermediate queues.

87
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
View File

@ -19,7 +19,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.webapp; package org.apache.hadoop.yarn.server.resourcemanager.webapp;
import java.io.IOException; import java.io.IOException;
import java.util.Collection;
import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentMap;
import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequest;
@ -32,8 +31,6 @@ import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Context; import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType; import javax.ws.rs.core.MediaType;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationId;
@ -75,7 +72,6 @@ import com.google.inject.Singleton;
@Path("/ws/v1/cluster") @Path("/ws/v1/cluster")
public class RMWebServices { public class RMWebServices {
private static final String EMPTY = ""; private static final String EMPTY = "";
private static final Log LOG = LogFactory.getLog(RMWebServices.class);
private final ResourceManager rm; private final ResourceManager rm;
private static RecordFactory recordFactory = RecordFactoryProvider private static RecordFactory recordFactory = RecordFactoryProvider
.getRecordFactory(null); .getRecordFactory(null);
@ -152,60 +148,57 @@ public class RMWebServices {
return new SchedulerTypeInfo(sinfo); return new SchedulerTypeInfo(sinfo);
} }
/**
* If no params are given, returns all active nodes, which includes
* nodes in the NEW and RUNNING states. If state param is "all", returns all
* nodes in all states. Otherwise, if the state param is set to a state name,
* returns all nodes that are in that state.
*/
@GET @GET
@Path("/nodes") @Path("/nodes")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public NodesInfo getNodes(@QueryParam("state") String filterState, public NodesInfo getNodes(@QueryParam("state") String state) {
@QueryParam("healthy") String healthState) {
init(); init();
ResourceScheduler sched = this.rm.getResourceScheduler(); ResourceScheduler sched = this.rm.getResourceScheduler();
if (sched == null) { if (sched == null) {
throw new NotFoundException("Null ResourceScheduler instance"); throw new NotFoundException("Null ResourceScheduler instance");
} }
Collection<RMNode> rmNodes = this.rm.getRMContext().getRMNodes().values();
boolean isInactive = false; NodeState acceptedState = null;
if (filterState != null && !filterState.isEmpty()) { boolean all = false;
NodeState nodeState = NodeState.valueOf(filterState.toUpperCase());
switch (nodeState) { if (state != null && !state.isEmpty()) {
case DECOMMISSIONED: if (state.equalsIgnoreCase("all")) {
case LOST: all = true;
case REBOOTED:
rmNodes = this.rm.getRMContext().getInactiveRMNodes().values();
isInactive = true;
break;
}
}
NodesInfo allNodes = new NodesInfo();
for (RMNode ni : rmNodes) {
NodeInfo nodeInfo = new NodeInfo(ni, sched);
if (filterState != null) {
if (!(nodeInfo.getState().equalsIgnoreCase(filterState))) {
continue;
}
} else { } else {
// No filter. User is asking for all nodes. Make sure you skip the acceptedState = NodeState.valueOf(state.toUpperCase());
// unhealthy nodes.
if (ni.getState() == NodeState.UNHEALTHY) {
continue;
}
} }
if ((healthState != null) && (!healthState.isEmpty())) {
LOG.info("heatlh state is : " + healthState);
if (!healthState.equalsIgnoreCase("true")
&& !healthState.equalsIgnoreCase("false")) {
String msg = "Error: You must specify either true or false to query on health";
throw new BadRequestException(msg);
}
if ((ni.getState() != NodeState.UNHEALTHY)
!= Boolean.parseBoolean(healthState)) {
continue;
}
}
if (isInactive) {
nodeInfo.setNodeHTTPAddress(EMPTY);
}
allNodes.add(nodeInfo);
} }
// getRMNodes() contains nodes that are NEW, RUNNING OR UNHEALTHY
NodesInfo allNodes = new NodesInfo();
for (RMNode ni : this.rm.getRMContext().getRMNodes().values()) {
if (all || (acceptedState == null && ni.getState() != NodeState.UNHEALTHY)
|| acceptedState == ni.getState()) {
NodeInfo nodeInfo = new NodeInfo(ni, sched);
allNodes.add(nodeInfo);
}
}
// getInactiveNodes() contains nodes that are DECOMMISSIONED, LOST, OR REBOOTED
if (all || (acceptedState != null &&
(acceptedState == NodeState.DECOMMISSIONED ||
acceptedState == NodeState.LOST ||
acceptedState == NodeState.REBOOTED))) {
for (RMNode ni : this.rm.getRMContext().getInactiveRMNodes().values()) {
if (all || acceptedState == ni.getState()) {
NodeInfo nodeInfo = new NodeInfo(ni, sched);
nodeInfo.setNodeHTTPAddress(EMPTY);
allNodes.add(nodeInfo);
}
}
}
return allNodes; return allNodes;
} }

121
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
View File

@ -163,7 +163,7 @@ public class TestRMWebServicesNodes extends JerseyTest {
} }
@Test @Test
public void testNodesQueryState() throws JSONException, Exception { public void testNodesQueryNew() throws JSONException, Exception {
WebResource r = resource(); WebResource r = resource();
MockNM nm1 = rm.registerNode("h1:1234", 5120); MockNM nm1 = rm.registerNode("h1:1234", 5120);
MockNM nm2 = rm.registerNode("h2:1235", 5121); MockNM nm2 = rm.registerNode("h2:1235", 5121);
@ -172,7 +172,7 @@ public class TestRMWebServicesNodes extends JerseyTest {
rm.NMwaitForState(nm2.getNodeId(), NodeState.NEW); rm.NMwaitForState(nm2.getNodeId(), NodeState.NEW);
ClientResponse response = r.path("ws").path("v1").path("cluster") ClientResponse response = r.path("ws").path("v1").path("cluster")
.path("nodes").queryParam("state", NodeState.RUNNING.toString()) .path("nodes").queryParam("state", NodeState.NEW.toString())
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
@ -184,7 +184,7 @@ public class TestRMWebServicesNodes extends JerseyTest {
assertEquals("incorrect number of elements", 1, nodeArray.length()); assertEquals("incorrect number of elements", 1, nodeArray.length());
JSONObject info = nodeArray.getJSONObject(0); JSONObject info = nodeArray.getJSONObject(0);
verifyNodeInfo(info, nm1); verifyNodeInfo(info, nm2);
} }
@Test @Test
@ -306,7 +306,7 @@ public class TestRMWebServicesNodes extends JerseyTest {
} }
@Test @Test
public void testNodesQueryHealthy() throws JSONException, Exception { public void testNodesQueryRunning() throws JSONException, Exception {
WebResource r = resource(); WebResource r = resource();
MockNM nm1 = rm.registerNode("h1:1234", 5120); MockNM nm1 = rm.registerNode("h1:1234", 5120);
MockNM nm2 = rm.registerNode("h2:1235", 5121); MockNM nm2 = rm.registerNode("h2:1235", 5121);
@ -314,7 +314,7 @@ public class TestRMWebServicesNodes extends JerseyTest {
rm.NMwaitForState(nm1.getNodeId(), NodeState.RUNNING); rm.NMwaitForState(nm1.getNodeId(), NodeState.RUNNING);
rm.NMwaitForState(nm2.getNodeId(), NodeState.NEW); rm.NMwaitForState(nm2.getNodeId(), NodeState.NEW);
ClientResponse response = r.path("ws").path("v1").path("cluster") ClientResponse response = r.path("ws").path("v1").path("cluster")
.path("nodes").queryParam("healthy", "true") .path("nodes").queryParam("state", "running")
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
JSONObject json = response.getEntity(JSONObject.class); JSONObject json = response.getEntity(JSONObject.class);
@ -322,54 +322,7 @@ public class TestRMWebServicesNodes extends JerseyTest {
JSONObject nodes = json.getJSONObject("nodes"); JSONObject nodes = json.getJSONObject("nodes");
assertEquals("incorrect number of elements", 1, nodes.length()); assertEquals("incorrect number of elements", 1, nodes.length());
JSONArray nodeArray = nodes.getJSONArray("node"); JSONArray nodeArray = nodes.getJSONArray("node");
assertEquals("incorrect number of elements", 2, nodeArray.length()); assertEquals("incorrect number of elements", 1, nodeArray.length());
}
@Test
public void testNodesQueryHealthyCase() throws JSONException, Exception {
WebResource r = resource();
MockNM nm1 = rm.registerNode("h1:1234", 5120);
MockNM nm2 = rm.registerNode("h2:1235", 5121);
rm.sendNodeStarted(nm1);
rm.NMwaitForState(nm1.getNodeId(), NodeState.RUNNING);
rm.NMwaitForState(nm2.getNodeId(), NodeState.NEW);
ClientResponse response = r.path("ws").path("v1").path("cluster")
.path("nodes").queryParam("healthy", "TRUe")
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
JSONObject json = response.getEntity(JSONObject.class);
assertEquals("incorrect number of elements", 1, json.length());
JSONObject nodes = json.getJSONObject("nodes");
assertEquals("incorrect number of elements", 1, nodes.length());
JSONArray nodeArray = nodes.getJSONArray("node");
assertEquals("incorrect number of elements", 2, nodeArray.length());
}
@Test
public void testNodesQueryHealthyAndState() throws JSONException, Exception {
WebResource r = resource();
MockNM nm1 = rm.registerNode("h1:1234", 5120);
MockNM nm2 = rm.registerNode("h2:1235", 5121);
rm.sendNodeStarted(nm1);
rm.NMwaitForState(nm2.getNodeId(), NodeState.NEW);
rm.NMwaitForState(nm1.getNodeId(), NodeState.RUNNING);
RMNodeImpl node = (RMNodeImpl) rm.getRMContext().getRMNodes()
.get(nm1.getNodeId());
NodeHealthStatus nodeHealth = NodeHealthStatus.newInstance(false,
"test health report", System.currentTimeMillis());
node.handle(new RMNodeStatusEvent(nm1.getNodeId(), nodeHealth,
new ArrayList<ContainerStatus>(), null, null));
rm.NMwaitForState(nm1.getNodeId(), NodeState.UNHEALTHY);
ClientResponse response = r.path("ws").path("v1").path("cluster")
.path("nodes").queryParam("healthy", "true")
.queryParam("state", NodeState.RUNNING.toString())
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
JSONObject json = response.getEntity(JSONObject.class);
assertEquals("incorrect number of elements", 1, json.length());
assertEquals("nodes is not null", JSONObject.NULL, json.get("nodes"));
} }
@Test @Test
@ -381,7 +334,7 @@ public class TestRMWebServicesNodes extends JerseyTest {
rm.NMwaitForState(nm1.getNodeId(), NodeState.RUNNING); rm.NMwaitForState(nm1.getNodeId(), NodeState.RUNNING);
rm.NMwaitForState(nm2.getNodeId(), NodeState.NEW); rm.NMwaitForState(nm2.getNodeId(), NodeState.NEW);
ClientResponse response = r.path("ws").path("v1").path("cluster") ClientResponse response = r.path("ws").path("v1").path("cluster")
.path("nodes").queryParam("healthy", "false") .path("nodes").queryParam("state", "UNHEALTHY")
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
JSONObject json = response.getEntity(JSONObject.class); JSONObject json = response.getEntity(JSONObject.class);
@ -389,42 +342,6 @@ public class TestRMWebServicesNodes extends JerseyTest {
assertEquals("nodes is not null", JSONObject.NULL, json.get("nodes")); assertEquals("nodes is not null", JSONObject.NULL, json.get("nodes"));
} }
@Test
public void testNodesQueryHealthyInvalid() throws JSONException, Exception {
WebResource r = resource();
rm.registerNode("h1:1234", 5120);
rm.registerNode("h2:1235", 5121);
try {
r.path("ws").path("v1").path("cluster").path("nodes")
.queryParam("healthy", "tr").accept(MediaType.APPLICATION_JSON)
.get(JSONObject.class);
fail("should have thrown exception querying invalid healthy string");
} catch (UniformInterfaceException ue) {
ClientResponse response = ue.getResponse();
assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus());
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
JSONObject msg = response.getEntity(JSONObject.class);
JSONObject exception = msg.getJSONObject("RemoteException");
assertEquals("incorrect number of elements", 3, exception.length());
String message = exception.getString("message");
String type = exception.getString("exception");
String classname = exception.getString("javaClassName");
WebServicesTestUtils
.checkStringMatch(
"exception message",
"java.lang.Exception: Error: You must specify either true or false to query on health",
message);
WebServicesTestUtils.checkStringMatch("exception type",
"BadRequestException", type);
WebServicesTestUtils.checkStringMatch("exception classname",
"org.apache.hadoop.yarn.webapp.BadRequestException", classname);
} finally {
rm.stop();
}
}
public void testNodesHelper(String path, String media) throws JSONException, public void testNodesHelper(String path, String media) throws JSONException,
Exception { Exception {
WebResource r = resource(); WebResource r = resource();
@ -689,6 +606,30 @@ public class TestRMWebServicesNodes extends JerseyTest {
assertEquals("incorrect number of elements", 2, nodes.getLength()); assertEquals("incorrect number of elements", 2, nodes.getLength());
rm.stop(); rm.stop();
} }
@Test
public void testQueryAll() throws Exception {
WebResource r = resource();
MockNM nm1 = rm.registerNode("h1:1234", 5120);
MockNM nm2 = rm.registerNode("h2:1235", 5121);
MockNM nm3 = rm.registerNode("h3:1236", 5122);
rm.sendNodeStarted(nm1);
rm.sendNodeStarted(nm3);
rm.NMwaitForState(nm1.getNodeId(), NodeState.RUNNING);
rm.NMwaitForState(nm2.getNodeId(), NodeState.NEW);
rm.sendNodeLost(nm3);
ClientResponse response = r.path("ws").path("v1").path("cluster")
.path("nodes").queryParam("state", "aLl")
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
JSONObject json = response.getEntity(JSONObject.class);
JSONObject nodes = json.getJSONObject("nodes");
assertEquals("incorrect number of elements", 1, nodes.length());
JSONArray nodeArray = nodes.getJSONArray("node");
assertEquals("incorrect number of elements", 3, nodeArray.length());
}
public void verifyNodesXML(NodeList nodes, MockNM nm) throws JSONException, public void verifyNodesXML(NodeList nodes, MockNM nm) throws JSONException,
Exception { Exception {