YARN-4947. Test timeout is happening for TestRMWebServicesNodes. Contributed by Bibin A Chundatt

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java

@@ -105,6 +105,9 @@ public class MockRM extends ResourceManager {
 
   static final Logger LOG = Logger.getLogger(MockRM.class);
   static final String ENABLE_WEBAPP = "mockrm.webapp.enabled";
+  private static final int SECOND = 1000;
+  private static final int TIMEOUT_MS_FOR_CONTAINER_AND_NODE = 10 * SECOND;
+  private static final int WAIT_MS_PER_LOOP = 10;
 
   final private boolean useNullRMNodeLabelsManager;
 
@@ -620,6 +623,35 @@ public class MockRM extends ResourceManager {
     node.handle(new RMNodeEvent(nm.getNodeId(), RMNodeEventType.EXPIRE));
   }
 
+  /**
+   * Wait until a node has reached a specified state. The timeout is 10 seconds.
+   *
+   * @param nodeId the id of a node
+   * @param finalState the node state waited
+   * @throws InterruptedException if interrupted while waiting for the state
+   *           transition
+   */
+  public void waitForState(NodeId nodeId, NodeState finalState)
+      throws InterruptedException {
+    RMNode node = getRMContext().getRMNodes().get(nodeId);
+    Assert.assertNotNull("node shouldn't be null", node);
+    int timeWaiting = 0;
+    while (!finalState.equals(node.getState())) {
+      if (timeWaiting >= TIMEOUT_MS_FOR_CONTAINER_AND_NODE) {
+        break;
+      }
+
+      System.out.println("Node State is : " + node.getState()
+          + " Waiting for state : " + finalState);
+      Thread.sleep(WAIT_MS_PER_LOOP);
+      timeWaiting += WAIT_MS_PER_LOOP;
+    }
+
+    System.out.println("Node State is : " + node.getState());
+    Assert.assertEquals("Node state is not correct (timedout)", finalState,
+        node.getState());
+  }
+
   public void NMwaitForState(NodeId nodeid, NodeState finalState)
       throws Exception {
     RMNode node = getRMContext().getRMNodes().get(nodeid);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java

@@ -34,17 +34,22 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.records.ContainerStatus;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceUtilization;
 import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
 import org.apache.hadoop.yarn.server.api.records.NodeStatus;
-import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
 import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
 import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEventType;
 import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeStartedEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeStatusEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport;
+import org.apache.hadoop.yarn.util.RackResolver;
+import org.apache.hadoop.yarn.util.YarnVersionInfo;
 import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
 import org.apache.hadoop.yarn.webapp.JerseyTestBase;
 import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
@@ -130,24 +135,21 @@ public class TestRMWebServicesNodes extends JerseyTestBase {
       Exception {
 
     WebResource r = resource();
-    MockNM nm1 = rm.registerNode("h1:1234", 5120);
+    getRunningRMNode("h1", 1234, 5120);
-    MockNM nm2 = rm.registerNode("h2:1235", 5121);
+    // h2 will be in NEW state
-    rm.sendNodeStarted(nm1);
+    getNewRMNode("h2", 1235, 5121);
-    rm.NMwaitForState(nm1.getNodeId(), NodeState.RUNNING);
-    rm.NMwaitForState(nm2.getNodeId(), NodeState.NEW);
 
-    MockNM nm3 = rm.registerNode("h3:1236", 5122);
+    RMNode node3 = getRunningRMNode("h3", 1236, 5122);
-    rm.NMwaitForState(nm3.getNodeId(), NodeState.NEW);
+    NodeId nodeId3 = node3.getNodeID();
-    rm.sendNodeStarted(nm3);
+
-    rm.NMwaitForState(nm3.getNodeId(), NodeState.RUNNING);
+    RMNode node = rm.getRMContext().getRMNodes().get(nodeId3);
-    RMNodeImpl node = (RMNodeImpl) rm.getRMContext().getRMNodes()
-        .get(nm3.getNodeId());
     NodeHealthStatus nodeHealth = NodeHealthStatus.newInstance(false,
         "test health report", System.currentTimeMillis());
-    NodeStatus nodeStatus = NodeStatus.newInstance(nm3.getNodeId(), 1,
+    NodeStatus nodeStatus = NodeStatus.newInstance(nodeId3, 1,
       new ArrayList<ContainerStatus>(), null, nodeHealth, null, null, null);
-    node.handle(new RMNodeStatusEvent(nm3.getNodeId(), nodeStatus, null));
+    ((RMNodeImpl) node)
-    rm.NMwaitForState(nm3.getNodeId(), NodeState.UNHEALTHY);
+        .handle(new RMNodeStatusEvent(nodeId3, nodeStatus, null));
+    rm.waitForState(nodeId3, NodeState.UNHEALTHY);
 
     ClientResponse response =
         r.path("ws").path("v1").path("cluster").path("nodes")
@@ -163,14 +165,38 @@ public class TestRMWebServicesNodes extends JerseyTestBase {
     assertEquals("incorrect number of elements", 3, nodeArray.length());
   }
 
+  private RMNode getRunningRMNode(String host, int port, int memory) {
+    RMNodeImpl rmnode1 = getNewRMNode(host, port, memory);
+    sendStartedEvent(rmnode1);
+    return rmnode1;
+  }
+
+  private void sendStartedEvent(RMNode node) {
+    ((RMNodeImpl) node)
+        .handle(new RMNodeStartedEvent(node.getNodeID(), null, null));
+  }
+
+  private void sendLostEvent(RMNode node) {
+    ((RMNodeImpl) node)
+        .handle(new RMNodeEvent(node.getNodeID(), RMNodeEventType.EXPIRE));
+  }
+
+  private RMNodeImpl getNewRMNode(String host, int port, int memory) {
+    NodeId nodeId = NodeId.newInstance(host, port);
+    RMNodeImpl nodeImpl = new RMNodeImpl(nodeId, rm.getRMContext(),
+        nodeId.getHost(), nodeId.getPort(), nodeId.getPort() + 1,
+        RackResolver.resolve(nodeId.getHost()), Resource.newInstance(memory, 4),
+        YarnVersionInfo.getVersion());
+    rm.getRMContext().getRMNodes().put(nodeId, nodeImpl);
+    return nodeImpl;
+  }
+
   @Test
   public void testNodesQueryNew() throws JSONException, Exception {
     WebResource r = resource();
-    MockNM nm1 = rm.registerNode("h1:1234", 5120);
+    getRunningRMNode("h1", 1234, 5120);
-    MockNM nm2 = rm.registerNode("h2:1235", 5121);
+    // h2 will be in NEW state
-    rm.sendNodeStarted(nm1);
+    RMNode rmnode2 = getNewRMNode("h2", 1235, 5121);
-    rm.NMwaitForState(nm1.getNodeId(), NodeState.RUNNING);
-    rm.NMwaitForState(nm2.getNodeId(), NodeState.NEW);
 
     ClientResponse response = r.path("ws").path("v1").path("cluster")
         .path("nodes").queryParam("states", NodeState.NEW.toString())
@@ -185,14 +211,14 @@ public class TestRMWebServicesNodes extends JerseyTestBase {
     assertEquals("incorrect number of elements", 1, nodeArray.length());
     JSONObject info = nodeArray.getJSONObject(0);
 
-    verifyNodeInfo(info, nm2);
+    verifyNodeInfo(info, rmnode2);
   }
 
   @Test
   public void testNodesQueryStateNone() throws JSONException, Exception {
     WebResource r = resource();
-    rm.registerNode("h1:1234", 5120);
+    getNewRMNode("h1", 1234, 5120);
-    rm.registerNode("h2:1235", 5121);
+    getNewRMNode("h2", 1235, 5121);
 
     ClientResponse response = r.path("ws").path("v1").path("cluster")
         .path("nodes")
@@ -207,8 +233,8 @@ public class TestRMWebServicesNodes extends JerseyTestBase {
   @Test
   public void testNodesQueryStateInvalid() throws JSONException, Exception {
     WebResource r = resource();
-    rm.registerNode("h1:1234", 5120);
+    getNewRMNode("h1", 1234, 5120);
-    rm.registerNode("h2:1235", 5121);
+    getNewRMNode("h2", 1235, 5121);
 
     try {
       r.path("ws").path("v1").path("cluster").path("nodes")
@@ -238,22 +264,16 @@ public class TestRMWebServicesNodes extends JerseyTestBase {
       WebServicesTestUtils.checkStringMatch("exception classname",
           "java.lang.IllegalArgumentException", classname);
 
-    } finally {
-      rm.stop();
     }
   }
   
   @Test
   public void testNodesQueryStateLost() throws JSONException, Exception {
     WebResource r = resource();
-    MockNM nm1 = rm.registerNode("h1:1234", 5120);
+    RMNode rmnode1 = getRunningRMNode("h1", 1234, 5120);
-    MockNM nm2 = rm.registerNode("h2:1234", 5120);
+    sendLostEvent(rmnode1);
-    rm.sendNodeStarted(nm1);
+    RMNode rmnode2 = getRunningRMNode("h2", 1235, 5121);
-    rm.sendNodeStarted(nm2);
+    sendLostEvent(rmnode2);
-    rm.NMwaitForState(nm1.getNodeId(), NodeState.RUNNING);
-    rm.NMwaitForState(nm2.getNodeId(), NodeState.RUNNING);
-    rm.sendNodeLost(nm1);
-    rm.sendNodeLost(nm2);
 
     ClientResponse response = r.path("ws").path("v1").path("cluster")
         .path("nodes").queryParam("states", NodeState.LOST.toString())
@@ -282,14 +302,9 @@ public class TestRMWebServicesNodes extends JerseyTestBase {
   @Test
   public void testSingleNodeQueryStateLost() throws JSONException, Exception {
     WebResource r = resource();
-    MockNM nm1 = rm.registerNode("h1:1234", 5120);
+    getRunningRMNode("h1", 1234, 5120);
-    MockNM nm2 = rm.registerNode("h2:1234", 5120);
+    RMNode rmnode2 = getRunningRMNode("h2", 1234, 5121);
-    rm.sendNodeStarted(nm1);
+    sendLostEvent(rmnode2);
-    rm.sendNodeStarted(nm2);
-    rm.NMwaitForState(nm1.getNodeId(), NodeState.RUNNING);
-    rm.NMwaitForState(nm2.getNodeId(), NodeState.RUNNING);
-    rm.sendNodeLost(nm1);
-    rm.sendNodeLost(nm2);
 
     ClientResponse response = r.path("ws").path("v1").path("cluster")
         .path("nodes").path("h2:1234").accept(MediaType.APPLICATION_JSON)
@@ -302,8 +317,8 @@ public class TestRMWebServicesNodes extends JerseyTestBase {
 
     assertEquals("Incorrect Node Information.", "h2:1234", id);
 
-    NodeId nodeId = NodeId.newInstance("h2", 1234);
+    RMNode rmNode =
-    RMNode rmNode = rm.getRMContext().getInactiveRMNodes().get(nodeId);
+        rm.getRMContext().getInactiveRMNodes().get(rmnode2.getNodeID());
     WebServicesTestUtils.checkStringMatch("nodeHTTPAddress", "",
         info.getString("nodeHTTPAddress"));
     if (rmNode != null) {
@@ -315,11 +330,9 @@ public class TestRMWebServicesNodes extends JerseyTestBase {
   @Test
   public void testNodesQueryRunning() throws JSONException, Exception {
     WebResource r = resource();
-    MockNM nm1 = rm.registerNode("h1:1234", 5120);
+    getRunningRMNode("h1", 1234, 5120);
-    MockNM nm2 = rm.registerNode("h2:1235", 5121);
+    // h2 will be in NEW state
-    rm.sendNodeStarted(nm1);
+    getNewRMNode("h2", 1235, 5121);
-    rm.NMwaitForState(nm1.getNodeId(), NodeState.RUNNING);
-    rm.NMwaitForState(nm2.getNodeId(), NodeState.NEW);
     ClientResponse response = r.path("ws").path("v1").path("cluster")
         .path("nodes").queryParam("states", "running")
         .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
@@ -335,11 +348,9 @@ public class TestRMWebServicesNodes extends JerseyTestBase {
   @Test
   public void testNodesQueryHealthyFalse() throws JSONException, Exception {
     WebResource r = resource();
-    MockNM nm1 = rm.registerNode("h1:1234", 5120);
+    getRunningRMNode("h1", 1234, 5120);
-    MockNM nm2 = rm.registerNode("h2:1235", 5121);
+    // h2 will be in NEW state
-    rm.sendNodeStarted(nm1);
+    getNewRMNode("h2", 1235, 5121);
-    rm.NMwaitForState(nm1.getNodeId(), NodeState.RUNNING);
-    rm.NMwaitForState(nm2.getNodeId(), NodeState.NEW);
     ClientResponse response = r.path("ws").path("v1").path("cluster")
         .path("nodes").queryParam("states", "UNHEALTHY")
         .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
@@ -352,12 +363,8 @@ public class TestRMWebServicesNodes extends JerseyTestBase {
   public void testNodesHelper(String path, String media) throws JSONException,
       Exception {
     WebResource r = resource();
-    MockNM nm1 = rm.registerNode("h1:1234", 5120);
+    RMNode rmnode1 = getRunningRMNode("h1", 1234, 5120);
-    MockNM nm2 = rm.registerNode("h2:1235", 5121);
+    RMNode rmnode2 = getRunningRMNode("h2", 1235, 5121);
-    rm.sendNodeStarted(nm1);
-    rm.sendNodeStarted(nm2);
-    rm.NMwaitForState(nm1.getNodeId(), NodeState.RUNNING);
-    rm.NMwaitForState(nm2.getNodeId(), NodeState.RUNNING);
 
     ClientResponse response = r.path("ws").path("v1").path("cluster")
         .path(path).accept(media).get(ClientResponse.class);
@@ -372,36 +379,36 @@ public class TestRMWebServicesNodes extends JerseyTestBase {
     String id = info.get("id").toString();
 
     if (id.matches("h1:1234")) {
-      verifyNodeInfo(info, nm1);
+      verifyNodeInfo(info, rmnode1);
-      verifyNodeInfo(nodeArray.getJSONObject(1), nm2);
+      verifyNodeInfo(nodeArray.getJSONObject(1), rmnode2);
     } else {
-      verifyNodeInfo(info, nm2);
+      verifyNodeInfo(info, rmnode2);
-      verifyNodeInfo(nodeArray.getJSONObject(1), nm1);
+      verifyNodeInfo(nodeArray.getJSONObject(1), rmnode1);
     }
   }
 
   @Test
   public void testSingleNode() throws JSONException, Exception {
-    rm.registerNode("h1:1234", 5120);
+    getRunningRMNode("h1", 1234, 5120);
-    MockNM nm2 = rm.registerNode("h2:1235", 5121);
+    RMNode rmnode2 = getRunningRMNode("h2", 1235, 5121);
-    testSingleNodeHelper("h2:1235", nm2, MediaType.APPLICATION_JSON);
+    testSingleNodeHelper("h2:1235", rmnode2, MediaType.APPLICATION_JSON);
   }
 
   @Test
   public void testSingleNodeSlash() throws JSONException, Exception {
-    MockNM nm1 = rm.registerNode("h1:1234", 5120);
+    RMNode rmnode1 = getRunningRMNode("h1", 1234, 5120);
-    rm.registerNode("h2:1235", 5121);
+    getRunningRMNode("h2", 1235, 5121);
-    testSingleNodeHelper("h1:1234/", nm1, MediaType.APPLICATION_JSON);
+    testSingleNodeHelper("h1:1234/", rmnode1, MediaType.APPLICATION_JSON);
   }
 
   @Test
   public void testSingleNodeDefault() throws JSONException, Exception {
-    MockNM nm1 = rm.registerNode("h1:1234", 5120);
+    RMNode rmnode1 = getRunningRMNode("h1", 1234, 5120);
-    rm.registerNode("h2:1235", 5121);
+    getRunningRMNode("h2", 1235, 5121);
-    testSingleNodeHelper("h1:1234/", nm1, "");
+    testSingleNodeHelper("h1:1234/", rmnode1, "");
   }
 
-  public void testSingleNodeHelper(String nodeid, MockNM nm, String media)
+  public void testSingleNodeHelper(String nodeid, RMNode nm, String media)
       throws JSONException, Exception {
     WebResource r = resource();
     ClientResponse response = r.path("ws").path("v1").path("cluster")
@@ -416,8 +423,10 @@ public class TestRMWebServicesNodes extends JerseyTestBase {
 
   @Test
   public void testNonexistNode() throws JSONException, Exception {
-    rm.registerNode("h1:1234", 5120);
+    // add h1 node in NEW state
-    rm.registerNode("h2:1235", 5121);
+    getNewRMNode("h1", 1234, 5120);
+    // add h2 node in NEW state
+    getNewRMNode("h2", 1235, 5121);
     WebResource r = resource();
     try {
       r.path("ws").path("v1").path("cluster").path("nodes")
@@ -436,17 +445,14 @@ public class TestRMWebServicesNodes extends JerseyTestBase {
       String type = exception.getString("exception");
       String classname = exception.getString("javaClassName");
       verifyNonexistNodeException(message, type, classname);
-
-    } finally {
-      rm.stop();
     }
   }
 
   // test that the exception output defaults to JSON
   @Test
   public void testNonexistNodeDefault() throws JSONException, Exception {
-    rm.registerNode("h1:1234", 5120);
+    getNewRMNode("h1", 1234, 5120);
-    rm.registerNode("h2:1235", 5121);
+    getNewRMNode("h2", 1235, 5121);
     WebResource r = resource();
     try {
       r.path("ws").path("v1").path("cluster").path("nodes")
@@ -464,16 +470,14 @@ public class TestRMWebServicesNodes extends JerseyTestBase {
       String type = exception.getString("exception");
       String classname = exception.getString("javaClassName");
       verifyNonexistNodeException(message, type, classname);
-    } finally {
-      rm.stop();
     }
   }
 
   // test that the exception output works in XML
   @Test
   public void testNonexistNodeXML() throws JSONException, Exception {
-    rm.registerNode("h1:1234", 5120);
+    getNewRMNode("h1", 1234, 5120);
-    rm.registerNode("h2:1235", 5121);
+    getNewRMNode("h2", 1235, 5121);
     WebResource r = resource();
     try {
       r.path("ws").path("v1").path("cluster").path("nodes")
@@ -499,8 +503,6 @@ public class TestRMWebServicesNodes extends JerseyTestBase {
       String classname = WebServicesTestUtils.getXmlString(element,
           "javaClassName");
       verifyNonexistNodeException(message, type, classname);
-    } finally {
-      rm.stop();
     }
   }
 
@@ -515,8 +517,8 @@ public class TestRMWebServicesNodes extends JerseyTestBase {
 
   @Test
   public void testInvalidNode() throws JSONException, Exception {
-    rm.registerNode("h1:1234", 5120);
+    getNewRMNode("h1", 1234, 5120);
-    rm.registerNode("h2:1235", 5121);
+    getNewRMNode("h2", 1235, 5121);
 
     WebResource r = resource();
     try {
@@ -542,16 +544,13 @@ public class TestRMWebServicesNodes extends JerseyTestBase {
           "IllegalArgumentException", type);
       WebServicesTestUtils.checkStringMatch("exception classname",
           "java.lang.IllegalArgumentException", classname);
-    } finally {
-      rm.stop();
     }
   }
 
   @Test
   public void testNodesXML() throws JSONException, Exception {
-    rm.start();
     WebResource r = resource();
-    MockNM nm1 = rm.registerNode("h1:1234", 5120);
+    RMNodeImpl rmnode1 = getNewRMNode("h1", 1234, 5120);
     // MockNM nm2 = rm.registerNode("h2:1235", 5121);
     ClientResponse response = r.path("ws").path("v1").path("cluster")
         .path("nodes").accept(MediaType.APPLICATION_XML)
@@ -567,15 +566,14 @@ public class TestRMWebServicesNodes extends JerseyTestBase {
     assertEquals("incorrect number of elements", 1, nodesApps.getLength());
     NodeList nodes = dom.getElementsByTagName("node");
     assertEquals("incorrect number of elements", 1, nodes.getLength());
-    verifyNodesXML(nodes, nm1);
+    verifyNodesXML(nodes, rmnode1);
-    rm.stop();
   }
 
   @Test
   public void testSingleNodesXML() throws JSONException, Exception {
-    rm.start();
     WebResource r = resource();
-    MockNM nm1 = rm.registerNode("h1:1234", 5120);
+    // add h2 node in NEW state
+    RMNodeImpl rmnode1 = getNewRMNode("h1", 1234, 5120);
     // MockNM nm2 = rm.registerNode("h2:1235", 5121);
     ClientResponse response = r.path("ws").path("v1").path("cluster")
         .path("nodes").path("h1:1234").accept(MediaType.APPLICATION_XML)
@@ -591,16 +589,14 @@ public class TestRMWebServicesNodes extends JerseyTestBase {
     Document dom = db.parse(is);
     NodeList nodes = dom.getElementsByTagName("node");
     assertEquals("incorrect number of elements", 1, nodes.getLength());
-    verifyNodesXML(nodes, nm1);
+    verifyNodesXML(nodes, rmnode1);
-    rm.stop();
   }
 
   @Test
   public void testNodes2XML() throws JSONException, Exception {
-    rm.start();
     WebResource r = resource();
-    rm.registerNode("h1:1234", 5120);
+    getNewRMNode("h1", 1234, 5120);
-    rm.registerNode("h2:1235", 5121);
+    getNewRMNode("h2", 1235, 5121);
     ClientResponse response = r.path("ws").path("v1").path("cluster")
         .path("nodes").accept(MediaType.APPLICATION_XML)
         .get(ClientResponse.class);
@@ -616,20 +612,17 @@ public class TestRMWebServicesNodes extends JerseyTestBase {
     assertEquals("incorrect number of elements", 1, nodesApps.getLength());
     NodeList nodes = dom.getElementsByTagName("node");
     assertEquals("incorrect number of elements", 2, nodes.getLength());
-    rm.stop();
   }
   
   @Test
   public void testQueryAll() throws Exception {
     WebResource r = resource();
-    MockNM nm1 = rm.registerNode("h1:1234", 5120);
+    getRunningRMNode("h1", 1234, 5120);
-    MockNM nm2 = rm.registerNode("h2:1235", 5121);
+    // add h2 node in NEW state
-    MockNM nm3 = rm.registerNode("h3:1236", 5122);
+    getNewRMNode("h2", 1235, 5121);
-    rm.sendNodeStarted(nm1);
+    // add lost node
-    rm.sendNodeStarted(nm3);
+    RMNode nm3 = getRunningRMNode("h3", 1236, 5122);
-    rm.NMwaitForState(nm1.getNodeId(), NodeState.RUNNING);
+    sendLostEvent(nm3);
-    rm.NMwaitForState(nm2.getNodeId(), NodeState.NEW);
-    rm.sendNodeLost(nm3);
 
     ClientResponse response = r.path("ws").path("v1").path("cluster")
         .path("nodes")
@@ -647,23 +640,22 @@ public class TestRMWebServicesNodes extends JerseyTestBase {
   @Test
   public void testNodesResourceUtilization() throws JSONException, Exception {
     WebResource r = resource();
-    MockNM nm1 = rm.registerNode("h1:1234", 5120);
+    RMNode rmnode1 = getRunningRMNode("h1", 1234, 5120);
-    rm.sendNodeStarted(nm1);
+    NodeId nodeId1 = rmnode1.getNodeID();
-    rm.NMwaitForState(nm1.getNodeId(), NodeState.RUNNING);
 
-    RMNodeImpl node = (RMNodeImpl) rm.getRMContext().getRMNodes()
+    RMNodeImpl node = (RMNodeImpl) rm.getRMContext().getRMNodes().get(nodeId1);
-        .get(nm1.getNodeId());
     NodeHealthStatus nodeHealth = NodeHealthStatus.newInstance(true,
         "test health report", System.currentTimeMillis());
     ResourceUtilization nodeResource = ResourceUtilization.newInstance(4096, 0,
         (float) 10.5);
     ResourceUtilization containerResource = ResourceUtilization.newInstance(
         2048, 0, (float) 5.05);
-    NodeStatus nodeStatus = NodeStatus.newInstance(nm1.getNodeId(), 0,
+    NodeStatus nodeStatus =
+        NodeStatus.newInstance(nodeId1, 0,
         new ArrayList<ContainerStatus>(), null, nodeHealth, containerResource,
         nodeResource, null);
-    node.handle(new RMNodeStatusEvent(nm1.getNodeId(), nodeStatus, null));
+    node.handle(new RMNodeStatusEvent(nodeId1, nodeStatus, null));
-    rm.NMwaitForState(nm1.getNodeId(), NodeState.RUNNING);
+    rm.waitForState(nodeId1, NodeState.RUNNING);
 
     ClientResponse response = r.path("ws").path("v1").path("cluster")
         .path("nodes").accept(MediaType.APPLICATION_JSON)
@@ -679,10 +671,11 @@ public class TestRMWebServicesNodes extends JerseyTestBase {
     JSONObject info = nodeArray.getJSONObject(0);
 
     // verify the resource utilization
-    verifyNodeInfo(info, nm1);
+    verifyNodeInfo(info, rmnode1);
   }
 
-  public void verifyNodesXML(NodeList nodes, MockNM nm) throws JSONException,
+  public void verifyNodesXML(NodeList nodes, RMNode nm)
+      throws JSONException,
       Exception {
     for (int i = 0; i < nodes.getLength(); i++) {
       Element element = (Element) nodes.item(i);
@@ -711,7 +704,7 @@ public class TestRMWebServicesNodes extends JerseyTestBase {
     }
   }
 
-  public void verifyNodeInfo(JSONObject nodeInfo, MockNM nm)
+  public void verifyNodeInfo(JSONObject nodeInfo, RMNode nm)
       throws JSONException, Exception {
     assertEquals("incorrect number of elements", 14, nodeInfo.length());
 
@@ -733,7 +726,7 @@ public class TestRMWebServicesNodes extends JerseyTestBase {
         resourceInfo.getDouble("containersCPUUsage"));
   }
 
-  public void verifyNodeInfoGeneric(MockNM nm, String state, String rack,
+  public void verifyNodeInfoGeneric(RMNode node, String state, String rack,
       String id, String nodeHostName,
       String nodeHTTPAddress, long lastHealthUpdate, String healthReport,
       int numContainers, long usedMemoryMB, long availMemoryMB, long usedVirtualCores, 
@@ -743,20 +736,20 @@ public class TestRMWebServicesNodes extends JerseyTestBase {
       double containersCPUUsage)
       throws JSONException, Exception {
 
-    RMNode node = rm.getRMContext().getRMNodes().get(nm.getNodeId());
     ResourceScheduler sched = rm.getResourceScheduler();
-    SchedulerNodeReport report = sched.getNodeReport(nm.getNodeId());
+    SchedulerNodeReport report = sched.getNodeReport(node.getNodeID());
 
     WebServicesTestUtils.checkStringMatch("state", node.getState().toString(),
         state);
     WebServicesTestUtils.checkStringMatch("rack", node.getRackName(), rack);
-    WebServicesTestUtils.checkStringMatch("id", nm.getNodeId().toString(), id);
+    WebServicesTestUtils.checkStringMatch("id", node.getNodeID().toString(),
-    WebServicesTestUtils.checkStringMatch("nodeHostName", nm.getNodeId()
+        id);
-        .getHost(), nodeHostName);
+    WebServicesTestUtils.checkStringMatch("nodeHostName",
+        node.getNodeID().getHost(), nodeHostName);
     WebServicesTestUtils.checkStringMatch("healthReport",
         String.valueOf(node.getHealthReport()), healthReport);
-    String expectedHttpAddress = nm.getNodeId().getHost() + ":"
+    String expectedHttpAddress =
-        + nm.getHttpPort();
+        node.getNodeID().getHost() + ":" + node.getHttpPort();
     WebServicesTestUtils.checkStringMatch("nodeHTTPAddress",
         expectedHttpAddress, nodeHTTPAddress);
     WebServicesTestUtils.checkStringMatch("version",