Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-1199. Make NM/RM Versions Available (Mit Desai via jeagles) 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1529008 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Jonathan Turner Eagles 2013-10-03 21:18:18 +00:00
parent 53df9bdb14
commit 8a93fa55b6
15 changed files with 69 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestRMNMInfo.java
View File

@ -120,6 +120,7 @@ public class TestRMNMInfo {
Assert.assertNotNull(n.get("NodeHTTPAddress"));
Assert.assertNotNull(n.get("LastHealthUpdate"));
Assert.assertNotNull(n.get("HealthReport"));
Assert.assertNotNull(n.get("NodeManagerVersion"));
Assert.assertNotNull(n.get("NumContainers"));
Assert.assertEquals(
n.get("NodeId") + ": Unexpected number of used containers",
@ -156,6 +157,7 @@ public class TestRMNMInfo {
Assert.assertNotNull(n.get("NodeHTTPAddress"));
Assert.assertNotNull(n.get("LastHealthUpdate"));
Assert.assertNotNull(n.get("HealthReport"));
Assert.assertNotNull(n.get("NodeManagerVersion"));
Assert.assertNull(n.get("NumContainers"));
Assert.assertNull(n.get("UsedMemoryMB"));
Assert.assertNull(n.get("AvailableMemoryMB"));

6
hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
View File

@ -147,6 +147,12 @@ public class NodeInfo {
list2));
return list;
}
@Override
public String getNodeManagerVersion() {
// TODO Auto-generated method stub
return null;
}
}
public static RMNode newNodeInfo(String rackName, String hostName,

6
hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
View File

@ -138,4 +138,10 @@ public class RMNodeWrapper implements RMNode {
return updates;
}
@Override
public String getNodeManagerVersion() {
// TODO Auto-generated method stub
return null;
}
}

2
hadoop-yarn-project/CHANGES.txt
View File

@ -31,6 +31,8 @@ Release 2.3.0 - UNRELEASED
YARN-425. coverage fix for yarn api (Aleksey Gorshkov via jeagles)
YARN-1199. Make NM/RM Versions Available (Mit Desai via jeagles)
OPTIMIZATIONS
BUG FIXES

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java
View File

@ -90,6 +90,8 @@ public class RMNMInfo implements RMNMInfoBeans {
ni.getLastHealthReportTime());
info.put("HealthReport",
ni.getHealthReport());
info.put("NodeManagerVersion",
ni.getNodeManagerVersion());
if(report != null) {
info.put("NumContainers", report.getNumContainers());
info.put("UsedMemoryMB", report.getUsedResource().getMemory());

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
View File

@ -234,7 +234,7 @@ public class ResourceTrackerService extends AbstractService implements
.getCurrentKey());
RMNode rmNode = new RMNodeImpl(nodeId, rmContext, host, cmPort, httpPort,
resolve(host), capability);
resolve(host), capability, nodeManagerVersion);
RMNode oldNode = this.rmContext.getRMNodes().putIfAbsent(nodeId, rmNode);
if (oldNode == null) {

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
View File

@ -27,7 +27,6 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
/**
* Node managers information on available resources
@ -84,7 +83,13 @@ public interface RMNode {
* @return the time of the latest health report received from this node.
*/
public long getLastHealthReportTime();
/**
* the node manager version of the node received as part of the
* registration with the resource manager
*/
public String getNodeManagerVersion();
/**
* the total available resource.
* @return the total available resource.

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
View File

@ -97,6 +97,7 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
private String healthReport;
private long lastHealthReportTime;
private String nodeManagerVersion;
/* set of containers that have just launched */
private final Map<ContainerId, ContainerStatus> justLaunchedContainers =
@ -172,7 +173,7 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
RMNodeEvent> stateMachine;
public RMNodeImpl(NodeId nodeId, RMContext context, String hostName,
int cmPort, int httpPort, Node node, Resource capability) {
int cmPort, int httpPort, Node node, Resource capability, String nodeManagerVersion) {
this.nodeId = nodeId;
this.context = context;
this.hostName = hostName;
@ -184,6 +185,7 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
this.node = node;
this.healthReport = "Healthy";
this.lastHealthReportTime = System.currentTimeMillis();
this.nodeManagerVersion = nodeManagerVersion;
this.latestNodeHeartBeatResponse.setResponseId(0);
@ -288,6 +290,11 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
}
}
@Override
public String getNodeManagerVersion() {
return nodeManagerVersion;
}
@Override
public NodeState getState() {
this.readLock.lock();

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
View File

@ -76,6 +76,7 @@ class NodesPage extends RmView {
th(".containers", "Containers").
th(".mem", "Mem Used").
th(".mem", "Mem Avail").
th(".nodeManagerVersion", "Version").
_()._().
tbody();
NodeState stateFilter = null;
@ -129,6 +130,7 @@ class NodesPage extends RmView {
_(StringUtils.byteDesc(usedMemory * BYTES_IN_MB))._().
td().br().$title(String.valueOf(usedMemory))._().
_(StringUtils.byteDesc(availableMemory * BYTES_IN_MB))._().
td(ni.getNodeManagerVersion()).
_();
}
tbody._()._();

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
View File

@ -38,6 +38,7 @@ public class NodeInfo {
protected String nodeHostName;
protected String nodeHTTPAddress;
protected long lastHealthUpdate;
protected String version;
protected String healthReport;
protected int numContainers;
protected long usedMemoryMB;
@ -64,6 +65,7 @@ public class NodeInfo {
this.nodeHTTPAddress = ni.getHttpAddress();
this.lastHealthUpdate = ni.getLastHealthReportTime();
this.healthReport = String.valueOf(ni.getHealthReport());
this.version = ni.getNodeManagerVersion();
}
public String getRack() {
@ -90,6 +92,10 @@ public class NodeInfo {
return this.lastHealthUpdate;
}
public String getVersion() {
return this.version;
}
public String getHealthReport() {
return this.healthReport;
}

11
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
View File

@ -40,6 +40,7 @@ import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
import org.apache.hadoop.yarn.server.api.records.NodeStatus;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.Records;
import org.apache.hadoop.yarn.util.YarnVersionInfo;
public class MockNM {
@ -51,6 +52,7 @@ public class MockNM {
private final int httpPort = 2;
private MasterKey currentContainerTokenMasterKey;
private MasterKey currentNMTokenMasterKey;
private String version;
public MockNM(String nodeIdStr, int memory, ResourceTrackerService resourceTracker) {
// scale vcores based on the requested memory
@ -61,10 +63,16 @@ public class MockNM {
}
public MockNM(String nodeIdStr, int memory, int vcores,
ResourceTrackerService resourceTracker) {
ResourceTrackerService resourceTracker) {
this(nodeIdStr, memory, vcores, resourceTracker, YarnVersionInfo.getVersion());
}
public MockNM(String nodeIdStr, int memory, int vcores,
ResourceTrackerService resourceTracker, String version) {
this.memory = memory;
this.vCores = vcores;
this.resourceTracker = resourceTracker;
this.version = version;
String[] splits = nodeIdStr.split(":");
nodeId = BuilderUtils.newNodeId(splits[0], Integer.parseInt(splits[1]));
}
@ -96,6 +104,7 @@ public class MockNM {
req.setHttpPort(httpPort);
Resource resource = BuilderUtils.newResource(memory, vCores);
req.setResource(resource);
req.setNMVersion(version);
RegisterNodeManagerResponse registrationResponse =
resourceTracker.registerNodeManager(req);
this.currentContainerTokenMasterKey =

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
View File

@ -183,6 +183,11 @@ public class MockNodes {
return null;
}
@Override
public String getNodeManagerVersion() {
return null;
}
@Override
public List<UpdatedContainerInfo> pullContainerUpdates() {
return new ArrayList<UpdatedContainerInfo>();

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
View File

@ -110,7 +110,7 @@ public class TestRMNodeTransitions {
new TestSchedulerEventDispatcher());
NodeId nodeId = BuilderUtils.newNodeId("localhost", 0);
node = new RMNodeImpl(nodeId, rmContext, null, 0, 0, null, null);
node = new RMNodeImpl(nodeId, rmContext, null, 0, 0, null, null, null);
}
@ -166,7 +166,7 @@ public class TestRMNodeTransitions {
node.handle(new RMNodeEvent(null,RMNodeEventType.STARTED));
NodeId nodeId = BuilderUtils.newNodeId("localhost:1", 1);
RMNodeImpl node2 = new RMNodeImpl(nodeId, rmContext, null, 0, 0, null, null);
RMNodeImpl node2 = new RMNodeImpl(nodeId, rmContext, null, 0, 0, null, null, null);
node2.handle(new RMNodeEvent(null,RMNodeEventType.STARTED));
ContainerId completedContainerIdFromNode1 = BuilderUtils.newContainerId(
@ -432,7 +432,7 @@ public class TestRMNodeTransitions {
private RMNodeImpl getRunningNode() {
NodeId nodeId = BuilderUtils.newNodeId("localhost", 0);
RMNodeImpl node = new RMNodeImpl(nodeId, rmContext,null, 0, 0,
null, null);
null, null, null);
node.handle(new RMNodeEvent(node.getNodeID(), RMNodeEventType.STARTED));
Assert.assertEquals(NodeState.RUNNING, node.getState());
return node;

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java
View File

@ -49,7 +49,7 @@ public class TestNodesPage {
// Number of Actual Table Headers for NodesPage.NodesBlock might change in
// future. In that case this value should be adjusted to the new value.
final int numberOfThInMetricsTable = 13;
final int numberOfActualTableHeaders = 9;
final int numberOfActualTableHeaders = 10;
private Injector injector;

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
View File

@ -655,13 +655,14 @@ public class TestRMWebServicesNodes extends JerseyTest {
WebServicesTestUtils.getXmlString(element, "healthReport"),
WebServicesTestUtils.getXmlInt(element, "numContainers"),
WebServicesTestUtils.getXmlLong(element, "usedMemoryMB"),
WebServicesTestUtils.getXmlLong(element, "availMemoryMB"));
WebServicesTestUtils.getXmlLong(element, "availMemoryMB"),
WebServicesTestUtils.getXmlString(element, "version"));
}
}
public void verifyNodeInfo(JSONObject nodeInfo, MockNM nm)
throws JSONException, Exception {
assertEquals("incorrect number of elements", 10, nodeInfo.length());
assertEquals("incorrect number of elements", 11, nodeInfo.length());
verifyNodeInfoGeneric(nm, nodeInfo.getString("state"),
nodeInfo.getString("rack"),
@ -669,14 +670,15 @@ public class TestRMWebServicesNodes extends JerseyTest {
nodeInfo.getString("nodeHTTPAddress"),
nodeInfo.getLong("lastHealthUpdate"),
nodeInfo.getString("healthReport"), nodeInfo.getInt("numContainers"),
nodeInfo.getLong("usedMemoryMB"), nodeInfo.getLong("availMemoryMB"));
nodeInfo.getLong("usedMemoryMB"), nodeInfo.getLong("availMemoryMB"),
nodeInfo.getString("version"));
}
public void verifyNodeInfoGeneric(MockNM nm, String state, String rack,
String id, String nodeHostName,
String nodeHTTPAddress, long lastHealthUpdate, String healthReport,
int numContainers, long usedMemoryMB, long availMemoryMB)
int numContainers, long usedMemoryMB, long availMemoryMB, String version)
throws JSONException, Exception {
RMNode node = rm.getRMContext().getRMNodes().get(nm.getNodeId());
@ -695,6 +697,8 @@ public class TestRMWebServicesNodes extends JerseyTest {
+ nm.getHttpPort();
WebServicesTestUtils.checkStringMatch("nodeHTTPAddress",
expectedHttpAddress, nodeHTTPAddress);
WebServicesTestUtils.checkStringMatch("version",
node.getNodeManagerVersion(), version);
long expectedHealthUpdate = node.getLastHealthReportTime();
assertEquals("lastHealthUpdate doesn't match, got: " + lastHealthUpdate