Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-1199. Make NM/RM Versions Available (Mit Desai via jeagles) 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1529003 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Jonathan Turner Eagles 2013-10-03 21:04:12 +00:00
parent dbdb8c6f1f
commit 7b687dda09
15 changed files with 69 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestRMNMInfo.java
View File

@ -120,6 +120,7 @@ public void testRMNMInfo() throws Exception {
Assert.assertNotNull(n.get("NodeHTTPAddress"));
Assert.assertNotNull(n.get("LastHealthUpdate"));
Assert.assertNotNull(n.get("HealthReport"));
Assert.assertNotNull(n.get("NodeManagerVersion"));
Assert.assertNotNull(n.get("NumContainers"));
Assert.assertEquals(
n.get("NodeId") + ": Unexpected number of used containers",
@ -156,6 +157,7 @@ public void testRMNMInfoMissmatch() throws Exception {
Assert.assertNotNull(n.get("NodeHTTPAddress"));
Assert.assertNotNull(n.get("LastHealthUpdate"));
Assert.assertNotNull(n.get("HealthReport"));
Assert.assertNotNull(n.get("NodeManagerVersion"));
Assert.assertNull(n.get("NumContainers"));
Assert.assertNull(n.get("UsedMemoryMB"));
Assert.assertNull(n.get("AvailableMemoryMB"));

6
hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
View File

@ -147,6 +147,12 @@ public List<UpdatedContainerInfo> pullContainerUpdates() {
list2));
return list;
}
@Override
public String getNodeManagerVersion() {
// TODO Auto-generated method stub
return null;
}
}
public static RMNode newNodeInfo(String rackName, String hostName,

6
hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
View File

@ -138,4 +138,10 @@ List<UpdatedContainerInfo> getContainerUpdates() {
return updates;
}
@Override
public String getNodeManagerVersion() {
// TODO Auto-generated method stub
return null;
}
}

2
hadoop-yarn-project/CHANGES.txt
View File

@ -46,6 +46,8 @@ Release 2.3.0 - UNRELEASED
YARN-425. coverage fix for yarn api (Aleksey Gorshkov via jeagles)
YARN-1199. Make NM/RM Versions Available (Mit Desai via jeagles)
OPTIMIZATIONS
BUG FIXES

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java
View File

@ -90,6 +90,8 @@ public String getLiveNodeManagers() {
ni.getLastHealthReportTime());
info.put("HealthReport",
ni.getHealthReport());
info.put("NodeManagerVersion",
ni.getNodeManagerVersion());
if(report != null) {
info.put("NumContainers", report.getNumContainers());
info.put("UsedMemoryMB", report.getUsedResource().getMemory());

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
View File

@ -234,7 +234,7 @@ public RegisterNodeManagerResponse registerNodeManager(
.getCurrentKey());
RMNode rmNode = new RMNodeImpl(nodeId, rmContext, host, cmPort, httpPort,
resolve(host), capability);
resolve(host), capability, nodeManagerVersion);
RMNode oldNode = this.rmContext.getRMNodes().putIfAbsent(nodeId, rmNode);
if (oldNode == null) {

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
View File

@ -27,7 +27,6 @@
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
/**
* Node managers information on available resources
@ -84,7 +83,13 @@ public interface RMNode {
* @return the time of the latest health report received from this node.
*/
public long getLastHealthReportTime();
/**
* the node manager version of the node received as part of the
* registration with the resource manager
*/
public String getNodeManagerVersion();
/**
* the total available resource.
* @return the total available resource.

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
View File

@ -97,6 +97,7 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
private String healthReport;
private long lastHealthReportTime;
private String nodeManagerVersion;
/* set of containers that have just launched */
private final Map<ContainerId, ContainerStatus> justLaunchedContainers =
@ -172,7 +173,7 @@ RMNodeEventType.CLEANUP_CONTAINER, new CleanUpContainerTransition())
RMNodeEvent> stateMachine;
public RMNodeImpl(NodeId nodeId, RMContext context, String hostName,
int cmPort, int httpPort, Node node, Resource capability) {
int cmPort, int httpPort, Node node, Resource capability, String nodeManagerVersion) {
this.nodeId = nodeId;
this.context = context;
this.hostName = hostName;
@ -184,6 +185,7 @@ public RMNodeImpl(NodeId nodeId, RMContext context, String hostName,
this.node = node;
this.healthReport = "Healthy";
this.lastHealthReportTime = System.currentTimeMillis();
this.nodeManagerVersion = nodeManagerVersion;
this.latestNodeHeartBeatResponse.setResponseId(0);
@ -288,6 +290,11 @@ public long getLastHealthReportTime() {
}
}
@Override
public String getNodeManagerVersion() {
return nodeManagerVersion;
}
@Override
public NodeState getState() {
this.readLock.lock();

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
View File

@ -76,6 +76,7 @@ protected void render(Block html) {
th(".containers", "Containers").
th(".mem", "Mem Used").
th(".mem", "Mem Avail").
th(".nodeManagerVersion", "Version").
_()._().
tbody();
NodeState stateFilter = null;
@ -129,6 +130,7 @@ protected void render(Block html) {
_(StringUtils.byteDesc(usedMemory * BYTES_IN_MB))._().
td().br().$title(String.valueOf(usedMemory))._().
_(StringUtils.byteDesc(availableMemory * BYTES_IN_MB))._().
td(ni.getNodeManagerVersion()).
_();
}
tbody._()._();

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
View File

@ -38,6 +38,7 @@ public class NodeInfo {
protected String nodeHostName;
protected String nodeHTTPAddress;
protected long lastHealthUpdate;
protected String version;
protected String healthReport;
protected int numContainers;
protected long usedMemoryMB;
@ -64,6 +65,7 @@ public NodeInfo(RMNode ni, ResourceScheduler sched) {
this.nodeHTTPAddress = ni.getHttpAddress();
this.lastHealthUpdate = ni.getLastHealthReportTime();
this.healthReport = String.valueOf(ni.getHealthReport());
this.version = ni.getNodeManagerVersion();
}
public String getRack() {
@ -90,6 +92,10 @@ public long getLastHealthUpdate() {
return this.lastHealthUpdate;
}
public String getVersion() {
return this.version;
}
public String getHealthReport() {
return this.healthReport;
}

11
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
View File

@ -40,6 +40,7 @@
import org.apache.hadoop.yarn.server.api.records.NodeStatus;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.Records;
import org.apache.hadoop.yarn.util.YarnVersionInfo;
public class MockNM {
@ -51,6 +52,7 @@ public class MockNM {
private final int httpPort = 2;
private MasterKey currentContainerTokenMasterKey;
private MasterKey currentNMTokenMasterKey;
private String version;
public MockNM(String nodeIdStr, int memory, ResourceTrackerService resourceTracker) {
// scale vcores based on the requested memory
@ -61,10 +63,16 @@ public MockNM(String nodeIdStr, int memory, ResourceTrackerService resourceTrack
}
public MockNM(String nodeIdStr, int memory, int vcores,
ResourceTrackerService resourceTracker) {
ResourceTrackerService resourceTracker) {
this(nodeIdStr, memory, vcores, resourceTracker, YarnVersionInfo.getVersion());
}
public MockNM(String nodeIdStr, int memory, int vcores,
ResourceTrackerService resourceTracker, String version) {
this.memory = memory;
this.vCores = vcores;
this.resourceTracker = resourceTracker;
this.version = version;
String[] splits = nodeIdStr.split(":");
nodeId = BuilderUtils.newNodeId(splits[0], Integer.parseInt(splits[1]));
}
@ -96,6 +104,7 @@ public RegisterNodeManagerResponse registerNode() throws Exception {
req.setHttpPort(httpPort);
Resource resource = BuilderUtils.newResource(memory, vCores);
req.setResource(resource);
req.setNMVersion(version);
RegisterNodeManagerResponse registrationResponse =
resourceTracker.registerNodeManager(req);
this.currentContainerTokenMasterKey =

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
View File

@ -183,6 +183,11 @@ public NodeHeartbeatResponse getLastNodeHeartBeatResponse() {
return null;
}
@Override
public String getNodeManagerVersion() {
return null;
}
@Override
public List<UpdatedContainerInfo> pullContainerUpdates() {
return new ArrayList<UpdatedContainerInfo>();

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
View File

@ -110,7 +110,7 @@ public Void answer(InvocationOnMock invocation) throws Throwable {
new TestSchedulerEventDispatcher());
NodeId nodeId = BuilderUtils.newNodeId("localhost", 0);
node = new RMNodeImpl(nodeId, rmContext, null, 0, 0, null, null);
node = new RMNodeImpl(nodeId, rmContext, null, 0, 0, null, null, null);
}
@ -166,7 +166,7 @@ public void testContainerUpdate() throws InterruptedException{
node.handle(new RMNodeEvent(null,RMNodeEventType.STARTED));
NodeId nodeId = BuilderUtils.newNodeId("localhost:1", 1);
RMNodeImpl node2 = new RMNodeImpl(nodeId, rmContext, null, 0, 0, null, null);
RMNodeImpl node2 = new RMNodeImpl(nodeId, rmContext, null, 0, 0, null, null, null);
node2.handle(new RMNodeEvent(null,RMNodeEventType.STARTED));
ContainerId completedContainerIdFromNode1 = BuilderUtils.newContainerId(
@ -432,7 +432,7 @@ public void testUpdateHeartbeatResponseForCleanup() {
private RMNodeImpl getRunningNode() {
NodeId nodeId = BuilderUtils.newNodeId("localhost", 0);
RMNodeImpl node = new RMNodeImpl(nodeId, rmContext,null, 0, 0,
null, null);
null, null, null);
node.handle(new RMNodeEvent(node.getNodeID(), RMNodeEventType.STARTED));
Assert.assertEquals(NodeState.RUNNING, node.getState());
return node;

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java
View File

@ -49,7 +49,7 @@ public class TestNodesPage {
// Number of Actual Table Headers for NodesPage.NodesBlock might change in
// future. In that case this value should be adjusted to the new value.
final int numberOfThInMetricsTable = 13;
final int numberOfActualTableHeaders = 9;
final int numberOfActualTableHeaders = 10;
private Injector injector;

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
View File

@ -655,13 +655,14 @@ public void verifyNodesXML(NodeList nodes, MockNM nm) throws JSONException,
WebServicesTestUtils.getXmlString(element, "healthReport"),
WebServicesTestUtils.getXmlInt(element, "numContainers"),
WebServicesTestUtils.getXmlLong(element, "usedMemoryMB"),
WebServicesTestUtils.getXmlLong(element, "availMemoryMB"));
WebServicesTestUtils.getXmlLong(element, "availMemoryMB"),
WebServicesTestUtils.getXmlString(element, "version"));
}
}
public void verifyNodeInfo(JSONObject nodeInfo, MockNM nm)
throws JSONException, Exception {
assertEquals("incorrect number of elements", 10, nodeInfo.length());
assertEquals("incorrect number of elements", 11, nodeInfo.length());
verifyNodeInfoGeneric(nm, nodeInfo.getString("state"),
nodeInfo.getString("rack"),
@ -669,14 +670,15 @@ public void verifyNodeInfo(JSONObject nodeInfo, MockNM nm)
nodeInfo.getString("nodeHTTPAddress"),
nodeInfo.getLong("lastHealthUpdate"),
nodeInfo.getString("healthReport"), nodeInfo.getInt("numContainers"),
nodeInfo.getLong("usedMemoryMB"), nodeInfo.getLong("availMemoryMB"));
nodeInfo.getLong("usedMemoryMB"), nodeInfo.getLong("availMemoryMB"),
nodeInfo.getString("version"));
}
public void verifyNodeInfoGeneric(MockNM nm, String state, String rack,
String id, String nodeHostName,
String nodeHTTPAddress, long lastHealthUpdate, String healthReport,
int numContainers, long usedMemoryMB, long availMemoryMB)
int numContainers, long usedMemoryMB, long availMemoryMB, String version)
throws JSONException, Exception {
RMNode node = rm.getRMContext().getRMNodes().get(nm.getNodeId());
@ -695,6 +697,8 @@ public void verifyNodeInfoGeneric(MockNM nm, String state, String rack,
+ nm.getHttpPort();
WebServicesTestUtils.checkStringMatch("nodeHTTPAddress",
expectedHttpAddress, nodeHTTPAddress);
WebServicesTestUtils.checkStringMatch("version",
node.getNodeManagerVersion(), version);
long expectedHealthUpdate = node.getLastHealthReportTime();
assertEquals("lastHealthUpdate doesn't match, got: " + lastHealthUpdate