Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-686. Flatten NodeReport. (sandyr via tucu) 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1490827 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Alejandro Abdelnur 2013-06-07 20:59:23 +00:00
parent 2a76cddcd5
commit af8514eef2
24 changed files with 171 additions and 166 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-yarn-project/CHANGES.txt
View File

@ -301,6 +301,8 @@ Release 2.1.0-beta - UNRELEASED
YARN-750. Allow for black-listing resources in YARN API and Impl in CS
(acmurthy via bikas)
YARN-686. Flatten NodeReport. (sandyr via tucu)
OPTIMIZATIONS
YARN-512. Log aggregation root directory check is more expensive than it

30
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java
View File

@ -51,7 +51,7 @@ public abstract class NodeReport {
@Private
public static NodeReport newInstance(NodeId nodeId, NodeState nodeState,
String httpAddress, String rackName, Resource used, Resource capability,
int numContainers, NodeHealthStatus nodeHealthStatus) {
int numContainers, String healthReport, long lastHealthReportTime) {
NodeReport nodeReport = Records.newRecord(NodeReport.class);
nodeReport.setNodeId(nodeId);
nodeReport.setNodeState(nodeState);
@ -60,7 +60,8 @@ public static NodeReport newInstance(NodeId nodeId, NodeState nodeState,
nodeReport.setUsed(used);
nodeReport.setCapability(capability);
nodeReport.setNumContainers(numContainers);
nodeReport.setNodeHealthStatus(nodeHealthStatus);
nodeReport.setHealthReport(healthReport);
nodeReport.setLastHealthReportTime(lastHealthReportTime);
return nodeReport;
}
@ -144,15 +145,28 @@ public static NodeReport newInstance(NodeId nodeId, NodeState nodeState,
@Unstable
public abstract void setNumContainers(int numContainers);
/**
* Get the <code>NodeHealthStatus</code> of the node.
* @return <code>NodeHealthStatus</code> of the node
/**
* Get the <em>diagnostic health report</em> of the node.
* @return <em>diagnostic health report</em> of the node
*/
@Public
@Stable
public abstract NodeHealthStatus getNodeHealthStatus();
public abstract String getHealthReport();
@Private
@Unstable
public abstract void setNodeHealthStatus(NodeHealthStatus nodeHealthStatus);
public abstract void setHealthReport(String healthReport);
/**
* Get the <em>last timestamp</em> at which the health report was received.
* @return <em>last timestamp</em> at which the health report was received
*/
@Public
@Stable
public abstract long getLastHealthReportTime();
@Private
@Unstable
public abstract void setLastHealthReportTime(long lastHealthReport);
}

59
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java
View File

@ -18,12 +18,10 @@
package org.apache.hadoop.yarn.api.records.impl.pb;
import org.apache.hadoop.yarn.api.records.NodeHealthStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.proto.YarnProtos.NodeHealthStatusProto;
import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.NodeReportProto;
import org.apache.hadoop.yarn.proto.YarnProtos.NodeReportProtoOrBuilder;
@ -38,7 +36,6 @@ public class NodeReportPBImpl extends NodeReport {
private NodeId nodeId;
private Resource used;
private Resource capability;
private NodeHealthStatus nodeHealthStatus;
public NodeReportPBImpl() {
builder = NodeReportProto.newBuilder();
@ -64,19 +61,33 @@ public Resource getCapability() {
}
@Override
public NodeHealthStatus getNodeHealthStatus() {
if (this.nodeHealthStatus != null) {
return this.nodeHealthStatus;
}
public String getHealthReport() {
NodeReportProtoOrBuilder p = viaProto ? proto : builder;
if (!p.hasNodeHealthStatus()) {
return null;
}
this.nodeHealthStatus = convertFromProtoFormat(p.getNodeHealthStatus());
return this.nodeHealthStatus;
return p.getHealthReport();
}
@Override
public void setHealthReport(String healthReport) {
maybeInitBuilder();
if (healthReport == null) {
builder.clearHealthReport();
return;
}
builder.setHealthReport(healthReport);
}
@Override
public long getLastHealthReportTime() {
NodeReportProtoOrBuilder p = viaProto ? proto : builder;
return p.getLastHealthReportTime();
}
@Override
public void setLastHealthReportTime(long lastHealthReportTime) {
maybeInitBuilder();
builder.setLastHealthReportTime(lastHealthReportTime);
}
@Override
public String getHttpAddress() {
NodeReportProtoOrBuilder p = viaProto ? proto : builder;
@ -158,14 +169,6 @@ public void setCapability(Resource capability) {
this.capability = capability;
}
@Override
public void setNodeHealthStatus(NodeHealthStatus healthStatus) {
maybeInitBuilder();
if (healthStatus == null)
builder.clearNodeHealthStatus();
this.nodeHealthStatus = healthStatus;
}
@Override
public void setHttpAddress(String httpAddress) {
maybeInitBuilder();
@ -247,11 +250,6 @@ private void mergeLocalToBuilder() {
builder.getCapability())) {
builder.setCapability(convertToProtoFormat(this.capability));
}
if (this.nodeHealthStatus != null
&& !((NodeHealthStatusPBImpl) this.nodeHealthStatus).getProto().equals(
builder.getNodeHealthStatus())) {
builder.setNodeHealthStatus(convertToProtoFormat(this.nodeHealthStatus));
}
}
private void mergeLocalToProto() {
@ -286,11 +284,4 @@ private ResourceProto convertToProtoFormat(Resource r) {
return ((ResourcePBImpl) r).getProto();
}
private NodeHealthStatusPBImpl convertFromProtoFormat(NodeHealthStatusProto p) {
return new NodeHealthStatusPBImpl(p);
}
private NodeHealthStatusProto convertToProtoFormat(NodeHealthStatus r) {
return ((NodeHealthStatusPBImpl) r).getProto();
}
}

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
View File

@ -191,8 +191,9 @@ message NodeReportProto {
optional ResourceProto used = 4;
optional ResourceProto capability = 5;
optional int32 numContainers = 6;
optional NodeHealthStatusProto node_health_status = 8;
optional NodeStateProto node_state = 9;
optional NodeStateProto node_state = 7;
optional string health_report = 8;
optional int64 last_health_report_time = 9;
}

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
View File

@ -330,8 +330,7 @@ public boolean run() throws IOException, YarnException {
+ ", nodeId=" + node.getNodeId()
+ ", nodeAddress" + node.getHttpAddress()
+ ", nodeRackName" + node.getRackName()
+ ", nodeNumContainers" + node.getNumContainers()
+ ", nodeHealthStatus" + node.getNodeHealthStatus());
+ ", nodeNumContainers" + node.getNumContainers());
}
QueueInfo queueInfo = super.getQueueInfo(this.amQueue);

14
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java
View File

@ -35,7 +35,7 @@
import org.apache.hadoop.yarn.util.ConverterUtils;
public class NodeCLI extends YarnCLI {
private static final String NODES_PATTERN = "%16s\t%10s\t%17s\t%26s\t%18s" +
private static final String NODES_PATTERN = "%16s\t%10s\t%17s\t%18s" +
System.getProperty("line.separator");
public static void main(String[] args) throws Exception {
@ -91,11 +91,10 @@ private void listClusterNodes() throws YarnException, IOException {
List<NodeReport> nodesReport = client.getNodeReports();
writer.println("Total Nodes:" + nodesReport.size());
writer.printf(NODES_PATTERN, "Node-Id", "Node-State", "Node-Http-Address",
"Health-Status(isNodeHealthy)", "Running-Containers");
"Running-Containers");
for (NodeReport nodeReport : nodesReport) {
writer.printf(NODES_PATTERN, nodeReport.getNodeId(), nodeReport
.getNodeState(), nodeReport.getHttpAddress(), nodeReport
.getNodeHealthStatus().getIsNodeHealthy(), nodeReport
.getNumContainers());
}
writer.flush();
@ -129,16 +128,13 @@ private void printNodeStatus(String nodeIdStr) throws YarnException,
nodeReportStr.println(nodeReport.getNodeState());
nodeReportStr.print("\tNode-Http-Address : ");
nodeReportStr.println(nodeReport.getHttpAddress());
nodeReportStr.print("\tHealth-Status(isNodeHealthy) : ");
nodeReportStr.println(nodeReport.getNodeHealthStatus()
.getIsNodeHealthy());
nodeReportStr.print("\tLast-Health-Update : ");
nodeReportStr.println(DateFormatUtils.format(
new Date(nodeReport.getNodeHealthStatus().
getLastHealthReportTime()),"E dd/MMM/yy hh:mm:ss:SSzz"));
new Date(nodeReport.getLastHealthReportTime()),
"E dd/MMM/yy hh:mm:ss:SSzz"));
nodeReportStr.print("\tHealth-Report : ");
nodeReportStr
.println(nodeReport.getNodeHealthStatus().getHealthReport());
.println(nodeReport.getHealthReport());
nodeReportStr.print("\tContainers : ");
nodeReportStr.println(nodeReport.getNumContainers());
nodeReportStr.print("\tMemory-Used : ");

15
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
View File

@ -41,7 +41,6 @@
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeHealthStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.NodeState;
@ -163,13 +162,13 @@ public void testListClusterNodes() throws Exception {
PrintWriter pw = new PrintWriter(baos);
pw.println("Total Nodes:3");
pw.print(" Node-Id\tNode-State\tNode-Http-Address\t");
pw.println("Health-Status(isNodeHealthy)\tRunning-Containers");
pw.println("Running-Containers");
pw.print(" host0:0\t RUNNING\t host1:8888");
pw.println("\t false\t 0");
pw.println("\t 0");
pw.print(" host1:0\t RUNNING\t host1:8888");
pw.println("\t false\t 0");
pw.println("\t 0");
pw.print(" host2:0\t RUNNING\t host1:8888");
pw.println("\t false\t 0");
pw.println("\t 0");
pw.close();
String nodesReportStr = baos.toString("UTF-8");
Assert.assertEquals(nodesReportStr, sysOutStream.toString());
@ -194,10 +193,9 @@ public void testNodeStatus() throws Exception {
pw.println("\tRack : rack1");
pw.println("\tNode-State : RUNNING");
pw.println("\tNode-Http-Address : host1:8888");
pw.println("\tHealth-Status(isNodeHealthy) : false");
pw.println("\tLast-Health-Update : "
+ DateFormatUtils.format(new Date(0), "E dd/MMM/yy hh:mm:ss:SSzz"));
pw.println("\tHealth-Report : null");
pw.println("\tHealth-Report : ");
pw.println("\tContainers : 0");
pw.println("\tMemory-Used : 0M");
pw.println("\tMemory-Capacity : 0");
@ -246,8 +244,7 @@ private List<NodeReport> getNodeReports(int noOfNodes) {
NodeReport nodeReport = NodeReport.newInstance(NodeId
.newInstance("host" + i, 0), NodeState.RUNNING, "host" + 1 + ":8888",
"rack1", Records.newRecord(Resource.class), Records
.newRecord(Resource.class), 0, Records
.newRecord(NodeHealthStatus.class));
.newRecord(Resource.class), 0, "", 0);
nodeReports.add(nodeReport);
}
return nodeReports;

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
View File

@ -171,7 +171,7 @@ public static NodeId newNodeId(String host, int port) {
public static NodeReport newNodeReport(NodeId nodeId, NodeState nodeState,
String httpAddress, String rackName, Resource used, Resource capability,
int numContainers, NodeHealthStatus nodeHealthStatus) {
int numContainers, String healthReport, long lastHealthReportTime) {
NodeReport nodeReport = recordFactory.newRecordInstance(NodeReport.class);
nodeReport.setNodeId(nodeId);
nodeReport.setNodeState(nodeState);
@ -180,7 +180,8 @@ public static NodeReport newNodeReport(NodeId nodeId, NodeState nodeState,
nodeReport.setUsed(used);
nodeReport.setCapability(capability);
nodeReport.setNumContainers(numContainers);
nodeReport.setNodeHealthStatus(nodeHealthStatus);
nodeReport.setHealthReport(healthReport);
nodeReport.setLastHealthReportTime(lastHealthReportTime);
return nodeReport;
}

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
View File

@ -343,7 +343,8 @@ public AllocateResponse allocate(AllocateRequest request)
rmNode.getState(),
rmNode.getHttpAddress(), rmNode.getRackName(), used,
rmNode.getTotalCapability(), numContainers,
rmNode.getNodeHealthStatus());
rmNode.getHealthReport(),
rmNode.getLastHealthReportTime());
updatedNodeReports.add(report);
}

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
View File

@ -475,7 +475,8 @@ private NodeReport createNodeReports(RMNode rmNode) {
rmNode.getState(),
rmNode.getHttpAddress(), rmNode.getRackName(), used,
rmNode.getTotalCapability(), numContainers,
rmNode.getNodeHealthStatus());
rmNode.getHealthReport(),
rmNode.getLastHealthReportTime());
return report;
}

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java
View File

@ -86,13 +86,10 @@ public String getLiveNodeManagers() {
info.put("State", ni.getState().toString());
info.put("NodeId", ni.getNodeID());
info.put("NodeHTTPAddress", ni.getHttpAddress());
info.put("HealthStatus",
ni.getNodeHealthStatus().getIsNodeHealthy() ?
"Healthy" : "Unhealthy");
info.put("LastHealthUpdate",
ni.getNodeHealthStatus().getLastHealthReportTime());
ni.getLastHealthReportTime());
info.put("HealthReport",
ni.getNodeHealthStatus().getHealthReport());
ni.getHealthReport());
if(report != null) {
info.put("NumContainers", report.getNumContainers());
info.put("UsedMemoryMB", report.getUsedResource().getMemory());

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
View File

@ -74,10 +74,16 @@ public interface RMNode {
public String getHttpAddress();
/**
* the health-status for this node
* @return the health-status for this node.
* the latest health report received from this node.
* @return the latest health report received from this node.
*/
public NodeHealthStatus getNodeHealthStatus();
public String getHealthReport();
/**
* the time of the latest health report received from this node.
* @return the time of the latest health report received from this node.
*/
public long getLastHealthReportTime();
/**
* the total available resource.

55
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
View File

@ -93,9 +93,10 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
private final String httpAddress;
private final Resource totalCapability;
private final Node node;
private final NodeHealthStatus nodeHealthStatus = recordFactory
.newRecordInstance(NodeHealthStatus.class);
private String healthReport;
private long lastHealthReportTime;
/* set of containers that have just launched */
private final Map<ContainerId, ContainerStatus> justLaunchedContainers =
new HashMap<ContainerId, ContainerStatus>();
@ -180,9 +181,8 @@ public RMNodeImpl(NodeId nodeId, RMContext context, String hostName,
this.nodeAddress = hostName + ":" + cmPort;
this.httpAddress = hostName + ":" + httpPort;
this.node = node;
this.nodeHealthStatus.setIsNodeHealthy(true);
this.nodeHealthStatus.setHealthReport("Healthy");
this.nodeHealthStatus.setLastHealthReportTime(System.currentTimeMillis());
this.healthReport = "Healthy";
this.lastHealthReportTime = System.currentTimeMillis();
this.latestNodeHeartBeatResponse.setResponseId(0);
@ -246,27 +246,46 @@ public Node getNode() {
}
@Override
public NodeHealthStatus getNodeHealthStatus() {
public String getHealthReport() {
this.readLock.lock();
try {
return this.nodeHealthStatus;
return this.healthReport;
} finally {
this.readLock.unlock();
}
}
private void setNodeHealthStatus(NodeHealthStatus status)
{
public void setHealthReport(String healthReport) {
this.writeLock.lock();
try {
this.nodeHealthStatus.setHealthReport(status.getHealthReport());
this.nodeHealthStatus.setIsNodeHealthy(status.getIsNodeHealthy());
this.nodeHealthStatus.setLastHealthReportTime(status.getLastHealthReportTime());
this.healthReport = healthReport;
} finally {
this.writeLock.unlock();
}
}
public void setLastHealthReportTime(long lastHealthReportTime) {
this.writeLock.lock();
try {
this.lastHealthReportTime = lastHealthReportTime;
} finally {
this.writeLock.unlock();
}
}
@Override
public long getLastHealthReportTime() {
this.readLock.lock();
try {
return this.lastHealthReportTime;
} finally {
this.readLock.unlock();
}
}
@Override
public NodeState getState() {
@ -511,7 +530,9 @@ public NodeState transition(RMNodeImpl rmNode, RMNodeEvent event) {
NodeHealthStatus remoteNodeHealthStatus =
statusEvent.getNodeHealthStatus();
rmNode.setNodeHealthStatus(remoteNodeHealthStatus);
rmNode.setHealthReport(remoteNodeHealthStatus.getHealthReport());
rmNode.setLastHealthReportTime(
remoteNodeHealthStatus.getLastHealthReportTime());
if (!remoteNodeHealthStatus.getIsNodeHealthy()) {
LOG.info("Node " + rmNode.nodeId + " reported UNHEALTHY with details: "
+ remoteNodeHealthStatus.getHealthReport());
@ -593,7 +614,9 @@ public NodeState transition(RMNodeImpl rmNode, RMNodeEvent event) {
// Switch the last heartbeatresponse.
rmNode.latestNodeHeartBeatResponse = statusEvent.getLatestResponse();
NodeHealthStatus remoteNodeHealthStatus = statusEvent.getNodeHealthStatus();
rmNode.setNodeHealthStatus(remoteNodeHealthStatus);
rmNode.setHealthReport(remoteNodeHealthStatus.getHealthReport());
rmNode.setLastHealthReportTime(
remoteNodeHealthStatus.getLastHealthReportTime());
if (remoteNodeHealthStatus.getIsNodeHealthy()) {
rmNode.context.getDispatcher().getEventHandler().handle(
new NodeAddedSchedulerEvent(rmNode));

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
View File

@ -71,7 +71,6 @@ protected void render(Block html) {
th(".state", "Node State").
th(".nodeaddress", "Node Address").
th(".nodehttpaddress", "Node HTTP Address").
th(".healthStatus", "Health-status").
th(".lastHealthUpdate", "Last health-update").
th(".healthReport", "Health-report").
th(".containers", "Containers").
@ -122,8 +121,7 @@ protected void render(Block html) {
row.td().a(HttpConfig.getSchemePrefix() + httpAddress,
httpAddress)._();
}
row.td(info.getHealthStatus()).
td().br().$title(String.valueOf(info.getLastHealthUpdate()))._().
row.td().br().$title(String.valueOf(info.getLastHealthUpdate()))._().
_(Times.format(info.getLastHealthUpdate()))._().
td(info.getHealthReport()).
td(String.valueOf(info.getNumContainers())).

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
View File

@ -196,7 +196,8 @@ public NodesInfo getNodes(@QueryParam("state") String filterState,
String msg = "Error: You must specify either true or false to query on health";
throw new BadRequestException(msg);
}
if (nodeInfo.isHealthy() != Boolean.parseBoolean(healthState)) {
if ((ni.getState() != NodeState.UNHEALTHY)
!= Boolean.parseBoolean(healthState)) {
continue;
}
}

21
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
View File

@ -21,9 +21,7 @@
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.XmlTransient;
import org.apache.hadoop.yarn.api.records.NodeHealthStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
@ -39,23 +37,18 @@ public class NodeInfo {
protected String id;
protected String nodeHostName;
protected String nodeHTTPAddress;
protected String healthStatus;
protected long lastHealthUpdate;
protected String healthReport;
protected int numContainers;
protected long usedMemoryMB;
protected long availMemoryMB;
@XmlTransient
protected boolean healthy;
public NodeInfo() {
} // JAXB needs this
public NodeInfo(RMNode ni, ResourceScheduler sched) {
NodeId id = ni.getNodeID();
SchedulerNodeReport report = sched.getNodeReport(id);
NodeHealthStatus health = ni.getNodeHealthStatus();
this.numContainers = 0;
this.usedMemoryMB = 0;
this.availMemoryMB = 0;
@ -69,14 +62,8 @@ public NodeInfo(RMNode ni, ResourceScheduler sched) {
this.nodeHostName = ni.getHostName();
this.state = ni.getState();
this.nodeHTTPAddress = ni.getHttpAddress();
this.healthy = health.getIsNodeHealthy();
this.healthStatus = health.getIsNodeHealthy() ? "Healthy" : "Unhealthy";
this.lastHealthUpdate = health.getLastHealthReportTime();
this.healthReport = String.valueOf(health.getHealthReport());
}
public boolean isHealthy() {
return this.healthy;
this.lastHealthUpdate = ni.getLastHealthReportTime();
this.healthReport = String.valueOf(ni.getHealthReport());
}
public String getRack() {
@ -99,10 +86,6 @@ public void setNodeHTTPAddress(String nodeHTTPAddress) {
this.nodeHTTPAddress = nodeHTTPAddress;
}
public String getHealthStatus() {
return this.healthStatus;
}
public long getLastHealthUpdate() {
return this.lastHealthUpdate;
}

35
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
View File

@ -24,7 +24,6 @@
import org.apache.hadoop.net.Node;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.NodeHealthStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Resource;
@ -96,18 +95,20 @@ private static class MockRMNodeImpl implements RMNode {
private int cmdPort;
private Resource perNode;
private String rackName;
private NodeHealthStatus nodeHealthStatus;
private String healthReport;
private long lastHealthReportTime;
private NodeState state;
public MockRMNodeImpl(NodeId nodeId, String nodeAddr, String httpAddress,
Resource perNode, String rackName, NodeHealthStatus nodeHealthStatus,
int cmdPort, String hostName, NodeState state) {
Resource perNode, String rackName, String healthReport,
long lastHealthReportTime, int cmdPort, String hostName, NodeState state) {
this.nodeId = nodeId;
this.nodeAddr = nodeAddr;
this.httpAddress = httpAddress;
this.perNode = perNode;
this.rackName = rackName;
this.nodeHealthStatus = nodeHealthStatus;
this.healthReport = healthReport;
this.lastHealthReportTime = lastHealthReportTime;
this.cmdPort = cmdPort;
this.hostName = hostName;
this.state = state;
@ -143,11 +144,6 @@ public String getHttpAddress() {
return this.httpAddress;
}
@Override
public NodeHealthStatus getNodeHealthStatus() {
return this.nodeHealthStatus;
}
@Override
public Resource getTotalCapability() {
return this.perNode;
@ -191,6 +187,16 @@ public NodeHeartbeatResponse getLastNodeHeartBeatResponse() {
public List<UpdatedContainerInfo> pullContainerUpdates() {
return new ArrayList<UpdatedContainerInfo>();
}
@Override
public String getHealthReport() {
return healthReport;
}
@Override
public long getLastHealthReportTime() {
return lastHealthReportTime;
}
};
private static RMNode buildRMNode(int rack, final Resource perNode, NodeState state, String httpAddr) {
@ -209,14 +215,9 @@ private static RMNode buildRMNode(int rack, final Resource perNode,
final NodeId nodeID = NodeId.newInstance(hostName, port);
final String httpAddress = httpAddr;
final NodeHealthStatus nodeHealthStatus =
recordFactory.newRecordInstance(NodeHealthStatus.class);
if (state != NodeState.UNHEALTHY) {
nodeHealthStatus.setIsNodeHealthy(true);
nodeHealthStatus.setHealthReport("HealthyMe");
}
String healthReport = (state == NodeState.UNHEALTHY) ? null : "HealthyMe";
return new MockRMNodeImpl(nodeID, nodeAddr, httpAddress, perNode, rackName,
nodeHealthStatus, nid, hostName, state);
healthReport, 0, nid, hostName, state);
}
public static RMNode nodeInfo(int rack, final Resource perNode,

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
View File

@ -53,6 +53,7 @@
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@ -133,8 +134,8 @@ protected ClientRMService createClientRMService() {
List<NodeReport> nodeReports =
client.getClusterNodes(request).getNodeReports();
Assert.assertEquals(1, nodeReports.size());
Assert.assertTrue("Node is expected to be healthy!", nodeReports.get(0)
.getNodeHealthStatus().getIsNodeHealthy());
Assert.assertNotSame("Node is expected to be healthy!", NodeState.UNHEALTHY,
nodeReports.get(0).getNodeState());
// Now make the node unhealthy.
node.nodeHeartbeat(false);
@ -142,8 +143,8 @@ protected ClientRMService createClientRMService() {
// Call again
nodeReports = client.getClusterNodes(request).getNodeReports();
Assert.assertEquals(1, nodeReports.size());
Assert.assertFalse("Node is expected to be unhealthy!", nodeReports.get(0)
.getNodeHealthStatus().getIsNodeHealthy());
Assert.assertEquals("Node is expected to be unhealthy!", NodeState.UNHEALTHY,
nodeReports.get(0).getNodeState());
}
@Test

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
View File

@ -346,9 +346,8 @@ private RMNodeImpl getRunningNode() {
private RMNodeImpl getUnhealthyNode() {
RMNodeImpl node = getRunningNode();
NodeHealthStatus status = node.getNodeHealthStatus();
status.setHealthReport("sick");
status.setIsNodeHealthy(false);
NodeHealthStatus status = NodeHealthStatus.newInstance(false, "sick",
System.currentTimeMillis());
node.handle(new RMNodeStatusEvent(node.getNodeID(), status,
new ArrayList<ContainerStatus>(), null, null));
Assert.assertEquals(NodeState.UNHEALTHY, node.getState());

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java
View File

@ -176,11 +176,8 @@ public void testNodeHealthReportIsNotNull() throws Exception{
nm1.heartbeat();
nm1.heartbeat();
Collection<RMNode> values = resourceManager.getRMContext().getRMNodes().values();
for (RMNode ni : values)
{
NodeHealthStatus nodeHealthStatus = ni.getNodeHealthStatus();
String healthReport = nodeHealthStatus.getHealthReport();
assertNotNull(healthReport);
for (RMNode ni : values) {
assertNotNull(ni.getHealthReport());
}
}

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
View File

@ -32,6 +32,7 @@
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
@ -391,15 +392,15 @@ private void checkUnealthyNMCount(MockRM rm, MockNM nm1, boolean health,
int count) throws Exception {
int waitCount = 0;
while(rm.getRMContext().getRMNodes().get(nm1.getNodeId())
.getNodeHealthStatus().getIsNodeHealthy() == health
while((rm.getRMContext().getRMNodes().get(nm1.getNodeId())
.getState() != NodeState.UNHEALTHY) == health
&& waitCount++ < 20) {
synchronized (this) {
wait(100);
}
}
Assert.assertFalse(rm.getRMContext().getRMNodes().get(nm1.getNodeId())
.getNodeHealthStatus().getIsNodeHealthy() == health);
Assert.assertFalse((rm.getRMContext().getRMNodes().get(nm1.getNodeId())
.getState() != NodeState.UNHEALTHY) == health);
Assert.assertEquals("Unhealthy metrics not incremented", count,
ClusterMetrics.getMetrics().getUnhealthyNMs());
}

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java
View File

@ -49,7 +49,7 @@ public class TestNodesPage {
// Number of Actual Table Headers for NodesPage.NodesBlock might change in
// future. In that case this value should be adjusted to the new value.
final int numberOfThInMetricsTable = 13;
final int numberOfActualTableHeaders = 10;
final int numberOfActualTableHeaders = 9;
private Injector injector;

24
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
View File

@ -142,9 +142,8 @@ public void testNodesDefaultWithUnHealthyNode() throws JSONException,
rm.NMwaitForState(nm3.getNodeId(), NodeState.RUNNING);
RMNodeImpl node = (RMNodeImpl) rm.getRMContext().getRMNodes()
.get(nm3.getNodeId());
NodeHealthStatus nodeHealth = node.getNodeHealthStatus();
nodeHealth.setHealthReport("test health report");
nodeHealth.setIsNodeHealthy(false);
NodeHealthStatus nodeHealth = NodeHealthStatus.newInstance(false,
"test health report", System.currentTimeMillis());
node.handle(new RMNodeStatusEvent(nm3.getNodeId(), nodeHealth,
new ArrayList<ContainerStatus>(), null, null));
rm.NMwaitForState(nm3.getNodeId(), NodeState.UNHEALTHY);
@ -357,9 +356,8 @@ public void testNodesQueryHealthyAndState() throws JSONException, Exception {
rm.NMwaitForState(nm1.getNodeId(), NodeState.RUNNING);
RMNodeImpl node = (RMNodeImpl) rm.getRMContext().getRMNodes()
.get(nm1.getNodeId());
NodeHealthStatus nodeHealth = node.getNodeHealthStatus();
nodeHealth.setHealthReport("test health report");
nodeHealth.setIsNodeHealthy(false);
NodeHealthStatus nodeHealth = NodeHealthStatus.newInstance(false,
"test health report", System.currentTimeMillis());
node.handle(new RMNodeStatusEvent(nm1.getNodeId(), nodeHealth,
new ArrayList<ContainerStatus>(), null, null));
rm.NMwaitForState(nm1.getNodeId(), NodeState.UNHEALTHY);
@ -699,7 +697,6 @@ public void verifyNodesXML(NodeList nodes, MockNM nm) throws JSONException,
verifyNodeInfoGeneric(nm,
WebServicesTestUtils.getXmlString(element, "state"),
WebServicesTestUtils.getXmlString(element, "rack"),
WebServicesTestUtils.getXmlString(element, "healthStatus"),
WebServicesTestUtils.getXmlString(element, "id"),
WebServicesTestUtils.getXmlString(element, "nodeHostName"),
WebServicesTestUtils.getXmlString(element, "nodeHTTPAddress"),
@ -713,10 +710,10 @@ public void verifyNodesXML(NodeList nodes, MockNM nm) throws JSONException,
public void verifyNodeInfo(JSONObject nodeInfo, MockNM nm)
throws JSONException, Exception {
assertEquals("incorrect number of elements", 11, nodeInfo.length());
assertEquals("incorrect number of elements", 10, nodeInfo.length());
verifyNodeInfoGeneric(nm, nodeInfo.getString("state"),
nodeInfo.getString("rack"), nodeInfo.getString("healthStatus"),
nodeInfo.getString("rack"),
nodeInfo.getString("id"), nodeInfo.getString("nodeHostName"),
nodeInfo.getString("nodeHTTPAddress"),
nodeInfo.getLong("lastHealthUpdate"),
@ -726,32 +723,29 @@ public void verifyNodeInfo(JSONObject nodeInfo, MockNM nm)
}
public void verifyNodeInfoGeneric(MockNM nm, String state, String rack,
String healthStatus, String id, String nodeHostName,
String id, String nodeHostName,
String nodeHTTPAddress, long lastHealthUpdate, String healthReport,
int numContainers, long usedMemoryMB, long availMemoryMB)
throws JSONException, Exception {
RMNode node = rm.getRMContext().getRMNodes().get(nm.getNodeId());
NodeHealthStatus health = node.getNodeHealthStatus();
ResourceScheduler sched = rm.getResourceScheduler();
SchedulerNodeReport report = sched.getNodeReport(nm.getNodeId());
WebServicesTestUtils.checkStringMatch("state", node.getState().toString(),
state);
WebServicesTestUtils.checkStringMatch("rack", node.getRackName(), rack);
WebServicesTestUtils.checkStringMatch("healthStatus", "Healthy",
healthStatus);
WebServicesTestUtils.checkStringMatch("id", nm.getNodeId().toString(), id);
WebServicesTestUtils.checkStringMatch("nodeHostName", nm.getNodeId()
.getHost(), nodeHostName);
WebServicesTestUtils.checkStringMatch("healthReport",
String.valueOf(health.getHealthReport()), healthReport);
String.valueOf(node.getHealthReport()), healthReport);
String expectedHttpAddress = nm.getNodeId().getHost() + ":"
+ nm.getHttpPort();
WebServicesTestUtils.checkStringMatch("nodeHTTPAddress",
expectedHttpAddress, nodeHTTPAddress);
long expectedHealthUpdate = health.getLastHealthReportTime();
long expectedHealthUpdate = node.getLastHealthReportTime();
assertEquals("lastHealthUpdate doesn't match, got: " + lastHealthUpdate
+ " expected: " + expectedHealthUpdate, expectedHealthUpdate,
lastHealthUpdate);

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestDiskFailures.java
View File

@ -27,6 +27,7 @@
import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.MiniYARNCluster;
import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
@ -243,7 +244,7 @@ private void verifyDisksHealth(boolean localORLogDirs, String expectedDirs,
for (int i = 0; i < 10; i++) {
Iterator<RMNode> iter = yarnCluster.getResourceManager().getRMContext()
.getRMNodes().values().iterator();
if (iter.next().getNodeHealthStatus().getIsNodeHealthy() == isHealthy) {
if ((iter.next().getState() != NodeState.UNHEALTHY) == isHealthy) {
break;
}
// wait for the node health info to go to RM
@ -256,7 +257,7 @@ private void verifyDisksHealth(boolean localORLogDirs, String expectedDirs,
Iterator<RMNode> iter = yarnCluster.getResourceManager().getRMContext()
.getRMNodes().values().iterator();
Assert.assertEquals("RM is not updated with the health status of a node",
isHealthy, iter.next().getNodeHealthStatus().getIsNodeHealthy());
isHealthy, iter.next().getState() != NodeState.UNHEALTHY);
}
/**