YARN-2072. RM/NM UIs and webservices are missing vcore information. (Nathan Roberts via tgraves)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1605162 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Thomas Graves 2014-06-24 19:34:34 +00:00
parent fdbff67856
commit 1f9a0fd927
23 changed files with 252 additions and 29 deletions

View File

@ -183,6 +183,9 @@ Release 2.5.0 - UNRELEASED
YARN-2109. Fix TestRM to work with both schedulers. (Anubhav Dhoot via kasha)
YARN-2072. RM/NM UIs and webservices are missing vcore information.
(Nathan Roberts via tgraves)
OPTIMIZATIONS
BUG FIXES

View File

@ -27,4 +27,6 @@ public interface ResourceView {
long getPmemAllocatedForContainers();
boolean isPmemCheckEnabled();
long getVCoresAllocatedForContainers();
}

View File

@ -70,6 +70,8 @@ public class ContainersMonitorImpl extends AbstractService implements
private boolean pmemCheckEnabled;
private boolean vmemCheckEnabled;
private long maxVCoresAllottedForContainers;
private static final long UNKNOWN_MEMORY_LIMIT = -1L;
public ContainersMonitorImpl(ContainerExecutor exec,
@ -108,10 +110,16 @@ public class ContainersMonitorImpl extends AbstractService implements
YarnConfiguration.NM_PMEM_MB,
YarnConfiguration.DEFAULT_NM_PMEM_MB) * 1024 * 1024l;
long configuredVCoresForContainers = conf.getLong(
YarnConfiguration.NM_VCORES,
YarnConfiguration.DEFAULT_NM_VCORES);
// Setting these irrespective of whether checks are enabled. Required in
// the UI.
// ///////// Physical memory configuration //////
this.maxPmemAllottedForContainers = configuredPMemForContainers;
this.maxVCoresAllottedForContainers = configuredVCoresForContainers;
// ///////// Virtual memory configuration //////
float vmemRatio = conf.getFloat(YarnConfiguration.NM_VMEM_PMEM_RATIO,
@ -518,6 +526,11 @@ public class ContainersMonitorImpl extends AbstractService implements
return this.maxPmemAllottedForContainers;
}
@Override
public long getVCoresAllocatedForContainers() {
return this.maxVCoresAllottedForContainers;
}
/**
* Is the total virtual memory check enabled?
*

View File

@ -40,6 +40,9 @@ public class NodeManagerMetrics {
@Metric("Current # of allocated containers")
MutableGaugeInt allocatedContainers;
@Metric MutableGaugeInt availableGB;
@Metric("Current allocated Virtual Cores")
MutableGaugeInt allocatedVCores;
@Metric MutableGaugeInt availableVCores;
public static NodeManagerMetrics create() {
return create(DefaultMetricsSystem.instance());
@ -88,16 +91,21 @@ public class NodeManagerMetrics {
allocatedContainers.incr();
allocatedGB.incr(res.getMemory() / 1024);
availableGB.decr(res.getMemory() / 1024);
allocatedVCores.incr(res.getVirtualCores());
availableVCores.decr(res.getVirtualCores());
}
public void releaseContainer(Resource res) {
allocatedContainers.decr();
allocatedGB.decr(res.getMemory() / 1024);
availableGB.incr(res.getMemory() / 1024);
allocatedVCores.decr(res.getVirtualCores());
availableVCores.incr(res.getVirtualCores());
}
public void addResource(Resource res) {
availableGB.incr(res.getMemory() / 1024);
availableVCores.incr(res.getVirtualCores());
}
public int getRunningContainers() {

View File

@ -85,6 +85,7 @@ public class ContainerPage extends NMView implements YarnWebParams {
._("Diagnostics", info.getDiagnostics())
._("User", info.getUser())
._("TotalMemoryNeeded", info.getMemoryNeeded())
._("TotalVCoresNeeded", info.getVCoresNeeded())
._("logs", info.getShortLogLink(), "Link to logs");
html._(InfoBlock.class);
}

View File

@ -73,6 +73,8 @@ public class NodePage extends NMView {
StringUtils.byteDesc(info.getTotalPmemAllocated() * BYTES_IN_MB))
._("Pmem enforcement enabled",
info.isVmemCheckEnabled())
._("Total VCores allocated for Containers",
String.valueOf(info.getTotalVCoresAllocated()))
._("NodeHealthyStatus",
info.getHealthStatus())
._("LastNodeHealthTime", new Date(

View File

@ -42,6 +42,7 @@ public class ContainerInfo {
protected String diagnostics;
protected String user;
protected long totalMemoryNeededMB;
protected long totalVCoresNeeded;
protected String containerLogsLink;
protected String nodeId;
@XmlTransient
@ -76,6 +77,7 @@ public class ContainerInfo {
Resource res = container.getResource();
if (res != null) {
this.totalMemoryNeededMB = res.getMemory();
this.totalVCoresNeeded = res.getVirtualCores();
}
this.containerLogsShortLink = ujoin("containerlogs", this.id,
container.getUser());
@ -130,4 +132,8 @@ public class ContainerInfo {
return this.totalMemoryNeededMB;
}
public long getVCoresNeeded() {
return this.totalVCoresNeeded;
}
}

View File

@ -36,6 +36,7 @@ public class NodeInfo {
protected String healthReport;
protected long totalVmemAllocatedContainersMB;
protected long totalPmemAllocatedContainersMB;
protected long totalVCoresAllocatedContainers;
protected boolean vmemCheckEnabled;
protected boolean pmemCheckEnabled;
protected long lastNodeUpdateTime;
@ -62,6 +63,8 @@ public class NodeInfo {
this.totalPmemAllocatedContainersMB = resourceView
.getPmemAllocatedForContainers() / BYTES_IN_MB;
this.pmemCheckEnabled = resourceView.isPmemCheckEnabled();
this.totalVCoresAllocatedContainers = resourceView
.getVCoresAllocatedForContainers();
this.nodeHealthy = context.getNodeHealthStatus().getIsNodeHealthy();
this.lastNodeUpdateTime = context.getNodeHealthStatus()
.getLastHealthReportTime();
@ -124,6 +127,10 @@ public class NodeInfo {
return this.totalVmemAllocatedContainersMB;
}
public long getTotalVCoresAllocated() {
return this.totalVCoresAllocatedContainers;
}
public boolean isVmemCheckEnabled() {
return this.vmemCheckEnabled;
}

View File

@ -31,8 +31,11 @@ public class TestNodeManagerMetrics {
NodeManagerMetrics metrics = NodeManagerMetrics.create();
Resource total = Records.newRecord(Resource.class);
total.setMemory(8*GiB);
total.setVirtualCores(16);
Resource resource = Records.newRecord(Resource.class);
resource.setMemory(1*GiB);
resource.setVirtualCores(2);
metrics.addResource(total);
@ -57,12 +60,12 @@ public class TestNodeManagerMetrics {
metrics.initingContainer();
metrics.runningContainer();
checkMetrics(5, 1, 1, 1, 1, 1, 2, 2, 6);
checkMetrics(5, 1, 1, 1, 1, 1, 2, 2, 6, 4, 12);
}
private void checkMetrics(int launched, int completed, int failed, int killed,
int initing, int running, int allocatedGB,
int allocatedContainers, int availableGB) {
int allocatedContainers, int availableGB, int allocatedVCores, int availableVCores) {
MetricsRecordBuilder rb = getMetrics("NodeManagerMetrics");
assertCounter("ContainersLaunched", launched, rb);
assertCounter("ContainersCompleted", completed, rb);
@ -71,7 +74,10 @@ public class TestNodeManagerMetrics {
assertGauge("ContainersIniting", initing, rb);
assertGauge("ContainersRunning", running, rb);
assertGauge("AllocatedGB", allocatedGB, rb);
assertGauge("AllocatedVCores", allocatedVCores, rb);
assertGauge("AllocatedContainers", allocatedContainers, rb);
assertGauge("AvailableGB", availableGB, rb);
assertGauge("AvailableVCores",availableVCores, rb);
}
}

View File

@ -91,6 +91,10 @@ public class TestNMWebServer {
return 0;
}
@Override
public long getVCoresAllocatedForContainers() {
return 0;
}
@Override
public boolean isVmemCheckEnabled() {
return true;
}
@ -150,6 +154,10 @@ public class TestNMWebServer {
return 0;
}
@Override
public long getVCoresAllocatedForContainers() {
return 0;
}
@Override
public boolean isVmemCheckEnabled() {
return true;
}

View File

@ -124,6 +124,10 @@ public class TestNMWebServices extends JerseyTest {
return new Long("17179869184");
}
@Override
public long getVCoresAllocatedForContainers() {
return new Long("4000");
}
@Override
public boolean isVmemCheckEnabled() {
return true;
}
@ -375,6 +379,8 @@ public class TestNMWebServices extends JerseyTest {
"totalVmemAllocatedContainersMB"),
WebServicesTestUtils.getXmlLong(element,
"totalPmemAllocatedContainersMB"),
WebServicesTestUtils.getXmlLong(element,
"totalVCoresAllocatedContainers"),
WebServicesTestUtils.getXmlBoolean(element, "vmemCheckEnabled"),
WebServicesTestUtils.getXmlBoolean(element, "pmemCheckEnabled"),
WebServicesTestUtils.getXmlLong(element, "lastNodeUpdateTime"),
@ -393,10 +399,11 @@ public class TestNMWebServices extends JerseyTest {
public void verifyNodeInfo(JSONObject json) throws JSONException, Exception {
assertEquals("incorrect number of elements", 1, json.length());
JSONObject info = json.getJSONObject("nodeInfo");
assertEquals("incorrect number of elements", 15, info.length());
assertEquals("incorrect number of elements", 16, info.length());
verifyNodeInfoGeneric(info.getString("id"), info.getString("healthReport"),
info.getLong("totalVmemAllocatedContainersMB"),
info.getLong("totalPmemAllocatedContainersMB"),
info.getLong("totalVCoresAllocatedContainers"),
info.getBoolean("vmemCheckEnabled"),
info.getBoolean("pmemCheckEnabled"),
info.getLong("lastNodeUpdateTime"), info.getBoolean("nodeHealthy"),
@ -410,6 +417,7 @@ public class TestNMWebServices extends JerseyTest {
public void verifyNodeInfoGeneric(String id, String healthReport,
long totalVmemAllocatedContainersMB, long totalPmemAllocatedContainersMB,
long totalVCoresAllocatedContainers,
boolean vmemCheckEnabled, boolean pmemCheckEnabled,
long lastNodeUpdateTime, Boolean nodeHealthy, String nodeHostName,
String hadoopVersionBuiltOn, String hadoopBuildVersion,
@ -423,6 +431,8 @@ public class TestNMWebServices extends JerseyTest {
totalVmemAllocatedContainersMB);
assertEquals("totalPmemAllocatedContainersMB incorrect", 16384,
totalPmemAllocatedContainersMB);
assertEquals("totalVCoresAllocatedContainers incorrect", 4000,
totalVCoresAllocatedContainers);
assertEquals("vmemCheckEnabled incorrect", true, vmemCheckEnabled);
assertEquals("pmemCheckEnabled incorrect", true, pmemCheckEnabled);
assertTrue("lastNodeUpdateTime incorrect", lastNodeUpdateTime == nmContext

View File

@ -116,6 +116,12 @@ public class TestNMWebServicesApps extends JerseyTest {
return new Long("17179869184");
}
@Override
public long getVCoresAllocatedForContainers() {
return new Long("4000");
}
@Override
public boolean isVmemCheckEnabled() {
return true;

View File

@ -106,6 +106,11 @@ public class TestNMWebServicesContainers extends JerseyTest {
return new Long("17179869184");
}
@Override
public long getVCoresAllocatedForContainers() {
return new Long("4000");
}
@Override
public boolean isVmemCheckEnabled() {
return true;
@ -461,24 +466,27 @@ public class TestNMWebServicesContainers extends JerseyTest {
WebServicesTestUtils.getXmlString(element, "diagnostics"),
WebServicesTestUtils.getXmlString(element, "nodeId"),
WebServicesTestUtils.getXmlInt(element, "totalMemoryNeededMB"),
WebServicesTestUtils.getXmlInt(element, "totalVCoresNeeded"),
WebServicesTestUtils.getXmlString(element, "containerLogsLink"));
}
}
public void verifyNodeContainerInfo(JSONObject info, Container cont)
throws JSONException, Exception {
assertEquals("incorrect number of elements", 8, info.length());
assertEquals("incorrect number of elements", 9, info.length());
verifyNodeContainerInfoGeneric(cont, info.getString("id"),
info.getString("state"), info.getString("user"),
info.getInt("exitCode"), info.getString("diagnostics"),
info.getString("nodeId"), info.getInt("totalMemoryNeededMB"),
info.getInt("totalVCoresNeeded"),
info.getString("containerLogsLink"));
}
public void verifyNodeContainerInfoGeneric(Container cont, String id,
String state, String user, int exitCode, String diagnostics,
String nodeId, int totalMemoryNeededMB, String logsLink)
String nodeId, int totalMemoryNeededMB, int totalVCoresNeeded,
String logsLink)
throws JSONException, Exception {
WebServicesTestUtils.checkStringMatch("id", cont.getContainerId()
.toString(), id);
@ -495,6 +503,9 @@ public class TestNMWebServicesContainers extends JerseyTest {
assertEquals("totalMemoryNeededMB wrong",
YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
totalMemoryNeededMB);
assertEquals("totalVCoresNeeded wrong",
YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
totalVCoresNeeded);
String shortLink =
ujoin("containerlogs", cont.getContainerId().toString(),
cont.getUser());

View File

@ -72,6 +72,9 @@ public class MetricsOverviewTable extends HtmlBlock {
th().$class("ui-state-default")._("Memory Used")._().
th().$class("ui-state-default")._("Memory Total")._().
th().$class("ui-state-default")._("Memory Reserved")._().
th().$class("ui-state-default")._("VCores Used")._().
th().$class("ui-state-default")._("VCores Total")._().
th().$class("ui-state-default")._("VCores Reserved")._().
th().$class("ui-state-default")._("Active Nodes")._().
th().$class("ui-state-default")._("Decommissioned Nodes")._().
th().$class("ui-state-default")._("Lost Nodes")._().
@ -94,6 +97,9 @@ public class MetricsOverviewTable extends HtmlBlock {
td(StringUtils.byteDesc(clusterMetrics.getAllocatedMB() * BYTES_IN_MB)).
td(StringUtils.byteDesc(clusterMetrics.getTotalMB() * BYTES_IN_MB)).
td(StringUtils.byteDesc(clusterMetrics.getReservedMB() * BYTES_IN_MB)).
td(String.valueOf(clusterMetrics.getAllocatedVirtualCores())).
td(String.valueOf(clusterMetrics.getTotalVirtualCores())).
td(String.valueOf(clusterMetrics.getReservedVirtualCores())).
td().a(url("nodes"),String.valueOf(clusterMetrics.getActiveNodes()))._().
td().a(url("nodes/decommissioned"),String.valueOf(clusterMetrics.getDecommissionedNodes()))._().
td().a(url("nodes/lost"),String.valueOf(clusterMetrics.getLostNodes()))._().
@ -120,6 +126,9 @@ public class MetricsOverviewTable extends HtmlBlock {
th().$class("ui-state-default")._("Memory Used")._().
th().$class("ui-state-default")._("Memory Pending")._().
th().$class("ui-state-default")._("Memory Reserved")._().
th().$class("ui-state-default")._("VCores Used")._().
th().$class("ui-state-default")._("VCores Pending")._().
th().$class("ui-state-default")._("VCores Reserved")._().
_().
_().
tbody().$class("ui-widget-content").
@ -139,6 +148,9 @@ public class MetricsOverviewTable extends HtmlBlock {
td(StringUtils.byteDesc(userMetrics.getAllocatedMB() * BYTES_IN_MB)).
td(StringUtils.byteDesc(userMetrics.getPendingMB() * BYTES_IN_MB)).
td(StringUtils.byteDesc(userMetrics.getReservedMB() * BYTES_IN_MB)).
td(String.valueOf(userMetrics.getAllocatedVirtualCores())).
td(String.valueOf(userMetrics.getPendingVirtualCores())).
td(String.valueOf(userMetrics.getReservedVirtualCores())).
_().
_()._();

View File

@ -75,6 +75,8 @@ class NodesPage extends RmView {
th(".containers", "Containers").
th(".mem", "Mem Used").
th(".mem", "Mem Avail").
th(".vcores", "VCores Used").
th(".vcores", "VCores Avail").
th(".nodeManagerVersion", "Version").
_()._().
tbody();
@ -127,8 +129,10 @@ class NodesPage extends RmView {
td(String.valueOf(info.getNumContainers())).
td().br().$title(String.valueOf(usedMemory))._().
_(StringUtils.byteDesc(usedMemory * BYTES_IN_MB))._().
td().br().$title(String.valueOf(usedMemory))._().
td().br().$title(String.valueOf(availableMemory))._().
_(StringUtils.byteDesc(availableMemory * BYTES_IN_MB))._().
td(String.valueOf(info.getUsedVirtualCores())).
td(String.valueOf(info.getAvailableVirtualCores())).
td(ni.getNodeManagerVersion()).
_();
}

View File

@ -37,16 +37,21 @@ public class ClusterMetricsInfo {
protected int appsRunning;
protected int appsFailed;
protected int appsKilled;
protected long reservedMB;
protected long availableMB;
protected long allocatedMB;
protected long reservedVirtualCores;
protected long availableVirtualCores;
protected long allocatedVirtualCores;
protected int containersAllocated;
protected int containersReserved;
protected int containersPending;
protected long totalMB;
protected long totalVirtualCores;
protected int totalNodes;
protected int lostNodes;
protected int unhealthyNodes;
@ -68,16 +73,21 @@ public class ClusterMetricsInfo {
this.appsRunning = metrics.getAppsRunning();
this.appsFailed = metrics.getAppsFailed();
this.appsKilled = metrics.getAppsKilled();
this.reservedMB = metrics.getReservedMB();
this.availableMB = metrics.getAvailableMB();
this.allocatedMB = metrics.getAllocatedMB();
this.reservedVirtualCores = metrics.getReservedVirtualCores();
this.availableVirtualCores = metrics.getAvailableVirtualCores();
this.allocatedVirtualCores = metrics.getAllocatedVirtualCores();
this.containersAllocated = metrics.getAllocatedContainers();
this.containersPending = metrics.getPendingContainers();
this.containersReserved = metrics.getReservedContainers();
this.totalMB = availableMB + allocatedMB;
this.totalVirtualCores = availableVirtualCores + allocatedVirtualCores;
this.activeNodes = clusterMetrics.getNumActiveNMs();
this.lostNodes = clusterMetrics.getNumLostNMs();
this.unhealthyNodes = clusterMetrics.getUnhealthyNMs();
@ -123,6 +133,18 @@ public class ClusterMetricsInfo {
return this.allocatedMB;
}
public long getReservedVirtualCores() {
return this.reservedVirtualCores;
}
public long getAvailableVirtualCores() {
return this.availableVirtualCores;
}
public long getAllocatedVirtualCores() {
return this.allocatedVirtualCores;
}
public int getContainersAllocated() {
return this.containersAllocated;
}
@ -134,15 +156,19 @@ public class ClusterMetricsInfo {
public int getPendingContainers() {
return this.containersPending;
}
public long getTotalMB() {
return this.totalMB;
}
public long getTotalVirtualCores() {
return this.totalVirtualCores;
}
public int getTotalNodes() {
return this.totalNodes;
}
public int getActiveNodes() {
return this.activeNodes;
}

View File

@ -43,6 +43,8 @@ public class NodeInfo {
protected int numContainers;
protected long usedMemoryMB;
protected long availMemoryMB;
protected long usedVirtualCores;
protected long availableVirtualCores;
public NodeInfo() {
} // JAXB needs this
@ -57,6 +59,8 @@ public class NodeInfo {
this.numContainers = report.getNumContainers();
this.usedMemoryMB = report.getUsedResource().getMemory();
this.availMemoryMB = report.getAvailableResource().getMemory();
this.usedVirtualCores = report.getUsedResource().getVirtualCores();
this.availableVirtualCores = report.getAvailableResource().getVirtualCores();
}
this.id = id.toString();
this.rack = ni.getRackName();
@ -83,7 +87,7 @@ public class NodeInfo {
public String getNodeHTTPAddress() {
return this.nodeHTTPAddress;
}
public void setNodeHTTPAddress(String nodeHTTPAddress) {
this.nodeHTTPAddress = nodeHTTPAddress;
}
@ -112,4 +116,12 @@ public class NodeInfo {
return this.availMemoryMB;
}
public long getUsedVirtualCores() {
return this.usedVirtualCores;
}
public long getAvailableVirtualCores() {
return this.availableVirtualCores;
}
}

View File

@ -43,6 +43,9 @@ public class UserMetricsInfo {
protected long reservedMB;
protected long pendingMB;
protected long allocatedMB;
protected long reservedVirtualCores;
protected long pendingVirtualCores;
protected long allocatedVirtualCores;
@XmlTransient
protected boolean userMetricsAvailable;
@ -59,7 +62,7 @@ public class UserMetricsInfo {
if (userMetrics != null) {
this.userMetricsAvailable = true;
this.appsSubmitted = userMetrics.getAppsSubmitted();
this.appsCompleted = metrics.getAppsCompleted();
this.appsPending = metrics.getAppsPending();
@ -70,10 +73,14 @@ public class UserMetricsInfo {
this.runningContainers = userMetrics.getAllocatedContainers();
this.pendingContainers = userMetrics.getPendingContainers();
this.reservedContainers = userMetrics.getReservedContainers();
this.reservedMB = userMetrics.getReservedMB();
this.pendingMB = userMetrics.getPendingMB();
this.allocatedMB = userMetrics.getAllocatedMB();
this.reservedVirtualCores = userMetrics.getReservedVirtualCores();
this.pendingVirtualCores = userMetrics.getPendingVirtualCores();
this.allocatedVirtualCores = userMetrics.getAllocatedVirtualCores();
}
}
@ -117,6 +124,18 @@ public class UserMetricsInfo {
return this.pendingMB;
}
public long getReservedVirtualCores() {
return this.reservedVirtualCores;
}
public long getAllocatedVirtualCores() {
return this.allocatedVirtualCores;
}
public long getPendingVirtualCores() {
return this.pendingVirtualCores;
}
public int getReservedContainers() {
return this.reservedContainers;
}

View File

@ -48,8 +48,8 @@ public class TestNodesPage {
// Number of Actual Table Headers for NodesPage.NodesBlock might change in
// future. In that case this value should be adjusted to the new value.
final int numberOfThInMetricsTable = 13;
final int numberOfActualTableHeaders = 10;
final int numberOfThInMetricsTable = 16;
final int numberOfActualTableHeaders = 12;
private Injector injector;

View File

@ -389,6 +389,10 @@ public class TestRMWebServices extends JerseyTest {
WebServicesTestUtils.getXmlInt(element, "reservedMB"),
WebServicesTestUtils.getXmlInt(element, "availableMB"),
WebServicesTestUtils.getXmlInt(element, "allocatedMB"),
WebServicesTestUtils.getXmlInt(element, "reservedVirtualCores"),
WebServicesTestUtils.getXmlInt(element, "availableVirtualCores"),
WebServicesTestUtils.getXmlInt(element, "allocatedVirtualCores"),
WebServicesTestUtils.getXmlInt(element, "totalVirtualCores"),
WebServicesTestUtils.getXmlInt(element, "containersAllocated"),
WebServicesTestUtils.getXmlInt(element, "totalMB"),
WebServicesTestUtils.getXmlInt(element, "totalNodes"),
@ -404,11 +408,13 @@ public class TestRMWebServices extends JerseyTest {
Exception {
assertEquals("incorrect number of elements", 1, json.length());
JSONObject clusterinfo = json.getJSONObject("clusterMetrics");
assertEquals("incorrect number of elements", 19, clusterinfo.length());
assertEquals("incorrect number of elements", 23, clusterinfo.length());
verifyClusterMetrics(
clusterinfo.getInt("appsSubmitted"), clusterinfo.getInt("appsCompleted"),
clusterinfo.getInt("reservedMB"), clusterinfo.getInt("availableMB"),
clusterinfo.getInt("allocatedMB"),
clusterinfo.getInt("reservedVirtualCores"), clusterinfo.getInt("availableVirtualCores"),
clusterinfo.getInt("allocatedVirtualCores"), clusterinfo.getInt("totalVirtualCores"),
clusterinfo.getInt("containersAllocated"),
clusterinfo.getInt("totalMB"), clusterinfo.getInt("totalNodes"),
clusterinfo.getInt("lostNodes"), clusterinfo.getInt("unhealthyNodes"),
@ -418,7 +424,9 @@ public class TestRMWebServices extends JerseyTest {
public void verifyClusterMetrics(int submittedApps, int completedApps,
int reservedMB, int availableMB,
int allocMB, int containersAlloc, int totalMB, int totalNodes,
int allocMB, int reservedVirtualCores, int availableVirtualCores,
int allocVirtualCores, int totalVirtualCores,
int containersAlloc, int totalMB, int totalNodes,
int lostNodes, int unhealthyNodes, int decommissionedNodes,
int rebootedNodes, int activeNodes) throws JSONException, Exception {
@ -428,7 +436,8 @@ public class TestRMWebServices extends JerseyTest {
long totalMBExpect =
metrics.getAvailableMB() + metrics.getAllocatedMB();
long totalVirtualCoresExpect =
metrics.getAvailableVirtualCores() + metrics.getAllocatedVirtualCores();
assertEquals("appsSubmitted doesn't match",
metrics.getAppsSubmitted(), submittedApps);
assertEquals("appsCompleted doesn't match",
@ -439,6 +448,12 @@ public class TestRMWebServices extends JerseyTest {
metrics.getAvailableMB(), availableMB);
assertEquals("allocatedMB doesn't match",
metrics.getAllocatedMB(), allocMB);
assertEquals("reservedVirtualCores doesn't match",
metrics.getReservedVirtualCores(), reservedVirtualCores);
assertEquals("availableVirtualCores doesn't match",
metrics.getAvailableVirtualCores(), availableVirtualCores);
assertEquals("allocatedVirtualCores doesn't match",
totalVirtualCoresExpect, allocVirtualCores);
assertEquals("containersAllocated doesn't match", 0, containersAlloc);
assertEquals("totalMB doesn't match", totalMBExpect, totalMB);
assertEquals(

View File

@ -656,13 +656,15 @@ public class TestRMWebServicesNodes extends JerseyTest {
WebServicesTestUtils.getXmlInt(element, "numContainers"),
WebServicesTestUtils.getXmlLong(element, "usedMemoryMB"),
WebServicesTestUtils.getXmlLong(element, "availMemoryMB"),
WebServicesTestUtils.getXmlLong(element, "usedVirtualCores"),
WebServicesTestUtils.getXmlLong(element, "availableVirtualCores"),
WebServicesTestUtils.getXmlString(element, "version"));
}
}
public void verifyNodeInfo(JSONObject nodeInfo, MockNM nm)
throws JSONException, Exception {
assertEquals("incorrect number of elements", 11, nodeInfo.length());
assertEquals("incorrect number of elements", 13, nodeInfo.length());
verifyNodeInfoGeneric(nm, nodeInfo.getString("state"),
nodeInfo.getString("rack"),
@ -671,6 +673,7 @@ public class TestRMWebServicesNodes extends JerseyTest {
nodeInfo.getLong("lastHealthUpdate"),
nodeInfo.getString("healthReport"), nodeInfo.getInt("numContainers"),
nodeInfo.getLong("usedMemoryMB"), nodeInfo.getLong("availMemoryMB"),
nodeInfo.getLong("usedVirtualCores"), nodeInfo.getLong("availableVirtualCores"),
nodeInfo.getString("version"));
}
@ -678,7 +681,8 @@ public class TestRMWebServicesNodes extends JerseyTest {
public void verifyNodeInfoGeneric(MockNM nm, String state, String rack,
String id, String nodeHostName,
String nodeHTTPAddress, long lastHealthUpdate, String healthReport,
int numContainers, long usedMemoryMB, long availMemoryMB, String version)
int numContainers, long usedMemoryMB, long availMemoryMB, long usedVirtualCores,
long availVirtualCores, String version)
throws JSONException, Exception {
RMNode node = rm.getRMContext().getRMNodes().get(nm.getNodeId());
@ -712,6 +716,10 @@ public class TestRMWebServicesNodes extends JerseyTest {
.getUsedResource().getMemory(), usedMemoryMB);
assertEquals("availMemoryMB doesn't match: " + availMemoryMB, report
.getAvailableResource().getMemory(), availMemoryMB);
assertEquals("usedVirtualCores doesn't match: " + usedVirtualCores, report
.getUsedResource().getVirtualCores(), usedVirtualCores);
assertEquals("availVirtualCores doesn't match: " + availVirtualCores, report
.getAvailableResource().getVirtualCores(), availVirtualCores);
}
}

View File

@ -62,6 +62,8 @@ NodeManager REST API's.
*---------------+--------------+-------------------------------+
| totalVmemAllocatedContainersMB | long | The amount of virtual memory allocated for use by containers in MB |
*---------------+--------------+-------------------------------+
| totalVCoresAllocatedContainers | long | The number of virtual cores allocated for use by containers |
*---------------+--------------+-------------------------------+
| lastNodeUpdateTime | long | The last timestamp at which the health report was received (in ms since epoch)|
*---------------+--------------+-------------------------------+
| healthReport | string | The diagnostic health report of the node |
@ -109,6 +111,7 @@ NodeManager REST API's.
"nodeManagerBuildVersion" : "0.23.1-SNAPSHOT from 1228355 by user1 source checksum 20647f76c36430e888cc7204826a445c",
"lastNodeUpdateTime" : 1326222266126,
"totalVmemAllocatedContainersMB" : 17203,
"totalVCoresAllocatedContainers" : 8,
"nodeHealthy" : true,
"healthReport" : "",
"totalPmemAllocatedContainersMB" : 8192,
@ -148,6 +151,7 @@ NodeManager REST API's.
<healthReport/>
<totalVmemAllocatedContainersMB>17203</totalVmemAllocatedContainersMB>
<totalPmemAllocatedContainersMB>8192</totalPmemAllocatedContainersMB>
<totalVCoresAllocatedContainers>8</totalVCoresAllocatedContainers>
<lastNodeUpdateTime>1326222386134</lastNodeUpdateTime>
<nodeHealthy>true</nodeHealthy>
<nodeManagerVersion>0.23.1-SNAPSHOT</nodeManagerVersion>
@ -446,6 +450,7 @@ NodeManager REST API's.
{
"nodeId" : "host.domain.com:8041",
"totalMemoryNeededMB" : 2048,
"totalVCoresNeeded" : 1,
"state" : "RUNNING",
"diagnostics" : "",
"containerLogsLink" : "http://host.domain.com:8042/node/containerlogs/container_1326121700862_0006_01_000001/user1",
@ -456,6 +461,7 @@ NodeManager REST API's.
{
"nodeId" : "host.domain.com:8041",
"totalMemoryNeededMB" : 2048,
"totalVCoresNeeded" : 2,
"state" : "RUNNING",
"diagnostics" : "",
"containerLogsLink" : "http://host.domain.com:8042/node/containerlogs/container_1326121700862_0006_01_000003/user1",
@ -498,6 +504,7 @@ NodeManager REST API's.
<diagnostics/>
<user>user1</user>
<totalMemoryNeededMB>2048</totalMemoryNeededMB>
<totalVCoresNeeded>1</totalVCoresNeeded>
<containerLogsLink>http://host.domain.com:8042/node/containerlogs/container_1326121700862_0006_01_000001/user1</containerLogsLink>
<nodeId>host.domain.com:8041</nodeId>
</container>
@ -508,6 +515,7 @@ NodeManager REST API's.
<diagnostics>Container killed by the ApplicationMaster.</diagnostics>
<user>user1</user>
<totalMemoryNeededMB>2048</totalMemoryNeededMB>
<totalVCoresNeeded>2</totalVCoresNeeded>
<containerLogsLink>http://host.domain.com:8042/node/containerlogs/container_1326121700862_0006_01_000003/user1</containerLogsLink>
<nodeId>host.domain.com:8041</nodeId>
</container>
@ -560,6 +568,8 @@ NodeManager REST API's.
*---------------+--------------+-------------------------------+
| totalMemoryNeededMB | long | Total amout of memory needed by the container (in MB) |
*---------------+--------------+-------------------------------+
| totalVCoresNeeded | long | Total number of virtual cores needed by the container |
*---------------+--------------+-------------------------------+
** Response Examples
@ -587,6 +597,7 @@ NodeManager REST API's.
"container" : {
"nodeId" : "host.domain.com:8041",
"totalMemoryNeededMB" : 2048,
"totalVCoresNeeded" : 1,
"state" : "RUNNING",
"diagnostics" : "",
"containerLogsLink" : "http://host.domain.com:8042/node/containerlogs/container_1326121700862_0007_01_000001/user1",
@ -626,6 +637,7 @@ NodeManager REST API's.
<diagnostics/>
<user>user1</user>
<totalMemoryNeededMB>2048</totalMemoryNeededMB>
<totalVCoresNeeded>1</totalVCoresNeeded>
<containerLogsLink>http://host.domain.com:8042/node/containerlogs/container_1326121700862_0007_01_000001/user1</containerLogsLink>
<nodeId>host.domain.com:8041</nodeId>
</container>

View File

@ -195,6 +195,14 @@ ResourceManager REST API's.
*---------------+--------------+-------------------------------+
| totalMB | long | The amount of total memory in MB |
*---------------+--------------+-------------------------------+
| reservedVirtualCores | long | The number of reserved virtual cores |
*---------------+--------------+-------------------------------+
| availableVirtualCores | long | The number of available virtual cores |
*---------------+--------------+-------------------------------+
| allocatedVirtualCores | long | The number of allocated virtual cores |
*---------------+--------------+-------------------------------+
| totalVirtualCores | long | The total number of virtual cores |
*---------------+--------------+-------------------------------+
| containersAllocated | int | The number of containers allocated |
*---------------+--------------+-------------------------------+
| containersReserved | int | The number of containers reserved |
@ -247,10 +255,14 @@ ResourceManager REST API's.
"reservedMB":0,
"availableMB":17408,
"allocatedMB":0,
"reservedVirtualCores":0,
"availableVirtualCores":7,
"allocatedVirtualCores":1,
"containersAllocated":0,
"containersReserved":0,
"containersPending":0,
"totalMB":17408,
"totalVirtualCores":8,
"totalNodes":1,
"lostNodes":0,
"unhealthyNodes":0,
@ -293,10 +305,14 @@ ResourceManager REST API's.
<reservedMB>0</reservedMB>
<availableMB>17408</availableMB>
<allocatedMB>0</allocatedMB>
<reservedVirtualCores>0</reservedVirtualCores>
<availableVirtualCores>7</availableVirtualCores>
<allocatedVirtualCores>1</allocatedVirtualCores>
<containersAllocated>0</containersAllocated>
<containersReserved>0</containersReserved>
<containersPending>0</containersPending>
<totalMB>17408</totalMB>
<totalVirtualCores>8</totalVirtualCores>
<totalNodes>1</totalNodes>
<lostNodes>0</lostNodes>
<unhealthyNodes>0</unhealthyNodes>
@ -2040,8 +2056,10 @@ Server: Jetty(6.1.26)
"lastHealthUpdate":1324056895432,
"healthReport":"Healthy",
"numContainers":0,
"usedMemoryMB":0
"availMemoryMB":8192
"usedMemoryMB":0,
"availMemoryMB":8192,
"usedVirtualCores":0,
"availableVirtualCores":8
},
{
"rack":"\/default-rack",
@ -2054,7 +2072,9 @@ Server: Jetty(6.1.26)
"healthReport":"Healthy",
"numContainers":0,
"usedMemoryMB":0,
"availMemoryMB":8192
"availMemoryMB":8192,
"usedVirtualCores":0,
"availableVirtualCores":8
}
]
}
@ -2096,6 +2116,8 @@ Server: Jetty(6.1.26)
<numContainers>0</numContainers>
<usedMemoryMB>0</usedMemoryMB>
<availMemoryMB>5120</availMemoryMB>
<usedVirtualCores>0</usedVirtualCores>
<availableVirtualCores>8</availableVirtualCores>
</node>
<node>
<rack>/default-rack</rack>
@ -2109,6 +2131,8 @@ Server: Jetty(6.1.26)
<numContainers>0</numContainers>
<usedMemoryMB>0</usedMemoryMB>
<availMemoryMB>5120</availMemoryMB>
<usedVirtualCores>0</usedVirtualCores>
<availableVirtualCores>8</availableVirtualCores>
</node>
</nodes>
+---+
@ -2159,10 +2183,14 @@ Server: Jetty(6.1.26)
*---------------+--------------+-------------------------------+
| lastHealthUpdate | long | The last time the node reported its health (in ms since epoch)|
*---------------+--------------+-------------------------------+
| usedMemoryMB | long | The total about of memory currently used on the node (in MB)|
| usedMemoryMB | long | The total amount of memory currently used on the node (in MB)|
*---------------+--------------+-------------------------------+
| availMemoryMB | long | The total amount of memory currently available on the node (in MB)|
*---------------+--------------+-------------------------------+
| usedVirtualCores | long | The total number of vCores currently used on the node |
*---------------+--------------+-------------------------------+
| availableVirtualCores | long | The total number of vCores available on the node |
*---------------+--------------+-------------------------------+
| numContainers | int | The total number of containers currently running on the node|
*---------------+--------------+-------------------------------+
@ -2201,7 +2229,9 @@ Server: Jetty(6.1.26)
"healthReport":"Healthy",
"numContainers":0,
"usedMemoryMB":0,
"availMemoryMB":5120
"availMemoryMB":5120,
"usedVirtualCores":0,
"availableVirtualCores":8
}
}
+---+
@ -2240,6 +2270,8 @@ Server: Jetty(6.1.26)
<numContainers>0</numContainers>
<usedMemoryMB>0</usedMemoryMB>
<availMemoryMB>5120</availMemoryMB>
<usedVirtualCores>0</usedVirtualCores>
<availableVirtualCores>5120</availableVirtualCores>
</node>
+---+