YARN-2072. RM/NM UIs and webservices are missing vcore information. (Nathan Roberts via tgraves)
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1605166 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Thomas Graves
parent
805fca1240
commit
1c2052e200
|
|
@ -168,6 +168,9 @@ Release 2.5.0 - UNRELEASED
|
|||
|
||||
YARN-2109. Fix TestRM to work with both schedulers. (Anubhav Dhoot via kasha)
|
||||
|
||||
YARN-2072. RM/NM UIs and webservices are missing vcore information.
|
||||
(Nathan Roberts via tgraves)
|
||||
|
||||
OPTIMIZATIONS
|
||||
|
||||
BUG FIXES
|
||||
|
|
|
|||
|
|
@ -27,4 +27,6 @@ public interface ResourceView {
|
|||
long getPmemAllocatedForContainers();
|
||||
|
||||
boolean isPmemCheckEnabled();
|
||||
|
||||
long getVCoresAllocatedForContainers();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -70,6 +70,8 @@ public class ContainersMonitorImpl extends AbstractService implements
|
|||
private boolean pmemCheckEnabled;
|
||||
private boolean vmemCheckEnabled;
|
||||
|
||||
private long maxVCoresAllottedForContainers;
|
||||
|
||||
private static final long UNKNOWN_MEMORY_LIMIT = -1L;
|
||||
|
||||
public ContainersMonitorImpl(ContainerExecutor exec,
|
||||
|
|
@ -108,10 +110,16 @@ public class ContainersMonitorImpl extends AbstractService implements
|
|||
YarnConfiguration.NM_PMEM_MB,
|
||||
YarnConfiguration.DEFAULT_NM_PMEM_MB) * 1024 * 1024l;
|
||||
|
||||
long configuredVCoresForContainers = conf.getLong(
|
||||
YarnConfiguration.NM_VCORES,
|
||||
YarnConfiguration.DEFAULT_NM_VCORES);
|
||||
|
||||
|
||||
// Setting these irrespective of whether checks are enabled. Required in
|
||||
// the UI.
|
||||
// ///////// Physical memory configuration //////
|
||||
this.maxPmemAllottedForContainers = configuredPMemForContainers;
|
||||
this.maxVCoresAllottedForContainers = configuredVCoresForContainers;
|
||||
|
||||
// ///////// Virtual memory configuration //////
|
||||
float vmemRatio = conf.getFloat(YarnConfiguration.NM_VMEM_PMEM_RATIO,
|
||||
|
|
@ -518,6 +526,11 @@ public class ContainersMonitorImpl extends AbstractService implements
|
|||
return this.maxPmemAllottedForContainers;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getVCoresAllocatedForContainers() {
|
||||
return this.maxVCoresAllottedForContainers;
|
||||
}
|
||||
|
||||
/**
|
||||
* Is the total virtual memory check enabled?
|
||||
*
|
||||
|
|
|
|||
|
|
@ -40,6 +40,9 @@ public class NodeManagerMetrics {
|
|||
@Metric("Current # of allocated containers")
|
||||
MutableGaugeInt allocatedContainers;
|
||||
@Metric MutableGaugeInt availableGB;
|
||||
@Metric("Current allocated Virtual Cores")
|
||||
MutableGaugeInt allocatedVCores;
|
||||
@Metric MutableGaugeInt availableVCores;
|
||||
|
||||
public static NodeManagerMetrics create() {
|
||||
return create(DefaultMetricsSystem.instance());
|
||||
|
|
@ -88,16 +91,21 @@ public class NodeManagerMetrics {
|
|||
allocatedContainers.incr();
|
||||
allocatedGB.incr(res.getMemory() / 1024);
|
||||
availableGB.decr(res.getMemory() / 1024);
|
||||
allocatedVCores.incr(res.getVirtualCores());
|
||||
availableVCores.decr(res.getVirtualCores());
|
||||
}
|
||||
|
||||
public void releaseContainer(Resource res) {
|
||||
allocatedContainers.decr();
|
||||
allocatedGB.decr(res.getMemory() / 1024);
|
||||
availableGB.incr(res.getMemory() / 1024);
|
||||
allocatedVCores.decr(res.getVirtualCores());
|
||||
availableVCores.incr(res.getVirtualCores());
|
||||
}
|
||||
|
||||
public void addResource(Resource res) {
|
||||
availableGB.incr(res.getMemory() / 1024);
|
||||
availableVCores.incr(res.getVirtualCores());
|
||||
}
|
||||
|
||||
public int getRunningContainers() {
|
||||
|
|
|
|||
|
|
@ -85,6 +85,7 @@ public class ContainerPage extends NMView implements YarnWebParams {
|
|||
._("Diagnostics", info.getDiagnostics())
|
||||
._("User", info.getUser())
|
||||
._("TotalMemoryNeeded", info.getMemoryNeeded())
|
||||
._("TotalVCoresNeeded", info.getVCoresNeeded())
|
||||
._("logs", info.getShortLogLink(), "Link to logs");
|
||||
html._(InfoBlock.class);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -73,6 +73,8 @@ public class NodePage extends NMView {
|
|||
StringUtils.byteDesc(info.getTotalPmemAllocated() * BYTES_IN_MB))
|
||||
._("Pmem enforcement enabled",
|
||||
info.isVmemCheckEnabled())
|
||||
._("Total VCores allocated for Containers",
|
||||
String.valueOf(info.getTotalVCoresAllocated()))
|
||||
._("NodeHealthyStatus",
|
||||
info.getHealthStatus())
|
||||
._("LastNodeHealthTime", new Date(
|
||||
|
|
|
|||
|
|
@ -42,6 +42,7 @@ public class ContainerInfo {
|
|||
protected String diagnostics;
|
||||
protected String user;
|
||||
protected long totalMemoryNeededMB;
|
||||
protected long totalVCoresNeeded;
|
||||
protected String containerLogsLink;
|
||||
protected String nodeId;
|
||||
@XmlTransient
|
||||
|
|
@ -76,6 +77,7 @@ public class ContainerInfo {
|
|||
Resource res = container.getResource();
|
||||
if (res != null) {
|
||||
this.totalMemoryNeededMB = res.getMemory();
|
||||
this.totalVCoresNeeded = res.getVirtualCores();
|
||||
}
|
||||
this.containerLogsShortLink = ujoin("containerlogs", this.id,
|
||||
container.getUser());
|
||||
|
|
@ -130,4 +132,8 @@ public class ContainerInfo {
|
|||
return this.totalMemoryNeededMB;
|
||||
}
|
||||
|
||||
public long getVCoresNeeded() {
|
||||
return this.totalVCoresNeeded;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@ public class NodeInfo {
|
|||
protected String healthReport;
|
||||
protected long totalVmemAllocatedContainersMB;
|
||||
protected long totalPmemAllocatedContainersMB;
|
||||
protected long totalVCoresAllocatedContainers;
|
||||
protected boolean vmemCheckEnabled;
|
||||
protected boolean pmemCheckEnabled;
|
||||
protected long lastNodeUpdateTime;
|
||||
|
|
@ -62,6 +63,8 @@ public class NodeInfo {
|
|||
this.totalPmemAllocatedContainersMB = resourceView
|
||||
.getPmemAllocatedForContainers() / BYTES_IN_MB;
|
||||
this.pmemCheckEnabled = resourceView.isPmemCheckEnabled();
|
||||
this.totalVCoresAllocatedContainers = resourceView
|
||||
.getVCoresAllocatedForContainers();
|
||||
this.nodeHealthy = context.getNodeHealthStatus().getIsNodeHealthy();
|
||||
this.lastNodeUpdateTime = context.getNodeHealthStatus()
|
||||
.getLastHealthReportTime();
|
||||
|
|
@ -124,6 +127,10 @@ public class NodeInfo {
|
|||
return this.totalVmemAllocatedContainersMB;
|
||||
}
|
||||
|
||||
public long getTotalVCoresAllocated() {
|
||||
return this.totalVCoresAllocatedContainers;
|
||||
}
|
||||
|
||||
public boolean isVmemCheckEnabled() {
|
||||
return this.vmemCheckEnabled;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -31,8 +31,11 @@ public class TestNodeManagerMetrics {
|
|||
NodeManagerMetrics metrics = NodeManagerMetrics.create();
|
||||
Resource total = Records.newRecord(Resource.class);
|
||||
total.setMemory(8*GiB);
|
||||
total.setVirtualCores(16);
|
||||
Resource resource = Records.newRecord(Resource.class);
|
||||
resource.setMemory(1*GiB);
|
||||
resource.setVirtualCores(2);
|
||||
|
||||
|
||||
metrics.addResource(total);
|
||||
|
||||
|
|
@ -57,12 +60,12 @@ public class TestNodeManagerMetrics {
|
|||
metrics.initingContainer();
|
||||
metrics.runningContainer();
|
||||
|
||||
checkMetrics(5, 1, 1, 1, 1, 1, 2, 2, 6);
|
||||
checkMetrics(5, 1, 1, 1, 1, 1, 2, 2, 6, 4, 12);
|
||||
}
|
||||
|
||||
private void checkMetrics(int launched, int completed, int failed, int killed,
|
||||
int initing, int running, int allocatedGB,
|
||||
int allocatedContainers, int availableGB) {
|
||||
int allocatedContainers, int availableGB, int allocatedVCores, int availableVCores) {
|
||||
MetricsRecordBuilder rb = getMetrics("NodeManagerMetrics");
|
||||
assertCounter("ContainersLaunched", launched, rb);
|
||||
assertCounter("ContainersCompleted", completed, rb);
|
||||
|
|
@ -71,7 +74,10 @@ public class TestNodeManagerMetrics {
|
|||
assertGauge("ContainersIniting", initing, rb);
|
||||
assertGauge("ContainersRunning", running, rb);
|
||||
assertGauge("AllocatedGB", allocatedGB, rb);
|
||||
assertGauge("AllocatedVCores", allocatedVCores, rb);
|
||||
assertGauge("AllocatedContainers", allocatedContainers, rb);
|
||||
assertGauge("AvailableGB", availableGB, rb);
|
||||
assertGauge("AvailableVCores",availableVCores, rb);
|
||||
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -91,6 +91,10 @@ public class TestNMWebServer {
|
|||
return 0;
|
||||
}
|
||||
@Override
|
||||
public long getVCoresAllocatedForContainers() {
|
||||
return 0;
|
||||
}
|
||||
@Override
|
||||
public boolean isVmemCheckEnabled() {
|
||||
return true;
|
||||
}
|
||||
|
|
@ -150,6 +154,10 @@ public class TestNMWebServer {
|
|||
return 0;
|
||||
}
|
||||
@Override
|
||||
public long getVCoresAllocatedForContainers() {
|
||||
return 0;
|
||||
}
|
||||
@Override
|
||||
public boolean isVmemCheckEnabled() {
|
||||
return true;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -124,6 +124,10 @@ public class TestNMWebServices extends JerseyTest {
|
|||
return new Long("17179869184");
|
||||
}
|
||||
@Override
|
||||
public long getVCoresAllocatedForContainers() {
|
||||
return new Long("4000");
|
||||
}
|
||||
@Override
|
||||
public boolean isVmemCheckEnabled() {
|
||||
return true;
|
||||
}
|
||||
|
|
@ -375,6 +379,8 @@ public class TestNMWebServices extends JerseyTest {
|
|||
"totalVmemAllocatedContainersMB"),
|
||||
WebServicesTestUtils.getXmlLong(element,
|
||||
"totalPmemAllocatedContainersMB"),
|
||||
WebServicesTestUtils.getXmlLong(element,
|
||||
"totalVCoresAllocatedContainers"),
|
||||
WebServicesTestUtils.getXmlBoolean(element, "vmemCheckEnabled"),
|
||||
WebServicesTestUtils.getXmlBoolean(element, "pmemCheckEnabled"),
|
||||
WebServicesTestUtils.getXmlLong(element, "lastNodeUpdateTime"),
|
||||
|
|
@ -393,10 +399,11 @@ public class TestNMWebServices extends JerseyTest {
|
|||
public void verifyNodeInfo(JSONObject json) throws JSONException, Exception {
|
||||
assertEquals("incorrect number of elements", 1, json.length());
|
||||
JSONObject info = json.getJSONObject("nodeInfo");
|
||||
assertEquals("incorrect number of elements", 15, info.length());
|
||||
assertEquals("incorrect number of elements", 16, info.length());
|
||||
verifyNodeInfoGeneric(info.getString("id"), info.getString("healthReport"),
|
||||
info.getLong("totalVmemAllocatedContainersMB"),
|
||||
info.getLong("totalPmemAllocatedContainersMB"),
|
||||
info.getLong("totalVCoresAllocatedContainers"),
|
||||
info.getBoolean("vmemCheckEnabled"),
|
||||
info.getBoolean("pmemCheckEnabled"),
|
||||
info.getLong("lastNodeUpdateTime"), info.getBoolean("nodeHealthy"),
|
||||
|
|
@ -410,6 +417,7 @@ public class TestNMWebServices extends JerseyTest {
|
|||
|
||||
public void verifyNodeInfoGeneric(String id, String healthReport,
|
||||
long totalVmemAllocatedContainersMB, long totalPmemAllocatedContainersMB,
|
||||
long totalVCoresAllocatedContainers,
|
||||
boolean vmemCheckEnabled, boolean pmemCheckEnabled,
|
||||
long lastNodeUpdateTime, Boolean nodeHealthy, String nodeHostName,
|
||||
String hadoopVersionBuiltOn, String hadoopBuildVersion,
|
||||
|
|
@ -423,6 +431,8 @@ public class TestNMWebServices extends JerseyTest {
|
|||
totalVmemAllocatedContainersMB);
|
||||
assertEquals("totalPmemAllocatedContainersMB incorrect", 16384,
|
||||
totalPmemAllocatedContainersMB);
|
||||
assertEquals("totalVCoresAllocatedContainers incorrect", 4000,
|
||||
totalVCoresAllocatedContainers);
|
||||
assertEquals("vmemCheckEnabled incorrect", true, vmemCheckEnabled);
|
||||
assertEquals("pmemCheckEnabled incorrect", true, pmemCheckEnabled);
|
||||
assertTrue("lastNodeUpdateTime incorrect", lastNodeUpdateTime == nmContext
|
||||
|
|
|
|||
|
|
@ -116,6 +116,12 @@ public class TestNMWebServicesApps extends JerseyTest {
|
|||
return new Long("17179869184");
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getVCoresAllocatedForContainers() {
|
||||
return new Long("4000");
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public boolean isVmemCheckEnabled() {
|
||||
return true;
|
||||
|
|
|
|||
|
|
@ -106,6 +106,11 @@ public class TestNMWebServicesContainers extends JerseyTest {
|
|||
return new Long("17179869184");
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getVCoresAllocatedForContainers() {
|
||||
return new Long("4000");
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isVmemCheckEnabled() {
|
||||
return true;
|
||||
|
|
@ -461,24 +466,27 @@ public class TestNMWebServicesContainers extends JerseyTest {
|
|||
WebServicesTestUtils.getXmlString(element, "diagnostics"),
|
||||
WebServicesTestUtils.getXmlString(element, "nodeId"),
|
||||
WebServicesTestUtils.getXmlInt(element, "totalMemoryNeededMB"),
|
||||
WebServicesTestUtils.getXmlInt(element, "totalVCoresNeeded"),
|
||||
WebServicesTestUtils.getXmlString(element, "containerLogsLink"));
|
||||
}
|
||||
}
|
||||
|
||||
public void verifyNodeContainerInfo(JSONObject info, Container cont)
|
||||
throws JSONException, Exception {
|
||||
assertEquals("incorrect number of elements", 8, info.length());
|
||||
assertEquals("incorrect number of elements", 9, info.length());
|
||||
|
||||
verifyNodeContainerInfoGeneric(cont, info.getString("id"),
|
||||
info.getString("state"), info.getString("user"),
|
||||
info.getInt("exitCode"), info.getString("diagnostics"),
|
||||
info.getString("nodeId"), info.getInt("totalMemoryNeededMB"),
|
||||
info.getInt("totalVCoresNeeded"),
|
||||
info.getString("containerLogsLink"));
|
||||
}
|
||||
|
||||
public void verifyNodeContainerInfoGeneric(Container cont, String id,
|
||||
String state, String user, int exitCode, String diagnostics,
|
||||
String nodeId, int totalMemoryNeededMB, String logsLink)
|
||||
String nodeId, int totalMemoryNeededMB, int totalVCoresNeeded,
|
||||
String logsLink)
|
||||
throws JSONException, Exception {
|
||||
WebServicesTestUtils.checkStringMatch("id", cont.getContainerId()
|
||||
.toString(), id);
|
||||
|
|
@ -495,6 +503,9 @@ public class TestNMWebServicesContainers extends JerseyTest {
|
|||
assertEquals("totalMemoryNeededMB wrong",
|
||||
YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
|
||||
totalMemoryNeededMB);
|
||||
assertEquals("totalVCoresNeeded wrong",
|
||||
YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
|
||||
totalVCoresNeeded);
|
||||
String shortLink =
|
||||
ujoin("containerlogs", cont.getContainerId().toString(),
|
||||
cont.getUser());
|
||||
|
|
|
|||
|
|
@ -72,6 +72,9 @@ public class MetricsOverviewTable extends HtmlBlock {
|
|||
th().$class("ui-state-default")._("Memory Used")._().
|
||||
th().$class("ui-state-default")._("Memory Total")._().
|
||||
th().$class("ui-state-default")._("Memory Reserved")._().
|
||||
th().$class("ui-state-default")._("VCores Used")._().
|
||||
th().$class("ui-state-default")._("VCores Total")._().
|
||||
th().$class("ui-state-default")._("VCores Reserved")._().
|
||||
th().$class("ui-state-default")._("Active Nodes")._().
|
||||
th().$class("ui-state-default")._("Decommissioned Nodes")._().
|
||||
th().$class("ui-state-default")._("Lost Nodes")._().
|
||||
|
|
@ -94,6 +97,9 @@ public class MetricsOverviewTable extends HtmlBlock {
|
|||
td(StringUtils.byteDesc(clusterMetrics.getAllocatedMB() * BYTES_IN_MB)).
|
||||
td(StringUtils.byteDesc(clusterMetrics.getTotalMB() * BYTES_IN_MB)).
|
||||
td(StringUtils.byteDesc(clusterMetrics.getReservedMB() * BYTES_IN_MB)).
|
||||
td(String.valueOf(clusterMetrics.getAllocatedVirtualCores())).
|
||||
td(String.valueOf(clusterMetrics.getTotalVirtualCores())).
|
||||
td(String.valueOf(clusterMetrics.getReservedVirtualCores())).
|
||||
td().a(url("nodes"),String.valueOf(clusterMetrics.getActiveNodes()))._().
|
||||
td().a(url("nodes/decommissioned"),String.valueOf(clusterMetrics.getDecommissionedNodes()))._().
|
||||
td().a(url("nodes/lost"),String.valueOf(clusterMetrics.getLostNodes()))._().
|
||||
|
|
@ -120,6 +126,9 @@ public class MetricsOverviewTable extends HtmlBlock {
|
|||
th().$class("ui-state-default")._("Memory Used")._().
|
||||
th().$class("ui-state-default")._("Memory Pending")._().
|
||||
th().$class("ui-state-default")._("Memory Reserved")._().
|
||||
th().$class("ui-state-default")._("VCores Used")._().
|
||||
th().$class("ui-state-default")._("VCores Pending")._().
|
||||
th().$class("ui-state-default")._("VCores Reserved")._().
|
||||
_().
|
||||
_().
|
||||
tbody().$class("ui-widget-content").
|
||||
|
|
@ -139,6 +148,9 @@ public class MetricsOverviewTable extends HtmlBlock {
|
|||
td(StringUtils.byteDesc(userMetrics.getAllocatedMB() * BYTES_IN_MB)).
|
||||
td(StringUtils.byteDesc(userMetrics.getPendingMB() * BYTES_IN_MB)).
|
||||
td(StringUtils.byteDesc(userMetrics.getReservedMB() * BYTES_IN_MB)).
|
||||
td(String.valueOf(userMetrics.getAllocatedVirtualCores())).
|
||||
td(String.valueOf(userMetrics.getPendingVirtualCores())).
|
||||
td(String.valueOf(userMetrics.getReservedVirtualCores())).
|
||||
_().
|
||||
_()._();
|
||||
|
||||
|
|
|
|||
|
|
@ -75,6 +75,8 @@ class NodesPage extends RmView {
|
|||
th(".containers", "Containers").
|
||||
th(".mem", "Mem Used").
|
||||
th(".mem", "Mem Avail").
|
||||
th(".vcores", "VCores Used").
|
||||
th(".vcores", "VCores Avail").
|
||||
th(".nodeManagerVersion", "Version").
|
||||
_()._().
|
||||
tbody();
|
||||
|
|
@ -127,8 +129,10 @@ class NodesPage extends RmView {
|
|||
td(String.valueOf(info.getNumContainers())).
|
||||
td().br().$title(String.valueOf(usedMemory))._().
|
||||
_(StringUtils.byteDesc(usedMemory * BYTES_IN_MB))._().
|
||||
td().br().$title(String.valueOf(usedMemory))._().
|
||||
td().br().$title(String.valueOf(availableMemory))._().
|
||||
_(StringUtils.byteDesc(availableMemory * BYTES_IN_MB))._().
|
||||
td(String.valueOf(info.getUsedVirtualCores())).
|
||||
td(String.valueOf(info.getAvailableVirtualCores())).
|
||||
td(ni.getNodeManagerVersion()).
|
||||
_();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -42,11 +42,16 @@ public class ClusterMetricsInfo {
|
|||
protected long availableMB;
|
||||
protected long allocatedMB;
|
||||
|
||||
protected long reservedVirtualCores;
|
||||
protected long availableVirtualCores;
|
||||
protected long allocatedVirtualCores;
|
||||
|
||||
protected int containersAllocated;
|
||||
protected int containersReserved;
|
||||
protected int containersPending;
|
||||
|
||||
protected long totalMB;
|
||||
protected long totalVirtualCores;
|
||||
protected int totalNodes;
|
||||
protected int lostNodes;
|
||||
protected int unhealthyNodes;
|
||||
|
|
@ -73,11 +78,16 @@ public class ClusterMetricsInfo {
|
|||
this.availableMB = metrics.getAvailableMB();
|
||||
this.allocatedMB = metrics.getAllocatedMB();
|
||||
|
||||
this.reservedVirtualCores = metrics.getReservedVirtualCores();
|
||||
this.availableVirtualCores = metrics.getAvailableVirtualCores();
|
||||
this.allocatedVirtualCores = metrics.getAllocatedVirtualCores();
|
||||
|
||||
this.containersAllocated = metrics.getAllocatedContainers();
|
||||
this.containersPending = metrics.getPendingContainers();
|
||||
this.containersReserved = metrics.getReservedContainers();
|
||||
|
||||
this.totalMB = availableMB + allocatedMB;
|
||||
this.totalVirtualCores = availableVirtualCores + allocatedVirtualCores;
|
||||
this.activeNodes = clusterMetrics.getNumActiveNMs();
|
||||
this.lostNodes = clusterMetrics.getNumLostNMs();
|
||||
this.unhealthyNodes = clusterMetrics.getUnhealthyNMs();
|
||||
|
|
@ -123,6 +133,18 @@ public class ClusterMetricsInfo {
|
|||
return this.allocatedMB;
|
||||
}
|
||||
|
||||
public long getReservedVirtualCores() {
|
||||
return this.reservedVirtualCores;
|
||||
}
|
||||
|
||||
public long getAvailableVirtualCores() {
|
||||
return this.availableVirtualCores;
|
||||
}
|
||||
|
||||
public long getAllocatedVirtualCores() {
|
||||
return this.allocatedVirtualCores;
|
||||
}
|
||||
|
||||
public int getContainersAllocated() {
|
||||
return this.containersAllocated;
|
||||
}
|
||||
|
|
@ -139,6 +161,10 @@ public class ClusterMetricsInfo {
|
|||
return this.totalMB;
|
||||
}
|
||||
|
||||
public long getTotalVirtualCores() {
|
||||
return this.totalVirtualCores;
|
||||
}
|
||||
|
||||
public int getTotalNodes() {
|
||||
return this.totalNodes;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -43,6 +43,8 @@ public class NodeInfo {
|
|||
protected int numContainers;
|
||||
protected long usedMemoryMB;
|
||||
protected long availMemoryMB;
|
||||
protected long usedVirtualCores;
|
||||
protected long availableVirtualCores;
|
||||
|
||||
public NodeInfo() {
|
||||
} // JAXB needs this
|
||||
|
|
@ -57,6 +59,8 @@ public class NodeInfo {
|
|||
this.numContainers = report.getNumContainers();
|
||||
this.usedMemoryMB = report.getUsedResource().getMemory();
|
||||
this.availMemoryMB = report.getAvailableResource().getMemory();
|
||||
this.usedVirtualCores = report.getUsedResource().getVirtualCores();
|
||||
this.availableVirtualCores = report.getAvailableResource().getVirtualCores();
|
||||
}
|
||||
this.id = id.toString();
|
||||
this.rack = ni.getRackName();
|
||||
|
|
@ -112,4 +116,12 @@ public class NodeInfo {
|
|||
return this.availMemoryMB;
|
||||
}
|
||||
|
||||
public long getUsedVirtualCores() {
|
||||
return this.usedVirtualCores;
|
||||
}
|
||||
|
||||
public long getAvailableVirtualCores() {
|
||||
return this.availableVirtualCores;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -43,6 +43,9 @@ public class UserMetricsInfo {
|
|||
protected long reservedMB;
|
||||
protected long pendingMB;
|
||||
protected long allocatedMB;
|
||||
protected long reservedVirtualCores;
|
||||
protected long pendingVirtualCores;
|
||||
protected long allocatedVirtualCores;
|
||||
|
||||
@XmlTransient
|
||||
protected boolean userMetricsAvailable;
|
||||
|
|
@ -74,6 +77,10 @@ public class UserMetricsInfo {
|
|||
this.reservedMB = userMetrics.getReservedMB();
|
||||
this.pendingMB = userMetrics.getPendingMB();
|
||||
this.allocatedMB = userMetrics.getAllocatedMB();
|
||||
|
||||
this.reservedVirtualCores = userMetrics.getReservedVirtualCores();
|
||||
this.pendingVirtualCores = userMetrics.getPendingVirtualCores();
|
||||
this.allocatedVirtualCores = userMetrics.getAllocatedVirtualCores();
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -117,6 +124,18 @@ public class UserMetricsInfo {
|
|||
return this.pendingMB;
|
||||
}
|
||||
|
||||
public long getReservedVirtualCores() {
|
||||
return this.reservedVirtualCores;
|
||||
}
|
||||
|
||||
public long getAllocatedVirtualCores() {
|
||||
return this.allocatedVirtualCores;
|
||||
}
|
||||
|
||||
public long getPendingVirtualCores() {
|
||||
return this.pendingVirtualCores;
|
||||
}
|
||||
|
||||
public int getReservedContainers() {
|
||||
return this.reservedContainers;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -48,8 +48,8 @@ public class TestNodesPage {
|
|||
|
||||
// Number of Actual Table Headers for NodesPage.NodesBlock might change in
|
||||
// future. In that case this value should be adjusted to the new value.
|
||||
final int numberOfThInMetricsTable = 13;
|
||||
final int numberOfActualTableHeaders = 10;
|
||||
final int numberOfThInMetricsTable = 16;
|
||||
final int numberOfActualTableHeaders = 12;
|
||||
|
||||
private Injector injector;
|
||||
|
||||
|
|
|
|||
|
|
@ -389,6 +389,10 @@ public class TestRMWebServices extends JerseyTest {
|
|||
WebServicesTestUtils.getXmlInt(element, "reservedMB"),
|
||||
WebServicesTestUtils.getXmlInt(element, "availableMB"),
|
||||
WebServicesTestUtils.getXmlInt(element, "allocatedMB"),
|
||||
WebServicesTestUtils.getXmlInt(element, "reservedVirtualCores"),
|
||||
WebServicesTestUtils.getXmlInt(element, "availableVirtualCores"),
|
||||
WebServicesTestUtils.getXmlInt(element, "allocatedVirtualCores"),
|
||||
WebServicesTestUtils.getXmlInt(element, "totalVirtualCores"),
|
||||
WebServicesTestUtils.getXmlInt(element, "containersAllocated"),
|
||||
WebServicesTestUtils.getXmlInt(element, "totalMB"),
|
||||
WebServicesTestUtils.getXmlInt(element, "totalNodes"),
|
||||
|
|
@ -404,11 +408,13 @@ public class TestRMWebServices extends JerseyTest {
|
|||
Exception {
|
||||
assertEquals("incorrect number of elements", 1, json.length());
|
||||
JSONObject clusterinfo = json.getJSONObject("clusterMetrics");
|
||||
assertEquals("incorrect number of elements", 19, clusterinfo.length());
|
||||
assertEquals("incorrect number of elements", 23, clusterinfo.length());
|
||||
verifyClusterMetrics(
|
||||
clusterinfo.getInt("appsSubmitted"), clusterinfo.getInt("appsCompleted"),
|
||||
clusterinfo.getInt("reservedMB"), clusterinfo.getInt("availableMB"),
|
||||
clusterinfo.getInt("allocatedMB"),
|
||||
clusterinfo.getInt("reservedVirtualCores"), clusterinfo.getInt("availableVirtualCores"),
|
||||
clusterinfo.getInt("allocatedVirtualCores"), clusterinfo.getInt("totalVirtualCores"),
|
||||
clusterinfo.getInt("containersAllocated"),
|
||||
clusterinfo.getInt("totalMB"), clusterinfo.getInt("totalNodes"),
|
||||
clusterinfo.getInt("lostNodes"), clusterinfo.getInt("unhealthyNodes"),
|
||||
|
|
@ -418,7 +424,9 @@ public class TestRMWebServices extends JerseyTest {
|
|||
|
||||
public void verifyClusterMetrics(int submittedApps, int completedApps,
|
||||
int reservedMB, int availableMB,
|
||||
int allocMB, int containersAlloc, int totalMB, int totalNodes,
|
||||
int allocMB, int reservedVirtualCores, int availableVirtualCores,
|
||||
int allocVirtualCores, int totalVirtualCores,
|
||||
int containersAlloc, int totalMB, int totalNodes,
|
||||
int lostNodes, int unhealthyNodes, int decommissionedNodes,
|
||||
int rebootedNodes, int activeNodes) throws JSONException, Exception {
|
||||
|
||||
|
|
@ -428,7 +436,8 @@ public class TestRMWebServices extends JerseyTest {
|
|||
|
||||
long totalMBExpect =
|
||||
metrics.getAvailableMB() + metrics.getAllocatedMB();
|
||||
|
||||
long totalVirtualCoresExpect =
|
||||
metrics.getAvailableVirtualCores() + metrics.getAllocatedVirtualCores();
|
||||
assertEquals("appsSubmitted doesn't match",
|
||||
metrics.getAppsSubmitted(), submittedApps);
|
||||
assertEquals("appsCompleted doesn't match",
|
||||
|
|
@ -439,6 +448,12 @@ public class TestRMWebServices extends JerseyTest {
|
|||
metrics.getAvailableMB(), availableMB);
|
||||
assertEquals("allocatedMB doesn't match",
|
||||
metrics.getAllocatedMB(), allocMB);
|
||||
assertEquals("reservedVirtualCores doesn't match",
|
||||
metrics.getReservedVirtualCores(), reservedVirtualCores);
|
||||
assertEquals("availableVirtualCores doesn't match",
|
||||
metrics.getAvailableVirtualCores(), availableVirtualCores);
|
||||
assertEquals("allocatedVirtualCores doesn't match",
|
||||
totalVirtualCoresExpect, allocVirtualCores);
|
||||
assertEquals("containersAllocated doesn't match", 0, containersAlloc);
|
||||
assertEquals("totalMB doesn't match", totalMBExpect, totalMB);
|
||||
assertEquals(
|
||||
|
|
|
|||
|
|
@ -656,13 +656,15 @@ public class TestRMWebServicesNodes extends JerseyTest {
|
|||
WebServicesTestUtils.getXmlInt(element, "numContainers"),
|
||||
WebServicesTestUtils.getXmlLong(element, "usedMemoryMB"),
|
||||
WebServicesTestUtils.getXmlLong(element, "availMemoryMB"),
|
||||
WebServicesTestUtils.getXmlLong(element, "usedVirtualCores"),
|
||||
WebServicesTestUtils.getXmlLong(element, "availableVirtualCores"),
|
||||
WebServicesTestUtils.getXmlString(element, "version"));
|
||||
}
|
||||
}
|
||||
|
||||
public void verifyNodeInfo(JSONObject nodeInfo, MockNM nm)
|
||||
throws JSONException, Exception {
|
||||
assertEquals("incorrect number of elements", 11, nodeInfo.length());
|
||||
assertEquals("incorrect number of elements", 13, nodeInfo.length());
|
||||
|
||||
verifyNodeInfoGeneric(nm, nodeInfo.getString("state"),
|
||||
nodeInfo.getString("rack"),
|
||||
|
|
@ -671,6 +673,7 @@ public class TestRMWebServicesNodes extends JerseyTest {
|
|||
nodeInfo.getLong("lastHealthUpdate"),
|
||||
nodeInfo.getString("healthReport"), nodeInfo.getInt("numContainers"),
|
||||
nodeInfo.getLong("usedMemoryMB"), nodeInfo.getLong("availMemoryMB"),
|
||||
nodeInfo.getLong("usedVirtualCores"), nodeInfo.getLong("availableVirtualCores"),
|
||||
nodeInfo.getString("version"));
|
||||
|
||||
}
|
||||
|
|
@ -678,7 +681,8 @@ public class TestRMWebServicesNodes extends JerseyTest {
|
|||
public void verifyNodeInfoGeneric(MockNM nm, String state, String rack,
|
||||
String id, String nodeHostName,
|
||||
String nodeHTTPAddress, long lastHealthUpdate, String healthReport,
|
||||
int numContainers, long usedMemoryMB, long availMemoryMB, String version)
|
||||
int numContainers, long usedMemoryMB, long availMemoryMB, long usedVirtualCores,
|
||||
long availVirtualCores, String version)
|
||||
throws JSONException, Exception {
|
||||
|
||||
RMNode node = rm.getRMContext().getRMNodes().get(nm.getNodeId());
|
||||
|
|
@ -712,6 +716,10 @@ public class TestRMWebServicesNodes extends JerseyTest {
|
|||
.getUsedResource().getMemory(), usedMemoryMB);
|
||||
assertEquals("availMemoryMB doesn't match: " + availMemoryMB, report
|
||||
.getAvailableResource().getMemory(), availMemoryMB);
|
||||
assertEquals("usedVirtualCores doesn't match: " + usedVirtualCores, report
|
||||
.getUsedResource().getVirtualCores(), usedVirtualCores);
|
||||
assertEquals("availVirtualCores doesn't match: " + availVirtualCores, report
|
||||
.getAvailableResource().getVirtualCores(), availVirtualCores);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -62,6 +62,8 @@ NodeManager REST API's.
|
|||
*---------------+--------------+-------------------------------+
|
||||
| totalVmemAllocatedContainersMB | long | The amount of virtual memory allocated for use by containers in MB |
|
||||
*---------------+--------------+-------------------------------+
|
||||
| totalVCoresAllocatedContainers | long | The number of virtual cores allocated for use by containers |
|
||||
*---------------+--------------+-------------------------------+
|
||||
| lastNodeUpdateTime | long | The last timestamp at which the health report was received (in ms since epoch)|
|
||||
*---------------+--------------+-------------------------------+
|
||||
| healthReport | string | The diagnostic health report of the node |
|
||||
|
|
@ -109,6 +111,7 @@ NodeManager REST API's.
|
|||
"nodeManagerBuildVersion" : "0.23.1-SNAPSHOT from 1228355 by user1 source checksum 20647f76c36430e888cc7204826a445c",
|
||||
"lastNodeUpdateTime" : 1326222266126,
|
||||
"totalVmemAllocatedContainersMB" : 17203,
|
||||
"totalVCoresAllocatedContainers" : 8,
|
||||
"nodeHealthy" : true,
|
||||
"healthReport" : "",
|
||||
"totalPmemAllocatedContainersMB" : 8192,
|
||||
|
|
@ -148,6 +151,7 @@ NodeManager REST API's.
|
|||
<healthReport/>
|
||||
<totalVmemAllocatedContainersMB>17203</totalVmemAllocatedContainersMB>
|
||||
<totalPmemAllocatedContainersMB>8192</totalPmemAllocatedContainersMB>
|
||||
<totalVCoresAllocatedContainers>8</totalVCoresAllocatedContainers>
|
||||
<lastNodeUpdateTime>1326222386134</lastNodeUpdateTime>
|
||||
<nodeHealthy>true</nodeHealthy>
|
||||
<nodeManagerVersion>0.23.1-SNAPSHOT</nodeManagerVersion>
|
||||
|
|
@ -446,6 +450,7 @@ NodeManager REST API's.
|
|||
{
|
||||
"nodeId" : "host.domain.com:8041",
|
||||
"totalMemoryNeededMB" : 2048,
|
||||
"totalVCoresNeeded" : 1,
|
||||
"state" : "RUNNING",
|
||||
"diagnostics" : "",
|
||||
"containerLogsLink" : "http://host.domain.com:8042/node/containerlogs/container_1326121700862_0006_01_000001/user1",
|
||||
|
|
@ -456,6 +461,7 @@ NodeManager REST API's.
|
|||
{
|
||||
"nodeId" : "host.domain.com:8041",
|
||||
"totalMemoryNeededMB" : 2048,
|
||||
"totalVCoresNeeded" : 2,
|
||||
"state" : "RUNNING",
|
||||
"diagnostics" : "",
|
||||
"containerLogsLink" : "http://host.domain.com:8042/node/containerlogs/container_1326121700862_0006_01_000003/user1",
|
||||
|
|
@ -498,6 +504,7 @@ NodeManager REST API's.
|
|||
<diagnostics/>
|
||||
<user>user1</user>
|
||||
<totalMemoryNeededMB>2048</totalMemoryNeededMB>
|
||||
<totalVCoresNeeded>1</totalVCoresNeeded>
|
||||
<containerLogsLink>http://host.domain.com:8042/node/containerlogs/container_1326121700862_0006_01_000001/user1</containerLogsLink>
|
||||
<nodeId>host.domain.com:8041</nodeId>
|
||||
</container>
|
||||
|
|
@ -508,6 +515,7 @@ NodeManager REST API's.
|
|||
<diagnostics>Container killed by the ApplicationMaster.</diagnostics>
|
||||
<user>user1</user>
|
||||
<totalMemoryNeededMB>2048</totalMemoryNeededMB>
|
||||
<totalVCoresNeeded>2</totalVCoresNeeded>
|
||||
<containerLogsLink>http://host.domain.com:8042/node/containerlogs/container_1326121700862_0006_01_000003/user1</containerLogsLink>
|
||||
<nodeId>host.domain.com:8041</nodeId>
|
||||
</container>
|
||||
|
|
@ -560,6 +568,8 @@ NodeManager REST API's.
|
|||
*---------------+--------------+-------------------------------+
|
||||
| totalMemoryNeededMB | long | Total amout of memory needed by the container (in MB) |
|
||||
*---------------+--------------+-------------------------------+
|
||||
| totalVCoresNeeded | long | Total number of virtual cores needed by the container |
|
||||
*---------------+--------------+-------------------------------+
|
||||
|
||||
** Response Examples
|
||||
|
||||
|
|
@ -587,6 +597,7 @@ NodeManager REST API's.
|
|||
"container" : {
|
||||
"nodeId" : "host.domain.com:8041",
|
||||
"totalMemoryNeededMB" : 2048,
|
||||
"totalVCoresNeeded" : 1,
|
||||
"state" : "RUNNING",
|
||||
"diagnostics" : "",
|
||||
"containerLogsLink" : "http://host.domain.com:8042/node/containerlogs/container_1326121700862_0007_01_000001/user1",
|
||||
|
|
@ -626,6 +637,7 @@ NodeManager REST API's.
|
|||
<diagnostics/>
|
||||
<user>user1</user>
|
||||
<totalMemoryNeededMB>2048</totalMemoryNeededMB>
|
||||
<totalVCoresNeeded>1</totalVCoresNeeded>
|
||||
<containerLogsLink>http://host.domain.com:8042/node/containerlogs/container_1326121700862_0007_01_000001/user1</containerLogsLink>
|
||||
<nodeId>host.domain.com:8041</nodeId>
|
||||
</container>
|
||||
|
|
|
|||
|
|
@ -195,6 +195,14 @@ ResourceManager REST API's.
|
|||
*---------------+--------------+-------------------------------+
|
||||
| totalMB | long | The amount of total memory in MB |
|
||||
*---------------+--------------+-------------------------------+
|
||||
| reservedVirtualCores | long | The number of reserved virtual cores |
|
||||
*---------------+--------------+-------------------------------+
|
||||
| availableVirtualCores | long | The number of available virtual cores |
|
||||
*---------------+--------------+-------------------------------+
|
||||
| allocatedVirtualCores | long | The number of allocated virtual cores |
|
||||
*---------------+--------------+-------------------------------+
|
||||
| totalVirtualCores | long | The total number of virtual cores |
|
||||
*---------------+--------------+-------------------------------+
|
||||
| containersAllocated | int | The number of containers allocated |
|
||||
*---------------+--------------+-------------------------------+
|
||||
| containersReserved | int | The number of containers reserved |
|
||||
|
|
@ -247,10 +255,14 @@ ResourceManager REST API's.
|
|||
"reservedMB":0,
|
||||
"availableMB":17408,
|
||||
"allocatedMB":0,
|
||||
"reservedVirtualCores":0,
|
||||
"availableVirtualCores":7,
|
||||
"allocatedVirtualCores":1,
|
||||
"containersAllocated":0,
|
||||
"containersReserved":0,
|
||||
"containersPending":0,
|
||||
"totalMB":17408,
|
||||
"totalVirtualCores":8,
|
||||
"totalNodes":1,
|
||||
"lostNodes":0,
|
||||
"unhealthyNodes":0,
|
||||
|
|
@ -293,10 +305,14 @@ ResourceManager REST API's.
|
|||
<reservedMB>0</reservedMB>
|
||||
<availableMB>17408</availableMB>
|
||||
<allocatedMB>0</allocatedMB>
|
||||
<reservedVirtualCores>0</reservedVirtualCores>
|
||||
<availableVirtualCores>7</availableVirtualCores>
|
||||
<allocatedVirtualCores>1</allocatedVirtualCores>
|
||||
<containersAllocated>0</containersAllocated>
|
||||
<containersReserved>0</containersReserved>
|
||||
<containersPending>0</containersPending>
|
||||
<totalMB>17408</totalMB>
|
||||
<totalVirtualCores>8</totalVirtualCores>
|
||||
<totalNodes>1</totalNodes>
|
||||
<lostNodes>0</lostNodes>
|
||||
<unhealthyNodes>0</unhealthyNodes>
|
||||
|
|
@ -2040,8 +2056,10 @@ Server: Jetty(6.1.26)
|
|||
"lastHealthUpdate":1324056895432,
|
||||
"healthReport":"Healthy",
|
||||
"numContainers":0,
|
||||
"usedMemoryMB":0
|
||||
"availMemoryMB":8192
|
||||
"usedMemoryMB":0,
|
||||
"availMemoryMB":8192,
|
||||
"usedVirtualCores":0,
|
||||
"availableVirtualCores":8
|
||||
},
|
||||
{
|
||||
"rack":"\/default-rack",
|
||||
|
|
@ -2054,7 +2072,9 @@ Server: Jetty(6.1.26)
|
|||
"healthReport":"Healthy",
|
||||
"numContainers":0,
|
||||
"usedMemoryMB":0,
|
||||
"availMemoryMB":8192
|
||||
"availMemoryMB":8192,
|
||||
"usedVirtualCores":0,
|
||||
"availableVirtualCores":8
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -2096,6 +2116,8 @@ Server: Jetty(6.1.26)
|
|||
<numContainers>0</numContainers>
|
||||
<usedMemoryMB>0</usedMemoryMB>
|
||||
<availMemoryMB>5120</availMemoryMB>
|
||||
<usedVirtualCores>0</usedVirtualCores>
|
||||
<availableVirtualCores>8</availableVirtualCores>
|
||||
</node>
|
||||
<node>
|
||||
<rack>/default-rack</rack>
|
||||
|
|
@ -2109,6 +2131,8 @@ Server: Jetty(6.1.26)
|
|||
<numContainers>0</numContainers>
|
||||
<usedMemoryMB>0</usedMemoryMB>
|
||||
<availMemoryMB>5120</availMemoryMB>
|
||||
<usedVirtualCores>0</usedVirtualCores>
|
||||
<availableVirtualCores>8</availableVirtualCores>
|
||||
</node>
|
||||
</nodes>
|
||||
+---+
|
||||
|
|
@ -2159,10 +2183,14 @@ Server: Jetty(6.1.26)
|
|||
*---------------+--------------+-------------------------------+
|
||||
| lastHealthUpdate | long | The last time the node reported its health (in ms since epoch)|
|
||||
*---------------+--------------+-------------------------------+
|
||||
| usedMemoryMB | long | The total about of memory currently used on the node (in MB)|
|
||||
| usedMemoryMB | long | The total amount of memory currently used on the node (in MB)|
|
||||
*---------------+--------------+-------------------------------+
|
||||
| availMemoryMB | long | The total amount of memory currently available on the node (in MB)|
|
||||
*---------------+--------------+-------------------------------+
|
||||
| usedVirtualCores | long | The total number of vCores currently used on the node |
|
||||
*---------------+--------------+-------------------------------+
|
||||
| availableVirtualCores | long | The total number of vCores available on the node |
|
||||
*---------------+--------------+-------------------------------+
|
||||
| numContainers | int | The total number of containers currently running on the node|
|
||||
*---------------+--------------+-------------------------------+
|
||||
|
||||
|
|
@ -2201,7 +2229,9 @@ Server: Jetty(6.1.26)
|
|||
"healthReport":"Healthy",
|
||||
"numContainers":0,
|
||||
"usedMemoryMB":0,
|
||||
"availMemoryMB":5120
|
||||
"availMemoryMB":5120,
|
||||
"usedVirtualCores":0,
|
||||
"availableVirtualCores":8
|
||||
}
|
||||
}
|
||||
+---+
|
||||
|
|
@ -2240,6 +2270,8 @@ Server: Jetty(6.1.26)
|
|||
<numContainers>0</numContainers>
|
||||
<usedMemoryMB>0</usedMemoryMB>
|
||||
<availMemoryMB>5120</availMemoryMB>
|
||||
<usedVirtualCores>0</usedVirtualCores>
|
||||
<availableVirtualCores>5120</availableVirtualCores>
|
||||
</node>
|
||||
+---+
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue