diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 170af0d3ec3..d014b9ada83 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -183,6 +183,9 @@ Release 2.5.0 - UNRELEASED YARN-2109. Fix TestRM to work with both schedulers. (Anubhav Dhoot via kasha) + YARN-2072. RM/NM UIs and webservices are missing vcore information. + (Nathan Roberts via tgraves) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ResourceView.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ResourceView.java index a41db4324bf..4fde7b926a2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ResourceView.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ResourceView.java @@ -27,4 +27,6 @@ public interface ResourceView { long getPmemAllocatedForContainers(); boolean isPmemCheckEnabled(); + + long getVCoresAllocatedForContainers(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java index 04887ed7576..02a63acfb1a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java @@ -70,6 +70,8 @@ public class ContainersMonitorImpl extends AbstractService implements private boolean pmemCheckEnabled; private boolean vmemCheckEnabled; + private long maxVCoresAllottedForContainers; + private static final long UNKNOWN_MEMORY_LIMIT = -1L; public ContainersMonitorImpl(ContainerExecutor exec, @@ -108,10 +110,16 @@ public class ContainersMonitorImpl extends AbstractService implements YarnConfiguration.NM_PMEM_MB, YarnConfiguration.DEFAULT_NM_PMEM_MB) * 1024 * 1024l; + long configuredVCoresForContainers = conf.getLong( + YarnConfiguration.NM_VCORES, + YarnConfiguration.DEFAULT_NM_VCORES); + + // Setting these irrespective of whether checks are enabled. Required in // the UI. // ///////// Physical memory configuration ////// this.maxPmemAllottedForContainers = configuredPMemForContainers; + this.maxVCoresAllottedForContainers = configuredVCoresForContainers; // ///////// Virtual memory configuration ////// float vmemRatio = conf.getFloat(YarnConfiguration.NM_VMEM_PMEM_RATIO, @@ -518,6 +526,11 @@ public class ContainersMonitorImpl extends AbstractService implements return this.maxPmemAllottedForContainers; } + @Override + public long getVCoresAllocatedForContainers() { + return this.maxVCoresAllottedForContainers; + } + /** * Is the total virtual memory check enabled? * diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java index 1feb8c70e77..3da21f0fe50 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java @@ -40,6 +40,9 @@ public class NodeManagerMetrics { @Metric("Current # of allocated containers") MutableGaugeInt allocatedContainers; @Metric MutableGaugeInt availableGB; + @Metric("Current allocated Virtual Cores") + MutableGaugeInt allocatedVCores; + @Metric MutableGaugeInt availableVCores; public static NodeManagerMetrics create() { return create(DefaultMetricsSystem.instance()); @@ -88,16 +91,21 @@ public class NodeManagerMetrics { allocatedContainers.incr(); allocatedGB.incr(res.getMemory() / 1024); availableGB.decr(res.getMemory() / 1024); + allocatedVCores.incr(res.getVirtualCores()); + availableVCores.decr(res.getVirtualCores()); } public void releaseContainer(Resource res) { allocatedContainers.decr(); allocatedGB.decr(res.getMemory() / 1024); availableGB.incr(res.getMemory() / 1024); + allocatedVCores.decr(res.getVirtualCores()); + availableVCores.incr(res.getVirtualCores()); } public void addResource(Resource res) { availableGB.incr(res.getMemory() / 1024); + availableVCores.incr(res.getVirtualCores()); } public int getRunningContainers() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerPage.java index 060d72ac98f..f4367bcb892 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerPage.java @@ -85,6 +85,7 @@ public class ContainerPage extends NMView implements YarnWebParams { ._("Diagnostics", info.getDiagnostics()) ._("User", info.getUser()) ._("TotalMemoryNeeded", info.getMemoryNeeded()) + ._("TotalVCoresNeeded", info.getVCoresNeeded()) ._("logs", info.getShortLogLink(), "Link to logs"); html._(InfoBlock.class); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NodePage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NodePage.java index 2831a18d5d0..5e5408d8de5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NodePage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NodePage.java @@ -73,6 +73,8 @@ public class NodePage extends NMView { StringUtils.byteDesc(info.getTotalPmemAllocated() * BYTES_IN_MB)) ._("Pmem enforcement enabled", info.isVmemCheckEnabled()) + ._("Total VCores allocated for Containers", + String.valueOf(info.getTotalVCoresAllocated())) ._("NodeHealthyStatus", info.getHealthStatus()) ._("LastNodeHealthTime", new Date( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java index 0da0a88fa7e..78982deb19f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java @@ -42,6 +42,7 @@ public class ContainerInfo { protected String diagnostics; protected String user; protected long totalMemoryNeededMB; + protected long totalVCoresNeeded; protected String containerLogsLink; protected String nodeId; @XmlTransient @@ -76,6 +77,7 @@ public class ContainerInfo { Resource res = container.getResource(); if (res != null) { this.totalMemoryNeededMB = res.getMemory(); + this.totalVCoresNeeded = res.getVirtualCores(); } this.containerLogsShortLink = ujoin("containerlogs", this.id, container.getUser()); @@ -130,4 +132,8 @@ public class ContainerInfo { return this.totalMemoryNeededMB; } + public long getVCoresNeeded() { + return this.totalVCoresNeeded; + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/NodeInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/NodeInfo.java index 9b1e8e5768a..5d67c9e6d6b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/NodeInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/NodeInfo.java @@ -36,6 +36,7 @@ public class NodeInfo { protected String healthReport; protected long totalVmemAllocatedContainersMB; protected long totalPmemAllocatedContainersMB; + protected long totalVCoresAllocatedContainers; protected boolean vmemCheckEnabled; protected boolean pmemCheckEnabled; protected long lastNodeUpdateTime; @@ -62,6 +63,8 @@ public class NodeInfo { this.totalPmemAllocatedContainersMB = resourceView .getPmemAllocatedForContainers() / BYTES_IN_MB; this.pmemCheckEnabled = resourceView.isPmemCheckEnabled(); + this.totalVCoresAllocatedContainers = resourceView + .getVCoresAllocatedForContainers(); this.nodeHealthy = context.getNodeHealthStatus().getIsNodeHealthy(); this.lastNodeUpdateTime = context.getNodeHealthStatus() .getLastHealthReportTime(); @@ -124,6 +127,10 @@ public class NodeInfo { return this.totalVmemAllocatedContainersMB; } + public long getTotalVCoresAllocated() { + return this.totalVCoresAllocatedContainers; + } + public boolean isVmemCheckEnabled() { return this.vmemCheckEnabled; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/metrics/TestNodeManagerMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/metrics/TestNodeManagerMetrics.java index 8384220b6c5..a54450dc1f5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/metrics/TestNodeManagerMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/metrics/TestNodeManagerMetrics.java @@ -31,8 +31,11 @@ public class TestNodeManagerMetrics { NodeManagerMetrics metrics = NodeManagerMetrics.create(); Resource total = Records.newRecord(Resource.class); total.setMemory(8*GiB); + total.setVirtualCores(16); Resource resource = Records.newRecord(Resource.class); resource.setMemory(1*GiB); + resource.setVirtualCores(2); + metrics.addResource(total); @@ -57,12 +60,12 @@ public class TestNodeManagerMetrics { metrics.initingContainer(); metrics.runningContainer(); - checkMetrics(5, 1, 1, 1, 1, 1, 2, 2, 6); + checkMetrics(5, 1, 1, 1, 1, 1, 2, 2, 6, 4, 12); } private void checkMetrics(int launched, int completed, int failed, int killed, int initing, int running, int allocatedGB, - int allocatedContainers, int availableGB) { + int allocatedContainers, int availableGB, int allocatedVCores, int availableVCores) { MetricsRecordBuilder rb = getMetrics("NodeManagerMetrics"); assertCounter("ContainersLaunched", launched, rb); assertCounter("ContainersCompleted", completed, rb); @@ -71,7 +74,10 @@ public class TestNodeManagerMetrics { assertGauge("ContainersIniting", initing, rb); assertGauge("ContainersRunning", running, rb); assertGauge("AllocatedGB", allocatedGB, rb); + assertGauge("AllocatedVCores", allocatedVCores, rb); assertGauge("AllocatedContainers", allocatedContainers, rb); assertGauge("AvailableGB", availableGB, rb); + assertGauge("AvailableVCores",availableVCores, rb); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java index 13a77afa9b4..4d6ac7788d5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java @@ -91,6 +91,10 @@ public class TestNMWebServer { return 0; } @Override + public long getVCoresAllocatedForContainers() { + return 0; + } + @Override public boolean isVmemCheckEnabled() { return true; } @@ -150,6 +154,10 @@ public class TestNMWebServer { return 0; } @Override + public long getVCoresAllocatedForContainers() { + return 0; + } + @Override public boolean isVmemCheckEnabled() { return true; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java index e90fe7e4fdc..61bdf1036c4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java @@ -124,6 +124,10 @@ public class TestNMWebServices extends JerseyTest { return new Long("17179869184"); } @Override + public long getVCoresAllocatedForContainers() { + return new Long("4000"); + } + @Override public boolean isVmemCheckEnabled() { return true; } @@ -375,6 +379,8 @@ public class TestNMWebServices extends JerseyTest { "totalVmemAllocatedContainersMB"), WebServicesTestUtils.getXmlLong(element, "totalPmemAllocatedContainersMB"), + WebServicesTestUtils.getXmlLong(element, + "totalVCoresAllocatedContainers"), WebServicesTestUtils.getXmlBoolean(element, "vmemCheckEnabled"), WebServicesTestUtils.getXmlBoolean(element, "pmemCheckEnabled"), WebServicesTestUtils.getXmlLong(element, "lastNodeUpdateTime"), @@ -393,10 +399,11 @@ public class TestNMWebServices extends JerseyTest { public void verifyNodeInfo(JSONObject json) throws JSONException, Exception { assertEquals("incorrect number of elements", 1, json.length()); JSONObject info = json.getJSONObject("nodeInfo"); - assertEquals("incorrect number of elements", 15, info.length()); + assertEquals("incorrect number of elements", 16, info.length()); verifyNodeInfoGeneric(info.getString("id"), info.getString("healthReport"), info.getLong("totalVmemAllocatedContainersMB"), info.getLong("totalPmemAllocatedContainersMB"), + info.getLong("totalVCoresAllocatedContainers"), info.getBoolean("vmemCheckEnabled"), info.getBoolean("pmemCheckEnabled"), info.getLong("lastNodeUpdateTime"), info.getBoolean("nodeHealthy"), @@ -410,6 +417,7 @@ public class TestNMWebServices extends JerseyTest { public void verifyNodeInfoGeneric(String id, String healthReport, long totalVmemAllocatedContainersMB, long totalPmemAllocatedContainersMB, + long totalVCoresAllocatedContainers, boolean vmemCheckEnabled, boolean pmemCheckEnabled, long lastNodeUpdateTime, Boolean nodeHealthy, String nodeHostName, String hadoopVersionBuiltOn, String hadoopBuildVersion, @@ -423,6 +431,8 @@ public class TestNMWebServices extends JerseyTest { totalVmemAllocatedContainersMB); assertEquals("totalPmemAllocatedContainersMB incorrect", 16384, totalPmemAllocatedContainersMB); + assertEquals("totalVCoresAllocatedContainers incorrect", 4000, + totalVCoresAllocatedContainers); assertEquals("vmemCheckEnabled incorrect", true, vmemCheckEnabled); assertEquals("pmemCheckEnabled incorrect", true, pmemCheckEnabled); assertTrue("lastNodeUpdateTime incorrect", lastNodeUpdateTime == nmContext diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java index a5ceaa4c842..87aa8526894 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java @@ -116,6 +116,12 @@ public class TestNMWebServicesApps extends JerseyTest { return new Long("17179869184"); } + @Override + public long getVCoresAllocatedForContainers() { + return new Long("4000"); + } + + @Override public boolean isVmemCheckEnabled() { return true; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java index 91400a6096b..62d9cb7b3b4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java @@ -106,6 +106,11 @@ public class TestNMWebServicesContainers extends JerseyTest { return new Long("17179869184"); } + @Override + public long getVCoresAllocatedForContainers() { + return new Long("4000"); + } + @Override public boolean isVmemCheckEnabled() { return true; @@ -461,24 +466,27 @@ public class TestNMWebServicesContainers extends JerseyTest { WebServicesTestUtils.getXmlString(element, "diagnostics"), WebServicesTestUtils.getXmlString(element, "nodeId"), WebServicesTestUtils.getXmlInt(element, "totalMemoryNeededMB"), + WebServicesTestUtils.getXmlInt(element, "totalVCoresNeeded"), WebServicesTestUtils.getXmlString(element, "containerLogsLink")); } } public void verifyNodeContainerInfo(JSONObject info, Container cont) throws JSONException, Exception { - assertEquals("incorrect number of elements", 8, info.length()); + assertEquals("incorrect number of elements", 9, info.length()); verifyNodeContainerInfoGeneric(cont, info.getString("id"), info.getString("state"), info.getString("user"), info.getInt("exitCode"), info.getString("diagnostics"), info.getString("nodeId"), info.getInt("totalMemoryNeededMB"), + info.getInt("totalVCoresNeeded"), info.getString("containerLogsLink")); } public void verifyNodeContainerInfoGeneric(Container cont, String id, String state, String user, int exitCode, String diagnostics, - String nodeId, int totalMemoryNeededMB, String logsLink) + String nodeId, int totalMemoryNeededMB, int totalVCoresNeeded, + String logsLink) throws JSONException, Exception { WebServicesTestUtils.checkStringMatch("id", cont.getContainerId() .toString(), id); @@ -495,6 +503,9 @@ public class TestNMWebServicesContainers extends JerseyTest { assertEquals("totalMemoryNeededMB wrong", YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB, totalMemoryNeededMB); + assertEquals("totalVCoresNeeded wrong", + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, + totalVCoresNeeded); String shortLink = ujoin("containerlogs", cont.getContainerId().toString(), cont.getUser()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java index 7a8b681200a..7e41e5384f8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java @@ -72,6 +72,9 @@ public class MetricsOverviewTable extends HtmlBlock { th().$class("ui-state-default")._("Memory Used")._(). th().$class("ui-state-default")._("Memory Total")._(). th().$class("ui-state-default")._("Memory Reserved")._(). + th().$class("ui-state-default")._("VCores Used")._(). + th().$class("ui-state-default")._("VCores Total")._(). + th().$class("ui-state-default")._("VCores Reserved")._(). th().$class("ui-state-default")._("Active Nodes")._(). th().$class("ui-state-default")._("Decommissioned Nodes")._(). th().$class("ui-state-default")._("Lost Nodes")._(). @@ -94,6 +97,9 @@ public class MetricsOverviewTable extends HtmlBlock { td(StringUtils.byteDesc(clusterMetrics.getAllocatedMB() * BYTES_IN_MB)). td(StringUtils.byteDesc(clusterMetrics.getTotalMB() * BYTES_IN_MB)). td(StringUtils.byteDesc(clusterMetrics.getReservedMB() * BYTES_IN_MB)). + td(String.valueOf(clusterMetrics.getAllocatedVirtualCores())). + td(String.valueOf(clusterMetrics.getTotalVirtualCores())). + td(String.valueOf(clusterMetrics.getReservedVirtualCores())). td().a(url("nodes"),String.valueOf(clusterMetrics.getActiveNodes()))._(). td().a(url("nodes/decommissioned"),String.valueOf(clusterMetrics.getDecommissionedNodes()))._(). td().a(url("nodes/lost"),String.valueOf(clusterMetrics.getLostNodes()))._(). @@ -120,6 +126,9 @@ public class MetricsOverviewTable extends HtmlBlock { th().$class("ui-state-default")._("Memory Used")._(). th().$class("ui-state-default")._("Memory Pending")._(). th().$class("ui-state-default")._("Memory Reserved")._(). + th().$class("ui-state-default")._("VCores Used")._(). + th().$class("ui-state-default")._("VCores Pending")._(). + th().$class("ui-state-default")._("VCores Reserved")._(). _(). _(). tbody().$class("ui-widget-content"). @@ -139,6 +148,9 @@ public class MetricsOverviewTable extends HtmlBlock { td(StringUtils.byteDesc(userMetrics.getAllocatedMB() * BYTES_IN_MB)). td(StringUtils.byteDesc(userMetrics.getPendingMB() * BYTES_IN_MB)). td(StringUtils.byteDesc(userMetrics.getReservedMB() * BYTES_IN_MB)). + td(String.valueOf(userMetrics.getAllocatedVirtualCores())). + td(String.valueOf(userMetrics.getPendingVirtualCores())). + td(String.valueOf(userMetrics.getReservedVirtualCores())). _(). _()._(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java index d5c75d04c28..f10e255938a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java @@ -75,6 +75,8 @@ class NodesPage extends RmView { th(".containers", "Containers"). th(".mem", "Mem Used"). th(".mem", "Mem Avail"). + th(".vcores", "VCores Used"). + th(".vcores", "VCores Avail"). th(".nodeManagerVersion", "Version"). _()._(). tbody(); @@ -127,8 +129,10 @@ class NodesPage extends RmView { td(String.valueOf(info.getNumContainers())). td().br().$title(String.valueOf(usedMemory))._(). _(StringUtils.byteDesc(usedMemory * BYTES_IN_MB))._(). - td().br().$title(String.valueOf(usedMemory))._(). + td().br().$title(String.valueOf(availableMemory))._(). _(StringUtils.byteDesc(availableMemory * BYTES_IN_MB))._(). + td(String.valueOf(info.getUsedVirtualCores())). + td(String.valueOf(info.getAvailableVirtualCores())). td(ni.getNodeManagerVersion()). _(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java index 92c9678371b..db553ae0918 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java @@ -37,16 +37,21 @@ public class ClusterMetricsInfo { protected int appsRunning; protected int appsFailed; protected int appsKilled; - + protected long reservedMB; protected long availableMB; protected long allocatedMB; - + + protected long reservedVirtualCores; + protected long availableVirtualCores; + protected long allocatedVirtualCores; + protected int containersAllocated; protected int containersReserved; protected int containersPending; - + protected long totalMB; + protected long totalVirtualCores; protected int totalNodes; protected int lostNodes; protected int unhealthyNodes; @@ -68,16 +73,21 @@ public class ClusterMetricsInfo { this.appsRunning = metrics.getAppsRunning(); this.appsFailed = metrics.getAppsFailed(); this.appsKilled = metrics.getAppsKilled(); - + this.reservedMB = metrics.getReservedMB(); this.availableMB = metrics.getAvailableMB(); this.allocatedMB = metrics.getAllocatedMB(); - + + this.reservedVirtualCores = metrics.getReservedVirtualCores(); + this.availableVirtualCores = metrics.getAvailableVirtualCores(); + this.allocatedVirtualCores = metrics.getAllocatedVirtualCores(); + this.containersAllocated = metrics.getAllocatedContainers(); this.containersPending = metrics.getPendingContainers(); this.containersReserved = metrics.getReservedContainers(); - + this.totalMB = availableMB + allocatedMB; + this.totalVirtualCores = availableVirtualCores + allocatedVirtualCores; this.activeNodes = clusterMetrics.getNumActiveNMs(); this.lostNodes = clusterMetrics.getNumLostNMs(); this.unhealthyNodes = clusterMetrics.getUnhealthyNMs(); @@ -123,6 +133,18 @@ public class ClusterMetricsInfo { return this.allocatedMB; } + public long getReservedVirtualCores() { + return this.reservedVirtualCores; + } + + public long getAvailableVirtualCores() { + return this.availableVirtualCores; + } + + public long getAllocatedVirtualCores() { + return this.allocatedVirtualCores; + } + public int getContainersAllocated() { return this.containersAllocated; } @@ -134,15 +156,19 @@ public class ClusterMetricsInfo { public int getPendingContainers() { return this.containersPending; } - + public long getTotalMB() { return this.totalMB; } + public long getTotalVirtualCores() { + return this.totalVirtualCores; + } + public int getTotalNodes() { return this.totalNodes; } - + public int getActiveNodes() { return this.activeNodes; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java index 3354ff20308..73a2db17242 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java @@ -43,6 +43,8 @@ public class NodeInfo { protected int numContainers; protected long usedMemoryMB; protected long availMemoryMB; + protected long usedVirtualCores; + protected long availableVirtualCores; public NodeInfo() { } // JAXB needs this @@ -57,6 +59,8 @@ public class NodeInfo { this.numContainers = report.getNumContainers(); this.usedMemoryMB = report.getUsedResource().getMemory(); this.availMemoryMB = report.getAvailableResource().getMemory(); + this.usedVirtualCores = report.getUsedResource().getVirtualCores(); + this.availableVirtualCores = report.getAvailableResource().getVirtualCores(); } this.id = id.toString(); this.rack = ni.getRackName(); @@ -83,7 +87,7 @@ public class NodeInfo { public String getNodeHTTPAddress() { return this.nodeHTTPAddress; } - + public void setNodeHTTPAddress(String nodeHTTPAddress) { this.nodeHTTPAddress = nodeHTTPAddress; } @@ -112,4 +116,12 @@ public class NodeInfo { return this.availMemoryMB; } + public long getUsedVirtualCores() { + return this.usedVirtualCores; + } + + public long getAvailableVirtualCores() { + return this.availableVirtualCores; + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/UserMetricsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/UserMetricsInfo.java index 9d4d77ce08b..e05cd85220c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/UserMetricsInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/UserMetricsInfo.java @@ -43,6 +43,9 @@ public class UserMetricsInfo { protected long reservedMB; protected long pendingMB; protected long allocatedMB; + protected long reservedVirtualCores; + protected long pendingVirtualCores; + protected long allocatedVirtualCores; @XmlTransient protected boolean userMetricsAvailable; @@ -59,7 +62,7 @@ public class UserMetricsInfo { if (userMetrics != null) { this.userMetricsAvailable = true; - + this.appsSubmitted = userMetrics.getAppsSubmitted(); this.appsCompleted = metrics.getAppsCompleted(); this.appsPending = metrics.getAppsPending(); @@ -70,10 +73,14 @@ public class UserMetricsInfo { this.runningContainers = userMetrics.getAllocatedContainers(); this.pendingContainers = userMetrics.getPendingContainers(); this.reservedContainers = userMetrics.getReservedContainers(); - + this.reservedMB = userMetrics.getReservedMB(); this.pendingMB = userMetrics.getPendingMB(); this.allocatedMB = userMetrics.getAllocatedMB(); + + this.reservedVirtualCores = userMetrics.getReservedVirtualCores(); + this.pendingVirtualCores = userMetrics.getPendingVirtualCores(); + this.allocatedVirtualCores = userMetrics.getAllocatedVirtualCores(); } } @@ -117,6 +124,18 @@ public class UserMetricsInfo { return this.pendingMB; } + public long getReservedVirtualCores() { + return this.reservedVirtualCores; + } + + public long getAllocatedVirtualCores() { + return this.allocatedVirtualCores; + } + + public long getPendingVirtualCores() { + return this.pendingVirtualCores; + } + public int getReservedContainers() { return this.reservedContainers; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java index 1979273ce12..09743114a9d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java @@ -48,8 +48,8 @@ public class TestNodesPage { // Number of Actual Table Headers for NodesPage.NodesBlock might change in // future. In that case this value should be adjusted to the new value. - final int numberOfThInMetricsTable = 13; - final int numberOfActualTableHeaders = 10; + final int numberOfThInMetricsTable = 16; + final int numberOfActualTableHeaders = 12; private Injector injector; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java index 4f1208cd145..561b1478f74 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java @@ -389,6 +389,10 @@ public class TestRMWebServices extends JerseyTest { WebServicesTestUtils.getXmlInt(element, "reservedMB"), WebServicesTestUtils.getXmlInt(element, "availableMB"), WebServicesTestUtils.getXmlInt(element, "allocatedMB"), + WebServicesTestUtils.getXmlInt(element, "reservedVirtualCores"), + WebServicesTestUtils.getXmlInt(element, "availableVirtualCores"), + WebServicesTestUtils.getXmlInt(element, "allocatedVirtualCores"), + WebServicesTestUtils.getXmlInt(element, "totalVirtualCores"), WebServicesTestUtils.getXmlInt(element, "containersAllocated"), WebServicesTestUtils.getXmlInt(element, "totalMB"), WebServicesTestUtils.getXmlInt(element, "totalNodes"), @@ -404,11 +408,13 @@ public class TestRMWebServices extends JerseyTest { Exception { assertEquals("incorrect number of elements", 1, json.length()); JSONObject clusterinfo = json.getJSONObject("clusterMetrics"); - assertEquals("incorrect number of elements", 19, clusterinfo.length()); + assertEquals("incorrect number of elements", 23, clusterinfo.length()); verifyClusterMetrics( clusterinfo.getInt("appsSubmitted"), clusterinfo.getInt("appsCompleted"), clusterinfo.getInt("reservedMB"), clusterinfo.getInt("availableMB"), clusterinfo.getInt("allocatedMB"), + clusterinfo.getInt("reservedVirtualCores"), clusterinfo.getInt("availableVirtualCores"), + clusterinfo.getInt("allocatedVirtualCores"), clusterinfo.getInt("totalVirtualCores"), clusterinfo.getInt("containersAllocated"), clusterinfo.getInt("totalMB"), clusterinfo.getInt("totalNodes"), clusterinfo.getInt("lostNodes"), clusterinfo.getInt("unhealthyNodes"), @@ -418,7 +424,9 @@ public class TestRMWebServices extends JerseyTest { public void verifyClusterMetrics(int submittedApps, int completedApps, int reservedMB, int availableMB, - int allocMB, int containersAlloc, int totalMB, int totalNodes, + int allocMB, int reservedVirtualCores, int availableVirtualCores, + int allocVirtualCores, int totalVirtualCores, + int containersAlloc, int totalMB, int totalNodes, int lostNodes, int unhealthyNodes, int decommissionedNodes, int rebootedNodes, int activeNodes) throws JSONException, Exception { @@ -428,7 +436,8 @@ public class TestRMWebServices extends JerseyTest { long totalMBExpect = metrics.getAvailableMB() + metrics.getAllocatedMB(); - + long totalVirtualCoresExpect = + metrics.getAvailableVirtualCores() + metrics.getAllocatedVirtualCores(); assertEquals("appsSubmitted doesn't match", metrics.getAppsSubmitted(), submittedApps); assertEquals("appsCompleted doesn't match", @@ -439,6 +448,12 @@ public class TestRMWebServices extends JerseyTest { metrics.getAvailableMB(), availableMB); assertEquals("allocatedMB doesn't match", metrics.getAllocatedMB(), allocMB); + assertEquals("reservedVirtualCores doesn't match", + metrics.getReservedVirtualCores(), reservedVirtualCores); + assertEquals("availableVirtualCores doesn't match", + metrics.getAvailableVirtualCores(), availableVirtualCores); + assertEquals("allocatedVirtualCores doesn't match", + totalVirtualCoresExpect, allocVirtualCores); assertEquals("containersAllocated doesn't match", 0, containersAlloc); assertEquals("totalMB doesn't match", totalMBExpect, totalMB); assertEquals( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java index da2e2b1022f..e2605039e23 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java @@ -656,13 +656,15 @@ public class TestRMWebServicesNodes extends JerseyTest { WebServicesTestUtils.getXmlInt(element, "numContainers"), WebServicesTestUtils.getXmlLong(element, "usedMemoryMB"), WebServicesTestUtils.getXmlLong(element, "availMemoryMB"), + WebServicesTestUtils.getXmlLong(element, "usedVirtualCores"), + WebServicesTestUtils.getXmlLong(element, "availableVirtualCores"), WebServicesTestUtils.getXmlString(element, "version")); } } public void verifyNodeInfo(JSONObject nodeInfo, MockNM nm) throws JSONException, Exception { - assertEquals("incorrect number of elements", 11, nodeInfo.length()); + assertEquals("incorrect number of elements", 13, nodeInfo.length()); verifyNodeInfoGeneric(nm, nodeInfo.getString("state"), nodeInfo.getString("rack"), @@ -671,6 +673,7 @@ public class TestRMWebServicesNodes extends JerseyTest { nodeInfo.getLong("lastHealthUpdate"), nodeInfo.getString("healthReport"), nodeInfo.getInt("numContainers"), nodeInfo.getLong("usedMemoryMB"), nodeInfo.getLong("availMemoryMB"), + nodeInfo.getLong("usedVirtualCores"), nodeInfo.getLong("availableVirtualCores"), nodeInfo.getString("version")); } @@ -678,7 +681,8 @@ public class TestRMWebServicesNodes extends JerseyTest { public void verifyNodeInfoGeneric(MockNM nm, String state, String rack, String id, String nodeHostName, String nodeHTTPAddress, long lastHealthUpdate, String healthReport, - int numContainers, long usedMemoryMB, long availMemoryMB, String version) + int numContainers, long usedMemoryMB, long availMemoryMB, long usedVirtualCores, + long availVirtualCores, String version) throws JSONException, Exception { RMNode node = rm.getRMContext().getRMNodes().get(nm.getNodeId()); @@ -712,6 +716,10 @@ public class TestRMWebServicesNodes extends JerseyTest { .getUsedResource().getMemory(), usedMemoryMB); assertEquals("availMemoryMB doesn't match: " + availMemoryMB, report .getAvailableResource().getMemory(), availMemoryMB); + assertEquals("usedVirtualCores doesn't match: " + usedVirtualCores, report + .getUsedResource().getVirtualCores(), usedVirtualCores); + assertEquals("availVirtualCores doesn't match: " + availVirtualCores, report + .getAvailableResource().getVirtualCores(), availVirtualCores); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/NodeManagerRest.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/NodeManagerRest.apt.vm index 36208f4e414..36b86217b33 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/NodeManagerRest.apt.vm +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/NodeManagerRest.apt.vm @@ -62,6 +62,8 @@ NodeManager REST API's. *---------------+--------------+-------------------------------+ | totalVmemAllocatedContainersMB | long | The amount of virtual memory allocated for use by containers in MB | *---------------+--------------+-------------------------------+ +| totalVCoresAllocatedContainers | long | The number of virtual cores allocated for use by containers | +*---------------+--------------+-------------------------------+ | lastNodeUpdateTime | long | The last timestamp at which the health report was received (in ms since epoch)| *---------------+--------------+-------------------------------+ | healthReport | string | The diagnostic health report of the node | @@ -109,6 +111,7 @@ NodeManager REST API's. "nodeManagerBuildVersion" : "0.23.1-SNAPSHOT from 1228355 by user1 source checksum 20647f76c36430e888cc7204826a445c", "lastNodeUpdateTime" : 1326222266126, "totalVmemAllocatedContainersMB" : 17203, + "totalVCoresAllocatedContainers" : 8, "nodeHealthy" : true, "healthReport" : "", "totalPmemAllocatedContainersMB" : 8192, @@ -148,6 +151,7 @@ NodeManager REST API's. 17203 8192 + 8 1326222386134 true 0.23.1-SNAPSHOT @@ -446,6 +450,7 @@ NodeManager REST API's. { "nodeId" : "host.domain.com:8041", "totalMemoryNeededMB" : 2048, + "totalVCoresNeeded" : 1, "state" : "RUNNING", "diagnostics" : "", "containerLogsLink" : "http://host.domain.com:8042/node/containerlogs/container_1326121700862_0006_01_000001/user1", @@ -456,6 +461,7 @@ NodeManager REST API's. { "nodeId" : "host.domain.com:8041", "totalMemoryNeededMB" : 2048, + "totalVCoresNeeded" : 2, "state" : "RUNNING", "diagnostics" : "", "containerLogsLink" : "http://host.domain.com:8042/node/containerlogs/container_1326121700862_0006_01_000003/user1", @@ -498,6 +504,7 @@ NodeManager REST API's. user1 2048 + 1 http://host.domain.com:8042/node/containerlogs/container_1326121700862_0006_01_000001/user1 host.domain.com:8041 @@ -508,6 +515,7 @@ NodeManager REST API's. Container killed by the ApplicationMaster. user1 2048 + 2 http://host.domain.com:8042/node/containerlogs/container_1326121700862_0006_01_000003/user1 host.domain.com:8041 @@ -560,6 +568,8 @@ NodeManager REST API's. *---------------+--------------+-------------------------------+ | totalMemoryNeededMB | long | Total amout of memory needed by the container (in MB) | *---------------+--------------+-------------------------------+ +| totalVCoresNeeded | long | Total number of virtual cores needed by the container | +*---------------+--------------+-------------------------------+ ** Response Examples @@ -587,6 +597,7 @@ NodeManager REST API's. "container" : { "nodeId" : "host.domain.com:8041", "totalMemoryNeededMB" : 2048, + "totalVCoresNeeded" : 1, "state" : "RUNNING", "diagnostics" : "", "containerLogsLink" : "http://host.domain.com:8042/node/containerlogs/container_1326121700862_0007_01_000001/user1", @@ -626,6 +637,7 @@ NodeManager REST API's. user1 2048 + 1 http://host.domain.com:8042/node/containerlogs/container_1326121700862_0007_01_000001/user1 host.domain.com:8041 diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm index 08de51ef7fd..f17b1df6e1a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm @@ -195,6 +195,14 @@ ResourceManager REST API's. *---------------+--------------+-------------------------------+ | totalMB | long | The amount of total memory in MB | *---------------+--------------+-------------------------------+ +| reservedVirtualCores | long | The number of reserved virtual cores | +*---------------+--------------+-------------------------------+ +| availableVirtualCores | long | The number of available virtual cores | +*---------------+--------------+-------------------------------+ +| allocatedVirtualCores | long | The number of allocated virtual cores | +*---------------+--------------+-------------------------------+ +| totalVirtualCores | long | The total number of virtual cores | +*---------------+--------------+-------------------------------+ | containersAllocated | int | The number of containers allocated | *---------------+--------------+-------------------------------+ | containersReserved | int | The number of containers reserved | @@ -247,10 +255,14 @@ ResourceManager REST API's. "reservedMB":0, "availableMB":17408, "allocatedMB":0, + "reservedVirtualCores":0, + "availableVirtualCores":7, + "allocatedVirtualCores":1, "containersAllocated":0, "containersReserved":0, "containersPending":0, "totalMB":17408, + "totalVirtualCores":8, "totalNodes":1, "lostNodes":0, "unhealthyNodes":0, @@ -293,10 +305,14 @@ ResourceManager REST API's. 0 17408 0 + 0 + 7 + 1 0 0 0 17408 + 8 1 0 0 @@ -2040,8 +2056,10 @@ Server: Jetty(6.1.26) "lastHealthUpdate":1324056895432, "healthReport":"Healthy", "numContainers":0, - "usedMemoryMB":0 - "availMemoryMB":8192 + "usedMemoryMB":0, + "availMemoryMB":8192, + "usedVirtualCores":0, + "availableVirtualCores":8 }, { "rack":"\/default-rack", @@ -2054,7 +2072,9 @@ Server: Jetty(6.1.26) "healthReport":"Healthy", "numContainers":0, "usedMemoryMB":0, - "availMemoryMB":8192 + "availMemoryMB":8192, + "usedVirtualCores":0, + "availableVirtualCores":8 } ] } @@ -2096,6 +2116,8 @@ Server: Jetty(6.1.26) 0 0 5120 + 0 + 8 /default-rack @@ -2109,6 +2131,8 @@ Server: Jetty(6.1.26) 0 0 5120 + 0 + 8 +---+ @@ -2159,10 +2183,14 @@ Server: Jetty(6.1.26) *---------------+--------------+-------------------------------+ | lastHealthUpdate | long | The last time the node reported its health (in ms since epoch)| *---------------+--------------+-------------------------------+ -| usedMemoryMB | long | The total about of memory currently used on the node (in MB)| +| usedMemoryMB | long | The total amount of memory currently used on the node (in MB)| *---------------+--------------+-------------------------------+ | availMemoryMB | long | The total amount of memory currently available on the node (in MB)| *---------------+--------------+-------------------------------+ +| usedVirtualCores | long | The total number of vCores currently used on the node | +*---------------+--------------+-------------------------------+ +| availableVirtualCores | long | The total number of vCores available on the node | +*---------------+--------------+-------------------------------+ | numContainers | int | The total number of containers currently running on the node| *---------------+--------------+-------------------------------+ @@ -2201,7 +2229,9 @@ Server: Jetty(6.1.26) "healthReport":"Healthy", "numContainers":0, "usedMemoryMB":0, - "availMemoryMB":5120 + "availMemoryMB":5120, + "usedVirtualCores":0, + "availableVirtualCores":8 } } +---+ @@ -2240,6 +2270,8 @@ Server: Jetty(6.1.26) 0 0 5120 + 0 + 5120 +---+