diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java index db3214f3fee..edb98a8bef2 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java @@ -35,6 +35,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerExitStatus; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -860,5 +861,11 @@ public class ResourceSchedulerWrapper implements ResourceScheduler, QueueACL acl, String queueName) { return scheduler.checkAccess(callerUGI, acl, queueName); } + + @Override + public ApplicationResourceUsageReport getAppResourceUsageReport( + ApplicationAttemptId appAttemptId) { + return scheduler.getAppResourceUsageReport(appAttemptId); + } } diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 8b1363f5b55..fd7a096c9d9 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -80,6 +80,8 @@ Release 2.3.0 - UNRELEASED YARN-1290. Let continuous scheduling achieve more balanced task assignment (Wei Yan via Sandy Ryza) + YARN-786. Expose application resource usage in RM REST API (Sandy Ryza) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index f741a6e6782..1247bb77d45 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -24,7 +24,6 @@ import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.EnumSet; import java.util.HashSet; @@ -54,7 +53,6 @@ import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Priority; -import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.factories.RecordFactory; @@ -84,9 +82,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAt import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptNewSavedEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptUnregistrationEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptUpdateSavedEvent; -import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent; @@ -98,7 +94,6 @@ import org.apache.hadoop.yarn.state.MultipleArcTransition; import org.apache.hadoop.yarn.state.SingleArcTransition; import org.apache.hadoop.yarn.state.StateMachine; import org.apache.hadoop.yarn.state.StateMachineFactory; -import org.apache.hadoop.yarn.util.resource.Resources; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; @SuppressWarnings({"unchecked", "rawtypes"}) @@ -673,38 +668,8 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { @Override public ApplicationResourceUsageReport getApplicationResourceUsageReport() { this.readLock.lock(); - try { - int numUsedContainers = 0; - int numReservedContainers = 0; - Resource currentConsumption = Resources.createResource(0, 0); - Resource reservedResources = Resources.createResource(0, 0); - - SchedulerAppReport schedApp = - scheduler.getSchedulerAppInfo(this.getAppAttemptId()); - Collection liveContainers; - Collection reservedContainers; - if (schedApp != null) { - liveContainers = schedApp.getLiveContainers(); - reservedContainers = schedApp.getReservedContainers(); - if (liveContainers != null) { - numUsedContainers = liveContainers.size(); - for (RMContainer lc : liveContainers) { - Resources.addTo(currentConsumption, lc.getContainer().getResource()); - } - } - if (reservedContainers != null) { - numReservedContainers = reservedContainers.size(); - for (RMContainer rc : reservedContainers) { - Resources.addTo(reservedResources, rc.getContainer().getResource()); - } - } - } - - return BuilderUtils.newApplicationResourceUsageReport( - numUsedContainers, numReservedContainers, - currentConsumption, reservedResources, - Resources.add(currentConsumption, reservedResources)); + return scheduler.getAppResourceUsageReport(this.getAppAttemptId()); } finally { this.readLock.unlock(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java index ade40c16c74..0fb8acbfbc1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java @@ -30,6 +30,7 @@ import org.apache.hadoop.classification.InterfaceStability.Stable; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.NodeId; @@ -397,5 +398,12 @@ public abstract class SchedulerApplication { lastScheduledContainer.put(priority, currentTimeMs); schedulingOpportunities.setCount(priority, 0); } + + public synchronized ApplicationResourceUsageReport getResourceUsageReport() { + return ApplicationResourceUsageReport.newInstance(liveContainers.size(), + reservedContainers.size(), Resources.clone(currentConsumption), + Resources.clone(currentReservation), + Resources.add(currentConsumption, currentReservation)); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java index 3ef8b9660ca..bdeaf74f6b5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java @@ -27,6 +27,7 @@ import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.classification.InterfaceStability.Stable; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.QueueACL; @@ -129,6 +130,16 @@ public interface YarnScheduler extends EventHandler { @Stable SchedulerAppReport getSchedulerAppInfo(ApplicationAttemptId appAttemptId); + /** + * Get a resource usage report from a given app attempt ID. + * @param appAttemptId the id of the application attempt + * @return resource usage report for this given attempt + */ + @LimitedPrivate("yarn") + @Evolving + ApplicationResourceUsageReport getAppResourceUsageReport( + ApplicationAttemptId appAttemptId); + /** * Get the root queue for the scheduler. * @return the root queue for the scheduler. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index a8a47c9b12c..241fe2f538a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -36,6 +36,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerStatus; @@ -856,6 +857,13 @@ public class CapacityScheduler return app == null ? null : new SchedulerAppReport(app); } + @Override + public ApplicationResourceUsageReport getAppResourceUsageReport( + ApplicationAttemptId applicationAttemptId) { + FiCaSchedulerApp app = getApplication(applicationAttemptId); + return app == null ? null : app.getResourceUsageReport(); + } + @Lock(Lock.NoLock.class) FiCaSchedulerNode getNode(NodeId nodeId) { return nodes.get(nodeId); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index 2efc57e511f..71c6cbd236e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -38,6 +38,7 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerStatus; @@ -1051,6 +1052,17 @@ public class FairScheduler implements ResourceScheduler { return new SchedulerAppReport(applications.get(appAttemptId)); } + @Override + public ApplicationResourceUsageReport getAppResourceUsageReport( + ApplicationAttemptId appAttemptId) { + FSSchedulerApp app = applications.get(appAttemptId); + if (app == null) { + LOG.error("Request for appInfo of unknown attempt" + appAttemptId); + return null; + } + return app.getResourceUsageReport(); + } + /** * Subqueue metrics might be a little out of date because fair shares are * recalculated at the update interval, but the root queue metrics needs to diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java index a2171fb8b57..626c482e219 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java @@ -37,6 +37,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerStatus; @@ -327,6 +328,13 @@ public class FifoScheduler implements ResourceScheduler, Configurable { return app == null ? null : new SchedulerAppReport(app); } + @Override + public ApplicationResourceUsageReport getAppResourceUsageReport( + ApplicationAttemptId applicationAttemptId) { + FiCaSchedulerApp app = getApplication(applicationAttemptId); + return app == null ? null : app.getResourceUsageReport(); + } + private FiCaSchedulerNode getNode(NodeId nodeId) { return nodes.get(nodeId); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java index 21f02714715..0a408df5435 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java @@ -26,8 +26,10 @@ import javax.xml.bind.annotation.XmlTransient; import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; @@ -71,6 +73,9 @@ public class AppInfo { protected long elapsedTime; protected String amContainerLogs; protected String amHostHttpAddress; + protected int allocatedMB; + protected int allocatedVCores; + protected int runningContainers; public AppInfo() { } // JAXB needs this @@ -132,6 +137,15 @@ public class AppInfo { this.amContainerLogs = url; this.amHostHttpAddress = masterContainer.getNodeHttpAddress(); } + + ApplicationResourceUsageReport resourceReport = attempt + .getApplicationResourceUsageReport(); + if (resourceReport != null) { + Resource usedResources = resourceReport.getUsedResources(); + allocatedMB = usedResources.getMemory(); + allocatedVCores = usedResources.getVirtualCores(); + runningContainers = resourceReport.getNumUsedContainers(); + } } } } @@ -224,5 +238,17 @@ public class AppInfo { public String getApplicationType() { return this.applicationType; } - + + public int getRunningContainers() { + return this.runningContainers; + } + + public int getAllocatedMB() { + return this.allocatedMB; + } + + public int getAllocatedVCores() { + return this.allocatedVCores; + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index 6fe6a14be1d..6d1d30d84a3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -78,6 +78,8 @@ import com.sun.jersey.test.framework.WebAppDescriptor; public class TestRMWebServicesApps extends JerseyTest { private static MockRM rm; + + private static final int CONTAINER_MB = 1024; private Injector injector = Guice.createInjector(new ServletModule() { @Override @@ -126,7 +128,7 @@ public class TestRMWebServicesApps extends JerseyTest { public void testApps() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(1024); + RMApp app1 = rm.submitApp(CONTAINER_MB); amNodeManager.nodeHeartbeat(true); testAppsHelper("apps", app1, MediaType.APPLICATION_JSON); rm.stop(); @@ -136,7 +138,7 @@ public class TestRMWebServicesApps extends JerseyTest { public void testAppsSlash() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(1024); + RMApp app1 = rm.submitApp(CONTAINER_MB); amNodeManager.nodeHeartbeat(true); testAppsHelper("apps/", app1, MediaType.APPLICATION_JSON); rm.stop(); @@ -146,7 +148,7 @@ public class TestRMWebServicesApps extends JerseyTest { public void testAppsDefault() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(1024); + RMApp app1 = rm.submitApp(CONTAINER_MB); amNodeManager.nodeHeartbeat(true); testAppsHelper("apps/", app1, ""); rm.stop(); @@ -156,7 +158,7 @@ public class TestRMWebServicesApps extends JerseyTest { public void testAppsXML() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(1024, "testwordcount", "user1"); + RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("cluster") @@ -181,7 +183,7 @@ public class TestRMWebServicesApps extends JerseyTest { public void testAppsXMLMulti() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - rm.submitApp(1024, "testwordcount", "user1"); + rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); rm.submitApp(2048, "testwordcount2", "user1"); amNodeManager.nodeHeartbeat(true); @@ -225,7 +227,7 @@ public class TestRMWebServicesApps extends JerseyTest { public void testAppsQueryState() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(1024); + RMApp app1 = rm.submitApp(CONTAINER_MB); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); @@ -248,8 +250,8 @@ public class TestRMWebServicesApps extends JerseyTest { public void testAppsQueryStates() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - rm.submitApp(1024); - RMApp killedApp = rm.submitApp(1024); + rm.submitApp(CONTAINER_MB); + RMApp killedApp = rm.submitApp(CONTAINER_MB); rm.killApp(killedApp.getApplicationId()); amNodeManager.nodeHeartbeat(true); @@ -297,8 +299,8 @@ public class TestRMWebServicesApps extends JerseyTest { public void testAppsQueryStatesComma() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - rm.submitApp(1024); - RMApp killedApp = rm.submitApp(1024); + rm.submitApp(CONTAINER_MB); + RMApp killedApp = rm.submitApp(CONTAINER_MB); rm.killApp(killedApp.getApplicationId()); amNodeManager.nodeHeartbeat(true); @@ -346,7 +348,7 @@ public class TestRMWebServicesApps extends JerseyTest { public void testAppsQueryStatesNone() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - rm.submitApp(1024); + rm.submitApp(CONTAINER_MB); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); @@ -365,7 +367,7 @@ public class TestRMWebServicesApps extends JerseyTest { public void testAppsQueryStateNone() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - rm.submitApp(1024); + rm.submitApp(CONTAINER_MB); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); @@ -384,7 +386,7 @@ public class TestRMWebServicesApps extends JerseyTest { public void testAppsQueryStatesInvalid() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - rm.submitApp(1024); + rm.submitApp(CONTAINER_MB); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); @@ -421,7 +423,7 @@ public class TestRMWebServicesApps extends JerseyTest { public void testAppsQueryStateInvalid() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - rm.submitApp(1024); + rm.submitApp(CONTAINER_MB); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); @@ -458,7 +460,7 @@ public class TestRMWebServicesApps extends JerseyTest { public void testAppsQueryFinalStatus() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(1024); + RMApp app1 = rm.submitApp(CONTAINER_MB); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); @@ -481,7 +483,7 @@ public class TestRMWebServicesApps extends JerseyTest { public void testAppsQueryFinalStatusNone() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - rm.submitApp(1024); + rm.submitApp(CONTAINER_MB); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); @@ -499,7 +501,7 @@ public class TestRMWebServicesApps extends JerseyTest { public void testAppsQueryFinalStatusInvalid() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - rm.submitApp(1024); + rm.submitApp(CONTAINER_MB); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); @@ -537,8 +539,8 @@ public class TestRMWebServicesApps extends JerseyTest { public void testAppsQueryUser() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - rm.submitApp(1024); - rm.submitApp(1024); + rm.submitApp(CONTAINER_MB); + rm.submitApp(CONTAINER_MB); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); @@ -565,8 +567,8 @@ public class TestRMWebServicesApps extends JerseyTest { public void testAppsQueryQueue() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - rm.submitApp(1024); - rm.submitApp(1024); + rm.submitApp(CONTAINER_MB); + rm.submitApp(CONTAINER_MB); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); @@ -588,9 +590,9 @@ public class TestRMWebServicesApps extends JerseyTest { public void testAppsQueryLimit() throws JSONException, Exception { rm.start(); rm.registerNode("127.0.0.1:1234", 2048); - rm.submitApp(1024); - rm.submitApp(1024); - rm.submitApp(1024); + rm.submitApp(CONTAINER_MB); + rm.submitApp(CONTAINER_MB); + rm.submitApp(CONTAINER_MB); WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("cluster") .path("apps").queryParam("limit", "2") @@ -611,9 +613,9 @@ public class TestRMWebServicesApps extends JerseyTest { long start = System.currentTimeMillis(); Thread.sleep(1); rm.registerNode("127.0.0.1:1234", 2048); - rm.submitApp(1024); - rm.submitApp(1024); - rm.submitApp(1024); + rm.submitApp(CONTAINER_MB); + rm.submitApp(CONTAINER_MB); + rm.submitApp(CONTAINER_MB); WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("cluster") .path("apps").queryParam("startedTimeBegin", String.valueOf(start)) @@ -632,11 +634,11 @@ public class TestRMWebServicesApps extends JerseyTest { public void testAppsQueryStartBeginSome() throws JSONException, Exception { rm.start(); rm.registerNode("127.0.0.1:1234", 2048); - rm.submitApp(1024); - rm.submitApp(1024); + rm.submitApp(CONTAINER_MB); + rm.submitApp(CONTAINER_MB); long start = System.currentTimeMillis(); Thread.sleep(1); - rm.submitApp(1024); + rm.submitApp(CONTAINER_MB); WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("cluster") .path("apps").queryParam("startedTimeBegin", String.valueOf(start)) @@ -657,9 +659,9 @@ public class TestRMWebServicesApps extends JerseyTest { rm.registerNode("127.0.0.1:1234", 2048); long end = System.currentTimeMillis(); Thread.sleep(1); - rm.submitApp(1024); - rm.submitApp(1024); - rm.submitApp(1024); + rm.submitApp(CONTAINER_MB); + rm.submitApp(CONTAINER_MB); + rm.submitApp(CONTAINER_MB); WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("cluster") .path("apps").queryParam("startedTimeEnd", String.valueOf(end)) @@ -677,11 +679,11 @@ public class TestRMWebServicesApps extends JerseyTest { rm.registerNode("127.0.0.1:1234", 2048); long start = System.currentTimeMillis(); Thread.sleep(1); - rm.submitApp(1024); - rm.submitApp(1024); + rm.submitApp(CONTAINER_MB); + rm.submitApp(CONTAINER_MB); long end = System.currentTimeMillis(); Thread.sleep(1); - rm.submitApp(1024); + rm.submitApp(CONTAINER_MB); WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("cluster") .path("apps").queryParam("startedTimeBegin", String.valueOf(start)) @@ -703,7 +705,7 @@ public class TestRMWebServicesApps extends JerseyTest { MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); long start = System.currentTimeMillis(); Thread.sleep(1); - RMApp app1 = rm.submitApp(1024); + RMApp app1 = rm.submitApp(CONTAINER_MB); amNodeManager.nodeHeartbeat(true); // finish App MockAM am = rm @@ -712,8 +714,8 @@ public class TestRMWebServicesApps extends JerseyTest { am.unregisterAppAttempt(); amNodeManager.nodeHeartbeat(app1.getCurrentAppAttempt().getAppAttemptId(), 1, ContainerState.COMPLETE); - rm.submitApp(1024); - rm.submitApp(1024); + rm.submitApp(CONTAINER_MB); + rm.submitApp(CONTAINER_MB); WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("cluster") @@ -733,7 +735,7 @@ public class TestRMWebServicesApps extends JerseyTest { public void testAppsQueryFinishEnd() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(1024); + RMApp app1 = rm.submitApp(CONTAINER_MB); amNodeManager.nodeHeartbeat(true); // finish App MockAM am = rm @@ -743,8 +745,8 @@ public class TestRMWebServicesApps extends JerseyTest { amNodeManager.nodeHeartbeat(app1.getCurrentAppAttempt().getAppAttemptId(), 1, ContainerState.COMPLETE); - rm.submitApp(1024); - rm.submitApp(1024); + rm.submitApp(CONTAINER_MB); + rm.submitApp(CONTAINER_MB); long end = System.currentTimeMillis(); WebResource r = resource(); @@ -767,7 +769,7 @@ public class TestRMWebServicesApps extends JerseyTest { MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); long start = System.currentTimeMillis(); Thread.sleep(1); - RMApp app1 = rm.submitApp(1024); + RMApp app1 = rm.submitApp(CONTAINER_MB); amNodeManager.nodeHeartbeat(true); // finish App MockAM am = rm @@ -777,8 +779,8 @@ public class TestRMWebServicesApps extends JerseyTest { amNodeManager.nodeHeartbeat(app1.getCurrentAppAttempt().getAppAttemptId(), 1, ContainerState.COMPLETE); - rm.submitApp(1024); - rm.submitApp(1024); + rm.submitApp(CONTAINER_MB); + rm.submitApp(CONTAINER_MB); long end = System.currentTimeMillis(); WebResource r = resource(); @@ -801,7 +803,7 @@ public class TestRMWebServicesApps extends JerseyTest { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); Thread.sleep(1); - RMApp app1 = rm.submitApp(1024); + RMApp app1 = rm.submitApp(CONTAINER_MB); amNodeManager.nodeHeartbeat(true); // finish App MockAM am = rm @@ -811,9 +813,9 @@ public class TestRMWebServicesApps extends JerseyTest { amNodeManager.nodeHeartbeat(app1.getCurrentAppAttempt().getAppAttemptId(), 1, ContainerState.COMPLETE); - rm.submitApp(1024, "", UserGroupInformation.getCurrentUser() + rm.submitApp(CONTAINER_MB, "", UserGroupInformation.getCurrentUser() .getShortUserName(), null, false, null, 2, null, "MAPREDUCE"); - rm.submitApp(1024, "", UserGroupInformation.getCurrentUser() + rm.submitApp(CONTAINER_MB, "", UserGroupInformation.getCurrentUser() .getShortUserName(), null, false, null, 2, null, "NON-YARN"); WebResource r = resource(); @@ -987,7 +989,7 @@ public class TestRMWebServicesApps extends JerseyTest { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 4096); Thread.sleep(1); - RMApp app1 = rm.submitApp(1024, "", UserGroupInformation.getCurrentUser() + RMApp app1 = rm.submitApp(CONTAINER_MB, "", UserGroupInformation.getCurrentUser() .getShortUserName(), null, false, null, 2, null, "MAPREDUCE"); amNodeManager.nodeHeartbeat(true); // finish App @@ -998,9 +1000,9 @@ public class TestRMWebServicesApps extends JerseyTest { amNodeManager.nodeHeartbeat(app1.getCurrentAppAttempt().getAppAttemptId(), 1, ContainerState.COMPLETE); - rm.submitApp(1024, "", UserGroupInformation.getCurrentUser() + rm.submitApp(CONTAINER_MB, "", UserGroupInformation.getCurrentUser() .getShortUserName(), null, false, null, 2, null, "MAPREDUCE"); - rm.submitApp(1024, "", UserGroupInformation.getCurrentUser() + rm.submitApp(CONTAINER_MB, "", UserGroupInformation.getCurrentUser() .getShortUserName(), null, false, null, 2, null, "OTHER"); // zero type, zero state @@ -1148,7 +1150,7 @@ public class TestRMWebServicesApps extends JerseyTest { public void testSingleApp() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(1024, "testwordcount", "user1"); + RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); amNodeManager.nodeHeartbeat(true); testSingleAppsHelper(app1.getApplicationId().toString(), app1, MediaType.APPLICATION_JSON); @@ -1159,7 +1161,7 @@ public class TestRMWebServicesApps extends JerseyTest { public void testSingleAppsSlash() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(1024); + RMApp app1 = rm.submitApp(CONTAINER_MB); amNodeManager.nodeHeartbeat(true); testSingleAppsHelper(app1.getApplicationId().toString() + "/", app1, MediaType.APPLICATION_JSON); @@ -1170,7 +1172,7 @@ public class TestRMWebServicesApps extends JerseyTest { public void testSingleAppsDefault() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(1024); + RMApp app1 = rm.submitApp(CONTAINER_MB); amNodeManager.nodeHeartbeat(true); testSingleAppsHelper(app1.getApplicationId().toString() + "/", app1, ""); rm.stop(); @@ -1180,7 +1182,7 @@ public class TestRMWebServicesApps extends JerseyTest { public void testInvalidApp() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - rm.submitApp(1024); + rm.submitApp(CONTAINER_MB); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); @@ -1216,7 +1218,7 @@ public class TestRMWebServicesApps extends JerseyTest { public void testNonexistApp() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - rm.submitApp(1024, "testwordcount", "user1"); + rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); @@ -1265,7 +1267,7 @@ public class TestRMWebServicesApps extends JerseyTest { public void testSingleAppsXML() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(1024, "testwordcount", "user1"); + RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("cluster") @@ -1307,7 +1309,10 @@ public class TestRMWebServicesApps extends JerseyTest { WebServicesTestUtils.getXmlLong(element, "finishedTime"), WebServicesTestUtils.getXmlLong(element, "elapsedTime"), WebServicesTestUtils.getXmlString(element, "amHostHttpAddress"), - WebServicesTestUtils.getXmlString(element, "amContainerLogs")); + WebServicesTestUtils.getXmlString(element, "amContainerLogs"), + WebServicesTestUtils.getXmlInt(element, "allocatedMB"), + WebServicesTestUtils.getXmlInt(element, "allocatedVCores"), + WebServicesTestUtils.getXmlInt(element, "runningContainers")); } } @@ -1315,7 +1320,7 @@ public class TestRMWebServicesApps extends JerseyTest { Exception { // 15 because trackingUrl not assigned yet - assertEquals("incorrect number of elements", 16, info.length()); + assertEquals("incorrect number of elements", 19, info.length()); verifyAppInfoGeneric(app, info.getString("id"), info.getString("user"), info.getString("name"), info.getString("applicationType"), info.getString("queue"), @@ -1324,14 +1329,16 @@ public class TestRMWebServicesApps extends JerseyTest { info.getString("diagnostics"), info.getLong("clusterId"), info.getLong("startedTime"), info.getLong("finishedTime"), info.getLong("elapsedTime"), info.getString("amHostHttpAddress"), - info.getString("amContainerLogs")); + info.getString("amContainerLogs"), info.getInt("allocatedMB"), + info.getInt("allocatedVCores"), info.getInt("runningContainers")); } public void verifyAppInfoGeneric(RMApp app, String id, String user, String name, String applicationType, String queue, String state, String finalStatus, float progress, String trackingUI, String diagnostics, long clusterId, long startedTime, long finishedTime, long elapsedTime, - String amHostHttpAddress, String amContainerLogs) throws JSONException, + String amHostHttpAddress, String amContainerLogs, int allocatedMB, + int allocatedVCores, int numContainers) throws JSONException, Exception { WebServicesTestUtils.checkStringMatch("id", app.getApplicationId() @@ -1363,13 +1370,16 @@ public class TestRMWebServicesApps extends JerseyTest { amContainerLogs.startsWith("http://")); assertTrue("amContainerLogs doesn't contain user info", amContainerLogs.endsWith("/" + app.getUser())); + assertEquals("allocatedMB doesn't match", 1024, allocatedMB); + assertEquals("allocatedVCores doesn't match", 1, allocatedVCores); + assertEquals("numContainers doesn't match", 1, numContainers); } @Test public void testAppAttempts() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(1024, "testwordcount", "user1"); + RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); amNodeManager.nodeHeartbeat(true); testAppAttemptsHelper(app1.getApplicationId().toString(), app1, MediaType.APPLICATION_JSON); @@ -1380,7 +1390,7 @@ public class TestRMWebServicesApps extends JerseyTest { public void testMultipleAppAttempts() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(1024, "testwordcount", "user1"); + RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); amNodeManager.nodeHeartbeat(true); int maxAppAttempts = rm.getConfig().getInt( YarnConfiguration.RM_AM_MAX_ATTEMPTS, @@ -1406,7 +1416,7 @@ public class TestRMWebServicesApps extends JerseyTest { public void testAppAttemptsSlash() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(1024); + RMApp app1 = rm.submitApp(CONTAINER_MB); amNodeManager.nodeHeartbeat(true); testAppAttemptsHelper(app1.getApplicationId().toString() + "/", app1, MediaType.APPLICATION_JSON); @@ -1417,7 +1427,7 @@ public class TestRMWebServicesApps extends JerseyTest { public void testAppAttemtpsDefault() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(1024); + RMApp app1 = rm.submitApp(CONTAINER_MB); amNodeManager.nodeHeartbeat(true); testAppAttemptsHelper(app1.getApplicationId().toString() + "/", app1, ""); rm.stop(); @@ -1427,7 +1437,7 @@ public class TestRMWebServicesApps extends JerseyTest { public void testInvalidAppAttempts() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - rm.submitApp(1024); + rm.submitApp(CONTAINER_MB); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); @@ -1463,7 +1473,7 @@ public class TestRMWebServicesApps extends JerseyTest { public void testNonexistAppAttempts() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - rm.submitApp(1024, "testwordcount", "user1"); + rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); @@ -1526,7 +1536,7 @@ public class TestRMWebServicesApps extends JerseyTest { rm.start(); String user = "user1"; MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); - RMApp app1 = rm.submitApp(1024, "testwordcount", user); + RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", user); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("cluster") diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm index b3eb94f21be..fff16541edf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm @@ -1177,7 +1177,10 @@ ResourceManager REST API's. "elapsedTime" : 25196, "diagnostics" : "", "trackingUrl" : "http://host.domain.com:8088/proxy/application_1326815542473_0001/jobhistory/job/job_1326815542473_1_1", - "queue" : "default" + "queue" : "default", + "allocatedMB" : 0, + "allocatedVCores" : 0, + "runningContainers" : 0 }, { "finishedTime" : 1326815789546, @@ -1195,7 +1198,10 @@ ResourceManager REST API's. "elapsedTime" : 148166, "diagnostics" : "", "trackingUrl" : "http://host.domain.com:8088/proxy/application_1326815542473_0002/jobhistory/job/job_1326815542473_2_2", - "queue" : "default" + "queue" : "default", + "allocatedMB" : 0, + "allocatedVCores" : 0, + "runningContainers" : 1 } ] } @@ -1245,6 +1251,9 @@ ResourceManager REST API's. http://host.domain.com:8042/node/containerlogs/container_1326815542473_0001 _01_000001 host.domain.com:8042 + 0 + 0 + 0 application_1326815542473_0002 @@ -1264,6 +1273,9 @@ _01_000001 148166 http://host.domain.com:8042/node/containerlogs/container_1326815542473_0002_01_000001 host.domain.com:8042 + 0 + 0 + 0 @@ -1457,6 +1469,12 @@ _01_000001 *---------------+--------------+--------------------------------+ | amHostHttpAddress | string | The nodes http address of the application master | *---------------+--------------+--------------------------------+ +| allocatedMB | int | The sum of memory in MB allocated to the application's running containers | +*---------------------------------------------------------------+ +| allocatedVCores | int | The sum of virtual cores allocated to the application's running containers | ++---------------------------------------------------------------+ +| runningContainers | int | The number of containers currently running for the application | ++---------------------------------------------------------------+ ** Response Examples