diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 3f58ef5b348..30ed0939a23 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -136,6 +136,10 @@ Release 2.5.0 - UNRELEASED YARN-2030. Augmented RMStateStore with state machine.(Binglin Chang via jianhe) + YARN-1424. RMAppAttemptImpl should return the + DummyApplicationResourceUsageReport for all invalid accesses. + (Ray Chiang via kasha) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java index 8de9ff3baf1..6e9c76fb012 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java @@ -47,7 +47,7 @@ public abstract class ApplicationResourceUsageReport { } /** - * Get the number of used containers + * Get the number of used containers. -1 for invalid/inaccessible reports. * @return the number of used containers */ @Public @@ -63,7 +63,7 @@ public abstract class ApplicationResourceUsageReport { public abstract void setNumUsedContainers(int num_containers); /** - * Get the number of reserved containers + * Get the number of reserved containers. -1 for invalid/inaccessible reports. * @return the number of reserved containers */ @Private @@ -79,7 +79,7 @@ public abstract class ApplicationResourceUsageReport { public abstract void setNumReservedContainers(int num_reserved_containers); /** - * Get the used Resource + * Get the used Resource. -1 for invalid/inaccessible reports. * @return the used Resource */ @Public @@ -91,7 +91,7 @@ public abstract class ApplicationResourceUsageReport { public abstract void setUsedResources(Resource resources); /** - * Get the reserved Resource + * Get the reserved Resource. -1 for invalid/inaccessible reports. * @return the reserved Resource */ @Public @@ -103,7 +103,7 @@ public abstract class ApplicationResourceUsageReport { public abstract void setReservedResources(Resource reserved_resources); /** - * Get the needed Resource + * Get the needed Resource. -1 for invalid/inaccessible reports. * @return the needed Resource */ @Public diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java index e884d29e303..d93c45d0d79 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java @@ -28,6 +28,7 @@ import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.Resource; @@ -43,6 +44,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils; +import org.apache.hadoop.yarn.server.utils.BuilderUtils; +import org.apache.hadoop.yarn.util.resource.Resources; /** * Utility methods to aid serving RM data through the REST and RPC APIs @@ -225,4 +228,13 @@ public class RMServerUtils { } } + /** + * Statically defined dummy ApplicationResourceUsageREport. Used as + * a return value when a valid report cannot be found. + */ + public static final ApplicationResourceUsageReport + DUMMY_APPLICATION_RESOURCE_USAGE_REPORT = + BuilderUtils.newApplicationResourceUsageReport(-1, -1, + Resources.createResource(-1, -1), Resources.createResource(-1, -1), + Resources.createResource(-1, -1)); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index bbd135b9901..3318f1582a0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -71,6 +71,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeCleanAppEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.state.InvalidStateTransitonException; import org.apache.hadoop.yarn.state.MultipleArcTransition; @@ -293,11 +294,6 @@ public class RMAppImpl implements RMApp, Recoverable { private final StateMachine stateMachine; - private static final ApplicationResourceUsageReport - DUMMY_APPLICATION_RESOURCE_USAGE_REPORT = - BuilderUtils.newApplicationResourceUsageReport(-1, -1, - Resources.createResource(-1, -1), Resources.createResource(-1, -1), - Resources.createResource(-1, -1)); private static final int DUMMY_APPLICATION_ATTEMPT_NUMBER = -1; public RMAppImpl(ApplicationId applicationId, RMContext rmContext, @@ -498,7 +494,7 @@ public class RMAppImpl implements RMApp, Recoverable { String origTrackingUrl = UNAVAILABLE; int rpcPort = -1; ApplicationResourceUsageReport appUsageReport = - DUMMY_APPLICATION_RESOURCE_USAGE_REPORT; + RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT; FinalApplicationStatus finishState = getFinalApplicationStatus(); String diags = UNAVAILABLE; float progress = 0.0f; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 5b1a17d930b..2a1170d41df 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -675,9 +675,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { ApplicationResourceUsageReport report = scheduler.getAppResourceUsageReport(this.getAppAttemptId()); if (report == null) { - Resource none = Resource.newInstance(0, 0); - report = ApplicationResourceUsageReport.newInstance(0, 0, none, none, - none); + report = RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT; } return report; } finally { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index 045b5b4ae94..4b1f59c3039 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -77,6 +77,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -258,6 +259,28 @@ public class TestClientRMService { } } + @Test + public void testGetApplicationResourceUsageReportDummy() throws YarnException, + IOException { + ApplicationAttemptId attemptId = getApplicationAttemptId(1); + YarnScheduler yarnScheduler = mockYarnScheduler(); + RMContext rmContext = mock(RMContext.class); + mockRMContext(yarnScheduler, rmContext); + when(rmContext.getDispatcher().getEventHandler()).thenReturn( + new EventHandler() { + public void handle(Event event) { + } + }); + ApplicationSubmissionContext asContext = + mock(ApplicationSubmissionContext.class); + YarnConfiguration config = new YarnConfiguration(); + RMAppAttemptImpl rmAppAttemptImpl = new RMAppAttemptImpl(attemptId, + rmContext, yarnScheduler, null, asContext, config, false); + ApplicationResourceUsageReport report = rmAppAttemptImpl + .getApplicationResourceUsageReport(); + assertEquals(report, RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT); + } + @Test public void testGetApplicationAttempts() throws YarnException, IOException { ClientRMService rmService = createRMService(); @@ -964,6 +987,8 @@ public class TestClientRMService { Arrays.asList(getApplicationAttemptId(101), getApplicationAttemptId(102))); when(yarnScheduler.getAppsInQueue(QUEUE_2)).thenReturn( Arrays.asList(getApplicationAttemptId(103))); + ApplicationAttemptId attemptId = getApplicationAttemptId(1); + when(yarnScheduler.getAppResourceUsageReport(attemptId)).thenReturn(null); return yarnScheduler; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java index e89b71b4105..0fd3c3c5c99 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java @@ -51,6 +51,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEvent; import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEventType; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; +import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils; import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.ApplicationState; @@ -921,6 +922,7 @@ public class TestRMAppTransitions { assertAppState(RMAppState.NEW, app); ApplicationReport report = app.createAndGetApplicationReport(null, true); Assert.assertNotNull(report.getApplicationResourceUsageReport()); + Assert.assertEquals(report.getApplicationResourceUsageReport(),RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT); report = app.createAndGetApplicationReport("clientuser", true); Assert.assertNotNull(report.getApplicationResourceUsageReport()); }