From e5b56e2af6b48cbc8501f624ae4b4f9d13eccacf Mon Sep 17 00:00:00 2001 From: Zhijie Shen Date: Tue, 3 Feb 2015 17:33:53 -0800 Subject: [PATCH] YARN-2543. Made resource usage be published to the timeline server too. Contributed by Naganarasimha G R. --- hadoop-yarn-project/CHANGES.txt | 3 +++ ...pplicationHistoryManagerOnTimelineStore.java | 12 +++++++++++- .../TestApplicationHistoryClientService.java | 4 ++++ ...pplicationHistoryManagerOnTimelineStore.java | 9 +++++++++ .../metrics/ApplicationMetricsConstants.java | 6 ++++++ .../metrics/ApplicationFinishedEvent.java | 9 ++++++++- .../metrics/SystemMetricsPublisher.java | 10 +++++++++- .../metrics/TestSystemMetricsPublisher.java | 17 +++++++++++++++-- 8 files changed, 65 insertions(+), 5 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 30f5a1e360a..33e98d1b5ed 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -470,6 +470,9 @@ Release 2.7.0 - UNRELEASED YARN-3056. Add verification for containerLaunchDuration in TestNodeManagerMetrics. (zhihai xu via xgong) + YARN-2543. Made resource usage be published to the timeline server too. + (Naganarasimha G R via zjshen) + Release 2.6.0 - 2014-11-18 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java index c88ccccf727..22418a8535c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java @@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.ContainerExitStatus; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerReport; @@ -222,6 +223,7 @@ private static ApplicationReportExt convertToApplicationReport( String diagnosticsInfo = null; FinalApplicationStatus finalStatus = FinalApplicationStatus.UNDEFINED; YarnApplicationState state = null; + ApplicationResourceUsageReport appResources = null; Map appViewACLs = new HashMap(); Map entityInfo = entity.getOtherInfo(); @@ -260,6 +262,14 @@ private static ApplicationReportExt convertToApplicationReport( entityInfo.get(ApplicationMetricsConstants.TYPE_ENTITY_INFO) .toString(); } + if (entityInfo.containsKey(ApplicationMetricsConstants.APP_CPU_METRICS)) { + long vcoreSeconds=Long.parseLong(entityInfo.get( + ApplicationMetricsConstants.APP_CPU_METRICS).toString()); + long memorySeconds=Long.parseLong(entityInfo.get( + ApplicationMetricsConstants.APP_MEM_METRICS).toString()); + appResources=ApplicationResourceUsageReport + .newInstance(0, 0, null, null, null, memorySeconds, vcoreSeconds); + } } List events = entity.getEvents(); if (events != null) { @@ -310,7 +320,7 @@ private static ApplicationReportExt convertToApplicationReport( return new ApplicationReportExt(ApplicationReport.newInstance( ConverterUtils.toApplicationId(entity.getEntityId()), latestApplicationAttemptId, user, queue, name, null, -1, null, state, - diagnosticsInfo, null, createdTime, finishedTime, finalStatus, null, + diagnosticsInfo, null, createdTime, finishedTime, finalStatus, appResources, null, 1.0F, type, null), appViewACLs); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java index 2a38d0a4bd7..32d011eaa68 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java @@ -80,6 +80,10 @@ public void testApplicationReport() throws IOException, YarnException { clientService.getClientHandler().getApplicationReport(request); ApplicationReport appReport = response.getApplicationReport(); Assert.assertNotNull(appReport); + Assert.assertEquals(123, appReport.getApplicationResourceUsageReport() + .getMemorySeconds()); + Assert.assertEquals(345, appReport.getApplicationResourceUsageReport() + .getVcoreSeconds()); Assert.assertEquals("application_0_0001", appReport.getApplicationId() .toString()); Assert.assertEquals("test app type", diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java index a093f19f9ea..50a15f1bfd3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java @@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerReport; import org.apache.hadoop.yarn.api.records.ContainerState; @@ -201,6 +202,12 @@ public ApplicationReport run() throws Exception { app.getOriginalTrackingUrl()); Assert.assertEquals("test diagnostics info", app.getDiagnostics()); } + ApplicationResourceUsageReport applicationResourceUsageReport = + app.getApplicationResourceUsageReport(); + Assert.assertEquals(123, + applicationResourceUsageReport.getMemorySeconds()); + Assert + .assertEquals(345, applicationResourceUsageReport.getVcoreSeconds()); Assert.assertEquals(FinalApplicationStatus.UNDEFINED, app.getFinalApplicationStatus()); Assert.assertEquals(YarnApplicationState.FINISHED, @@ -416,6 +423,8 @@ private static TimelineEntity createApplicationTimelineEntity( entityInfo.put(ApplicationMetricsConstants.QUEUE_ENTITY_INFO, "test queue"); entityInfo.put(ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO, Integer.MAX_VALUE + 1L); + entityInfo.put(ApplicationMetricsConstants.APP_MEM_METRICS,123); + entityInfo.put(ApplicationMetricsConstants.APP_CPU_METRICS,345); if (emptyACLs) { entityInfo.put(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO, ""); } else { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java index ee34c495af2..df8eecb40b2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java @@ -63,6 +63,12 @@ public class ApplicationMetricsConstants { public static final String STATE_EVENT_INFO = "YARN_APPLICATION_STATE"; + + public static final String APP_CPU_METRICS = + "YARN_APPLICATION_CPU_METRIC"; + + public static final String APP_MEM_METRICS = + "YARN_APPLICATION_MEM_METRIC"; public static final String LATEST_APP_ATTEMPT_EVENT_INFO = "YARN_APPLICATION_LATEST_APP_ATTEMPT"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationFinishedEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationFinishedEvent.java index c457710db6b..8d75f92e426 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationFinishedEvent.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationFinishedEvent.java @@ -22,6 +22,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics; public class ApplicationFinishedEvent extends SystemMetricsEvent { @@ -31,6 +32,7 @@ public class ApplicationFinishedEvent extends private FinalApplicationStatus appStatus; private YarnApplicationState state; private ApplicationAttemptId latestAppAttemptId; + private RMAppMetrics appMetrics; public ApplicationFinishedEvent( ApplicationId appId, @@ -38,13 +40,15 @@ public ApplicationFinishedEvent( FinalApplicationStatus appStatus, YarnApplicationState state, ApplicationAttemptId latestAppAttemptId, - long finishedTime) { + long finishedTime, + RMAppMetrics appMetrics) { super(SystemMetricsEventType.APP_FINISHED, finishedTime); this.appId = appId; this.diagnosticsInfo = diagnosticsInfo; this.appStatus = appStatus; this.latestAppAttemptId = latestAppAttemptId; this.state = state; + this.appMetrics=appMetrics; } @Override @@ -72,4 +76,7 @@ public ApplicationAttemptId getLatestApplicationAttemptId() { return latestAppAttemptId; } + public RMAppMetrics getAppMetrics() { + return appMetrics; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java index d8e7ef6f202..3adf519a6c3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java @@ -45,6 +45,7 @@ import org.apache.hadoop.yarn.server.metrics.ContainerMetricsConstants; import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; @@ -120,7 +121,8 @@ public void appFinished(RMApp app, RMAppState state, long finishedTime) { RMServerUtils.createApplicationState(state), app.getCurrentAppAttempt() == null ? null : app.getCurrentAppAttempt().getAppAttemptId(), - finishedTime)); + finishedTime, + app.getRMAppMetrics())); } } @@ -276,6 +278,12 @@ private void publishApplicationFinishedEvent(ApplicationFinishedEvent event) { eventInfo.put(ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO, event.getLatestApplicationAttemptId().toString()); } + RMAppMetrics appMetrics = event.getAppMetrics(); + entity.addOtherInfo(ApplicationMetricsConstants.APP_CPU_METRICS, + appMetrics.getVcoreSeconds()); + entity.addOtherInfo(ApplicationMetricsConstants.APP_MEM_METRICS, + appMetrics.getMemorySeconds()); + tEvent.setEventInfo(eventInfo); entity.addEvent(tEvent); putEntity(entity); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java index 946d526142a..9f02721399a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java @@ -43,6 +43,7 @@ import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants; import org.apache.hadoop.yarn.server.metrics.ContainerMetricsConstants; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; @@ -148,8 +149,18 @@ public void testPublishApplicationMetrics() throws Exception { entity.getOtherInfo().get( ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO)); } else { - Assert.assertEquals("", entity.getOtherInfo().get( - ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO)); + Assert.assertEquals( + "", + entity.getOtherInfo().get( + ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO)); + Assert.assertEquals( + app.getRMAppMetrics().getMemorySeconds(), + Long.parseLong(entity.getOtherInfo() + .get(ApplicationMetricsConstants.APP_MEM_METRICS).toString())); + Assert.assertEquals( + app.getRMAppMetrics().getVcoreSeconds(), + Long.parseLong(entity.getOtherInfo() + .get(ApplicationMetricsConstants.APP_CPU_METRICS).toString())); } boolean hasCreatedEvent = false; boolean hasFinishedEvent = false; @@ -340,6 +351,8 @@ private static RMApp createRMApp(ApplicationId appId) { when(app.getCurrentAppAttempt()).thenReturn(appAttempt); when(app.getFinalApplicationStatus()).thenReturn( FinalApplicationStatus.UNDEFINED); + when(app.getRMAppMetrics()).thenReturn( + new RMAppMetrics(null, 0, 0, Integer.MAX_VALUE, Long.MAX_VALUE)); return app; }