From 6984d899e35ecf2a32f4d50ff2aa5356b043bc02 Mon Sep 17 00:00:00 2001 From: Jason Lowe Date: Wed, 12 Nov 2014 17:01:15 +0000 Subject: [PATCH] YARN-2780. Log aggregated resource allocation in rm-appsummary.log. Contributed by Eric Payne (cherry picked from commit f8aefa5e9c8c6d2817205b5ed8d914db31f56ae7) --- hadoop-yarn-project/CHANGES.txt | 3 +++ .../yarn/server/resourcemanager/RMAppManager.java | 9 ++++++++- .../yarn/server/resourcemanager/TestAppManager.java | 12 +++++++++++- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 51413ca2fa4..f9a00fafafd 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -41,6 +41,9 @@ Release 2.7.0 - UNRELEASED YARN-570. Time strings are formated in different timezone. (Akira Ajisaka and Peng Zhang via kasha) + YARN-2780. Log aggregated resource allocation in rm-appsummary.log (Eric + Payne via jlowe) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index 02c6d2f0036..ab8df62947c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppRecoverEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppRejectedEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; @@ -155,6 +156,7 @@ public static SummaryBuilder createAppSummary(RMApp app) { trackingUrl = attempt.getTrackingUrl(); host = attempt.getHost(); } + RMAppMetrics metrics = app.getRMAppMetrics(); SummaryBuilder summary = new SummaryBuilder() .add("appId", app.getApplicationId()) .add("name", app.getName()) @@ -165,7 +167,12 @@ public static SummaryBuilder createAppSummary(RMApp app) { .add("appMasterHost", host) .add("startTime", app.getStartTime()) .add("finishTime", app.getFinishTime()) - .add("finalStatus", app.getFinalApplicationStatus()); + .add("finalStatus", app.getFinalApplicationStatus()) + .add("memorySeconds", metrics.getMemorySeconds()) + .add("vcoreSeconds", metrics.getVcoreSeconds()) + .add("preemptedAMContainers", metrics.getNumAMContainersPreempted()) + .add("preemptedNonAMContainers", metrics.getNumNonAMContainersPreempted()) + .add("preemptedResources", metrics.getResourcePreempted()); return summary; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java index e1466114c02..6b3eea20ff2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java @@ -59,6 +59,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AMLivelinessMonitor; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer; @@ -572,6 +573,10 @@ public void testEscapeApplicationSummary() { when(app.getQueue()).thenReturn("Multiline\n\n\r\rQueueName"); when(app.getState()).thenReturn(RMAppState.RUNNING); + RMAppMetrics metrics = + new RMAppMetrics(Resource.newInstance(1234, 56), 10, 1, 16384, 64); + when(app.getRMAppMetrics()).thenReturn(metrics); + RMAppManager.ApplicationSummary.SummaryBuilder summary = new RMAppManager.ApplicationSummary().createAppSummary(app); String msg = summary.toString(); @@ -583,7 +588,12 @@ public void testEscapeApplicationSummary() { Assert.assertTrue(msg.contains("Multiline" + escaped +"AppName")); Assert.assertTrue(msg.contains("Multiline" + escaped +"UserName")); Assert.assertTrue(msg.contains("Multiline" + escaped +"QueueName")); - } + Assert.assertTrue(msg.contains("memorySeconds=16384")); + Assert.assertTrue(msg.contains("vcoreSeconds=64")); + Assert.assertTrue(msg.contains("preemptedAMContainers=1")); + Assert.assertTrue(msg.contains("preemptedNonAMContainers=10")); + Assert.assertTrue(msg.contains("preemptedResources=")); + } private static ResourceScheduler mockResourceScheduler() { ResourceScheduler scheduler = mock(ResourceScheduler.class);