From 0306d902f53582320aa5895ca9f5c31f64aaaff6 Mon Sep 17 00:00:00 2001 From: Xuan Date: Mon, 3 Aug 2015 15:46:00 -0700 Subject: [PATCH] YARN-3543. ApplicationReport should be able to tell whether the Application is AM managed or not. Contributed by Rohith Sharma K S --- hadoop-yarn-project/CHANGES.txt | 3 +++ .../yarn/api/records/ApplicationReport.java | 17 ++++++++++++++- .../src/main/proto/yarn_protos.proto | 1 + .../yarn/client/cli/ApplicationCLI.java | 4 +++- .../yarn/client/api/impl/TestYarnClient.java | 10 +++++---- .../hadoop/yarn/client/cli/TestYarnCLI.java | 3 ++- .../impl/pb/ApplicationReportPBImpl.java | 12 +++++++++++ .../hadoop/yarn/api/TestApplicatonReport.java | 3 ++- ...licationHistoryManagerOnTimelineStore.java | 13 ++++++++++-- ...licationHistoryManagerOnTimelineStore.java | 2 ++ .../metrics/ApplicationMetricsConstants.java | 3 +++ .../hadoop/yarn/server/webapp/AppBlock.java | 1 + .../yarn/server/webapp/dao/AppInfo.java | 6 ++++++ .../metrics/ApplicationCreatedEvent.java | 9 +++++++- .../metrics/SystemMetricsPublisher.java | 6 +++++- .../resourcemanager/rmapp/RMAppImpl.java | 3 ++- .../resourcemanager/webapp/dao/AppInfo.java | 7 +++++++ .../metrics/TestSystemMetricsPublisher.java | 10 +++++++++ .../webapp/TestRMWebServicesApps.java | 14 +++++++++---- .../src/site/markdown/ResourceManagerRest.md | 13 +++++++++--- .../src/site/markdown/TimelineServer.md | 21 ++++++++++++++----- 21 files changed, 136 insertions(+), 25 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 14961440a18..a77a0813718 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -363,6 +363,9 @@ Release 2.8.0 - UNRELEASED YARN-3965. Add startup timestamp to nodemanager UI (Hong Zhiguo via jlowe) + YARN-3543. ApplicationReport should be able to tell whether the Application + is AM managed or not. (Rohith Sharma K S via xgong) + OPTIMIZATIONS YARN-3339. TestDockerContainerExecutor should pull a single image and not diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java index 444a202e12f..e7a2ad4b835 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java @@ -92,13 +92,14 @@ public abstract class ApplicationReport { long startTime, long finishTime, FinalApplicationStatus finalStatus, ApplicationResourceUsageReport appResources, String origTrackingUrl, float progress, String applicationType, Token amRmToken, - Set tags) { + Set tags, boolean unmanagedApplication) { ApplicationReport report = newInstance(applicationId, applicationAttemptId, user, queue, name, host, rpcPort, clientToAMToken, state, diagnostics, url, startTime, finishTime, finalStatus, appResources, origTrackingUrl, progress, applicationType, amRmToken); report.setApplicationTags(tags); + report.setUnmanagedApp(unmanagedApplication); return report; } @@ -393,4 +394,18 @@ public abstract class ApplicationReport { @Unstable public abstract void setLogAggregationStatus( LogAggregationStatus logAggregationStatus); + + /** + * @return true if the AM is not managed by the RM + */ + @Public + @Unstable + public abstract boolean isUnmanagedApp(); + + /** + * @param value true if RM should not manage the AM + */ + @Public + @Unstable + public abstract void setUnmanagedApp(boolean unmanagedApplication); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 33cc2551bf8..7bd03975a1a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -195,6 +195,7 @@ message ApplicationReportProto { optional hadoop.common.TokenProto am_rm_token = 19; repeated string applicationTags = 20; optional LogAggregationStatusProto log_aggregation_status = 21; + optional bool unmanaged_application = 22 [default = false]; } enum LogAggregationStatusProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java index 8ef88c39ff4..013b88e00ef 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java @@ -534,7 +534,9 @@ public class ApplicationCLI extends YarnCLI { appReportStr.println(appReport.getLogAggregationStatus() == null ? "N/A" : appReport.getLogAggregationStatus()); appReportStr.print("\tDiagnostics : "); - appReportStr.print(appReport.getDiagnostics()); + appReportStr.println(appReport.getDiagnostics()); + appReportStr.print("\tUnmanaged Application : "); + appReportStr.print(appReport.isUnmanagedApp()); } else { appReportStr.print("Application with id '" + applicationId + "' doesn't exist in RM."); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java index bc40b9ae6c9..e584cf9efff 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java @@ -895,12 +895,12 @@ public class TestYarnClient { rmClient.start(); ApplicationId appId = createApp(rmClient, false); - waitTillAccepted(rmClient, appId); + waitTillAccepted(rmClient, appId, false); //managed AMs don't return AMRM token Assert.assertNull(rmClient.getAMRMToken(appId)); appId = createApp(rmClient, true); - waitTillAccepted(rmClient, appId); + waitTillAccepted(rmClient, appId, true); long start = System.currentTimeMillis(); while (rmClient.getAMRMToken(appId) == null) { if (System.currentTimeMillis() - start > 20 * 1000) { @@ -921,7 +921,7 @@ public class TestYarnClient { rmClient.init(yarnConf); rmClient.start(); ApplicationId appId = createApp(rmClient, true); - waitTillAccepted(rmClient, appId); + waitTillAccepted(rmClient, appId, true); long start = System.currentTimeMillis(); while (rmClient.getAMRMToken(appId) == null) { if (System.currentTimeMillis() - start > 20 * 1000) { @@ -981,7 +981,8 @@ public class TestYarnClient { return appId; } - private void waitTillAccepted(YarnClient rmClient, ApplicationId appId) + private void waitTillAccepted(YarnClient rmClient, ApplicationId appId, + boolean unmanagedApplication) throws Exception { try { long start = System.currentTimeMillis(); @@ -994,6 +995,7 @@ public class TestYarnClient { Thread.sleep(200); report = rmClient.getApplicationReport(appId); } + Assert.assertEquals(unmanagedApplication, report.isUnmanagedApp()); } catch (Exception ex) { throw new Exception(ex); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java index 1013958bdf7..aadcd4dc27c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java @@ -104,7 +104,7 @@ public class TestYarnCLI { "user", "queue", "appname", "host", 124, null, YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, FinalApplicationStatus.SUCCEEDED, usageReport, "N/A", 0.53789f, "YARN", - null); + null, null, false); newApplicationReport.setLogAggregationStatus(LogAggregationStatus.SUCCEEDED); when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( newApplicationReport); @@ -131,6 +131,7 @@ public class TestYarnCLI { (i == 0 ? "N/A" : "123456 MB-seconds, 4567 vcore-seconds")); pw.println("\tLog Aggregation Status : SUCCEEDED"); pw.println("\tDiagnostics : diagnostics"); + pw.println("\tUnmanaged Application : false"); pw.close(); String appReportStr = baos.toString("UTF-8"); Assert.assertEquals(appReportStr, sysOutStream.toString()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java index 751dd90e16c..69435b5d7b7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java @@ -581,4 +581,16 @@ public class ApplicationReportPBImpl extends ApplicationReport { convertToProtoFormat(LogAggregationStatus s) { return ProtoUtils.convertToProtoFormat(s); } + + @Override + public boolean isUnmanagedApp() { + ApplicationReportProtoOrBuilder p = viaProto ? proto : builder; + return p.getUnmanagedApplication(); + } + + @Override + public void setUnmanagedApp(boolean unmanagedApplication) { + maybeInitBuilder(); + builder.setUnmanagedApplication(unmanagedApplication); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java index 9302d4b6791..e2fa64805de 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java @@ -58,7 +58,8 @@ public class TestApplicatonReport { ApplicationReport.newInstance(appId, appAttemptId, "user", "queue", "appname", "host", 124, null, YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, FinalApplicationStatus.SUCCEEDED, null, - "N/A", 0.53789f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null); + "N/A", 0.53789f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null, + null,false); return appReport; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java index c7b19d360f4..9ae99c5ef33 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java @@ -247,6 +247,7 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService String queue = null; String name = null; String type = null; + boolean unmanagedApplication = false; long createdTime = 0; long finishedTime = 0; float progress = 0.0f; @@ -277,7 +278,8 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService ConverterUtils.toApplicationId(entity.getEntityId()), latestApplicationAttemptId, user, queue, name, null, -1, null, state, diagnosticsInfo, null, createdTime, finishedTime, finalStatus, null, - null, progress, type, null, appTags), appViewACLs); + null, progress, type, null, appTags, + unmanagedApplication), appViewACLs); } if (entityInfo.containsKey(ApplicationMetricsConstants.QUEUE_ENTITY_INFO)) { queue = @@ -294,6 +296,13 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService entityInfo.get(ApplicationMetricsConstants.TYPE_ENTITY_INFO) .toString(); } + if (entityInfo + .containsKey(ApplicationMetricsConstants.UNMANAGED_APPLICATION_ENTITY_INFO)) { + unmanagedApplication = + Boolean.parseBoolean(entityInfo.get( + ApplicationMetricsConstants.UNMANAGED_APPLICATION_ENTITY_INFO) + .toString()); + } if (entityInfo.containsKey(ApplicationMetricsConstants.APP_CPU_METRICS)) { long vcoreSeconds=Long.parseLong(entityInfo.get( ApplicationMetricsConstants.APP_CPU_METRICS).toString()); @@ -365,7 +374,7 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService ConverterUtils.toApplicationId(entity.getEntityId()), latestApplicationAttemptId, user, queue, name, null, -1, null, state, diagnosticsInfo, null, createdTime, finishedTime, finalStatus, appResources, - null, progress, type, null, appTags), appViewACLs); + null, progress, type, null, appTags, unmanagedApplication), appViewACLs); } private static ApplicationAttemptReport convertToApplicationAttemptReport( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java index f553b79eaa3..fb4af85bb39 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java @@ -475,6 +475,8 @@ public class TestApplicationHistoryManagerOnTimelineStore { "test app type"); entityInfo.put(ApplicationMetricsConstants.USER_ENTITY_INFO, "user1"); entityInfo.put(ApplicationMetricsConstants.QUEUE_ENTITY_INFO, "test queue"); + entityInfo.put( + ApplicationMetricsConstants.UNMANAGED_APPLICATION_ENTITY_INFO, "false"); entityInfo.put(ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO, Integer.MAX_VALUE + 1L); entityInfo.put(ApplicationMetricsConstants.APP_MEM_METRICS,123); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java index f4524106175..26b113f1325 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java @@ -74,4 +74,7 @@ public class ApplicationMetricsConstants { "YARN_APPLICATION_LATEST_APP_ATTEMPT"; public static final String APP_TAGS_INFO = "YARN_APPLICATION_TAGS"; + + public static final String UNMANAGED_APPLICATION_ENTITY_INFO = + "YARN_APPLICATION_UNMANAGED_APPLICATION"; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java index 871bac35964..111842c2f2f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java @@ -206,6 +206,7 @@ public class AppBlock extends HtmlBlock { } overviewTable._("Diagnostics:", app.getDiagnosticsInfo() == null ? "" : app.getDiagnosticsInfo()); + overviewTable._("Unmanaged Application:", app.isUnmanagedApp()); Collection attempts; try { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java index 8f332a4cdd4..77553d497d0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java @@ -60,6 +60,7 @@ public class AppInfo { protected String applicationTags; private int allocatedCpuVcores; private int allocatedMemoryMB; + protected boolean unmanagedApplication; public AppInfo() { // JAXB needs this @@ -99,6 +100,7 @@ public class AppInfo { if (app.getApplicationTags() != null && !app.getApplicationTags().isEmpty()) { this.applicationTags = CSV_JOINER.join(app.getApplicationTags()); } + unmanagedApplication = app.isUnmanagedApp(); } public String getAppId() { @@ -188,4 +190,8 @@ public class AppInfo { public String getApplicationTags() { return applicationTags; } + + public boolean isUnmanagedApp() { + return unmanagedApplication; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java index 7c43aa4ba62..df0e39671e4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java @@ -32,6 +32,7 @@ public class ApplicationCreatedEvent extends private String queue; private long submittedTime; private Set appTags; + private boolean unmanagedApplication; public ApplicationCreatedEvent(ApplicationId appId, String name, @@ -40,7 +41,8 @@ public class ApplicationCreatedEvent extends String queue, long submittedTime, long createdTime, - Set appTags) { + Set appTags, + boolean unmanagedApplication) { super(SystemMetricsEventType.APP_CREATED, createdTime); this.appId = appId; this.name = name; @@ -49,6 +51,7 @@ public class ApplicationCreatedEvent extends this.queue = queue; this.submittedTime = submittedTime; this.appTags = appTags; + this.unmanagedApplication = unmanagedApplication; } @Override @@ -83,4 +86,8 @@ public class ApplicationCreatedEvent extends public Set getAppTags() { return appTags; } + + public boolean isUnmanagedApp() { + return unmanagedApplication; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java index faf6d7265f7..07553a73bba 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java @@ -106,7 +106,8 @@ public class SystemMetricsPublisher extends CompositeService { app.getUser(), app.getQueue(), app.getSubmitTime(), - createdTime, app.getApplicationTags())); + createdTime, app.getApplicationTags(), + app.getApplicationSubmissionContext().getUnmanagedAM())); } } @@ -253,6 +254,9 @@ public class SystemMetricsPublisher extends CompositeService { event.getSubmittedTime()); entityInfo.put(ApplicationMetricsConstants.APP_TAGS_INFO, event.getAppTags()); + entityInfo.put( + ApplicationMetricsConstants.UNMANAGED_APPLICATION_ENTITY_INFO, + event.isUnmanagedApp()); entity.setOtherInfo(entityInfo); TimelineEvent tEvent = new TimelineEvent(); tEvent.setEventType( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index d480c24237d..6b7526eab1c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -663,8 +663,9 @@ public class RMAppImpl implements RMApp, Recoverable { createApplicationState(), diags, trackingUrl, this.startTime, this.finishTime, finishState, appUsageReport, origTrackingUrl, progress, this.applicationType, - amrmToken, applicationTags); + amrmToken, applicationTags); report.setLogAggregationStatus(logAggregationStatus); + report.setUnmanagedApp(submissionContext.getUnmanagedAM()); return report; } finally { this.readLock.unlock(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java index bd3b046ab99..231f955444c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java @@ -96,6 +96,7 @@ public class AppInfo { protected List resourceRequests; protected LogAggregationStatus logAggregationStatus; + protected boolean unmanagedApplication; public AppInfo() { } // JAXB needs this @@ -183,6 +184,8 @@ public class AppInfo { appMetrics.getResourcePreempted().getVirtualCores(); memorySeconds = appMetrics.getMemorySeconds(); vcoreSeconds = appMetrics.getVcoreSeconds(); + unmanagedApplication = + app.getApplicationSubmissionContext().getUnmanagedAM(); } } @@ -321,4 +324,8 @@ public class AppInfo { public LogAggregationStatus getLogAggregationStatus() { return this.logAggregationStatus; } + + public boolean isUnmanagedApp() { + return unmanagedApplication; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java index 5197902a9b5..890dfb105d2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java @@ -29,6 +29,7 @@ import java.util.Set; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerState; @@ -134,6 +135,12 @@ public class TestSystemMetricsPublisher { Assert.assertEquals(app.getQueue(), entity.getOtherInfo() .get(ApplicationMetricsConstants.QUEUE_ENTITY_INFO)); + + Assert.assertEquals( + app.getApplicationSubmissionContext().getUnmanagedAM(), + entity.getOtherInfo().get( + ApplicationMetricsConstants.UNMANAGED_APPLICATION_ENTITY_INFO)); + Assert .assertEquals( app.getUser(), @@ -362,6 +369,9 @@ public class TestSystemMetricsPublisher { appTags.add("test"); appTags.add("tags"); when(app.getApplicationTags()).thenReturn(appTags); + ApplicationSubmissionContext asc = mock(ApplicationSubmissionContext.class); + when(asc.getUnmanagedAM()).thenReturn(false); + when(app.getApplicationSubmissionContext()).thenReturn(asc); return app; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index 549b9e0ac6e..879ce4cbca3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -1308,14 +1308,15 @@ public class TestRMWebServicesApps extends JerseyTestBase { WebServicesTestUtils.getXmlInt(element, "preemptedResourceVCores"), WebServicesTestUtils.getXmlInt(element, "numNonAMContainerPreempted"), WebServicesTestUtils.getXmlInt(element, "numAMContainerPreempted"), - WebServicesTestUtils.getXmlString(element, "logAggregationStatus")); + WebServicesTestUtils.getXmlString(element, "logAggregationStatus"), + WebServicesTestUtils.getXmlBoolean(element, "unmanagedApplication")); } } public void verifyAppInfo(JSONObject info, RMApp app) throws JSONException, Exception { - assertEquals("incorrect number of elements", 28, info.length()); + assertEquals("incorrect number of elements", 29, info.length()); verifyAppInfoGeneric(app, info.getString("id"), info.getString("user"), info.getString("name"), info.getString("applicationType"), @@ -1331,7 +1332,8 @@ public class TestRMWebServicesApps extends JerseyTestBase { info.getInt("preemptedResourceVCores"), info.getInt("numNonAMContainerPreempted"), info.getInt("numAMContainerPreempted"), - info.getString("logAggregationStatus")); + info.getString("logAggregationStatus"), + info.getBoolean("unmanagedApplication")); } public void verifyAppInfoGeneric(RMApp app, String id, String user, @@ -1342,7 +1344,8 @@ public class TestRMWebServicesApps extends JerseyTestBase { int allocatedMB, int allocatedVCores, int numContainers, int preemptedResourceMB, int preemptedResourceVCores, int numNonAMContainerPreempted, int numAMContainerPreempted, - String logAggregationStatus) throws JSONException, + String logAggregationStatus, boolean unmanagedApplication) + throws JSONException, Exception { WebServicesTestUtils.checkStringMatch("id", app.getApplicationId() @@ -1392,6 +1395,9 @@ public class TestRMWebServicesApps extends JerseyTestBase { assertEquals("Log aggregation Status doesn't match", app .getLogAggregationStatusForAppReport().toString(), logAggregationStatus); + assertEquals("unmanagedApplication doesn't match", app + .getApplicationSubmissionContext().getUnmanagedAM(), + unmanagedApplication); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md index c8fe8550c9f..ddf2bf44813 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md @@ -1380,7 +1380,8 @@ Response Body: "applicationType" : "MAPREDUCE", "applicationTags" : "" "memorySeconds" : 151730, - "vcoreSeconds" : 103 + "vcoreSeconds" : 103, + "unmanagedApplication":"false" }, { "finishedTime" : 1326815789546, @@ -1406,7 +1407,8 @@ Response Body: "applicationTags" : "tag1" "memorySeconds" : 640064, "vcoreSeconds" : 442 - } + "unmanagedApplication":"false" + } ] } } @@ -1455,6 +1457,7 @@ Response Body: 0 151730 103 + false application_1326815542473_0002 @@ -1480,6 +1483,7 @@ Response Body: 0 640064 442 + false ``` @@ -1639,6 +1643,7 @@ Note that depending on security settings a user might not be able to see all the | runningContainers | int | The number of containers currently running for the application | | memorySeconds | long | The amount of memory the application has allocated (megabyte-seconds) | | vcoreSeconds | long | The amount of CPU resources the application has allocated (virtual core-seconds) | +| unmanagedApplication | boolean | Is the application unmanaged. | ### Response Examples @@ -1678,7 +1683,8 @@ Response Body: "trackingUrl" : "http://host.domain.com:8088/proxy/application_1326821518301_0005/jobhistory/job/job_1326821518301_5_5", "queue" : "a1", "memorySeconds" : 151730, - "vcoreSeconds" : 103 + "vcoreSeconds" : 103, + "unmanagedApplication":"false" } } ``` @@ -1720,6 +1726,7 @@ Response Body: host.domain.com:8042 151730 103 + false ``` diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md index 1b666068a9d..bbf3e5936e4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md @@ -1087,7 +1087,8 @@ Response Body: "submittedTime":1430425001004, "startedTime":1430425001004, "finishedTime":1430425008861, - "elapsedTime":7857}, + "elapsedTime":7857, + "unmanagedApplication":"false"}, { "appId":"application_1430424020775_0003", "currentAppAttemptId":"appattempt_1430424020775_0003_000001", @@ -1106,7 +1107,8 @@ Response Body: "submittedTime":1430424956650, "startedTime":1430424956650, "finishedTime":1430424963907, - "elapsedTime":7257}, + "elapsedTime":7257, + "unmanagedApplication":"false"}, { "appId":"application_1430424020775_0002", "currentAppAttemptId":"appattempt_1430424020775_0002_000001", @@ -1125,7 +1127,8 @@ Response Body: "submittedTime":1430424769395, "startedTime":1430424769395, "finishedTime":1430424776594, - "elapsedTime":7199}, + "elapsedTime":7199, + "unmanagedApplication":"false"}, { "appId":"application_1430424020775_0001", "currentAppAttemptId":"appattempt_1430424020775_0001_000001", @@ -1145,7 +1148,8 @@ Response Body: "startedTime":1430424072153, "finishedTime":1430424776594, "elapsedTime":18344, - "applicationTags":"mrapplication,ta-example" + "applicationTags":"mrapplication,ta-example", + "unmanagedApplication":"false" } ] } @@ -1187,6 +1191,7 @@ Response Body: 1430425001004 1430425008861 7857 + false application_1430424020775_0003 @@ -1207,6 +1212,7 @@ Response Body: 1430424956650 1430424963907 7257 + false application_1430424020775_0002 @@ -1227,6 +1233,7 @@ Response Body: 1430424769395 1430424776594 7199 + false application_1430424020775_0001 @@ -1248,6 +1255,7 @@ Response Body: 1430424072153 18344 mrapplication,ta-example + false @@ -1298,6 +1306,7 @@ None | `host` | string | The host of the ApplicationMaster | | `rpcPort` | int | The RPC port of the ApplicationMaster; zero if no IPC service declared | | `applicationTags` | string | The application tags. | +| `unmanagedApplication` | boolean | Is the application unmanaged. | ### Response Examples: @@ -1334,7 +1343,8 @@ Response Body: "startedTime": 1430424053809, "finishedTime": 1430424072153, "elapsedTime": 18344, - "applicationTags": mrapplication,tag-example + "applicationTags": mrapplication,tag-example, + "unmanagedApplication":"false" } #### XML response @@ -1373,6 +1383,7 @@ Response Body: 1430424072153 18344 mrapplication,ta-example + false ## Application Attempt List