From 1ad0d43fb42861403403c175bbb7eff61cb6c386 Mon Sep 17 00:00:00 2001 From: Xuan Date: Mon, 1 Jun 2015 11:29:36 -0700 Subject: [PATCH] Revert "YARN-1462. Made RM write application tags to timeline server and exposed them to users via generic history web UI and REST API. Contributed by Xuan Gong." This reverts commit 2032e8d1a0c6abe154a64df69b0124cdae9f78e0. --- .../apache/hadoop/mapred/NotRunningJob.java | 2 +- .../mapred/TestClientServiceDelegate.java | 4 +- .../apache/hadoop/mapred/TestYARNRunner.java | 2 +- hadoop-yarn-project/CHANGES.txt | 3 -- .../yarn/api/records/ApplicationReport.java | 4 +- .../yarn/client/ProtocolHATestBase.java | 2 +- .../yarn/client/api/impl/TestAHSClient.java | 8 ++-- .../yarn/client/api/impl/TestYarnClient.java | 9 ++--- .../hadoop/yarn/client/cli/TestYarnCLI.java | 27 ++++++-------- .../hadoop/yarn/api/TestApplicatonReport.java | 3 +- .../ApplicationHistoryManagerImpl.java | 2 +- ...licationHistoryManagerOnTimelineStore.java | 19 +--------- ...licationHistoryManagerOnTimelineStore.java | 9 ----- .../metrics/ApplicationMetricsConstants.java | 1 - .../metrics/ApplicationCreatedEvent.java | 10 +---- .../metrics/SystemMetricsPublisher.java | 4 +- .../applicationsmanager/MockAsm.java | 2 +- .../metrics/TestSystemMetricsPublisher.java | 37 ------------------- .../resourcemanager/webapp/TestRMWebApp.java | 2 +- .../src/site/markdown/TimelineServer.md | 30 ++------------- 20 files changed, 36 insertions(+), 144 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java index 1b26cd3b1e2..03552e4f2bc 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java @@ -90,7 +90,7 @@ public class NotRunningJob implements MRClientProtocol { return ApplicationReport.newInstance(unknownAppId, unknownAttemptId, "N/A", "N/A", "N/A", "N/A", 0, null, YarnApplicationState.NEW, "N/A", "N/A", 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A", 0.0f, - YarnConfiguration.DEFAULT_APPLICATION_TYPE, null, null); + YarnConfiguration.DEFAULT_APPLICATION_TYPE, null); } NotRunningJob(ApplicationReport applicationReport, JobState jobState) { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java index c1c03df278a..b85f18db4eb 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java @@ -516,7 +516,7 @@ public class TestClientServiceDelegate { return ApplicationReport.newInstance(appId, attemptId, "user", "queue", "appname", "host", 124, null, YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, FinalApplicationStatus.SUCCEEDED, null, - "N/A", 0.0f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null, null); + "N/A", 0.0f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null); } private ApplicationReport getRunningApplicationReport(String host, int port) { @@ -526,7 +526,7 @@ public class TestClientServiceDelegate { return ApplicationReport.newInstance(appId, attemptId, "user", "queue", "appname", host, port, null, YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A", 0.0f, - YarnConfiguration.DEFAULT_APPLICATION_TYPE, null, null); + YarnConfiguration.DEFAULT_APPLICATION_TYPE, null); } private ResourceMgrDelegate getRMDelegate() throws IOException { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java index 30a8f015ec1..c427975e353 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java @@ -196,7 +196,7 @@ public class TestYARNRunner extends TestCase { ApplicationReport.newInstance(appId, null, "tmp", "tmp", "tmp", "tmp", 0, null, YarnApplicationState.FINISHED, "tmp", "tmp", 0l, 0l, FinalApplicationStatus.SUCCEEDED, null, null, 0f, - "tmp", null, null)); + "tmp", null)); yarnRunner.killJob(jobId); verify(clientDelegate).killJob(jobId); } diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index b57eed262f2..67e06283823 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -20,9 +20,6 @@ Release 2.7.1 - UNRELEASED YARN-3489. RMServerUtils.validateResourceRequests should only obtain queue info once. (Varun Saxena via wangda) - YARN-1462. Made RM write application tags to timeline server and exposed them - to users via generic history web UI and REST API. (Xuan Gong via zjshen) - OPTIMIZATIONS YARN-3006. Improve the error message when attempting manual failover with diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java index df7ab15ddf3..ff4fb52a555 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java @@ -59,8 +59,7 @@ public abstract class ApplicationReport { YarnApplicationState state, String diagnostics, String url, long startTime, long finishTime, FinalApplicationStatus finalStatus, ApplicationResourceUsageReport appResources, String origTrackingUrl, - float progress, String applicationType, Token amRmToken, - Set tags) { + float progress, String applicationType, Token amRmToken) { ApplicationReport report = Records.newRecord(ApplicationReport.class); report.setApplicationId(applicationId); report.setCurrentApplicationAttemptId(applicationAttemptId); @@ -81,7 +80,6 @@ public abstract class ApplicationReport { report.setProgress(progress); report.setApplicationType(applicationType); report.setAMRMToken(amRmToken); - report.setApplicationTags(tags); return report; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java index a680a110d6a..f468bc115a9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java @@ -647,7 +647,7 @@ public abstract class ProtocolHATestBase extends ClientBaseWithFixes { "fakeQueue", "fakeApplicationName", "localhost", 0, null, YarnApplicationState.FINISHED, "fake an application report", "", 1000l, 1200l, FinalApplicationStatus.FAILED, null, "", 50f, - "fakeApplicationType", null, null); + "fakeApplicationType", null); return report; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java index 12d06028212..c3e3c414771 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java @@ -334,7 +334,7 @@ public class TestAHSClient { "queue", "appname", "host", 124, null, YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", - null, null); + null); List applicationReports = new ArrayList(); applicationReports.add(newApplicationReport); @@ -391,7 +391,7 @@ public class TestAHSClient { "queue2", "appname2", "host2", 125, null, YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.63789f, - "NON-YARN", null, null); + "NON-YARN", null); applicationReports.add(newApplicationReport2); ApplicationId applicationId3 = ApplicationId.newInstance(1234, 7); @@ -401,7 +401,7 @@ public class TestAHSClient { "queue3", "appname3", "host3", 126, null, YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, - "MAPREDUCE", null, null); + "MAPREDUCE", null); applicationReports.add(newApplicationReport3); ApplicationId applicationId4 = ApplicationId.newInstance(1234, 8); @@ -411,7 +411,7 @@ public class TestAHSClient { "queue4", "appname4", "host4", 127, null, YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f, - "NON-MAPREDUCE", null, null); + "NON-MAPREDUCE", null); applicationReports.add(newApplicationReport4); reports = applicationReports; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java index 8739385a0fd..738b0a8dd32 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java @@ -580,8 +580,7 @@ public class TestYarnClient { applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, - FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null, - null); + FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); List applicationReports = new ArrayList(); applicationReports.add(newApplicationReport); List appAttempts = new ArrayList(); @@ -656,7 +655,7 @@ public class TestYarnClient { "user2", "queue2", "appname2", "host2", 125, null, YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.63789f, "NON-YARN", - null, null); + null); applicationReports.add(newApplicationReport2); ApplicationId applicationId3 = ApplicationId.newInstance(1234, 7); @@ -665,7 +664,7 @@ public class TestYarnClient { "user3", "queue3", "appname3", "host3", 126, null, YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE", - null, null); + null); applicationReports.add(newApplicationReport3); ApplicationId applicationId4 = ApplicationId.newInstance(1234, 8); @@ -676,7 +675,7 @@ public class TestYarnClient { "user4", "queue4", "appname4", "host4", 127, null, YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f, - "NON-MAPREDUCE", null, null); + "NON-MAPREDUCE", null); applicationReports.add(newApplicationReport4); return applicationReports; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java index 597531753cd..4b60c521673 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java @@ -103,7 +103,7 @@ public class TestYarnCLI { "user", "queue", "appname", "host", 124, null, YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, FinalApplicationStatus.SUCCEEDED, usageReport, "N/A", 0.53789f, "YARN", - null, null); + null); when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( newApplicationReport); int result = cli.run(new String[] { "application", "-status", applicationId.toString() }); @@ -347,8 +347,7 @@ public class TestYarnCLI { applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, - FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null, - null); + FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); List applicationReports = new ArrayList(); applicationReports.add(newApplicationReport); @@ -358,7 +357,7 @@ public class TestYarnCLI { "user2", "queue2", "appname2", "host2", 125, null, YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.63789f, "NON-YARN", - null, null); + null); applicationReports.add(newApplicationReport2); ApplicationId applicationId3 = ApplicationId.newInstance(1234, 7); @@ -367,7 +366,7 @@ public class TestYarnCLI { "user3", "queue3", "appname3", "host3", 126, null, YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE", - null, null); + null); applicationReports.add(newApplicationReport3); ApplicationId applicationId4 = ApplicationId.newInstance(1234, 8); @@ -376,7 +375,7 @@ public class TestYarnCLI { "user4", "queue4", "appname4", "host4", 127, null, YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f, "NON-MAPREDUCE", - null, null); + null); applicationReports.add(newApplicationReport4); ApplicationId applicationId5 = ApplicationId.newInstance(1234, 9); @@ -385,7 +384,7 @@ public class TestYarnCLI { "user5", "queue5", "appname5", "host5", 128, null, YarnApplicationState.ACCEPTED, "diagnostics5", "url5", 5, 5, FinalApplicationStatus.KILLED, null, "N/A", 0.93789f, "HIVE", - null, null); + null); applicationReports.add(newApplicationReport5); ApplicationId applicationId6 = ApplicationId.newInstance(1234, 10); @@ -394,7 +393,7 @@ public class TestYarnCLI { "user6", "queue6", "appname6", "host6", 129, null, YarnApplicationState.SUBMITTED, "diagnostics6", "url6", 6, 6, FinalApplicationStatus.KILLED, null, "N/A", 0.99789f, "PIG", - null, null); + null); applicationReports.add(newApplicationReport6); // Test command yarn application -list @@ -818,8 +817,7 @@ public class TestYarnCLI { applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, - FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null, - null); + FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( newApplicationReport2); int result = cli.run(new String[] { "application","-kill", applicationId.toString() }); @@ -832,8 +830,7 @@ public class TestYarnCLI { applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, - FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null, - null); + FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( newApplicationReport); result = cli.run(new String[] { "application","-kill", applicationId.toString() }); @@ -868,8 +865,7 @@ public class TestYarnCLI { applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, - FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null, - null); + FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( newApplicationReport2); int result = cli.run(new String[] { "application", "-movetoqueue", @@ -884,8 +880,7 @@ public class TestYarnCLI { applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, - FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null, - null); + FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( newApplicationReport); result = cli.run(new String[] { "application", "-movetoqueue", diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java index a2ea0a7c064..9302d4b6791 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java @@ -58,8 +58,7 @@ public class TestApplicatonReport { ApplicationReport.newInstance(appId, appAttemptId, "user", "queue", "appname", "host", 124, null, YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, FinalApplicationStatus.SUCCEEDED, null, - "N/A", 0.53789f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null, - null); + "N/A", 0.53789f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null); return appReport; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java index 8b36e9ae436..c7cf07b1a88 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java @@ -139,7 +139,7 @@ public class ApplicationHistoryManagerImpl extends AbstractService implements appHistory.getYarnApplicationState(), appHistory.getDiagnosticsInfo(), trackingUrl, appHistory.getStartTime(), appHistory.getFinishTime(), appHistory.getFinalApplicationStatus(), null, "", 100, - appHistory.getApplicationType(), null, null); + appHistory.getApplicationType(), null); } private ApplicationAttemptHistoryData getLastAttempt(ApplicationId appId) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java index 52bc83f32b2..5f4510bd60a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java @@ -19,14 +19,11 @@ package org.apache.hadoop.yarn.server.applicationhistoryservice; import java.io.IOException; -import java.util.Collection; import java.util.EnumSet; import java.util.HashMap; -import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -246,7 +243,6 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService FinalApplicationStatus finalStatus = FinalApplicationStatus.UNDEFINED; YarnApplicationState state = null; ApplicationResourceUsageReport appResources = null; - Set appTags = null; Map appViewACLs = new HashMap(); Map entityInfo = entity.getOtherInfo(); @@ -268,7 +264,7 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService ConverterUtils.toApplicationId(entity.getEntityId()), latestApplicationAttemptId, user, queue, name, null, -1, null, state, diagnosticsInfo, null, createdTime, finishedTime, finalStatus, null, - null, 1.0F, type, null, appTags), appViewACLs); + null, 1.0F, type, null), appViewACLs); } if (entityInfo.containsKey(ApplicationMetricsConstants.QUEUE_ENTITY_INFO)) { queue = @@ -293,17 +289,6 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService appResources=ApplicationResourceUsageReport .newInstance(0, 0, null, null, null, memorySeconds, vcoreSeconds); } - if (entityInfo.containsKey(ApplicationMetricsConstants.APP_TAGS_INFO)) { - appTags = new HashSet(); - Object obj = entityInfo.get(ApplicationMetricsConstants.APP_TAGS_INFO); - if (obj != null && obj instanceof Collection) { - for(Object o : (Collection)obj) { - if (o != null) { - appTags.add(o.toString()); - } - } - } - } } List events = entity.getEvents(); if (events != null) { @@ -355,7 +340,7 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService ConverterUtils.toApplicationId(entity.getEntityId()), latestApplicationAttemptId, user, queue, name, null, -1, null, state, diagnosticsInfo, null, createdTime, finishedTime, finalStatus, appResources, - null, 1.0F, type, null, appTags), appViewACLs); + null, 1.0F, type, null), appViewACLs); } private static ApplicationAttemptReport convertToApplicationAttemptReport( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java index c55751fb3e7..fcdafc29442 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java @@ -22,9 +22,7 @@ import java.security.PrivilegedExceptionAction; import java.util.Arrays; import java.util.Collection; import java.util.HashMap; -import java.util.HashSet; import java.util.Map; -import java.util.Set; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.SaslRpcServer.AuthMethod; @@ -185,9 +183,6 @@ public class TestApplicationHistoryManagerOnTimelineStore { Assert.assertEquals(Integer.MAX_VALUE + 2L, app.getStartTime()); Assert.assertEquals(Integer.MAX_VALUE + 3L, app.getFinishTime()); Assert.assertTrue(Math.abs(app.getProgress() - 1.0F) < 0.0001); - Assert.assertEquals(2, app.getApplicationTags().size()); - Assert.assertTrue(app.getApplicationTags().contains("Test_APP_TAGS_1")); - Assert.assertTrue(app.getApplicationTags().contains("Test_APP_TAGS_2")); // App 2 doesn't have the ACLs, such that the default ACLs " " will be used. // Nobody except admin and owner has access to the details of the app. if ((i == 1 && callerUGI != null && @@ -474,10 +469,6 @@ public class TestApplicationHistoryManagerOnTimelineStore { entityInfo.put(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO, "user2"); } - Set appTags = new HashSet(); - appTags.add("Test_APP_TAGS_1"); - appTags.add("Test_APP_TAGS_2"); - entityInfo.put(ApplicationMetricsConstants.APP_TAGS_INFO, appTags); entity.setOtherInfo(entityInfo); TimelineEvent tEvent = new TimelineEvent(); tEvent.setEventType(ApplicationMetricsConstants.CREATED_EVENT_TYPE); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java index f4524106175..df8eecb40b2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java @@ -73,5 +73,4 @@ public class ApplicationMetricsConstants { public static final String LATEST_APP_ATTEMPT_EVENT_INFO = "YARN_APPLICATION_LATEST_APP_ATTEMPT"; - public static final String APP_TAGS_INFO = "YARN_APPLICATION_TAGS"; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java index 7c43aa4ba62..2373b3bf1e8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java @@ -18,8 +18,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.metrics; -import java.util.Set; - import org.apache.hadoop.yarn.api.records.ApplicationId; public class ApplicationCreatedEvent extends @@ -31,7 +29,6 @@ public class ApplicationCreatedEvent extends private String user; private String queue; private long submittedTime; - private Set appTags; public ApplicationCreatedEvent(ApplicationId appId, String name, @@ -39,8 +36,7 @@ public class ApplicationCreatedEvent extends String user, String queue, long submittedTime, - long createdTime, - Set appTags) { + long createdTime) { super(SystemMetricsEventType.APP_CREATED, createdTime); this.appId = appId; this.name = name; @@ -48,7 +44,6 @@ public class ApplicationCreatedEvent extends this.user = user; this.queue = queue; this.submittedTime = submittedTime; - this.appTags = appTags; } @Override @@ -80,7 +75,4 @@ public class ApplicationCreatedEvent extends return submittedTime; } - public Set getAppTags() { - return appTags; - } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java index faf6d7265f7..b849b0035db 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java @@ -106,7 +106,7 @@ public class SystemMetricsPublisher extends CompositeService { app.getUser(), app.getQueue(), app.getSubmitTime(), - createdTime, app.getApplicationTags())); + createdTime)); } } @@ -251,8 +251,6 @@ public class SystemMetricsPublisher extends CompositeService { event.getQueue()); entityInfo.put(ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO, event.getSubmittedTime()); - entityInfo.put(ApplicationMetricsConstants.APP_TAGS_INFO, - event.getAppTags()); entity.setOtherInfo(entityInfo); TimelineEvent tEvent = new TimelineEvent(); tEvent.setEventType( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java index 33eab9f7296..f8d92aa2f17 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java @@ -289,7 +289,7 @@ public abstract class MockAsm extends MockApps { getName(), null, 0, null, null, getDiagnostics().toString(), getTrackingUrl(), getStartTime(), getFinishTime(), getFinalApplicationStatus(), usageReport , null, getProgress(), - type, null, null); + type, null); return report; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java index 5197902a9b5..7ed3835bdb7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java @@ -21,11 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.metrics; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import java.util.Collection; import java.util.EnumSet; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -147,8 +143,6 @@ public class TestSystemMetricsPublisher { Assert.assertEquals(app.getSubmitTime(), entity.getOtherInfo().get( ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO)); - Assert.assertTrue(verifyAppTags(app.getApplicationTags(), - entity.getOtherInfo())); if (i == 1) { Assert.assertEquals("uers1,user2", entity.getOtherInfo().get( @@ -358,10 +352,6 @@ public class TestSystemMetricsPublisher { FinalApplicationStatus.UNDEFINED); when(app.getRMAppMetrics()).thenReturn( new RMAppMetrics(null, 0, 0, Integer.MAX_VALUE, Long.MAX_VALUE)); - Set appTags = new HashSet(); - appTags.add("test"); - appTags.add("tags"); - when(app.getApplicationTags()).thenReturn(appTags); return app; } @@ -402,31 +392,4 @@ public class TestSystemMetricsPublisher { return container; } - private static boolean verifyAppTags(Set appTags, - Map entityInfo) { - if (!entityInfo.containsKey(ApplicationMetricsConstants.APP_TAGS_INFO)) { - return false; - } - Object obj = entityInfo.get(ApplicationMetricsConstants.APP_TAGS_INFO); - if (obj instanceof Collection) { - Collection collection = (Collection) obj; - if (collection.size() != appTags.size()) { - return false; - } - for (String appTag : appTags) { - boolean match = false; - for (Object o : collection) { - if (o.toString().equals(appTag)) { - match = true; - break; - } - } - if (!match) { - return false; - } - } - return true; - } - return false; - } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java index 79e38d97cb0..481a53b6221 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java @@ -253,7 +253,7 @@ public class TestRMWebApp { app.getStartTime(), app.getFinishTime(), app.getFinalApplicationStatus(), (ApplicationResourceUsageReport) null, app.getTrackingUrl(), - app.getProgress(), app.getApplicationType(), (Token) null, null); + app.getProgress(), app.getApplicationType(), (Token) null); appReports.add(appReport); } GetApplicationsResponse response = mock(GetApplicationsResponse.class); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md index 87893cdcdd9..fde654b3b27 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md @@ -1030,27 +1030,7 @@ Response Body: "submittedTime":1430424769395, "startedTime":1430424769395, "finishedTime":1430424776594, - "elapsedTime":7199}, - { - "appId":"application_1430424020775_0001", - "currentAppAttemptId":"appattempt_1430424020775_0001_000001", - "user":"zshen", - "name":"QuasiMonteCarlo", - "queue":"default", - "type":"MAPREDUCE", - "host":"localhost", - "rpcPort":56264, - "appState":"FINISHED", - "progress":100.0, - "diagnosticsInfo":"", - "originalTrackingUrl":"http://d-69-91-129-173.dhcp4.washington.edu:19888/jobhistory/job/job_1430424020775_0001", - "trackingUrl":"http://d-69-91-129-173.dhcp4.washington.edu:8088/proxy/application_1430424020775_0001/", - "finalAppStatus":"SUCCEEDED", - "submittedTime":1430424053809, - "startedTime":1430424072153, - "finishedTime":1430424776594, - "elapsedTime":18344, - "applicationTags":"mrapplication,ta-example" + "elapsedTime":7199 } ] } @@ -1152,7 +1132,6 @@ Response Body: 1430424053809 1430424072153 18344 - mrapplication,ta-example @@ -1200,8 +1179,7 @@ None | `allocatedVCores` | int | The sum of virtual cores allocated to the application's running containers | | `currentAppAttemptId` | string | The latest application attempt ID | | `host` | string | The host of the ApplicationMaster | -| `rpcPort` | int | The RPC port of the ApplicationMaster; zero if no IPC service declared | -| `applicationTags` | string | The application tags. | +| `rpcPort` | int | The RPC port of the ApplicationMaster; zero if no IPC service declared. | ### Response Examples: @@ -1237,8 +1215,7 @@ Response Body: "submittedTime": 1430424053809, "startedTime": 1430424053809, "finishedTime": 1430424072153, - "elapsedTime": 18344, - "applicationTags": mrapplication,tag-example + "elapsedTime": 18344 } #### XML response @@ -1276,7 +1253,6 @@ Response Body: 1430424053809 1430424072153 18344 - mrapplication,ta-example ## Application Attempt List