From bbfd81503cfd98aaeb0334ba356076bfdf6bbd9f Mon Sep 17 00:00:00 2001 From: Mahadev Konar Date: Mon, 19 Sep 2011 23:56:52 +0000 Subject: [PATCH] MAPREDUCE-3017. The Web UI shows FINISHED for killed/successful/failed jobs. (mahadev) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1172906 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 + .../server/resourcemanager/rmapp/RMApp.java | 80 +++++++++++++++++++ .../resourcemanager/rmapp/RMAppImpl.java | 18 ++++- .../rmapp/attempt/RMAppAttempt.java | 72 ++++++++++++++++- .../rmapp/attempt/RMAppAttemptImpl.java | 13 ++- .../resourcemanager/webapp/AppsBlock.java | 4 +- .../resourcemanager/webapp/RmController.java | 5 +- .../applicationsmanager/MockAsm.java | 5 ++ .../resourcemanager/rmapp/MockRMApp.java | 5 ++ 9 files changed, 198 insertions(+), 7 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index d6c112324ff..98ef209d94a 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -1363,6 +1363,9 @@ Release 0.23.0 - Unreleased MAPREDUCE-3004. Fix ReduceTask to not assume 'local' mode in YARN. (Hitesh Shah via acmurthy) + MAPREDUCE-3017. The Web UI shows FINISHED for killed/successful/failed jobs. + (mahadev) + Release 0.22.0 - Unreleased INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java index 46cb74b93f7..2e739a98b99 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java @@ -18,40 +18,120 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmapp; +import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.event.EventHandler; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.ApplicationsStore; import org.apache.hadoop.yarn.server.resourcemanager.recovery.ApplicationsStore.ApplicationStore; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; +/** + * The read interface to an Application in the ResourceManager. Take a + * look at {@link RMAppImpl} for its implementation. This interface + * exposes methods to access various updates in application status/report. + */ public interface RMApp extends EventHandler{ + /** + * The application id for this {@link RMApp}. + * @return the {@link ApplicationId} for this {@link RMApp}. + */ ApplicationId getApplicationId(); + /** + * The current state of the {@link RMApp}. + * @return the current state {@link RMAppState} for this application. + */ RMAppState getState(); + /** + * The user who submitted this application. + * @return the user who submitted the application. + */ String getUser(); + /** + * Progress of application. + * @return the progress of the {@link RMApp}. + */ float getProgress(); + /** + * {@link RMApp} can have multiple application attempts {@link RMAppAttempt}. + * This method returns the {@link RMAppAttempt} corresponding to + * {@link ApplicationAttemptId}. + * @param appAttemptId the application attempt id + * @return the {@link RMAppAttempt} corresponding to the {@link ApplicationAttemptId}. + */ RMAppAttempt getRMAppAttempt(ApplicationAttemptId appAttemptId); + /** + * Each Application is submitted to a queue decided by {@link + * ApplicationSubmissionContext#setQueue(String)}. + * This method returns the queue to which an application was submitted. + * @return the queue to which the application was submitted to. + */ String getQueue(); + /** + * The name of the application as set in {@link + * ApplicationSubmissionContext#setApplicationName(String)}. + * @return the name of the application. + */ String getName(); + /** + * {@link RMApp} can have multiple application attempts {@link RMAppAttempt}. + * This method returns the current {@link RMAppAttempt}. + * @return the current {@link RMAppAttempt} + */ RMAppAttempt getCurrentAppAttempt(); + /** + * To get the status of an application in the RM, this method can be used. + * @return the {@link ApplicationReport} detailing the status of the application. + */ ApplicationReport createAndGetApplicationReport(); + /** + * Application level metadata is stored in {@link ApplicationStore} whicn + * can persist the information. + * @return the {@link ApplicationStore} for this {@link RMApp}. + */ ApplicationStore getApplicationStore(); + /** + * The finish time of the {@link RMApp} + * @return the finish time of the application., + */ long getFinishTime(); + /** + * the start time of the application. + * @return the start time of the application. + */ long getStartTime(); + /** + * The tracking url for the application master. + * @return the tracking url for the application master. + */ String getTrackingUrl(); + /** + * the diagnostics information for the application master. + * @return the diagnostics information for the application master. + */ StringBuilder getDiagnostics(); + + /** + * The final state of the AM when unregistering as in + * {@link FinishApplicationMasterRequest#setFinalState(String)}. + * @return the final state of the AM as set in + * {@link FinishApplicationMasterRequest#setFinalState(String)}. + */ + String getAMFinalState(); } diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index 65ee9945e29..9246d1838c7 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -40,11 +40,10 @@ import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService; -import org.apache.hadoop.yarn.server.resourcemanager.RMContext; -import org.apache.hadoop.yarn.server.resourcemanager.recovery.ApplicationsStore.ApplicationStore; import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEvent; import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEventType; -import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AMLivelinessMonitor; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.ApplicationsStore.ApplicationStore; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType; @@ -194,6 +193,19 @@ public class RMAppImpl implements RMApp { return this.applicationId; } + @Override + public String getAMFinalState() { + this.readLock.lock(); + try { + if (currentAttempt != null) { + return currentAttempt.getAMFinalState(); + } + return "UNKNOWN"; + } finally { + this.readLock.unlock(); + } + } + @Override public RMAppState getState() { this.readLock.lock(); diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java index 98025fb6ca8..70747deacba 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java @@ -26,33 +26,103 @@ import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.EventHandler; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +/** + * Interface to an Application Attempt in the Resource Manager. + * A {@link RMApp} can have multiple app attempts based on + * {@link YarnConfiguration#RM_AM_MAX_RETRIES}. For specific + * implementation take a look at {@link RMAppAttemptImpl}. + */ public interface RMAppAttempt extends EventHandler{ + /** + * Get the application attempt id for this {@link RMAppAttempt}. + * @return the {@link ApplicationAttemptId} for this RM attempt. + */ ApplicationAttemptId getAppAttemptId(); + /** + * The state of the {@link RMAppAttempt}. + * @return the state {@link RMAppAttemptState} of this {@link RMAppAttempt} + */ RMAppAttemptState getAppAttemptState(); - + + /** + * The host on which the {@link RMAppAttempt} is running/ran on. + * @return the host on which the {@link RMAppAttempt} ran/is running on. + */ String getHost(); + /** + * The rpc port of the {@link RMAppAttempt}. + * @return the rpc port of the {@link RMAppAttempt} to which the clients can connect + * to. + */ int getRpcPort(); + /** + * The url at which the status of the application attempt can be accessed. + * @return the url at which the status of the attempt can be accessed. + */ String getTrackingUrl(); + /** + * The token required by the clients to talk to the application attempt + * @return the token required by the clients to talk to the application attempt + */ String getClientToken(); + /** + * Diagnostics information for the application attempt. + * @return diagnostics information for the application attempt. + */ StringBuilder getDiagnostics(); + /** + * Progress for the application attempt. + * @return the progress for this {@link RMAppAttempt} + */ float getProgress(); + /** + * The final state set by the AM. + * @return the final state that is set by the AM when unregistering itself. + */ + String getAMFinalState(); + + /** + * Nodes on which the containers for this {@link RMAppAttempt} ran. + * @return the set of nodes that ran any containers from this {@link RMAppAttempt} + */ Set getRanNodes(); + /** + * Return a list of the last set of finished containers, resetting the + * finished containers to empty. + * @return the list of just finished containers, re setting the finished containers. + */ List pullJustFinishedContainers(); + /** + * Return the list of last set of finished containers. This does not reset the + * finished containers. + * @return the list of just finished contianers, this does not reset the + * finished containers. + */ List getJustFinishedContainers(); + /** + * The container on which the Application Master is running. + * @return the {@link Container} on which the application master is running. + */ Container getMasterContainer(); + /** + * The application submission context for this {@link RMAppAttempt}. + * @return the application submission context for this Application. + */ ApplicationSubmissionContext getSubmissionContext(); } diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 12eca4d82f3..7c6357defab 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -263,6 +263,16 @@ public class RMAppAttemptImpl implements RMAppAttempt { public ApplicationSubmissionContext getSubmissionContext() { return this.submissionContext; } + + @Override + public String getAMFinalState() { + this.readLock.lock(); + try { + return this.finalState; + } finally { + this.readLock.unlock(); + } + } @Override public RMAppAttemptState getAppAttemptState() { @@ -413,7 +423,8 @@ public class RMAppAttemptImpl implements RMAppAttempt { } private static final class AttemptStartedTransition extends BaseTransition { - @Override + @SuppressWarnings("unchecked") + @Override public void transition(RMAppAttemptImpl appAttempt, RMAppAttemptEvent event) { diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlock.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlock.java index afdec298a1d..8d2d0957db1 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlock.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlock.java @@ -23,6 +23,7 @@ import static org.apache.hadoop.yarn.webapp.view.JQueryUI._PROGRESSBAR; import static org.apache.hadoop.yarn.webapp.view.JQueryUI._PROGRESSBAR_VALUE; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY; @@ -69,7 +70,8 @@ class AppsBlock extends HtmlBlock { td(app.getUser().toString()). td(app.getName().toString()). td(app.getQueue().toString()). - td(app.getState().toString()). + td(app.getState() == RMAppState.FINISHED ? app.getAMFinalState() : + app.getState().toString()). td(). br().$title(percent)._(). // for sorting div(_PROGRESSBAR). diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java index cbbf8af9ca4..234f93e2f0c 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java @@ -29,6 +29,7 @@ import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.util.Apps; @@ -88,7 +89,9 @@ public class RmController extends Controller { ResponseInfo info = info("Application Overview"). _("User:", app.getUser()). _("Name:", app.getName()). - _("State:", app.getState()). + _("State:", (app.getState() == RMAppState.FINISHED ? + app.getAMFinalState() : app.getState().toString()) + ). _("Started:", Times.format(app.getStartTime())). _("Elapsed:", StringUtils.formatTime( Times.elapsed(app.getStartTime(), app.getFinishTime()))). diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java index fcc2c08344d..01dd078b772 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java @@ -209,6 +209,11 @@ public abstract class MockAsm extends MockApps { public void handle(RMAppEvent event) { throw new UnsupportedOperationException("Not supported yet."); } + + @Override + public String getAMFinalState() { + throw new UnsupportedOperationException("Not supported yet."); + } } public static RMApp newApplication(int i) { diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java index 6df30629ea3..4fb6486c2c7 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java @@ -163,6 +163,11 @@ public class MockRMApp implements RMApp { } public void handle(RMAppEvent event) { + } + + @Override + public String getAMFinalState() { + return "UNKNOWN"; }; }