From 3dc10a25d9f7eba6cdca2e86fca8cf6dc85b33ff Mon Sep 17 00:00:00 2001 From: Thomas Graves Date: Mon, 4 Jun 2012 17:42:45 +0000 Subject: [PATCH] merge -r 1346046:1346047 from trunk. FIXES: MAPREDUCE-3350 git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1346048 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 + .../mapreduce/v2/hs/webapp/HsJobBlock.java | 3 +- .../server/resourcemanager/rmapp/RMApp.java | 9 + .../resourcemanager/rmapp/RMAppImpl.java | 12 + .../rmapp/attempt/RMAppAttempt.java | 8 +- .../rmapp/attempt/RMAppAttemptImpl.java | 15 +- .../resourcemanager/webapp/AppBlock.java | 55 +++- .../webapp/JAXBContextResolver.java | 5 +- .../resourcemanager/webapp/RMWebServices.java | 30 +++ .../applicationsmanager/MockAsm.java | 5 + .../resourcemanager/rmapp/MockRMApp.java | 11 + .../webapp/TestRMWebServicesApps.java | 240 +++++++++++++++++- .../src/site/apt/ResourceManagerRest.apt.vm | 124 +++++++++ 13 files changed, 507 insertions(+), 13 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 8d8d27587ae..f61c314ef83 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -443,6 +443,9 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-4302. NM goes down if error encountered during log aggregation (Daryn Sharp via bobby) + MAPREDUCE-3350. Per-app RM page should have the list of application-attempts + like on the app JHS page (Jonathon Eagles via tgraves) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsJobBlock.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsJobBlock.java index 972b295cae6..b21218e8222 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsJobBlock.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsJobBlock.java @@ -135,10 +135,11 @@ public class HsJobBlock extends HtmlBlock { th(_TH, "Node"). th(_TH, "Logs"). _(); + boolean odd = false; for (AMInfo amInfo : amInfos) { AMAttemptInfo attempt = new AMAttemptInfo(amInfo, job.getId(), job.getUserName(), "", ""); - table.tr(). + table.tr((odd = !odd) ? _ODD : _EVEN). td(String.valueOf(attempt.getAttemptId())). td(new Date(attempt.getStartTime()).toString()). td().a(".nodelink", url("http://", attempt.getNodeHttpAddress()), diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java index 46df0c71ee1..63ff84422ab 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java @@ -20,6 +20,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmapp; import java.util.Collection; +import java.util.Map; + import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; @@ -93,6 +95,13 @@ public interface RMApp extends EventHandler { */ RMAppAttempt getCurrentAppAttempt(); + /** + * {@link RMApp} can have multiple application attempts {@link RMAppAttempt}. + * This method returns the all {@link RMAppAttempt}s for the RMApp. + * @return all {@link RMAppAttempt}s for the RMApp. + */ + Map getAppAttempts(); + /** * To get the status of an application in the RM, this method can be used. * If full access is not allowed then the following fields in the report diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index 43fc99196eb..1cf1ca293cc 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmapp; import java.util.Collection; +import java.util.Collections; import java.util.EnumSet; import java.util.HashSet; import java.util.LinkedHashMap; @@ -311,6 +312,17 @@ public class RMAppImpl implements RMApp { } } + @Override + public Map getAppAttempts() { + this.readLock.lock(); + + try { + return Collections.unmodifiableMap(this.attempts); + } finally { + this.readLock.unlock(); + } + } + @Override public ApplicationStore getApplicationStore() { return this.appStore; diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java index 7c5d1810106..57d78e9816f 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java @@ -145,9 +145,15 @@ public interface RMAppAttempt extends EventHandler { */ ApplicationSubmissionContext getSubmissionContext(); - /* + /** * Get application container and resource usage information. * @return an ApplicationResourceUsageReport object. */ ApplicationResourceUsageReport getApplicationResourceUsageReport(); + + /** + * the start time of the application. + * @return the start time of the application. + */ + long getStartTime(); } diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 334f7977a6c..151c8156f7c 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -119,7 +119,8 @@ public class RMAppAttemptImpl implements RMAppAttempt { private int rpcPort; private String origTrackingUrl = "N/A"; private String proxiedTrackingUrl = "N/A"; - + private long startTime = 0; + // Set to null initially. Will eventually get set // if an RMAppAttemptUnregistrationEvent occurs private FinalApplicationStatus finalStatus = null; @@ -543,6 +544,8 @@ public class RMAppAttemptImpl implements RMAppAttempt { public void transition(RMAppAttemptImpl appAttempt, RMAppAttemptEvent event) { + appAttempt.startTime = System.currentTimeMillis(); + // Register with the ApplicationMasterService appAttempt.masterService .registerAppAttempt(appAttempt.applicationAttemptId); @@ -912,4 +915,14 @@ public class RMAppAttemptImpl implements RMAppAttempt { return RMAppAttemptState.RUNNING; } } + + @Override + public long getStartTime() { + this.readLock.lock(); + try { + return this.startTime; + } finally { + this.readLock.unlock(); + } + } } diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java index 54ac79bc887..3dcd2f0268b 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java @@ -20,6 +20,13 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp; import static org.apache.hadoop.yarn.util.StringHelper.join; import static org.apache.hadoop.yarn.webapp.YarnWebParams.APPLICATION_ID; +import static org.apache.hadoop.yarn.webapp.view.JQueryUI._EVEN; +import static org.apache.hadoop.yarn.webapp.view.JQueryUI._INFO_WRAP; +import static org.apache.hadoop.yarn.webapp.view.JQueryUI._ODD; +import static org.apache.hadoop.yarn.webapp.view.JQueryUI._TH; + + +import java.util.Collection; import com.google.inject.Inject; @@ -29,19 +36,23 @@ import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.util.Apps; import org.apache.hadoop.yarn.util.Times; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE; import org.apache.hadoop.yarn.webapp.view.HtmlBlock; import org.apache.hadoop.yarn.webapp.view.InfoBlock; -import org.apache.hadoop.yarn.webapp.ResponseInfo; public class AppBlock extends HtmlBlock { private ApplicationACLsManager aclsManager; - + @Inject AppBlock(ResourceManager rm, ViewContext ctx, ApplicationACLsManager aclsManager) { super(ctx); @@ -88,7 +99,7 @@ public class AppBlock extends HtmlBlock { setTitle(join("Application ", aid)); - ResponseInfo info = info("Application Overview"). + info("Application Overview"). _("User:", app.getUser()). _("Name:", app.getName()). _("State:", app.getState()). @@ -99,12 +110,40 @@ public class AppBlock extends HtmlBlock { _("Tracking URL:", !app.isTrackingUrlReady() ? "#" : app.getTrackingUrlPretty(), app.getTrackingUI()). _("Diagnostics:", app.getNote()); - if (app.amContainerLogsExist()) { - info._("AM container logs:", app.getAMContainerLogs(), app.getAMContainerLogs()); - } else { - info._("AM container logs:", ""); + + Collection attempts = rmApp.getAppAttempts().values(); + String amString = + attempts.size() == 1 ? "ApplicationMaster" : "ApplicationMasters"; + + DIV div = html. + _(InfoBlock.class). + div(_INFO_WRAP); + // MRAppMasters Table + TABLE> table = div.table("#app"); + table. + tr(). + th(amString). + _(). + tr(). + th(_TH, "Attempt Number"). + th(_TH, "Start Time"). + th(_TH, "Node"). + th(_TH, "Logs"). + _(); + + boolean odd = false; + for (RMAppAttempt attempt : attempts) { + AppAttemptInfo attemptInfo = new AppAttemptInfo(attempt); + table.tr((odd = !odd) ? _ODD : _EVEN). + td(String.valueOf(attemptInfo.getAttemptId())). + td(Times.format(attemptInfo.getStartTime())). + td().a(".nodelink", url("http://", attemptInfo.getNodeHttpAddress()), + attemptInfo.getNodeHttpAddress())._(). + td().a(".logslink", url(attemptInfo.getLogsLink()), "logs")._(). + _(); } - html._(InfoBlock.class); + table._(); + div._(); } } diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java index c7ce9c87e44..12e77a7c49c 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java @@ -31,6 +31,8 @@ import javax.ws.rs.ext.Provider; import javax.xml.bind.JAXBContext; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerQueueInfo; @@ -53,7 +55,8 @@ public class JAXBContextResolver implements ContextResolver { private final Set types; // you have to specify all the dao classes here - private final Class[] cTypes = { AppInfo.class, ClusterInfo.class, + private final Class[] cTypes = { AppInfo.class, AppAttemptInfo.class, + AppAttemptsInfo.class, ClusterInfo.class, CapacitySchedulerQueueInfo.class, FifoSchedulerInfo.class, SchedulerTypeInfo.class, NodeInfo.class, UserMetricsInfo.class, CapacitySchedulerInfo.class, ClusterMetricsInfo.class, diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index 857367a8f45..334d3a8d331 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -45,11 +45,14 @@ import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerInfo; @@ -385,4 +388,31 @@ public class RMWebServices { return new AppInfo(app, hasAccess(app, hsr)); } + @GET + @Path("/apps/{appid}/appattempts") + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + public AppAttemptsInfo getAppAttempts(@PathParam("appid") String appId) { + + init(); + if (appId == null || appId.isEmpty()) { + throw new NotFoundException("appId, " + appId + ", is empty or null"); + } + ApplicationId id; + id = ConverterUtils.toApplicationId(recordFactory, appId); + if (id == null) { + throw new NotFoundException("appId is null"); + } + RMApp app = rm.getRMContext().getRMApps().get(id); + if (app == null) { + throw new NotFoundException("app with id: " + appId + " not found"); + } + + AppAttemptsInfo appAttemptsInfo = new AppAttemptsInfo(); + for (RMAppAttempt attempt : app.getAppAttempts().values()) { + AppAttemptInfo attemptInfo = new AppAttemptInfo(attempt); + appAttemptsInfo.add(attemptInfo); + } + + return appAttemptsInfo; + } } diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java index db203d29ea8..81aba399748 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager; import java.util.Collection; import java.util.List; +import java.util.Map; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.yarn.MockApps; @@ -189,6 +190,10 @@ public abstract class MockAsm extends MockApps { throw new UnsupportedOperationException("Not supported yet."); } @Override + public Map getAppAttempts() { + throw new UnsupportedOperationException("Not supported yet."); + } + @Override public ApplicationStore getApplicationStore() { throw new UnsupportedOperationException("Not supported yet."); } diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java index 983b29c9253..e5ea2b86f4f 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java @@ -20,6 +20,9 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmapp; import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.Map; + import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -112,6 +115,14 @@ public class MockRMApp implements RMApp { this.name = name; } + @Override + public Map getAppAttempts() { + Map attempts = + new LinkedHashMap(); + attempts.put(attempt.getAppAttemptId(), attempt); + return attempts; + } + @Override public RMAppAttempt getCurrentAppAttempt() { return attempt; diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index 3a30b928873..427dcf8345f 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -23,6 +23,7 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.StringReader; +import java.util.Collection; import javax.ws.rs.core.MediaType; import javax.xml.parsers.DocumentBuilder; @@ -31,13 +32,18 @@ import javax.xml.parsers.DocumentBuilderFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppFailedAttemptEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.WebServicesTestUtils; @@ -73,7 +79,9 @@ public class TestRMWebServicesApps extends JerseyTest { bind(JAXBContextResolver.class); bind(RMWebServices.class); bind(GenericExceptionHandler.class); - rm = new MockRM(new Configuration()); + Configuration conf = new Configuration(); + conf.setInt(YarnConfiguration.RM_AM_MAX_RETRIES, 2); + rm = new MockRM(conf); bind(ResourceManager.class).toInstance(rm); bind(RMContext.class).toInstance(rm.getRMContext()); bind(ApplicationACLsManager.class).toInstance( @@ -835,4 +843,234 @@ public class TestRMWebServicesApps extends JerseyTest { amContainerLogs.endsWith("/" + app.getUser())); } + @Test + public void testAppAttempts() throws JSONException, Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); + RMApp app1 = rm.submitApp(1024, "testwordcount", "user1"); + amNodeManager.nodeHeartbeat(true); + testAppAttemptsHelper(app1.getApplicationId().toString(), app1, + MediaType.APPLICATION_JSON); + rm.stop(); + } + + @Test + public void testMultipleAppAttempts() throws JSONException, Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); + RMApp app1 = rm.submitApp(1024, "testwordcount", "user1"); + amNodeManager.nodeHeartbeat(true); + int maxRetries = rm.getConfig().getInt(YarnConfiguration.RM_AM_MAX_RETRIES, + YarnConfiguration.DEFAULT_RM_AM_MAX_RETRIES); + int retriesLeft = maxRetries; + while (--retriesLeft > 0) { + RMAppEvent event = + new RMAppFailedAttemptEvent(app1.getApplicationId(), + RMAppEventType.ATTEMPT_FAILED, ""); + app1.handle(event); + rm.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED); + amNodeManager.nodeHeartbeat(true); + } + assertEquals("incorrect number of attempts", maxRetries, + app1.getAppAttempts().values().size()); + testAppAttemptsHelper(app1.getApplicationId().toString(), app1, + MediaType.APPLICATION_JSON); + rm.stop(); + } + + @Test + public void testAppAttemptsSlash() throws JSONException, Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); + RMApp app1 = rm.submitApp(1024); + amNodeManager.nodeHeartbeat(true); + testAppAttemptsHelper(app1.getApplicationId().toString() + "/", app1, + MediaType.APPLICATION_JSON); + rm.stop(); + } + + @Test + public void testAppAttemtpsDefault() throws JSONException, Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); + RMApp app1 = rm.submitApp(1024); + amNodeManager.nodeHeartbeat(true); + testAppAttemptsHelper(app1.getApplicationId().toString() + "/", app1, ""); + rm.stop(); + } + + @Test + public void testInvalidAppAttempts() throws JSONException, Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); + rm.submitApp(1024); + amNodeManager.nodeHeartbeat(true); + WebResource r = resource(); + + try { + r.path("ws").path("v1").path("cluster").path("apps") + .path("application_invalid_12").accept(MediaType.APPLICATION_JSON) + .get(JSONObject.class); + fail("should have thrown exception on invalid appid"); + } catch (UniformInterfaceException ue) { + ClientResponse response = ue.getResponse(); + + assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject msg = response.getEntity(JSONObject.class); + JSONObject exception = msg.getJSONObject("RemoteException"); + assertEquals("incorrect number of elements", 3, exception.length()); + String message = exception.getString("message"); + String type = exception.getString("exception"); + String classname = exception.getString("javaClassName"); + WebServicesTestUtils.checkStringMatch("exception message", + "For input string: \"invalid\"", message); + WebServicesTestUtils.checkStringMatch("exception type", + "NumberFormatException", type); + WebServicesTestUtils.checkStringMatch("exception classname", + "java.lang.NumberFormatException", classname); + + } finally { + rm.stop(); + } + } + + @Test + public void testNonexistAppAttempts() throws JSONException, Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); + rm.submitApp(1024, "testwordcount", "user1"); + amNodeManager.nodeHeartbeat(true); + WebResource r = resource(); + + try { + r.path("ws").path("v1").path("cluster").path("apps") + .path("application_00000_0099").accept(MediaType.APPLICATION_JSON) + .get(JSONObject.class); + fail("should have thrown exception on invalid appid"); + } catch (UniformInterfaceException ue) { + ClientResponse response = ue.getResponse(); + + assertEquals(Status.NOT_FOUND, response.getClientResponseStatus()); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + + JSONObject msg = response.getEntity(JSONObject.class); + JSONObject exception = msg.getJSONObject("RemoteException"); + assertEquals("incorrect number of elements", 3, exception.length()); + String message = exception.getString("message"); + String type = exception.getString("exception"); + String classname = exception.getString("javaClassName"); + WebServicesTestUtils.checkStringMatch("exception message", + "java.lang.Exception: app with id: application_00000_0099 not found", + message); + WebServicesTestUtils.checkStringMatch("exception type", + "NotFoundException", type); + WebServicesTestUtils.checkStringMatch("exception classname", + "org.apache.hadoop.yarn.webapp.NotFoundException", classname); + } finally { + rm.stop(); + } + } + + public void testAppAttemptsHelper(String path, RMApp app, String media) + throws JSONException, Exception { + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("apps").path(path).path("appattempts").accept(media) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + JSONObject jsonAppAttempts = json.getJSONObject("appAttempts"); + assertEquals("incorrect number of elements", 1, jsonAppAttempts.length()); + JSONArray jsonArray = jsonAppAttempts.getJSONArray("appAttempt"); + + Collection attempts = app.getAppAttempts().values(); + assertEquals("incorrect number of elements", attempts.size(), + jsonArray.length()); + + // Verify these parallel arrays are the same + int i = 0; + for (RMAppAttempt attempt : attempts) { + verifyAppAttemptsInfo(jsonArray.getJSONObject(i), attempt); + ++i; + } + } + + @Test + public void testAppAttemptsXML() throws JSONException, Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); + RMApp app1 = rm.submitApp(1024, "testwordcount", "user1"); + amNodeManager.nodeHeartbeat(true); + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("apps").path(app1.getApplicationId().toString()) + .path("appattempts").accept(MediaType.APPLICATION_XML) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType()); + String xml = response.getEntity(String.class); + + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + DocumentBuilder db = dbf.newDocumentBuilder(); + InputSource is = new InputSource(); + is.setCharacterStream(new StringReader(xml)); + Document dom = db.parse(is); + NodeList nodes = dom.getElementsByTagName("appAttempts"); + assertEquals("incorrect number of elements", 1, nodes.getLength()); + NodeList attempt = dom.getElementsByTagName("appAttempt"); + assertEquals("incorrect number of elements", 1, attempt.getLength()); + verifyAppAttemptsXML(attempt, app1.getCurrentAppAttempt()); + rm.stop(); + } + + public void verifyAppAttemptsXML(NodeList nodes, RMAppAttempt appAttempt) + throws JSONException, Exception { + + for (int i = 0; i < nodes.getLength(); i++) { + Element element = (Element) nodes.item(i); + + verifyAppAttemptInfoGeneric(appAttempt, + WebServicesTestUtils.getXmlInt(element, "id"), + WebServicesTestUtils.getXmlLong(element, "startTime"), + WebServicesTestUtils.getXmlString(element, "containerId"), + WebServicesTestUtils.getXmlString(element, "nodeHttpAddress"), + WebServicesTestUtils.getXmlString(element, "nodeId"), + WebServicesTestUtils.getXmlString(element, "logsLink")); + } + } + + public void verifyAppAttemptsInfo(JSONObject info, RMAppAttempt appAttempt) + throws JSONException, Exception { + + assertEquals("incorrect number of elements", 6, info.length()); + + verifyAppAttemptInfoGeneric(appAttempt, info.getInt("id"), + info.getLong("startTime"), info.getString("containerId"), + info.getString("nodeHttpAddress"), info.getString("nodeId"), + info.getString("logsLink")); + } + + public void verifyAppAttemptInfoGeneric(RMAppAttempt appAttempt, int id, + long startTime, String containerId, String nodeHttpAddress, String nodeId, + String logsLink) + throws JSONException, Exception { + + assertEquals("id doesn't match", appAttempt.getAppAttemptId() + .getAttemptId(), id); + assertEquals("startedTime doesn't match", appAttempt.getStartTime(), + startTime); + WebServicesTestUtils.checkStringMatch("containerId", appAttempt + .getMasterContainer().getId().toString(), containerId); + WebServicesTestUtils.checkStringMatch("nodeHttpAddress", appAttempt + .getMasterContainer().getNodeHttpAddress(), nodeHttpAddress); + WebServicesTestUtils.checkStringMatch("nodeId", appAttempt + .getMasterContainer().getNodeId().toString(), nodeId); + assertTrue("logsLink doesn't match", + logsLink.startsWith("http://")); + assertTrue("logsLink doesn't contain user info", + logsLink.endsWith("/" + appAttempt.getSubmissionContext().getUser())); + } + } + diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm index 977bd52b684..36600b8aa5f 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm @@ -1236,6 +1236,130 @@ _01_000001 +---+ +* Cluster Application Attempts API + + With the application attempts API, you can obtain a collection of resources that represent an application attempt. When you run a GET operation on this resource, you obtain a collection of App Attempt Objects. + +** URI + +------ + * http:///ws/v1/cluster/apps/{appid}/appattempts +------ + +** HTTP Operations Supported + +------ + * GET +------ + +** Query Parameters Supported + +------ + None +------ + +** Elements of the object + + When you make a request for the list of app attempts, the information will be returned as an array of app attempt objects. + + appAttempts: + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| appAttempt | array of app attempt objects(JSON)/zero or more app attempt objects(XML) | The collection of app attempt objects | +*---------------+--------------+--------------------------------+ + +** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| id | string | The app attempt id | +*---------------+--------------+--------------------------------+ +| nodeId | string | The node id of the node the attempt ran on| +*---------------+--------------+--------------------------------+ +| nodeHttpAddress | string | The node http address of the node the attempt ran on| +*---------------+--------------+--------------------------------+ +| logsLink | string | The http link to the app attempt logs | +*---------------+--------------+--------------------------------+ +| containerId | string | The id of the container for the app attempt | +*---------------+--------------+--------------------------------+ +| startTime | long | The start time of the attempt (in ms since epoch)| +*---------------+--------------+--------------------------------+ + +** Response Examples + + <> + + HTTP Request: + +------ + GET http:///ws/v1/cluster/apps/application_1326821518301_0005/appattempts +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "appAttempts" : { + "appAttempt" : [ + { + "nodeId" : "host.domain.com:8041", + "nodeHttpAddress" : "host.domain.com:8042", + "startTime" : 1326381444693, + "id" : 1, + "logsLink" : "http://host.domain.com:8042/node/containerlogs/container_1326821518301_0005_01_000001/user1", + "containerId" : "container_1326821518301_0005_01_000001" + } + ] + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///ws/v1/cluster/apps/application_1326821518301_0005/appattempts + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 575 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + + host.domain.com:8042 + host.domain.com:8041 + 1 + 1326381444693 + container_1326821518301_0005_01_000001 + http://host.domain.com:8042/node/containerlogs/container_1326821518301_0005_01_000001/user1 + + ++---+ + * Cluster Nodes API With the Nodes API, you can obtain a collection of resources, each of which represents a node. When you run a GET operation on this resource, you obtain a collection of Node Objects.