diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 9113eb4bd9d..4075c5e8d8a 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -1259,6 +1259,9 @@ Release 2.7.3 - UNRELEASED YARN-4598. Invalid event: RESOURCE_FAILED at CONTAINER_CLEANEDUP_AFTER_KILL (tangshangwen via jlowe) + YARN-4428. Redirect RM page to AHS page when AHS turned on and RM page is + not available (Chang Li via jlowe) + Release 2.7.2 - 2016-01-25 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 1ff4d75de96..1648349f74c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -58,6 +58,7 @@ import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.factories.RecordFactory; @@ -628,6 +629,21 @@ private void setTrackingUrlToRMAppPage(RMAppAttemptState stateToBeStored) { } } + private void setTrackingUrlToAHSPage(RMAppAttemptState stateToBeStored) { + originalTrackingUrl = pjoin( + WebAppUtils.getHttpSchemePrefix(conf) + + WebAppUtils.getAHSWebAppURLWithoutScheme(conf), + "applicationhistory", "app", getAppAttemptId().getApplicationId()); + switch (stateToBeStored) { + case KILLED: + case FAILED: + proxiedTrackingUrl = originalTrackingUrl; + break; + default: + break; + } + } + private void invalidateAMHostAndPort() { this.host = "N/A"; this.rpcPort = -1; @@ -1211,7 +1227,12 @@ private void rememberTargetTransitionsAndStoreState(RMAppAttemptEvent event, String diags = null; // don't leave the tracking URL pointing to a non-existent AM - setTrackingUrlToRMAppPage(stateToBeStored); + if (conf.getBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, + YarnConfiguration.DEFAULT_APPLICATION_HISTORY_ENABLED)) { + setTrackingUrlToAHSPage(stateToBeStored); + } else { + setTrackingUrlToRMAppPage(stateToBeStored); + } String finalTrackingUrl = getOriginalTrackingUrl(); FinalApplicationStatus finalStatus = null; int exitStatus = ContainerExitStatus.INVALID; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java index 91c3d72e06c..106065bac81 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java @@ -26,6 +26,7 @@ import org.apache.hadoop.yarn.api.ApplicationBaseProtocol; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.util.RMHAUtils; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.WebApp; @@ -122,4 +123,8 @@ public String getHAZookeeperConnectionState() { return rm.getRMContext().getRMAdminService() .getHAZookeeperConnectionState(); } + + public RMContext getRMContext() { + return rm.getRMContext(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java index 9f35b6ce566..74137480b43 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java @@ -18,6 +18,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp; +import static org.apache.hadoop.yarn.util.StringHelper.pjoin; + import java.io.IOException; import java.io.PrintWriter; import java.net.InetSocketAddress; @@ -35,9 +37,19 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.http.HtmlQuoting; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils; +import org.apache.hadoop.yarn.util.Apps; +import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.webapp.YarnWebParams; +import org.apache.hadoop.yarn.webapp.util.WebAppUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.google.common.collect.Sets; import com.google.inject.Injector; @@ -45,6 +57,8 @@ @Singleton public class RMWebAppFilter extends GuiceContainer { + private static final Logger LOG = + LoggerFactory.getLogger(RMWebAppFilter.class); private Injector injector; /** @@ -56,6 +70,8 @@ public class RMWebAppFilter extends GuiceContainer { private static final Set NON_REDIRECTED_URIS = Sets.newHashSet( "/conf", "/stacks", "/logLevel", "/logs"); private String path; + private boolean ahsEnabled; + private String ahsPageURLPrefix; private static final int BASIC_SLEEP_TIME = 5; private static final int MAX_SLEEP_TIME = 5 * 60; private static final Random randnum = new Random(); @@ -76,6 +92,13 @@ public RMWebAppFilter(Injector injector, Configuration conf) { path = YarnConfiguration.useHttps(conf) ? "https://" + path : "http://" + path; + ahsEnabled = conf.getBoolean( + YarnConfiguration.APPLICATION_HISTORY_ENABLED, + YarnConfiguration.DEFAULT_APPLICATION_HISTORY_ENABLED); + ahsPageURLPrefix = pjoin( + WebAppUtils.getHttpSchemePrefix(conf) + + WebAppUtils.getAHSWebAppURLWithoutScheme( + conf), "applicationhistory"); } @Override @@ -137,11 +160,78 @@ && shouldRedirect(rmWebApp, uri)) { } } return; + } else if (ahsEnabled) { + String ahsRedirectUrl = ahsRedirectPath(uri, rmWebApp); + if(ahsRedirectUrl != null) { + response.setHeader("Location", ahsRedirectUrl); + response.setStatus(HttpServletResponse.SC_TEMPORARY_REDIRECT); + return; + } } super.doFilter(request, response, chain); } + private String ahsRedirectPath(String uri, RMWebApp rmWebApp) { + // TODO: Commonize URL parsing code. Will be done in YARN-4642. + String redirectPath = null; + if(uri.contains("/cluster/")) { + String[] parts = uri.split("/"); + if(parts.length > 3) { + RMContext context = rmWebApp.getRMContext(); + String type = parts[2]; + ApplicationId appId = null; + ApplicationAttemptId appAttemptId = null; + ContainerId containerId = null; + switch(type){ + case "app": + try { + appId = Apps.toAppID(parts[3]); + } catch (YarnRuntimeException | NumberFormatException e) { + LOG.debug("Error parsing {} as an ApplicationId", + parts[3], e); + return redirectPath; + } + if(!context.getRMApps().containsKey(appId)) { + redirectPath = pjoin(ahsPageURLPrefix, "app", appId); + } + break; + case "appattempt": + try{ + appAttemptId = ConverterUtils.toApplicationAttemptId(parts[3]); + } catch (IllegalArgumentException e) { + LOG.debug("Error parsing {} as an ApplicationAttemptId", + parts[3], e); + return redirectPath; + } + if(!context.getRMApps().containsKey( + appAttemptId.getApplicationId())) { + redirectPath = pjoin(ahsPageURLPrefix, + "appattempt", appAttemptId); + } + break; + case "container": + try { + containerId = ContainerId.fromString(parts[3]); + } catch (IllegalArgumentException e) { + LOG.debug("Error parsing {} as an ContainerId", + parts[3], e); + return redirectPath; + } + if(!context.getRMApps().containsKey( + containerId.getApplicationAttemptId().getApplicationId())) { + redirectPath = pjoin(ahsPageURLPrefix, + "container", containerId); + } + break; + default: + break; + } + } + } + return redirectPath; + } + private boolean shouldRedirect(RMWebApp rmWebApp, String uri) { return !uri.equals("/" + rmWebApp.wsName() + "/v1/cluster/info") && !uri.equals("/" + rmWebApp.name() + "/cluster") diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java index e7985fa078b..93f17e7fa22 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java @@ -68,6 +68,7 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.event.InlineDispatcher; @@ -137,7 +138,10 @@ public class TestRMAppAttemptTransitions { private static final String FAILED_DIAGNOSTICS = "Attempt failed by user."; private static final String RM_WEBAPP_ADDR = WebAppUtils.getResolvedRMWebAppURLWithScheme(new Configuration()); - + private static final String AHS_WEBAPP_ADDR = + WebAppUtils.getHttpSchemePrefix(new Configuration()) + + WebAppUtils.getAHSWebAppURLWithoutScheme(new Configuration()); + private boolean isSecurityEnabled; private RMContext rmContext; private RMContext spyRMContext; @@ -1090,6 +1094,78 @@ public void testLaunchedExpire() { verifyApplicationAttemptFinished(RMAppAttemptState.FAILED); } + @SuppressWarnings("unchecked") + @Test(timeout=10000) + public void testLaunchedFailWhileAHSEnabled() { + Configuration myConf = new Configuration(conf); + myConf.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, true); + ApplicationId applicationId = MockApps.newAppID(appId); + ApplicationAttemptId applicationAttemptId = + ApplicationAttemptId.newInstance(applicationId, 2); + RMAppAttempt myApplicationAttempt = + new RMAppAttemptImpl(applicationAttempt.getAppAttemptId(), + spyRMContext, scheduler,masterService, + submissionContext, myConf, false, + BuilderUtils.newResourceRequest( + RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY, + submissionContext.getResource(), 1)); + + //submit, schedule and allocate app attempt + myApplicationAttempt.handle( + new RMAppAttemptEvent(myApplicationAttempt.getAppAttemptId(), + RMAppAttemptEventType.START)); + myApplicationAttempt.handle( + new RMAppAttemptEvent( + myApplicationAttempt.getAppAttemptId(), + RMAppAttemptEventType.ATTEMPT_ADDED)); + + Container amContainer = mock(Container.class); + Resource resource = BuilderUtils.newResource(2048, 1); + when(amContainer.getId()).thenReturn( + BuilderUtils.newContainerId(myApplicationAttempt.getAppAttemptId(), 1)); + when(amContainer.getResource()).thenReturn(resource); + Allocation allocation = mock(Allocation.class); + when(allocation.getContainers()). + thenReturn(Collections.singletonList(amContainer)); + when(scheduler.allocate(any(ApplicationAttemptId.class), any(List.class), + any(List.class), any(List.class), any(List.class), any(List.class), + any(List.class))).thenReturn(allocation); + RMContainer rmContainer = mock(RMContainerImpl.class); + when(scheduler.getRMContainer(amContainer.getId())).thenReturn(rmContainer); + + myApplicationAttempt.handle( + new RMAppAttemptEvent(myApplicationAttempt.getAppAttemptId(), + RMAppAttemptEventType.CONTAINER_ALLOCATED)); + assertEquals(RMAppAttemptState.ALLOCATED_SAVING, + myApplicationAttempt.getAppAttemptState()); + myApplicationAttempt.handle( + new RMAppAttemptEvent(myApplicationAttempt.getAppAttemptId(), + RMAppAttemptEventType.ATTEMPT_NEW_SAVED)); + + // launch app attempt + myApplicationAttempt.handle( + new RMAppAttemptEvent(myApplicationAttempt.getAppAttemptId(), + RMAppAttemptEventType.LAUNCHED)); + assertEquals(YarnApplicationAttemptState.LAUNCHED, + myApplicationAttempt.createApplicationAttemptState()); + + //fail container right after launched + NodeId anyNodeId = NodeId.newInstance("host", 1234); + myApplicationAttempt.handle( + new RMAppAttemptContainerFinishedEvent( + myApplicationAttempt.getAppAttemptId(), + BuilderUtils.newContainerStatus(amContainer.getId(), + ContainerState.COMPLETE, "", 0, + amContainer.getResource()), anyNodeId)); + sendAttemptUpdateSavedEvent(myApplicationAttempt); + assertEquals(RMAppAttemptState.FAILED, + myApplicationAttempt.getAppAttemptState()); + String rmAppPageUrl = pjoin(AHS_WEBAPP_ADDR, "applicationhistory", "app", + myApplicationAttempt.getAppAttemptId().getApplicationId()); + assertEquals(rmAppPageUrl, myApplicationAttempt.getOriginalTrackingUrl()); + assertEquals(rmAppPageUrl, myApplicationAttempt.getTrackingUrl()); + } + @Test(timeout=20000) public void testRunningExpire() { Container amContainer = allocateApplicationAttempt();