From db5322695a77884f3010f77c30cb2fa3162481d3 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Thu, 26 Sep 2013 04:18:00 +0000 Subject: [PATCH] YARN-1157. Fixed ResourceManager UI to behave correctly when apps like distributed-shell do not set tracking urls. Contributed by Xuan Gong. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1526371 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../FinishApplicationMasterRequest.java | 21 +++-- .../RegisterApplicationMasterRequest.java | 21 +++-- .../rmapp/attempt/RMAppAttemptImpl.java | 12 ++- .../rmapp/attempt/TestRMAppAttemptImpl.java | 77 ---------------- .../attempt/TestRMAppAttemptTransitions.java | 87 +++++++++++++++---- 6 files changed, 115 insertions(+), 106 deletions(-) delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptImpl.java diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index e075c18a7eb..034030ed6c8 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -83,6 +83,9 @@ Release 2.1.2 - UNRELEASED YARN-49. Improve distributed shell application to work on a secure cluster. (Vinod Kumar Vavilapalli via hitesh) + YARN-1157. Fixed ResourceManager UI to behave correctly when apps like + distributed-shell do not set tracking urls. (Xuan Gong via vinodkv) + Release 2.1.1-beta - 2013-09-23 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/FinishApplicationMasterRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/FinishApplicationMasterRequest.java index 3c72568b1b4..15c36802bbe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/FinishApplicationMasterRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/FinishApplicationMasterRequest.java @@ -100,11 +100,22 @@ public abstract class FinishApplicationMasterRequest { public abstract String getTrackingUrl(); /** - * Set the tracking URLfor the ApplicationMaster - * This url if contains scheme then that will be used by resource manager - * web application proxy otherwise it will default to http. - * @param url tracking URLfor the - * ApplicationMaster + * Set the final tracking URLfor the ApplicationMaster. + * This is the web-URL to which ResourceManager or web-application proxy will + * redirect client/users once the application is finished and the + * ApplicationMaster is gone. + *

+ * If the passed url has a scheme then that will be used by the + * ResourceManager and web-application proxy, otherwise the scheme will + * default to http. + *

+ *

+ * Empty, null, "N/A" strings are all valid besides a real URL. In case an url + * isn't explicitly passed, it defaults to "N/A" on the ResourceManager. + *

+ * + * @param url + * tracking URLfor the ApplicationMaster */ @Public @Stable diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java index 05668dd16bf..0b485d18fd1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java @@ -112,11 +112,22 @@ public abstract class RegisterApplicationMasterRequest { public abstract String getTrackingUrl(); /** - * Set the tracking URL for the ApplicationMaster. - * This url if contains scheme then that will be used by resource manager - * web application proxy otherwise it will default to http. - * @param trackingUrl tracking URL for the - * ApplicationMaster + * Set the tracking URLfor the ApplicationMaster while + * it is running. This is the web-URL to which ResourceManager or + * web-application proxy will redirect client/users while the application and + * the ApplicationMaster are still running. + *

+ * If the passed url has a scheme then that will be used by the + * ResourceManager and web-application proxy, otherwise the scheme will + * default to http. + *

+ *

+ * Empty, null, "N/A" strings are all valid besides a real URL. In case an url + * isn't explicitly passed, it defaults to "N/A" on the ResourceManager. + *

+ * + * @param trackingUrl + * tracking URLfor the ApplicationMaster */ @Public @Stable diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 530b343afe2..ae3bc9f7698 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -994,7 +994,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { } } - static final class AMRegisteredTransition extends BaseTransition { + private static final class AMRegisteredTransition extends BaseTransition { @Override public void transition(RMAppAttemptImpl appAttempt, RMAppAttemptEvent event) { @@ -1003,7 +1003,8 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { = (RMAppAttemptRegistrationEvent) event; appAttempt.host = registrationEvent.getHost(); appAttempt.rpcPort = registrationEvent.getRpcport(); - appAttempt.origTrackingUrl = registrationEvent.getTrackingurl(); + appAttempt.origTrackingUrl = + sanitizeTrackingUrl(registrationEvent.getTrackingurl()); appAttempt.proxiedTrackingUrl = appAttempt.generateProxyUriWithoutScheme(appAttempt.origTrackingUrl); @@ -1138,7 +1139,8 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { RMAppAttemptUnregistrationEvent unregisterEvent = (RMAppAttemptUnregistrationEvent) event; appAttempt.diagnostics.append(unregisterEvent.getDiagnostics()); - appAttempt.origTrackingUrl = unregisterEvent.getTrackingUrl(); + appAttempt.origTrackingUrl = + sanitizeTrackingUrl(unregisterEvent.getTrackingUrl()); appAttempt.proxiedTrackingUrl = appAttempt.generateProxyUriWithoutScheme(appAttempt.origTrackingUrl); appAttempt.finalStatus = unregisterEvent.getFinalApplicationStatus(); @@ -1292,4 +1294,8 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { appAttempt.rmContext.getAMRMTokenSecretManager() .applicationMasterFinished(appAttempt.getAppAttemptId()); } + + private static String sanitizeTrackingUrl(String url) { + return (url == null || url.trim().isEmpty()) ? "N/A" : url; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptImpl.java deleted file mode 100644 index e69d867e0f4..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptImpl.java +++ /dev/null @@ -1,77 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt; - -import junit.framework.Assert; -import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; -import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.event.Dispatcher; -import org.apache.hadoop.yarn.event.EventHandler; -import org.apache.hadoop.yarn.server.resourcemanager.RMContext; -import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptRegistrationEvent; - -import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils; -import org.junit.Test; -import org.mockito.Mockito; - -public class TestRMAppAttemptImpl { - - private void testTrackingUrl(String url, boolean unmanaged) { - ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance - (ApplicationId.newInstance(1, 2), 1); - EventHandler handler = Mockito.mock(EventHandler.class); - Dispatcher dispatcher = Mockito.mock(Dispatcher.class); - Mockito.when(dispatcher.getEventHandler()).thenReturn(handler); - RMContext rmContext = Mockito.mock(RMContext.class); - Mockito.when(rmContext.getDispatcher()).thenReturn(dispatcher); - - ApplicationSubmissionContext appContext = - Mockito.mock(ApplicationSubmissionContext.class); - Mockito.when(appContext.getUnmanagedAM()).thenReturn(unmanaged); - - RMAppAttemptImpl attempt = new RMAppAttemptImpl(attemptId, rmContext, null, - null, appContext, new YarnConfiguration(), null); - RMAppAttemptRegistrationEvent event = - Mockito.mock(RMAppAttemptRegistrationEvent.class); - Mockito.when(event.getHost()).thenReturn("h"); - Mockito.when(event.getRpcport()).thenReturn(0); - Mockito.when(event.getTrackingurl()).thenReturn(url); - new RMAppAttemptImpl.AMRegisteredTransition().transition(attempt, event); - if (unmanaged) { - Assert.assertEquals(url, attempt.getTrackingUrl()); - } else { - Assert.assertNotSame(url, attempt.getTrackingUrl()); - Assert.assertTrue(attempt.getTrackingUrl().contains( - ProxyUriUtils.PROXY_SERVLET_NAME)); - Assert.assertTrue(attempt.getTrackingUrl().contains( - attemptId.getApplicationId().toString())); - } - } - - @Test - public void testTrackingUrlUnmanagedAM() { - testTrackingUrl("http://foo:8000/x", true); - } - - @Test - public void testTrackingUrlManagedAM() { - testTrackingUrl("bar:8000/x", false); - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java index 056589a8e3f..a878be11dd8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java @@ -30,14 +30,18 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.net.URI; +import java.net.URISyntaxException; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.List; +import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; @@ -85,8 +89,10 @@ import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSec import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.server.utils.BuilderUtils; +import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -261,8 +267,22 @@ public class TestRMAppAttemptTransitions { private String getProxyUrl(RMAppAttempt appAttempt) { - return pjoin(RM_WEBAPP_ADDR, "proxy", - appAttempt.getAppAttemptId().getApplicationId(), ""); + String url = null; + try { + URI trackingUri = + StringUtils.isEmpty(appAttempt.getOriginalTrackingUrl()) ? null : + ProxyUriUtils + .getUriFromAMUrl(appAttempt.getOriginalTrackingUrl()); + String proxy = WebAppUtils.getProxyHostAndPort(conf); + URI proxyUri = ProxyUriUtils.getUriFromAMUrl(proxy); + URI result = ProxyUriUtils.getProxyUri(trackingUri, proxyUri, + appAttempt.getAppAttemptId().getApplicationId()); + url = result.toASCIIString().substring( + HttpConfig.getSchemePrefix().length()); + } catch (URISyntaxException ex) { + Assert.fail(); + } + return url; } /** @@ -448,9 +468,9 @@ public class TestRMAppAttemptTransitions { assertEquals(container, applicationAttempt.getMasterContainer()); assertEquals(host, applicationAttempt.getHost()); assertEquals(rpcPort, applicationAttempt.getRpcPort()); - assertEquals(trackingUrl, applicationAttempt.getOriginalTrackingUrl()); + verifyUrl(trackingUrl, applicationAttempt.getOriginalTrackingUrl()); if (unmanagedAM) { - assertEquals("oldtrackingurl", applicationAttempt.getTrackingUrl()); + verifyUrl(trackingUrl, applicationAttempt.getTrackingUrl()); } else { assertEquals(getProxyUrl(applicationAttempt), applicationAttempt.getTrackingUrl()); @@ -468,7 +488,7 @@ public class TestRMAppAttemptTransitions { assertEquals(RMAppAttemptState.FINISHING, applicationAttempt.getAppAttemptState()); assertEquals(diagnostics, applicationAttempt.getDiagnostics()); - assertEquals(trackingUrl, applicationAttempt.getOriginalTrackingUrl()); + verifyUrl(trackingUrl, applicationAttempt.getOriginalTrackingUrl()); assertEquals(getProxyUrl(applicationAttempt), applicationAttempt.getTrackingUrl()); assertEquals(container, applicationAttempt.getMasterContainer()); @@ -487,9 +507,9 @@ public class TestRMAppAttemptTransitions { assertEquals(RMAppAttemptState.FINISHED, applicationAttempt.getAppAttemptState()); assertEquals(diagnostics, applicationAttempt.getDiagnostics()); - assertEquals(trackingUrl, applicationAttempt.getOriginalTrackingUrl()); + verifyUrl(trackingUrl, applicationAttempt.getOriginalTrackingUrl()); if (unmanagedAM) { - assertEquals("mytrackingurl", applicationAttempt.getTrackingUrl()); + verifyUrl(trackingUrl, applicationAttempt.getTrackingUrl()); } else { assertEquals(getProxyUrl(applicationAttempt), @@ -603,9 +623,7 @@ public class TestRMAppAttemptTransitions { trackingUrl, diagnostics); } - - @Test - public void testUnmanagedAMSuccess() { + private void testUnmanagedAMSuccess(String url) { unmanagedAM = true; when(submissionContext.getUnmanagedAM()).thenReturn(true); // submit AM and check it goes to LAUNCHED state @@ -615,7 +633,7 @@ public class TestRMAppAttemptTransitions { applicationAttempt.getAppAttemptId()); // launch AM - runApplicationAttempt(null, "host", 8042, "oldtrackingurl", true); + runApplicationAttempt(null, "host", 8042, url, true); // complete a container applicationAttempt.handle(new RMAppAttemptContainerAcquiredEvent( @@ -623,13 +641,12 @@ public class TestRMAppAttemptTransitions { applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent( applicationAttempt.getAppAttemptId(), mock(ContainerStatus.class))); // complete AM - String trackingUrl = "mytrackingurl"; String diagnostics = "Successful"; FinalApplicationStatus finalStatus = FinalApplicationStatus.SUCCEEDED; applicationAttempt.handle(new RMAppAttemptUnregistrationEvent( - applicationAttempt.getAppAttemptId(), trackingUrl, finalStatus, + applicationAttempt.getAppAttemptId(), url, finalStatus, diagnostics)); - testAppAttemptFinishedState(null, finalStatus, trackingUrl, diagnostics, 1, + testAppAttemptFinishedState(null, finalStatus, url, diagnostics, 1, true); } @@ -824,12 +841,42 @@ public class TestRMAppAttemptTransitions { "Killed by user"); } + @Test + public void testTrackingUrlUnmanagedAM() { + testUnmanagedAMSuccess("oldTrackingUrl"); + } @Test - public void testNoTrackingUrl() { + public void testEmptyTrackingUrlUnmanagedAM() { + testUnmanagedAMSuccess(""); + } + + @Test + public void testNullTrackingUrlUnmanagedAM() { + testUnmanagedAMSuccess(null); + } + + @Test + public void testManagedAMWithTrackingUrl() { + testTrackingUrlManagedAM("theTrackingUrl"); + } + + @Test + public void testManagedAMWithEmptyTrackingUrl() { + testTrackingUrlManagedAM(""); + } + + @Test + public void testManagedAMWithNullTrackingUrl() { + testTrackingUrlManagedAM(null); + } + + private void testTrackingUrlManagedAM(String url) { Container amContainer = allocateApplicationAttempt(); launchApplicationAttempt(amContainer); - runApplicationAttempt(amContainer, "host", 8042, "", false); + runApplicationAttempt(amContainer, "host", 8042, url, false); + unregisterApplicationAttempt(amContainer, + FinalApplicationStatus.SUCCEEDED, url, "Successful"); } @Test @@ -927,4 +974,12 @@ public class TestRMAppAttemptTransitions { } } } + + private void verifyUrl(String url1, String url2) { + if (url1 == null || url1.trim().isEmpty()) { + assertEquals("N/A", url2); + } else { + assertEquals(url1, url2); + } + } }