From feb3a23c71a5e000f1100e1ca6459c1fade8befc Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Fri, 16 Dec 2011 23:41:36 +0000 Subject: [PATCH] MAPREDUCE-3387. Fixed AM's tracking URL to always go through the proxy, even before the job started, so that it works properly with oozie throughout the job execution. Contributed by Robert Joseph Evans. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1215345 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 4 ++ .../rmapp/attempt/RMAppAttemptImpl.java | 9 ++- .../resourcemanager/webapp/dao/AppInfo.java | 5 +- .../attempt/TestRMAppAttemptTransitions.java | 5 +- .../yarn/server/webproxy/ProxyUriUtils.java | 8 ++- .../server/webproxy/TestProxyUriUtils.java | 58 +++++++------------ 6 files changed, 43 insertions(+), 46 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 6d198a745eb..010b34aa654 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -327,6 +327,10 @@ Release 0.23.1 - Unreleased MAPREDUCE-3366. Mapreduce component should use consistent directory structure layout as HDFS/common (Eric Yang via mahadev) + MAPREDUCE-3387. Fixed AM's tracking URL to always go through the proxy, even + before the job started, so that it works properly with oozie throughout + the job execution. (Robert Joseph Evans via vinodkv) + Release 0.23.0 - 2011-11-01 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 71dd982b607..0f695fda9f1 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -273,6 +273,8 @@ public class RMAppAttemptImpl implements RMAppAttempt { this.readLock = lock.readLock(); this.writeLock = lock.writeLock(); + this.proxiedTrackingUrl = generateProxyUriWithoutScheme(); + this.stateMachine = stateMachineFactory.make(this); } @@ -358,11 +360,16 @@ public class RMAppAttemptImpl implements RMAppAttempt { } } + private String generateProxyUriWithoutScheme() { + return generateProxyUriWithoutScheme(null); + } + private String generateProxyUriWithoutScheme( final String trackingUriWithoutScheme) { this.readLock.lock(); try { - URI trackingUri = ProxyUriUtils.getUriFromAMUrl(trackingUriWithoutScheme); + URI trackingUri = trackingUriWithoutScheme == null ? null : + ProxyUriUtils.getUriFromAMUrl(trackingUriWithoutScheme); URI proxyUri = ProxyUriUtils.getUriFromAMUrl(proxy); URI result = ProxyUriUtils.getProxyUri(trackingUri, proxyUri, applicationAttemptId.getApplicationId()); diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java index b2600ae0eaf..f7f54aae3f1 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java @@ -80,8 +80,10 @@ public class AppInfo { if (app != null) { String trackingUrl = app.getTrackingUrl(); + this.state = app.getState(); this.trackingUrlIsNotReady = trackingUrl == null || trackingUrl.isEmpty() - || "N/A".equalsIgnoreCase(trackingUrl); + || RMAppState.NEW == this.state || RMAppState.SUBMITTED == this.state + || RMAppState.ACCEPTED == this.state; this.trackingUI = this.trackingUrlIsNotReady ? "UNASSIGNED" : (app .getFinishTime() == 0 ? "ApplicationMaster" : "History"); if (!trackingUrlIsNotReady) { @@ -95,7 +97,6 @@ public class AppInfo { this.user = app.getUser().toString(); this.name = app.getName().toString(); this.queue = app.getQueue().toString(); - this.state = app.getState(); this.progress = app.getProgress() * 100; this.diagnostics = app.getDiagnostics().toString(); if (diagnostics == null || diagnostics.isEmpty()) { diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java index 09699fbf91f..1059e58ab96 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java @@ -17,8 +17,7 @@ */ package org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; +import static org.junit.Assert.*; import static org.mockito.Matchers.*; import static org.mockito.Mockito.*; @@ -203,6 +202,8 @@ public class TestRMAppAttemptTransitions { assertEquals(0.0, (double)applicationAttempt.getProgress(), 0.0001); assertEquals(0, applicationAttempt.getRanNodes().size()); assertNull(applicationAttempt.getFinalApplicationStatus()); + assertNotNull(applicationAttempt.getTrackingUrl()); + assertFalse("N/A".equals(applicationAttempt.getTrackingUrl())); } /** diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyUriUtils.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyUriUtils.java index e9bc0c81f8e..61e31eee93c 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyUriUtils.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyUriUtils.java @@ -114,7 +114,8 @@ public class ProxyUriUtils { /** * Get a proxied URI for the original URI. - * @param originalUri the original URI to go through the proxy + * @param originalUri the original URI to go through the proxy, or null if + * a default path "/" can be used. * @param proxyUri the URI of the proxy itself, scheme, host and port are used. * @param id the id of the application * @return the proxied URI @@ -122,9 +123,10 @@ public class ProxyUriUtils { public static URI getProxyUri(URI originalUri, URI proxyUri, ApplicationId id) { try { - String path = getPath(id, originalUri.getPath()); + String path = getPath(id, originalUri == null ? "/" : originalUri.getPath()); return new URI(proxyUri.getScheme(), proxyUri.getAuthority(), path, - originalUri.getQuery(), originalUri.getFragment()); + originalUri == null ? null : originalUri.getQuery(), + originalUri == null ? null : originalUri.getFragment()); } catch (URISyntaxException e) { throw new RuntimeException("Could not proxify "+originalUri,e); } diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestProxyUriUtils.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestProxyUriUtils.java index 2f83b6e38c4..16ee7bededd 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestProxyUriUtils.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestProxyUriUtils.java @@ -23,44 +23,16 @@ import static org.junit.Assert.*; import java.net.URI; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.util.BuilderUtils; import org.junit.Test; public class TestProxyUriUtils { - public static class TestAppId extends ApplicationId { - private long timestamp; - private int id; - - public TestAppId(int id, long timestamp) { - setId(id); - setClusterTimestamp(timestamp); - } - @Override - public int getId() { - return id; - } - - @Override - public void setId(int id) { - this.id = id; - } - - @Override - public long getClusterTimestamp() { - return timestamp; - } - - @Override - public void setClusterTimestamp(long clusterTimestamp) { - this.timestamp = clusterTimestamp; - } - } - @Test public void testGetPathApplicationId() { assertEquals("/proxy/application_100_0001", - ProxyUriUtils.getPath(new TestAppId(1, 100l))); + ProxyUriUtils.getPath(BuilderUtils.newApplicationId(100l, 1))); assertEquals("/proxy/application_6384623_0005", - ProxyUriUtils.getPath(new TestAppId(5, 6384623l))); + ProxyUriUtils.getPath(BuilderUtils.newApplicationId(6384623l, 5))); } @Test(expected = IllegalArgumentException.class) @@ -71,23 +43,23 @@ public class TestProxyUriUtils { @Test public void testGetPathApplicationIdString() { assertEquals("/proxy/application_6384623_0005", - ProxyUriUtils.getPath(new TestAppId(5, 6384623l), null)); + ProxyUriUtils.getPath(BuilderUtils.newApplicationId(6384623l, 5), null)); assertEquals("/proxy/application_6384623_0005/static/app", - ProxyUriUtils.getPath(new TestAppId(5, 6384623l), "/static/app")); + ProxyUriUtils.getPath(BuilderUtils.newApplicationId(6384623l, 5), "/static/app")); assertEquals("/proxy/application_6384623_0005/", - ProxyUriUtils.getPath(new TestAppId(5, 6384623l), "/")); + ProxyUriUtils.getPath(BuilderUtils.newApplicationId(6384623l, 5), "/")); assertEquals("/proxy/application_6384623_0005/some/path", - ProxyUriUtils.getPath(new TestAppId(5, 6384623l), "some/path")); + ProxyUriUtils.getPath(BuilderUtils.newApplicationId(6384623l, 5), "some/path")); } @Test public void testGetPathAndQuery() { assertEquals("/proxy/application_6384623_0005/static/app?foo=bar", - ProxyUriUtils.getPathAndQuery(new TestAppId(5, 6384623l), "/static/app", + ProxyUriUtils.getPathAndQuery(BuilderUtils.newApplicationId(6384623l, 5), "/static/app", "?foo=bar", false)); assertEquals("/proxy/application_6384623_0005/static/app?foo=bar&bad=good&proxyapproved=true", - ProxyUriUtils.getPathAndQuery(new TestAppId(5, 6384623l), "/static/app", + ProxyUriUtils.getPathAndQuery(BuilderUtils.newApplicationId(6384623l, 5), "/static/app", "foo=bar&bad=good", true)); } @@ -95,10 +67,20 @@ public class TestProxyUriUtils { public void testGetProxyUri() throws Exception { URI originalUri = new URI("http://host.com/static/foo?bar=bar"); URI proxyUri = new URI("http://proxy.net:8080/"); - TestAppId id = new TestAppId(5, 6384623l); + ApplicationId id = BuilderUtils.newApplicationId(6384623l, 5); URI expected = new URI("http://proxy.net:8080/proxy/application_6384623_0005/static/foo?bar=bar"); URI result = ProxyUriUtils.getProxyUri(originalUri, proxyUri, id); assertEquals(expected, result); } + + @Test + public void testGetProxyUriNull() throws Exception { + URI originalUri = null; + URI proxyUri = new URI("http://proxy.net:8080/"); + ApplicationId id = BuilderUtils.newApplicationId(6384623l, 5); + URI expected = new URI("http://proxy.net:8080/proxy/application_6384623_0005/"); + URI result = ProxyUriUtils.getProxyUri(originalUri, proxyUri, id); + assertEquals(expected, result); + } }