diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 6d198a745eb..010b34aa654 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -327,6 +327,10 @@ Release 0.23.1 - Unreleased MAPREDUCE-3366. Mapreduce component should use consistent directory structure layout as HDFS/common (Eric Yang via mahadev) + MAPREDUCE-3387. Fixed AM's tracking URL to always go through the proxy, even + before the job started, so that it works properly with oozie throughout + the job execution. (Robert Joseph Evans via vinodkv) + Release 0.23.0 - 2011-11-01 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 71dd982b607..0f695fda9f1 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -273,6 +273,8 @@ public RMAppAttemptImpl(ApplicationAttemptId appAttemptId, this.readLock = lock.readLock(); this.writeLock = lock.writeLock(); + this.proxiedTrackingUrl = generateProxyUriWithoutScheme(); + this.stateMachine = stateMachineFactory.make(this); } @@ -358,11 +360,16 @@ public String getWebProxyBase() { } } + private String generateProxyUriWithoutScheme() { + return generateProxyUriWithoutScheme(null); + } + private String generateProxyUriWithoutScheme( final String trackingUriWithoutScheme) { this.readLock.lock(); try { - URI trackingUri = ProxyUriUtils.getUriFromAMUrl(trackingUriWithoutScheme); + URI trackingUri = trackingUriWithoutScheme == null ? null : + ProxyUriUtils.getUriFromAMUrl(trackingUriWithoutScheme); URI proxyUri = ProxyUriUtils.getUriFromAMUrl(proxy); URI result = ProxyUriUtils.getProxyUri(trackingUri, proxyUri, applicationAttemptId.getApplicationId()); diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java index b2600ae0eaf..f7f54aae3f1 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java @@ -80,8 +80,10 @@ public AppInfo(RMApp app, Boolean hasAccess) { if (app != null) { String trackingUrl = app.getTrackingUrl(); + this.state = app.getState(); this.trackingUrlIsNotReady = trackingUrl == null || trackingUrl.isEmpty() - || "N/A".equalsIgnoreCase(trackingUrl); + || RMAppState.NEW == this.state || RMAppState.SUBMITTED == this.state + || RMAppState.ACCEPTED == this.state; this.trackingUI = this.trackingUrlIsNotReady ? "UNASSIGNED" : (app .getFinishTime() == 0 ? "ApplicationMaster" : "History"); if (!trackingUrlIsNotReady) { @@ -95,7 +97,6 @@ public AppInfo(RMApp app, Boolean hasAccess) { this.user = app.getUser().toString(); this.name = app.getName().toString(); this.queue = app.getQueue().toString(); - this.state = app.getState(); this.progress = app.getProgress() * 100; this.diagnostics = app.getDiagnostics().toString(); if (diagnostics == null || diagnostics.isEmpty()) { diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java index 09699fbf91f..1059e58ab96 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java @@ -17,8 +17,7 @@ */ package org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; +import static org.junit.Assert.*; import static org.mockito.Matchers.*; import static org.mockito.Mockito.*; @@ -203,6 +202,8 @@ private void testAppAttemptNewState() { assertEquals(0.0, (double)applicationAttempt.getProgress(), 0.0001); assertEquals(0, applicationAttempt.getRanNodes().size()); assertNull(applicationAttempt.getFinalApplicationStatus()); + assertNotNull(applicationAttempt.getTrackingUrl()); + assertFalse("N/A".equals(applicationAttempt.getTrackingUrl())); } /** diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyUriUtils.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyUriUtils.java index e9bc0c81f8e..61e31eee93c 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyUriUtils.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyUriUtils.java @@ -114,7 +114,8 @@ private static boolean appendQuery(StringBuilder builder, String query, /** * Get a proxied URI for the original URI. - * @param originalUri the original URI to go through the proxy + * @param originalUri the original URI to go through the proxy, or null if + * a default path "/" can be used. * @param proxyUri the URI of the proxy itself, scheme, host and port are used. * @param id the id of the application * @return the proxied URI @@ -122,9 +123,10 @@ private static boolean appendQuery(StringBuilder builder, String query, public static URI getProxyUri(URI originalUri, URI proxyUri, ApplicationId id) { try { - String path = getPath(id, originalUri.getPath()); + String path = getPath(id, originalUri == null ? "/" : originalUri.getPath()); return new URI(proxyUri.getScheme(), proxyUri.getAuthority(), path, - originalUri.getQuery(), originalUri.getFragment()); + originalUri == null ? null : originalUri.getQuery(), + originalUri == null ? null : originalUri.getFragment()); } catch (URISyntaxException e) { throw new RuntimeException("Could not proxify "+originalUri,e); } diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestProxyUriUtils.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestProxyUriUtils.java index 2f83b6e38c4..16ee7bededd 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestProxyUriUtils.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestProxyUriUtils.java @@ -23,44 +23,16 @@ import java.net.URI; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.util.BuilderUtils; import org.junit.Test; public class TestProxyUriUtils { - public static class TestAppId extends ApplicationId { - private long timestamp; - private int id; - - public TestAppId(int id, long timestamp) { - setId(id); - setClusterTimestamp(timestamp); - } - @Override - public int getId() { - return id; - } - - @Override - public void setId(int id) { - this.id = id; - } - - @Override - public long getClusterTimestamp() { - return timestamp; - } - - @Override - public void setClusterTimestamp(long clusterTimestamp) { - this.timestamp = clusterTimestamp; - } - } - @Test public void testGetPathApplicationId() { assertEquals("/proxy/application_100_0001", - ProxyUriUtils.getPath(new TestAppId(1, 100l))); + ProxyUriUtils.getPath(BuilderUtils.newApplicationId(100l, 1))); assertEquals("/proxy/application_6384623_0005", - ProxyUriUtils.getPath(new TestAppId(5, 6384623l))); + ProxyUriUtils.getPath(BuilderUtils.newApplicationId(6384623l, 5))); } @Test(expected = IllegalArgumentException.class) @@ -71,23 +43,23 @@ public void testGetPathApplicationIdBad() { @Test public void testGetPathApplicationIdString() { assertEquals("/proxy/application_6384623_0005", - ProxyUriUtils.getPath(new TestAppId(5, 6384623l), null)); + ProxyUriUtils.getPath(BuilderUtils.newApplicationId(6384623l, 5), null)); assertEquals("/proxy/application_6384623_0005/static/app", - ProxyUriUtils.getPath(new TestAppId(5, 6384623l), "/static/app")); + ProxyUriUtils.getPath(BuilderUtils.newApplicationId(6384623l, 5), "/static/app")); assertEquals("/proxy/application_6384623_0005/", - ProxyUriUtils.getPath(new TestAppId(5, 6384623l), "/")); + ProxyUriUtils.getPath(BuilderUtils.newApplicationId(6384623l, 5), "/")); assertEquals("/proxy/application_6384623_0005/some/path", - ProxyUriUtils.getPath(new TestAppId(5, 6384623l), "some/path")); + ProxyUriUtils.getPath(BuilderUtils.newApplicationId(6384623l, 5), "some/path")); } @Test public void testGetPathAndQuery() { assertEquals("/proxy/application_6384623_0005/static/app?foo=bar", - ProxyUriUtils.getPathAndQuery(new TestAppId(5, 6384623l), "/static/app", + ProxyUriUtils.getPathAndQuery(BuilderUtils.newApplicationId(6384623l, 5), "/static/app", "?foo=bar", false)); assertEquals("/proxy/application_6384623_0005/static/app?foo=bar&bad=good&proxyapproved=true", - ProxyUriUtils.getPathAndQuery(new TestAppId(5, 6384623l), "/static/app", + ProxyUriUtils.getPathAndQuery(BuilderUtils.newApplicationId(6384623l, 5), "/static/app", "foo=bar&bad=good", true)); } @@ -95,10 +67,20 @@ public void testGetPathAndQuery() { public void testGetProxyUri() throws Exception { URI originalUri = new URI("http://host.com/static/foo?bar=bar"); URI proxyUri = new URI("http://proxy.net:8080/"); - TestAppId id = new TestAppId(5, 6384623l); + ApplicationId id = BuilderUtils.newApplicationId(6384623l, 5); URI expected = new URI("http://proxy.net:8080/proxy/application_6384623_0005/static/foo?bar=bar"); URI result = ProxyUriUtils.getProxyUri(originalUri, proxyUri, id); assertEquals(expected, result); } + + @Test + public void testGetProxyUriNull() throws Exception { + URI originalUri = null; + URI proxyUri = new URI("http://proxy.net:8080/"); + ApplicationId id = BuilderUtils.newApplicationId(6384623l, 5); + URI expected = new URI("http://proxy.net:8080/proxy/application_6384623_0005/"); + URI result = ProxyUriUtils.getProxyUri(originalUri, proxyUri, id); + assertEquals(expected, result); + } }