Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

MAPREDUCE-3387. Fixed AM's tracking URL to always go through the proxy, even before the job started, so that it works properly with oozie throughout the job execution. Contributed by Robert Joseph Evans. 

svn merge -c 1215345 --ignore-ancestry ../../trunk/


git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.23@1215347 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Vinod Kumar Vavilapalli 2011-12-16 23:42:56 +00:00
parent e0657fb161
commit 84138f3864
6 changed files with 43 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-mapreduce-project/CHANGES.txt
View File

@ -275,6 +275,10 @@ Release 0.23.1 - Unreleased
MAPREDUCE-3366. Mapreduce component should use consistent directory structure
layout as HDFS/common (Eric Yang via mahadev)
MAPREDUCE-3387. Fixed AM's tracking URL to always go through the proxy, even
before the job started, so that it works properly with oozie throughout
the job execution. (Robert Joseph Evans via vinodkv)
Release 0.23.0 - 2011-11-01
INCOMPATIBLE CHANGES

9
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
View File

@ -273,6 +273,8 @@ public class RMAppAttemptImpl implements RMAppAttempt {
this.readLock = lock.readLock();
this.writeLock = lock.writeLock();
this.proxiedTrackingUrl = generateProxyUriWithoutScheme();
this.stateMachine = stateMachineFactory.make(this);
}
@ -358,11 +360,16 @@ public class RMAppAttemptImpl implements RMAppAttempt {
}
}
private String generateProxyUriWithoutScheme() {
return generateProxyUriWithoutScheme(null);
}
private String generateProxyUriWithoutScheme(
final String trackingUriWithoutScheme) {
this.readLock.lock();
try {
URI trackingUri = ProxyUriUtils.getUriFromAMUrl(trackingUriWithoutScheme);
URI trackingUri = trackingUriWithoutScheme == null ? null :
ProxyUriUtils.getUriFromAMUrl(trackingUriWithoutScheme);
URI proxyUri = ProxyUriUtils.getUriFromAMUrl(proxy);
URI result = ProxyUriUtils.getProxyUri(trackingUri, proxyUri,
applicationAttemptId.getApplicationId());

5
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
View File

@ -80,8 +80,10 @@ public class AppInfo {
if (app != null) {
String trackingUrl = app.getTrackingUrl();
this.state = app.getState();
this.trackingUrlIsNotReady = trackingUrl == null || trackingUrl.isEmpty()
|| "N/A".equalsIgnoreCase(trackingUrl);
|| RMAppState.NEW == this.state || RMAppState.SUBMITTED == this.state
|| RMAppState.ACCEPTED == this.state;
this.trackingUI = this.trackingUrlIsNotReady ? "UNASSIGNED" : (app
.getFinishTime() == 0 ? "ApplicationMaster" : "History");
if (!trackingUrlIsNotReady) {
@ -95,7 +97,6 @@ public class AppInfo {
this.user = app.getUser().toString();
this.name = app.getName().toString();
this.queue = app.getQueue().toString();
this.state = app.getState();
this.progress = app.getProgress() * 100;
this.diagnostics = app.getDiagnostics().toString();
if (diagnostics == null || diagnostics.isEmpty()) {

5
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
View File

@ -17,8 +17,7 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.*;
import static org.mockito.Matchers.*;
import static org.mockito.Mockito.*;
@ -203,6 +202,8 @@ public class TestRMAppAttemptTransitions {
assertEquals(0.0, (double)applicationAttempt.getProgress(), 0.0001);
assertEquals(0, applicationAttempt.getRanNodes().size());
assertNull(applicationAttempt.getFinalApplicationStatus());
assertNotNull(applicationAttempt.getTrackingUrl());
assertFalse("N/A".equals(applicationAttempt.getTrackingUrl()));
}
/**

8
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyUriUtils.java
View File

@ -114,7 +114,8 @@ public class ProxyUriUtils {
/**
* Get a proxied URI for the original URI.
* @param originalUri the original URI to go through the proxy
* @param originalUri the original URI to go through the proxy, or null if
* a default path "/" can be used.
* @param proxyUri the URI of the proxy itself, scheme, host and port are used.
* @param id the id of the application
* @return the proxied URI
@ -122,9 +123,10 @@ public class ProxyUriUtils {
public static URI getProxyUri(URI originalUri, URI proxyUri,
ApplicationId id) {
try {
String path = getPath(id, originalUri.getPath());
String path = getPath(id, originalUri == null ? "/" : originalUri.getPath());
return new URI(proxyUri.getScheme(), proxyUri.getAuthority(), path,
originalUri.getQuery(), originalUri.getFragment());
originalUri == null ? null : originalUri.getQuery(),
originalUri == null ? null : originalUri.getFragment());
} catch (URISyntaxException e) {
throw new RuntimeException("Could not proxify "+originalUri,e);
}

58
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestProxyUriUtils.java
View File

@ -23,44 +23,16 @@ import static org.junit.Assert.*;
import java.net.URI;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.util.BuilderUtils;
import org.junit.Test;
public class TestProxyUriUtils {
public static class TestAppId extends ApplicationId {
private long timestamp;
private int id;
public TestAppId(int id, long timestamp) {
setId(id);
setClusterTimestamp(timestamp);
}
@Override
public int getId() {
return id;
}
@Override
public void setId(int id) {
this.id = id;
}
@Override
public long getClusterTimestamp() {
return timestamp;
}
@Override
public void setClusterTimestamp(long clusterTimestamp) {
this.timestamp = clusterTimestamp;
}
}
@Test
public void testGetPathApplicationId() {
assertEquals("/proxy/application_100_0001",
ProxyUriUtils.getPath(new TestAppId(1, 100l)));
ProxyUriUtils.getPath(BuilderUtils.newApplicationId(100l, 1)));
assertEquals("/proxy/application_6384623_0005",
ProxyUriUtils.getPath(new TestAppId(5, 6384623l)));
ProxyUriUtils.getPath(BuilderUtils.newApplicationId(6384623l, 5)));
}
@Test(expected = IllegalArgumentException.class)
@ -71,23 +43,23 @@ public class TestProxyUriUtils {
@Test
public void testGetPathApplicationIdString() {
assertEquals("/proxy/application_6384623_0005",
ProxyUriUtils.getPath(new TestAppId(5, 6384623l), null));
ProxyUriUtils.getPath(BuilderUtils.newApplicationId(6384623l, 5), null));
assertEquals("/proxy/application_6384623_0005/static/app",
ProxyUriUtils.getPath(new TestAppId(5, 6384623l), "/static/app"));
ProxyUriUtils.getPath(BuilderUtils.newApplicationId(6384623l, 5), "/static/app"));
assertEquals("/proxy/application_6384623_0005/",
ProxyUriUtils.getPath(new TestAppId(5, 6384623l), "/"));
ProxyUriUtils.getPath(BuilderUtils.newApplicationId(6384623l, 5), "/"));
assertEquals("/proxy/application_6384623_0005/some/path",
ProxyUriUtils.getPath(new TestAppId(5, 6384623l), "some/path"));
ProxyUriUtils.getPath(BuilderUtils.newApplicationId(6384623l, 5), "some/path"));
}
@Test
public void testGetPathAndQuery() {
assertEquals("/proxy/application_6384623_0005/static/app?foo=bar",
ProxyUriUtils.getPathAndQuery(new TestAppId(5, 6384623l), "/static/app",
ProxyUriUtils.getPathAndQuery(BuilderUtils.newApplicationId(6384623l, 5), "/static/app",
"?foo=bar", false));
assertEquals("/proxy/application_6384623_0005/static/app?foo=bar&bad=good&proxyapproved=true",
ProxyUriUtils.getPathAndQuery(new TestAppId(5, 6384623l), "/static/app",
ProxyUriUtils.getPathAndQuery(BuilderUtils.newApplicationId(6384623l, 5), "/static/app",
"foo=bar&bad=good", true));
}
@ -95,10 +67,20 @@ public class TestProxyUriUtils {
public void testGetProxyUri() throws Exception {
URI originalUri = new URI("http://host.com/static/foo?bar=bar");
URI proxyUri = new URI("http://proxy.net:8080/");
TestAppId id = new TestAppId(5, 6384623l);
ApplicationId id = BuilderUtils.newApplicationId(6384623l, 5);
URI expected = new URI("http://proxy.net:8080/proxy/application_6384623_0005/static/foo?bar=bar");
URI result = ProxyUriUtils.getProxyUri(originalUri, proxyUri, id);
assertEquals(expected, result);
}
@Test
public void testGetProxyUriNull() throws Exception {
URI originalUri = null;
URI proxyUri = new URI("http://proxy.net:8080/");
ApplicationId id = BuilderUtils.newApplicationId(6384623l, 5);
URI expected = new URI("http://proxy.net:8080/proxy/application_6384623_0005/");
URI result = ProxyUriUtils.getProxyUri(originalUri, proxyUri, id);
assertEquals(expected, result);
}
}