Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-1283. Fixed RM to give a fully-qualified proxy URL for an application so that clients don't need to do scheme-mangling. Contributed by Omkar Vinit Joshi. 

svn merge --ignore-ancestry -c 1530819 ../../trunk/


git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1530820 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Vinod Kumar Vavilapalli 2013-10-09 22:49:50 +00:00
parent 9b5a023fc2
commit 82c4a9438d
7 changed files with 34 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
View File

@ -33,7 +33,6 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.mapreduce.JobStatus;
import org.apache.hadoop.mapreduce.MRJobConfig;
@ -424,9 +423,6 @@ public class ClientServiceDelegate {
String historyTrackingUrl = report.getTrackingUrl();
String url = StringUtils.isNotEmpty(historyTrackingUrl)
? historyTrackingUrl : trackingUrl;
if (!UNAVAILABLE.equals(url)) {
url = HttpConfig.getSchemePrefix() + url;
}
jobStatus = TypeConverter.fromYarn(report, url);
}
return jobStatus;

2
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java
View File

@ -514,7 +514,7 @@ public class TestClientServiceDelegate {
jobReport.setMapProgress(1.0f);
jobReport.setReduceProgress(1.0f);
jobReport.setJobFile("TestJobFilePath");
jobReport.setTrackingUrl("TestTrackingUrl");
jobReport.setTrackingUrl("http://TestTrackingUrl");
jobReportResponse.setJobReport(jobReport);
return jobReportResponse;
}

4
hadoop-yarn-project/CHANGES.txt
View File

@ -76,6 +76,10 @@ Release 2.2.1 - UNRELEASED
YARN-1253. Changes to LinuxContainerExecutor to run containers as a single
dedicated user in non-secure mode. (rvs via tucu)
YARN-1283. Fixed RM to give a fully-qualified proxy URL for an application
so that clients don't need to do scheme-mangling. (Omkar Vinit Joshi via
vinodkv)
Release 2.2.0 - 2013-10-13
INCOMPATIBLE CHANGES

17
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
View File

@ -143,6 +143,21 @@ public class WebAppUtils {
return conf.get(YarnConfiguration.NM_WEBAPP_ADDRESS,
YarnConfiguration.DEFAULT_NM_WEBAPP_ADDRESS);
}
}
/**
* if url has scheme then it will be returned as it is else it will return
* url with scheme.
* @param schemePrefix eg. http:// or https://
* @param url
* @return url with scheme
*/
public static String getURLWithScheme(String schemePrefix, String url) {
// If scheme is provided then it will be returned as it is
if (url.indexOf("://") > 0) {
return url;
} else {
return schemePrefix + url;
}
}
}

15
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
View File

@ -381,7 +381,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
this.readLock = lock.readLock();
this.writeLock = lock.writeLock();
this.proxiedTrackingUrl = generateProxyUriWithoutScheme();
this.proxiedTrackingUrl = generateProxyUriWithScheme(null);
this.stateMachine = stateMachineFactory.make(this);
this.user = user;
@ -470,11 +470,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
}
}
private String generateProxyUriWithoutScheme() {
return generateProxyUriWithoutScheme(null);
}
private String generateProxyUriWithoutScheme(
private String generateProxyUriWithScheme(
final String trackingUriWithoutScheme) {
this.readLock.lock();
try {
@ -484,8 +480,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
URI proxyUri = ProxyUriUtils.getUriFromAMUrl(proxy);
URI result = ProxyUriUtils.getProxyUri(trackingUri, proxyUri,
applicationAttemptId.getApplicationId());
//We need to strip off the scheme to have it match what was there before
return result.toASCIIString().substring(HttpConfig.getSchemePrefix().length());
return result.toASCIIString();
} catch (URISyntaxException e) {
LOG.warn("Could not proxify "+trackingUriWithoutScheme,e);
return trackingUriWithoutScheme;
@ -1006,7 +1001,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
appAttempt.origTrackingUrl =
sanitizeTrackingUrl(registrationEvent.getTrackingurl());
appAttempt.proxiedTrackingUrl =
appAttempt.generateProxyUriWithoutScheme(appAttempt.origTrackingUrl);
appAttempt.generateProxyUriWithScheme(appAttempt.origTrackingUrl);
// Let the app know
appAttempt.eventHandler.handle(new RMAppEvent(appAttempt
@ -1142,7 +1137,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
appAttempt.origTrackingUrl =
sanitizeTrackingUrl(unregisterEvent.getTrackingUrl());
appAttempt.proxiedTrackingUrl =
appAttempt.generateProxyUriWithoutScheme(appAttempt.origTrackingUrl);
appAttempt.generateProxyUriWithScheme(appAttempt.origTrackingUrl);
appAttempt.finalStatus = unregisterEvent.getFinalApplicationStatus();
// Tell the app

10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
View File

@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.Times;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
@XmlRootElement(name = "app")
@XmlAccessorType(XmlAccessType.FIELD)
@ -91,10 +92,13 @@ public class AppInfo {
this.trackingUI = this.trackingUrlIsNotReady ? "UNASSIGNED" : (app
.getFinishTime() == 0 ? "ApplicationMaster" : "History");
if (!trackingUrlIsNotReady) {
this.trackingUrl = join(HttpConfig.getSchemePrefix(), trackingUrl);
this.trackingUrl =
WebAppUtils.getURLWithScheme(HttpConfig.getSchemePrefix(),
trackingUrl);
this.trackingUrlPretty = this.trackingUrl;
} else {
this.trackingUrlPretty = "UNASSIGNED";
}
this.trackingUrlPretty = trackingUrlIsNotReady ? "UNASSIGNED" : join(
HttpConfig.getSchemePrefix(), trackingUrl);
this.applicationId = app.getApplicationId();
this.applicationType = app.getApplicationType();
this.appIdNum = String.valueOf(app.getApplicationId().getId());

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
View File

@ -41,7 +41,6 @@ import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
@ -277,8 +276,7 @@ public class TestRMAppAttemptTransitions {
URI proxyUri = ProxyUriUtils.getUriFromAMUrl(proxy);
URI result = ProxyUriUtils.getProxyUri(trackingUri, proxyUri,
appAttempt.getAppAttemptId().getApplicationId());
url = result.toASCIIString().substring(
HttpConfig.getSchemePrefix().length());
url = result.toASCIIString();
} catch (URISyntaxException ex) {
Assert.fail();
}