YARN-7207. Cache the RM proxy server address. (Yufei Gu)
(cherry picked from commit 72d22b753abde4d07a727479d3f3d5d84d5dd6b2) # Conflicts: # hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java # hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
This commit is contained in:
Yufei Gu
parent
9bb9831347
commit
4cfc7d0af6
@ -159,4 +159,6 @@ void setRMDelegatedNodeLabelsUpdater(
|
||||
String getHAZookeeperConnectionState();
|
||||
|
||||
ResourceManager getResourceManager();
|
||||
|
||||
String getAppProxyUrl(Configuration conf, ApplicationId applicationId);
|
||||
}
|
||||
|
||||
@ -18,9 +18,13 @@
|
||||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager;
|
||||
|
||||
import java.net.URI;
|
||||
import java.net.URISyntaxException;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.concurrent.ConcurrentMap;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
||||
import org.apache.hadoop.classification.InterfaceStability.Unstable;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
@ -52,7 +56,9 @@
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.security.RMDelegationTokenSecretManager;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.timelineservice.RMTimelineCollectorManager;
|
||||
import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils;
|
||||
import org.apache.hadoop.yarn.util.Clock;
|
||||
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
|
||||
@ -70,6 +76,8 @@
|
||||
*/
|
||||
public class RMContextImpl implements RMContext {
|
||||
|
||||
private static final Log LOG = LogFactory.getLog(RMContextImpl.class);
|
||||
private static final String UNAVAILABLE = "N/A";
|
||||
/**
|
||||
* RM service contexts which runs through out RM life span. These are created
|
||||
* once during start of RM.
|
||||
@ -82,6 +90,8 @@ public class RMContextImpl implements RMContext {
|
||||
*/
|
||||
private RMActiveServiceContext activeServiceContext;
|
||||
|
||||
private String proxyHostAndPort = null;
|
||||
|
||||
/**
|
||||
* Default constructor. To be used in conjunction with setter methods for
|
||||
* individual fields.
|
||||
@ -551,5 +561,28 @@ public RMAppLifetimeMonitor getRMAppLifetimeMonitor() {
|
||||
return this.activeServiceContext.getRMAppLifetimeMonitor();
|
||||
}
|
||||
|
||||
String getProxyHostAndPort(Configuration conf) {
|
||||
if (proxyHostAndPort == null) {
|
||||
proxyHostAndPort = WebAppUtils.getProxyHostAndPort(conf);
|
||||
}
|
||||
return proxyHostAndPort;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getAppProxyUrl(Configuration conf, ApplicationId applicationId)
|
||||
{
|
||||
try {
|
||||
final String scheme = WebAppUtils.getHttpSchemePrefix(conf);
|
||||
URI proxyUri = ProxyUriUtils.getUriFromAMUrl(scheme,
|
||||
getProxyHostAndPort(conf));
|
||||
URI result = ProxyUriUtils.getProxyUri(null, proxyUri, applicationId);
|
||||
return result.toASCIIString();
|
||||
} catch(URISyntaxException e) {
|
||||
LOG.warn("Could not generate default proxy tracking URL for " +
|
||||
applicationId);
|
||||
return UNAVAILABLE;
|
||||
}
|
||||
}
|
||||
|
||||
// Note: Read java doc before adding any services over here.
|
||||
}
|
||||
|
||||
@ -1068,7 +1068,7 @@ protected void startWepApp() {
|
||||
.withCSRFProtection(YarnConfiguration.RM_CSRF_PREFIX)
|
||||
.withXFSProtection(YarnConfiguration.RM_XFS_PREFIX)
|
||||
.at(webAppAddress);
|
||||
String proxyHostAndPort = WebAppUtils.getProxyHostAndPort(conf);
|
||||
String proxyHostAndPort = rmContext.getProxyHostAndPort(conf);
|
||||
if(WebAppUtils.getResolvedRMWebAppURLWithoutScheme(conf).
|
||||
equals(proxyHostAndPort)) {
|
||||
if (HAUtil.isHAEnabled(conf)) {
|
||||
|
||||
@ -19,8 +19,6 @@
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.rmapp;
|
||||
|
||||
import java.net.InetAddress;
|
||||
import java.net.URI;
|
||||
import java.net.URISyntaxException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
@ -103,7 +101,6 @@
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent;
|
||||
import org.apache.hadoop.yarn.server.timelineservice.collector.AppLevelTimelineCollector;
|
||||
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
|
||||
import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils;
|
||||
import org.apache.hadoop.yarn.state.InvalidStateTransitionException;
|
||||
import org.apache.hadoop.yarn.state.MultipleArcTransition;
|
||||
import org.apache.hadoop.yarn.state.SingleArcTransition;
|
||||
@ -113,7 +110,6 @@
|
||||
import org.apache.hadoop.yarn.util.SystemClock;
|
||||
import org.apache.hadoop.yarn.util.Times;
|
||||
import org.apache.hadoop.yarn.util.resource.Resources;
|
||||
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
|
||||
@ -707,7 +703,7 @@ public ApplicationReport createAndGetApplicationReport(String clientUserName,
|
||||
float progress = 0.0f;
|
||||
org.apache.hadoop.yarn.api.records.Token amrmToken = null;
|
||||
if (allowAccess) {
|
||||
trackingUrl = getDefaultProxyTrackingUrl();
|
||||
trackingUrl = rmContext.getAppProxyUrl(conf, applicationId);
|
||||
if (this.currentAttempt != null) {
|
||||
currentApplicationAttemptId = this.currentAttempt.getAppAttemptId();
|
||||
trackingUrl = this.currentAttempt.getTrackingUrl();
|
||||
@ -801,20 +797,6 @@ public ApplicationReport createAndGetApplicationReport(String clientUserName,
|
||||
}
|
||||
}
|
||||
|
||||
private String getDefaultProxyTrackingUrl() {
|
||||
try {
|
||||
final String scheme = WebAppUtils.getHttpSchemePrefix(conf);
|
||||
String proxy = WebAppUtils.getProxyHostAndPort(conf);
|
||||
URI proxyUri = ProxyUriUtils.getUriFromAMUrl(scheme, proxy);
|
||||
URI result = ProxyUriUtils.getProxyUri(null, proxyUri, applicationId);
|
||||
return result.toASCIIString();
|
||||
} catch (URISyntaxException e) {
|
||||
LOG.warn("Could not generate default proxy tracking URL for "
|
||||
+ applicationId);
|
||||
return UNAVAILABLE;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getFinishTime() {
|
||||
this.readLock.lock();
|
||||
|
||||
@ -21,8 +21,6 @@
|
||||
import static org.apache.hadoop.yarn.util.StringHelper.pjoin;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URI;
|
||||
import java.net.URISyntaxException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
@ -521,7 +519,8 @@ public RMAppAttemptImpl(ApplicationAttemptId appAttemptId,
|
||||
this.readLock = lock.readLock();
|
||||
this.writeLock = lock.writeLock();
|
||||
|
||||
this.proxiedTrackingUrl = generateProxyUriWithScheme();
|
||||
this.proxiedTrackingUrl = rmContext.getAppProxyUrl(conf,
|
||||
appAttemptId.getApplicationId());
|
||||
this.stateMachine = stateMachineFactory.make(this);
|
||||
|
||||
this.attemptMetrics =
|
||||
@ -654,24 +653,6 @@ public String getWebProxyBase() {
|
||||
}
|
||||
}
|
||||
|
||||
private String generateProxyUriWithScheme() {
|
||||
this.readLock.lock();
|
||||
try {
|
||||
final String scheme = WebAppUtils.getHttpSchemePrefix(conf);
|
||||
String proxy = WebAppUtils.getProxyHostAndPort(conf);
|
||||
URI proxyUri = ProxyUriUtils.getUriFromAMUrl(scheme, proxy);
|
||||
URI result = ProxyUriUtils.getProxyUri(null, proxyUri,
|
||||
applicationAttemptId.getApplicationId());
|
||||
return result.toASCIIString();
|
||||
} catch (URISyntaxException e) {
|
||||
LOG.warn("Could not proxify the uri for "
|
||||
+ applicationAttemptId.getApplicationId(), e);
|
||||
return null;
|
||||
} finally {
|
||||
this.readLock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
private void setTrackingUrlToRMAppPage(RMAppAttemptState stateToBeStored) {
|
||||
originalTrackingUrl = pjoin(
|
||||
WebAppUtils.getResolvedRMWebAppURLWithScheme(conf),
|
||||
|
||||
@ -35,8 +35,6 @@
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.net.URI;
|
||||
import java.net.URISyntaxException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collection;
|
||||
@ -113,7 +111,6 @@
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
|
||||
import org.apache.hadoop.yarn.server.security.MasterKeyData;
|
||||
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
|
||||
import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils;
|
||||
import org.apache.hadoop.yarn.util.resource.Resources;
|
||||
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
|
||||
import org.junit.After;
|
||||
@ -344,19 +341,10 @@ public void tearDown() throws Exception {
|
||||
((AsyncDispatcher)this.spyRMContext.getDispatcher()).stop();
|
||||
}
|
||||
|
||||
|
||||
private String getProxyUrl(RMAppAttempt appAttempt) {
|
||||
String url = null;
|
||||
final String scheme = WebAppUtils.getHttpSchemePrefix(conf);
|
||||
try {
|
||||
String proxy = WebAppUtils.getProxyHostAndPort(conf);
|
||||
URI proxyUri = ProxyUriUtils.getUriFromAMUrl(scheme, proxy);
|
||||
URI result = ProxyUriUtils.getProxyUri(null, proxyUri, appAttempt
|
||||
.getAppAttemptId().getApplicationId());
|
||||
url = result.toASCIIString();
|
||||
} catch (URISyntaxException ex) {
|
||||
Assert.fail();
|
||||
}
|
||||
String url = rmContext.getAppProxyUrl(conf,
|
||||
appAttempt.getAppAttemptId().getApplicationId());
|
||||
Assert.assertNotEquals("N/A", url);
|
||||
return url;
|
||||
}
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user