YARN-6137. Yarn client implicitly invoke ATS client which accesses HDFS. Contributed by Li Lu
This commit is contained in:
parent
eec52e158b
commit
37b4acf7ce
|
@ -136,7 +136,7 @@ public class YarnClientImpl extends YarnClient {
|
|||
private long asyncApiPollTimeoutMillis;
|
||||
protected AHSClient historyClient;
|
||||
private boolean historyServiceEnabled;
|
||||
protected TimelineClient timelineClient;
|
||||
protected volatile TimelineClient timelineClient;
|
||||
@VisibleForTesting
|
||||
Text timelineService;
|
||||
@VisibleForTesting
|
||||
|
@ -169,24 +169,9 @@ public class YarnClientImpl extends YarnClient {
|
|||
|
||||
if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
|
||||
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
|
||||
try {
|
||||
timelineServiceEnabled = true;
|
||||
timelineClient = createTimelineClient();
|
||||
timelineClient.init(conf);
|
||||
timelineDTRenewer = getTimelineDelegationTokenRenewer(conf);
|
||||
timelineService = TimelineUtils.buildTimelineTokenService(conf);
|
||||
} catch (NoClassDefFoundError error) {
|
||||
// When attempt to initiate the timeline client with
|
||||
// different set of dependencies, it may fail with
|
||||
// NoClassDefFoundError. When some of them are not compatible
|
||||
// with timeline server. This is not necessarily a fatal error
|
||||
// to the client.
|
||||
LOG.warn("Timeline client could not be initialized "
|
||||
+ "because dependency missing or incompatible,"
|
||||
+ " disabling timeline client.",
|
||||
error);
|
||||
timelineServiceEnabled = false;
|
||||
}
|
||||
timelineServiceEnabled = true;
|
||||
timelineDTRenewer = getTimelineDelegationTokenRenewer(conf);
|
||||
timelineService = TimelineUtils.buildTimelineTokenService(conf);
|
||||
}
|
||||
|
||||
// The AHSClientService is enabled by default when we start the
|
||||
|
@ -219,9 +204,6 @@ public class YarnClientImpl extends YarnClient {
|
|||
if (historyServiceEnabled) {
|
||||
historyClient.start();
|
||||
}
|
||||
if (timelineServiceEnabled) {
|
||||
timelineClient.start();
|
||||
}
|
||||
} catch (IOException e) {
|
||||
throw new YarnRuntimeException(e);
|
||||
}
|
||||
|
@ -236,7 +218,7 @@ public class YarnClientImpl extends YarnClient {
|
|||
if (historyServiceEnabled) {
|
||||
historyClient.stop();
|
||||
}
|
||||
if (timelineServiceEnabled) {
|
||||
if (timelineClient != null) {
|
||||
timelineClient.stop();
|
||||
}
|
||||
super.serviceStop();
|
||||
|
@ -375,16 +357,26 @@ public class YarnClientImpl extends YarnClient {
|
|||
@VisibleForTesting
|
||||
org.apache.hadoop.security.token.Token<TimelineDelegationTokenIdentifier>
|
||||
getTimelineDelegationToken() throws IOException, YarnException {
|
||||
try {
|
||||
return timelineClient.getDelegationToken(timelineDTRenewer);
|
||||
} catch (Exception e ) {
|
||||
if (timelineServiceBestEffort) {
|
||||
LOG.warn("Failed to get delegation token from the timeline server: "
|
||||
+ e.getMessage());
|
||||
return null;
|
||||
try {
|
||||
// Only reachable when both security and timeline service are enabled.
|
||||
if (timelineClient == null) {
|
||||
synchronized (this) {
|
||||
if (timelineClient == null) {
|
||||
timelineClient = createTimelineClient();
|
||||
timelineClient.init(getConfig());
|
||||
timelineClient.start();
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
return timelineClient.getDelegationToken(timelineDTRenewer);
|
||||
} catch (Exception e) {
|
||||
if (timelineServiceBestEffort) {
|
||||
LOG.warn("Failed to get delegation token from the timeline server: "
|
||||
+ e.getMessage());
|
||||
return null;
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
private static String getTimelineDelegationTokenRenewer(Configuration conf)
|
||||
|
|
|
@ -18,7 +18,6 @@
|
|||
|
||||
package org.apache.hadoop.yarn.client.api.impl;
|
||||
|
||||
import static org.junit.Assert.assertFalse;
|
||||
import static org.mockito.Matchers.any;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
|
@ -157,25 +156,36 @@ public class TestYarnClient {
|
|||
}
|
||||
|
||||
@Test
|
||||
public void testTimelineClientInitFailure() throws Exception{
|
||||
public void testStartWithTimelineV15Failure() throws Exception{
|
||||
Configuration conf = new Configuration();
|
||||
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
|
||||
conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 1.5f);
|
||||
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_CLIENT_BEST_EFFORT,
|
||||
true);
|
||||
YarnClient client = YarnClient.createYarnClient();
|
||||
if(client instanceof YarnClientImpl) {
|
||||
YarnClientImpl impl = (YarnClientImpl) client;
|
||||
YarnClientImpl spyClient = spy(impl);
|
||||
when(spyClient.createTimelineClient()).thenThrow(
|
||||
new NoClassDefFoundError(
|
||||
"Mock a failure when init timeline instance"));
|
||||
new IOException("ATS v1.5 client initialization failed. "));
|
||||
spyClient.init(conf);
|
||||
spyClient.start();
|
||||
assertFalse("Timeline client should be disabled when"
|
||||
+ "it is failed to init",
|
||||
spyClient.timelineServiceEnabled);
|
||||
spyClient.getTimelineDelegationToken();
|
||||
spyClient.stop();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testStartWithTimelineV15() throws Exception {
|
||||
Configuration conf = new Configuration();
|
||||
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
|
||||
conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 1.5f);
|
||||
YarnClientImpl client = (YarnClientImpl) YarnClient.createYarnClient();
|
||||
client.init(conf);
|
||||
client.start();
|
||||
client.stop();
|
||||
}
|
||||
|
||||
@SuppressWarnings("deprecation")
|
||||
@Test (timeout = 30000)
|
||||
public void testSubmitApplication() throws Exception {
|
||||
|
|
Loading…
Reference in New Issue