Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-6137. Yarn client implicitly invoke ATS client which accesses HDFS. Contributed by Li Lu

This commit is contained in:
Jason Lowe 2017-02-08 14:59:09 -06:00
parent eec52e158b
commit 37b4acf7ce
2 changed files with 40 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
View File

@ -136,7 +136,7 @@ public class YarnClientImpl extends YarnClient {
private long asyncApiPollTimeoutMillis;
protected AHSClient historyClient;
private boolean historyServiceEnabled;
protected TimelineClient timelineClient;
protected volatile TimelineClient timelineClient;
@VisibleForTesting
Text timelineService;
@VisibleForTesting
@ -169,24 +169,9 @@ public class YarnClientImpl extends YarnClient {
if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
try {
timelineServiceEnabled = true;
timelineClient = createTimelineClient();
timelineClient.init(conf);
timelineDTRenewer = getTimelineDelegationTokenRenewer(conf);
timelineService = TimelineUtils.buildTimelineTokenService(conf);
} catch (NoClassDefFoundError error) {
// When attempt to initiate the timeline client with
// different set of dependencies, it may fail with
// NoClassDefFoundError. When some of them are not compatible
// with timeline server. This is not necessarily a fatal error
// to the client.
LOG.warn("Timeline client could not be initialized "
+ "because dependency missing or incompatible,"
+ " disabling timeline client.",
error);
timelineServiceEnabled = false;
}
}
// The AHSClientService is enabled by default when we start the
@ -219,9 +204,6 @@ public class YarnClientImpl extends YarnClient {
if (historyServiceEnabled) {
historyClient.start();
}
if (timelineServiceEnabled) {
timelineClient.start();
}
} catch (IOException e) {
throw new YarnRuntimeException(e);
}
@ -236,7 +218,7 @@ public class YarnClientImpl extends YarnClient {
if (historyServiceEnabled) {
historyClient.stop();
}
if (timelineServiceEnabled) {
if (timelineClient != null) {
timelineClient.stop();
}
super.serviceStop();
@ -376,6 +358,16 @@ public class YarnClientImpl extends YarnClient {
org.apache.hadoop.security.token.Token<TimelineDelegationTokenIdentifier>
getTimelineDelegationToken() throws IOException, YarnException {
try {
// Only reachable when both security and timeline service are enabled.
if (timelineClient == null) {
synchronized (this) {
if (timelineClient == null) {
timelineClient = createTimelineClient();
timelineClient.init(getConfig());
timelineClient.start();
}
}
}
return timelineClient.getDelegationToken(timelineDTRenewer);
} catch (Exception e) {
if (timelineServiceBestEffort) {

24
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
View File

@ -18,7 +18,6 @@
package org.apache.hadoop.yarn.client.api.impl;
import static org.junit.Assert.assertFalse;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
@ -157,25 +156,36 @@ public class TestYarnClient {
}
@Test
public void testTimelineClientInitFailure() throws Exception{
public void testStartWithTimelineV15Failure() throws Exception{
Configuration conf = new Configuration();
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 1.5f);
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_CLIENT_BEST_EFFORT,
true);
YarnClient client = YarnClient.createYarnClient();
if(client instanceof YarnClientImpl) {
YarnClientImpl impl = (YarnClientImpl) client;
YarnClientImpl spyClient = spy(impl);
when(spyClient.createTimelineClient()).thenThrow(
new NoClassDefFoundError(
"Mock a failure when init timeline instance"));
new IOException("ATS v1.5 client initialization failed. "));
spyClient.init(conf);
spyClient.start();
assertFalse("Timeline client should be disabled when"
+ "it is failed to init",
spyClient.timelineServiceEnabled);
spyClient.getTimelineDelegationToken();
spyClient.stop();
}
}
@Test
public void testStartWithTimelineV15() throws Exception {
Configuration conf = new Configuration();
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 1.5f);
YarnClientImpl client = (YarnClientImpl) YarnClient.createYarnClient();
client.init(conf);
client.start();
client.stop();
}
@SuppressWarnings("deprecation")
@Test (timeout = 30000)
public void testSubmitApplication() throws Exception {