From af65f8bb59863089d0b8f2accd44c27785652751 Mon Sep 17 00:00:00 2001 From: Jason Lowe Date: Wed, 8 Feb 2017 14:59:09 -0600 Subject: [PATCH] YARN-6137. Yarn client implicitly invoke ATS client which accesses HDFS. Contributed by Li Lu (cherry picked from commit 37b4acf7cee1f05599a84bbb1ebf07979a71f82f) --- .../yarn/client/api/impl/YarnClientImpl.java | 54 ++++++++----------- .../yarn/client/api/impl/TestYarnClient.java | 24 ++++++--- 2 files changed, 40 insertions(+), 38 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java index 6d57994f18b..93893f9221c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java @@ -134,7 +134,7 @@ public class YarnClientImpl extends YarnClient { private long asyncApiPollTimeoutMillis; protected AHSClient historyClient; private boolean historyServiceEnabled; - protected TimelineClient timelineClient; + protected volatile TimelineClient timelineClient; @VisibleForTesting Text timelineService; @VisibleForTesting @@ -174,24 +174,9 @@ public class YarnClientImpl extends YarnClient { if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) { - try { - timelineServiceEnabled = true; - timelineClient = createTimelineClient(); - timelineClient.init(conf); - timelineDTRenewer = getTimelineDelegationTokenRenewer(conf); - timelineService = TimelineUtils.buildTimelineTokenService(conf); - } catch (NoClassDefFoundError error) { - // When attempt to initiate the timeline client with - // different set of dependencies, it may fail with - // NoClassDefFoundError. When some of them are not compatible - // with timeline server. This is not necessarily a fatal error - // to the client. - LOG.warn("Timeline client could not be initialized " - + "because dependency missing or incompatible," - + " disabling timeline client.", - error); - timelineServiceEnabled = false; - } + timelineServiceEnabled = true; + timelineDTRenewer = getTimelineDelegationTokenRenewer(conf); + timelineService = TimelineUtils.buildTimelineTokenService(conf); } timelineServiceBestEffort = conf.getBoolean( @@ -212,9 +197,6 @@ public class YarnClientImpl extends YarnClient { if (historyServiceEnabled) { historyClient.start(); } - if (timelineServiceEnabled) { - timelineClient.start(); - } } catch (IOException e) { throw new YarnRuntimeException(e); } @@ -229,7 +211,7 @@ public class YarnClientImpl extends YarnClient { if (historyServiceEnabled) { historyClient.stop(); } - if (timelineServiceEnabled) { + if (timelineClient != null) { timelineClient.stop(); } super.serviceStop(); @@ -368,16 +350,26 @@ public class YarnClientImpl extends YarnClient { @VisibleForTesting org.apache.hadoop.security.token.Token getTimelineDelegationToken() throws IOException, YarnException { - try { - return timelineClient.getDelegationToken(timelineDTRenewer); - } catch (Exception e ) { - if (timelineServiceBestEffort) { - LOG.warn("Failed to get delegation token from the timeline server: " - + e.getMessage()); - return null; + try { + // Only reachable when both security and timeline service are enabled. + if (timelineClient == null) { + synchronized (this) { + if (timelineClient == null) { + timelineClient = createTimelineClient(); + timelineClient.init(getConfig()); + timelineClient.start(); } - throw e; } + } + return timelineClient.getDelegationToken(timelineDTRenewer); + } catch (Exception e) { + if (timelineServiceBestEffort) { + LOG.warn("Failed to get delegation token from the timeline server: " + + e.getMessage()); + return null; + } + throw e; + } } private static String getTimelineDelegationTokenRenewer(Configuration conf) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java index 1e90a2a533b..8d199d2cb91 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java @@ -18,7 +18,6 @@ package org.apache.hadoop.yarn.client.api.impl; -import static org.junit.Assert.assertFalse; import static org.mockito.Matchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; @@ -157,25 +156,36 @@ public class TestYarnClient { } @Test - public void testTimelineClientInitFailure() throws Exception{ + public void testStartWithTimelineV15Failure() throws Exception{ Configuration conf = new Configuration(); conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); + conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 1.5f); + conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_CLIENT_BEST_EFFORT, + true); YarnClient client = YarnClient.createYarnClient(); if(client instanceof YarnClientImpl) { YarnClientImpl impl = (YarnClientImpl) client; YarnClientImpl spyClient = spy(impl); when(spyClient.createTimelineClient()).thenThrow( - new NoClassDefFoundError( - "Mock a failure when init timeline instance")); + new IOException("ATS v1.5 client initialization failed. ")); spyClient.init(conf); spyClient.start(); - assertFalse("Timeline client should be disabled when" - + "it is failed to init", - spyClient.timelineServiceEnabled); + spyClient.getTimelineDelegationToken(); spyClient.stop(); } } + @Test + public void testStartWithTimelineV15() throws Exception { + Configuration conf = new Configuration(); + conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); + conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 1.5f); + YarnClientImpl client = (YarnClientImpl) YarnClient.createYarnClient(); + client.init(conf); + client.start(); + client.stop(); + } + @SuppressWarnings("deprecation") @Test (timeout = 30000) public void testSubmitApplication() throws Exception {