From 09f383254c8459071533f2118debd6d3b8538a13 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Tue, 25 Mar 2014 02:22:39 +0000 Subject: [PATCH] YARN-1850. Introduced the ability to optionally disable sending out timeline-events in the TimelineClient. Contributed by Zhijie Shen. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1581189 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 ++ .../hadoop/yarn/conf/YarnConfiguration.java | 5 +++ .../client/api/impl/TimelineClientImpl.java | 35 ++++++++++----- .../client/api/impl/TestTimelineClient.java | 45 ++++++++++++++----- .../src/main/resources/yarn-default.xml | 8 ++++ 5 files changed, 76 insertions(+), 20 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 127f75bb6df..0ab36911f9e 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -323,6 +323,9 @@ Release 2.4.0 - UNRELEASED YARN-1536. Cleanup: Get rid of ResourceManager#get*SecretManager() methods and use the RMContext methods instead. (Anubhav Dhoot via kasha) + YARN-1850. Introduced the ability to optionally disable sending out timeline- + events in the TimelineClient. (Zhijie Shen via vinodkv) + OPTIMIZATIONS YARN-1771. Reduce the number of NameNode operations during localization of diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index d3afb704287..f7d6b6ba97e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1095,6 +1095,11 @@ public class YarnConfiguration extends Configuration { public static final String DEFAULT_FS_APPLICATION_HISTORY_STORE_COMPRESSION_TYPE = "none"; + /** The setting that controls whether timeline service is enabled or not. */ + public static final String TIMELINE_SERVICE_ENABLED = + TIMELINE_SERVICE_PREFIX + "enabled"; + public static final boolean DEFAULT_TIMELINE_SERVICE_ENABLED = true; + /** host:port address for timeline service RPC APIs. */ public static final String TIMELINE_SERVICE_ADDRESS = TIMELINE_SERVICE_PREFIX + "address"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java index 55dc55bf42d..64cc041aaea 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java @@ -55,6 +55,7 @@ public class TimelineClientImpl extends TimelineClient { private Client client; private URI resURI; + private boolean isEnabled; public TimelineClientImpl() { super(TimelineClientImpl.class.getName()); @@ -64,24 +65,38 @@ public class TimelineClientImpl extends TimelineClient { } protected void serviceInit(Configuration conf) throws Exception { - if (YarnConfiguration.useHttps(conf)) { - resURI = URI - .create(JOINER.join("https://", conf.get( - YarnConfiguration.TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS, - YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS), - RESOURCE_URI_STR)); + isEnabled = conf.getBoolean( + YarnConfiguration.TIMELINE_SERVICE_ENABLED, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED); + if (!isEnabled) { + LOG.info("Timeline service is not enabled"); } else { - resURI = URI.create(JOINER.join("http://", conf.get( - YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS, - YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_ADDRESS), RESOURCE_URI_STR)); + if (YarnConfiguration.useHttps(conf)) { + resURI = URI + .create(JOINER.join("https://", conf.get( + YarnConfiguration.TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS), + RESOURCE_URI_STR)); + } else { + resURI = URI.create(JOINER.join("http://", conf.get( + YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_ADDRESS), + RESOURCE_URI_STR)); + } + LOG.info("Timeline service address: " + resURI); } - LOG.info("Timeline service address: " + resURI); super.serviceInit(conf); } @Override public TimelinePutResponse putEntities( TimelineEntity... entities) throws IOException, YarnException { + if (!isEnabled) { + if (LOG.isDebugEnabled()) { + LOG.debug("Nothing will be put because timeline service is not enabled"); + } + return new TimelinePutResponse(); + } TimelineEntities entitiesContainer = new TimelineEntities(); entitiesContainer.addEntities(Arrays.asList(entities)); ClientResponse resp; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java index 0fb3f38f86d..8f61a05b27e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java @@ -49,19 +49,19 @@ public class TestTimelineClient { @Before public void setup() { - client = spy((TimelineClientImpl) TimelineClient.createTimelineClient()); - client.init(new YarnConfiguration()); - client.start(); + client = createTimelineClient(new YarnConfiguration()); } @After public void tearDown() { - client.stop(); + if (client != null) { + client.stop(); + } } @Test public void testPostEntities() throws Exception { - mockClientResponse(ClientResponse.Status.OK, false, false); + mockClientResponse(client, ClientResponse.Status.OK, false, false); try { TimelinePutResponse response = client.putEntities(generateEntity()); Assert.assertEquals(0, response.getErrors().size()); @@ -72,7 +72,7 @@ public class TestTimelineClient { @Test public void testPostEntitiesWithError() throws Exception { - mockClientResponse(ClientResponse.Status.OK, true, false); + mockClientResponse(client, ClientResponse.Status.OK, true, false); try { TimelinePutResponse response = client.putEntities(generateEntity()); Assert.assertEquals(1, response.getErrors().size()); @@ -90,7 +90,7 @@ public class TestTimelineClient { @Test public void testPostEntitiesNoResponse() throws Exception { mockClientResponse( - ClientResponse.Status.INTERNAL_SERVER_ERROR, false, false); + client, ClientResponse.Status.INTERNAL_SERVER_ERROR, false, false); try { client.putEntities(generateEntity()); Assert.fail("Exception is expected"); @@ -102,7 +102,7 @@ public class TestTimelineClient { @Test public void testPostEntitiesConnectionRefused() throws Exception { - mockClientResponse(null, false, true); + mockClientResponse(client, null, false, true); try { client.putEntities(generateEntity()); Assert.fail("RuntimeException is expected"); @@ -111,8 +111,24 @@ public class TestTimelineClient { } } - private ClientResponse mockClientResponse(ClientResponse.Status status, - boolean hasError, boolean hasRuntimeError) { + @Test + public void testPostEntitiesTimelineServiceNotEnabled() throws Exception { + YarnConfiguration conf = new YarnConfiguration(); + conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false); + TimelineClientImpl client = createTimelineClient(conf); + mockClientResponse( + client, ClientResponse.Status.INTERNAL_SERVER_ERROR, false, false); + try { + TimelinePutResponse response = client.putEntities(generateEntity()); + Assert.assertEquals(0, response.getErrors().size()); + } catch (YarnException e) { + Assert.fail( + "putEntities should already return before throwing the exception"); + } + } + + private static ClientResponse mockClientResponse(TimelineClientImpl client, + ClientResponse.Status status, boolean hasError, boolean hasRuntimeError) { ClientResponse response = mock(ClientResponse.class); if (hasRuntimeError) { doThrow(new ClientHandlerException(new ConnectException())).when(client) @@ -157,4 +173,13 @@ public class TestTimelineClient { return entity; } + private static TimelineClientImpl createTimelineClient( + YarnConfiguration conf) { + TimelineClientImpl client = + spy((TimelineClientImpl) TimelineClient.createTimelineClient()); + client.init(conf); + client.start(); + return client; + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index a3ac3d5c40e..dc41a161d21 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1087,6 +1087,14 @@ + + Indicate to clients whether timeline service is enabled or not. + If enabled, clients will put entities and events to the timeline server. + + yarn.timeline-service.enabled + true + + The hostname of the timeline service web application. yarn.timeline-service.hostname