Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-1850. Introduced the ability to optionally disable sending out timeline-events in the TimelineClient. Contributed by Zhijie Shen. 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1581189 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Vinod Kumar Vavilapalli 2014-03-25 02:22:39 +00:00
parent 9db6f98e0b
commit 09f383254c
5 changed files with 76 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/CHANGES.txt
View File

@ -323,6 +323,9 @@ Release 2.4.0 - UNRELEASED
YARN-1536. Cleanup: Get rid of ResourceManager#get*SecretManager() methods
and use the RMContext methods instead. (Anubhav Dhoot via kasha)
YARN-1850. Introduced the ability to optionally disable sending out timeline-
events in the TimelineClient. (Zhijie Shen via vinodkv)
OPTIMIZATIONS
YARN-1771. Reduce the number of NameNode operations during localization of

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
View File

@ -1095,6 +1095,11 @@ public class YarnConfiguration extends Configuration {
public static final String DEFAULT_FS_APPLICATION_HISTORY_STORE_COMPRESSION_TYPE =
"none";
/** The setting that controls whether timeline service is enabled or not. */
public static final String TIMELINE_SERVICE_ENABLED =
TIMELINE_SERVICE_PREFIX + "enabled";
public static final boolean DEFAULT_TIMELINE_SERVICE_ENABLED = true;
/** host:port address for timeline service RPC APIs. */
public static final String TIMELINE_SERVICE_ADDRESS =
TIMELINE_SERVICE_PREFIX + "address";

35
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
View File

@ -55,6 +55,7 @@ public class TimelineClientImpl extends TimelineClient {
private Client client;
private URI resURI;
private boolean isEnabled;
public TimelineClientImpl() {
super(TimelineClientImpl.class.getName());
@ -64,24 +65,38 @@ public class TimelineClientImpl extends TimelineClient {
}
protected void serviceInit(Configuration conf) throws Exception {
if (YarnConfiguration.useHttps(conf)) {
resURI = URI
.create(JOINER.join("https://", conf.get(
YarnConfiguration.TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS,
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS),
RESOURCE_URI_STR));
isEnabled = conf.getBoolean(
YarnConfiguration.TIMELINE_SERVICE_ENABLED,
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED);
if (!isEnabled) {
LOG.info("Timeline service is not enabled");
} else {
resURI = URI.create(JOINER.join("http://", conf.get(
YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS,
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_ADDRESS), RESOURCE_URI_STR));
if (YarnConfiguration.useHttps(conf)) {
resURI = URI
.create(JOINER.join("https://", conf.get(
YarnConfiguration.TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS,
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS),
RESOURCE_URI_STR));
} else {
resURI = URI.create(JOINER.join("http://", conf.get(
YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS,
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_ADDRESS),
RESOURCE_URI_STR));
}
LOG.info("Timeline service address: " + resURI);
}
LOG.info("Timeline service address: " + resURI);
super.serviceInit(conf);
}
@Override
public TimelinePutResponse putEntities(
TimelineEntity... entities) throws IOException, YarnException {
if (!isEnabled) {
if (LOG.isDebugEnabled()) {
LOG.debug("Nothing will be put because timeline service is not enabled");
}
return new TimelinePutResponse();
}
TimelineEntities entitiesContainer = new TimelineEntities();
entitiesContainer.addEntities(Arrays.asList(entities));
ClientResponse resp;

45
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java
View File

@ -49,19 +49,19 @@ public class TestTimelineClient {
@Before
public void setup() {
client = spy((TimelineClientImpl) TimelineClient.createTimelineClient());
client.init(new YarnConfiguration());
client.start();
client = createTimelineClient(new YarnConfiguration());
}
@After
public void tearDown() {
client.stop();
if (client != null) {
client.stop();
}
}
@Test
public void testPostEntities() throws Exception {
mockClientResponse(ClientResponse.Status.OK, false, false);
mockClientResponse(client, ClientResponse.Status.OK, false, false);
try {
TimelinePutResponse response = client.putEntities(generateEntity());
Assert.assertEquals(0, response.getErrors().size());
@ -72,7 +72,7 @@ public class TestTimelineClient {
@Test
public void testPostEntitiesWithError() throws Exception {
mockClientResponse(ClientResponse.Status.OK, true, false);
mockClientResponse(client, ClientResponse.Status.OK, true, false);
try {
TimelinePutResponse response = client.putEntities(generateEntity());
Assert.assertEquals(1, response.getErrors().size());
@ -90,7 +90,7 @@ public class TestTimelineClient {
@Test
public void testPostEntitiesNoResponse() throws Exception {
mockClientResponse(
ClientResponse.Status.INTERNAL_SERVER_ERROR, false, false);
client, ClientResponse.Status.INTERNAL_SERVER_ERROR, false, false);
try {
client.putEntities(generateEntity());
Assert.fail("Exception is expected");
@ -102,7 +102,7 @@ public class TestTimelineClient {
@Test
public void testPostEntitiesConnectionRefused() throws Exception {
mockClientResponse(null, false, true);
mockClientResponse(client, null, false, true);
try {
client.putEntities(generateEntity());
Assert.fail("RuntimeException is expected");
@ -111,8 +111,24 @@ public class TestTimelineClient {
}
}
private ClientResponse mockClientResponse(ClientResponse.Status status,
boolean hasError, boolean hasRuntimeError) {
@Test
public void testPostEntitiesTimelineServiceNotEnabled() throws Exception {
YarnConfiguration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false);
TimelineClientImpl client = createTimelineClient(conf);
mockClientResponse(
client, ClientResponse.Status.INTERNAL_SERVER_ERROR, false, false);
try {
TimelinePutResponse response = client.putEntities(generateEntity());
Assert.assertEquals(0, response.getErrors().size());
} catch (YarnException e) {
Assert.fail(
"putEntities should already return before throwing the exception");
}
}
private static ClientResponse mockClientResponse(TimelineClientImpl client,
ClientResponse.Status status, boolean hasError, boolean hasRuntimeError) {
ClientResponse response = mock(ClientResponse.class);
if (hasRuntimeError) {
doThrow(new ClientHandlerException(new ConnectException())).when(client)
@ -157,4 +173,13 @@ public class TestTimelineClient {
return entity;
}
private static TimelineClientImpl createTimelineClient(
YarnConfiguration conf) {
TimelineClientImpl client =
spy((TimelineClientImpl) TimelineClient.createTimelineClient());
client.init(conf);
client.start();
return client;
}
}

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
View File

@ -1087,6 +1087,14 @@
<!-- Timeline Service's Configuration-->
<property>
<description>Indicate to clients whether timeline service is enabled or not.
If enabled, clients will put entities and events to the timeline server.
</description>
<name>yarn.timeline-service.enabled</name>
<value>true</value>
</property>
<property>
<description>The hostname of the timeline service web application.</description>
<name>yarn.timeline-service.hostname</name>