diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 6bfe71b96dd..f458f03916a 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -250,6 +250,9 @@ Release 2.4.0 - UNRELEASED YARN-1761. Modified RMAdmin CLI to check whether HA is enabled or not before it executes any of the HA admin related commands. (Xuan Gong via vinodkv) + YARN-1780. Improved logging in the Timeline client and server. (Zhijie Shen + via vinodkv) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java index 7323e15f00d..55dc55bf42d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java @@ -75,6 +75,7 @@ protected void serviceInit(Configuration conf) throws Exception { YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS, YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_ADDRESS), RESOURCE_URI_STR)); } + LOG.info("Timeline service address: " + resURI); super.serviceInit(conf); } @@ -83,12 +84,22 @@ public TimelinePutResponse putEntities( TimelineEntity... entities) throws IOException, YarnException { TimelineEntities entitiesContainer = new TimelineEntities(); entitiesContainer.addEntities(Arrays.asList(entities)); - ClientResponse resp = doPostingEntities(entitiesContainer); - if (resp.getClientResponseStatus() != ClientResponse.Status.OK) { + ClientResponse resp; + try { + resp = doPostingEntities(entitiesContainer); + } catch (RuntimeException re) { + // runtime exception is expected if the client cannot connect the server + String msg = + "Failed to get the response from the timeline server."; + LOG.error(msg, re); + throw re; + } + if (resp == null || + resp.getClientResponseStatus() != ClientResponse.Status.OK) { String msg = "Failed to get the response from the timeline server."; LOG.error(msg); - if (LOG.isDebugEnabled()) { + if (LOG.isDebugEnabled() && resp != null) { String output = resp.getEntity(String.class); LOG.debug("HTTP error code: " + resp.getStatus() + " Server response : \n" + output); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java index 3804757c38e..605856f5379 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java @@ -20,9 +20,13 @@ import static org.mockito.Matchers.any; import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; + +import java.net.ConnectException; + import junit.framework.Assert; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; @@ -36,6 +40,7 @@ import org.junit.Before; import org.junit.Test; +import com.sun.jersey.api.client.ClientHandlerException; import com.sun.jersey.api.client.ClientResponse; public class TestTimelineClient { @@ -56,7 +61,7 @@ public void tearDown() { @Test public void testPostEntities() throws Exception { - mockClientResponse(ClientResponse.Status.OK, false); + mockClientResponse(ClientResponse.Status.OK, false, false); try { TimelinePutResponse response = client.putEntities(generateEntity()); Assert.assertEquals(0, response.getErrors().size()); @@ -67,7 +72,7 @@ public void testPostEntities() throws Exception { @Test public void testPostEntitiesWithError() throws Exception { - mockClientResponse(ClientResponse.Status.OK, true); + mockClientResponse(ClientResponse.Status.OK, true, false); try { TimelinePutResponse response = client.putEntities(generateEntity()); Assert.assertEquals(1, response.getErrors().size()); @@ -84,7 +89,8 @@ public void testPostEntitiesWithError() throws Exception { @Test public void testPostEntitiesNoResponse() throws Exception { - mockClientResponse(ClientResponse.Status.INTERNAL_SERVER_ERROR, false); + mockClientResponse( + ClientResponse.Status.INTERNAL_SERVER_ERROR, false, false); try { client.putEntities(generateEntity()); Assert.fail("Exception is expected"); @@ -94,9 +100,25 @@ public void testPostEntitiesNoResponse() throws Exception { } } + @Test + public void testPostEntitiesConnectionRefused() throws Exception { + mockClientResponse(null, false, true); + try { + client.putEntities(generateEntity()); + Assert.fail("RuntimeException is expected"); + } catch (RuntimeException re) { + Assert.assertTrue(re instanceof ClientHandlerException); + } + } + private ClientResponse mockClientResponse(ClientResponse.Status status, - boolean hasError) { + boolean hasError, boolean hasRuntimeError) { ClientResponse response = mock(ClientResponse.class); + if (hasRuntimeError) { + doThrow(new ClientHandlerException(new ConnectException())).when(client) + .doPostingEntities(any(TimelineEntities.class)); + return response; + } doReturn(response).when(client) .doPostingEntities(any(TimelineEntities.class)); when(response.getClientResponseStatus()).thenReturn(status); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java index 4902cf2e95d..4e1f3c04a24 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java @@ -18,6 +18,8 @@ package org.apache.hadoop.yarn.server.applicationhistoryservice.webapp; +import static org.apache.hadoop.yarn.util.StringHelper.CSV_JOINER; + import java.io.IOException; import java.util.ArrayList; import java.util.Collection; @@ -54,10 +56,12 @@ import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; +import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.EntityIdentifier; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.GenericObjectMapper; -import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineStore; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.NameValuePair; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineReader.Field; +import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineStore; +import org.apache.hadoop.yarn.util.timeline.TimelineUtils; import org.apache.hadoop.yarn.webapp.BadRequestException; import com.google.inject.Inject; @@ -245,6 +249,17 @@ public TimelinePutResponse postEntities( return new TimelinePutResponse(); } try { + List entityIDs = new ArrayList(); + for (TimelineEntity entity : entities.getEntities()) { + EntityIdentifier entityID = + new EntityIdentifier(entity.getEntityId(), entity.getEntityType()); + entityIDs.add(entityID); + if (LOG.isDebugEnabled()) { + LOG.debug("Storing the entity " + entityID + ", JSON-style content: " + + TimelineUtils.dumpTimelineRecordtoJSON(entity)); + } + } + LOG.info("Storing entities: " + CSV_JOINER.join(entityIDs)); return store.put(entities); } catch (IOException e) { LOG.error("Error putting entities", e);