From 04e056f4f55649456f70c35bd506118346ca637a Mon Sep 17 00:00:00 2001 From: rohithsharmaks Date: Mon, 7 Dec 2015 15:16:56 +0530 Subject: [PATCH] YARN-3456. Improve handling of incomplete TimelineEntities. (Varun Saxena via rohithsharmaks) (cherry picked from commit 01a641bc447c464b2830d58addd482f47dbd92ae) --- hadoop-yarn-project/CHANGES.txt | 3 +++ .../client/api/impl/TimelineClientImpl.java | 8 +++++-- .../client/api/impl/TestTimelineClient.java | 9 ++++++++ .../server/timeline/TimelineDataManager.java | 6 +++++- .../timeline/webapp/TimelineWebServices.java | 2 ++ .../webapp/TestTimelineWebServices.java | 21 +++++++++++++++++++ 6 files changed, 46 insertions(+), 3 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 5634edb66db..1851776a70a 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -547,6 +547,9 @@ Release 2.8.0 - UNRELEASED YARN-4358. Reservation System: Improve relationship between SharingPolicy and ReservationAgent. (Carlo Curino via asuresh) + YARN-3456. Improve handling of incomplete TimelineEntities. (Varun Saxena + via rohithsharmaks) + OPTIMIZATIONS YARN-3339. TestDockerContainerExecutor should pull a single image and not diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java index 04c84ca5f73..019c7a5a119 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java @@ -29,7 +29,6 @@ import java.net.URL; import java.net.URLConnection; import java.security.GeneralSecurityException; import java.security.PrivilegedExceptionAction; -import java.util.Arrays; import javax.net.ssl.HostnameVerifier; import javax.net.ssl.HttpsURLConnection; @@ -301,7 +300,12 @@ public class TimelineClientImpl extends TimelineClient { public TimelinePutResponse putEntities( TimelineEntity... entities) throws IOException, YarnException { TimelineEntities entitiesContainer = new TimelineEntities(); - entitiesContainer.addEntities(Arrays.asList(entities)); + for (TimelineEntity entity : entities) { + if (entity.getEntityId() == null || entity.getEntityType() == null) { + throw new YarnException("Incomplete entity without entity id/type"); + } + entitiesContainer.addEntity(entity); + } ClientResponse resp = doPosting(entitiesContainer, null); return resp.getEntity(TimelinePutResponse.class); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java index 859a6c9e222..4c74c618ac1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java @@ -95,6 +95,15 @@ public class TestTimelineClient { } } + @Test + public void testPostIncompleteEntities() throws Exception { + try { + client.putEntities(new TimelineEntity()); + Assert.fail("Exception should have been thrown"); + } catch (YarnException e) { + } + } + @Test public void testPostEntitiesNoResponse() throws Exception { mockEntityClientResponse( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineDataManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineDataManager.java index 459fd4e75ce..23ff8e4ddac 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineDataManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineDataManager.java @@ -42,6 +42,7 @@ import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.timeline.TimelineReader.Field; import org.apache.hadoop.yarn.server.timeline.security.TimelineACLsManager; +import org.apache.hadoop.yarn.webapp.BadRequestException; import com.google.common.annotations.VisibleForTesting; @@ -338,7 +339,10 @@ public class TimelineDataManager extends AbstractService { entity.getDomainId().length() == 0) { entity.setDomainId(DEFAULT_DOMAIN_ID); } - + if (entity.getEntityId() == null || entity.getEntityType() == null) { + throw new BadRequestException("Incomplete entity without entity" + + " id/type"); + } // check if there is existing entity TimelineEntity existingEntity = null; try { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java index 90f4d396135..eb47ef241e1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java @@ -229,6 +229,8 @@ public class TimelineWebServices { } try { return timelineDataManager.postEntities(entities, callerUGI); + } catch (BadRequestException bre) { + throw bre; } catch (Exception e) { LOG.error("Error putting entities", e); throw new WebApplicationException(e, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServices.java index ab7cffd878d..74466a89b01 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServices.java @@ -502,6 +502,27 @@ public class TestTimelineWebServices extends JerseyTestBase { Assert.assertEquals("test type 1", entity.getEntityType()); } + @Test + public void testPostIncompleteEntities() throws Exception { + TimelineEntities entities = new TimelineEntities(); + TimelineEntity entity1 = new TimelineEntity(); + entity1.setEntityId("test id 1"); + entity1.setEntityType("test type 1"); + entity1.setStartTime(System.currentTimeMillis()); + entity1.setDomainId("domain_id_1"); + entities.addEntity(entity1); + // Add an entity with no id or type. + entities.addEntity(new TimelineEntity()); + WebResource r = resource(); + // One of the entities has no id or type. HTTP 400 will be returned + ClientResponse response = r.path("ws").path("v1").path("timeline") + .queryParam("user.name", "tester").accept(MediaType.APPLICATION_JSON) + .type(MediaType.APPLICATION_JSON).post(ClientResponse.class, entities); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + assertEquals(ClientResponse.Status.BAD_REQUEST, + response.getClientResponseStatus()); + } + @Test public void testPostEntitiesWithYarnACLsEnabled() throws Exception { AdminACLsManager oldAdminACLsManager =