From 7f07c4d81023e3bf4bf8980e64cc9420ec31cf55 Mon Sep 17 00:00:00 2001 From: Xuan Date: Mon, 27 Apr 2015 10:36:42 -0700 Subject: [PATCH] YARN-3530. ATS throws exception on trying to filter results without otherinfo. Contributed by zhijie shen --- hadoop-yarn-project/CHANGES.txt | 3 + .../server/timeline/LeveldbTimelineStore.java | 34 ++++++- .../timeline/TimelineStoreTestUtils.java | 97 +++++++++++++------ 3 files changed, 103 insertions(+), 31 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 87db291b895..fdc3f4a6f2a 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -271,6 +271,9 @@ Release 2.8.0 - UNRELEASED YARN-3464. Race condition in LocalizerRunner kills localizer before localizing all resources. (Zhihai Xu via kasha) + YARN-3530. ATS throws exception on trying to filter results without otherinfo. + (zhijie shen via xgong) + Release 2.7.1 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java index d521f7012c6..8cfa0c76a99 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java @@ -357,6 +357,9 @@ public class LeveldbTimelineStore extends AbstractService iterator = new LeveldbIterator(db); iterator.seek(prefix); + if (fields == null) { + fields = EnumSet.allOf(Field.class); + } return getEntity(entityId, entityType, revStartTime, fields, iterator, prefix, prefix.length); } catch(DBException e) { @@ -373,10 +376,6 @@ public class LeveldbTimelineStore extends AbstractService private static TimelineEntity getEntity(String entityId, String entityType, Long startTime, EnumSet fields, LeveldbIterator iterator, byte[] prefix, int prefixlen) throws IOException { - if (fields == null) { - fields = EnumSet.allOf(Field.class); - } - TimelineEntity entity = new TimelineEntity(); boolean events = false; boolean lastEvent = false; @@ -590,6 +589,25 @@ public class LeveldbTimelineStore extends AbstractService String entityType, Long limit, Long starttime, Long endtime, String fromId, Long fromTs, Collection secondaryFilters, EnumSet fields, CheckAcl checkAcl) throws IOException { + // Even if other info and primary filter fields are not included, we + // still need to load them to match secondary filters when they are + // non-empty + if (fields == null) { + fields = EnumSet.allOf(Field.class); + } + boolean addPrimaryFilters = false; + boolean addOtherInfo = false; + if (secondaryFilters != null && secondaryFilters.size() > 0) { + if (!fields.contains(Field.PRIMARY_FILTERS)) { + fields.add(Field.PRIMARY_FILTERS); + addPrimaryFilters = true; + } + if (!fields.contains(Field.OTHER_INFO)) { + fields.add(Field.OTHER_INFO); + addOtherInfo = true; + } + } + LeveldbIterator iterator = null; try { KeyBuilder kb = KeyBuilder.newInstance().add(base).add(entityType); @@ -690,6 +708,14 @@ public class LeveldbTimelineStore extends AbstractService entity.setDomainId(DEFAULT_DOMAIN_ID); } if (checkAcl == null || checkAcl.check(entity)) { + // Remove primary filter and other info if they are added for + // matching secondary filters + if (addPrimaryFilters) { + entity.setPrimaryFilters(null); + } + if (addOtherInfo) { + entity.setOtherInfo(null); + } entities.addEntity(entity); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TimelineStoreTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TimelineStoreTestUtils.java index da71f46f161..6ac5a35bdce 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TimelineStoreTestUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TimelineStoreTestUtils.java @@ -28,6 +28,7 @@ import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -487,6 +488,13 @@ public class TimelineStoreTestUtils { primaryFilter, secondaryFilters, null, null).getEntities(); } + protected List getEntitiesWithFilters(String entityType, + NameValuePair primaryFilter, Collection secondaryFilters, + EnumSet fields) throws IOException { + return store.getEntities(entityType, null, null, null, null, null, + primaryFilter, secondaryFilters, fields, null).getEntities(); + } + protected List getEntities(String entityType, Long limit, Long windowStart, Long windowEnd, NameValuePair primaryFilter, EnumSet fields) throws IOException { @@ -751,38 +759,73 @@ public class TimelineStoreTestUtils { } public void testGetEntitiesWithSecondaryFilters() throws IOException { - // test using secondary filter - List entities = getEntitiesWithFilters("type_1", null, - goodTestingFilters); - assertEquals(3, entities.size()); - verifyEntityInfo(entityId1, entityType1, events1, EMPTY_REL_ENTITIES, - primaryFilters, otherInfo, entities.get(0), domainId1); - verifyEntityInfo(entityId1b, entityType1, events1, EMPTY_REL_ENTITIES, - primaryFilters, otherInfo, entities.get(1), domainId1); - verifyEntityInfo(entityId6, entityType1, EMPTY_EVENTS, EMPTY_REL_ENTITIES, - primaryFilters, otherInfo, entities.get(2), domainId2); + for (int i = 0; i < 4; ++i) { + // Verify the secondary filter works both other info is included or not. + EnumSet fields = null; + if (i == 1) { + fields = EnumSet.noneOf(Field.class); + } else if (i == 2) { + fields = EnumSet.of(Field.PRIMARY_FILTERS); + } else if (i == 3) { + fields = EnumSet.of(Field.OTHER_INFO); + } + // test using secondary filter + List entities = getEntitiesWithFilters("type_1", null, + goodTestingFilters, fields); + assertEquals(3, entities.size()); + verifyEntityInfo(entityId1, entityType1, + (i == 0 ? events1 : null), + (i == 0 ? EMPTY_REL_ENTITIES : null), + (i == 0 || i == 2 ? primaryFilters : null), + (i == 0 || i == 3 ? otherInfo : null), entities.get(0), domainId1); + verifyEntityInfo(entityId1b, entityType1, + (i == 0 ? events1 : null), + (i == 0 ? EMPTY_REL_ENTITIES : null), + (i == 0 || i == 2 ? primaryFilters : null), + (i == 0 || i == 3 ? otherInfo : null), entities.get(1), domainId1); + verifyEntityInfo(entityId6, entityType1, + (i == 0 ? EMPTY_EVENTS : null), + (i == 0 ? EMPTY_REL_ENTITIES : null), + (i == 0 || i == 2 ? primaryFilters : null), + (i == 0 || i == 3 ? otherInfo : null), entities.get(2), domainId2); - entities = getEntitiesWithFilters("type_1", userFilter, goodTestingFilters); - assertEquals(3, entities.size()); - verifyEntityInfo(entityId1, entityType1, events1, EMPTY_REL_ENTITIES, - primaryFilters, otherInfo, entities.get(0), domainId1); - verifyEntityInfo(entityId1b, entityType1, events1, EMPTY_REL_ENTITIES, - primaryFilters, otherInfo, entities.get(1), domainId1); - verifyEntityInfo(entityId6, entityType1, EMPTY_EVENTS, EMPTY_REL_ENTITIES, - primaryFilters, otherInfo, entities.get(2), domainId2); + entities = + getEntitiesWithFilters("type_1", userFilter, goodTestingFilters, fields); + assertEquals(3, entities.size()); + if (i == 0) { + verifyEntityInfo(entityId1, entityType1, + (i == 0 ? events1 : null), + (i == 0 ? EMPTY_REL_ENTITIES : null), + (i == 0 || i == 2 ? primaryFilters : null), + (i == 0 || i == 3 ? otherInfo : null), entities.get(0), domainId1); + verifyEntityInfo(entityId1b, entityType1, + (i == 0 ? events1 : null), + (i == 0 ? EMPTY_REL_ENTITIES : null), + (i == 0 || i == 2 ? primaryFilters : null), + (i == 0 || i == 3 ? otherInfo : null), entities.get(1), domainId1); + verifyEntityInfo(entityId6, entityType1, + (i == 0 ? EMPTY_EVENTS : null), + (i == 0 ? EMPTY_REL_ENTITIES : null), + (i == 0 || i == 2 ? primaryFilters : null), + (i == 0 || i == 3 ? otherInfo : null), entities.get(2), domainId2); + } - entities = getEntitiesWithFilters("type_1", null, - Collections.singleton(new NameValuePair("user", "none"))); - assertEquals(0, entities.size()); + entities = getEntitiesWithFilters("type_1", null, + Collections.singleton(new NameValuePair("user", "none")), fields); + assertEquals(0, entities.size()); - entities = getEntitiesWithFilters("type_1", null, badTestingFilters); - assertEquals(0, entities.size()); + entities = + getEntitiesWithFilters("type_1", null, badTestingFilters, fields); + assertEquals(0, entities.size()); - entities = getEntitiesWithFilters("type_1", userFilter, badTestingFilters); - assertEquals(0, entities.size()); + entities = + getEntitiesWithFilters("type_1", userFilter, badTestingFilters, fields); + assertEquals(0, entities.size()); - entities = getEntitiesWithFilters("type_5", null, badTestingFilters); - assertEquals(0, entities.size()); + entities = + getEntitiesWithFilters("type_5", null, badTestingFilters, fields); + assertEquals(0, entities.size()); + } } public void testGetEvents() throws IOException {