Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-5093. created time shows 0 in most REST output (Varun Saxena via sjlee)

This commit is contained in:
Sangjin Lee 2016-05-24 10:33:04 -07:00
parent d5dfee24b1
commit 8c8183e515
7 changed files with 45 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
View File

@ -144,7 +144,7 @@ public class TimelineEntity implements Comparable<TimelineEntity> {
private NavigableSet<TimelineEvent> events = new TreeSet<>(); private NavigableSet<TimelineEvent> events = new TreeSet<>();
private HashMap<String, Set<String>> isRelatedToEntities = new HashMap<>(); private HashMap<String, Set<String>> isRelatedToEntities = new HashMap<>();
private HashMap<String, Set<String>> relatesToEntities = new HashMap<>(); private HashMap<String, Set<String>> relatesToEntities = new HashMap<>();
private long createdTime; private Long createdTime;
public TimelineEntity() { public TimelineEntity() {
identifier = new Identifier(); identifier = new Identifier();
@ -490,7 +490,7 @@ public class TimelineEntity implements Comparable<TimelineEntity> {
} }
@XmlElement(name = "createdtime") @XmlElement(name = "createdtime")
public long getCreatedTime() { public Long getCreatedTime() {
if (real == null) { if (real == null) {
return createdTime; return createdTime;
} else { } else {
@ -499,7 +499,7 @@ public class TimelineEntity implements Comparable<TimelineEntity> {
} }
@JsonSetter("createdtime") @JsonSetter("createdtime")
public void setCreatedTime(long createdTs) { public void setCreatedTime(Long createdTs) {
if (real == null) { if (real == null) {
this.createdTime = createdTs; this.createdTime = createdTs;
} else { } else {
@ -547,6 +547,16 @@ public class TimelineEntity implements Comparable<TimelineEntity> {
public int compareTo(TimelineEntity other) { public int compareTo(TimelineEntity other) {
int comparison = getType().compareTo(other.getType()); int comparison = getType().compareTo(other.getType());
if (comparison == 0) { if (comparison == 0) {
if (getCreatedTime() == null) {
if (other.getCreatedTime() == null) {
return getId().compareTo(other.getId());
} else {
return 1;
}
}
if (other.getCreatedTime() == null) {
return -1;
}
if (getCreatedTime() > other.getCreatedTime()) { if (getCreatedTime() > other.getCreatedTime()) {
// Order by created time desc // Order by created time desc
return -1; return -1;

30
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java
View File

@ -488,7 +488,7 @@ public class TestHBaseTimelineStorage {
ApplicationEntity entity = new ApplicationEntity(); ApplicationEntity entity = new ApplicationEntity();
String appId = "application_1000178881110_2002"; String appId = "application_1000178881110_2002";
entity.setId(appId); entity.setId(appId);
long cTime = 1425016501000L; Long cTime = 1425016501000L;
entity.setCreatedTime(cTime); entity.setCreatedTime(cTime);
// add the info map in Timeline Entity // add the info map in Timeline Entity
@ -546,7 +546,6 @@ public class TestHBaseTimelineStorage {
aggEntity.setId(appId); aggEntity.setId(appId);
aggEntity.setType(type); aggEntity.setType(type);
long cTime2 = 1425016502000L; long cTime2 = 1425016502000L;
long mTime2 = 1425026902000L;
aggEntity.setCreatedTime(cTime2); aggEntity.setCreatedTime(cTime2);
TimelineMetric aggMetric = new TimelineMetric(); TimelineMetric aggMetric = new TimelineMetric();
@ -574,8 +573,21 @@ public class TestHBaseTimelineStorage {
String flowVersion = "AB7822C10F1111"; String flowVersion = "AB7822C10F1111";
long runid = 1002345678919L; long runid = 1002345678919L;
hbi.write(cluster, user, flow, flowVersion, runid, appId, te); hbi.write(cluster, user, flow, flowVersion, runid, appId, te);
// Write entity again, this time without created time.
entity = new ApplicationEntity();
appId = "application_1000178881110_2002";
entity.setId(appId);
// add the info map in Timeline Entity
Map<String, Object> infoMap1 = new HashMap<>();
infoMap1.put("infoMapKey3", "infoMapValue1");
entity.addInfo(infoMap1);
te = new TimelineEntities();
te.addEntity(entity);
hbi.write(cluster, user, flow, flowVersion, runid, appId, te);
hbi.stop(); hbi.stop();
infoMap.putAll(infoMap1);
// retrieve the row // retrieve the row
byte[] rowKey = byte[] rowKey =
ApplicationRowKey.getRowKey(cluster, user, flow, runid, appId); ApplicationRowKey.getRowKey(cluster, user, flow, runid, appId);
@ -585,7 +597,7 @@ public class TestHBaseTimelineStorage {
Result result = new ApplicationTable().getResult(c1, conn, get); Result result = new ApplicationTable().getResult(c1, conn, get);
assertTrue(result != null); assertTrue(result != null);
assertEquals(16, result.size()); assertEquals(17, result.size());
// check the row key // check the row key
byte[] row1 = result.getRow(); byte[] row1 = result.getRow();
@ -596,10 +608,9 @@ public class TestHBaseTimelineStorage {
String id1 = ApplicationColumn.ID.readResult(result).toString(); String id1 = ApplicationColumn.ID.readResult(result).toString();
assertEquals(appId, id1); assertEquals(appId, id1);
Number val = Long cTime1 =
(Number) ApplicationColumn.CREATED_TIME.readResult(result); (Long) ApplicationColumn.CREATED_TIME.readResult(result);
long cTime1 = val.longValue(); assertEquals(cTime, cTime1);
assertEquals(cTime1, cTime);
Map<String, Object> infoColumns = Map<String, Object> infoColumns =
ApplicationColumnPrefix.INFO.readResults(result); ApplicationColumnPrefix.INFO.readResults(result);
@ -701,7 +712,7 @@ public class TestHBaseTimelineStorage {
String type = "world"; String type = "world";
entity.setId(id); entity.setId(id);
entity.setType(type); entity.setType(type);
long cTime = 1425016501000L; Long cTime = 1425016501000L;
entity.setCreatedTime(cTime); entity.setCreatedTime(cTime);
// add the info map in Timeline Entity // add the info map in Timeline Entity
@ -796,8 +807,7 @@ public class TestHBaseTimelineStorage {
String type1 = EntityColumn.TYPE.readResult(result).toString(); String type1 = EntityColumn.TYPE.readResult(result).toString();
assertEquals(type, type1); assertEquals(type, type1);
Number val = (Number) EntityColumn.CREATED_TIME.readResult(result); Long cTime1 = (Long) EntityColumn.CREATED_TIME.readResult(result);
long cTime1 = val.longValue();
assertEquals(cTime1, cTime); assertEquals(cTime1, cTime);
Map<String, Object> infoColumns = Map<String, Object> infoColumns =

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
View File

@ -204,7 +204,7 @@ public class FileSystemTimelineReaderImpl extends AbstractService
private static void mergeEntities(TimelineEntity entity1, private static void mergeEntities(TimelineEntity entity1,
TimelineEntity entity2) { TimelineEntity entity2) {
// Ideally created time wont change except in the case of issue from client. // Ideally created time wont change except in the case of issue from client.
if (entity2.getCreatedTime() > 0) { if (entity2.getCreatedTime() != null && entity2.getCreatedTime() > 0) {
entity1.setCreatedTime(entity2.getCreatedTime()); entity1.setCreatedTime(entity2.getCreatedTime());
} }
for (Entry<String, String> configEntry : entity2.getConfigs().entrySet()) { for (Entry<String, String> configEntry : entity2.getConfigs().entrySet()) {

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java
View File

@ -397,9 +397,8 @@ class ApplicationEntityReader extends GenericEntityReader {
TimelineEntityFilters filters = getFilters(); TimelineEntityFilters filters = getFilters();
// fetch created time // fetch created time
Number createdTime = Long createdTime = (Long) ApplicationColumn.CREATED_TIME.readResult(result);
(Number)ApplicationColumn.CREATED_TIME.readResult(result); entity.setCreatedTime(createdTime);
entity.setCreatedTime(createdTime.longValue());
EnumSet<Field> fieldsToRetrieve = getDataToRetrieve().getFieldsToRetrieve(); EnumSet<Field> fieldsToRetrieve = getDataToRetrieve().getFieldsToRetrieve();
// fetch is related to entities and match isRelatedTo filter. If isRelatedTo // fetch is related to entities and match isRelatedTo filter. If isRelatedTo

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java
View File

@ -526,8 +526,8 @@ class GenericEntityReader extends TimelineEntityReader {
TimelineEntityFilters filters = getFilters(); TimelineEntityFilters filters = getFilters();
// fetch created time // fetch created time
Number createdTime = (Number)EntityColumn.CREATED_TIME.readResult(result); Long createdTime = (Long) EntityColumn.CREATED_TIME.readResult(result);
entity.setCreatedTime(createdTime.longValue()); entity.setCreatedTime(createdTime);
EnumSet<Field> fieldsToRetrieve = getDataToRetrieve().getFieldsToRetrieve(); EnumSet<Field> fieldsToRetrieve = getDataToRetrieve().getFieldsToRetrieve();
// fetch is related to entities and match isRelatedTo filter. If isRelatedTo // fetch is related to entities and match isRelatedTo filter. If isRelatedTo

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java
View File

@ -173,7 +173,7 @@ public class TestTimelineReaderWebServices {
assertNotNull(entity); assertNotNull(entity);
assertEquals("id_1", entity.getId()); assertEquals("id_1", entity.getId());
assertEquals("app", entity.getType()); assertEquals("app", entity.getType());
assertEquals(1425016502000L, entity.getCreatedTime()); assertEquals((Long)1425016502000L, entity.getCreatedTime());
// Default view i.e. when no fields are specified, entity contains only // Default view i.e. when no fields are specified, entity contains only
// entity id, entity type and created time. // entity id, entity type and created time.
assertEquals(0, entity.getConfigs().size()); assertEquals(0, entity.getConfigs().size());
@ -196,7 +196,7 @@ public class TestTimelineReaderWebServices {
assertNotNull(entity); assertNotNull(entity);
assertEquals("id_1", entity.getId()); assertEquals("id_1", entity.getId());
assertEquals("app", entity.getType()); assertEquals("app", entity.getType());
assertEquals(1425016502000L, entity.getCreatedTime()); assertEquals((Long)1425016502000L, entity.getCreatedTime());
} finally { } finally {
client.destroy(); client.destroy();
} }

10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java
View File

@ -278,7 +278,7 @@ public class TestFileSystemTimelineReaderImpl {
Assert.assertEquals( Assert.assertEquals(
(new TimelineEntity.Identifier("app", "id_1")).toString(), (new TimelineEntity.Identifier("app", "id_1")).toString(),
result.getIdentifier().toString()); result.getIdentifier().toString());
Assert.assertEquals(1425016502000L, result.getCreatedTime()); Assert.assertEquals((Long)1425016502000L, result.getCreatedTime());
Assert.assertEquals(0, result.getConfigs().size()); Assert.assertEquals(0, result.getConfigs().size());
Assert.assertEquals(0, result.getMetrics().size()); Assert.assertEquals(0, result.getMetrics().size());
} }
@ -293,7 +293,7 @@ public class TestFileSystemTimelineReaderImpl {
Assert.assertEquals( Assert.assertEquals(
(new TimelineEntity.Identifier("app", "id_1")).toString(), (new TimelineEntity.Identifier("app", "id_1")).toString(),
result.getIdentifier().toString()); result.getIdentifier().toString());
Assert.assertEquals(1425016502000L, result.getCreatedTime()); Assert.assertEquals((Long)1425016502000L, result.getCreatedTime());
Assert.assertEquals(0, result.getConfigs().size()); Assert.assertEquals(0, result.getConfigs().size());
Assert.assertEquals(0, result.getMetrics().size()); Assert.assertEquals(0, result.getMetrics().size());
} }
@ -310,7 +310,7 @@ public class TestFileSystemTimelineReaderImpl {
Assert.assertEquals( Assert.assertEquals(
(new TimelineEntity.Identifier("app", "id_5")).toString(), (new TimelineEntity.Identifier("app", "id_5")).toString(),
result.getIdentifier().toString()); result.getIdentifier().toString());
Assert.assertEquals(1425016502050L, result.getCreatedTime()); Assert.assertEquals((Long)1425016502050L, result.getCreatedTime());
} }
@Test @Test
@ -324,7 +324,7 @@ public class TestFileSystemTimelineReaderImpl {
Assert.assertEquals( Assert.assertEquals(
(new TimelineEntity.Identifier("app", "id_1")).toString(), (new TimelineEntity.Identifier("app", "id_1")).toString(),
result.getIdentifier().toString()); result.getIdentifier().toString());
Assert.assertEquals(1425016502000L, result.getCreatedTime()); Assert.assertEquals((Long)1425016502000L, result.getCreatedTime());
Assert.assertEquals(3, result.getConfigs().size()); Assert.assertEquals(3, result.getConfigs().size());
Assert.assertEquals(3, result.getMetrics().size()); Assert.assertEquals(3, result.getMetrics().size());
Assert.assertEquals(2, result.getInfo().size()); Assert.assertEquals(2, result.getInfo().size());
@ -342,7 +342,7 @@ public class TestFileSystemTimelineReaderImpl {
Assert.assertEquals( Assert.assertEquals(
(new TimelineEntity.Identifier("app", "id_1")).toString(), (new TimelineEntity.Identifier("app", "id_1")).toString(),
result.getIdentifier().toString()); result.getIdentifier().toString());
Assert.assertEquals(1425016502000L, result.getCreatedTime()); Assert.assertEquals((Long)1425016502000L, result.getCreatedTime());
Assert.assertEquals(3, result.getConfigs().size()); Assert.assertEquals(3, result.getConfigs().size());
Assert.assertEquals(3, result.getMetrics().size()); Assert.assertEquals(3, result.getMetrics().size());
// All fields including events will be returned. // All fields including events will be returned.