diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java index e9bd9651d2a..18e3d865c64 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java @@ -186,8 +186,6 @@ public class NMTimelinePublisher extends CompositeService { Math.round(cpuUsagePercentPerCore)); entity.addMetric(cpuMetric); } - entity.setIdPrefix(TimelineServiceHelper. - invertLong(container.getContainerStartTime())); ApplicationId appId = container.getContainerId(). getApplicationAttemptId().getApplicationId(); try { @@ -253,7 +251,6 @@ public class NMTimelinePublisher extends CompositeService { long containerStartTime = container.getContainerStartTime(); entity.addEvent(tEvent); entity.setCreatedTime(containerStartTime); - entity.setIdPrefix(TimelineServiceHelper.invertLong(containerStartTime)); dispatcher.getEventHandler().handle(new TimelinePublishEvent(entity, containerId.getApplicationAttemptId().getApplicationId())); } @@ -281,7 +278,6 @@ public class NMTimelinePublisher extends CompositeService { tEvent.setId(ContainerMetricsConstants.FINISHED_EVENT_TYPE); tEvent.setTimestamp(containerFinishTime); entity.addEvent(tEvent); - entity.setIdPrefix(TimelineServiceHelper.invertLong(containerStartTime)); dispatcher.getEventHandler().handle(new TimelinePublishEvent(entity, containerId.getApplicationAttemptId().getApplicationId())); @@ -299,8 +295,6 @@ public class NMTimelinePublisher extends CompositeService { tEvent.setId(eventType); tEvent.setTimestamp(event.getTimestamp()); entity.addEvent(tEvent); - entity.setIdPrefix(TimelineServiceHelper. - invertLong(container.getContainerStartTime())); ApplicationId appId = container.getContainerId(). getApplicationAttemptId().getApplicationId(); @@ -336,6 +330,8 @@ public class NMTimelinePublisher extends CompositeService { ContainerId containerId) { ContainerEntity entity = new ContainerEntity(); entity.setId(containerId.toString()); + entity.setIdPrefix(TimelineServiceHelper.invertLong( + containerId.getContainerId())); Identifier parentIdentifier = new Identifier(); parentIdentifier .setType(TimelineEntityType.YARN_APPLICATION_ATTEMPT.name()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/TestNMTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/TestNMTimelinePublisher.java index cf9ede0805f..ae51f8598f6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/TestNMTimelinePublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/TestNMTimelinePublisher.java @@ -46,6 +46,7 @@ import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationContainerFinishedEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree; +import org.apache.hadoop.yarn.util.TimelineServiceHelper; import org.junit.Assert; import org.junit.Test; import org.junit.After; @@ -140,6 +141,8 @@ public class TestNMTimelinePublisher { entity.getInfo().get(ContainerMetricsConstants.DIAGNOSTICS_INFO)); Assert.assertEquals(exitStatus, entity.getInfo().get(ContainerMetricsConstants.EXIT_STATUS_INFO)); + Assert.assertEquals(TimelineServiceHelper.invertLong( + cId.getContainerId()), entity.getIdPrefix()); } @Test public void testContainerResourceUsage() { @@ -148,29 +151,31 @@ public class TestNMTimelinePublisher { Container aContainer = mock(Container.class); when(aContainer.getContainerId()).thenReturn(ContainerId .newContainerId(ApplicationAttemptId.newInstance(appId, 1), 0L)); + long idPrefix = TimelineServiceHelper.invertLong( + aContainer.getContainerId().getContainerId()); publisher.reportContainerResourceUsage(aContainer, 1024L, 8F); - verifyPublishedResourceUsageMetrics(timelineClient, 1024L, 8); + verifyPublishedResourceUsageMetrics(timelineClient, 1024L, 8, idPrefix); timelineClient.reset(); publisher.reportContainerResourceUsage(aContainer, 1024L, 0.8F); - verifyPublishedResourceUsageMetrics(timelineClient, 1024L, 1); + verifyPublishedResourceUsageMetrics(timelineClient, 1024L, 1, idPrefix); timelineClient.reset(); publisher.reportContainerResourceUsage(aContainer, 1024L, 0.49F); - verifyPublishedResourceUsageMetrics(timelineClient, 1024L, 0); + verifyPublishedResourceUsageMetrics(timelineClient, 1024L, 0, idPrefix); timelineClient.reset(); publisher.reportContainerResourceUsage(aContainer, 1024L, (float) ResourceCalculatorProcessTree.UNAVAILABLE); verifyPublishedResourceUsageMetrics(timelineClient, 1024L, - ResourceCalculatorProcessTree.UNAVAILABLE); + ResourceCalculatorProcessTree.UNAVAILABLE, idPrefix); } - private void verifyPublishedResourceUsageMetrics( - DummyTimelineClient timelineClient, long memoryUsage, int cpuUsage) { + private void verifyPublishedResourceUsageMetrics(DummyTimelineClient + dummyTimelineClient, long memoryUsage, int cpuUsage, long idPrefix) { TimelineEntity[] entities = null; for (int i = 0; i < 10; i++) { - entities = timelineClient.getLastPublishedEntities(); + entities = dummyTimelineClient.getLastPublishedEntities(); if (entities != null) { break; } @@ -188,6 +193,7 @@ public class TestNMTimelinePublisher { assertNotNull("entities are expected to be published", entities); assertEquals("Expected number of metrics notpublished", numberOfResourceMetrics, entities[0].getMetrics().size()); + assertEquals(idPrefix, entities[0].getIdPrefix()); Iterator metrics = entities[0].getMetrics().iterator(); while (metrics.hasNext()) { TimelineMetric metric = metrics.next(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java index 11db6b19a41..35093e390f4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java @@ -441,6 +441,8 @@ public class TimelineServiceV2Publisher extends AbstractSystemMetricsPublisher { ContainerId containerId) { ContainerEntity entity = new ContainerEntity(); entity.setId(containerId.toString()); + entity.setIdPrefix(TimelineServiceHelper.invertLong( + containerId.getContainerId())); entity.setParent(new Identifier(TimelineEntityType.YARN_APPLICATION_ATTEMPT .name(), containerId.getApplicationAttemptId().toString())); return entity; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java index ca26f58c06c..5a19410f2d7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java @@ -293,7 +293,8 @@ public class TestSystemMetricsPublisherForV2 { File appFile = new File(outputDirApp, timelineServiceFileName); Assert.assertTrue(appFile.exists()); verifyEntity(appFile, 2, - ContainerMetricsConstants.CREATED_IN_RM_EVENT_TYPE, 0, 0); + ContainerMetricsConstants.CREATED_IN_RM_EVENT_TYPE, 0, + TimelineServiceHelper.invertLong(containerId.getContainerId())); } @Test(timeout = 10000)