diff --git a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-policy.xml b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-policy.xml index 2bf5c02e110..d282c5841c1 100644 --- a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-policy.xml +++ b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-policy.xml @@ -223,4 +223,15 @@ group list is separated by a blank. For e.g. "alice,bob users,wheel". A special value of "*" means all users are allowed. + + + security.collector-nodemanager.protocol.acl + * + ACL for CollectorNodemanagerProtocol, used by nodemanager + if timeline service v2 is enabled, for the timeline collector and nodemanager + to communicate with each other. + The ACL is a comma-separated list of user and group names. The user and + group list is separated by a blank. For e.g. "alice,bob users,wheel". + A special value of "*" means all users are allowed. + diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java index 53fe055f75e..cfa91f530a3 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java @@ -78,6 +78,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.util.TimelineServiceHelper; import com.google.common.annotations.VisibleForTesting; import com.sun.jersey.api.client.ClientHandlerException; @@ -1124,7 +1125,7 @@ public class JobHistoryEventHandler extends AbstractService private org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity createTaskEntity(HistoryEvent event, long timestamp, String taskId, String entityType, String relatedJobEntity, JobId jobId, - boolean setCreatedTime) { + boolean setCreatedTime, long taskIdPrefix) { org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity = createBaseEntity(event, timestamp, entityType, setCreatedTime); entity.setId(taskId); @@ -1133,6 +1134,7 @@ public class JobHistoryEventHandler extends AbstractService ((TaskStartedEvent)event).getTaskType().toString()); } entity.addIsRelatedToEntity(relatedJobEntity, jobId.toString()); + entity.setIdPrefix(taskIdPrefix); return entity; } @@ -1141,11 +1143,12 @@ public class JobHistoryEventHandler extends AbstractService private org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity createTaskAttemptEntity(HistoryEvent event, long timestamp, String taskAttemptId, String entityType, String relatedTaskEntity, - String taskId, boolean setCreatedTime) { + String taskId, boolean setCreatedTime, long taskAttemptIdPrefix) { org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity = createBaseEntity(event, timestamp, entityType, setCreatedTime); entity.setId(taskAttemptId); entity.addIsRelatedToEntity(relatedTaskEntity, taskId); + entity.setIdPrefix(taskAttemptIdPrefix); return entity; } @@ -1196,6 +1199,8 @@ public class JobHistoryEventHandler extends AbstractService String taskId = null; String taskAttemptId = null; boolean setCreatedTime = false; + long taskIdPrefix = 0; + long taskAttemptIdPrefix = 0; switch (event.getEventType()) { // Handle job events @@ -1218,15 +1223,21 @@ public class JobHistoryEventHandler extends AbstractService case TASK_STARTED: setCreatedTime = true; taskId = ((TaskStartedEvent)event).getTaskId().toString(); + taskIdPrefix = TimelineServiceHelper. + invertLong(((TaskStartedEvent)event).getStartTime()); break; case TASK_FAILED: taskId = ((TaskFailedEvent)event).getTaskId().toString(); + taskIdPrefix = TimelineServiceHelper. + invertLong(((TaskFailedEvent)event).getStartTime()); break; case TASK_UPDATED: taskId = ((TaskUpdatedEvent)event).getTaskId().toString(); break; case TASK_FINISHED: taskId = ((TaskFinishedEvent)event).getTaskId().toString(); + taskIdPrefix = TimelineServiceHelper. + invertLong(((TaskFinishedEvent)event).getStartTime()); break; case MAP_ATTEMPT_STARTED: case REDUCE_ATTEMPT_STARTED: @@ -1234,6 +1245,8 @@ public class JobHistoryEventHandler extends AbstractService taskId = ((TaskAttemptStartedEvent)event).getTaskId().toString(); taskAttemptId = ((TaskAttemptStartedEvent)event). getTaskAttemptId().toString(); + taskAttemptIdPrefix = TimelineServiceHelper. + invertLong(((TaskAttemptStartedEvent)event).getStartTime()); break; case CLEANUP_ATTEMPT_STARTED: case SETUP_ATTEMPT_STARTED: @@ -1253,16 +1266,22 @@ public class JobHistoryEventHandler extends AbstractService getTaskId().toString(); taskAttemptId = ((TaskAttemptUnsuccessfulCompletionEvent)event). getTaskAttemptId().toString(); + taskAttemptIdPrefix = TimelineServiceHelper.invertLong( + ((TaskAttemptUnsuccessfulCompletionEvent)event).getStartTime()); break; case MAP_ATTEMPT_FINISHED: taskId = ((MapAttemptFinishedEvent)event).getTaskId().toString(); taskAttemptId = ((MapAttemptFinishedEvent)event). getAttemptId().toString(); + taskAttemptIdPrefix = TimelineServiceHelper. + invertLong(((MapAttemptFinishedEvent)event).getStartTime()); break; case REDUCE_ATTEMPT_FINISHED: taskId = ((ReduceAttemptFinishedEvent)event).getTaskId().toString(); taskAttemptId = ((ReduceAttemptFinishedEvent)event). getAttemptId().toString(); + taskAttemptIdPrefix = TimelineServiceHelper. + invertLong(((ReduceAttemptFinishedEvent)event).getStartTime()); break; case SETUP_ATTEMPT_FINISHED: case CLEANUP_ATTEMPT_FINISHED: @@ -1291,12 +1310,12 @@ public class JobHistoryEventHandler extends AbstractService // TaskEntity tEntity = createTaskEntity(event, timestamp, taskId, MAPREDUCE_TASK_ENTITY_TYPE, MAPREDUCE_JOB_ENTITY_TYPE, - jobId, setCreatedTime); + jobId, setCreatedTime, taskIdPrefix); } else { // TaskAttemptEntity tEntity = createTaskAttemptEntity(event, timestamp, taskAttemptId, MAPREDUCE_TASK_ATTEMPT_ENTITY_TYPE, MAPREDUCE_TASK_ENTITY_TYPE, - taskId, setCreatedTime); + taskId, setCreatedTime, taskAttemptIdPrefix); } } try { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java index 9ea1b9aa922..3faad480b9d 100755 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java @@ -1530,7 +1530,7 @@ public abstract class TaskAttemptImpl implements StringUtils.join( LINE_SEPARATOR, taskAttempt.getDiagnostics()), taskAttempt.getCounters(), taskAttempt - .getProgressSplitBlock().burst()); + .getProgressSplitBlock().burst(), taskAttempt.launchTime); return tauce; } @@ -1943,35 +1943,35 @@ public abstract class TaskAttemptImpl implements this.container == null ? -1 : this.container.getNodeId().getPort(); if (attemptId.getTaskId().getTaskType() == TaskType.MAP) { MapAttemptFinishedEvent mfe = - new MapAttemptFinishedEvent(TypeConverter.fromYarn(attemptId), - TypeConverter.fromYarn(attemptId.getTaskId().getTaskType()), - state.toString(), - this.reportedStatus.mapFinishTime, - finishTime, - containerHostName, - containerNodePort, - this.nodeRackName == null ? "UNKNOWN" : this.nodeRackName, - this.reportedStatus.stateString, - getCounters(), - getProgressSplitBlock().burst()); - eventHandler.handle( - new JobHistoryEvent(attemptId.getTaskId().getJobId(), mfe)); + new MapAttemptFinishedEvent(TypeConverter.fromYarn(attemptId), + TypeConverter.fromYarn(attemptId.getTaskId().getTaskType()), + state.toString(), + this.reportedStatus.mapFinishTime, + finishTime, + containerHostName, + containerNodePort, + this.nodeRackName == null ? "UNKNOWN" : this.nodeRackName, + this.reportedStatus.stateString, + getCounters(), + getProgressSplitBlock().burst(), launchTime); + eventHandler.handle( + new JobHistoryEvent(attemptId.getTaskId().getJobId(), mfe)); } else { - ReduceAttemptFinishedEvent rfe = - new ReduceAttemptFinishedEvent(TypeConverter.fromYarn(attemptId), - TypeConverter.fromYarn(attemptId.getTaskId().getTaskType()), - state.toString(), - this.reportedStatus.shuffleFinishTime, - this.reportedStatus.sortFinishTime, - finishTime, - containerHostName, - containerNodePort, - this.nodeRackName == null ? "UNKNOWN" : this.nodeRackName, - this.reportedStatus.stateString, - getCounters(), - getProgressSplitBlock().burst()); - eventHandler.handle( - new JobHistoryEvent(attemptId.getTaskId().getJobId(), rfe)); + ReduceAttemptFinishedEvent rfe = + new ReduceAttemptFinishedEvent(TypeConverter.fromYarn(attemptId), + TypeConverter.fromYarn(attemptId.getTaskId().getTaskType()), + state.toString(), + this.reportedStatus.shuffleFinishTime, + this.reportedStatus.sortFinishTime, + finishTime, + containerHostName, + containerNodePort, + this.nodeRackName == null ? "UNKNOWN" : this.nodeRackName, + this.reportedStatus.stateString, + getCounters(), + getProgressSplitBlock().burst(), launchTime); + eventHandler.handle( + new JobHistoryEvent(attemptId.getTaskId().getJobId(), rfe)); } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java index 8a6fa304d4e..228ae24a955 100755 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java @@ -139,6 +139,8 @@ public abstract class TaskImpl implements Task, EventHandler { private final Set inProgressAttempts; private boolean historyTaskStartGenerated = false; + // Launch time reported in history events. + private long launchTime; private static final SingleArcTransition ATTEMPT_KILLED_TRANSITION = new AttemptKilledTransition(); @@ -705,8 +707,9 @@ public abstract class TaskImpl implements Task, EventHandler { } private void sendTaskStartedEvent() { + launchTime = getLaunchTime(); TaskStartedEvent tse = new TaskStartedEvent( - TypeConverter.fromYarn(taskId), getLaunchTime(), + TypeConverter.fromYarn(taskId), launchTime, TypeConverter.fromYarn(taskId.getTaskType()), getSplitsAsString()); eventHandler @@ -714,18 +717,19 @@ public abstract class TaskImpl implements Task, EventHandler { historyTaskStartGenerated = true; } - private static TaskFinishedEvent createTaskFinishedEvent(TaskImpl task, TaskStateInternal taskState) { + private static TaskFinishedEvent createTaskFinishedEvent(TaskImpl task, + TaskStateInternal taskState) { TaskFinishedEvent tfe = new TaskFinishedEvent(TypeConverter.fromYarn(task.taskId), TypeConverter.fromYarn(task.successfulAttempt), task.getFinishTime(task.successfulAttempt), TypeConverter.fromYarn(task.taskId.getTaskType()), - taskState.toString(), - task.getCounters()); + taskState.toString(), task.getCounters(), task.launchTime); return tfe; } - private static TaskFailedEvent createTaskFailedEvent(TaskImpl task, List diag, TaskStateInternal taskState, TaskAttemptId taId) { + private static TaskFailedEvent createTaskFailedEvent(TaskImpl task, + List diag, TaskStateInternal taskState, TaskAttemptId taId) { StringBuilder errorSb = new StringBuilder(); if (diag != null) { for (String d : diag) { @@ -740,7 +744,7 @@ public abstract class TaskImpl implements Task, EventHandler { errorSb.toString(), taskState.toString(), taId == null ? null : TypeConverter.fromYarn(taId), - task.getCounters()); + task.getCounters(), task.launchTime); return taskFailedEvent; } @@ -861,7 +865,8 @@ public abstract class TaskImpl implements Task, EventHandler { TaskFailedEvent tfe = new TaskFailedEvent(taskInfo.getTaskId(), taskInfo.getFinishTime(), taskInfo.getTaskType(), taskInfo.getError(), taskInfo.getTaskStatus(), - taskInfo.getFailedDueToAttemptId(), taskInfo.getCounters()); + taskInfo.getFailedDueToAttemptId(), taskInfo.getCounters(), + launchTime); eventHandler.handle(new JobHistoryEvent(taskId.getJobId(), tfe)); eventHandler.handle( new JobTaskEvent(taskId, getExternalState(taskState))); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java index 09527977def..0dc7642418c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java @@ -848,7 +848,8 @@ public class RMContainerAllocator extends RMContainerRequestor updateAMRMToken(response.getAMRMToken()); } - List finishedContainers = response.getCompletedContainersStatuses(); + List finishedContainers = + response.getCompletedContainersStatuses(); // propagate preemption requests final PreemptionMessage preemptReq = response.getPreemptionMessage(); @@ -877,16 +878,13 @@ public class RMContainerAllocator extends RMContainerRequestor handleUpdatedNodes(response); handleJobPriorityChange(response); - // handle receiving the timeline collector address for this app - String collectorAddr = response.getCollectorAddr(); + // Handle receiving the timeline collector address and token for this app. MRAppMaster.RunningAppContext appContext = (MRAppMaster.RunningAppContext)this.getContext(); - if (collectorAddr != null && !collectorAddr.isEmpty() - && appContext.getTimelineV2Client() != null) { - appContext.getTimelineV2Client().setTimelineServiceAddress( - response.getCollectorAddr()); + if (appContext.getTimelineV2Client() != null) { + appContext.getTimelineV2Client(). + setTimelineCollectorInfo(response.getCollectorInfo()); } - for (ContainerStatus cont : finishedContainers) { processFinishedContainer(cont); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java index ac510b39434..e2713191ac7 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java @@ -58,7 +58,7 @@ public class TestEvents { Counters counters = new Counters(); TaskAttemptFinishedEvent test = new TaskAttemptFinishedEvent(taskAttemptId, TaskType.REDUCE, "TEST", 123L, "RAKNAME", "HOSTNAME", "STATUS", - counters); + counters, 234); assertEquals(test.getAttemptId().toString(), taskAttemptId.toString()); assertEquals(test.getCounters(), counters); @@ -69,7 +69,7 @@ public class TestEvents { assertEquals(test.getTaskId(), tid); assertEquals(test.getTaskStatus(), "TEST"); assertEquals(test.getTaskType(), TaskType.REDUCE); - + assertEquals(234, test.getStartTime()); } /** diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java index caf8c6718a9..e35a84d537e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java @@ -148,7 +148,7 @@ public class TestJobHistoryEventHandler { // First completion event, but min-queue-size for batching flushes is 10 handleEvent(jheh, new JobHistoryEvent(t.jobId, new TaskFinishedEvent( - t.taskID, t.taskAttemptID, 0, TaskType.MAP, "", null))); + t.taskID, t.taskAttemptID, 0, TaskType.MAP, "", null, 0))); verify(mockWriter).flush(); } finally { @@ -184,7 +184,7 @@ public class TestJobHistoryEventHandler { for (int i = 0 ; i < 100 ; i++) { queueEvent(jheh, new JobHistoryEvent(t.jobId, new TaskFinishedEvent( - t.taskID, t.taskAttemptID, 0, TaskType.MAP, "", null))); + t.taskID, t.taskAttemptID, 0, TaskType.MAP, "", null, 0))); } handleNextNEvents(jheh, 9); @@ -229,7 +229,7 @@ public class TestJobHistoryEventHandler { for (int i = 0 ; i < 100 ; i++) { queueEvent(jheh, new JobHistoryEvent(t.jobId, new TaskFinishedEvent( - t.taskID, t.taskAttemptID, 0, TaskType.MAP, "", null))); + t.taskID, t.taskAttemptID, 0, TaskType.MAP, "", null, 0))); } handleNextNEvents(jheh, 9); @@ -272,7 +272,7 @@ public class TestJobHistoryEventHandler { for (int i = 0 ; i < 100 ; i++) { queueEvent(jheh, new JobHistoryEvent(t.jobId, new TaskFinishedEvent( - t.taskID, t.taskAttemptID, 0, TaskType.MAP, "", null))); + t.taskID, t.taskAttemptID, 0, TaskType.MAP, "", null, 0))); } queueEvent(jheh, new JobHistoryEvent(t.jobId, new JobFinishedEvent( TypeConverter.fromYarn(t.jobId), 0, 10, 10, 0, 0, null, null, new Counters()))); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java index 6c51626af9d..e4a8a1a90b9 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java @@ -18,6 +18,7 @@ package org.apache.hadoop.mapreduce.v2.app.rm; +import static org.junit.Assert.assertEquals; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyFloat; import static org.mockito.Matchers.anyInt; @@ -27,6 +28,7 @@ import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -99,6 +101,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRespo import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.CollectorInfo; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerExitStatus; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -110,6 +113,7 @@ import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.client.api.TimelineV2Client; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.DrainDispatcher; @@ -121,6 +125,7 @@ import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; +import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; import org.apache.hadoop.yarn.server.api.records.NodeAction; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; @@ -140,6 +145,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretMan import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.Clock; import org.apache.hadoop.yarn.util.ControlledClock; +import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.util.SystemClock; import org.junit.After; import org.junit.Assert; @@ -748,6 +754,96 @@ public class TestRMContainerAllocator { } } + @Test + public void testUpdateCollectorInfo() throws Exception { + LOG.info("Running testUpdateCollectorInfo"); + Configuration conf = new Configuration(); + conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); + conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f); + ApplicationId appId = ApplicationId.newInstance(1, 1); + ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1); + JobId jobId = MRBuilderUtils.newJobId(appId, 0); + Job mockJob = mock(Job.class); + when(mockJob.getReport()).thenReturn( + MRBuilderUtils.newJobReport(jobId, "job", "user", JobState.RUNNING, 0, + 0, 0, 0, 0, 0, 0, "jobfile", null, false, "")); + String localAddr = "localhost:1234"; + UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + + // Generate a timeline delegation token. + TimelineDelegationTokenIdentifier ident = + new TimelineDelegationTokenIdentifier(new Text(ugi.getUserName()), + new Text("renewer"), null); + ident.setSequenceNumber(1); + Token collectorToken = + new Token(ident.getBytes(), + new byte[0], TimelineDelegationTokenIdentifier.KIND_NAME, + new Text(localAddr)); + org.apache.hadoop.yarn.api.records.Token token = + org.apache.hadoop.yarn.api.records.Token.newInstance( + collectorToken.getIdentifier(), collectorToken.getKind().toString(), + collectorToken.getPassword(), + collectorToken.getService().toString()); + CollectorInfo collectorInfo = CollectorInfo.newInstance(localAddr, token); + // Mock scheduler to server Allocate request. + final MockSchedulerForTimelineCollector mockScheduler = + new MockSchedulerForTimelineCollector(collectorInfo); + MyContainerAllocator allocator = + new MyContainerAllocator(null, conf, attemptId, mockJob, + SystemClock.getInstance()) { + @Override + protected void register() { + } + + @Override + protected ApplicationMasterProtocol createSchedulerProxy() { + return mockScheduler; + } + }; + // Initially UGI should have no tokens. + ArrayList> tokens = + new ArrayList<>(ugi.getTokens()); + assertEquals(0, tokens.size()); + TimelineV2Client client = spy(TimelineV2Client.createTimelineClient(appId)); + client.init(conf); + when(((RunningAppContext)allocator.getContext()).getTimelineV2Client()). + thenReturn(client); + + // Send allocate request to RM and fetch collector address and token. + allocator.schedule(); + verify(client).setTimelineCollectorInfo(collectorInfo); + // Verify if token has been updated in UGI. + tokens = new ArrayList<>(ugi.getTokens()); + assertEquals(1, tokens.size()); + assertEquals(TimelineDelegationTokenIdentifier.KIND_NAME, + tokens.get(0).getKind()); + assertEquals(collectorToken.decodeIdentifier(), + tokens.get(0).decodeIdentifier()); + + // Generate new collector token, send allocate request to RM and fetch the + // new token. + ident.setSequenceNumber(100); + Token collectorToken1 = + new Token(ident.getBytes(), + new byte[0], TimelineDelegationTokenIdentifier.KIND_NAME, + new Text(localAddr)); + token = org.apache.hadoop.yarn.api.records.Token.newInstance( + collectorToken1.getIdentifier(), collectorToken1.getKind().toString(), + collectorToken1.getPassword(), collectorToken1.getService().toString()); + collectorInfo = CollectorInfo.newInstance(localAddr, token); + mockScheduler.updateCollectorInfo(collectorInfo); + allocator.schedule(); + verify(client).setTimelineCollectorInfo(collectorInfo); + // Verify if new token has been updated in UGI. + tokens = new ArrayList<>(ugi.getTokens()); + assertEquals(1, tokens.size()); + assertEquals(TimelineDelegationTokenIdentifier.KIND_NAME, + tokens.get(0).getKind()); + assertEquals(collectorToken1.decodeIdentifier(), + tokens.get(0).decodeIdentifier()); + allocator.close(); + } + @Test public void testMapReduceScheduling() throws Exception { @@ -3488,6 +3584,46 @@ public class TestRMContainerAllocator { } } + private final static class MockSchedulerForTimelineCollector + implements ApplicationMasterProtocol { + private CollectorInfo collectorInfo; + + private MockSchedulerForTimelineCollector(CollectorInfo info) { + this.collectorInfo = info; + } + + private void updateCollectorInfo(CollectorInfo info) { + collectorInfo = info; + } + + @Override + public RegisterApplicationMasterResponse registerApplicationMaster( + RegisterApplicationMasterRequest request) throws YarnException, + IOException { + return Records.newRecord(RegisterApplicationMasterResponse.class); + } + + @Override + public FinishApplicationMasterResponse finishApplicationMaster( + FinishApplicationMasterRequest request) throws YarnException, + IOException { + return FinishApplicationMasterResponse.newInstance(false); + } + + @Override + public AllocateResponse allocate(AllocateRequest request) + throws YarnException, IOException { + AllocateResponse response = AllocateResponse.newInstance( + request.getResponseId(), Collections.emptyList(), + Collections.emptyList(), + Collections.emptyList(), + Resource.newInstance(512000, 1024), null, 10, null, + Collections.emptyList()); + response.setCollectorInfo(collectorInfo); + return response; + } + } + public static void main(String[] args) throws Exception { TestRMContainerAllocator t = new TestRMContainerAllocator(); t.testSimple(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/MapAttemptFinishedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/MapAttemptFinishedEvent.java index 3121c4e0016..2b1357ea859 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/MapAttemptFinishedEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/MapAttemptFinishedEvent.java @@ -32,9 +32,10 @@ import org.apache.hadoop.mapreduce.util.JobHistoryEventUtils; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric; +import org.apache.hadoop.yarn.util.SystemClock; /** - * Event to record successful completion of a map attempt + * Event to record successful completion of a map attempt. * */ @InterfaceAudience.Private @@ -58,9 +59,10 @@ public class MapAttemptFinishedEvent implements HistoryEvent { int[] cpuUsages; int[] vMemKbytes; int[] physMemKbytes; + private long startTime; /** - * Create an event for successful completion of map attempts + * Create an event for successful completion of map attempts. * @param id Task Attempt ID * @param taskType Type of the task * @param taskStatus Status of the task @@ -77,12 +79,13 @@ public class MapAttemptFinishedEvent implements HistoryEvent { * virtual memory and physical memory. * * If you have no splits data, code {@code null} for this - * parameter. + * parameter. + * @param startTs Task start time to be used for writing entity to ATSv2. */ - public MapAttemptFinishedEvent - (TaskAttemptID id, TaskType taskType, String taskStatus, - long mapFinishTime, long finishTime, String hostname, int port, - String rackName, String state, Counters counters, int[][] allSplits) { + public MapAttemptFinishedEvent(TaskAttemptID id, TaskType taskType, + String taskStatus, long mapFinishTime, long finishTime, String hostname, + int port, String rackName, String state, Counters counters, + int[][] allSplits, long startTs) { this.attemptId = id; this.taskType = taskType; this.taskStatus = taskStatus; @@ -98,6 +101,16 @@ public class MapAttemptFinishedEvent implements HistoryEvent { this.cpuUsages = ProgressSplitsBlock.arrayGetCPUTime(allSplits); this.vMemKbytes = ProgressSplitsBlock.arrayGetVMemKbytes(allSplits); this.physMemKbytes = ProgressSplitsBlock.arrayGetPhysMemKbytes(allSplits); + this.startTime = startTs; + } + + public MapAttemptFinishedEvent(TaskAttemptID id, TaskType taskType, + String taskStatus, long mapFinishTime, long finishTime, String hostname, + int port, String rackName, String state, Counters counters, + int[][] allSplits) { + this(id, taskType, taskStatus, mapFinishTime, finishTime, hostname, port, + rackName, state, counters, allSplits, + SystemClock.getInstance().getTime()); } /** @@ -117,15 +130,13 @@ public class MapAttemptFinishedEvent implements HistoryEvent { * @param counters Counters for the attempt */ @Deprecated - public MapAttemptFinishedEvent - (TaskAttemptID id, TaskType taskType, String taskStatus, - long mapFinishTime, long finishTime, String hostname, - String state, Counters counters) { + public MapAttemptFinishedEvent(TaskAttemptID id, TaskType taskType, + String taskStatus, long mapFinishTime, long finishTime, String hostname, + String state, Counters counters) { this(id, taskType, taskStatus, mapFinishTime, finishTime, hostname, -1, "", state, counters, null); } - - + MapAttemptFinishedEvent() {} public Object getDatum() { @@ -175,38 +186,56 @@ public class MapAttemptFinishedEvent implements HistoryEvent { this.physMemKbytes = AvroArrayUtils.fromAvro(datum.getPhysMemKbytes()); } - /** Get the task ID */ - public TaskID getTaskId() { return attemptId.getTaskID(); } - /** Get the attempt id */ + /** Gets the task ID. */ + public TaskID getTaskId() { + return attemptId.getTaskID(); + } + /** Gets the attempt id. */ public TaskAttemptID getAttemptId() { return attemptId; } - /** Get the task type */ + /** Gets the task type. */ public TaskType getTaskType() { return taskType; } - /** Get the task status */ + /** Gets the task status. */ public String getTaskStatus() { return taskStatus.toString(); } - /** Get the map phase finish time */ + /** Gets the map phase finish time. */ public long getMapFinishTime() { return mapFinishTime; } - /** Get the attempt finish time */ + /** Gets the attempt finish time. */ public long getFinishTime() { return finishTime; } - /** Get the host name */ + /** + * Gets the task attempt start time. + * @return task attempt start time. + */ + public long getStartTime() { + return startTime; + } + /** Gets the host name. */ public String getHostname() { return hostname.toString(); } - /** Get the tracker rpc port */ + /** Gets the tracker rpc port. */ public int getPort() { return port; } - /** Get the rack name */ + /** Gets the rack name. */ public String getRackName() { return rackName == null ? null : rackName.toString(); } - - /** Get the state string */ - public String getState() { return state.toString(); } - /** Get the counters */ - Counters getCounters() { return counters; } - /** Get the event type */ + /** + * Gets the attempt state string. + * @return map attempt state + */ + public String getState() { + return state.toString(); + } + /** + * Gets the counters. + * @return counters + */ + Counters getCounters() { + return counters; + } + /** Gets the event type. */ public EventType getEventType() { return EventType.MAP_ATTEMPT_FINISHED; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/ReduceAttemptFinishedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/ReduceAttemptFinishedEvent.java index 9c0f09b017d..5a16f834acb 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/ReduceAttemptFinishedEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/ReduceAttemptFinishedEvent.java @@ -32,6 +32,7 @@ import org.apache.hadoop.mapreduce.util.JobHistoryEventUtils; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric; +import org.apache.hadoop.yarn.util.SystemClock; /** * Event to record successful completion of a reduce attempt @@ -59,6 +60,7 @@ public class ReduceAttemptFinishedEvent implements HistoryEvent { int[] cpuUsages; int[] vMemKbytes; int[] physMemKbytes; + private long startTime; /** * Create an event to record completion of a reduce attempt @@ -76,13 +78,13 @@ public class ReduceAttemptFinishedEvent implements HistoryEvent { * @param allSplits the "splits", or a pixelated graph of various * measurable worker node state variables against progress. * Currently there are four; wallclock time, CPU time, - * virtual memory and physical memory. + * virtual memory and physical memory. + * @param startTs Task start time to be used for writing entity to ATSv2. */ - public ReduceAttemptFinishedEvent - (TaskAttemptID id, TaskType taskType, String taskStatus, - long shuffleFinishTime, long sortFinishTime, long finishTime, - String hostname, int port, String rackName, String state, - Counters counters, int[][] allSplits) { + public ReduceAttemptFinishedEvent(TaskAttemptID id, TaskType taskType, + String taskStatus, long shuffleFinishTime, long sortFinishTime, + long finishTime, String hostname, int port, String rackName, + String state, Counters counters, int[][] allSplits, long startTs) { this.attemptId = id; this.taskType = taskType; this.taskStatus = taskStatus; @@ -99,6 +101,16 @@ public class ReduceAttemptFinishedEvent implements HistoryEvent { this.cpuUsages = ProgressSplitsBlock.arrayGetCPUTime(allSplits); this.vMemKbytes = ProgressSplitsBlock.arrayGetVMemKbytes(allSplits); this.physMemKbytes = ProgressSplitsBlock.arrayGetPhysMemKbytes(allSplits); + this.startTime = startTs; + } + + public ReduceAttemptFinishedEvent(TaskAttemptID id, TaskType taskType, + String taskStatus, long shuffleFinishTime, long sortFinishTime, + long finishTime, String hostname, int port, String rackName, + String state, Counters counters, int[][] allSplits) { + this(id, taskType, taskStatus, shuffleFinishTime, sortFinishTime, + finishTime, hostname, port, rackName, state, counters, allSplits, + SystemClock.getInstance().getTime()); } /** @@ -118,13 +130,12 @@ public class ReduceAttemptFinishedEvent implements HistoryEvent { * @param state State of the attempt * @param counters Counters for the attempt */ - public ReduceAttemptFinishedEvent - (TaskAttemptID id, TaskType taskType, String taskStatus, - long shuffleFinishTime, long sortFinishTime, long finishTime, - String hostname, String state, Counters counters) { + public ReduceAttemptFinishedEvent(TaskAttemptID id, TaskType taskType, + String taskStatus, long shuffleFinishTime, long sortFinishTime, + long finishTime, String hostname, String state, Counters counters) { this(id, taskType, taskStatus, - shuffleFinishTime, sortFinishTime, finishTime, - hostname, -1, "", state, counters, null); + shuffleFinishTime, sortFinishTime, finishTime, + hostname, -1, "", state, counters, null); } ReduceAttemptFinishedEvent() {} @@ -178,39 +189,55 @@ public class ReduceAttemptFinishedEvent implements HistoryEvent { this.physMemKbytes = AvroArrayUtils.fromAvro(datum.getPhysMemKbytes()); } - /** Get the Task ID */ + /** Gets the Task ID. */ public TaskID getTaskId() { return attemptId.getTaskID(); } - /** Get the attempt id */ + /** Gets the attempt id. */ public TaskAttemptID getAttemptId() { return attemptId; } - /** Get the task type */ + /** Gets the task type. */ public TaskType getTaskType() { return taskType; } - /** Get the task status */ + /** Gets the task status. */ public String getTaskStatus() { return taskStatus.toString(); } - /** Get the finish time of the sort phase */ + /** Gets the finish time of the sort phase. */ public long getSortFinishTime() { return sortFinishTime; } - /** Get the finish time of the shuffle phase */ + /** Gets the finish time of the shuffle phase. */ public long getShuffleFinishTime() { return shuffleFinishTime; } - /** Get the finish time of the attempt */ + /** Gets the finish time of the attempt. */ public long getFinishTime() { return finishTime; } - /** Get the name of the host where the attempt ran */ + /** + * Gets the start time. + * @return task attempt start time. + */ + public long getStartTime() { + return startTime; + } + /** Gets the name of the host where the attempt ran. */ public String getHostname() { return hostname.toString(); } - /** Get the tracker rpc port */ + /** Gets the tracker rpc port. */ public int getPort() { return port; } - /** Get the rack name of the node where the attempt ran */ + /** Gets the rack name of the node where the attempt ran. */ public String getRackName() { return rackName == null ? null : rackName.toString(); } - - /** Get the state string */ - public String getState() { return state.toString(); } - /** Get the counters for the attempt */ - Counters getCounters() { return counters; } - /** Get the event type */ + /** + * Gets the state string. + * @return reduce attempt state + */ + public String getState() { + return state.toString(); + } + /** + * Gets the counters. + * @return counters + */ + Counters getCounters() { + return counters; + } + /** Gets the event type. */ public EventType getEventType() { return EventType.REDUCE_ATTEMPT_FINISHED; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptFinishedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptFinishedEvent.java index a931ca24033..c28c21605df 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptFinishedEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptFinishedEvent.java @@ -31,6 +31,7 @@ import org.apache.hadoop.mapreduce.util.JobHistoryEventUtils; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric; +import org.apache.hadoop.yarn.util.SystemClock; /** * Event to record successful task completion @@ -50,10 +51,11 @@ public class TaskAttemptFinishedEvent implements HistoryEvent { private String hostname; private String state; private Counters counters; + private long startTime; /** - * Create an event to record successful finishes for setup and cleanup - * attempts + * Create an event to record successful finishes for setup and cleanup + * attempts. * @param id Attempt ID * @param taskType Type of task * @param taskStatus Status of task @@ -61,11 +63,12 @@ public class TaskAttemptFinishedEvent implements HistoryEvent { * @param hostname Host where the attempt executed * @param state State string * @param counters Counters for the attempt + * @param startTs Task start time to be used for writing entity to ATSv2. */ public TaskAttemptFinishedEvent(TaskAttemptID id, TaskType taskType, String taskStatus, long finishTime, String rackName, - String hostname, String state, Counters counters) { + String hostname, String state, Counters counters, long startTs) { this.attemptId = id; this.taskType = taskType; this.taskStatus = taskStatus; @@ -74,6 +77,14 @@ public class TaskAttemptFinishedEvent implements HistoryEvent { this.hostname = hostname; this.state = state; this.counters = counters; + this.startTime = startTs; + } + + public TaskAttemptFinishedEvent(TaskAttemptID id, TaskType taskType, + String taskStatus, long finishTime, String rackName, String hostname, + String state, Counters counters) { + this(id, taskType, taskStatus, finishTime, rackName, hostname, state, + counters, SystemClock.getInstance().getTime()); } TaskAttemptFinishedEvent() {} @@ -107,33 +118,43 @@ public class TaskAttemptFinishedEvent implements HistoryEvent { this.counters = EventReader.fromAvro(datum.getCounters()); } - /** Get the task ID */ + /** Gets the task ID. */ public TaskID getTaskId() { return attemptId.getTaskID(); } - /** Get the task attempt id */ + /** Gets the task attempt id. */ public TaskAttemptID getAttemptId() { return attemptId; } - /** Get the task type */ + /** Gets the task type. */ public TaskType getTaskType() { return taskType; } - /** Get the task status */ + /** Gets the task status. */ public String getTaskStatus() { return taskStatus.toString(); } - /** Get the attempt finish time */ + /** Gets the attempt finish time. */ public long getFinishTime() { return finishTime; } - /** Get the host where the attempt executed */ + /** + * Gets the task attempt start time to be used while publishing to ATSv2. + * @return task attempt start time. + */ + public long getStartTime() { + return startTime; + } + /** Gets the host where the attempt executed. */ public String getHostname() { return hostname.toString(); } - /** Get the rackname where the attempt executed */ + /** Gets the rackname where the attempt executed. */ public String getRackName() { return rackName == null ? null : rackName.toString(); } - /** Get the state string */ + /** + * Gets the state string. + * @return task attempt state. + */ public String getState() { return state.toString(); } - /** Get the counters for the attempt */ + /** Gets the counters for the attempt. */ Counters getCounters() { return counters; } - /** Get the event type */ + /** Gets the event type. */ public EventType getEventType() { // Note that the task type can be setup/map/reduce/cleanup but the // attempt-type can only be map/reduce. diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java index 175296725ff..9afa09384cc 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java @@ -33,6 +33,7 @@ import org.apache.hadoop.mapreduce.util.JobHistoryEventUtils; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric; +import org.apache.hadoop.yarn.util.SystemClock; /** * Event to record unsuccessful (Killed/Failed) completion of task attempts @@ -58,10 +59,11 @@ public class TaskAttemptUnsuccessfulCompletionEvent implements HistoryEvent { int[] cpuUsages; int[] vMemKbytes; int[] physMemKbytes; + private long startTime; private static final Counters EMPTY_COUNTERS = new Counters(); /** - * Create an event to record the unsuccessful completion of attempts + * Create an event to record the unsuccessful completion of attempts. * @param id Attempt ID * @param taskType Type of the task * @param status Status of the attempt @@ -75,12 +77,13 @@ public class TaskAttemptUnsuccessfulCompletionEvent implements HistoryEvent { * measurable worker node state variables against progress. * Currently there are four; wallclock time, CPU time, * virtual memory and physical memory. + * @param startTs Task start time to be used for writing entity to ATSv2. */ public TaskAttemptUnsuccessfulCompletionEvent (TaskAttemptID id, TaskType taskType, String status, long finishTime, String hostname, int port, String rackName, - String error, Counters counters, int[][] allSplits) { + String error, Counters counters, int[][] allSplits, long startTs) { this.attemptId = id; this.taskType = taskType; this.status = status; @@ -99,6 +102,15 @@ public class TaskAttemptUnsuccessfulCompletionEvent implements HistoryEvent { ProgressSplitsBlock.arrayGetVMemKbytes(allSplits); this.physMemKbytes = ProgressSplitsBlock.arrayGetPhysMemKbytes(allSplits); + this.startTime = startTs; + } + + public TaskAttemptUnsuccessfulCompletionEvent(TaskAttemptID id, + TaskType taskType, String status, long finishTime, String hostname, + int port, String rackName, String error, Counters counters, + int[][] allSplits) { + this(id, taskType, status, finishTime, hostname, port, rackName, error, + counters, allSplits, SystemClock.getInstance().getTime()); } /** @@ -190,39 +202,49 @@ public class TaskAttemptUnsuccessfulCompletionEvent implements HistoryEvent { AvroArrayUtils.fromAvro(datum.getPhysMemKbytes()); } - /** Get the task id */ + /** Gets the task id. */ public TaskID getTaskId() { return attemptId.getTaskID(); } - /** Get the task type */ + /** Gets the task type. */ public TaskType getTaskType() { return TaskType.valueOf(taskType.toString()); } - /** Get the attempt id */ + /** Gets the attempt id. */ public TaskAttemptID getTaskAttemptId() { return attemptId; } - /** Get the finish time */ + /** Gets the finish time. */ public long getFinishTime() { return finishTime; } - /** Get the name of the host where the attempt executed */ + /** + * Gets the task attempt start time to be used while publishing to ATSv2. + * @return task attempt start time. + */ + public long getStartTime() { + return startTime; + } + /** Gets the name of the host where the attempt executed. */ public String getHostname() { return hostname; } - /** Get the rpc port for the host where the attempt executed */ + /** Gets the rpc port for the host where the attempt executed. */ public int getPort() { return port; } - /** Get the rack name of the node where the attempt ran */ + /** Gets the rack name of the node where the attempt ran. */ public String getRackName() { return rackName == null ? null : rackName.toString(); } - /** Get the error string */ + /** Gets the error string. */ public String getError() { return error.toString(); } - /** Get the task status */ + /** + * Gets the task attempt status. + * @return task attempt status. + */ public String getTaskStatus() { return status.toString(); } - /** Get the counters */ + /** Gets the counters. */ Counters getCounters() { return counters; } - /** Get the event type */ + /** Gets the event type. */ public EventType getEventType() { // Note that the task type can be setup/map/reduce/cleanup but the // attempt-type can only be map/reduce. diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFailedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFailedEvent.java index d14350df54f..b4d9e410da2 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFailedEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFailedEvent.java @@ -32,6 +32,7 @@ import org.apache.hadoop.mapreduce.util.JobHistoryEventUtils; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric; +import org.apache.hadoop.yarn.util.SystemClock; /** * Event to record the failure of a task @@ -49,11 +50,12 @@ public class TaskFailedEvent implements HistoryEvent { private String status; private String error; private Counters counters; + private long startTime; private static final Counters EMPTY_COUNTERS = new Counters(); /** - * Create an event to record task failure + * Create an event to record task failure. * @param id Task ID * @param finishTime Finish time of the task * @param taskType Type of the task @@ -61,10 +63,11 @@ public class TaskFailedEvent implements HistoryEvent { * @param status Status * @param failedDueToAttempt The attempt id due to which the task failed * @param counters Counters for the task + * @param startTs task start time. */ public TaskFailedEvent(TaskID id, long finishTime, TaskType taskType, String error, String status, - TaskAttemptID failedDueToAttempt, Counters counters) { + TaskAttemptID failedDueToAttempt, Counters counters, long startTs) { this.id = id; this.finishTime = finishTime; this.taskType = taskType; @@ -72,15 +75,23 @@ public class TaskFailedEvent implements HistoryEvent { this.status = status; this.failedDueToAttempt = failedDueToAttempt; this.counters = counters; + this.startTime = startTs; + } + + public TaskFailedEvent(TaskID id, long finishTime, TaskType taskType, + String error, String status, TaskAttemptID failedDueToAttempt, + Counters counters) { + this(id, finishTime, taskType, error, status, failedDueToAttempt, counters, + SystemClock.getInstance().getTime()); } public TaskFailedEvent(TaskID id, long finishTime, - TaskType taskType, String error, String status, - TaskAttemptID failedDueToAttempt) { - this(id, finishTime, taskType, error, status, - failedDueToAttempt, EMPTY_COUNTERS); + TaskType taskType, String error, String status, + TaskAttemptID failedDueToAttempt) { + this(id, finishTime, taskType, error, status, failedDueToAttempt, + EMPTY_COUNTERS); } - + TaskFailedEvent() {} public Object getDatum() { @@ -118,27 +129,37 @@ public class TaskFailedEvent implements HistoryEvent { EventReader.fromAvro(datum.getCounters()); } - /** Get the task id */ + /** Gets the task id. */ public TaskID getTaskId() { return id; } - /** Get the error string */ + /** Gets the error string. */ public String getError() { return error; } - /** Get the finish time of the attempt */ + /** Gets the finish time of the attempt. */ public long getFinishTime() { return finishTime; } - /** Get the task type */ + /** + * Gets the task start time to be reported to ATSv2. + * @return task start time. + */ + public long getStartTime() { + return startTime; + } + /** Gets the task type. */ public TaskType getTaskType() { return taskType; } - /** Get the attempt id due to which the task failed */ + /** Gets the attempt id due to which the task failed. */ public TaskAttemptID getFailedAttemptID() { return failedDueToAttempt; } - /** Get the task status */ + /** + * Gets the task status. + * @return task status + */ public String getTaskStatus() { return status; } - /** Get task counters */ + /** Gets task counters. */ public Counters getCounters() { return counters; } - /** Get the event type */ + /** Gets the event type. */ public EventType getEventType() { return EventType.TASK_FAILED; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFinishedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFinishedEvent.java index 0bc43832636..97557c7e0b4 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFinishedEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFinishedEvent.java @@ -32,6 +32,7 @@ import org.apache.hadoop.mapreduce.util.JobHistoryEventUtils; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric; +import org.apache.hadoop.yarn.util.SystemClock; /** * Event to record the successful completion of a task @@ -49,27 +50,36 @@ public class TaskFinishedEvent implements HistoryEvent { private TaskType taskType; private String status; private Counters counters; - + private long startTime; + /** - * Create an event to record the successful completion of a task + * Create an event to record the successful completion of a task. * @param id Task ID * @param attemptId Task Attempt ID of the successful attempt for this task * @param finishTime Finish time of the task * @param taskType Type of the task * @param status Status string * @param counters Counters for the task + * @param startTs task start time */ public TaskFinishedEvent(TaskID id, TaskAttemptID attemptId, long finishTime, TaskType taskType, - String status, Counters counters) { + String status, Counters counters, long startTs) { this.taskid = id; this.successfulAttemptId = attemptId; this.finishTime = finishTime; this.taskType = taskType; this.status = status; this.counters = counters; + this.startTime = startTs; } - + + public TaskFinishedEvent(TaskID id, TaskAttemptID attemptId, long finishTime, + TaskType taskType, String status, Counters counters) { + this(id, attemptId, finishTime, taskType, status, counters, + SystemClock.getInstance().getTime()); + } + TaskFinishedEvent() {} public Object getDatum() { @@ -101,23 +111,33 @@ public class TaskFinishedEvent implements HistoryEvent { this.counters = EventReader.fromAvro(datum.getCounters()); } - /** Get task id */ + /** Gets task id. */ public TaskID getTaskId() { return taskid; } - /** Get successful task attempt id */ + /** Gets successful task attempt id. */ public TaskAttemptID getSuccessfulTaskAttemptId() { return successfulAttemptId; } - /** Get the task finish time */ + /** Gets the task finish time. */ public long getFinishTime() { return finishTime; } - /** Get task counters */ + /** + * Gets the task start time to be reported to ATSv2. + * @return task start time + */ + public long getStartTime() { + return startTime; + } + /** Gets task counters. */ public Counters getCounters() { return counters; } - /** Get task type */ + /** Gets task type. */ public TaskType getTaskType() { return taskType; } - /** Get task status */ + /** + * Gets task status. + * @return task status + */ public String getTaskStatus() { return status.toString(); } - /** Get event type */ + /** Gets event type. */ public EventType getEventType() { return EventType.TASK_FINISHED; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java index cbca3c8cdb1..19313d3c8d5 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java @@ -298,10 +298,10 @@ public class TestMRTimelineEventHandling { " does not exist.", jobEventFile.exists()); verifyEntity(jobEventFile, EventType.JOB_FINISHED.name(), - true, false, null); + true, false, null, false); Set cfgsToCheck = Sets.newHashSet("dummy_conf1", "dummy_conf2", "huge_dummy_conf1", "huge_dummy_conf2"); - verifyEntity(jobEventFile, null, false, true, cfgsToCheck); + verifyEntity(jobEventFile, null, false, true, cfgsToCheck, false); // for this test, we expect MR job metrics are published in YARN_APPLICATION String outputAppDir = @@ -322,8 +322,8 @@ public class TestMRTimelineEventHandling { "appEventFilePath: " + appEventFilePath + " does not exist.", appEventFile.exists()); - verifyEntity(appEventFile, null, true, false, null); - verifyEntity(appEventFile, null, false, true, cfgsToCheck); + verifyEntity(appEventFile, null, true, false, null, false); + verifyEntity(appEventFile, null, false, true, cfgsToCheck, false); // check for task event file String outputDirTask = @@ -344,7 +344,7 @@ public class TestMRTimelineEventHandling { " does not exist.", taskEventFile.exists()); verifyEntity(taskEventFile, EventType.TASK_FINISHED.name(), - true, false, null); + true, false, null, true); // check for task attempt event file String outputDirTaskAttempt = @@ -363,7 +363,7 @@ public class TestMRTimelineEventHandling { Assert.assertTrue("taskAttemptEventFileName: " + taskAttemptEventFilePath + " does not exist.", taskAttemptEventFile.exists()); verifyEntity(taskAttemptEventFile, EventType.MAP_ATTEMPT_FINISHED.name(), - true, false, null); + true, false, null, true); } /** @@ -380,12 +380,13 @@ public class TestMRTimelineEventHandling { * @throws IOException */ private void verifyEntity(File entityFile, String eventId, - boolean chkMetrics, boolean chkCfg, Set cfgsToVerify) - throws IOException { + boolean chkMetrics, boolean chkCfg, Set cfgsToVerify, + boolean checkIdPrefix) throws IOException { BufferedReader reader = null; String strLine; try { reader = new BufferedReader(new FileReader(entityFile)); + long idPrefix = -1; while ((strLine = reader.readLine()) != null) { if (strLine.trim().length() > 0) { org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity @@ -394,6 +395,19 @@ public class TestMRTimelineEventHandling { strLine.trim(), org.apache.hadoop.yarn.api.records.timelineservice. TimelineEntity.class); + + LOG.info("strLine.trim()= " + strLine.trim()); + if (checkIdPrefix) { + Assert.assertTrue("Entity ID prefix expected to be > 0", + entity.getIdPrefix() > 0); + if (idPrefix == -1) { + idPrefix = entity.getIdPrefix(); + } else { + Assert.assertEquals("Entity ID prefix should be same across " + + "each publish of same entity", + idPrefix, entity.getIdPrefix()); + } + } if (eventId == null) { // Job metrics are published without any events for // ApplicationEntity. There is also possibility that some other diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/JobHistoryFileReplayMapperV1.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/JobHistoryFileReplayMapperV1.java index 447ea4e6b49..d553596b2fa 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/JobHistoryFileReplayMapperV1.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/JobHistoryFileReplayMapperV1.java @@ -37,7 +37,6 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; import org.apache.hadoop.yarn.client.api.TimelineClient; -import org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl; import org.apache.hadoop.yarn.exceptions.YarnException; @@ -54,7 +53,7 @@ class JobHistoryFileReplayMapperV1 extends public void map(IntWritable key, IntWritable val, Context context) throws IOException { // collect the apps it needs to process - TimelineClient tlc = new TimelineClientImpl(); + TimelineClient tlc = TimelineClient.createTimelineClient(); TimelineEntityConverterV1 converter = new TimelineEntityConverterV1(); JobHistoryFileReplayHelper helper = new JobHistoryFileReplayHelper(context); int replayMode = helper.getReplayMode(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/SimpleEntityWriterV1.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/SimpleEntityWriterV1.java index 16d14a18c88..6d6151fa6a1 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/SimpleEntityWriterV1.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/SimpleEntityWriterV1.java @@ -32,7 +32,6 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent; import org.apache.hadoop.yarn.client.api.TimelineClient; -import org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl; /** * Adds simple entities with random string payload, events, metrics, and @@ -46,7 +45,7 @@ class SimpleEntityWriterV1 public void map(IntWritable key, IntWritable val, Context context) throws IOException { - TimelineClient tlc = new TimelineClientImpl(); + TimelineClient tlc = TimelineClient.createTimelineClient(); Configuration conf = context.getConfiguration(); final int kbs = conf.getInt(KBS_SENT, KBS_SENT_DEFAULT); diff --git a/hadoop-project/src/site/markdown/index.md.vm b/hadoop-project/src/site/markdown/index.md.vm index 62e21b2edf9..bb7bda2c822 100644 --- a/hadoop-project/src/site/markdown/index.md.vm +++ b/hadoop-project/src/site/markdown/index.md.vm @@ -55,17 +55,15 @@ documentation. YARN Timeline Service v.2 ------------------- -We are introducing an early preview (alpha 1) of a major revision of YARN +We are introducing an early preview (alpha 2) of a major revision of YARN Timeline Service: v.2. YARN Timeline Service v.2 addresses two major challenges: improving scalability and reliability of Timeline Service, and enhancing usability by introducing flows and aggregation. -YARN Timeline Service v.2 alpha 1 is provided so that users and developers +YARN Timeline Service v.2 alpha 2 is provided so that users and developers can test it and provide feedback and suggestions for making it a ready replacement for Timeline Service v.1.x. It should be used only in a test -capacity. Most importantly, security is not enabled. Do not set up or use -Timeline Service v.2 until security is implemented if security is a -critical requirement. +capacity. More details are available in the [YARN Timeline Service v.2](./hadoop-yarn/hadoop-yarn-site/TimelineServiceV2.html) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java index d3ca765f91e..9b254ae9072 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java @@ -27,6 +27,7 @@ import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.classification.InterfaceStability.Stable; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; +import org.apache.hadoop.yarn.api.records.CollectorInfo; import org.apache.hadoop.yarn.api.records.AMCommand; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerStatus; @@ -92,21 +93,21 @@ public abstract class AllocateResponse { .preemptionMessage(preempt).nmTokens(nmTokens).build(); } - @Public + @Private @Unstable public static AllocateResponse newInstance(int responseId, List completedContainers, List allocatedContainers, List updatedNodes, Resource availResources, AMCommand command, int numClusterNodes, PreemptionMessage preempt, List nmTokens, - List updatedContainers) { + CollectorInfo collectorInfo) { return AllocateResponse.newBuilder().numClusterNodes(numClusterNodes) .responseId(responseId) .completedContainersStatuses(completedContainers) .allocatedContainers(allocatedContainers).updatedNodes(updatedNodes) .availableResources(availResources).amCommand(command) .preemptionMessage(preempt).nmTokens(nmTokens) - .updatedContainers(updatedContainers).build(); + .collectorInfo(collectorInfo).build(); } @Private @@ -133,7 +134,7 @@ public abstract class AllocateResponse { List allocatedContainers, List updatedNodes, Resource availResources, AMCommand command, int numClusterNodes, PreemptionMessage preempt, List nmTokens, Token amRMToken, - List updatedContainers, String collectorAddr) { + List updatedContainers, CollectorInfo collectorInfo) { return AllocateResponse.newBuilder().numClusterNodes(numClusterNodes) .responseId(responseId) .completedContainersStatuses(completedContainers) @@ -141,7 +142,7 @@ public abstract class AllocateResponse { .availableResources(availResources).amCommand(command) .preemptionMessage(preempt).nmTokens(nmTokens) .updatedContainers(updatedContainers).amRmToken(amRMToken) - .collectorAddr(collectorAddr).build(); + .collectorInfo(collectorInfo).build(); } /** @@ -333,17 +334,18 @@ public abstract class AllocateResponse { public abstract void setApplicationPriority(Priority priority); /** - * The address of collector that belong to this app + * The data associated with the collector that belongs to this app. Contains + * address and token alongwith identification information. * - * @return The address of collector that belong to this attempt + * @return The data of collector that belong to this attempt */ @Public @Unstable - public abstract String getCollectorAddr(); + public abstract CollectorInfo getCollectorInfo(); @Private @Unstable - public abstract void setCollectorAddr(String collectorAddr); + public abstract void setCollectorInfo(CollectorInfo info); /** * Get the list of container update errors to inform the @@ -559,15 +561,17 @@ public abstract class AllocateResponse { } /** - * Set the collectorAddr of the response. - * @see AllocateResponse#setCollectorAddr(String) - * @param collectorAddr collectorAddr of the response + * Set the collectorInfo of the response. + * @see AllocateResponse#setCollectorInfo(CollectorInfo) + * @param collectorInfo collectorInfo of the response which + * contains collector address, RM id, version and collector token. * @return {@link AllocateResponseBuilder} */ @Private @Unstable - public AllocateResponseBuilder collectorAddr(String collectorAddr) { - allocateResponse.setCollectorAddr(collectorAddr); + public AllocateResponseBuilder collectorInfo( + CollectorInfo collectorInfo) { + allocateResponse.setCollectorInfo(collectorInfo); return this; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/AppCollectorsMap.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/CollectorInfo.java similarity index 50% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/AppCollectorsMap.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/CollectorInfo.java index 07e1d92898f..d22b9fb48db 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/AppCollectorsMap.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/CollectorInfo.java @@ -16,31 +16,44 @@ * limitations under the License. */ -package org.apache.hadoop.yarn.server.api.records; +package org.apache.hadoop.yarn.api.records; +import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.util.Records; - +/** + * Collector info containing collector address and collector token passed from + * RM to AM in Allocate Response. + */ @Private -public abstract class AppCollectorsMap { +@InterfaceStability.Unstable +public abstract class CollectorInfo { - public static AppCollectorsMap newInstance( - ApplicationId id, String collectorAddr) { - AppCollectorsMap appCollectorsMap = - Records.newRecord(AppCollectorsMap.class); - appCollectorsMap.setApplicationId(id); - appCollectorsMap.setCollectorAddr(collectorAddr); - return appCollectorsMap; + protected static final long DEFAULT_TIMESTAMP_VALUE = -1; + + public static CollectorInfo newInstance(String collectorAddr) { + return newInstance(collectorAddr, null); } - public abstract ApplicationId getApplicationId(); - - public abstract void setApplicationId(ApplicationId id); + public static CollectorInfo newInstance(String collectorAddr, Token token) { + CollectorInfo amCollectorInfo = + Records.newRecord(CollectorInfo.class); + amCollectorInfo.setCollectorAddr(collectorAddr); + amCollectorInfo.setCollectorToken(token); + return amCollectorInfo; + } public abstract String getCollectorAddr(); public abstract void setCollectorAddr(String addr); + /** + * Get delegation token for app collector which AM will use to publish + * entities. + * @return the delegation token for app collector. + */ + public abstract Token getCollectorToken(); + + public abstract void setCollectorToken(Token token); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationEntity.java index 6075ec4e145..20226aa8e9c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationEntity.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationEntity.java @@ -49,4 +49,32 @@ public class ApplicationEntity extends HierarchicalTimelineEntity { public void setQueue(String queue) { addInfo(QUEUE_INFO_KEY, queue); } + + /** + * Checks if the input TimelineEntity object is an ApplicationEntity. + * + * @param te TimelineEntity object. + * @return true if input is an ApplicationEntity, false otherwise + */ + public static boolean isApplicationEntity(TimelineEntity te) { + return (te == null ? false + : te.getType().equals(TimelineEntityType.YARN_APPLICATION.toString())); + } + + /** + * @param te TimelineEntity object. + * @param eventId event with this id needs to be fetched + * @return TimelineEvent if TimelineEntity contains the desired event. + */ + public static TimelineEvent getApplicationEvent(TimelineEntity te, + String eventId) { + if (isApplicationEntity(te)) { + for (TimelineEvent event : te.getEvents()) { + if (event.getId().equals(eventId)) { + return event; + } + } + } + return null; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java index e43c3acb6de..0af5ea47694 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java @@ -55,6 +55,7 @@ import com.fasterxml.jackson.annotation.JsonSetter; @InterfaceStability.Unstable public class TimelineEntity implements Comparable { protected final static String SYSTEM_INFO_KEY_PREFIX = "SYSTEM_INFO_"; + public final static long DEFAULT_ENTITY_PREFIX = 0L; /** * Identifier of timeline entity(entity id + entity type). @@ -146,6 +147,7 @@ public class TimelineEntity implements Comparable { private HashMap> isRelatedToEntities = new HashMap<>(); private HashMap> relatesToEntities = new HashMap<>(); private Long createdTime; + private long idPrefix; public TimelineEntity() { identifier = new Identifier(); @@ -548,20 +550,10 @@ public class TimelineEntity implements Comparable { public int compareTo(TimelineEntity other) { int comparison = getType().compareTo(other.getType()); if (comparison == 0) { - if (getCreatedTime() == null) { - if (other.getCreatedTime() == null) { - return getId().compareTo(other.getId()); - } else { - return 1; - } - } - if (other.getCreatedTime() == null) { + if (getIdPrefix() > other.getIdPrefix()) { + // Descending order by entity id prefix return -1; - } - if (getCreatedTime() > other.getCreatedTime()) { - // Order by created time desc - return -1; - } else if (getCreatedTime() < other.getCreatedTime()) { + } else if (getIdPrefix() < other.getIdPrefix()) { return 1; } else { return getId().compareTo(other.getId()); @@ -582,4 +574,38 @@ public class TimelineEntity implements Comparable { return real.toString(); } } + + @XmlElement(name = "idprefix") + public long getIdPrefix() { + if (real == null) { + return idPrefix; + } else { + return real.getIdPrefix(); + } + } + + /** + * Sets idPrefix for an entity. + *

+ * Note: Entities will be stored in the order of idPrefix specified. + * If users decide to set idPrefix for an entity, they MUST provide + * the same prefix for every update of this entity. + *

+ * Example: 
+   * TimelineEntity entity = new TimelineEntity();
+   * entity.setIdPrefix(value);
+   *
+ * Users can use {@link TimelineServiceHelper#invertLong(long)} to invert + * the prefix if necessary. + * + * @param entityIdPrefix prefix for an entity. + */ + @JsonSetter("idprefix") + public void setIdPrefix(long entityIdPrefix) { + if (real == null) { + this.idPrefix = entityIdPrefix; + } else { + real.setIdPrefix(entityIdPrefix); + } + } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 67dfeebafe6..49448217e9c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1782,6 +1782,10 @@ public class YarnConfiguration extends Configuration { YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONHISTORY_PROTOCOL = "security.applicationhistory.protocol.acl"; + public static final String + YARN_SECURITY_SERVICE_AUTHORIZATION_COLLECTOR_NODEMANAGER_PROTOCOL = + "security.collector-nodemanager.protocol.acl"; + /** No. of milliseconds to wait between sending a SIGTERM and SIGKILL * to a running container */ public static final String NM_SLEEP_DELAY_BEFORE_SIGKILL_MS = @@ -2099,7 +2103,7 @@ public class YarnConfiguration extends Configuration { TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_PREFIX + "with-user-dir"; /** - * Settings for timeline service v2.0 + * Settings for timeline service v2.0. */ public static final String TIMELINE_SERVICE_WRITER_CLASS = TIMELINE_SERVICE_PREFIX + "writer.class"; @@ -2112,9 +2116,20 @@ public class YarnConfiguration extends Configuration { TIMELINE_SERVICE_PREFIX + "reader.class"; public static final String DEFAULT_TIMELINE_SERVICE_READER_CLASS = - "org.apache.hadoop.yarn.server.timelineservice" + - ".storage.HBaseTimelineReaderImpl"; + "org.apache.hadoop.yarn.server.timelineservice.storage" + + ".HBaseTimelineReaderImpl"; + /** + * default schema prefix for hbase tables. + */ + public static final String DEFAULT_TIMELINE_SERVICE_HBASE_SCHEMA_PREFIX = + "prod."; + + /** + * config param name to override schema prefix. + */ + public static final String TIMELINE_SERVICE_HBASE_SCHEMA_PREFIX_NAME = + TIMELINE_SERVICE_PREFIX + "hbase-schema.prefix"; /** The setting that controls how often the timeline collector flushes the * timeline writer. @@ -2134,6 +2149,58 @@ public class YarnConfiguration extends Configuration { TIMELINE_SERVICE_PREFIX + "hbase.coprocessor.app-final-value-retention-milliseconds"; + /** + * The name of the setting for the location of the coprocessor + * jar on hdfs. + */ + public static final String FLOW_RUN_COPROCESSOR_JAR_HDFS_LOCATION = + TIMELINE_SERVICE_PREFIX + + "hbase.coprocessor.jar.hdfs.location"; + + /** default hdfs location for flowrun coprocessor jar. */ + public static final String DEFAULT_HDFS_LOCATION_FLOW_RUN_COPROCESSOR_JAR = + "/hbase/coprocessor/hadoop-yarn-server-timelineservice.jar"; + + /** + * The name for setting that points to an optional HBase configuration + * (hbase-site.xml file) with settings that will override the ones found on + * the classpath. + */ + public static final String TIMELINE_SERVICE_HBASE_CONFIGURATION_FILE = + TIMELINE_SERVICE_PREFIX + + "hbase.configuration.file"; + + /** + * The name for setting that enables or disables authentication checks + * for reading timeline service v2 data. + */ + public static final String TIMELINE_SERVICE_READ_AUTH_ENABLED = + TIMELINE_SERVICE_PREFIX + "read.authentication.enabled"; + + /** + * The default setting for authentication checks for reading timeline + * service v2 data. + */ + public static final Boolean DEFAULT_TIMELINE_SERVICE_READ_AUTH_ENABLED = + false; + + /** + * The name for setting that lists the users and groups who are allowed + * to read timeline service v2 data. It is a comma separated list of + * user, followed by space, then comma separated list of groups. + * It will allow this list of users and groups to read the data + * and reject everyone else. + */ + public static final String TIMELINE_SERVICE_READ_ALLOWED_USERS = + TIMELINE_SERVICE_PREFIX + "read.allowed.users"; + + /** + * The default value for list of the users who are allowed to read + * timeline service v2 data. + */ + public static final String DEFAULT_TIMELINE_SERVICE_READ_ALLOWED_USERS = + ""; + /** * The setting that controls how long the final value of a metric of a * completed app is retained before merging into the flow sum. Up to this time @@ -2154,6 +2221,8 @@ public class YarnConfiguration extends Configuration { public static final int DEFAULT_NUMBER_OF_ASYNC_ENTITIES_TO_MERGE = 10; + /** default version for any flow. */ + public static final String DEFAULT_FLOW_VERSION = "1"; /** * The time period for which timeline v2 client will wait for draining diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/TimelineServiceHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/TimelineServiceHelper.java index e0268a67b8f..65ed18a7a1e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/TimelineServiceHelper.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/TimelineServiceHelper.java @@ -46,4 +46,12 @@ public final class TimelineServiceHelper { (HashMap) originalMap : new HashMap(originalMap); } + /** + * Inverts the given key. + * @param key value to be inverted . + * @return inverted long + */ + public static long invertLong(long key) { + return Long.MAX_VALUE - key; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 81ebd798bb9..c5f485fc3f7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -613,3 +613,8 @@ message StringBytesMapProto { optional string key = 1; optional bytes value = 2; } + +message CollectorInfoProto { + optional string collector_addr = 1; + optional hadoop.common.TokenProto collector_token = 2; +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto index b92c46e945e..7a7f03503ca 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto @@ -112,7 +112,7 @@ message AllocateResponseProto { repeated NMTokenProto nm_tokens = 9; optional hadoop.common.TokenProto am_rm_token = 12; optional PriorityProto application_priority = 13; - optional string collector_addr = 14; + optional CollectorInfoProto collector_info = 14; repeated UpdateContainerErrorProto update_errors = 15; repeated UpdatedContainerProto updated_containers = 16; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestApplicationEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestApplicationEntity.java new file mode 100644 index 00000000000..c3f277718e7 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestApplicationEntity.java @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records.timelineservice; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertEquals; +import org.junit.Test; + +/** + * Various tests for the ApplicationEntity class. + * + */ +public class TestApplicationEntity { + + @Test + public void testIsApplicationEntity() { + TimelineEntity te = new TimelineEntity(); + te.setType(TimelineEntityType.YARN_APPLICATION.toString()); + assertTrue(ApplicationEntity.isApplicationEntity(te)); + + te = null; + assertEquals(false, ApplicationEntity.isApplicationEntity(te)); + + te = new TimelineEntity(); + te.setType(TimelineEntityType.YARN_CLUSTER.toString()); + assertEquals(false, ApplicationEntity.isApplicationEntity(te)); + } + + @Test + public void testGetApplicationEvent() { + TimelineEntity te = null; + TimelineEvent tEvent = ApplicationEntity.getApplicationEvent(te, + "no event"); + assertEquals(null, tEvent); + + te = new TimelineEntity(); + te.setType(TimelineEntityType.YARN_APPLICATION.toString()); + TimelineEvent event = new TimelineEvent(); + event.setId("start_event"); + event.setTimestamp(System.currentTimeMillis()); + te.addEvent(event); + tEvent = ApplicationEntity.getApplicationEvent(te, "start_event"); + assertEquals(event, tEvent); + + te = new TimelineEntity(); + te.setType(TimelineEntityType.YARN_CLUSTER.toString()); + event = new TimelineEvent(); + event.setId("start_event_cluster"); + event.setTimestamp(System.currentTimeMillis()); + te.addEvent(event); + tEvent = ApplicationEntity.getApplicationEvent(te, "start_event_cluster"); + assertEquals(null, tEvent); + + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java index d97c6ebd885..bd7bf93566f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java @@ -66,6 +66,8 @@ public class TestYarnConfigurationFields extends TestConfigurationFieldsBase { configurationPropsToSkipCompare .add(YarnConfiguration .YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCETRACKER_PROTOCOL); + configurationPropsToSkipCompare.add(YarnConfiguration + .YARN_SECURITY_SERVICE_AUTHORIZATION_COLLECTOR_NODEMANAGER_PROTOCOL); configurationPropsToSkipCompare.add(YarnConfiguration.CURATOR_LEADER_ELECTOR); // Federation default configs to be ignored diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java index ab4607aca6a..a02af709116 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java @@ -104,6 +104,8 @@ import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; +import org.apache.hadoop.yarn.util.SystemClock; +import org.apache.hadoop.yarn.util.TimelineServiceHelper; import org.apache.hadoop.yarn.util.timeline.TimelineUtils; import org.apache.log4j.LogManager; @@ -313,6 +315,17 @@ public class ApplicationMaster { protected final Set launchedContainers = Collections.newSetFromMap(new ConcurrentHashMap()); + /** + * Container start times used to set id prefix while publishing entity + * to ATSv2. + */ + private final ConcurrentMap containerStartTimes = + new ConcurrentHashMap(); + + private ConcurrentMap getContainerStartTimes() { + return containerStartTimes; + } + /** * @param args Command line args */ @@ -866,7 +879,15 @@ public class ApplicationMaster { + containerStatus.getContainerId()); } if (timelineServiceV2Enabled) { - publishContainerEndEventOnTimelineServiceV2(containerStatus); + Long containerStartTime = + containerStartTimes.get(containerStatus.getContainerId()); + if (containerStartTime == null) { + containerStartTime = SystemClock.getInstance().getTime(); + containerStartTimes.put(containerStatus.getContainerId(), + containerStartTime); + } + publishContainerEndEventOnTimelineServiceV2(containerStatus, + containerStartTime); } else if (timelineServiceV1Enabled) { publishContainerEndEvent(timelineClient, containerStatus, domainId, appSubmitterUgi); @@ -994,8 +1015,10 @@ public class ApplicationMaster { containerId, container.getNodeId()); } if (applicationMaster.timelineServiceV2Enabled) { - applicationMaster - .publishContainerStartEventOnTimelineServiceV2(container); + long startTime = SystemClock.getInstance().getTime(); + applicationMaster.getContainerStartTimes().put(containerId, startTime); + applicationMaster.publishContainerStartEventOnTimelineServiceV2( + container, startTime); } else if (applicationMaster.timelineServiceV1Enabled) { applicationMaster.publishContainerStartEvent( applicationMaster.timelineClient, container, @@ -1356,24 +1379,24 @@ public class ApplicationMaster { } private void publishContainerStartEventOnTimelineServiceV2( - Container container) { + Container container, long startTime) { final org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity = new org.apache.hadoop.yarn.api.records.timelineservice. TimelineEntity(); entity.setId(container.getId().toString()); entity.setType(DSEntity.DS_CONTAINER.toString()); - long ts = System.currentTimeMillis(); - entity.setCreatedTime(ts); + entity.setCreatedTime(startTime); entity.addInfo("user", appSubmitterUgi.getShortUserName()); org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent event = new org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent(); - event.setTimestamp(ts); + event.setTimestamp(startTime); event.setId(DSEvent.DS_CONTAINER_START.toString()); event.addInfo("Node", container.getNodeId().toString()); event.addInfo("Resources", container.getResource().toString()); entity.addEvent(event); + entity.setIdPrefix(TimelineServiceHelper.invertLong(startTime)); try { appSubmitterUgi.doAs(new PrivilegedExceptionAction() { @@ -1391,7 +1414,7 @@ public class ApplicationMaster { } private void publishContainerEndEventOnTimelineServiceV2( - final ContainerStatus container) { + final ContainerStatus container, long containerStartTime) { final org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity = new org.apache.hadoop.yarn.api.records.timelineservice. @@ -1407,6 +1430,7 @@ public class ApplicationMaster { event.addInfo("State", container.getState().name()); event.addInfo("Exit Status", container.getExitStatus()); entity.addEvent(event); + entity.setIdPrefix(TimelineServiceHelper.invertLong(containerStartTime)); try { appSubmitterUgi.doAs(new PrivilegedExceptionAction() { @@ -1441,6 +1465,8 @@ public class ApplicationMaster { event.setId(appEvent.toString()); event.setTimestamp(ts); entity.addEvent(event); + entity.setIdPrefix( + TimelineServiceHelper.invertLong(appAttemptID.getAttemptId())); try { appSubmitterUgi.doAs(new PrivilegedExceptionAction() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java index ef21c8786c5..fc270cb79f2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java @@ -64,7 +64,9 @@ import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; +import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType; +import org.apache.hadoop.yarn.applications.distributedshell.ApplicationMaster.DSEvent; import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.client.api.impl.DirectTimelineWriter; import org.apache.hadoop.yarn.client.api.impl.TestTimelineClient; @@ -81,6 +83,7 @@ import org.apache.hadoop.yarn.server.timeline.PluginStoreTestUtils; import org.apache.hadoop.yarn.server.timeline.TimelineVersion; import org.apache.hadoop.yarn.server.timeline.TimelineVersionWatcher; import org.apache.hadoop.yarn.server.timelineservice.collector.PerNodeTimelineCollectorsAuxService; +import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineReaderImpl; import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineWriterImpl; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.LinuxResourceCalculatorPlugin; @@ -523,15 +526,31 @@ public class TestDistributedShell { "appattempt_" + appId.getClusterTimestamp() + "_000" + appId.getId() + "_000001" + FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION; - verifyEntityTypeFileExists(basePath, "DS_APP_ATTEMPT", - appTimestampFileName); + File dsAppAttemptEntityFile = verifyEntityTypeFileExists(basePath, + "DS_APP_ATTEMPT", appTimestampFileName); + // Check if required events are published and same idprefix is sent for + // on each publish. + verifyEntityForTimelineV2(dsAppAttemptEntityFile, + DSEvent.DS_APP_ATTEMPT_START.toString(), 1, 1, 0, true); + // to avoid race condition of testcase, atleast check 40 times with sleep + // of 50ms + verifyEntityForTimelineV2(dsAppAttemptEntityFile, + DSEvent.DS_APP_ATTEMPT_END.toString(), 1, 40, 50, true); - // Verify DS_CONTAINER entities posted by the client + // Verify DS_CONTAINER entities posted by the client. String containerTimestampFileName = "container_" + appId.getClusterTimestamp() + "_000" + appId.getId() + "_01_000002.thist"; - verifyEntityTypeFileExists(basePath, "DS_CONTAINER", - containerTimestampFileName); + File dsContainerEntityFile = verifyEntityTypeFileExists(basePath, + "DS_CONTAINER", containerTimestampFileName); + // Check if required events are published and same idprefix is sent for + // on each publish. + verifyEntityForTimelineV2(dsContainerEntityFile, + DSEvent.DS_CONTAINER_START.toString(), 1, 1, 0, true); + // to avoid race condition of testcase, atleast check 40 times with sleep + // of 50ms + verifyEntityForTimelineV2(dsContainerEntityFile, + DSEvent.DS_CONTAINER_END.toString(), 1, 40, 50, true); // Verify NM posting container metrics info. String containerMetricsTimestampFileName = @@ -541,29 +560,13 @@ public class TestDistributedShell { File containerEntityFile = verifyEntityTypeFileExists(basePath, TimelineEntityType.YARN_CONTAINER.toString(), containerMetricsTimestampFileName); - Assert.assertEquals( - "Container created event needs to be published atleast once", - 1, - getNumOfStringOccurrences(containerEntityFile, - ContainerMetricsConstants.CREATED_EVENT_TYPE)); + verifyEntityForTimelineV2(containerEntityFile, + ContainerMetricsConstants.CREATED_EVENT_TYPE, 1, 1, 0, true); - // to avoid race condition of testcase, atleast check 4 times with sleep - // of 500ms - long numOfContainerFinishedOccurrences = 0; - for (int i = 0; i < 4; i++) { - numOfContainerFinishedOccurrences = - getNumOfStringOccurrences(containerEntityFile, - ContainerMetricsConstants.FINISHED_EVENT_TYPE); - if (numOfContainerFinishedOccurrences > 0) { - break; - } else { - Thread.sleep(500L); - } - } - Assert.assertEquals( - "Container finished event needs to be published atleast once", - 1, - numOfContainerFinishedOccurrences); + // to avoid race condition of testcase, atleast check 40 times with sleep + // of 50ms + verifyEntityForTimelineV2(containerEntityFile, + ContainerMetricsConstants.FINISHED_EVENT_TYPE, 1, 40, 50, true); // Verify RM posting Application life cycle Events are getting published String appMetricsTimestampFileName = @@ -573,29 +576,14 @@ public class TestDistributedShell { verifyEntityTypeFileExists(basePath, TimelineEntityType.YARN_APPLICATION.toString(), appMetricsTimestampFileName); - Assert.assertEquals( - "Application created event should be published atleast once", - 1, - getNumOfStringOccurrences(appEntityFile, - ApplicationMetricsConstants.CREATED_EVENT_TYPE)); + // No need to check idprefix for app. + verifyEntityForTimelineV2(appEntityFile, + ApplicationMetricsConstants.CREATED_EVENT_TYPE, 1, 1, 0, false); - // to avoid race condition of testcase, atleast check 4 times with sleep - // of 500ms - long numOfStringOccurrences = 0; - for (int i = 0; i < 4; i++) { - numOfStringOccurrences = - getNumOfStringOccurrences(appEntityFile, - ApplicationMetricsConstants.FINISHED_EVENT_TYPE); - if (numOfStringOccurrences > 0) { - break; - } else { - Thread.sleep(500L); - } - } - Assert.assertEquals( - "Application finished event should be published atleast once", - 1, - numOfStringOccurrences); + // to avoid race condition of testcase, atleast check 40 times with sleep + // of 50ms + verifyEntityForTimelineV2(appEntityFile, + ApplicationMetricsConstants.FINISHED_EVENT_TYPE, 1, 40, 50, false); // Verify RM posting AppAttempt life cycle Events are getting published String appAttemptMetricsTimestampFileName = @@ -606,17 +594,10 @@ public class TestDistributedShell { verifyEntityTypeFileExists(basePath, TimelineEntityType.YARN_APPLICATION_ATTEMPT.toString(), appAttemptMetricsTimestampFileName); - Assert.assertEquals( - "AppAttempt register event should be published atleast once", - 1, - getNumOfStringOccurrences(appAttemptEntityFile, - AppAttemptMetricsConstants.REGISTERED_EVENT_TYPE)); - - Assert.assertEquals( - "AppAttempt finished event should be published atleast once", - 1, - getNumOfStringOccurrences(appAttemptEntityFile, - AppAttemptMetricsConstants.FINISHED_EVENT_TYPE)); + verifyEntityForTimelineV2(appAttemptEntityFile, + AppAttemptMetricsConstants.REGISTERED_EVENT_TYPE, 1, 1, 0, true); + verifyEntityForTimelineV2(appAttemptEntityFile, + AppAttemptMetricsConstants.FINISHED_EVENT_TYPE, 1, 1, 0, true); } finally { FileUtils.deleteDirectory(tmpRootFolder.getParentFile()); } @@ -636,22 +617,64 @@ public class TestDistributedShell { return entityFile; } - private long getNumOfStringOccurrences(File entityFile, String searchString) - throws IOException { - BufferedReader reader = null; - String strLine; + /** + * Checks the events and idprefix published for an entity. + * + * @param entityFile Entity file. + * @param expectedEvent Expected event Id. + * @param numOfExpectedEvent Number of expected occurences of expected event + * id. + * @param checkTimes Number of times to check. + * @param sleepTime Sleep time for each iteration. + * @param checkIdPrefix Whether to check idprefix. + * @throws IOException if entity file reading fails. + * @throws InterruptedException if sleep is interrupted. + */ + private void verifyEntityForTimelineV2(File entityFile, String expectedEvent, + long numOfExpectedEvent, int checkTimes, long sleepTime, + boolean checkIdPrefix) throws IOException, InterruptedException { long actualCount = 0; - try { - reader = new BufferedReader(new FileReader(entityFile)); - while ((strLine = reader.readLine()) != null) { - if (strLine.trim().contains(searchString)) { - actualCount++; + for (int i = 0; i < checkTimes; i++) { + BufferedReader reader = null; + String strLine = null; + actualCount = 0; + try { + reader = new BufferedReader(new FileReader(entityFile)); + long idPrefix = -1; + while ((strLine = reader.readLine()) != null) { + String entityLine = strLine.trim(); + if (entityLine.isEmpty()) { + continue; + } + if (entityLine.contains(expectedEvent)) { + actualCount++; + } + if (checkIdPrefix) { + TimelineEntity entity = FileSystemTimelineReaderImpl. + getTimelineRecordFromJSON(entityLine, TimelineEntity.class); + Assert.assertTrue("Entity ID prefix expected to be > 0", + entity.getIdPrefix() > 0); + if (idPrefix == -1) { + idPrefix = entity.getIdPrefix(); + } else { + Assert.assertEquals("Entity ID prefix should be same across " + + "each publish of same entity", + idPrefix, entity.getIdPrefix()); + } + } } + } finally { + reader.close(); + } + if (numOfExpectedEvent == actualCount) { + break; + } + if (sleepTime > 0 && i < checkTimes - 1) { + Thread.sleep(sleepTime); } - } finally { - reader.close(); } - return actualCount; + Assert.assertEquals("Unexpected number of " + expectedEvent + + " event published.", numOfExpectedEvent, actualCount); } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java index 6711da28cf0..d12b108f3e2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java @@ -68,8 +68,6 @@ extends AMRMClientAsync { private volatile boolean keepRunning; private volatile float progress; - - private volatile String collectorAddr; /** * @@ -325,17 +323,16 @@ extends AMRMClientAsync { } AllocateResponse response = (AllocateResponse) object; - String collectorAddress = response.getCollectorAddr(); + String collectorAddress = null; + if (response.getCollectorInfo() != null) { + collectorAddress = response.getCollectorInfo().getCollectorAddr(); + } + TimelineV2Client timelineClient = client.getRegisteredTimelineV2Client(); - if (timelineClient != null && collectorAddress != null - && !collectorAddress.isEmpty()) { - if (collectorAddr == null - || !collectorAddr.equals(collectorAddress)) { - collectorAddr = collectorAddress; - timelineClient.setTimelineServiceAddress(collectorAddress); - LOG.info("collectorAddress " + collectorAddress); - } + if (timelineClient != null && response.getCollectorInfo() != null) { + timelineClient. + setTimelineCollectorInfo(response.getCollectorInfo()); } List updatedNodes = response.getUpdatedNodes(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ApplicationMasterServiceProtoTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ApplicationMasterServiceProtoTestBase.java new file mode 100644 index 00000000000..45210188b51 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ApplicationMasterServiceProtoTestBase.java @@ -0,0 +1,72 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.client; + +import java.io.IOException; + +import org.apache.hadoop.ipc.RPC; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; +import org.junit.After; + +/** + * Test Base for Application Master Service Protocol. + */ +public abstract class ApplicationMasterServiceProtoTestBase + extends ProtocolHATestBase { + + private ApplicationMasterProtocol amClient; + private ApplicationAttemptId attemptId; + + protected void startupHAAndSetupClient() throws Exception { + attemptId = this.cluster.createFakeApplicationAttemptId(); + + Token appToken = + this.cluster.getResourceManager().getRMContext() + .getAMRMTokenSecretManager().createAndGetAMRMToken(attemptId); + appToken.setService(ClientRMProxy.getAMRMTokenService(this.conf)); + UserGroupInformation.setLoginUser(UserGroupInformation + .createRemoteUser(UserGroupInformation.getCurrentUser().getUserName())); + UserGroupInformation.getCurrentUser().addToken(appToken); + syncToken(appToken); + amClient = ClientRMProxy + .createRMProxy(this.conf, ApplicationMasterProtocol.class); + } + + @After + public void shutDown() { + if(this.amClient != null) { + RPC.stopProxy(this.amClient); + } + } + + protected ApplicationMasterProtocol getAMClient() { + return amClient; + } + + private void syncToken(Token token) throws IOException { + for (int i = 0; i < this.cluster.getNumOfResourceManager(); i++) { + this.cluster.getResourceManager(i).getRMContext() + .getAMRMTokenSecretManager().addPersistedPassword(token); + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java index a8e91323a38..f4005e9021c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java @@ -25,6 +25,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterResponse; import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest; import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; +import org.apache.hadoop.yarn.api.records.CollectorInfo; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -804,11 +805,20 @@ public abstract class ProtocolHATestBase extends ClientBaseWithFixes { } public AllocateResponse createFakeAllocateResponse() { - return AllocateResponse.newInstance(-1, - new ArrayList(), - new ArrayList(), new ArrayList(), - Resource.newInstance(1024, 2), null, 1, - null, new ArrayList()); + if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) { + return AllocateResponse.newInstance(-1, + new ArrayList(), new ArrayList(), + new ArrayList(), Resource.newInstance(1024, 2), null, 1, + null, new ArrayList(), CollectorInfo.newInstance( + "host:port", Token.newInstance(new byte[] {0}, "TIMELINE", + new byte[] {0}, "rm"))); + } else { + return AllocateResponse.newInstance(-1, + new ArrayList(), + new ArrayList(), new ArrayList(), + Resource.newInstance(1024, 2), null, 1, + null, new ArrayList()); + } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationMasterServiceProtocolForTimelineV2.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationMasterServiceProtocolForTimelineV2.java new file mode 100644 index 00000000000..be8c3023d08 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationMasterServiceProtocolForTimelineV2.java @@ -0,0 +1,71 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.client; + +import java.io.IOException; +import java.util.ArrayList; + +import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; +import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; +import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.resourcemanager.HATestUtil; +import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineWriterImpl; +import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineWriter; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +/** + * Tests Application Master Protocol with timeline service v2 enabled. + */ +public class TestApplicationMasterServiceProtocolForTimelineV2 + extends ApplicationMasterServiceProtoTestBase { + + @Before + public void initialize() throws Exception { + HATestUtil.setRpcAddressForRM(RM1_NODE_ID, RM1_PORT_BASE + 200, conf); + HATestUtil.setRpcAddressForRM(RM2_NODE_ID, RM2_PORT_BASE + 200, conf); + conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); + conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f); + conf.setClass(YarnConfiguration.TIMELINE_SERVICE_WRITER_CLASS, + FileSystemTimelineWriterImpl.class, TimelineWriter.class); + startHACluster(0, false, false, true); + super.startupHAAndSetupClient(); + } + + @Test(timeout = 15000) + public void testAllocateForTimelineV2OnHA() + throws YarnException, IOException { + AllocateRequest request = AllocateRequest.newInstance(0, 50f, + new ArrayList(), + new ArrayList(), + ResourceBlacklistRequest.newInstance(new ArrayList(), + new ArrayList())); + AllocateResponse response = getAMClient().allocate(request); + Assert.assertEquals(response, this.cluster.createFakeAllocateResponse()); + Assert.assertNotNull(response.getCollectorInfo()); + Assert.assertEquals("host:port", + response.getCollectorInfo().getCollectorAddr()); + Assert.assertNotNull(response.getCollectorInfo().getCollectorToken()); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationMasterServiceProtocolOnHA.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationMasterServiceProtocolOnHA.java index ad86fb377b8..c2f39a1d4ff 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationMasterServiceProtocolOnHA.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationMasterServiceProtocolOnHA.java @@ -23,10 +23,6 @@ import java.util.ArrayList; import org.junit.Assert; -import org.apache.hadoop.ipc.RPC; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest; @@ -34,45 +30,20 @@ import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRespons import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest; import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; -import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; -import org.junit.After; import org.junit.Before; import org.junit.Test; public class TestApplicationMasterServiceProtocolOnHA - extends ProtocolHATestBase { - private ApplicationMasterProtocol amClient; - private ApplicationAttemptId attemptId ; - + extends ApplicationMasterServiceProtoTestBase { @Before public void initialize() throws Exception { startHACluster(0, false, false, true); - attemptId = this.cluster.createFakeApplicationAttemptId(); - - Token appToken = - this.cluster.getResourceManager().getRMContext() - .getAMRMTokenSecretManager().createAndGetAMRMToken(attemptId); - appToken.setService(ClientRMProxy.getAMRMTokenService(this.conf)); - UserGroupInformation.setLoginUser(UserGroupInformation - .createRemoteUser(UserGroupInformation.getCurrentUser().getUserName())); - UserGroupInformation.getCurrentUser().addToken(appToken); - syncToken(appToken); - - amClient = ClientRMProxy - .createRMProxy(this.conf, ApplicationMasterProtocol.class); - } - - @After - public void shutDown() { - if(this.amClient != null) { - RPC.stopProxy(this.amClient); - } + super.startupHAAndSetupClient(); } @Test(timeout = 15000) @@ -81,7 +52,7 @@ public class TestApplicationMasterServiceProtocolOnHA RegisterApplicationMasterRequest request = RegisterApplicationMasterRequest.newInstance("localhost", 0, ""); RegisterApplicationMasterResponse response = - amClient.registerApplicationMaster(request); + getAMClient().registerApplicationMaster(request); Assert.assertEquals(response, this.cluster.createFakeRegisterApplicationMasterResponse()); } @@ -93,7 +64,7 @@ public class TestApplicationMasterServiceProtocolOnHA FinishApplicationMasterRequest.newInstance( FinalApplicationStatus.SUCCEEDED, "", ""); FinishApplicationMasterResponse response = - amClient.finishApplicationMaster(request); + getAMClient().finishApplicationMaster(request); Assert.assertEquals(response, this.cluster.createFakeFinishApplicationMasterResponse()); } @@ -105,14 +76,7 @@ public class TestApplicationMasterServiceProtocolOnHA new ArrayList(), ResourceBlacklistRequest.newInstance(new ArrayList(), new ArrayList())); - AllocateResponse response = amClient.allocate(request); + AllocateResponse response = getAMClient().allocate(request); Assert.assertEquals(response, this.cluster.createFakeAllocateResponse()); } - - private void syncToken(Token token) throws IOException { - for (int i = 0; i < this.cluster.getNumOfResourceManager(); i++) { - this.cluster.getResourceManager(i).getRMContext() - .getAMRMTokenSecretManager().addPersistedPassword(token); - } - } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/async/impl/TestAMRMClientAsync.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/async/impl/TestAMRMClientAsync.java index ba383409752..56826c431c7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/async/impl/TestAMRMClientAsync.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/async/impl/TestAMRMClientAsync.java @@ -426,7 +426,7 @@ public class TestAMRMClientAsync { } AllocateResponse response = AllocateResponse.newInstance(0, completed, allocated, - new ArrayList(), null, null, 1, null, nmTokens, + new ArrayList(), null, null, 1, null, nmTokens, null, updatedContainers); return response; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java index 1455d6a82a6..ff35da81eb0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java @@ -27,6 +27,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.security.proto.SecurityProtos.TokenProto; import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; +import org.apache.hadoop.yarn.api.records.CollectorInfo; import org.apache.hadoop.yarn.api.records.AMCommand; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerStatus; @@ -38,6 +39,7 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.UpdateContainerError; import org.apache.hadoop.yarn.api.records.UpdatedContainer; +import org.apache.hadoop.yarn.api.records.impl.pb.CollectorInfoPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.ContainerPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.ContainerStatusPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.NMTokenPBImpl; @@ -48,6 +50,7 @@ import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils; import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.TokenPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.UpdatedContainerPBImpl; +import org.apache.hadoop.yarn.proto.YarnProtos.CollectorInfoProto; import org.apache.hadoop.yarn.proto.YarnProtos.ContainerProto; import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStatusProto; import org.apache.hadoop.yarn.proto.YarnProtos.NodeReportProto; @@ -80,6 +83,7 @@ public class AllocateResponsePBImpl extends AllocateResponse { private PreemptionMessage preempt; private Token amrmToken = null; private Priority appPriority = null; + private CollectorInfo collectorInfo = null; public AllocateResponsePBImpl() { builder = AllocateResponseProto.newBuilder(); @@ -162,6 +166,9 @@ public class AllocateResponsePBImpl extends AllocateResponse { if (this.amrmToken != null) { builder.setAmRmToken(convertToProtoFormat(this.amrmToken)); } + if (this.collectorInfo != null) { + builder.setCollectorInfo(convertToProtoFormat(this.collectorInfo)); + } if (this.appPriority != null) { builder.setApplicationPriority(convertToProtoFormat(this.appPriority)); } @@ -398,19 +405,25 @@ public class AllocateResponsePBImpl extends AllocateResponse { @Override - public synchronized String getCollectorAddr() { + public synchronized CollectorInfo getCollectorInfo() { AllocateResponseProtoOrBuilder p = viaProto ? proto : builder; - return p.getCollectorAddr(); + if (this.collectorInfo != null) { + return this.collectorInfo; + } + if (!p.hasCollectorInfo()) { + return null; + } + this.collectorInfo = convertFromProtoFormat(p.getCollectorInfo()); + return this.collectorInfo; } @Override - public synchronized void setCollectorAddr(String collectorAddr) { + public synchronized void setCollectorInfo(CollectorInfo info) { maybeInitBuilder(); - if (collectorAddr == null) { - builder.clearCollectorAddr(); - return; + if (info == null) { + builder.clearCollectorInfo(); } - builder.setCollectorAddr(collectorAddr); + this.collectorInfo = info; } @Override @@ -718,6 +731,16 @@ public class AllocateResponsePBImpl extends AllocateResponse { return ((NodeReportPBImpl)t).getProto(); } + private synchronized CollectorInfoPBImpl convertFromProtoFormat( + CollectorInfoProto p) { + return new CollectorInfoPBImpl(p); + } + + private synchronized CollectorInfoProto convertToProtoFormat( + CollectorInfo t) { + return ((CollectorInfoPBImpl)t).getProto(); + } + private synchronized ContainerPBImpl convertFromProtoFormat( ContainerProto p) { return new ContainerPBImpl(p); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/CollectorInfoPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/CollectorInfoPBImpl.java new file mode 100644 index 00000000000..5835d1a2b00 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/CollectorInfoPBImpl.java @@ -0,0 +1,152 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.api.records.impl.pb; + +import org.apache.hadoop.security.proto.SecurityProtos.TokenProto; +import org.apache.hadoop.yarn.api.records.CollectorInfo; +import org.apache.hadoop.yarn.api.records.Token; +import org.apache.hadoop.yarn.proto.YarnProtos.CollectorInfoProto; +import org.apache.hadoop.yarn.proto.YarnProtos.CollectorInfoProtoOrBuilder; + +import com.google.protobuf.TextFormat; + +/** + * Protocol record implementation of {@link CollectorInfo}. + */ +public class CollectorInfoPBImpl extends CollectorInfo { + + private CollectorInfoProto proto = CollectorInfoProto.getDefaultInstance(); + + private CollectorInfoProto.Builder builder = null; + private boolean viaProto = false; + + private String collectorAddr = null; + private Token collectorToken = null; + + + public CollectorInfoPBImpl() { + builder = CollectorInfoProto.newBuilder(); + } + + public CollectorInfoPBImpl(CollectorInfoProto proto) { + this.proto = proto; + viaProto = true; + } + + public CollectorInfoProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = CollectorInfoProto.newBuilder(proto); + } + viaProto = false; + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } + + @Override + public String getCollectorAddr() { + CollectorInfoProtoOrBuilder p = viaProto ? proto : builder; + if (this.collectorAddr == null && p.hasCollectorAddr()) { + this.collectorAddr = p.getCollectorAddr(); + } + return this.collectorAddr; + } + + @Override + public void setCollectorAddr(String addr) { + maybeInitBuilder(); + if (collectorAddr == null) { + builder.clearCollectorAddr(); + } + this.collectorAddr = addr; + } + + @Override + public Token getCollectorToken() { + CollectorInfoProtoOrBuilder p = viaProto ? proto : builder; + if (this.collectorToken != null) { + return this.collectorToken; + } + if (!p.hasCollectorToken()) { + return null; + } + this.collectorToken = convertFromProtoFormat(p.getCollectorToken()); + return this.collectorToken; + } + + @Override + public void setCollectorToken(Token token) { + maybeInitBuilder(); + if (token == null) { + builder.clearCollectorToken(); + } + this.collectorToken = token; + } + + private TokenPBImpl convertFromProtoFormat(TokenProto p) { + return new TokenPBImpl(p); + } + + private TokenProto convertToProtoFormat(Token t) { + return ((TokenPBImpl) t).getProto(); + } + + private void mergeLocalToBuilder() { + if (this.collectorAddr != null) { + builder.setCollectorAddr(this.collectorAddr); + } + if (this.collectorToken != null) { + builder.setCollectorToken(convertToProtoFormat(this.collectorToken)); + } + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineV2Client.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineV2Client.java index 32cf1e94bad..423c059319c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineV2Client.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineV2Client.java @@ -23,6 +23,7 @@ import java.io.IOException; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.service.CompositeService; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.CollectorInfo; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; import org.apache.hadoop.yarn.client.api.impl.TimelineV2ClientImpl; import org.apache.hadoop.yarn.exceptions.YarnException; @@ -83,10 +84,13 @@ public abstract class TimelineV2Client extends CompositeService { /** *

- * Update the timeline service address where the request will be sent to. + * Update collector info received in AllocateResponse which contains the + * timeline service address where the request will be sent to and the timeline + * delegation token which will be used to send the request. *

* - * @param address the timeline service address + * @param collectorInfo Collector info which contains the timeline service + * address and timeline delegation token. */ - public abstract void setTimelineServiceAddress(String address); + public abstract void setTimelineCollectorInfo(CollectorInfo collectorInfo); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineV2ClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineV2ClientImpl.java index 5d88f708b15..220d6af9ad6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineV2ClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineV2ClientImpl.java @@ -19,7 +19,11 @@ package org.apache.hadoop.yarn.client.api.impl; import java.io.IOException; +import java.io.InterruptedIOException; +import java.lang.reflect.UndeclaredThrowableException; +import java.net.InetSocketAddress; import java.net.URI; +import java.security.PrivilegedExceptionAction; import java.util.concurrent.BlockingQueue; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; @@ -36,15 +40,22 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.CollectorInfo; +import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; import org.apache.hadoop.yarn.client.api.TimelineV2Client; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier; +import com.google.common.annotations.VisibleForTesting; import com.sun.jersey.api.client.ClientResponse; import com.sun.jersey.core.util.MultivaluedMapImpl; @@ -59,6 +70,8 @@ public class TimelineV2ClientImpl extends TimelineV2Client { private TimelineEntityDispatcher entityDispatcher; private volatile String timelineServiceAddress; + @VisibleForTesting + volatile Token currentTimelineToken = null; // Retry parameters for identifying new timeline service // TODO consider to merge with connection retry @@ -69,6 +82,8 @@ public class TimelineV2ClientImpl extends TimelineV2Client { private ApplicationId contextAppId; + private UserGroupInformation authUgi; + public TimelineV2ClientImpl(ApplicationId appId) { super(TimelineV2ClientImpl.class.getName()); this.contextAppId = appId; @@ -88,7 +103,6 @@ public class TimelineV2ClientImpl extends TimelineV2Client { UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); UserGroupInformation realUgi = ugi.getRealUser(); String doAsUser = null; - UserGroupInformation authUgi = null; if (realUgi != null) { authUgi = realUgi; doAsUser = ugi.getShortUserName(); @@ -96,7 +110,6 @@ public class TimelineV2ClientImpl extends TimelineV2Client { authUgi = ugi; doAsUser = null; } - // TODO need to add/cleanup filter retry later for ATSV2. similar to V1 DelegationTokenAuthenticatedURL.Token token = new DelegationTokenAuthenticatedURL.Token(); @@ -140,8 +153,72 @@ public class TimelineV2ClientImpl extends TimelineV2Client { } @Override - public void setTimelineServiceAddress(String address) { - this.timelineServiceAddress = address; + public void setTimelineCollectorInfo(CollectorInfo collectorInfo) { + if (collectorInfo == null) { + LOG.warn("Not setting collector info as it is null."); + return; + } + // First update the token so that it is available when collector address is + // used. + if (collectorInfo.getCollectorToken() != null) { + // Use collector address to update token service if its not available. + setTimelineDelegationToken( + collectorInfo.getCollectorToken(), collectorInfo.getCollectorAddr()); + } + // Update timeline service address. + if (collectorInfo.getCollectorAddr() != null && + !collectorInfo.getCollectorAddr().isEmpty() && + !collectorInfo.getCollectorAddr().equals(timelineServiceAddress)) { + this.timelineServiceAddress = collectorInfo.getCollectorAddr(); + LOG.info("Updated timeline service address to " + timelineServiceAddress); + } + } + + private void setTimelineDelegationToken(Token delegationToken, + String collectorAddr) { + // Checks below are to ensure that an invalid token is not updated in UGI. + // This is required because timeline token is set via a public API. + if (!delegationToken.getKind().equals( + TimelineDelegationTokenIdentifier.KIND_NAME.toString())) { + LOG.warn("Timeline token to be updated should be of kind " + + TimelineDelegationTokenIdentifier.KIND_NAME); + return; + } + if (collectorAddr == null || collectorAddr.isEmpty()) { + collectorAddr = timelineServiceAddress; + } + // Token need not be updated if both address and token service do not exist. + String service = delegationToken.getService(); + if ((service == null || service.isEmpty()) && + (collectorAddr == null || collectorAddr.isEmpty())) { + LOG.warn("Timeline token does not have service and timeline service " + + "address is not yet set. Not updating the token"); + return; + } + // No need to update a duplicate token. + if (currentTimelineToken != null && + currentTimelineToken.equals(delegationToken)) { + return; + } + currentTimelineToken = delegationToken; + // Convert the token, sanitize the token service and add it to UGI. + org.apache.hadoop.security.token. + Token timelineToken = + new org.apache.hadoop.security.token. + Token( + delegationToken.getIdentifier().array(), + delegationToken.getPassword().array(), + new Text(delegationToken.getKind()), + service == null ? new Text() : new Text(service)); + // Prefer timeline service address over service coming in the token for + // updating the token service. + InetSocketAddress serviceAddr = + (collectorAddr != null && !collectorAddr.isEmpty()) ? + NetUtils.createSocketAddr(collectorAddr) : + SecurityUtil.getTokenServiceAddr(timelineToken); + SecurityUtil.setTokenService(timelineToken, serviceAddr); + authUgi.addToken(timelineToken); + LOG.info("Updated timeline delegation token " + timelineToken); } @Private @@ -192,19 +269,33 @@ public class TimelineV2ClientImpl extends TimelineV2Client { } } + private ClientResponse doPutObjects(URI base, String path, + MultivaluedMap params, Object obj) { + return connector.getClient().resource(base).path(path).queryParams(params) + .accept(MediaType.APPLICATION_JSON).type(MediaType.APPLICATION_JSON) + .put(ClientResponse.class, obj); + } + protected void putObjects(URI base, String path, MultivaluedMap params, Object obj) throws IOException, YarnException { - ClientResponse resp; + ClientResponse resp = null; try { - resp = connector.getClient().resource(base).path(path).queryParams(params) - .accept(MediaType.APPLICATION_JSON).type(MediaType.APPLICATION_JSON) - .put(ClientResponse.class, obj); - } catch (RuntimeException re) { - // runtime exception is expected if the client cannot connect the server - String msg = "Failed to get the response from the timeline server."; - LOG.error(msg, re); - throw new IOException(re); + resp = authUgi.doAs(new PrivilegedExceptionAction() { + @Override + public ClientResponse run() throws Exception { + return doPutObjects(base, path, params, obj); + } + }); + } catch (UndeclaredThrowableException ue) { + Throwable cause = ue.getCause(); + if (cause instanceof IOException) { + throw (IOException)cause; + } else { + throw new IOException(cause); + } + } catch (InterruptedException ie) { + throw (IOException) new InterruptedIOException().initCause(ie); } if (resp == null || resp.getStatusInfo() .getStatusCode() != ClientResponse.Status.OK.getStatusCode()) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 0823dfe7531..17d04965dda 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -2341,6 +2341,39 @@ 259200000 + + + The default hdfs location for flowrun coprocessor jar. + + yarn.timeline-service.hbase.coprocessor.jar.hdfs.location + + /hbase/coprocessor/hadoop-yarn-server-timelineservice.jar + + + + + The value of this parameter sets the prefix for all tables that are part of + timeline service in the hbase storage schema. It can be set to "dev." + or "staging." if it is to be used for development or staging instances. + This way the data in production tables stays in a separate set of tables + prefixed by "prod.". + + yarn.timeline-service.hbase-schema.prefix + prod. + + + + Optional URL to an hbase-site.xml configuration file to be + used to connect to the timeline-service hbase cluster. If empty or not + specified, then the HBase configuration will be loaded from the classpath. + When specified the values in the specified configuration file will override + those from the ones that are present on the classpath. + + yarn.timeline-service.hbase.configuration.file + + + + @@ -3145,6 +3178,17 @@ 64 + + + Flag to enable cross-origin (CORS) support for timeline service v1.x or + Timeline Reader in timeline service v2. For timeline service v2, also add + org.apache.hadoop.security.HttpCrossOriginFilterInitializer to the + configuration hadoop.http.filter.initializers in core-site.xml. + + yarn.timeline-service.http-cross-origin.enabled + false + + Flag to enable cross-origin (CORS) support for timeline service v1.x or diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java index bb688c93a9f..a3f5491cdc0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java @@ -101,6 +101,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StopContainersRequestP import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StopContainersResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SubmitApplicationRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SubmitApplicationResponsePBImpl; +import org.apache.hadoop.yarn.api.records.CollectorInfo; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -406,6 +407,7 @@ public class TestPBImplRecords extends BasePBImplRecordsTest { generateByNewInstance(CommitResponse.class); generateByNewInstance(ApplicationTimeout.class); generateByNewInstance(QueueConfigurations.class); + generateByNewInstance(CollectorInfo.class); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClientV2Impl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClientV2Impl.java index c5b02fd32ca..95595a9f4c4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClientV2Impl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClientV2Impl.java @@ -18,6 +18,11 @@ package org.apache.hadoop.yarn.client.api.impl; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; + import java.io.IOException; import java.net.URI; import java.util.ArrayList; @@ -27,11 +32,16 @@ import javax.ws.rs.core.MultivaluedMap; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.CollectorInfo; +import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -151,7 +161,7 @@ public class TestTimelineClientV2Impl { maxRetries); c.init(conf); c.start(); - c.setTimelineServiceAddress("localhost:12345"); + c.setTimelineCollectorInfo(CollectorInfo.newInstance("localhost:12345")); try { c.putEntities(new TimelineEntity()); } catch (IOException e) { @@ -310,6 +320,50 @@ public class TestTimelineClientV2Impl { } } + @Test + public void testSetTimelineToken() throws Exception { + UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + assertEquals(0, ugi.getTokens().size()); + assertNull("Timeline token in v2 client should not be set", + client.currentTimelineToken); + + Token token = Token.newInstance( + new byte[0], "kind", new byte[0], "service"); + client.setTimelineCollectorInfo(CollectorInfo.newInstance(null, token)); + assertNull("Timeline token in v2 client should not be set as token kind " + + "is unexepcted.", client.currentTimelineToken); + assertEquals(0, ugi.getTokens().size()); + + token = Token.newInstance(new byte[0], TimelineDelegationTokenIdentifier. + KIND_NAME.toString(), new byte[0], null); + client.setTimelineCollectorInfo(CollectorInfo.newInstance(null, token)); + assertNull("Timeline token in v2 client should not be set as serice is " + + "not set.", client.currentTimelineToken); + assertEquals(0, ugi.getTokens().size()); + + TimelineDelegationTokenIdentifier ident = + new TimelineDelegationTokenIdentifier(new Text(ugi.getUserName()), + new Text("renewer"), null); + ident.setSequenceNumber(1); + token = Token.newInstance(ident.getBytes(), + TimelineDelegationTokenIdentifier.KIND_NAME.toString(), new byte[0], + "localhost:1234"); + client.setTimelineCollectorInfo(CollectorInfo.newInstance(null, token)); + assertEquals(1, ugi.getTokens().size()); + assertNotNull("Timeline token should be set in v2 client.", + client.currentTimelineToken); + assertEquals(token, client.currentTimelineToken); + + ident.setSequenceNumber(20); + Token newToken = Token.newInstance(ident.getBytes(), + TimelineDelegationTokenIdentifier.KIND_NAME.toString(), new byte[0], + "localhost:1234"); + client.setTimelineCollectorInfo(CollectorInfo.newInstance(null, newToken)); + assertEquals(1, ugi.getTokens().size()); + assertNotEquals(token, client.currentTimelineToken); + assertEquals(newToken, client.currentTimelineToken); + } + @Test public void testAfterStop() throws Exception { client.setSleepBeforeReturn(true); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java index 85e5f2db0af..4e3a1e603a3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java @@ -20,14 +20,14 @@ package org.apache.hadoop.yarn.server.applicationhistoryservice; import java.io.IOException; import java.net.InetSocketAddress; -import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.Set; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.http.HttpServer2; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.metrics2.source.JvmMetrics; -import org.apache.hadoop.security.AuthenticationFilterInitializer; import org.apache.hadoop.security.HttpCrossOriginFilterInitializer; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.service.CompositeService; @@ -47,10 +47,9 @@ import org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore; import org.apache.hadoop.yarn.server.timeline.TimelineDataManager; import org.apache.hadoop.yarn.server.timeline.TimelineStore; import org.apache.hadoop.yarn.server.timeline.security.TimelineACLsManager; -import org.apache.hadoop.yarn.server.timeline.security.TimelineAuthenticationFilter; -import org.apache.hadoop.yarn.server.timeline.security.TimelineAuthenticationFilterInitializer; -import org.apache.hadoop.yarn.server.timeline.security.TimelineDelegationTokenSecretManagerService; +import org.apache.hadoop.yarn.server.timeline.security.TimelineV1DelegationTokenSecretManagerService; import org.apache.hadoop.yarn.server.timeline.webapp.CrossOriginFilterInitializer; +import org.apache.hadoop.yarn.server.util.timeline.TimelineServerUtils; import org.apache.hadoop.yarn.webapp.WebApp; import org.apache.hadoop.yarn.webapp.WebApps; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; @@ -75,7 +74,7 @@ public class ApplicationHistoryServer extends CompositeService { private ApplicationACLsManager aclsManager; private ApplicationHistoryManager historyManager; private TimelineStore timelineStore; - private TimelineDelegationTokenSecretManagerService secretManagerService; + private TimelineV1DelegationTokenSecretManagerService secretManagerService; private TimelineDataManager timelineDataManager; private WebApp webApp; private JvmPauseMonitor pauseMonitor; @@ -223,9 +222,9 @@ public class ApplicationHistoryServer extends CompositeService { TimelineStore.class), conf); } - private TimelineDelegationTokenSecretManagerService + private TimelineV1DelegationTokenSecretManagerService createTimelineDelegationTokenSecretManagerService(Configuration conf) { - return new TimelineDelegationTokenSecretManagerService(); + return new TimelineV1DelegationTokenSecretManagerService(); } private TimelineDataManager createTimelineDataManager(Configuration conf) { @@ -237,63 +236,33 @@ public class ApplicationHistoryServer extends CompositeService { @SuppressWarnings("unchecked") private void startWebApp() { Configuration conf = getConfig(); - TimelineAuthenticationFilter.setTimelineDelegationTokenSecretManager( - secretManagerService.getTimelineDelegationTokenSecretManager()); // Always load pseudo authentication filter to parse "user.name" in an URL // to identify a HTTP request's user in insecure mode. // When Kerberos authentication type is set (i.e., secure mode is turned on), // the customized filter will be loaded by the timeline server to do Kerberos // + DT authentication. - String initializers = conf.get("hadoop.http.filter.initializers"); - boolean modifiedInitializers = false; - - initializers = - initializers == null || initializers.length() == 0 ? "" : initializers; - + String initializers = conf.get("hadoop.http.filter.initializers", ""); + Set defaultInitializers = new LinkedHashSet(); + // Add CORS filter if (!initializers.contains(CrossOriginFilterInitializer.class.getName())) { - if(conf.getBoolean(YarnConfiguration - .TIMELINE_SERVICE_HTTP_CROSS_ORIGIN_ENABLED, YarnConfiguration - .TIMELINE_SERVICE_HTTP_CROSS_ORIGIN_ENABLED_DEFAULT)) { - if (initializers.contains(HttpCrossOriginFilterInitializer.class.getName())) { - initializers = - initializers.replaceAll(HttpCrossOriginFilterInitializer.class.getName(), + if(conf.getBoolean(YarnConfiguration. + TIMELINE_SERVICE_HTTP_CROSS_ORIGIN_ENABLED, + YarnConfiguration. + TIMELINE_SERVICE_HTTP_CROSS_ORIGIN_ENABLED_DEFAULT)) { + if (initializers.contains( + HttpCrossOriginFilterInitializer.class.getName())) { + initializers = initializers.replaceAll( + HttpCrossOriginFilterInitializer.class.getName(), CrossOriginFilterInitializer.class.getName()); + } else { + defaultInitializers.add(CrossOriginFilterInitializer.class.getName()); } - else { - if (initializers.length() != 0) { - initializers += ","; - } - initializers += CrossOriginFilterInitializer.class.getName(); - } - modifiedInitializers = true; } } - - if (!initializers.contains(TimelineAuthenticationFilterInitializer.class - .getName())) { - if (initializers.length() != 0) { - initializers += ","; - } - initializers += TimelineAuthenticationFilterInitializer.class.getName(); - modifiedInitializers = true; - } - - String[] parts = initializers.split(","); - ArrayList target = new ArrayList(); - for (String filterInitializer : parts) { - filterInitializer = filterInitializer.trim(); - if (filterInitializer.equals(AuthenticationFilterInitializer.class - .getName())) { - modifiedInitializers = true; - continue; - } - target.add(filterInitializer); - } - String actualInitializers = - org.apache.commons.lang.StringUtils.join(target, ","); - if (modifiedInitializers) { - conf.set("hadoop.http.filter.initializers", actualInitializers); - } + TimelineServerUtils.addTimelineAuthFilter( + initializers, defaultInitializers, secretManagerService); + TimelineServerUtils.setTimelineFilters( + conf, initializers, defaultInitializers); String bindAddress = WebAppUtils.getWebAppBindURL(conf, YarnConfiguration.TIMELINE_SERVICE_BIND_HOST, WebAppUtils.getAHSWebAppURLWithoutScheme(conf)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineDelegationTokenSecretManagerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineV1DelegationTokenSecretManagerService.java similarity index 79% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineDelegationTokenSecretManagerService.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineV1DelegationTokenSecretManagerService.java index 0c6892a19d3..85d8ccab3c3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineDelegationTokenSecretManagerService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineV1DelegationTokenSecretManagerService.java @@ -26,7 +26,6 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager; import org.apache.hadoop.security.token.delegation.DelegationKey; -import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier; @@ -37,18 +36,16 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * The service wrapper of {@link TimelineDelegationTokenSecretManager} + * The service wrapper of {@link TimelineV1DelegationTokenSecretManager}. */ @Private @Unstable -public class TimelineDelegationTokenSecretManagerService extends - AbstractService { - - private TimelineDelegationTokenSecretManager secretManager = null; +public class TimelineV1DelegationTokenSecretManagerService extends + TimelineDelgationTokenSecretManagerService { private TimelineStateStore stateStore = null; - public TimelineDelegationTokenSecretManagerService() { - super(TimelineDelegationTokenSecretManagerService.class.getName()); + public TimelineV1DelegationTokenSecretManagerService() { + super(TimelineV1DelegationTokenSecretManagerService.class.getName()); } @Override @@ -58,19 +55,7 @@ public class TimelineDelegationTokenSecretManagerService extends stateStore = createStateStore(conf); stateStore.init(conf); } - - long secretKeyInterval = - conf.getLong(YarnConfiguration.TIMELINE_DELEGATION_KEY_UPDATE_INTERVAL, - YarnConfiguration.DEFAULT_TIMELINE_DELEGATION_KEY_UPDATE_INTERVAL); - long tokenMaxLifetime = - conf.getLong(YarnConfiguration.TIMELINE_DELEGATION_TOKEN_MAX_LIFETIME, - YarnConfiguration.DEFAULT_TIMELINE_DELEGATION_TOKEN_MAX_LIFETIME); - long tokenRenewInterval = - conf.getLong(YarnConfiguration.TIMELINE_DELEGATION_TOKEN_RENEW_INTERVAL, - YarnConfiguration.DEFAULT_TIMELINE_DELEGATION_TOKEN_RENEW_INTERVAL); - secretManager = new TimelineDelegationTokenSecretManager(secretKeyInterval, - tokenMaxLifetime, tokenRenewInterval, 3600000, stateStore); - super.init(conf); + super.serviceInit(conf); } @Override @@ -78,10 +63,9 @@ public class TimelineDelegationTokenSecretManagerService extends if (stateStore != null) { stateStore.start(); TimelineServiceState state = stateStore.loadState(); - secretManager.recover(state); + ((TimelineV1DelegationTokenSecretManager) + getTimelineDelegationTokenSecretManager()).recover(state); } - - secretManager.startThreads(); super.serviceStart(); } @@ -90,9 +74,18 @@ public class TimelineDelegationTokenSecretManagerService extends if (stateStore != null) { stateStore.stop(); } + super.serviceStop(); + } - secretManager.stopThreads(); - super.stop(); + @Override + protected AbstractDelegationTokenSecretManager + + createTimelineDelegationTokenSecretManager(long secretKeyInterval, + long tokenMaxLifetime, long tokenRenewInterval, + long tokenRemovalScanInterval) { + return new TimelineV1DelegationTokenSecretManager(secretKeyInterval, + tokenMaxLifetime, tokenRenewInterval, tokenRemovalScanInterval, + stateStore); } protected TimelineStateStore createStateStore( @@ -104,27 +97,20 @@ public class TimelineDelegationTokenSecretManagerService extends } /** - * Ge the instance of {link #TimelineDelegationTokenSecretManager} - * - * @return the instance of {link #TimelineDelegationTokenSecretManager} + * Delegation token secret manager for ATSv1 and ATSv1.5. */ - public TimelineDelegationTokenSecretManager - getTimelineDelegationTokenSecretManager() { - return secretManager; - } - @Private @Unstable - public static class TimelineDelegationTokenSecretManager extends + public static class TimelineV1DelegationTokenSecretManager extends AbstractDelegationTokenSecretManager { public static final Logger LOG = - LoggerFactory.getLogger(TimelineDelegationTokenSecretManager.class); + LoggerFactory.getLogger(TimelineV1DelegationTokenSecretManager.class); private TimelineStateStore stateStore; /** - * Create a timeline secret manager + * Create a timeline v1 secret manager. * @param delegationKeyUpdateInterval the number of milliseconds for rolling * new secret keys. * @param delegationTokenMaxLifetime the maximum lifetime of the delegation @@ -135,7 +121,7 @@ public class TimelineDelegationTokenSecretManagerService extends * scanned for expired tokens in milliseconds * @param stateStore timeline service state store */ - public TimelineDelegationTokenSecretManager( + public TimelineV1DelegationTokenSecretManager( long delegationKeyUpdateInterval, long delegationTokenMaxLifetime, long delegationTokenRenewInterval, @@ -236,5 +222,4 @@ public class TimelineDelegationTokenSecretManagerService extends } } } - } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilterForV1.java similarity index 85% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilter.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilterForV1.java index 063f5121e50..d918e8ddde6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilterForV1.java @@ -55,27 +55,31 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +/** + * Test cases for authentication via TimelineAuthenticationFilter while + * publishing entities for ATSv1. + */ @RunWith(Parameterized.class) -public class TestTimelineAuthenticationFilter { +public class TestTimelineAuthenticationFilterForV1 { private static final String FOO_USER = "foo"; private static final String BAR_USER = "bar"; private static final String HTTP_USER = "HTTP"; - private static final File testRootDir = new File( + private static final File TEST_ROOT_DIR = new File( System.getProperty("test.build.dir", "target/test-dir"), - TestTimelineAuthenticationFilter.class.getName() + "-root"); + TestTimelineAuthenticationFilterForV1.class.getName() + "-root"); private static File httpSpnegoKeytabFile = new File( KerberosTestUtils.getKeytabFile()); private static String httpSpnegoPrincipal = KerberosTestUtils.getServerPrincipal(); private static final String BASEDIR = System.getProperty("test.build.dir", "target/test-dir") + "/" - + TestTimelineAuthenticationFilter.class.getSimpleName(); + + TestTimelineAuthenticationFilterForV1.class.getSimpleName(); @Parameterized.Parameters public static Collection withSsl() { - return Arrays.asList(new Object[][] { { false }, { true } }); + return Arrays.asList(new Object[][] {{false}, {true}}); } private static MiniKdc testMiniKDC; @@ -85,14 +89,14 @@ public class TestTimelineAuthenticationFilter { private static Configuration conf; private static boolean withSsl; - public TestTimelineAuthenticationFilter(boolean withSsl) { - TestTimelineAuthenticationFilter.withSsl = withSsl; + public TestTimelineAuthenticationFilterForV1(boolean withSsl) { + TestTimelineAuthenticationFilterForV1.withSsl = withSsl; } @BeforeClass public static void setup() { try { - testMiniKDC = new MiniKdc(MiniKdc.createConf(), testRootDir); + testMiniKDC = new MiniKdc(MiniKdc.createConf(), TEST_ROOT_DIR); testMiniKDC.start(); testMiniKDC.createPrincipal( httpSpnegoKeytabFile, HTTP_USER + "/localhost"); @@ -111,11 +115,11 @@ public class TestTimelineAuthenticationFilter { KerberosAuthenticationHandler.KEYTAB, httpSpnegoKeytabFile.getAbsolutePath()); conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, - "kerberos"); + "kerberos"); conf.set(YarnConfiguration.TIMELINE_SERVICE_PRINCIPAL, - httpSpnegoPrincipal); + httpSpnegoPrincipal); conf.set(YarnConfiguration.TIMELINE_SERVICE_KEYTAB, - httpSpnegoKeytabFile.getAbsolutePath()); + httpSpnegoKeytabFile.getAbsolutePath()); conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); conf.setClass(YarnConfiguration.TIMELINE_SERVICE_STORE, MemoryTimelineStore.class, TimelineStore.class); @@ -136,8 +140,8 @@ public class TestTimelineAuthenticationFilter { FileUtil.fullyDelete(base); base.mkdirs(); keystoresDir = new File(BASEDIR).getAbsolutePath(); - sslConfDir = - KeyStoreTestUtil.getClasspathDir(TestTimelineAuthenticationFilter.class); + sslConfDir = KeyStoreTestUtil.getClasspathDir( + TestTimelineAuthenticationFilterForV1.class); KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, conf, false); } @@ -145,6 +149,7 @@ public class TestTimelineAuthenticationFilter { testTimelineServer.init(conf); testTimelineServer.start(); } catch (Exception e) { + e.printStackTrace(); assertTrue("Couldn't setup TimelineServer", false); } } @@ -181,14 +186,14 @@ public class TestTimelineAuthenticationFilter { TimelineClient client = createTimelineClientForUGI(); TimelineEntity entityToStore = new TimelineEntity(); entityToStore.setEntityType( - TestTimelineAuthenticationFilter.class.getName()); + TestTimelineAuthenticationFilterForV1.class.getName()); entityToStore.setEntityId("entity1"); entityToStore.setStartTime(0L); TimelinePutResponse putResponse = client.putEntities(entityToStore); Assert.assertEquals(0, putResponse.getErrors().size()); TimelineEntity entityToRead = - testTimelineServer.getTimelineStore().getEntity( - "entity1", TestTimelineAuthenticationFilter.class.getName(), null); + testTimelineServer.getTimelineStore().getEntity("entity1", + TestTimelineAuthenticationFilterForV1.class.getName(), null); Assert.assertNotNull(entityToRead); return null; } @@ -202,13 +207,14 @@ public class TestTimelineAuthenticationFilter { public Void call() throws Exception { TimelineClient client = createTimelineClientForUGI(); TimelineDomain domainToStore = new TimelineDomain(); - domainToStore.setId(TestTimelineAuthenticationFilter.class.getName()); + domainToStore.setId( + TestTimelineAuthenticationFilterForV1.class.getName()); domainToStore.setReaders("*"); domainToStore.setWriters("*"); client.putDomain(domainToStore); TimelineDomain domainToRead = testTimelineServer.getTimelineStore().getDomain( - TestTimelineAuthenticationFilter.class.getName()); + TestTimelineAuthenticationFilterForV1.class.getName()); Assert.assertNotNull(domainToRead); return null; } @@ -218,22 +224,24 @@ public class TestTimelineAuthenticationFilter { @Test public void testDelegationTokenOperations() throws Exception { TimelineClient httpUserClient = - KerberosTestUtils.doAs(HTTP_USER + "/localhost", new Callable() { - @Override - public TimelineClient call() throws Exception { - return createTimelineClientForUGI(); - } - }); + KerberosTestUtils.doAs(HTTP_USER + "/localhost", + new Callable() { + @Override + public TimelineClient call() throws Exception { + return createTimelineClientForUGI(); + } + }); UserGroupInformation httpUser = - KerberosTestUtils.doAs(HTTP_USER + "/localhost", new Callable() { - @Override - public UserGroupInformation call() throws Exception { - return UserGroupInformation.getCurrentUser(); - } - }); + KerberosTestUtils.doAs(HTTP_USER + "/localhost", + new Callable() { + @Override + public UserGroupInformation call() throws Exception { + return UserGroupInformation.getCurrentUser(); + } + }); // Let HTTP user to get the delegation for itself Token token = - httpUserClient.getDelegationToken(httpUser.getShortUserName()); + httpUserClient.getDelegationToken(httpUser.getShortUserName()); Assert.assertNotNull(token); TimelineDelegationTokenIdentifier tDT = token.decodeIdentifier(); Assert.assertNotNull(tDT); @@ -317,7 +325,8 @@ public class TestTimelineAuthenticationFilter { barUserClient.getDelegationToken(httpUser.getShortUserName()); Assert.fail(); } catch (Exception e) { - Assert.assertTrue(e.getCause() instanceof AuthorizationException || e.getCause() instanceof AuthenticationException); + Assert.assertTrue(e.getCause() instanceof AuthorizationException || + e.getCause() instanceof AuthenticationException); } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatRequest.java index c795e556c46..f238f79f172 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatRequest.java @@ -24,6 +24,7 @@ import java.util.Set; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.NodeLabel; +import org.apache.hadoop.yarn.server.api.records.AppCollectorData; import org.apache.hadoop.yarn.server.api.records.MasterKey; import org.apache.hadoop.yarn.server.api.records.NodeStatus; import org.apache.hadoop.yarn.util.Records; @@ -47,7 +48,7 @@ public abstract class NodeHeartbeatRequest { public static NodeHeartbeatRequest newInstance(NodeStatus nodeStatus, MasterKey lastKnownContainerTokenMasterKey, MasterKey lastKnownNMTokenMasterKey, Set nodeLabels, - Map registeredCollectors) { + Map registeringCollectors) { NodeHeartbeatRequest nodeHeartbeatRequest = Records.newRecord(NodeHeartbeatRequest.class); nodeHeartbeatRequest.setNodeStatus(nodeStatus); @@ -56,7 +57,7 @@ public abstract class NodeHeartbeatRequest { nodeHeartbeatRequest .setLastKnownNMTokenMasterKey(lastKnownNMTokenMasterKey); nodeHeartbeatRequest.setNodeLabels(nodeLabels); - nodeHeartbeatRequest.setRegisteredCollectors(registeredCollectors); + nodeHeartbeatRequest.setRegisteringCollectors(registeringCollectors); return nodeHeartbeatRequest; } @@ -79,7 +80,9 @@ public abstract class NodeHeartbeatRequest { List logAggregationReportsForApps); // This tells RM registered collectors' address info on this node - public abstract Map getRegisteredCollectors(); - public abstract void setRegisteredCollectors(Map appCollectorsMap); + public abstract Map + getRegisteringCollectors(); + + public abstract void setRegisteringCollectors(Map appCollectorsMap); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java index 3b0ec10595b..2ebca570853 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java @@ -28,6 +28,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.server.api.records.AppCollectorData; import org.apache.hadoop.yarn.server.api.records.ContainerQueuingLimit; import org.apache.hadoop.yarn.server.api.records.MasterKey; import org.apache.hadoop.yarn.server.api.records.NodeAction; @@ -47,10 +48,9 @@ public abstract class NodeHeartbeatResponse { public abstract List getApplicationsToCleanup(); // This tells NM the collectors' address info of related apps - public abstract Map getAppCollectorsMap(); - - public abstract void setAppCollectorsMap( - Map appCollectorsMap); + public abstract Map getAppCollectors(); + public abstract void setAppCollectors( + Map appCollectorsMap); public abstract void setResponseId(int responseId); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/ReportNewCollectorInfoRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/ReportNewCollectorInfoRequest.java index 3498de90a3c..a4c1a384c26 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/ReportNewCollectorInfoRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/ReportNewCollectorInfoRequest.java @@ -22,14 +22,15 @@ import java.util.Arrays; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.server.api.records.AppCollectorsMap; +import org.apache.hadoop.yarn.api.records.Token; +import org.apache.hadoop.yarn.server.api.records.AppCollectorData; import org.apache.hadoop.yarn.util.Records; @Private public abstract class ReportNewCollectorInfoRequest { public static ReportNewCollectorInfoRequest newInstance( - List appCollectorsList) { + List appCollectorsList) { ReportNewCollectorInfoRequest request = Records.newRecord(ReportNewCollectorInfoRequest.class); request.setAppCollectorsList(appCollectorsList); @@ -37,17 +38,17 @@ public abstract class ReportNewCollectorInfoRequest { } public static ReportNewCollectorInfoRequest newInstance( - ApplicationId id, String collectorAddr) { + ApplicationId id, String collectorAddr, Token token) { ReportNewCollectorInfoRequest request = Records.newRecord(ReportNewCollectorInfoRequest.class); request.setAppCollectorsList( - Arrays.asList(AppCollectorsMap.newInstance(id, collectorAddr))); + Arrays.asList(AppCollectorData.newInstance(id, collectorAddr, token))); return request; } - public abstract List getAppCollectorsList(); + public abstract List getAppCollectorsList(); public abstract void setAppCollectorsList( - List appCollectorsList); + List appCollectorsList); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java index d0c11985db3..1ffd223f8a6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java @@ -26,16 +26,20 @@ import java.util.List; import java.util.Map; import java.util.Set; +import org.apache.hadoop.security.proto.SecurityProtos.TokenProto; +import org.apache.hadoop.yarn.server.api.records.AppCollectorData; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.NodeLabel; +import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.NodeLabelPBImpl; +import org.apache.hadoop.yarn.api.records.impl.pb.TokenPBImpl; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto; import org.apache.hadoop.yarn.proto.YarnProtos.NodeLabelProto; import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.MasterKeyProto; import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.NodeStatusProto; +import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.AppCollectorDataProto; import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.LogAggregationReportProto; -import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.AppCollectorsMapProto; import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NodeHeartbeatRequestProto; import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NodeHeartbeatRequestProtoOrBuilder; import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NodeLabelsProto; @@ -58,7 +62,7 @@ public class NodeHeartbeatRequestPBImpl extends NodeHeartbeatRequest { private Set labels = null; private List logAggregationReportsForApps = null; - private Map registeredCollectors = null; + private Map registeringCollectors = null; public NodeHeartbeatRequestPBImpl() { builder = NodeHeartbeatRequestProto.newBuilder(); @@ -114,8 +118,8 @@ public class NodeHeartbeatRequestPBImpl extends NodeHeartbeatRequest { if (this.logAggregationReportsForApps != null) { addLogAggregationStatusForAppsToProto(); } - if (this.registeredCollectors != null) { - addRegisteredCollectorsToProto(); + if (this.registeringCollectors != null) { + addRegisteringCollectorsToProto(); } } @@ -158,14 +162,23 @@ public class NodeHeartbeatRequestPBImpl extends NodeHeartbeatRequest { return ((LogAggregationReportPBImpl) value).getProto(); } - private void addRegisteredCollectorsToProto() { + private void addRegisteringCollectorsToProto() { maybeInitBuilder(); - builder.clearRegisteredCollectors(); - for (Map.Entry entry : - registeredCollectors.entrySet()) { - builder.addRegisteredCollectors(AppCollectorsMapProto.newBuilder() - .setAppId(convertToProtoFormat(entry.getKey())) - .setAppCollectorAddr(entry.getValue())); + builder.clearRegisteringCollectors(); + for (Map.Entry entry : + registeringCollectors.entrySet()) { + AppCollectorData data = entry.getValue(); + AppCollectorDataProto.Builder appCollectorDataBuilder = + AppCollectorDataProto.newBuilder() + .setAppId(convertToProtoFormat(entry.getKey())) + .setAppCollectorAddr(data.getCollectorAddr()) + .setRmIdentifier(data.getRMIdentifier()) + .setVersion(data.getVersion()); + if (data.getCollectorToken() != null) { + appCollectorDataBuilder.setAppCollectorToken( + convertToProtoFormat(data.getCollectorToken())); + } + builder.addRegisteringCollectors(appCollectorDataBuilder); } } @@ -251,35 +264,42 @@ public class NodeHeartbeatRequestPBImpl extends NodeHeartbeatRequest { } @Override - public Map getRegisteredCollectors() { - if (this.registeredCollectors != null) { - return this.registeredCollectors; + public Map getRegisteringCollectors() { + if (this.registeringCollectors != null) { + return this.registeringCollectors; } initRegisteredCollectors(); - return registeredCollectors; + return registeringCollectors; } private void initRegisteredCollectors() { NodeHeartbeatRequestProtoOrBuilder p = viaProto ? proto : builder; - List list = p.getRegisteredCollectorsList(); + List list = p.getRegisteringCollectorsList(); if (!list.isEmpty()) { - this.registeredCollectors = new HashMap<>(); - for (AppCollectorsMapProto c : list) { + this.registeringCollectors = new HashMap<>(); + for (AppCollectorDataProto c : list) { ApplicationId appId = convertFromProtoFormat(c.getAppId()); - this.registeredCollectors.put(appId, c.getAppCollectorAddr()); + Token collectorToken = null; + if (c.hasAppCollectorToken()){ + collectorToken = convertFromProtoFormat(c.getAppCollectorToken()); + } + AppCollectorData data = AppCollectorData.newInstance(appId, + c.getAppCollectorAddr(), c.getRmIdentifier(), c.getVersion(), + collectorToken); + this.registeringCollectors.put(appId, data); } } } @Override - public void setRegisteredCollectors( - Map registeredCollectors) { + public void setRegisteringCollectors( + Map registeredCollectors) { if (registeredCollectors == null || registeredCollectors.isEmpty()) { return; } maybeInitBuilder(); - this.registeredCollectors = new HashMap(); - this.registeredCollectors.putAll(registeredCollectors); + this.registeringCollectors = new HashMap<>(); + this.registeringCollectors.putAll(registeredCollectors); } private NodeStatusPBImpl convertFromProtoFormat(NodeStatusProto p) { @@ -306,6 +326,14 @@ public class NodeHeartbeatRequestPBImpl extends NodeHeartbeatRequest { return ((MasterKeyPBImpl)t).getProto(); } + private TokenPBImpl convertFromProtoFormat(TokenProto p) { + return new TokenPBImpl(p); + } + + private TokenProto convertToProtoFormat(Token t) { + return ((TokenPBImpl) t).getProto(); + } + @Override public Set getNodeLabels() { initNodeLabels(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java index 46c2b0b8789..11f5f61416f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java @@ -26,26 +26,30 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import org.apache.hadoop.security.proto.SecurityProtos.TokenProto; import org.apache.hadoop.yarn.api.protocolrecords.SignalContainerRequest; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SignalContainerRequestPBImpl; +import org.apache.hadoop.yarn.server.api.records.AppCollectorData; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.ContainerPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils; import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl; +import org.apache.hadoop.yarn.api.records.impl.pb.TokenPBImpl; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto; import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto; import org.apache.hadoop.yarn.proto.YarnProtos.ContainerProto; import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto; +import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.AppCollectorDataProto; import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.ContainerQueuingLimitProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.SignalContainerRequestProto; import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.MasterKeyProto; import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.NodeActionProto; -import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.AppCollectorsMapProto; import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NodeHeartbeatResponseProto; import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NodeHeartbeatResponseProtoOrBuilder; import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.SystemCredentialsForAppsProto; @@ -70,7 +74,7 @@ public class NodeHeartbeatResponsePBImpl extends NodeHeartbeatResponse { private List applicationsToCleanup = null; private Map systemCredentials = null; private Resource resource = null; - private Map appCollectorsMap = null; + private Map appCollectorsMap = null; private MasterKey containerTokenMasterKey = null; private MasterKey nmTokenMasterKey = null; @@ -146,11 +150,21 @@ public class NodeHeartbeatResponsePBImpl extends NodeHeartbeatResponse { private void addAppCollectorsMapToProto() { maybeInitBuilder(); - builder.clearAppCollectorsMap(); - for (Map.Entry entry : appCollectorsMap.entrySet()) { - builder.addAppCollectorsMap(AppCollectorsMapProto.newBuilder() - .setAppId(convertToProtoFormat(entry.getKey())) - .setAppCollectorAddr(entry.getValue())); + builder.clearAppCollectors(); + for (Map.Entry entry + : appCollectorsMap.entrySet()) { + AppCollectorData data = entry.getValue(); + AppCollectorDataProto.Builder appCollectorDataBuilder = + AppCollectorDataProto.newBuilder() + .setAppId(convertToProtoFormat(entry.getKey())) + .setAppCollectorAddr(data.getCollectorAddr()) + .setRmIdentifier(data.getRMIdentifier()) + .setVersion(data.getVersion()); + if (data.getCollectorToken() != null) { + appCollectorDataBuilder.setAppCollectorToken( + convertToProtoFormat(data.getCollectorToken())); + } + builder.addAppCollectors(appCollectorDataBuilder); } } @@ -568,7 +582,7 @@ public class NodeHeartbeatResponsePBImpl extends NodeHeartbeatResponse { } @Override - public Map getAppCollectorsMap() { + public Map getAppCollectors() { if (this.appCollectorsMap != null) { return this.appCollectorsMap; } @@ -589,12 +603,19 @@ public class NodeHeartbeatResponsePBImpl extends NodeHeartbeatResponse { private void initAppCollectorsMap() { NodeHeartbeatResponseProtoOrBuilder p = viaProto ? proto : builder; - List list = p.getAppCollectorsMapList(); + List list = p.getAppCollectorsList(); if (!list.isEmpty()) { this.appCollectorsMap = new HashMap<>(); - for (AppCollectorsMapProto c : list) { + for (AppCollectorDataProto c : list) { ApplicationId appId = convertFromProtoFormat(c.getAppId()); - this.appCollectorsMap.put(appId, c.getAppCollectorAddr()); + Token collectorToken = null; + if (c.hasAppCollectorToken()){ + collectorToken = convertFromProtoFormat(c.getAppCollectorToken()); + } + AppCollectorData data = AppCollectorData.newInstance(appId, + c.getAppCollectorAddr(), c.getRmIdentifier(), c.getVersion(), + collectorToken); + this.appCollectorsMap.put(appId, data); } } } @@ -611,14 +632,14 @@ public class NodeHeartbeatResponsePBImpl extends NodeHeartbeatResponse { } @Override - public void setAppCollectorsMap( - Map appCollectorsMap) { - if (appCollectorsMap == null || appCollectorsMap.isEmpty()) { + public void setAppCollectors( + Map appCollectors) { + if (appCollectors == null || appCollectors.isEmpty()) { return; } maybeInitBuilder(); - this.appCollectorsMap = new HashMap(); - this.appCollectorsMap.putAll(appCollectorsMap); + this.appCollectorsMap = new HashMap<>(); + this.appCollectorsMap.putAll(appCollectors); } @Override @@ -773,5 +794,13 @@ public class NodeHeartbeatResponsePBImpl extends NodeHeartbeatResponse { SignalContainerRequest t) { return ((SignalContainerRequestPBImpl)t).getProto(); } + + private TokenProto convertToProtoFormat(Token t) { + return ((TokenPBImpl) t).getProto(); + } + + private TokenPBImpl convertFromProtoFormat(TokenProto p) { + return new TokenPBImpl(p); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReportNewCollectorInfoRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReportNewCollectorInfoRequestPBImpl.java index c6f66194e5d..5ffc3a24089 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReportNewCollectorInfoRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReportNewCollectorInfoRequestPBImpl.java @@ -20,12 +20,12 @@ package org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb; import java.util.ArrayList; import java.util.List; -import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.AppCollectorsMapProto; +import org.apache.hadoop.yarn.server.api.records.AppCollectorData; +import org.apache.hadoop.yarn.server.api.records.impl.pb.AppCollectorDataPBImpl; +import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.AppCollectorDataProto; import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.ReportNewCollectorInfoRequestProto; import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.ReportNewCollectorInfoRequestProtoOrBuilder; import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewCollectorInfoRequest; -import org.apache.hadoop.yarn.server.api.records.AppCollectorsMap; -import org.apache.hadoop.yarn.server.api.records.impl.pb.AppCollectorsMapPBImpl; public class ReportNewCollectorInfoRequestPBImpl extends ReportNewCollectorInfoRequest { @@ -36,7 +36,7 @@ public class ReportNewCollectorInfoRequestPBImpl extends private ReportNewCollectorInfoRequestProto.Builder builder = null; private boolean viaProto = false; - private List collectorsList = null; + private List collectorsList = null; public ReportNewCollectorInfoRequestPBImpl() { builder = ReportNewCollectorInfoRequestProto.newBuilder(); @@ -96,9 +96,9 @@ public class ReportNewCollectorInfoRequestPBImpl extends private void addLocalCollectorsToProto() { maybeInitBuilder(); builder.clearAppCollectors(); - List protoList = - new ArrayList(); - for (AppCollectorsMap m : this.collectorsList) { + List protoList = + new ArrayList(); + for (AppCollectorData m : this.collectorsList) { protoList.add(convertToProtoFormat(m)); } builder.addAllAppCollectors(protoList); @@ -106,16 +106,16 @@ public class ReportNewCollectorInfoRequestPBImpl extends private void initLocalCollectorsList() { ReportNewCollectorInfoRequestProtoOrBuilder p = viaProto ? proto : builder; - List list = + List list = p.getAppCollectorsList(); - this.collectorsList = new ArrayList(); - for (AppCollectorsMapProto m : list) { + this.collectorsList = new ArrayList(); + for (AppCollectorDataProto m : list) { this.collectorsList.add(convertFromProtoFormat(m)); } } @Override - public List getAppCollectorsList() { + public List getAppCollectorsList() { if (this.collectorsList == null) { initLocalCollectorsList(); } @@ -123,7 +123,7 @@ public class ReportNewCollectorInfoRequestPBImpl extends } @Override - public void setAppCollectorsList(List appCollectorsList) { + public void setAppCollectorsList(List appCollectorsList) { maybeInitBuilder(); if (appCollectorsList == null) { builder.clearAppCollectors(); @@ -131,14 +131,14 @@ public class ReportNewCollectorInfoRequestPBImpl extends this.collectorsList = appCollectorsList; } - private AppCollectorsMapPBImpl convertFromProtoFormat( - AppCollectorsMapProto p) { - return new AppCollectorsMapPBImpl(p); + private AppCollectorDataPBImpl convertFromProtoFormat( + AppCollectorDataProto p) { + return new AppCollectorDataPBImpl(p); } - private AppCollectorsMapProto convertToProtoFormat( - AppCollectorsMap m) { - return ((AppCollectorsMapPBImpl) m).getProto(); + private AppCollectorDataProto convertToProtoFormat( + AppCollectorData m) { + return ((AppCollectorDataPBImpl) m).getProto(); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/AppCollectorData.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/AppCollectorData.java new file mode 100644 index 00000000000..5266dca1584 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/AppCollectorData.java @@ -0,0 +1,125 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.api.records; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.Token; +import org.apache.hadoop.yarn.util.Records; + + +@Private +@InterfaceStability.Unstable +public abstract class AppCollectorData { + + protected static final long DEFAULT_TIMESTAMP_VALUE = -1; + + public static AppCollectorData newInstance( + ApplicationId id, String collectorAddr, long rmIdentifier, long version, + Token token) { + AppCollectorData appCollectorData = + Records.newRecord(AppCollectorData.class); + appCollectorData.setApplicationId(id); + appCollectorData.setCollectorAddr(collectorAddr); + appCollectorData.setRMIdentifier(rmIdentifier); + appCollectorData.setVersion(version); + appCollectorData.setCollectorToken(token); + return appCollectorData; + } + + public static AppCollectorData newInstance( + ApplicationId id, String collectorAddr, long rmIdentifier, long version) { + return newInstance(id, collectorAddr, rmIdentifier, version, null); + } + + public static AppCollectorData newInstance(ApplicationId id, + String collectorAddr, Token token) { + return newInstance(id, collectorAddr, DEFAULT_TIMESTAMP_VALUE, + DEFAULT_TIMESTAMP_VALUE, token); + } + + public static AppCollectorData newInstance(ApplicationId id, + String collectorAddr) { + return newInstance(id, collectorAddr, null); + } + + /** + * Returns if a collector data item happens before another one. Null data + * items happens before any other non-null items. Non-null data items A + * happens before another non-null item B when A's rmIdentifier is less than + * B's rmIdentifier. Or A's version is less than B's if they have the same + * rmIdentifier. + * + * @param dataA first collector data item. + * @param dataB second collector data item. + * @return true if dataA happens before dataB. + */ + public static boolean happensBefore(AppCollectorData dataA, + AppCollectorData dataB) { + if (dataA == null && dataB == null) { + return false; + } else if (dataA == null || dataB == null) { + return dataA == null; + } + + return + (dataA.getRMIdentifier() < dataB.getRMIdentifier()) + || ((dataA.getRMIdentifier() == dataB.getRMIdentifier()) + && (dataA.getVersion() < dataB.getVersion())); + } + + /** + * Returns if the collector data has been stamped by the RM with a RM cluster + * timestamp and a version number. + * + * @return true if RM has already assigned a timestamp for this collector. + * Otherwise, it means the RM has not recognized the existence of this + * collector. + */ + public boolean isStamped() { + return (getRMIdentifier() != DEFAULT_TIMESTAMP_VALUE) + || (getVersion() != DEFAULT_TIMESTAMP_VALUE); + } + + public abstract ApplicationId getApplicationId(); + + public abstract void setApplicationId(ApplicationId id); + + public abstract String getCollectorAddr(); + + public abstract void setCollectorAddr(String addr); + + public abstract long getRMIdentifier(); + + public abstract void setRMIdentifier(long rmId); + + public abstract long getVersion(); + + public abstract void setVersion(long version); + + /** + * Get delegation token for app collector which AM will use to publish + * entities. + * @return the delegation token for app collector. + */ + public abstract Token getCollectorToken(); + + public abstract void setCollectorToken(Token token); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/AppCollectorsMapPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/AppCollectorDataPBImpl.java similarity index 56% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/AppCollectorsMapPBImpl.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/AppCollectorDataPBImpl.java index 3740035d836..c08e9ca0606 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/AppCollectorsMapPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/AppCollectorDataPBImpl.java @@ -19,39 +19,43 @@ package org.apache.hadoop.yarn.server.api.records.impl.pb; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.server.api.records.AppCollectorData; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl; -import org.apache.hadoop.yarn.server.api.records.AppCollectorsMap; - +import org.apache.hadoop.yarn.api.records.impl.pb.TokenPBImpl; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto; -import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.AppCollectorsMapProto; -import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.AppCollectorsMapProtoOrBuilder; +import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.AppCollectorDataProto; +import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.AppCollectorDataProtoOrBuilder; import com.google.protobuf.TextFormat; @Private @Unstable -public class AppCollectorsMapPBImpl extends AppCollectorsMap { +public class AppCollectorDataPBImpl extends AppCollectorData { - private AppCollectorsMapProto proto = - AppCollectorsMapProto.getDefaultInstance(); + private AppCollectorDataProto proto = + AppCollectorDataProto.getDefaultInstance(); - private AppCollectorsMapProto.Builder builder = null; + private AppCollectorDataProto.Builder builder = null; private boolean viaProto = false; private ApplicationId appId = null; private String collectorAddr = null; + private Long rmIdentifier = null; + private Long version = null; + private Token collectorToken = null; - public AppCollectorsMapPBImpl() { - builder = AppCollectorsMapProto.newBuilder(); + public AppCollectorDataPBImpl() { + builder = AppCollectorDataProto.newBuilder(); } - public AppCollectorsMapPBImpl(AppCollectorsMapProto proto) { + public AppCollectorDataPBImpl(AppCollectorDataProto proto) { this.proto = proto; viaProto = true; } - public AppCollectorsMapProto getProto() { + public AppCollectorDataProto getProto() { mergeLocalToProto(); proto = viaProto ? proto : builder.build(); viaProto = true; @@ -81,7 +85,7 @@ public class AppCollectorsMapPBImpl extends AppCollectorsMap { @Override public ApplicationId getApplicationId() { - AppCollectorsMapProtoOrBuilder p = viaProto ? proto : builder; + AppCollectorDataProtoOrBuilder p = viaProto ? proto : builder; if (this.appId == null && p.hasAppId()) { this.appId = convertFromProtoFormat(p.getAppId()); } @@ -90,7 +94,7 @@ public class AppCollectorsMapPBImpl extends AppCollectorsMap { @Override public String getCollectorAddr() { - AppCollectorsMapProtoOrBuilder p = viaProto ? proto : builder; + AppCollectorDataProtoOrBuilder p = viaProto ? proto : builder; if (this.collectorAddr == null && p.hasAppCollectorAddr()) { this.collectorAddr = p.getAppCollectorAddr(); @@ -116,6 +120,68 @@ public class AppCollectorsMapPBImpl extends AppCollectorsMap { this.collectorAddr = collectorAddr; } + @Override + public long getRMIdentifier() { + AppCollectorDataProtoOrBuilder p = viaProto ? proto : builder; + if (this.rmIdentifier == null && p.hasRmIdentifier()) { + this.rmIdentifier = p.getRmIdentifier(); + } + if (this.rmIdentifier != null) { + return this.rmIdentifier; + } else { + return AppCollectorData.DEFAULT_TIMESTAMP_VALUE; + } + } + + @Override + public void setRMIdentifier(long rmId) { + maybeInitBuilder(); + this.rmIdentifier = rmId; + builder.setRmIdentifier(rmId); + } + + @Override + public long getVersion() { + AppCollectorDataProtoOrBuilder p = viaProto ? proto : builder; + if (this.version == null && p.hasRmIdentifier()) { + this.version = p.getRmIdentifier(); + } + if (this.version != null) { + return this.version; + } else { + return AppCollectorData.DEFAULT_TIMESTAMP_VALUE; + } + } + + @Override + public void setVersion(long version) { + maybeInitBuilder(); + this.version = version; + builder.setVersion(version); + } + + @Override + public Token getCollectorToken() { + AppCollectorDataProtoOrBuilder p = viaProto ? proto : builder; + if (this.collectorToken != null) { + return this.collectorToken; + } + if (!p.hasAppCollectorToken()) { + return null; + } + this.collectorToken = new TokenPBImpl(p.getAppCollectorToken()); + return this.collectorToken; + } + + @Override + public void setCollectorToken(Token token) { + maybeInitBuilder(); + if (token == null) { + builder.clearAppCollectorToken(); + } + this.collectorToken = token; + } + private ApplicationIdPBImpl convertFromProtoFormat(ApplicationIdProto p) { return new ApplicationIdPBImpl(p); } @@ -126,7 +192,7 @@ public class AppCollectorsMapPBImpl extends AppCollectorsMap { private void maybeInitBuilder() { if (viaProto || builder == null) { - builder = AppCollectorsMapProto.newBuilder(proto); + builder = AppCollectorDataProto.newBuilder(proto); } viaProto = false; } @@ -147,6 +213,15 @@ public class AppCollectorsMapPBImpl extends AppCollectorsMap { if (this.collectorAddr != null) { builder.setAppCollectorAddr(this.collectorAddr); } + if (this.rmIdentifier != null) { + builder.setRmIdentifier(this.rmIdentifier); + } + if (this.version != null) { + builder.setVersion(this.version); + } + if (this.collectorToken != null) { + builder.setAppCollectorToken( + ((TokenPBImpl)this.collectorToken).getProto()); + } } - } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/package-info.java new file mode 100644 index 00000000000..4ce3896fbb9 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/package-info.java @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** Server records PB implementations. */ +package org.apache.hadoop.yarn.server.api.records.impl.pb; \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilter.java similarity index 69% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilter.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilter.java index ad8dc2c28dc..f6d1863cfc3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilter.java @@ -23,27 +23,33 @@ import javax.servlet.ServletException; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager; import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationFilter; -import org.apache.hadoop.yarn.server.timeline.security.TimelineDelegationTokenSecretManagerService.TimelineDelegationTokenSecretManager; +import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier; +/** + * Timeline authentication filter provides delegation token support for ATSv1 + * and ATSv2. + */ @Private @Unstable public class TimelineAuthenticationFilter extends DelegationTokenAuthenticationFilter { - private static TimelineDelegationTokenSecretManager secretManager; + private static AbstractDelegationTokenSecretManager + secretManager; @Override public void init(FilterConfig filterConfig) throws ServletException { filterConfig.getServletContext().setAttribute( - DelegationTokenAuthenticationFilter.DELEGATION_TOKEN_SECRET_MANAGER_ATTR, - secretManager); + DelegationTokenAuthenticationFilter. + DELEGATION_TOKEN_SECRET_MANAGER_ATTR, secretManager); super.init(filterConfig); } public static void setTimelineDelegationTokenSecretManager( - TimelineDelegationTokenSecretManager secretManager) { - TimelineAuthenticationFilter.secretManager = secretManager; + AbstractDelegationTokenSecretManager + secretMgr) { + TimelineAuthenticationFilter.secretManager = secretMgr; } - } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java similarity index 78% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java index 4e7c29ad585..3d8ce058952 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java @@ -51,30 +51,19 @@ import java.util.Map; public class TimelineAuthenticationFilterInitializer extends FilterInitializer { /** - * The configuration prefix of timeline HTTP authentication + * The configuration prefix of timeline HTTP authentication. */ - public static final String PREFIX = "yarn.timeline-service.http-authentication."; + public static final String PREFIX = + "yarn.timeline-service.http-authentication."; @VisibleForTesting Map filterConfig; - /** - * Initializes {@link TimelineAuthenticationFilter} - *

- * Propagates to {@link TimelineAuthenticationFilter} configuration all YARN - * configuration properties prefixed with {@value #PREFIX} - * - * @param container - * The filter container - * @param conf - * Configuration for run-time parameters - */ - @Override - public void initFilter(FilterContainer container, Configuration conf) { + protected void setAuthFilterConfig(Configuration conf) { filterConfig = new HashMap(); // setting the cookie path to root '/' so it is used for all resources. - filterConfig.put(TimelineAuthenticationFilter.COOKIE_PATH, "/"); + filterConfig.put(AuthenticationFilter.COOKIE_PATH, "/"); for (Map.Entry entry : conf) { String name = entry.getKey(); @@ -95,6 +84,41 @@ public class TimelineAuthenticationFilterInitializer extends FilterInitializer { } } + // Resolve _HOST into bind address + String bindAddress = conf.get(HttpServer2.BIND_ADDRESS); + String principal = + filterConfig.get(KerberosAuthenticationHandler.PRINCIPAL); + if (principal != null) { + try { + principal = SecurityUtil.getServerPrincipal(principal, bindAddress); + } catch (IOException ex) { + throw new RuntimeException("Could not resolve Kerberos principal " + + "name: " + ex.toString(), ex); + } + filterConfig.put(KerberosAuthenticationHandler.PRINCIPAL, + principal); + } + } + + protected Map getFilterConfig() { + return filterConfig; + } + + /** + * Initializes {@link TimelineAuthenticationFilter}. + *

+ * Propagates to {@link TimelineAuthenticationFilter} configuration all YARN + * configuration properties prefixed with {@value #PREFIX}. + * + * @param container + * The filter container. + * @param conf + * Configuration for run-time parameters. + */ + @Override + public void initFilter(FilterContainer container, Configuration conf) { + setAuthFilterConfig(conf); + String authType = filterConfig.get(AuthenticationFilter.AUTH_TYPE); if (authType.equals(PseudoAuthenticationHandler.TYPE)) { filterConfig.put(AuthenticationFilter.AUTH_TYPE, @@ -102,23 +126,7 @@ public class TimelineAuthenticationFilterInitializer extends FilterInitializer { } else if (authType.equals(KerberosAuthenticationHandler.TYPE)) { filterConfig.put(AuthenticationFilter.AUTH_TYPE, KerberosDelegationTokenAuthenticationHandler.class.getName()); - - // Resolve _HOST into bind address - String bindAddress = conf.get(HttpServer2.BIND_ADDRESS); - String principal = - filterConfig.get(KerberosAuthenticationHandler.PRINCIPAL); - if (principal != null) { - try { - principal = SecurityUtil.getServerPrincipal(principal, bindAddress); - } catch (IOException ex) { - throw new RuntimeException( - "Could not resolve Kerberos principal name: " + ex.toString(), ex); - } - filterConfig.put(KerberosAuthenticationHandler.PRINCIPAL, - principal); - } } - filterConfig.put(DelegationTokenAuthenticationHandler.TOKEN_KIND, TimelineDelegationTokenIdentifier.KIND_NAME.toString()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineDelgationTokenSecretManagerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineDelgationTokenSecretManagerService.java new file mode 100644 index 00000000000..2e95af2cdd0 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineDelgationTokenSecretManagerService.java @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.timeline.security; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager; +import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier; + +/** + * Abstract implementation of delegation token manager service for different + * versions of timeline service. + */ +public abstract class TimelineDelgationTokenSecretManagerService extends + AbstractService { + + public TimelineDelgationTokenSecretManagerService(String name) { + super(name); + } + + private static long delegationTokenRemovalScanInterval = 3600000L; + + private AbstractDelegationTokenSecretManager + secretManager = null; + + @Override + protected void serviceInit(Configuration conf) throws Exception { + long secretKeyInterval = + conf.getLong(YarnConfiguration.TIMELINE_DELEGATION_KEY_UPDATE_INTERVAL, + YarnConfiguration.DEFAULT_TIMELINE_DELEGATION_KEY_UPDATE_INTERVAL); + long tokenMaxLifetime = + conf.getLong(YarnConfiguration.TIMELINE_DELEGATION_TOKEN_MAX_LIFETIME, + YarnConfiguration.DEFAULT_TIMELINE_DELEGATION_TOKEN_MAX_LIFETIME); + long tokenRenewInterval = + conf.getLong(YarnConfiguration.TIMELINE_DELEGATION_TOKEN_RENEW_INTERVAL, + YarnConfiguration.DEFAULT_TIMELINE_DELEGATION_TOKEN_RENEW_INTERVAL); + secretManager = createTimelineDelegationTokenSecretManager( + secretKeyInterval, tokenMaxLifetime, tokenRenewInterval, + delegationTokenRemovalScanInterval); + super.init(conf); + } + + protected abstract + AbstractDelegationTokenSecretManager + createTimelineDelegationTokenSecretManager(long secretKeyInterval, + long tokenMaxLifetime, long tokenRenewInterval, + long tokenRemovalScanInterval); + + @Override + protected void serviceStart() throws Exception { + secretManager.startThreads(); + super.serviceStart(); + } + + @Override + protected void serviceStop() throws Exception { + secretManager.stopThreads(); + super.stop(); + } + + public AbstractDelegationTokenSecretManager + + getTimelineDelegationTokenSecretManager() { + return secretManager; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/package-info.java new file mode 100644 index 00000000000..14a52e342b3 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/package-info.java @@ -0,0 +1,26 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Package org.apache.hadoop.server.timeline.security contains classes related + * to timeline authentication filters and abstract delegation token service for + * ATSv1 and ATSv2. + */ +@InterfaceAudience.Private +package org.apache.hadoop.yarn.server.timeline.security; +import org.apache.hadoop.classification.InterfaceAudience; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/util/timeline/TimelineServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/util/timeline/TimelineServerUtils.java new file mode 100644 index 00000000000..78bf20f8ccd --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/util/timeline/TimelineServerUtils.java @@ -0,0 +1,92 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.util.timeline; + +import java.util.LinkedHashSet; +import java.util.Set; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.AuthenticationFilterInitializer; +import org.apache.hadoop.yarn.server.timeline.security.TimelineAuthenticationFilter; +import org.apache.hadoop.yarn.server.timeline.security.TimelineAuthenticationFilterInitializer; +import org.apache.hadoop.yarn.server.timeline.security.TimelineDelgationTokenSecretManagerService; + +/** + * Set of utility methods to be used across timeline reader and collector. + */ +public final class TimelineServerUtils { + private static final Log LOG = LogFactory.getLog(TimelineServerUtils.class); + + private TimelineServerUtils() { + } + + /** + * Sets filter initializers configuration based on existing configuration and + * default filters added by timeline service(such as timeline auth filter and + * CORS filter). + * @param conf Configuration object. + * @param configuredInitializers Comma separated list of filter initializers. + * @param defaultInitializers Set of initializers added by default by timeline + * service. + */ + public static void setTimelineFilters(Configuration conf, + String configuredInitializers, Set defaultInitializers) { + String[] parts = configuredInitializers.split(","); + Set target = new LinkedHashSet(); + for (String filterInitializer : parts) { + filterInitializer = filterInitializer.trim(); + if (filterInitializer.equals( + AuthenticationFilterInitializer.class.getName()) || + filterInitializer.isEmpty()) { + continue; + } + target.add(filterInitializer); + } + target.addAll(defaultInitializers); + String actualInitializers = + org.apache.commons.lang.StringUtils.join(target, ","); + LOG.info("Filter initializers set for timeline service: " + + actualInitializers); + conf.set("hadoop.http.filter.initializers", actualInitializers); + } + + /** + * Adds timeline authentication filter to the set of default filter + * initializers and assigns the delegation token manager service to it. + * @param initializers Comma separated list of filter initializers. + * @param defaultInitializers Set of initializers added by default by timeline + * service. + * @param delegationTokenMgrService Delegation token manager service. + * This will be used by timeline authentication filter to assign + * delegation tokens. + */ + public static void addTimelineAuthFilter(String initializers, + Set defaultInitializers, + TimelineDelgationTokenSecretManagerService delegationTokenMgrService) { + TimelineAuthenticationFilter.setTimelineDelegationTokenSecretManager( + delegationTokenMgrService.getTimelineDelegationTokenSecretManager()); + if (!initializers.contains( + TimelineAuthenticationFilterInitializer.class.getName())) { + defaultInitializers.add( + TimelineAuthenticationFilterInitializer.class.getName()); + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/util/timeline/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/util/timeline/package-info.java new file mode 100644 index 00000000000..75c69738c50 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/util/timeline/package-info.java @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Package org.apache.hadoop.server.util.timeline contains utility classes used + * by ATSv1 and ATSv2 on the server side. + */ +@InterfaceAudience.Private +package org.apache.hadoop.yarn.server.util.timeline; +import org.apache.hadoop.classification.InterfaceAudience; \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto index 4e05fbad787..c2ba6772265 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto @@ -22,6 +22,7 @@ option java_generic_services = true; option java_generate_equals_and_hash = true; package hadoop.yarn; +import "Security.proto"; import "yarn_protos.proto"; import "yarn_server_common_protos.proto"; import "yarn_service_protos.proto"; @@ -90,7 +91,7 @@ message NodeHeartbeatRequestProto { optional MasterKeyProto last_known_nm_token_master_key = 3; optional NodeLabelsProto nodeLabels = 4; repeated LogAggregationReportProto log_aggregation_reports_for_apps = 5; - repeated AppCollectorsMapProto registered_collectors = 6; + repeated AppCollectorDataProto registering_collectors = 6; } message LogAggregationReportProto { @@ -116,7 +117,7 @@ message NodeHeartbeatResponseProto { repeated SignalContainerRequestProto containers_to_signal = 13; optional ResourceProto resource = 14; optional ContainerQueuingLimitProto container_queuing_limit = 15; - repeated AppCollectorsMapProto app_collectors_map = 16; + repeated AppCollectorDataProto app_collectors = 16; // to be used in place of containers_to_decrease repeated ContainerProto containers_to_update = 17; } @@ -134,16 +135,19 @@ message SystemCredentialsForAppsProto { //////////////////////////////////////////////////////////////////////// ////// From collector_nodemanager_protocol //////////////////////////// //////////////////////////////////////////////////////////////////////// -message AppCollectorsMapProto { - optional ApplicationIdProto appId = 1; - optional string appCollectorAddr = 2; +message AppCollectorDataProto { + optional ApplicationIdProto app_id = 1; + optional string app_collector_addr = 2; + optional int64 rm_identifier = 3 [default = -1]; + optional int64 version = 4 [default = -1]; + optional hadoop.common.TokenProto app_collector_token = 5; } ////////////////////////////////////////////////////// /////// collector_nodemanager_protocol ////////////// ////////////////////////////////////////////////////// message ReportNewCollectorInfoRequestProto { - repeated AppCollectorsMapProto app_collectors = 1; + repeated AppCollectorDataProto app_collectors = 1; } message ReportNewCollectorInfoResponseProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java index 9775f5c15a4..82dfaea32a9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java @@ -30,6 +30,7 @@ import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.ipc.Server; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.util.Time; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.ContainerManagementProtocol; import org.apache.hadoop.yarn.api.ContainerManagementProtocolPB; @@ -72,12 +73,13 @@ import org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC; import org.apache.hadoop.yarn.ipc.RPCUtil; import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; +import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier; import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocol; import org.apache.hadoop.yarn.server.api.protocolrecords.GetTimelineCollectorContextRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.GetTimelineCollectorContextResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewCollectorInfoRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewCollectorInfoResponse; -import org.apache.hadoop.yarn.server.api.records.AppCollectorsMap; +import org.apache.hadoop.yarn.server.api.records.AppCollectorData; import org.apache.hadoop.yarn.util.Records; import org.junit.Assert; import org.junit.Test; @@ -93,6 +95,21 @@ public class TestRPC { "collectors' number in ReportNewCollectorInfoRequest is not ONE."; public static final String DEFAULT_COLLECTOR_ADDR = "localhost:0"; + private static final Token DEFAULT_COLLECTOR_TOKEN; + static { + TimelineDelegationTokenIdentifier identifier = + new TimelineDelegationTokenIdentifier(); + identifier.setOwner(new Text("user")); + identifier.setRenewer(new Text("user")); + identifier.setRealUser(new Text("user")); + long now = Time.now(); + identifier.setIssueDate(now); + identifier.setMaxDate(now + 1000L); + identifier.setMasterKeyId(500); + identifier.setSequenceNumber(5); + DEFAULT_COLLECTOR_TOKEN = Token.newInstance(identifier.getBytes(), + identifier.getKind().toString(), identifier.getBytes(), "localhost:0"); + } public static final ApplicationId DEFAULT_APP_ID = ApplicationId.newInstance(0, 0); @@ -173,7 +190,16 @@ public class TestRPC { try { ReportNewCollectorInfoRequest request = ReportNewCollectorInfoRequest.newInstance( - DEFAULT_APP_ID, DEFAULT_COLLECTOR_ADDR); + DEFAULT_APP_ID, DEFAULT_COLLECTOR_ADDR, null); + proxy.reportNewCollectorInfo(request); + } catch (YarnException e) { + Assert.fail("RPC call failured is not expected here."); + } + + try { + ReportNewCollectorInfoRequest request = + ReportNewCollectorInfoRequest.newInstance( + DEFAULT_APP_ID, DEFAULT_COLLECTOR_ADDR, DEFAULT_COLLECTOR_TOKEN); proxy.reportNewCollectorInfo(request); } catch (YarnException e) { Assert.fail("RPC call failured is not expected here."); @@ -429,14 +455,16 @@ public class TestRPC { public ReportNewCollectorInfoResponse reportNewCollectorInfo( ReportNewCollectorInfoRequest request) throws YarnException, IOException { - List appCollectors = request.getAppCollectorsList(); + List appCollectors = request.getAppCollectorsList(); if (appCollectors.size() == 1) { // check default appID and collectorAddr - AppCollectorsMap appCollector = appCollectors.get(0); + AppCollectorData appCollector = appCollectors.get(0); Assert.assertEquals(appCollector.getApplicationId(), DEFAULT_APP_ID); Assert.assertEquals(appCollector.getCollectorAddr(), DEFAULT_COLLECTOR_ADDR); + Assert.assertTrue(appCollector.getCollectorToken() == null || + appCollector.getCollectorToken().equals(DEFAULT_COLLECTOR_TOKEN)); } else { throw new YarnException(ILLEGAL_NUMBER_MESSAGE); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java index 8c0c73afd80..8b1d0bb49e5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java @@ -37,6 +37,7 @@ import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl; @@ -48,6 +49,7 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.NodeHeartbeatRe import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RegisterNodeManagerRequestPBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RegisterNodeManagerResponsePBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.UnRegisterNodeManagerRequestPBImpl; +import org.apache.hadoop.yarn.server.api.records.AppCollectorData; import org.apache.hadoop.yarn.server.api.records.MasterKey; import org.apache.hadoop.yarn.server.api.records.NodeAction; import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus; @@ -109,14 +111,14 @@ public class TestYarnServerApiClasses { original.setLastKnownNMTokenMasterKey(getMasterKey()); original.setNodeStatus(getNodeStatus()); original.setNodeLabels(getValidNodeLabels()); - Map collectors = getCollectors(); - original.setRegisteredCollectors(collectors); + Map collectors = getCollectors(false); + original.setRegisteringCollectors(collectors); NodeHeartbeatRequestPBImpl copy = new NodeHeartbeatRequestPBImpl( original.getProto()); assertEquals(1, copy.getLastKnownContainerTokenMasterKey().getKeyId()); assertEquals(1, copy.getLastKnownNMTokenMasterKey().getKeyId()); assertEquals("localhost", copy.getNodeStatus().getNodeId().getHost()); - assertEquals(collectors, copy.getRegisteredCollectors()); + assertEquals(collectors, copy.getRegisteringCollectors()); // check labels are coming with valid values Assert.assertTrue(original.getNodeLabels() .containsAll(copy.getNodeLabels())); @@ -128,6 +130,16 @@ public class TestYarnServerApiClasses { Assert.assertEquals(0, copy.getNodeLabels().size()); } + @Test + public void testNodeHBRequestPBImplWithNullCollectorToken() { + NodeHeartbeatRequestPBImpl original = new NodeHeartbeatRequestPBImpl(); + Map collectors = getCollectors(true); + original.setRegisteringCollectors(collectors); + NodeHeartbeatRequestPBImpl copy = new NodeHeartbeatRequestPBImpl( + original.getProto()); + assertEquals(collectors, copy.getRegisteringCollectors()); + } + /** * Test NodeHeartbeatRequestPBImpl. */ @@ -153,8 +165,8 @@ public class TestYarnServerApiClasses { original.setNextHeartBeatInterval(1000); original.setNodeAction(NodeAction.NORMAL); original.setResponseId(100); - Map collectors = getCollectors(); - original.setAppCollectorsMap(collectors); + Map collectors = getCollectors(false); + original.setAppCollectors(collectors); NodeHeartbeatResponsePBImpl copy = new NodeHeartbeatResponsePBImpl( original.getProto()); @@ -164,7 +176,7 @@ public class TestYarnServerApiClasses { assertEquals(1, copy.getContainerTokenMasterKey().getKeyId()); assertEquals(1, copy.getNMTokenMasterKey().getKeyId()); assertEquals("testDiagnosticMessage", copy.getDiagnosticsMessage()); - assertEquals(collectors, copy.getAppCollectorsMap()); + assertEquals(collectors, copy.getAppCollectors()); assertEquals(false, copy.getAreNodeLabelsAcceptedByRM()); } @@ -177,6 +189,16 @@ public class TestYarnServerApiClasses { assertTrue(copy.getAreNodeLabelsAcceptedByRM()); } + @Test + public void testNodeHBResponsePBImplWithNullCollectorToken() { + NodeHeartbeatResponsePBImpl original = new NodeHeartbeatResponsePBImpl(); + Map collectors = getCollectors(true); + original.setAppCollectors(collectors); + NodeHeartbeatResponsePBImpl copy = new NodeHeartbeatResponsePBImpl( + original.getProto()); + assertEquals(collectors, copy.getAppCollectors()); + } + @Test public void testNodeHeartbeatResponsePBImplWithDecreasedContainers() { NodeHeartbeatResponsePBImpl original = new NodeHeartbeatResponsePBImpl(); @@ -347,12 +369,18 @@ public class TestYarnServerApiClasses { return nodeLabels; } - private Map getCollectors() { + private Map getCollectors( + boolean hasNullCollectorToken) { ApplicationId appID = ApplicationId.newInstance(1L, 1); String collectorAddr = "localhost:0"; - Map collectorMap = - new HashMap(); - collectorMap.put(appID, collectorAddr); + AppCollectorData data = AppCollectorData.newInstance(appID, collectorAddr); + if (!hasNullCollectorToken) { + data.setCollectorToken( + Token.newInstance(new byte[0], "kind", new byte[0], "s")); + } + Map collectorMap = + new HashMap<>(); + collectorMap.put(appID, data); return collectorMap; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilterInitializer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilterInitializer.java similarity index 97% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilterInitializer.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilterInitializer.java index 430911eaa34..44f63ead34a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilterInitializer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilterInitializer.java @@ -27,7 +27,9 @@ import static org.apache.hadoop.yarn.server.timeline.security.TimelineAuthentica import org.junit.Test; import org.mockito.Mockito; - +/** + * Tests {@link TimelineAuthenticationFilterInitializer}. + */ public class TestTimelineAuthenticationFilterInitializer { @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java index 16a84973b69..00bd0efcb17 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java @@ -28,6 +28,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.server.api.protocolrecords.LogAggregationReport; +import org.apache.hadoop.yarn.server.api.records.AppCollectorData; import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus; import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManager; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application; @@ -65,11 +66,18 @@ public interface Context { Map getSystemCredentialsForApps(); /** - * Get the registered collectors that located on this NM. - * @return registered collectors, or null if the timeline service v.2 is not + * Get the list of collectors that are registering with the RM from this node. + * @return registering collectors, or null if the timeline service v.2 is not * enabled */ - Map getRegisteredCollectors(); + ConcurrentMap getRegisteringCollectors(); + + /** + * Get the list of collectors registered with the RM and known by this node. + * @return known collectors, or null if the timeline service v.2 is not + * enabled. + */ + ConcurrentMap getKnownCollectors(); ConcurrentMap getContainers(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java index bf4b43cb9a7..3e919c5cdad 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java @@ -57,11 +57,13 @@ import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.api.protocolrecords.LogAggregationReport; +import org.apache.hadoop.yarn.server.api.records.AppCollectorData; import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus; import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManager; import org.apache.hadoop.yarn.server.nodemanager.collectormanager.NMCollectorService; import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics; import org.apache.hadoop.yarn.server.nodemanager.nodelabels.ConfigurationNodeLabelsProvider; @@ -464,8 +466,14 @@ public class NodeManager extends CompositeService if (!rmWorkPreservingRestartEnabled) { LOG.info("Cleaning up running containers on resync"); containerManager.cleanupContainersOnNMResync(); + // Clear all known collectors for resync. + if (context.getKnownCollectors() != null) { + context.getKnownCollectors().clear(); + } } else { LOG.info("Preserving containers on resync"); + // Re-register known timeline collectors. + reregisterCollectors(); } ((NodeStatusUpdaterImpl) nodeStatusUpdater) .rebootNodeStatusUpdaterAndRegisterWithRM(); @@ -477,6 +485,38 @@ public class NodeManager extends CompositeService }.start(); } + /** + * Reregisters all collectors known by this node to the RM. This method is + * called when the RM needs to resync with the node. + */ + protected void reregisterCollectors() { + Map knownCollectors + = context.getKnownCollectors(); + if (knownCollectors == null) { + return; + } + ConcurrentMap registeringCollectors + = context.getRegisteringCollectors(); + for (Map.Entry entry + : knownCollectors.entrySet()) { + Application app = context.getApplications().get(entry.getKey()); + if ((app != null) + && !ApplicationState.FINISHED.equals(app.getApplicationState())) { + registeringCollectors.putIfAbsent(entry.getKey(), entry.getValue()); + AppCollectorData data = entry.getValue(); + if (LOG.isDebugEnabled()) { + LOG.debug(entry.getKey() + " : " + data.getCollectorAddr() + "@<" + + data.getRMIdentifier() + ", " + data.getVersion() + ">"); + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Remove collector data for done app " + entry.getKey()); + } + } + } + knownCollectors.clear(); + } + public static class NMContext implements Context { private NodeId nodeId = null; @@ -492,7 +532,10 @@ public class NodeManager extends CompositeService protected final ConcurrentMap containers = new ConcurrentSkipListMap(); - private Map registeredCollectors; + private ConcurrentMap + registeringCollectors; + + private ConcurrentMap knownCollectors; protected final ConcurrentMap increasedContainers = @@ -526,7 +569,8 @@ public class NodeManager extends CompositeService NMStateStoreService stateStore, boolean isDistSchedulingEnabled, Configuration conf) { if (YarnConfiguration.timelineServiceV2Enabled(conf)) { - this.registeredCollectors = new ConcurrentHashMap<>(); + this.registeringCollectors = new ConcurrentHashMap<>(); + this.knownCollectors = new ConcurrentHashMap<>(); } this.containerTokenSecretManager = containerTokenSecretManager; this.nmTokenSecretManager = nmTokenSecretManager; @@ -681,18 +725,14 @@ public class NodeManager extends CompositeService } @Override - public Map getRegisteredCollectors() { - return this.registeredCollectors; + public ConcurrentMap + getRegisteringCollectors() { + return this.registeringCollectors; } - public void addRegisteredCollectors( - Map newRegisteredCollectors) { - if (registeredCollectors != null) { - this.registeredCollectors.putAll(newRegisteredCollectors); - } else { - LOG.warn("collectors are added when the registered collectors are " + - "initialized"); - } + @Override + public ConcurrentMap getKnownCollectors() { + return this.knownCollectors; } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java index 3b465451319..35b7cb0e5f1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java @@ -71,7 +71,7 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.UnRegisterNodeManagerRequest; - +import org.apache.hadoop.yarn.server.api.records.AppCollectorData; import org.apache.hadoop.yarn.server.api.records.ContainerQueuingLimit; import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus; import org.apache.hadoop.yarn.server.api.records.MasterKey; @@ -760,7 +760,6 @@ public class NodeStatusUpdaterImpl extends AbstractService implements } protected void startStatusUpdater() { - statusUpdaterRunnable = new StatusUpdaterRunnable(); statusUpdater = new Thread(statusUpdaterRunnable, "Node Status Updater"); @@ -1043,7 +1042,7 @@ public class NodeStatusUpdaterImpl extends AbstractService implements .getNMTokenSecretManager().getCurrentKey(), nodeLabelsForHeartbeat, NodeStatusUpdaterImpl.this.context - .getRegisteredCollectors()); + .getRegisteringCollectors()); if (logAggregationEnabled) { // pull log aggregation status for application running in this NM @@ -1134,7 +1133,7 @@ public class NodeStatusUpdaterImpl extends AbstractService implements } } if (YarnConfiguration.timelineServiceV2Enabled(context.getConf())) { - updateTimelineClientsAddress(response); + updateTimelineCollectorData(response); } } catch (ConnectException e) { @@ -1164,40 +1163,48 @@ public class NodeStatusUpdaterImpl extends AbstractService implements } } - private void updateTimelineClientsAddress( + private void updateTimelineCollectorData( NodeHeartbeatResponse response) { - Map knownCollectorsMap = - response.getAppCollectorsMap(); - if (knownCollectorsMap == null) { + Map incomingCollectorsMap = + response.getAppCollectors(); + if (incomingCollectorsMap == null) { if (LOG.isDebugEnabled()) { LOG.debug("No collectors to update RM"); } - } else { - Set> rmKnownCollectors = - knownCollectorsMap.entrySet(); - for (Map.Entry entry : rmKnownCollectors) { - ApplicationId appId = entry.getKey(); - String collectorAddr = entry.getValue(); + return; + } + Map knownCollectors = + context.getKnownCollectors(); + for (Map.Entry entry + : incomingCollectorsMap.entrySet()) { + ApplicationId appId = entry.getKey(); + AppCollectorData collectorData = entry.getValue(); - // Only handle applications running on local node. - // Not include apps with timeline collectors running in local - Application application = context.getApplications().get(appId); - // TODO this logic could be problematic if the collector address - // gets updated due to NM restart or collector service failure - if (application != null && - !context.getRegisteredCollectors().containsKey(appId)) { + // Only handle applications running on local node. + Application application = context.getApplications().get(appId); + if (application != null) { + // Update collector data if the newly received data happens after + // the known data (updates the known data). + AppCollectorData existingData = knownCollectors.get(appId); + if (AppCollectorData.happensBefore(existingData, collectorData)) { if (LOG.isDebugEnabled()) { - LOG.debug("Sync a new collector address: " + collectorAddr + - " for application: " + appId + " from RM."); + LOG.debug("Sync a new collector address: " + + collectorData.getCollectorAddr() + + " for application: " + appId + " from RM."); } + // Update information for clients. NMTimelinePublisher nmTimelinePublisher = context.getNMTimelinePublisher(); if (nmTimelinePublisher != null) { nmTimelinePublisher.setTimelineServiceAddress( - application.getAppId(), collectorAddr); + application.getAppId(), collectorData.getCollectorAddr()); } + // Update information for the node manager itself. + knownCollectors.put(appId, collectorData); } } + // Remove the registering collector data + context.getRegisteringCollectors().remove(entry.getKey()); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java index e52e1ec1a16..4648a655504 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java @@ -26,6 +26,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.ipc.Server; import org.apache.hadoop.service.CompositeService; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -37,10 +38,10 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.GetTimelineCollectorCon import org.apache.hadoop.yarn.server.api.protocolrecords.GetTimelineCollectorContextResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewCollectorInfoRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewCollectorInfoResponse; -import org.apache.hadoop.yarn.server.api.records.AppCollectorsMap; +import org.apache.hadoop.yarn.server.api.records.AppCollectorData; import org.apache.hadoop.yarn.server.nodemanager.Context; -import org.apache.hadoop.yarn.server.nodemanager.NodeManager; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application; +import org.apache.hadoop.yarn.server.nodemanager.security.authorize.NMPolicyProvider; import org.apache.hadoop.yarn.server.nodemanager.timelineservice.NMTimelinePublisher; /** @@ -74,16 +75,21 @@ public class NMCollectorService extends CompositeService implements Configuration serverConf = new Configuration(conf); - // TODO Security settings. YarnRPC rpc = YarnRPC.create(conf); + // Kerberos based authentication to be used for CollectorNodemanager + // protocol if security is enabled. server = rpc.getServer(CollectorNodemanagerProtocol.class, this, - collectorServerAddress, serverConf, - this.context.getNMTokenSecretManager(), + collectorServerAddress, serverConf, null, conf.getInt(YarnConfiguration.NM_COLLECTOR_SERVICE_THREAD_COUNT, YarnConfiguration.DEFAULT_NM_COLLECTOR_SERVICE_THREAD_COUNT)); + if (conf.getBoolean( + CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) { + server.refreshServiceAcl(conf, new NMPolicyProvider()); + } + server.start(); collectorServerAddress = conf.updateConnectAddr( YarnConfiguration.NM_BIND_HOST, @@ -95,7 +101,6 @@ public class NMCollectorService extends CompositeService implements LOG.info("NMCollectorService started at " + collectorServerAddress); } - @Override public void serviceStop() throws Exception { if (server != null) { @@ -108,23 +113,31 @@ public class NMCollectorService extends CompositeService implements @Override public ReportNewCollectorInfoResponse reportNewCollectorInfo( ReportNewCollectorInfoRequest request) throws YarnException, IOException { - List newCollectorsList = request.getAppCollectorsList(); + List newCollectorsList = request.getAppCollectorsList(); if (newCollectorsList != null && !newCollectorsList.isEmpty()) { - Map newCollectorsMap = - new HashMap(); - for (AppCollectorsMap collector : newCollectorsList) { + Map newCollectorsMap = + new HashMap<>(); + for (AppCollectorData collector : newCollectorsList) { ApplicationId appId = collector.getApplicationId(); - String collectorAddr = collector.getCollectorAddr(); - newCollectorsMap.put(appId, collectorAddr); + newCollectorsMap.put(appId, collector); // set registered collector address to TimelineClient. + // TODO: Do we need to do this after we received confirmation from + // the RM? NMTimelinePublisher nmTimelinePublisher = context.getNMTimelinePublisher(); if (nmTimelinePublisher != null) { - nmTimelinePublisher.setTimelineServiceAddress(appId, collectorAddr); + nmTimelinePublisher.setTimelineServiceAddress(appId, + collector.getCollectorAddr()); } } - ((NodeManager.NMContext)context).addRegisteredCollectors( - newCollectorsMap); + Map registeringCollectors + = context.getRegisteringCollectors(); + if (registeringCollectors != null) { + registeringCollectors.putAll(newCollectorsMap); + } else { + LOG.warn("collectors are added when the registered collectors are " + + "initialized"); + } } return ReportNewCollectorInfoResponse.newInstance(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java index 2efc932353e..5e0f2936f83 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java @@ -244,7 +244,8 @@ public class AuxServices extends AbstractService for (AuxiliaryService serv : serviceMap.values()) { try { serv.initializeContainer(new ContainerInitializationContext( - event.getUser(), event.getContainer().getContainerId(), + event.getContainer().getUser(), + event.getContainer().getContainerId(), event.getContainer().getResource(), event.getContainer() .getContainerTokenIdentifier().getContainerType())); } catch (Throwable th) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java index ef36ba64408..e497f62264b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java @@ -155,6 +155,7 @@ import org.apache.hadoop.yarn.server.nodemanager.security.authorize.NMPolicyProv import org.apache.hadoop.yarn.server.nodemanager.timelineservice.NMTimelinePublisher; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.server.utils.YarnServerSecurityUtils; +import org.apache.hadoop.yarn.util.SystemClock; import org.apache.hadoop.yarn.util.resource.Resources; import org.apache.hadoop.yarn.util.timeline.TimelineUtils; @@ -401,6 +402,16 @@ public class ContainerManagerImpl extends CompositeService implements LOG.debug( "Recovering Flow context: " + fc + " for an application " + appId); } + } else { + // in upgrade situations, where there is no prior existing flow context, + // default would be used. + fc = new FlowContext(TimelineUtils.generateDefaultFlowName(null, appId), + YarnConfiguration.DEFAULT_FLOW_VERSION, appId.getClusterTimestamp()); + if (LOG.isDebugEnabled()) { + LOG.debug( + "No prior existing flow context found. Using default Flow context: " + + fc + " for an application " + appId); + } } LOG.info("Recovering application " + appId); @@ -1052,10 +1063,11 @@ public class ContainerManagerImpl extends CompositeService implements Credentials credentials = YarnServerSecurityUtils.parseCredentials(launchContext); + long containerStartTime = SystemClock.getInstance().getTime(); Container container = new ContainerImpl(getConfig(), this.dispatcher, launchContext, credentials, metrics, containerTokenIdentifier, - context); + context, containerStartTime); ApplicationId applicationID = containerId.getApplicationAttemptId().getApplicationId(); if (context.getContainers().putIfAbsent(containerId, container) != null) { @@ -1112,7 +1124,7 @@ public class ContainerManagerImpl extends CompositeService implements } this.context.getNMStateStore().storeContainer(containerId, - containerTokenIdentifier.getVersion(), request); + containerTokenIdentifier.getVersion(), containerStartTime, request); dispatcher.getEventHandler().handle( new ApplicationContainerInitEvent(container)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationContainerFinishedEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationContainerFinishedEvent.java index 0a8ffdff659..09c946b6829 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationContainerFinishedEvent.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationContainerFinishedEvent.java @@ -23,12 +23,16 @@ import org.apache.hadoop.yarn.api.records.ContainerStatus; public class ApplicationContainerFinishedEvent extends ApplicationEvent { private ContainerStatus containerStatus; + // Required by NMTimelinePublisher. + private long containerStartTime; - public ApplicationContainerFinishedEvent(ContainerStatus containerStatus) { + public ApplicationContainerFinishedEvent(ContainerStatus containerStatus, + long containerStartTs) { super(containerStatus.getContainerId().getApplicationAttemptId(). getApplicationId(), ApplicationEventType.APPLICATION_CONTAINER_FINISHED); this.containerStatus = containerStatus; + this.containerStartTime = containerStartTs; } public ContainerId getContainerID() { @@ -39,4 +43,7 @@ public class ApplicationContainerFinishedEvent extends ApplicationEvent { return containerStatus; } + public long getContainerStartTime() { + return containerStartTime; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java index dd200716f87..39be7a790c4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java @@ -45,6 +45,7 @@ import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.proto.YarnProtos; import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.ContainerManagerApplicationProto; import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.FlowContextProto; +import org.apache.hadoop.yarn.server.api.records.AppCollectorData; import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.containermanager.AuxServicesEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.AuxServicesEventType; @@ -558,6 +559,29 @@ public class ApplicationImpl implements Application { @SuppressWarnings("unchecked") static class AppCompletelyDoneTransition implements SingleArcTransition { + + private void updateCollectorStatus(ApplicationImpl app) { + // Remove collectors info for finished apps. + // TODO check we remove related collectors info in failure cases + // (YARN-3038) + Map registeringCollectors + = app.context.getRegisteringCollectors(); + if (registeringCollectors != null) { + registeringCollectors.remove(app.getAppId()); + } + Map knownCollectors = + app.context.getKnownCollectors(); + if (knownCollectors != null) { + knownCollectors.remove(app.getAppId()); + } + // stop timelineClient when application get finished. + NMTimelinePublisher nmTimelinePublisher = + app.context.getNMTimelinePublisher(); + if (nmTimelinePublisher != null) { + nmTimelinePublisher.stopTimelineClient(app.getAppId()); + } + } + @Override public void transition(ApplicationImpl app, ApplicationEvent event) { @@ -566,20 +590,7 @@ public class ApplicationImpl implements Application { new LogHandlerAppFinishedEvent(app.appId)); app.context.getNMTokenSecretManager().appFinished(app.getAppId()); - // Remove collectors info for finished apps. - // TODO check we remove related collectors info in failure cases - // (YARN-3038) - Map registeredCollectors = - app.context.getRegisteredCollectors(); - if (registeredCollectors != null) { - registeredCollectors.remove(app.getAppId()); - } - // stop timelineClient when application get finished. - NMTimelinePublisher nmTimelinePublisher = - app.context.getNMTimelinePublisher(); - if (nmTimelinePublisher != null) { - nmTimelinePublisher.stopTimelineClient(app.getAppId()); - } + updateCollectorStatus(app); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java index f6e567c19ec..ac9fbb7070c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java @@ -37,6 +37,8 @@ public interface Container extends EventHandler { ContainerId getContainerId(); + long getContainerStartTime(); + Resource getResource(); ContainerTokenIdentifier getContainerTokenIdentifier(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java index 6af8653aae1..772b6e7660f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java @@ -173,11 +173,11 @@ public class ContainerImpl implements Container { /** The NM-wide configuration - not specific to this container */ private final Configuration daemonConf; + private final long startTime; private static final Logger LOG = LoggerFactory.getLogger(ContainerImpl.class); - // whether container has been recovered after a restart private RecoveredContainerStatus recoveredStatus = RecoveredContainerStatus.REQUESTED; @@ -190,6 +190,16 @@ public class ContainerImpl implements Container { ContainerLaunchContext launchContext, Credentials creds, NodeManagerMetrics metrics, ContainerTokenIdentifier containerTokenIdentifier, Context context) { + this(conf, dispatcher, launchContext, creds, metrics, + containerTokenIdentifier, context, SystemClock.getInstance().getTime()); + } + + public ContainerImpl(Configuration conf, Dispatcher dispatcher, + ContainerLaunchContext launchContext, Credentials creds, + NodeManagerMetrics metrics, + ContainerTokenIdentifier containerTokenIdentifier, Context context, + long startTs) { + this.startTime = startTs; this.daemonConf = conf; this.dispatcher = dispatcher; this.stateStore = context.getNMStateStore(); @@ -263,7 +273,7 @@ public class ContainerImpl implements Container { ContainerTokenIdentifier containerTokenIdentifier, Context context, RecoveredContainerState rcs) { this(conf, dispatcher, launchContext, creds, metrics, - containerTokenIdentifier, context); + containerTokenIdentifier, context, rcs.getStartTime()); this.recoveredStatus = rcs.getStatus(); this.exitCode = rcs.getExitCode(); this.recoveredAsKilled = rcs.getKilled(); @@ -630,6 +640,11 @@ public class ContainerImpl implements Container { return this.containerId; } + @Override + public long getContainerStartTime() { + return this.startTime; + } + @Override public Resource getResource() { return Resources.clone( @@ -694,7 +709,8 @@ public class ContainerImpl implements Container { EventHandler eventHandler = dispatcher.getEventHandler(); ContainerStatus containerStatus = cloneAndGetContainerStatus(); - eventHandler.handle(new ApplicationContainerFinishedEvent(containerStatus)); + eventHandler.handle( + new ApplicationContainerFinishedEvent(containerStatus, startTime)); // Tell the scheduler the container is Done eventHandler.handle(new ContainerSchedulerEvent(this, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java index a2ee38d4a23..a31756e1c95 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java @@ -112,6 +112,7 @@ public class NMLeveldbStateStoreService extends NMStateStoreService { "ContainerManager/containers/"; private static final String CONTAINER_REQUEST_KEY_SUFFIX = "/request"; private static final String CONTAINER_VERSION_KEY_SUFFIX = "/version"; + private static final String CONTAINER_START_TIME_KEY_SUFFIX = "/starttime"; private static final String CONTAINER_DIAGS_KEY_SUFFIX = "/diagnostics"; private static final String CONTAINER_LAUNCHED_KEY_SUFFIX = "/launched"; private static final String CONTAINER_QUEUED_KEY_SUFFIX = "/queued"; @@ -257,6 +258,8 @@ public class NMLeveldbStateStoreService extends NMStateStoreService { StartContainerRequestProto.parseFrom(entry.getValue())); } else if (suffix.equals(CONTAINER_VERSION_KEY_SUFFIX)) { rcs.version = Integer.parseInt(asString(entry.getValue())); + } else if (suffix.equals(CONTAINER_START_TIME_KEY_SUFFIX)) { + rcs.setStartTime(Long.parseLong(asString(entry.getValue()))); } else if (suffix.equals(CONTAINER_DIAGS_KEY_SUFFIX)) { rcs.diagnostics = asString(entry.getValue()); } else if (suffix.equals(CONTAINER_QUEUED_KEY_SUFFIX)) { @@ -296,21 +299,23 @@ public class NMLeveldbStateStoreService extends NMStateStoreService { @Override public void storeContainer(ContainerId containerId, int containerVersion, - StartContainerRequest startRequest) throws IOException { + long startTime, StartContainerRequest startRequest) throws IOException { String idStr = containerId.toString(); if (LOG.isDebugEnabled()) { LOG.debug("storeContainer: containerId= " + idStr + ", startRequest= " + startRequest); } - String keyRequest = CONTAINERS_KEY_PREFIX + idStr - + CONTAINER_REQUEST_KEY_SUFFIX; + String keyRequest = getContainerKey(idStr, CONTAINER_REQUEST_KEY_SUFFIX); String keyVersion = getContainerVersionKey(idStr); + String keyStartTime = + getContainerKey(idStr, CONTAINER_START_TIME_KEY_SUFFIX); try { WriteBatch batch = db.createWriteBatch(); try { batch.put(bytes(keyRequest), - ((StartContainerRequestPBImpl) startRequest) - .getProto().toByteArray()); + ((StartContainerRequestPBImpl) startRequest).getProto(). + toByteArray()); + batch.put(bytes(keyStartTime), bytes(Long.toString(startTime))); if (containerVersion != 0) { batch.put(bytes(keyVersion), bytes(Integer.toString(containerVersion))); @@ -326,7 +331,11 @@ public class NMLeveldbStateStoreService extends NMStateStoreService { @VisibleForTesting String getContainerVersionKey(String containerId) { - return CONTAINERS_KEY_PREFIX + containerId + CONTAINER_VERSION_KEY_SUFFIX; + return getContainerKey(containerId, CONTAINER_VERSION_KEY_SUFFIX); + } + + private String getContainerKey(String containerId, String suffix) { + return CONTAINERS_KEY_PREFIX + containerId + suffix; } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMNullStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMNullStateStoreService.java index 96c3f9e5c6f..86dc99fdeaa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMNullStateStoreService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMNullStateStoreService.java @@ -71,7 +71,7 @@ public class NMNullStateStoreService extends NMStateStoreService { @Override public void storeContainer(ContainerId containerId, int version, - StartContainerRequest startRequest) throws IOException { + long startTime, StartContainerRequest startRequest) throws IOException { } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMStateStoreService.java index 9f87279fb25..ec534bff7ec 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMStateStoreService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMStateStoreService.java @@ -87,6 +87,7 @@ public abstract class NMStateStoreService extends AbstractService { int version; private RecoveredContainerType recoveryType = RecoveredContainerType.RECOVER; + private long startTime; public RecoveredContainerStatus getStatus() { return status; @@ -108,6 +109,14 @@ public abstract class NMStateStoreService extends AbstractService { return version; } + public long getStartTime() { + return startTime; + } + + public void setStartTime(long ts) { + startTime = ts; + } + public StartContainerRequest getStartRequest() { return startRequest; } @@ -145,6 +154,7 @@ public abstract class NMStateStoreService extends AbstractService { return new StringBuffer("Status: ").append(getStatus()) .append(", Exit code: ").append(exitCode) .append(", Version: ").append(version) + .append(", Start Time: ").append(startTime) .append(", Killed: ").append(getKilled()) .append(", Diagnostics: ").append(getDiagnostics()) .append(", Capability: ").append(getCapability()) @@ -365,11 +375,12 @@ public abstract class NMStateStoreService extends AbstractService { * Record a container start request * @param containerId the container ID * @param containerVersion the container Version + * @param startTime container start time * @param startRequest the container start request * @throws IOException */ public abstract void storeContainer(ContainerId containerId, - int containerVersion, StartContainerRequest startRequest) + int containerVersion, long startTime, StartContainerRequest startRequest) throws IOException; /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/authorize/NMPolicyProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/authorize/NMPolicyProvider.java index 89e3d78935a..7b28659131f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/authorize/NMPolicyProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/authorize/NMPolicyProvider.java @@ -23,6 +23,7 @@ import org.apache.hadoop.security.authorize.PolicyProvider; import org.apache.hadoop.security.authorize.Service; import org.apache.hadoop.yarn.api.ContainerManagementProtocolPB; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocolPB; import org.apache.hadoop.yarn.server.nodemanager.api.LocalizationProtocolPB; /** @@ -32,18 +33,21 @@ import org.apache.hadoop.yarn.server.nodemanager.api.LocalizationProtocolPB; @InterfaceStability.Unstable public class NMPolicyProvider extends PolicyProvider { - private static final Service[] nodeManagerServices = + private static final Service[] NODE_MANAGER_SERVICES = new Service[] { - new Service( - YarnConfiguration.YARN_SECURITY_SERVICE_AUTHORIZATION_CONTAINER_MANAGEMENT_PROTOCOL, - ContainerManagementProtocolPB.class), - new Service(YarnConfiguration.YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCE_LOCALIZER, - LocalizationProtocolPB.class) - }; + new Service(YarnConfiguration. + YARN_SECURITY_SERVICE_AUTHORIZATION_CONTAINER_MANAGEMENT_PROTOCOL, + ContainerManagementProtocolPB.class), + new Service(YarnConfiguration. + YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCE_LOCALIZER, + LocalizationProtocolPB.class), + new Service(YarnConfiguration. + YARN_SECURITY_SERVICE_AUTHORIZATION_COLLECTOR_NODEMANAGER_PROTOCOL, + CollectorNodemanagerProtocolPB.class) + }; @Override public Service[] getServices() { - return nodeManagerServices; + return NODE_MANAGER_SERVICES; } - } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java index a45995812bc..b8192ca9232 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.nodemanager.timelineservice; import java.io.IOException; +import java.security.PrivilegedExceptionAction; import java.util.HashMap; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -26,8 +27,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.service.CompositeService; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.CollectorInfo; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.ContainerStatus; @@ -55,6 +58,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.even import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizationEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorImpl.ContainerMetric; import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree; +import org.apache.hadoop.yarn.util.TimelineServiceHelper; import org.apache.hadoop.yarn.util.timeline.TimelineUtils; import com.google.common.annotations.VisibleForTesting; @@ -77,6 +81,8 @@ public class NMTimelinePublisher extends CompositeService { private String httpAddress; + private UserGroupInformation nmLoginUGI; + private final Map appToClientMap; public NMTimelinePublisher(Context context) { @@ -91,6 +97,9 @@ public class NMTimelinePublisher extends CompositeService { dispatcher.register(NMTimelineEventType.class, new ForwardingEventHandler()); addIfService(dispatcher); + this.nmLoginUGI = UserGroupInformation.isSecurityEnabled() ? + UserGroupInformation.getLoginUser() : + UserGroupInformation.getCurrentUser(); super.serviceInit(conf); } @@ -149,6 +158,8 @@ public class NMTimelinePublisher extends CompositeService { Math.round(cpuUsagePercentPerCore)); entity.addMetric(cpuMetric); } + entity.setIdPrefix(TimelineServiceHelper. + invertLong(container.getContainerStartTime())); ApplicationId appId = container.getContainerId().getApplicationAttemptId() .getApplicationId(); try { @@ -195,15 +206,17 @@ public class NMTimelinePublisher extends CompositeService { tEvent.setId(ContainerMetricsConstants.CREATED_EVENT_TYPE); tEvent.setTimestamp(event.getTimestamp()); + long containerStartTime = container.getContainerStartTime(); entity.addEvent(tEvent); - entity.setCreatedTime(event.getTimestamp()); + entity.setCreatedTime(containerStartTime); + entity.setIdPrefix(TimelineServiceHelper.invertLong(containerStartTime)); dispatcher.getEventHandler().handle(new TimelinePublishEvent(entity, containerId.getApplicationAttemptId().getApplicationId())); } @SuppressWarnings("unchecked") private void publishContainerFinishedEvent(ContainerStatus containerStatus, - long timeStamp) { + long containerFinishTime, long containerStartTime) { ContainerId containerId = containerStatus.getContainerId(); TimelineEntity entity = createContainerEntity(containerId); @@ -215,13 +228,14 @@ public class NMTimelinePublisher extends CompositeService { entityInfo.put(ContainerMetricsConstants.STATE_INFO, ContainerState.COMPLETE.toString()); entityInfo.put(ContainerMetricsConstants.CONTAINER_FINISHED_TIME, - timeStamp); + containerFinishTime); entity.setInfo(entityInfo); TimelineEvent tEvent = new TimelineEvent(); tEvent.setId(ContainerMetricsConstants.FINISHED_EVENT_TYPE); - tEvent.setTimestamp(timeStamp); + tEvent.setTimestamp(containerFinishTime); entity.addEvent(tEvent); + entity.setIdPrefix(TimelineServiceHelper.invertLong(containerStartTime)); dispatcher.getEventHandler().handle(new TimelinePublishEvent(entity, containerId.getApplicationAttemptId().getApplicationId())); @@ -237,6 +251,8 @@ public class NMTimelinePublisher extends CompositeService { tEvent.setId(eventType); tEvent.setTimestamp(event.getTimestamp()); entity.addEvent(tEvent); + entity.setIdPrefix(TimelineServiceHelper. + invertLong(container.getContainerStartTime())); ApplicationId appId = container.getContainerId().getApplicationAttemptId().getApplicationId(); @@ -300,7 +316,7 @@ public class NMTimelinePublisher extends CompositeService { ApplicationContainerFinishedEvent evnt = (ApplicationContainerFinishedEvent) event; publishContainerFinishedEvent(evnt.getContainerStatus(), - event.getTimestamp()); + event.getTimestamp(), evnt.getContainerStartTime()); break; default: @@ -391,11 +407,23 @@ public class NMTimelinePublisher extends CompositeService { public void createTimelineClient(ApplicationId appId) { if (!appToClientMap.containsKey(appId)) { - TimelineV2Client timelineClient = - TimelineV2Client.createTimelineClient(appId); - timelineClient.init(getConfig()); - timelineClient.start(); - appToClientMap.put(appId, timelineClient); + try { + TimelineV2Client timelineClient = + nmLoginUGI.doAs(new PrivilegedExceptionAction() { + @Override + public TimelineV2Client run() throws Exception { + TimelineV2Client timelineClient = + TimelineV2Client.createTimelineClient(appId); + timelineClient.init(getConfig()); + timelineClient.start(); + return timelineClient; + } + }); + appToClientMap.put(appId, timelineClient); + } catch (IOException | InterruptedException | RuntimeException | + Error e) { + LOG.warn("Unable to create timeline client for app " + appId, e); + } } } @@ -410,11 +438,11 @@ public class NMTimelinePublisher extends CompositeService { String collectorAddr) { TimelineV2Client client = appToClientMap.get(appId); if (client != null) { - client.setTimelineServiceAddress(collectorAddr); + client.setTimelineCollectorInfo(CollectorInfo.newInstance(collectorAddr)); } } private TimelineV2Client getTimelineClient(ApplicationId appId) { return appToClientMap.get(appId); } -} \ No newline at end of file +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java index 26e40859a57..7c8551e75b0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java @@ -59,6 +59,7 @@ import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; import org.apache.hadoop.yarn.server.api.protocolrecords.LogAggregationReport; +import org.apache.hadoop.yarn.server.api.records.AppCollectorData; import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus; import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor; import org.apache.hadoop.yarn.server.nodemanager.Context; @@ -659,7 +660,13 @@ public abstract class BaseAMRMProxyTest { } @Override - public Map getRegisteredCollectors() { + public ConcurrentMap + getRegisteringCollectors() { + return null; + } + + @Override + public ConcurrentMap getKnownCollectors() { return null; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java index 05ea03641eb..65558e937cb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java @@ -601,7 +601,7 @@ public class TestApplication { public void containerFinished(int containerNum) { app.handle(new ApplicationContainerFinishedEvent(containers.get( - containerNum).cloneAndGetContainerStatus())); + containerNum).cloneAndGetContainerStatus(), 0)); drainDispatcherEvents(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMMemoryStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMMemoryStateStoreService.java index 0e039947c49..c1638df7b5a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMMemoryStateStoreService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMMemoryStateStoreService.java @@ -126,10 +126,12 @@ public class NMMemoryStateStoreService extends NMStateStoreService { @Override public synchronized void storeContainer(ContainerId containerId, - int version, StartContainerRequest startRequest) throws IOException { + int version, long startTime, StartContainerRequest startRequest) + throws IOException { RecoveredContainerState rcs = new RecoveredContainerState(); rcs.startRequest = startRequest; rcs.version = version; + rcs.setStartTime(startTime); containerStates.put(containerId, rcs); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java index 01331560e6c..b0a9bc92b1e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java @@ -234,7 +234,8 @@ public class TestNMLeveldbStateStoreService { StartContainerRequest containerReq = createContainerRequest(containerId); // store a container and verify recovered - stateStore.storeContainer(containerId, 0, containerReq); + long containerStartTime = System.currentTimeMillis(); + stateStore.storeContainer(containerId, 0, containerStartTime, containerReq); // verify the container version key is not stored for new containers DB db = stateStore.getDB(); @@ -246,6 +247,7 @@ public class TestNMLeveldbStateStoreService { assertEquals(1, recoveredContainers.size()); RecoveredContainerState rcs = recoveredContainers.get(0); assertEquals(0, rcs.getVersion()); + assertEquals(containerStartTime, rcs.getStartTime()); assertEquals(RecoveredContainerStatus.REQUESTED, rcs.getStatus()); assertEquals(ContainerExitStatus.INVALID, rcs.getExitCode()); assertEquals(false, rcs.getKilled()); @@ -998,7 +1000,7 @@ public class TestNMLeveldbStateStoreService { StartContainerRequest containerReq = StartContainerRequest.newInstance(clc, containerToken); - stateStore.storeContainer(containerId, 0, containerReq); + stateStore.storeContainer(containerId, 0, 0, containerReq); // add a invalid key byte[] invalidKey = ("ContainerManager/containers/" diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java index 4561e85c87e..57bee8c665e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java @@ -235,4 +235,8 @@ public class MockContainer implements Container { public boolean isRecovering() { return false; } + + public long getContainerStartTime() { + return 0; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java index be1dae183ee..0a71a9179bb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java @@ -218,8 +218,8 @@ public class TestNMWebServer { Context context = mock(Context.class); Container container = new ContainerImpl(conf, dispatcher, launchContext, - null, metrics, - BuilderUtils.newContainerTokenIdentifier(containerToken), context) { + null, metrics, BuilderUtils.newContainerTokenIdentifier( + containerToken), context) { @Override public ContainerState getContainerState() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java index 0b13887cfc1..479aa43df08 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java @@ -45,8 +45,6 @@ import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterResponse; import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest; import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; - - import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Container; @@ -61,6 +59,7 @@ import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AMLivelinessMonitor; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl; @@ -254,7 +253,7 @@ public class ApplicationMasterService extends AbstractService implements // Remove collector address when app get finished. if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) { - rmApp.removeCollectorAddr(); + ((RMAppImpl) rmApp).removeCollectorData(); } // checking whether the app exits in RMStateStore at first not to throw // ApplicationDoesNotExistInCacheException before and after diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java index 052ec22c2f0..a993d694e28 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java @@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterReque import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.records.CollectorInfo; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerUpdateType; @@ -293,9 +294,10 @@ final class DefaultAMSProcessor implements ApplicationMasterServiceProcessor { // add collector address for this application if (YarnConfiguration.timelineServiceV2Enabled( getRmContext().getYarnConfiguration())) { - response.setCollectorAddr( - getRmContext().getRMApps().get(appAttemptId.getApplicationId()) - .getCollectorAddr()); + CollectorInfo collectorInfo = app.getCollectorInfo(); + if (collectorInfo != null) { + response.setCollectorInfo(collectorInfo); + } } // add preemption to the allocateResponse message (if any) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java index e6f2bb2436d..cc47e02cb19 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java @@ -27,6 +27,7 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock; import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock; @@ -63,12 +64,14 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequ import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.UnRegisterNodeManagerRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.UnRegisterNodeManagerResponse; +import org.apache.hadoop.yarn.server.api.records.AppCollectorData; import org.apache.hadoop.yarn.server.api.records.MasterKey; import org.apache.hadoop.yarn.server.api.records.NodeAction; import org.apache.hadoop.yarn.server.api.records.NodeStatus; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NodeLabelsUtils; import org.apache.hadoop.yarn.server.resourcemanager.resource.DynamicResourceConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptContainerFinishedEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; @@ -118,6 +121,8 @@ public class ResourceTrackerService extends AbstractService implements private boolean isDelegatedCentralizedNodeLabelsConf; private DynamicResourceConfiguration drConf; + private final AtomicLong timelineCollectorVersion = new AtomicLong(0); + public ResourceTrackerService(RMContext rmContext, NodesListManager nodesListManager, NMLivelinessMonitor nmLivelinessMonitor, @@ -521,16 +526,6 @@ public class ResourceTrackerService extends AbstractService implements message); } - boolean timelineV2Enabled = - YarnConfiguration.timelineServiceV2Enabled(getConfig()); - if (timelineV2Enabled) { - // Check & update collectors info from request. - // TODO make sure it won't have race condition issue for AM failed over - // case that the older registration could possible override the newer - // one. - updateAppCollectorsMap(request); - } - // Evaluate whether a DECOMMISSIONING node is ready to be DECOMMISSIONED. if (rmNode.getState() == NodeState.DECOMMISSIONING && decommissioningWatcher.checkReadyToBeDecommissioned( @@ -545,6 +540,13 @@ public class ResourceTrackerService extends AbstractService implements NodeAction.SHUTDOWN, message); } + boolean timelineV2Enabled = + YarnConfiguration.timelineServiceV2Enabled(getConfig()); + if (timelineV2Enabled) { + // Check & update collectors info from request. + updateAppCollectorsMap(request); + } + // Heartbeat response NodeHeartbeatResponse nodeHeartBeatResponse = YarnServerBuilderUtils .newNodeHeartbeatResponse(lastNodeHeartbeatResponse. @@ -613,44 +615,66 @@ public class ResourceTrackerService extends AbstractService implements private void setAppCollectorsMapToResponse( List runningApps, NodeHeartbeatResponse response) { - Map liveAppCollectorsMap = new - HashMap(); + Map liveAppCollectorsMap = new + HashMap<>(); Map rmApps = rmContext.getRMApps(); // Set collectors for all running apps on this node. for (ApplicationId appId : runningApps) { - String appCollectorAddr = rmApps.get(appId).getCollectorAddr(); - if (appCollectorAddr != null) { - liveAppCollectorsMap.put(appId, appCollectorAddr); - } else { - if (LOG.isDebugEnabled()) { - LOG.debug("Collector for applicaton: " + appId + - " hasn't registered yet!"); + RMApp app = rmApps.get(appId); + if (app != null) { + AppCollectorData appCollectorData = rmApps.get(appId) + .getCollectorData(); + if (appCollectorData != null) { + liveAppCollectorsMap.put(appId, appCollectorData); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Collector for applicaton: " + appId + + " hasn't registered yet!"); + } } } } - response.setAppCollectorsMap(liveAppCollectorsMap); + response.setAppCollectors(liveAppCollectorsMap); } private void updateAppCollectorsMap(NodeHeartbeatRequest request) { - Map registeredCollectorsMap = - request.getRegisteredCollectors(); - if (registeredCollectorsMap != null - && !registeredCollectorsMap.isEmpty()) { + Map registeringCollectorsMap = + request.getRegisteringCollectors(); + if (registeringCollectorsMap != null + && !registeringCollectorsMap.isEmpty()) { Map rmApps = rmContext.getRMApps(); - for (Map.Entry entry: - registeredCollectorsMap.entrySet()) { + for (Map.Entry entry: + registeringCollectorsMap.entrySet()) { ApplicationId appId = entry.getKey(); - String collectorAddr = entry.getValue(); - if (collectorAddr != null && !collectorAddr.isEmpty()) { + AppCollectorData collectorData = entry.getValue(); + if (collectorData != null) { + if (!collectorData.isStamped()) { + // Stamp the collector if we have not done so + collectorData.setRMIdentifier( + ResourceManager.getClusterTimeStamp()); + collectorData.setVersion( + timelineCollectorVersion.getAndIncrement()); + } RMApp rmApp = rmApps.get(appId); if (rmApp == null) { LOG.warn("Cannot update collector info because application ID: " + appId + " is not found in RMContext!"); } else { - String previousCollectorAddr = rmApp.getCollectorAddr(); - if (previousCollectorAddr == null - || !previousCollectorAddr.equals(collectorAddr)) { - rmApp.setCollectorAddr(collectorAddr); + synchronized (rmApp) { + AppCollectorData previousCollectorData = rmApp.getCollectorData(); + if (AppCollectorData.happensBefore(previousCollectorData, + collectorData)) { + // Sending collector update event. + // Note: RM has to store the newly received collector data + // synchronously. Otherwise, the RM may send out stale collector + // data before this update is done, and the RM then crashes, the + // newly updated collector data will get lost. + LOG.info("Update collector information for application " + appId + + " with new address: " + collectorData.getCollectorAddr() + + " timestamp: " + collectorData.getRMIdentifier() + + ", " + collectorData.getVersion()); + ((RMAppImpl) rmApp).setCollectorData(collectorData); + } } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java index a3a2ebc992e..7eaa6e7feb5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java @@ -58,6 +58,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptS import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.timelineservice.RMTimelineCollectorManager; import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollector; +import org.apache.hadoop.yarn.util.TimelineServiceHelper; import org.apache.hadoop.yarn.util.timeline.TimelineUtils; import com.google.common.annotations.VisibleForTesting; @@ -294,8 +295,8 @@ public class TimelineServiceV2Publisher extends AbstractSystemMetricsPublisher { @Override public void appAttemptRegistered(RMAppAttempt appAttempt, long registeredTime) { - TimelineEntity entity = - createAppAttemptEntity(appAttempt.getAppAttemptId()); + ApplicationAttemptId attemptId = appAttempt.getAppAttemptId(); + TimelineEntity entity = createAppAttemptEntity(attemptId); entity.setCreatedTime(registeredTime); TimelineEvent tEvent = new TimelineEvent(); @@ -317,6 +318,8 @@ public class TimelineServiceV2Publisher extends AbstractSystemMetricsPublisher { appAttempt.getMasterContainer().getId().toString()); } entity.setInfo(entityInfo); + entity.setIdPrefix( + TimelineServiceHelper.invertLong(attemptId.getAttemptId())); getDispatcher().getEventHandler().handle( new TimelineV2PublishEvent(SystemMetricsEventType.PUBLISH_ENTITY, @@ -327,7 +330,7 @@ public class TimelineServiceV2Publisher extends AbstractSystemMetricsPublisher { @Override public void appAttemptFinished(RMAppAttempt appAttempt, RMAppAttemptState appAttemtpState, RMApp app, long finishedTime) { - + ApplicationAttemptId attemptId = appAttempt.getAppAttemptId(); ApplicationAttemptEntity entity = createAppAttemptEntity(appAttempt.getAppAttemptId()); @@ -346,7 +349,8 @@ public class TimelineServiceV2Publisher extends AbstractSystemMetricsPublisher { entityInfo.put(AppAttemptMetricsConstants.STATE_INFO, RMServerUtils .createApplicationAttemptState(appAttemtpState).toString()); entity.setInfo(entityInfo); - + entity.setIdPrefix( + TimelineServiceHelper.invertLong(attemptId.getAttemptId())); getDispatcher().getEventHandler().handle( new TimelineV2PublishEvent(SystemMetricsEventType.PUBLISH_ENTITY, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java index 43fd1fbfbdf..93c41b6747c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java @@ -23,6 +23,8 @@ import java.util.List; import java.util.Map; import java.util.Set; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.ipc.CallerContext; import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; @@ -30,6 +32,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType; +import org.apache.hadoop.yarn.api.records.CollectorInfo; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.LogAggregationStatus; import org.apache.hadoop.yarn.api.records.NodeId; @@ -39,6 +42,7 @@ import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.server.api.protocolrecords.LogAggregationReport; +import org.apache.hadoop.yarn.server.api.records.AppCollectorData; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; @@ -180,28 +184,27 @@ public interface RMApp extends EventHandler { String getTrackingUrl(); /** - * The collector address for the application. It should be used only if the - * timeline service v.2 is enabled. + * The timeline collector information for the application. It should be used + * only if the timeline service v.2 is enabled. * - * @return the address for the application's collector, or null if the - * timeline service v.2 is not enabled. + * @return the data for the application's collector, including collector + * address, RM ID, version and collector token. Return null if the timeline + * service v.2 is not enabled. */ - String getCollectorAddr(); + @InterfaceAudience.Private + @InterfaceStability.Unstable + AppCollectorData getCollectorData(); /** - * Set collector address for the application. It should be used only if the - * timeline service v.2 is enabled. + * The timeline collector information to be sent to AM. It should be used + * only if the timeline service v.2 is enabled. * - * @param collectorAddr the address of collector + * @return collector info, including collector address and collector token. + * Return null if the timeline service v.2 is not enabled. */ - void setCollectorAddr(String collectorAddr); - - /** - * Remove collector address when application is finished or killed. It should - * be used only if the timeline service v.2 is enabled. - */ - void removeCollectorAddr(); - + @InterfaceAudience.Private + @InterfaceStability.Unstable + CollectorInfo getCollectorInfo(); /** * The original tracking url for the application master. * @return the original tracking url for the application master. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java index 04d2db522f6..514efd44dd6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java @@ -30,9 +30,6 @@ public enum RMAppEventType { // Source: Scheduler APP_ACCEPTED, - // TODO add source later - COLLECTOR_UPDATE, - // Source: RMAppAttempt ATTEMPT_REGISTERED, ATTEMPT_UNREGISTERED, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index af7cec4377f..13b079286fc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -55,6 +55,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ApplicationTimeout; import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType; +import org.apache.hadoop.yarn.api.records.CollectorInfo; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.LogAggregationStatus; import org.apache.hadoop.yarn.api.records.NodeId; @@ -72,6 +73,7 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier; import org.apache.hadoop.yarn.server.api.protocolrecords.LogAggregationReport; +import org.apache.hadoop.yarn.server.api.records.AppCollectorData; import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService; import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEvent; import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEventType; @@ -165,7 +167,8 @@ public class RMAppImpl implements RMApp, Recoverable { private long storedFinishTime = 0; private int firstAttemptIdInStateStore = 1; private int nextAttemptId = 1; - private volatile String collectorAddr; + private AppCollectorData collectorData; + private CollectorInfo collectorInfo; // This field isn't protected by readlock now. private volatile RMAppAttempt currentAttempt; private String queue; @@ -529,7 +532,7 @@ public class RMAppImpl implements RMApp, Recoverable { */ public void startTimelineCollector() { AppLevelTimelineCollector collector = - new AppLevelTimelineCollector(applicationId); + new AppLevelTimelineCollector(applicationId, user); rmContext.getRMTimelineCollectorManager().putIfAbsent( applicationId, collector); } @@ -611,18 +614,22 @@ public class RMAppImpl implements RMApp, Recoverable { } @Override - public String getCollectorAddr() { - return this.collectorAddr; + public AppCollectorData getCollectorData() { + return this.collectorData; } - @Override - public void setCollectorAddr(String collectorAddress) { - this.collectorAddr = collectorAddress; + public void setCollectorData(AppCollectorData incomingData) { + this.collectorData = incomingData; + this.collectorInfo = CollectorInfo.newInstance( + incomingData.getCollectorAddr(), incomingData.getCollectorToken()); } - @Override - public void removeCollectorAddr() { - this.collectorAddr = null; + public CollectorInfo getCollectorInfo() { + return this.collectorInfo; + } + + public void removeCollectorData() { + this.collectorData = null; } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java index 2d7612738ca..4a8ff00099b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java @@ -24,6 +24,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -39,6 +40,7 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerResponse; +import org.apache.hadoop.yarn.server.api.records.AppCollectorData; import org.apache.hadoop.yarn.server.api.records.MasterKey; import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus; import org.apache.hadoop.yarn.server.api.records.NodeStatus; @@ -60,6 +62,8 @@ public class MockNM { private String version; private Map containerStats = new HashMap(); + private Map registeringCollectors + = new ConcurrentHashMap<>(); public MockNM(String nodeIdStr, int memory, ResourceTrackerService resourceTracker) { // scale vcores based on the requested memory @@ -117,6 +121,15 @@ public class MockNM { true, ++responseId); } + public void addRegisteringCollector(ApplicationId appId, + AppCollectorData data) { + this.registeringCollectors.put(appId, data); + } + + public Map getRegisteringCollectors() { + return this.registeringCollectors; + } + public RegisterNodeManagerResponse registerNode() throws Exception { return registerNode(null, null); } @@ -229,6 +242,9 @@ public class MockNM { req.setNodeStatus(status); req.setLastKnownContainerTokenMasterKey(this.currentContainerTokenMasterKey); req.setLastKnownNMTokenMasterKey(this.currentNMTokenMasterKey); + + req.setRegisteringCollectors(this.registeringCollectors); + NodeHeartbeatResponse heartbeatResponse = resourceTracker.nodeHeartbeat(req); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHATimelineCollectors.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHATimelineCollectors.java new file mode 100644 index 00000000000..fa0d318f63f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHATimelineCollectors.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.api.records.AppCollectorData; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineWriterImpl; +import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineWriter; +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; + +/** + * Test if the new active RM could recover collector status on a state + * transition. + */ +public class TestRMHATimelineCollectors extends RMHATestBase { + public static final Log LOG = LogFactory + .getLog(TestSubmitApplicationWithRMHA.class); + + @Before + @Override + public void setup() throws Exception { + super.setup(); + confForRM1.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); + confForRM2.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); + confForRM1.setClass(YarnConfiguration.TIMELINE_SERVICE_WRITER_CLASS, + FileSystemTimelineWriterImpl.class, TimelineWriter.class); + confForRM1.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f); + confForRM2.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f); + confForRM2.setClass(YarnConfiguration.TIMELINE_SERVICE_WRITER_CLASS, + FileSystemTimelineWriterImpl.class, TimelineWriter.class); + } + + @Test + public void testRebuildCollectorDataOnFailover() throws Exception { + startRMs(); + MockNM nm1 + = new MockNM("127.0.0.1:1234", 15120, rm2.getResourceTrackerService()); + MockNM nm2 + = new MockNM("127.0.0.1:5678", 15121, rm2.getResourceTrackerService()); + RMApp app1 = rm1.submitApp(1024); + String collectorAddr1 = "1.2.3.4:5"; + AppCollectorData data1 = AppCollectorData.newInstance( + app1.getApplicationId(), collectorAddr1); + nm1.addRegisteringCollector(app1.getApplicationId(), data1); + + String collectorAddr2 = "5.4.3.2:1"; + RMApp app2 = rm1.submitApp(1024); + AppCollectorData data2 = AppCollectorData.newInstance( + app2.getApplicationId(), collectorAddr2, rm1.getStartTime(), 1); + nm1.addRegisteringCollector(app2.getApplicationId(), data2); + + explicitFailover(); + + List runningApps = new ArrayList<>(); + runningApps.add(app1.getApplicationId()); + runningApps.add(app2.getApplicationId()); + nm1.registerNode(runningApps); + nm2.registerNode(runningApps); + + String collectorAddr12 = "1.2.3.4:56"; + AppCollectorData data12 = AppCollectorData.newInstance( + app1.getApplicationId(), collectorAddr12, rm1.getStartTime(), 0); + nm2.addRegisteringCollector(app1.getApplicationId(), data12); + + String collectorAddr22 = "5.4.3.2:10"; + AppCollectorData data22 = AppCollectorData.newInstance( + app2.getApplicationId(), collectorAddr22, rm1.getStartTime(), 2); + nm2.addRegisteringCollector(app2.getApplicationId(), data22); + + Map results1 + = nm1.nodeHeartbeat(true).getAppCollectors(); + assertEquals(collectorAddr1, + results1.get(app1.getApplicationId()).getCollectorAddr()); + assertEquals(collectorAddr2, + results1.get(app2.getApplicationId()).getCollectorAddr()); + + Map results2 + = nm2.nodeHeartbeat(true).getAppCollectors(); + // addr of app1 should be collectorAddr1 since it's registering (no time + // stamp). + assertEquals(collectorAddr1, + results2.get(app1.getApplicationId()).getCollectorAddr()); + // addr of app2 should be collectorAddr22 since its version number is + // greater. + assertEquals(collectorAddr22, + results2.get(app2.getApplicationId()).getCollectorAddr()); + + // Now nm1 should get updated collector list + nm1.getRegisteringCollectors().clear(); + Map results12 + = nm1.nodeHeartbeat(true).getAppCollectors(); + assertEquals(collectorAddr1, + results12.get(app1.getApplicationId()).getCollectorAddr()); + assertEquals(collectorAddr22, + results12.get(app2.getApplicationId()).getCollectorAddr()); + + + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java index 1e63fdf577f..5ed327868c6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java @@ -63,6 +63,7 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.UnRegisterNodeManagerRequest; +import org.apache.hadoop.yarn.server.api.records.AppCollectorData; import org.apache.hadoop.yarn.server.api.records.NodeAction; import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus; import org.apache.hadoop.yarn.server.api.records.NodeStatus; @@ -70,6 +71,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NodeLabelsUtils; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; @@ -1011,13 +1013,23 @@ public class TestResourceTrackerService extends NodeLabelTestBase { RMNodeImpl node2 = (RMNodeImpl) rm.getRMContext().getRMNodes().get(nm2.getNodeId()); - RMApp app1 = rm.submitApp(1024); + RMAppImpl app1 = (RMAppImpl) rm.submitApp(1024); String collectorAddr1 = "1.2.3.4:5"; - app1.setCollectorAddr(collectorAddr1); + app1.setCollectorData(AppCollectorData.newInstance( + app1.getApplicationId(), collectorAddr1)); String collectorAddr2 = "5.4.3.2:1"; - RMApp app2 = rm.submitApp(1024); - app2.setCollectorAddr(collectorAddr2); + RMAppImpl app2 = (RMAppImpl) rm.submitApp(1024); + app2.setCollectorData(AppCollectorData.newInstance( + app2.getApplicationId(), collectorAddr2)); + + String collectorAddr3 = "5.4.3.2:2"; + app2.setCollectorData(AppCollectorData.newInstance( + app2.getApplicationId(), collectorAddr3, 0, 1)); + + String collectorAddr4 = "5.4.3.2:3"; + app2.setCollectorData(AppCollectorData.newInstance( + app2.getApplicationId(), collectorAddr4, 1, 0)); // Create a running container for app1 running on nm1 ContainerId runningContainerId1 = BuilderUtils.newContainerId( @@ -1055,14 +1067,18 @@ public class TestResourceTrackerService extends NodeLabelTestBase { Assert.assertEquals(app2.getApplicationId(), node2.getRunningApps().get(0)); nodeHeartbeat1 = nm1.nodeHeartbeat(true); - Map map1 = nodeHeartbeat1.getAppCollectorsMap(); + Map map1 + = nodeHeartbeat1.getAppCollectors(); Assert.assertEquals(1, map1.size()); - Assert.assertEquals(collectorAddr1, map1.get(app1.getApplicationId())); + Assert.assertEquals(collectorAddr1, + map1.get(app1.getApplicationId()).getCollectorAddr()); nodeHeartbeat2 = nm2.nodeHeartbeat(true); - Map map2 = nodeHeartbeat2.getAppCollectorsMap(); + Map map2 + = nodeHeartbeat2.getAppCollectors(); Assert.assertEquals(1, map2.size()); - Assert.assertEquals(collectorAddr2, map2.get(app2.getApplicationId())); + Assert.assertEquals(collectorAddr4, + map2.get(app2.getApplicationId()).getCollectorAddr()); } private void checkRebootedNMCount(MockRM rm2, int count) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java index 5246eb79a15..f826631a21d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java @@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType; +import org.apache.hadoop.yarn.api.records.CollectorInfo; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; @@ -43,6 +44,7 @@ import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.api.protocolrecords.LogAggregationReport; +import org.apache.hadoop.yarn.server.api.records.AppCollectorData; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics; @@ -97,15 +99,7 @@ public abstract class MockAsm extends MockApps { throw new UnsupportedOperationException("Not supported yet."); } @Override - public String getCollectorAddr() { - throw new UnsupportedOperationException("Not supported yet."); - } - @Override - public void setCollectorAddr(String collectorAddr) { - throw new UnsupportedOperationException("Not supported yet."); - } - @Override - public void removeCollectorAddr() { + public AppCollectorData getCollectorData() { throw new UnsupportedOperationException("Not supported yet."); } @Override @@ -246,6 +240,11 @@ public abstract class MockAsm extends MockApps { public boolean isAppInCompletedStates() { throw new UnsupportedOperationException("Not supported yet."); } + + @Override + public CollectorInfo getCollectorInfo() { + throw new UnsupportedOperationException("Not supported yet."); + } } public static RMApp newApplication(int i) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java index ec099457510..c6bfcc71b23 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java @@ -69,6 +69,7 @@ import org.apache.hadoop.yarn.server.timelineservice.collector.AppLevelTimelineC import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineReaderImpl; import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineWriterImpl; import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineWriter; +import org.apache.hadoop.yarn.util.TimelineServiceHelper; import org.apache.hadoop.yarn.util.timeline.TimelineUtils; import org.junit.AfterClass; import org.junit.Assert; @@ -216,7 +217,8 @@ public class TestSystemMetricsPublisherForV2 { + FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION; File appFile = new File(outputDirApp, timelineServiceFileName); Assert.assertTrue(appFile.exists()); - verifyEntity(appFile, 3, ApplicationMetricsConstants.CREATED_EVENT_TYPE, 8); + verifyEntity( + appFile, 3, ApplicationMetricsConstants.CREATED_EVENT_TYPE, 8, 0); } @Test(timeout = 10000) @@ -251,7 +253,7 @@ public class TestSystemMetricsPublisherForV2 { File appFile = new File(outputDirApp, timelineServiceFileName); Assert.assertTrue(appFile.exists()); verifyEntity(appFile, 2, AppAttemptMetricsConstants.REGISTERED_EVENT_TYPE, - 0); + 0, TimelineServiceHelper.invertLong(appAttemptId.getAttemptId())); } @Test(timeout = 10000) @@ -283,7 +285,7 @@ public class TestSystemMetricsPublisherForV2 { File appFile = new File(outputDirApp, timelineServiceFileName); Assert.assertTrue(appFile.exists()); verifyEntity(appFile, 2, - ContainerMetricsConstants.CREATED_IN_RM_EVENT_TYPE, 0); + ContainerMetricsConstants.CREATED_IN_RM_EVENT_TYPE, 0, 0); } private RMApp createAppAndRegister(ApplicationId appId) { @@ -297,7 +299,8 @@ public class TestSystemMetricsPublisherForV2 { } private static void verifyEntity(File entityFile, long expectedEvents, - String eventForCreatedTime, long expectedMetrics) throws IOException { + String eventForCreatedTime, long expectedMetrics, long idPrefix) + throws IOException { BufferedReader reader = null; String strLine; long count = 0; @@ -309,6 +312,7 @@ public class TestSystemMetricsPublisherForV2 { TimelineEntity entity = FileSystemTimelineReaderImpl. getTimelineRecordFromJSON(strLine.trim(), TimelineEntity.class); metricsCount = entity.getMetrics().size(); + assertEquals(idPrefix, entity.getIdPrefix()); for (TimelineEvent event : entity.getEvents()) { if (event.getId().equals(eventForCreatedTime)) { assertTrue(entity.getCreatedTime() > 0); @@ -394,6 +398,7 @@ public class TestSystemMetricsPublisherForV2 { when(appAttempt.getTrackingUrl()).thenReturn("test tracking url"); when(appAttempt.getOriginalTrackingUrl()).thenReturn( "test original tracking url"); + when(appAttempt.getStartTime()).thenReturn(200L); return appAttempt; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java index 9290ff8faa0..39a7f995ab6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java @@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType; +import org.apache.hadoop.yarn.api.records.CollectorInfo; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.LogAggregationStatus; import org.apache.hadoop.yarn.api.records.NodeId; @@ -43,6 +44,7 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPBImpl; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.api.protocolrecords.LogAggregationReport; +import org.apache.hadoop.yarn.server.api.records.AppCollectorData; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; @@ -305,17 +307,8 @@ public class MockRMApp implements RMApp { throw new UnsupportedOperationException("Not supported yet."); } - public String getCollectorAddr() { - throw new UnsupportedOperationException("Not supported yet."); - } - @Override - public void removeCollectorAddr() { - throw new UnsupportedOperationException("Not supported yet."); - } - - @Override - public void setCollectorAddr(String collectorAddr) { + public AppCollectorData getCollectorData() { throw new UnsupportedOperationException("Not supported yet."); } @@ -333,4 +326,9 @@ public class MockRMApp implements RMApp { public boolean isAppInCompletedStates() { return false; } + + @Override + public CollectorInfo getCollectorInfo() { + throw new UnsupportedOperationException("Not supported yet."); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml index f182236e32e..babe9fe8bf0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml @@ -125,6 +125,17 @@ commons-logging commons-logging + + org.bouncycastle + bcprov-jdk16 + test + + + org.apache.hadoop + hadoop-auth + test + test-jar + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestRMNMSecretKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestRMNMSecretKeys.java index 8cc2dee43ac..ba14491bf51 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestRMNMSecretKeys.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestRMNMSecretKeys.java @@ -18,10 +18,13 @@ package org.apache.hadoop.yarn.server; +import java.io.File; import java.io.IOException; +import java.util.UUID; +import org.junit.AfterClass; import org.junit.Assert; - +import org.junit.BeforeClass; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -32,9 +35,38 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerResp import org.apache.hadoop.yarn.server.api.records.MasterKey; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.kerby.util.IOUtil; import org.junit.Test; public class TestRMNMSecretKeys { + private static final String KRB5_CONF = "java.security.krb5.conf"; + private static final File KRB5_CONF_ROOT_DIR = new File( + System.getProperty("test.build.dir", "target/test-dir"), + UUID.randomUUID().toString()); + + @BeforeClass + public static void setup() throws IOException { + KRB5_CONF_ROOT_DIR.mkdir(); + File krb5ConfFile = new File(KRB5_CONF_ROOT_DIR, "krb5.conf"); + krb5ConfFile.createNewFile(); + String content = "[libdefaults]\n" + + " default_realm = APACHE.ORG\n" + + " udp_preference_limit = 1\n"+ + " extra_addresses = 127.0.0.1\n" + + "[realms]\n" + + " APACHE.ORG = {\n" + + " admin_server = localhost:88\n" + + " kdc = localhost:88\n}\n" + + "[domain_realm]\n" + + " localhost = APACHE.ORG"; + IOUtil.writeFile(content, krb5ConfFile); + System.setProperty(KRB5_CONF, krb5ConfFile.getAbsolutePath()); + } + + @AfterClass + public static void tearDown() throws IOException { + KRB5_CONF_ROOT_DIR.delete(); + } @Test(timeout = 1000000) public void testNMUpdation() throws Exception { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java index 07058f62ef7..6a5ef552510 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java @@ -33,6 +33,7 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.ExitUtil; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.CollectorInfo; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.timelineservice.ApplicationAttemptEntity; import org.apache.hadoop.yarn.api.records.timelineservice.ApplicationEntity; @@ -80,7 +81,7 @@ public class TestTimelineServiceClientIntegration { auxService = PerNodeTimelineCollectorsAuxService.launchServer(new String[0], collectorManager, conf); - auxService.addApplication(ApplicationId.newInstance(0, 1)); + auxService.addApplication(ApplicationId.newInstance(0, 1), "user"); } catch (ExitUtil.ExitException e) { fail(); } @@ -99,9 +100,9 @@ public class TestTimelineServiceClientIntegration { TimelineV2Client client = TimelineV2Client.createTimelineClient(ApplicationId.newInstance(0, 1)); try { - // set the timeline service address manually - client.setTimelineServiceAddress( - collectorManager.getRestServerBindAddress()); + // Set the timeline service address manually. + client.setTimelineCollectorInfo(CollectorInfo.newInstance( + collectorManager.getRestServerBindAddress())); client.init(conf); client.start(); TimelineEntity entity = new TimelineEntity(); @@ -126,9 +127,9 @@ public class TestTimelineServiceClientIntegration { TimelineV2Client client = TimelineV2Client.createTimelineClient(appId); try { - // set the timeline service address manually - client.setTimelineServiceAddress( - collectorManager.getRestServerBindAddress()); + // Set the timeline service address manually. + client.setTimelineCollectorInfo(CollectorInfo.newInstance( + collectorManager.getRestServerBindAddress())); client.init(conf); client.start(); ClusterEntity cluster = new ClusterEntity(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/security/TestTimelineAuthFilterForV2.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/security/TestTimelineAuthFilterForV2.java new file mode 100644 index 00000000000..75f17fb3967 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/security/TestTimelineAuthFilterForV2.java @@ -0,0 +1,478 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.timelineservice.security; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; +import java.util.UUID; +import java.util.concurrent.Callable; + +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.http.HttpConfig; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.minikdc.MiniKdc; +import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.KerberosTestUtils; +import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler; +import org.apache.hadoop.security.ssl.KeyStoreTestUtil; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.CollectorInfo; +import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; +import org.apache.hadoop.yarn.client.api.TimelineV2Client; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier; +import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocol; +import org.apache.hadoop.yarn.server.api.protocolrecords.GetTimelineCollectorContextRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.GetTimelineCollectorContextResponse; +import org.apache.hadoop.yarn.server.timeline.security.TimelineAuthenticationFilterInitializer; +import org.apache.hadoop.yarn.server.timelineservice.collector.AppLevelTimelineCollector; +import org.apache.hadoop.yarn.server.timelineservice.collector.NodeTimelineCollectorManager; +import org.apache.hadoop.yarn.server.timelineservice.collector.PerNodeTimelineCollectorsAuxService; +import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineReaderImpl; +import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineWriterImpl; +import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineWriter; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +/** + * Tests timeline authentication filter based security for timeline service v2. + */ +@RunWith(Parameterized.class) +public class TestTimelineAuthFilterForV2 { + + private static final String FOO_USER = "foo"; + private static final String HTTP_USER = "HTTP"; + private static final File TEST_ROOT_DIR = new File( + System.getProperty("test.build.dir", "target" + File.separator + + "test-dir"), UUID.randomUUID().toString()); + private static final String BASEDIR = + System.getProperty("test.build.dir", "target/test-dir") + "/" + + TestTimelineAuthFilterForV2.class.getSimpleName(); + private static File httpSpnegoKeytabFile = new File(KerberosTestUtils. + getKeytabFile()); + private static String httpSpnegoPrincipal = KerberosTestUtils. + getServerPrincipal(); + + // First param indicates whether HTTPS access or HTTP access and second param + // indicates whether it is kerberos access or token based access. + @Parameterized.Parameters + public static Collection params() { + return Arrays.asList(new Object[][] {{false, true}, {false, false}, + {true, false}, {true, true}}); + } + + private static MiniKdc testMiniKDC; + private static String keystoresDir; + private static String sslConfDir; + private static Configuration conf; + private static UserGroupInformation nonKerberosUser; + static { + try { + nonKerberosUser = UserGroupInformation.getCurrentUser(); + } catch (IOException e) {} + } + // Indicates whether HTTPS or HTTP access. + private boolean withSsl; + // Indicates whether Kerberos based login is used or token based access is + // done. + private boolean withKerberosLogin; + private NodeTimelineCollectorManager collectorManager; + private PerNodeTimelineCollectorsAuxService auxService; + public TestTimelineAuthFilterForV2(boolean withSsl, + boolean withKerberosLogin) { + this.withSsl = withSsl; + this.withKerberosLogin = withKerberosLogin; + } + + @BeforeClass + public static void setup() { + try { + testMiniKDC = new MiniKdc(MiniKdc.createConf(), TEST_ROOT_DIR); + testMiniKDC.start(); + testMiniKDC.createPrincipal( + httpSpnegoKeytabFile, HTTP_USER + "/localhost"); + } catch (Exception e) { + fail("Couldn't setup MiniKDC."); + } + + // Setup timeline service v2. + try { + conf = new Configuration(false); + conf.setStrings(TimelineAuthenticationFilterInitializer.PREFIX + "type", + "kerberos"); + conf.set(TimelineAuthenticationFilterInitializer.PREFIX + + KerberosAuthenticationHandler.PRINCIPAL, httpSpnegoPrincipal); + conf.set(TimelineAuthenticationFilterInitializer.PREFIX + + KerberosAuthenticationHandler.KEYTAB, + httpSpnegoKeytabFile.getAbsolutePath()); + conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, + "kerberos"); + conf.set(YarnConfiguration.TIMELINE_SERVICE_PRINCIPAL, + httpSpnegoPrincipal); + conf.set(YarnConfiguration.TIMELINE_SERVICE_KEYTAB, + httpSpnegoKeytabFile.getAbsolutePath()); + // Enable timeline service v2 + conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); + conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f); + conf.setClass(YarnConfiguration.TIMELINE_SERVICE_WRITER_CLASS, + FileSystemTimelineWriterImpl.class, TimelineWriter.class); + conf.set(YarnConfiguration.TIMELINE_SERVICE_BIND_HOST, "localhost"); + conf.set(FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_DIR_ROOT, + TEST_ROOT_DIR.getAbsolutePath()); + conf.set("hadoop.proxyuser.HTTP.hosts", "*"); + conf.set("hadoop.proxyuser.HTTP.users", FOO_USER); + UserGroupInformation.setConfiguration(conf); + } catch (Exception e) { + fail("Couldn't setup TimelineServer V2."); + } + } + + @Before + public void initialize() throws Exception { + if (withSsl) { + conf.set(YarnConfiguration.YARN_HTTP_POLICY_KEY, + HttpConfig.Policy.HTTPS_ONLY.name()); + File base = new File(BASEDIR); + FileUtil.fullyDelete(base); + base.mkdirs(); + keystoresDir = new File(BASEDIR).getAbsolutePath(); + sslConfDir = + KeyStoreTestUtil.getClasspathDir(TestTimelineAuthFilterForV2.class); + KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, conf, false); + } else { + conf.set(YarnConfiguration.YARN_HTTP_POLICY_KEY, + HttpConfig.Policy.HTTP_ONLY.name()); + } + if (!withKerberosLogin) { + // For timeline delegation token based access, set delegation token renew + // interval to 100 ms. to test if timeline delegation token for the app is + // renewed automatically if app is still alive. + conf.setLong( + YarnConfiguration.TIMELINE_DELEGATION_TOKEN_RENEW_INTERVAL, 100); + // Set token max lifetime to 4 seconds to test if timeline delegation + // token for the app is regenerated automatically if app is still alive. + conf.setLong( + YarnConfiguration.TIMELINE_DELEGATION_TOKEN_MAX_LIFETIME, 4000); + } + UserGroupInformation.setConfiguration(conf); + collectorManager = new DummyNodeTimelineCollectorManager(); + auxService = PerNodeTimelineCollectorsAuxService.launchServer( + new String[0], collectorManager, conf); + if (withKerberosLogin) { + SecurityUtil.login(conf, YarnConfiguration.TIMELINE_SERVICE_KEYTAB, + YarnConfiguration.TIMELINE_SERVICE_PRINCIPAL, "localhost"); + } + ApplicationId appId = ApplicationId.newInstance(0, 1); + auxService.addApplication( + appId, UserGroupInformation.getCurrentUser().getUserName()); + if (!withKerberosLogin) { + AppLevelTimelineCollector collector = + (AppLevelTimelineCollector)collectorManager.get(appId); + Token token = + collector.getDelegationTokenForApp(); + token.setService(new Text("localhost" + token.getService().toString(). + substring(token.getService().toString().indexOf(":")))); + UserGroupInformation.getCurrentUser().addToken(token); + } + } + + private TimelineV2Client createTimelineClientForUGI(ApplicationId appId) { + TimelineV2Client client = + TimelineV2Client.createTimelineClient(ApplicationId.newInstance(0, 1)); + // set the timeline service address. + String restBindAddr = collectorManager.getRestServerBindAddress(); + String addr = + "localhost" + restBindAddr.substring(restBindAddr.indexOf(":")); + client.setTimelineCollectorInfo(CollectorInfo.newInstance(addr)); + client.init(conf); + client.start(); + return client; + } + + @AfterClass + public static void tearDown() throws Exception { + if (testMiniKDC != null) { + testMiniKDC.stop(); + } + FileUtil.fullyDelete(TEST_ROOT_DIR); + } + + @After + public void destroy() throws Exception { + if (auxService != null) { + auxService.stop(); + } + if (withSsl) { + KeyStoreTestUtil.cleanupSSLConfig(keystoresDir, sslConfDir); + FileUtil.fullyDelete(new File(BASEDIR)); + } + if (withKerberosLogin) { + UserGroupInformation.getCurrentUser().logoutUserFromKeytab(); + } + // Reset the user for next run. + UserGroupInformation.setLoginUser( + UserGroupInformation.createRemoteUser(nonKerberosUser.getUserName())); + } + + private static TimelineEntity createEntity(String id, String type) { + TimelineEntity entityToStore = new TimelineEntity(); + entityToStore.setId(id); + entityToStore.setType(type); + entityToStore.setCreatedTime(0L); + return entityToStore; + } + + private static void verifyEntity(File entityTypeDir, String id, String type) + throws IOException { + File entityFile = new File(entityTypeDir, id + + FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION); + assertTrue(entityFile.exists()); + TimelineEntity entity = readEntityFile(entityFile); + assertNotNull(entity); + assertEquals(id, entity.getId()); + assertEquals(type, entity.getType()); + } + + private static TimelineEntity readEntityFile(File entityFile) + throws IOException { + BufferedReader reader = null; + String strLine; + try { + reader = new BufferedReader(new FileReader(entityFile)); + while ((strLine = reader.readLine()) != null) { + if (strLine.trim().length() > 0) { + return FileSystemTimelineReaderImpl. + getTimelineRecordFromJSON(strLine.trim(), TimelineEntity.class); + } + } + return null; + } finally { + reader.close(); + } + } + + private void publishAndVerifyEntity(ApplicationId appId, File entityTypeDir, + String entityType, int numEntities) throws Exception { + TimelineV2Client client = createTimelineClientForUGI(appId); + try { + // Sync call. Results available immediately. + client.putEntities(createEntity("entity1", entityType)); + assertEquals(numEntities, entityTypeDir.listFiles().length); + verifyEntity(entityTypeDir, "entity1", entityType); + // Async call. + client.putEntitiesAsync(createEntity("entity2", entityType)); + } finally { + client.stop(); + } + } + + private boolean publishWithRetries(ApplicationId appId, File entityTypeDir, + String entityType, int numEntities) throws Exception { + for (int i = 0; i < 10; i++) { + try { + publishAndVerifyEntity(appId, entityTypeDir, entityType, numEntities); + } catch (YarnException e) { + Thread.sleep(50); + continue; + } + return true; + } + return false; + } + + @Test + public void testPutTimelineEntities() throws Exception { + final String entityType = "dummy_type"; + ApplicationId appId = ApplicationId.newInstance(0, 1); + File entityTypeDir = new File(TEST_ROOT_DIR.getAbsolutePath() + + File.separator + "entities" + File.separator + + YarnConfiguration.DEFAULT_RM_CLUSTER_ID + File.separator + + UserGroupInformation.getCurrentUser().getUserName() + + File.separator + "test_flow_name" + File.separator + + "test_flow_version" + File.separator + "1" + File.separator + + appId.toString() + File.separator + entityType); + try { + if (withKerberosLogin) { + KerberosTestUtils.doAs(HTTP_USER + "/localhost", new Callable() { + @Override + public Void call() throws Exception { + publishAndVerifyEntity(appId, entityTypeDir, entityType, 1); + return null; + } + }); + } else { + assertTrue("Entities should have been published successfully.", + publishWithRetries(appId, entityTypeDir, entityType, 1)); + + AppLevelTimelineCollector collector = + (AppLevelTimelineCollector) collectorManager.get(appId); + Token token = + collector.getDelegationTokenForApp(); + assertNotNull(token); + + // Verify if token is renewed automatically and entities can still be + // published. + Thread.sleep(1000); + // Entities should publish successfully after renewal. + assertTrue("Entities should have been published successfully.", + publishWithRetries(appId, entityTypeDir, entityType, 2)); + assertNotNull(collector); + verify(collectorManager.getTokenManagerService(), atLeastOnce()). + renewToken(eq(collector.getDelegationTokenForApp()), + any(String.class)); + + // Wait to ensure lifetime of token expires and ensure its regenerated + // automatically. + Thread.sleep(3000); + for (int i = 0; i < 40; i++) { + if (!token.equals(collector.getDelegationTokenForApp())) { + break; + } + Thread.sleep(50); + } + assertNotEquals("Token should have been regenerated.", token, + collector.getDelegationTokenForApp()); + Thread.sleep(1000); + // Try publishing with the old token in UGI. Publishing should fail due + // to invalid token. + try { + publishAndVerifyEntity(appId, entityTypeDir, entityType, 2); + fail("Exception should have been thrown due to Invalid Token."); + } catch (YarnException e) { + assertTrue("Exception thrown should have been due to Invalid Token.", + e.getCause().getMessage().contains("InvalidToken")); + } + + // Update the regenerated token in UGI and retry publishing entities. + Token regeneratedToken = + collector.getDelegationTokenForApp(); + regeneratedToken.setService(new Text("localhost" + + regeneratedToken.getService().toString().substring( + regeneratedToken.getService().toString().indexOf(":")))); + UserGroupInformation.getCurrentUser().addToken(regeneratedToken); + assertTrue("Entities should have been published successfully.", + publishWithRetries(appId, entityTypeDir, entityType, 2)); + // Token was generated twice, once when app collector was created and + // later after token lifetime expiry. + verify(collectorManager.getTokenManagerService(), times(2)). + generateToken(any(UserGroupInformation.class), any(String.class)); + assertEquals(1, ((DummyNodeTimelineCollectorManager) collectorManager). + getTokenExpiredCnt()); + } + // Wait for async entity to be published. + for (int i = 0; i < 50; i++) { + if (entityTypeDir.listFiles().length == 2) { + break; + } + Thread.sleep(50); + } + assertEquals(2, entityTypeDir.listFiles().length); + verifyEntity(entityTypeDir, "entity2", entityType); + AppLevelTimelineCollector collector = + (AppLevelTimelineCollector)collectorManager.get(appId); + assertNotNull(collector); + auxService.removeApplication(appId); + verify(collectorManager.getTokenManagerService()).cancelToken( + eq(collector.getDelegationTokenForApp()), any(String.class)); + } finally { + FileUtils.deleteQuietly(entityTypeDir); + } + } + + private static class DummyNodeTimelineCollectorManager extends + NodeTimelineCollectorManager { + private volatile int tokenExpiredCnt = 0; + DummyNodeTimelineCollectorManager() { + super(); + } + + private int getTokenExpiredCnt() { + return tokenExpiredCnt; + } + + @Override + protected TimelineV2DelegationTokenSecretManagerService + createTokenManagerService() { + return spy(new TimelineV2DelegationTokenSecretManagerService() { + @Override + protected AbstractDelegationTokenSecretManager + + createTimelineDelegationTokenSecretManager(long secretKeyInterval, + long tokenMaxLifetime, long tokenRenewInterval, + long tokenRemovalScanInterval) { + return spy(new TimelineV2DelegationTokenSecretManager( + secretKeyInterval, tokenMaxLifetime, tokenRenewInterval, 2000L) { + @Override + protected void logExpireToken( + TimelineDelegationTokenIdentifier ident) throws IOException { + tokenExpiredCnt++; + } + }); + } + }); + } + + @Override + protected CollectorNodemanagerProtocol getNMCollectorService() { + CollectorNodemanagerProtocol protocol = + mock(CollectorNodemanagerProtocol.class); + try { + GetTimelineCollectorContextResponse response = + GetTimelineCollectorContextResponse.newInstance( + UserGroupInformation.getCurrentUser().getUserName(), + "test_flow_name", "test_flow_version", 1L); + when(protocol.getTimelineCollectorContext(any( + GetTimelineCollectorContextRequest.class))).thenReturn(response); + } catch (YarnException | IOException e) { + fail(); + } + return protocol; + } + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/resources/krb5.conf b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/resources/krb5.conf deleted file mode 100644 index 121ac6d9b98..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/resources/krb5.conf +++ /dev/null @@ -1,28 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -[libdefaults] - default_realm = APACHE.ORG - udp_preference_limit = 1 - extra_addresses = 127.0.0.1 -[realms] - APACHE.ORG = { - admin_server = localhost:88 - kdc = localhost:88 - } -[domain_realm] - localhost = APACHE.ORG diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/AbstractTimelineReaderHBaseTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/AbstractTimelineReaderHBaseTestBase.java new file mode 100644 index 00000000000..3519c3f3bf2 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/AbstractTimelineReaderHBaseTestBase.java @@ -0,0 +1,177 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.timelineservice.reader; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.lang.reflect.UndeclaredThrowableException; +import java.net.HttpURLConnection; +import java.net.URI; +import java.net.URL; +import java.util.List; + +import javax.ws.rs.core.MediaType; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.yarn.api.records.timelineservice.FlowActivityEntity; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.timelineservice.storage.DataGeneratorForTest; +import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; +import org.junit.Assert; + +import com.sun.jersey.api.client.Client; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.ClientResponse.Status; +import com.sun.jersey.api.client.GenericType; +import com.sun.jersey.api.client.config.ClientConfig; +import com.sun.jersey.api.client.config.DefaultClientConfig; +import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory; +import com.sun.jersey.client.urlconnection.URLConnectionClientHandler; + +/** + * Test Base for TimelineReaderServer HBase tests. + */ +public abstract class AbstractTimelineReaderHBaseTestBase { + private static int serverPort; + private static TimelineReaderServer server; + private static HBaseTestingUtility util; + + public static void setup() throws Exception { + util = new HBaseTestingUtility(); + Configuration conf = util.getConfiguration(); + conf.setInt("hfile.format.version", 3); + util.startMiniCluster(); + DataGeneratorForTest.createSchema(util.getConfiguration()); + } + + public static void tearDown() throws Exception { + if (server != null) { + server.stop(); + server = null; + } + if (util != null) { + util.shutdownMiniCluster(); + } + } + + protected static void initialize() throws Exception { + try { + Configuration config = util.getConfiguration(); + config.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); + config.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f); + config.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS, + "localhost:0"); + config.set(YarnConfiguration.RM_CLUSTER_ID, "cluster1"); + config.set(YarnConfiguration.TIMELINE_SERVICE_READER_CLASS, + "org.apache.hadoop.yarn.server.timelineservice.storage." + + "HBaseTimelineReaderImpl"); + config.setInt("hfile.format.version", 3); + server = new TimelineReaderServer() { + @Override + protected void addFilters(Configuration conf) { + // The parent code uses hadoop-common jar from this version of + // Hadoop, but the tests are using hadoop-common jar from + // ${hbase-compatible-hadoop.version}. This version uses Jetty 9 + // while ${hbase-compatible-hadoop.version} uses Jetty 6, and there + // are many differences, including classnames and packages. + // We do nothing here, so that we don't cause a NoSuchMethodError or + // NoClassDefFoundError. + // Once ${hbase-compatible-hadoop.version} is changed to Hadoop 3, + // we should be able to remove this @Override. + } + }; + server.init(config); + server.start(); + serverPort = server.getWebServerPort(); + } catch (Exception e) { + Assert.fail("Web server failed to start"); + } + } + + protected Client createClient() { + ClientConfig cfg = new DefaultClientConfig(); + cfg.getClasses().add(YarnJacksonJaxbJsonProvider.class); + return new Client( + new URLConnectionClientHandler(new DummyURLConnectionFactory()), cfg); + } + + protected ClientResponse getResponse(Client client, URI uri) + throws Exception { + ClientResponse resp = + client.resource(uri).accept(MediaType.APPLICATION_JSON) + .type(MediaType.APPLICATION_JSON).get(ClientResponse.class); + if (resp == null || resp.getStatusInfo() + .getStatusCode() != ClientResponse.Status.OK.getStatusCode()) { + String msg = ""; + if (resp != null) { + msg = String.valueOf(resp.getStatusInfo().getStatusCode()); + } + throw new IOException( + "Incorrect response from timeline reader. " + "Status=" + msg); + } + return resp; + } + + protected void verifyHttpResponse(Client client, URI uri, Status status) { + ClientResponse resp = + client.resource(uri).accept(MediaType.APPLICATION_JSON) + .type(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertNotNull(resp); + assertTrue("Response from server should have been " + status, + resp.getStatusInfo().getStatusCode() == status.getStatusCode()); + System.out.println("Response is: " + resp.getEntity(String.class)); + } + + protected List verifyFlowEntites(Client client, URI uri, + int noOfEntities) throws Exception { + ClientResponse resp = getResponse(client, uri); + List entities = + resp.getEntity(new GenericType>() { + }); + assertNotNull(entities); + assertEquals(noOfEntities, entities.size()); + return entities; + } + + protected static class DummyURLConnectionFactory + implements HttpURLConnectionFactory { + + @Override + public HttpURLConnection getHttpURLConnection(final URL url) + throws IOException { + try { + return (HttpURLConnection) url.openConnection(); + } catch (UndeclaredThrowableException e) { + throw new IOException(e.getCause()); + } + } + } + + protected static HBaseTestingUtility getHBaseTestingUtility() { + return util; + } + + public static int getServerPort() { + return serverPort; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java index 3f8978cb40c..b2029ca1f55 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java @@ -24,10 +24,7 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.IOException; -import java.lang.reflect.UndeclaredThrowableException; -import java.net.HttpURLConnection; import java.net.URI; -import java.net.URL; import java.text.DateFormat; import java.util.ArrayList; import java.util.HashMap; @@ -39,7 +36,9 @@ import java.util.Set; import javax.ws.rs.core.MediaType; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.timelineservice.ApplicationEntity; import org.apache.hadoop.yarn.api.records.timelineservice.FlowActivityEntity; import org.apache.hadoop.yarn.api.records.timelineservice.FlowRunEntity; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities; @@ -48,16 +47,12 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric.Type; -import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants; +import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorContext; import org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl; -import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineSchemaCreator; import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineStorageUtils; -import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; -import org.junit.After; +import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -67,27 +62,27 @@ import com.sun.jersey.api.client.Client; import com.sun.jersey.api.client.ClientResponse; import com.sun.jersey.api.client.ClientResponse.Status; import com.sun.jersey.api.client.GenericType; -import com.sun.jersey.api.client.config.ClientConfig; -import com.sun.jersey.api.client.config.DefaultClientConfig; -import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory; -import com.sun.jersey.client.urlconnection.URLConnectionClientHandler; -public class TestTimelineReaderWebServicesHBaseStorage { - private int serverPort; - private TimelineReaderServer server; - private static HBaseTestingUtility util; +/** + * Test TimelineReder Web Service REST API's. + */ +public class TestTimelineReaderWebServicesHBaseStorage + extends AbstractTimelineReaderHBaseTestBase { private static long ts = System.currentTimeMillis(); private static long dayTs = HBaseTimelineStorageUtils.getTopOfTheDayTimestamp(ts); + private static String doAsUser = "remoteuser"; @BeforeClass - public static void setup() throws Exception { - util = new HBaseTestingUtility(); - Configuration conf = util.getConfiguration(); - conf.setInt("hfile.format.version", 3); - util.startMiniCluster(); - TimelineSchemaCreator.createAllTables(util.getConfiguration(), false); + public static void setupBeforeClass() throws Exception { + setup(); loadData(); + initialize(); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + tearDown(); } private static void loadData() throws Exception { @@ -214,7 +209,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { entity4.addMetrics(metrics); te4.addEntity(entity4); - TimelineEntities te5 = new TimelineEntities(); + TimelineEntities userEntities = new TimelineEntities(); TimelineEntity entity5 = new TimelineEntity(); entity5.setId("entity1"); entity5.setType("type1"); @@ -270,7 +265,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { relatesTo1.put("type3", Sets.newHashSet("entity31", "entity35", "entity32", "entity33")); entity5.addRelatesToEntities(relatesTo1); - te5.addEntity(entity5); + userEntities.addEntity(entity5); TimelineEntity entity6 = new TimelineEntity(); entity6.setId("entity2"); @@ -329,20 +324,35 @@ public class TestTimelineReaderWebServicesHBaseStorage { relatesTo2.put("type6", Sets.newHashSet("entity61", "entity66")); relatesTo2.put("type3", Sets.newHashSet("entity31")); entity6.addRelatesToEntities(relatesTo2); - te5.addEntity(entity6); + userEntities.addEntity(entity6); + + for (long i = 1; i <= 10; i++) { + TimelineEntity userEntity = new TimelineEntity(); + userEntity.setType("entitytype"); + userEntity.setId("entityid-" + i); + userEntity.setIdPrefix(11 - i); + userEntity.setCreatedTime(ts); + userEntities.addEntity(userEntity); + } HBaseTimelineWriterImpl hbi = null; - Configuration c1 = util.getConfiguration(); + Configuration c1 = getHBaseTestingUtility().getConfiguration(); + UserGroupInformation remoteUser = + UserGroupInformation.createRemoteUser(doAsUser); try { hbi = new HBaseTimelineWriterImpl(); hbi.init(c1); - hbi.write(cluster, user, flow, flowVersion, runid, entity.getId(), te); - hbi.write(cluster, user, flow, flowVersion, runid, entity1.getId(), te1); - hbi.write(cluster, user, flow, flowVersion, runid1, entity4.getId(), te4); - hbi.write(cluster, user, flow2, - flowVersion2, runid2, entity3.getId(), te3); - hbi.write(cluster, user, flow, flowVersion, runid, - "application_1111111111_1111", te5); + hbi.write(new TimelineCollectorContext(cluster, user, flow, flowVersion, + runid, entity.getId()), te, remoteUser); + hbi.write(new TimelineCollectorContext(cluster, user, flow, flowVersion, + runid, entity1.getId()), te1, remoteUser); + hbi.write(new TimelineCollectorContext(cluster, user, flow, flowVersion, + runid1, entity4.getId()), te4, remoteUser); + hbi.write(new TimelineCollectorContext(cluster, user, flow2, flowVersion2, + runid2, entity3.getId()), te3, remoteUser); + hbi.write(new TimelineCollectorContext(cluster, user, flow, flowVersion, + runid, "application_1111111111_1111"), userEntities, remoteUser); + writeApplicationEntities(hbi, ts); hbi.flush(); } finally { if (hbi != null) { @@ -351,80 +361,32 @@ public class TestTimelineReaderWebServicesHBaseStorage { } } - @AfterClass - public static void tearDown() throws Exception { - util.shutdownMiniCluster(); - } + static void writeApplicationEntities(HBaseTimelineWriterImpl hbi, + long timestamp) throws IOException { + int count = 1; + for (long i = 1; i <= 3; i++) { + for (int j = 1; j <= 5; j++) { + TimelineEntities te = new TimelineEntities(); + ApplicationId appId = + BuilderUtils.newApplicationId(timestamp, count++); + ApplicationEntity appEntity = new ApplicationEntity(); + appEntity.setId( + HBaseTimelineStorageUtils.convertApplicationIdToString(appId)); + appEntity.setCreatedTime(timestamp); - @Before - public void init() throws Exception { - try { - Configuration config = util.getConfiguration(); - config.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); - config.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f); - config.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS, - "localhost:0"); - config.set(YarnConfiguration.RM_CLUSTER_ID, "cluster1"); - config.set(YarnConfiguration.TIMELINE_SERVICE_READER_CLASS, - "org.apache.hadoop.yarn.server.timelineservice.storage." + - "HBaseTimelineReaderImpl"); - config.setInt("hfile.format.version", 3); - server = new TimelineReaderServer() { - @Override - protected void setupOptions(Configuration conf) { - // The parent code tries to use HttpServer2 from this version of - // Hadoop, but the tests are loading in HttpServer2 from - // ${hbase-compatible-hadoop.version}. This version uses Jetty 9 - // while ${hbase-compatible-hadoop.version} uses Jetty 6, and there - // are many differences, including classnames and packages. - // We do nothing here, so that we don't cause a NoSuchMethodError. - // Once ${hbase-compatible-hadoop.version} is changed to Hadoop 3, - // we should be able to remove this @Override. - } - }; - server.init(config); - server.start(); - serverPort = server.getWebServerPort(); - } catch (Exception e) { - Assert.fail("Web server failed to start"); - } - } + TimelineEvent created = new TimelineEvent(); + created.setId(ApplicationMetricsConstants.CREATED_EVENT_TYPE); + created.setTimestamp(timestamp); + appEntity.addEvent(created); + TimelineEvent finished = new TimelineEvent(); + finished.setId(ApplicationMetricsConstants.FINISHED_EVENT_TYPE); + finished.setTimestamp(timestamp + i * j); - private static Client createClient() { - ClientConfig cfg = new DefaultClientConfig(); - cfg.getClasses().add(YarnJacksonJaxbJsonProvider.class); - return new Client(new URLConnectionClientHandler( - new DummyURLConnectionFactory()), cfg); - } - - private static ClientResponse getResponse(Client client, URI uri) - throws Exception { - ClientResponse resp = - client.resource(uri).accept(MediaType.APPLICATION_JSON) - .type(MediaType.APPLICATION_JSON).get(ClientResponse.class); - if (resp == null || - resp.getStatusInfo().getStatusCode() != - ClientResponse.Status.OK.getStatusCode()) { - String msg = ""; - if (resp != null) { - msg = String.valueOf(resp.getStatusInfo().getStatusCode()); - } - throw new IOException("Incorrect response from timeline reader. " + - "Status=" + msg); - } - return resp; - } - - private static class DummyURLConnectionFactory - implements HttpURLConnectionFactory { - - @Override - public HttpURLConnection getHttpURLConnection(final URL url) - throws IOException { - try { - return (HttpURLConnection)url.openConnection(); - } catch (UndeclaredThrowableException e) { - throw new IOException(e.getCause()); + appEntity.addEvent(finished); + te.addEntity(appEntity); + hbi.write(new TimelineCollectorContext("cluster1", "user1", "flow1", + "CF7022C10F1354", i, appEntity.getId()), te, + UserGroupInformation.createRemoteUser("user1")); } } } @@ -470,22 +432,11 @@ public class TestTimelineReaderWebServicesHBaseStorage { return false; } - private static void verifyHttpResponse(Client client, URI uri, - Status status) { - ClientResponse resp = - client.resource(uri).accept(MediaType.APPLICATION_JSON) - .type(MediaType.APPLICATION_JSON).get(ClientResponse.class); - assertNotNull(resp); - assertTrue("Response from server should have been " + status, - resp.getStatusInfo().getStatusCode() == status.getStatusCode()); - System.out.println("Response is: " + resp.getEntity(String.class)); - } - @Test public void testGetFlowRun() throws Exception { Client client = createClient(); try { - URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + URI uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/users/user1/flows/flow_name/runs/" + "1002345678919"); ClientResponse resp = getResponse(client, uri); @@ -506,7 +457,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { } // Query without specifying cluster ID. - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/users/user1/flows/flow_name/runs/1002345678919"); resp = getResponse(client, uri); entity = resp.getEntity(FlowRunEntity.class); @@ -531,7 +482,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { public void testGetFlowRuns() throws Exception { Client client = createClient(); try { - URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + URI uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/users/user1/flows/flow_name/runs"); ClientResponse resp = getResponse(client, uri); Set entities = @@ -551,8 +502,9 @@ public class TestTimelineReaderWebServicesHBaseStorage { assertEquals(0, entity.getMetrics().size()); } - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" + - "clusters/cluster1/users/user1/flows/flow_name/runs?limit=1"); + uri = + URI.create("http://localhost:" + getServerPort() + "/ws/v2/timeline/" + + "clusters/cluster1/users/user1/flows/flow_name/runs?limit=1"); resp = getResponse(client, uri); entities = resp.getEntity(new GenericType>(){}); assertEquals(MediaType.APPLICATION_JSON_TYPE + "; charset=utf-8", @@ -567,7 +519,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { assertEquals(0, entity.getMetrics().size()); } - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/users/user1/flows/flow_name/runs?" + "createdtimestart=1425016501030"); resp = getResponse(client, uri); @@ -584,7 +536,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { assertEquals(0, entity.getMetrics().size()); } - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/users/user1/flows/flow_name/runs?" + "createdtimestart=1425016500999&createdtimeend=1425016501035"); resp = getResponse(client, uri); @@ -604,7 +556,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { assertEquals(0, entity.getMetrics().size()); } - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/users/user1/flows/flow_name/runs?" + "createdtimeend=1425016501030"); resp = getResponse(client, uri); @@ -621,7 +573,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { assertEquals(0, entity.getMetrics().size()); } - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/users/user1/flows/flow_name/runs?" + "fields=metrics"); resp = getResponse(client, uri); @@ -644,7 +596,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { // fields as CONFIGS will lead to a HTTP 400 as it makes no sense for // flow runs. - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/users/user1/flows/flow_name/runs?" + "fields=CONFIGS"); verifyHttpResponse(client, uri, Status.BAD_REQUEST); @@ -657,7 +609,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { public void testGetFlowRunsMetricsToRetrieve() throws Exception { Client client = createClient(); try { - URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + URI uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/users/user1/flows/flow_name/runs?" + "metricstoretrieve=MAP_,HDFS_"); ClientResponse resp = getResponse(client, uri); @@ -677,7 +629,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { } assertEquals(3, metricCnt); - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/users/user1/flows/flow_name/runs?" + "metricstoretrieve=!(MAP_,HDFS_)"); resp = getResponse(client, uri); @@ -704,17 +656,17 @@ public class TestTimelineReaderWebServicesHBaseStorage { Client client = createClient(); try { // Query all flows. - URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + URI uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/flows"); ClientResponse resp = getResponse(client, uri); Set flowEntities = resp.getEntity(new GenericType>(){}); assertNotNull(flowEntities); - assertEquals(2, flowEntities.size()); + assertEquals(3, flowEntities.size()); List listFlowUIDs = new ArrayList(); for (FlowActivityEntity entity : flowEntities) { String flowUID = - (String)entity.getInfo().get(TimelineReaderManager.UID_KEY); + (String) entity.getInfo().get(TimelineReaderUtils.UID_KEY); listFlowUIDs.add(flowUID); assertEquals(TimelineUIDConverter.FLOW_UID.encodeUID( new TimelineReaderContext(entity.getCluster(), entity.getUser(), @@ -722,13 +674,15 @@ public class TestTimelineReaderWebServicesHBaseStorage { assertTrue((entity.getId().endsWith("@flow_name") && entity.getFlowRuns().size() == 2) || (entity.getId().endsWith("@flow_name2") && - entity.getFlowRuns().size() == 1)); + entity.getFlowRuns().size() == 1) + || (entity.getId().endsWith("@flow1") + && entity.getFlowRuns().size() == 3)); } // Query flowruns based on UID returned in query above. List listFlowRunUIDs = new ArrayList(); for (String flowUID : listFlowUIDs) { - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/flow-uid/" + flowUID + "/runs"); resp = getResponse(client, uri); Set frEntities = @@ -736,7 +690,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { assertNotNull(frEntities); for (FlowRunEntity entity : frEntities) { String flowRunUID = - (String)entity.getInfo().get(TimelineReaderManager.UID_KEY); + (String) entity.getInfo().get(TimelineReaderUtils.UID_KEY); listFlowRunUIDs.add(flowRunUID); assertEquals(TimelineUIDConverter.FLOWRUN_UID.encodeUID( new TimelineReaderContext("cluster1", entity.getUser(), @@ -744,11 +698,11 @@ public class TestTimelineReaderWebServicesHBaseStorage { flowRunUID); } } - assertEquals(3, listFlowRunUIDs.size()); + assertEquals(6, listFlowRunUIDs.size()); // Query single flowrun based on UIDs' returned in query to get flowruns. for (String flowRunUID : listFlowRunUIDs) { - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/run-uid/" + flowRunUID); resp = getResponse(client, uri); FlowRunEntity entity = resp.getEntity(FlowRunEntity.class); @@ -760,7 +714,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { for (String flowRunUID : listFlowRunUIDs) { TimelineReaderContext context = TimelineUIDConverter.FLOWRUN_UID.decodeUID(flowRunUID); - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/run-uid/" + flowRunUID + "/apps"); resp = getResponse(client, uri); Set appEntities = @@ -768,7 +722,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { assertNotNull(appEntities); for (TimelineEntity entity : appEntities) { String appUID = - (String)entity.getInfo().get(TimelineReaderManager.UID_KEY); + (String) entity.getInfo().get(TimelineReaderUtils.UID_KEY); listAppUIDs.add(appUID); assertEquals(TimelineUIDConverter.APPLICATION_UID.encodeUID( new TimelineReaderContext(context.getClusterId(), @@ -776,11 +730,11 @@ public class TestTimelineReaderWebServicesHBaseStorage { context.getFlowRunId(), entity.getId(), null, null)), appUID); } } - assertEquals(4, listAppUIDs.size()); + assertEquals(19, listAppUIDs.size()); // Query single app based on UIDs' returned in query to get apps. for (String appUID : listAppUIDs) { - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/app-uid/" + appUID); resp = getResponse(client, uri); TimelineEntity entity = resp.getEntity(TimelineEntity.class); @@ -793,7 +747,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { for (String appUID : listAppUIDs) { TimelineReaderContext context = TimelineUIDConverter.APPLICATION_UID.decodeUID(appUID); - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/app-uid/" + appUID + "/entities/type1"); resp = getResponse(client, uri); Set entities = @@ -801,12 +755,13 @@ public class TestTimelineReaderWebServicesHBaseStorage { assertNotNull(entities); for (TimelineEntity entity : entities) { String entityUID = - (String)entity.getInfo().get(TimelineReaderManager.UID_KEY); + (String) entity.getInfo().get(TimelineReaderUtils.UID_KEY); listEntityUIDs.add(entityUID); assertEquals(TimelineUIDConverter.GENERIC_ENTITY_UID.encodeUID( new TimelineReaderContext(context.getClusterId(), context.getUserId(), context.getFlowName(), - context.getFlowRunId(), context.getAppId(), "type1", + context.getFlowRunId(), context.getAppId(), "type1", + entity.getIdPrefix(), entity.getId())), entityUID); } } @@ -814,39 +769,39 @@ public class TestTimelineReaderWebServicesHBaseStorage { // Query single entity based on UIDs' returned in query to get entities. for (String entityUID : listEntityUIDs) { - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/entity-uid/" + entityUID); resp = getResponse(client, uri); TimelineEntity entity = resp.getEntity(TimelineEntity.class); assertNotNull(entity); } - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/flow-uid/dummy:flow/runs"); verifyHttpResponse(client, uri, Status.BAD_REQUEST); - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/run-uid/dummy:flowrun"); verifyHttpResponse(client, uri, Status.BAD_REQUEST); // Run Id is not a numerical value. - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/run-uid/some:dummy:flow:123v456"); verifyHttpResponse(client, uri, Status.BAD_REQUEST); - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/run-uid/dummy:flowrun/apps"); verifyHttpResponse(client, uri, Status.BAD_REQUEST); - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/app-uid/dummy:app"); verifyHttpResponse(client, uri, Status.BAD_REQUEST); - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/app-uid/dummy:app/entities/type1"); verifyHttpResponse(client, uri, Status.BAD_REQUEST); - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/entity-uid/dummy:entity"); verifyHttpResponse(client, uri, Status.BAD_REQUEST); } finally { @@ -860,7 +815,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { try { String appUIDWithFlowInfo = "cluster1!user1!flow_name!1002345678919!application_1111111111_1111"; - URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/"+ + URI uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/"+ "timeline/app-uid/" + appUIDWithFlowInfo); ClientResponse resp = getResponse(client, uri); TimelineEntity appEntity1 = resp.getEntity(TimelineEntity.class); @@ -869,8 +824,9 @@ public class TestTimelineReaderWebServicesHBaseStorage { TimelineEntityType.YARN_APPLICATION.toString(), appEntity1.getType()); assertEquals("application_1111111111_1111", appEntity1.getId()); - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" + - "app-uid/" + appUIDWithFlowInfo + "/entities/type1"); + uri = + URI.create("http://localhost:" + getServerPort() + "/ws/v2/timeline/" + + "app-uid/" + appUIDWithFlowInfo + "/entities/type1"); resp = getResponse(client, uri); Set entities1 = resp.getEntity(new GenericType>(){}); @@ -878,17 +834,17 @@ public class TestTimelineReaderWebServicesHBaseStorage { assertEquals(2, entities1.size()); for (TimelineEntity entity : entities1) { assertNotNull(entity.getInfo()); - assertEquals(1, entity.getInfo().size()); + assertEquals(2, entity.getInfo().size()); String uid = - (String) entity.getInfo().get(TimelineReaderManager.UID_KEY); + (String) entity.getInfo().get(TimelineReaderUtils.UID_KEY); assertNotNull(uid); - assertTrue(uid.equals(appUIDWithFlowInfo + "!type1!entity1") || - uid.equals(appUIDWithFlowInfo + "!type1!entity2")); + assertTrue(uid.equals(appUIDWithFlowInfo + "!type1!0!entity1") + || uid.equals(appUIDWithFlowInfo + "!type1!0!entity2")); } String appUIDWithoutFlowInfo = "cluster1!application_1111111111_1111"; - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/"+ - "app-uid/" + appUIDWithoutFlowInfo); + uri = URI.create("http://localhost:" + getServerPort() + + "/ws/v2/timeline/" + "app-uid/" + appUIDWithoutFlowInfo); resp = getResponse(client, uri); TimelineEntity appEntity2 = resp.getEntity(TimelineEntity.class); assertNotNull(appEntity2); @@ -896,8 +852,9 @@ public class TestTimelineReaderWebServicesHBaseStorage { TimelineEntityType.YARN_APPLICATION.toString(), appEntity2.getType()); assertEquals("application_1111111111_1111", appEntity2.getId()); - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" + - "app-uid/" + appUIDWithoutFlowInfo + "/entities/type1"); + uri = + URI.create("http://localhost:" + getServerPort() + "/ws/v2/timeline/" + + "app-uid/" + appUIDWithoutFlowInfo + "/entities/type1"); resp = getResponse(client, uri); Set entities2 = resp.getEntity(new GenericType>(){}); @@ -905,17 +862,17 @@ public class TestTimelineReaderWebServicesHBaseStorage { assertEquals(2, entities2.size()); for (TimelineEntity entity : entities2) { assertNotNull(entity.getInfo()); - assertEquals(1, entity.getInfo().size()); + assertEquals(2, entity.getInfo().size()); String uid = - (String) entity.getInfo().get(TimelineReaderManager.UID_KEY); + (String) entity.getInfo().get(TimelineReaderUtils.UID_KEY); assertNotNull(uid); - assertTrue(uid.equals(appUIDWithoutFlowInfo + "!type1!entity1") || - uid.equals(appUIDWithoutFlowInfo + "!type1!entity2")); + assertTrue(uid.equals(appUIDWithoutFlowInfo + "!type1!0!entity1") + || uid.equals(appUIDWithoutFlowInfo + "!type1!0!entity2")); } - String entityUIDWithFlowInfo = appUIDWithFlowInfo + "!type1!entity1"; - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/"+ - "entity-uid/" + entityUIDWithFlowInfo); + String entityUIDWithFlowInfo = appUIDWithFlowInfo + "!type1!0!entity1"; + uri = URI.create("http://localhost:" + getServerPort() + + "/ws/v2/timeline/" + "entity-uid/" + entityUIDWithFlowInfo); resp = getResponse(client, uri); TimelineEntity singleEntity1 = resp.getEntity(TimelineEntity.class); assertNotNull(singleEntity1); @@ -923,9 +880,9 @@ public class TestTimelineReaderWebServicesHBaseStorage { assertEquals("entity1", singleEntity1.getId()); String entityUIDWithoutFlowInfo = - appUIDWithoutFlowInfo + "!type1!entity1"; - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/"+ - "entity-uid/" + entityUIDWithoutFlowInfo); + appUIDWithoutFlowInfo + "!type1!0!entity1"; + uri = URI.create("http://localhost:" + getServerPort() + + "/ws/v2/timeline/" + "entity-uid/" + entityUIDWithoutFlowInfo); resp = getResponse(client, uri); TimelineEntity singleEntity2 = resp.getEntity(TimelineEntity.class); assertNotNull(singleEntity2); @@ -942,7 +899,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { try { String appUID = "cluster1!user*1!flow_name!1002345678919!application_1111111111_1111"; - URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/"+ + URI uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/"+ "timeline/app-uid/" + appUID); verifyHttpResponse(client, uri, Status.BAD_REQUEST); } finally { @@ -954,91 +911,64 @@ public class TestTimelineReaderWebServicesHBaseStorage { public void testGetFlows() throws Exception { Client client = createClient(); try { - URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + URI uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/flows"); - ClientResponse resp = getResponse(client, uri); - Set entities = - resp.getEntity(new GenericType>(){}); - assertNotNull(entities); - assertEquals(2, entities.size()); - for (FlowActivityEntity entity : entities) { - assertTrue((entity.getId().endsWith("@flow_name") && - entity.getFlowRuns().size() == 2) || - (entity.getId().endsWith("@flow_name2") && - entity.getFlowRuns().size() == 1)); - } + + verifyFlowEntites(client, uri, 3, new int[] {3, 2, 1}, + new String[] {"flow1", "flow_name", "flow_name2"}); // Query without specifying cluster ID. - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/flows/"); - resp = getResponse(client, uri); - entities = resp.getEntity(new GenericType>(){}); - assertNotNull(entities); - assertEquals(2, entities.size()); + verifyFlowEntites(client, uri, 3, new int[] {3, 2, 1}, + new String[] {"flow1", "flow_name", "flow_name2"}); - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/flows?limit=1"); - resp = getResponse(client, uri); - entities = resp.getEntity(new GenericType>(){}); - assertNotNull(entities); - assertEquals(1, entities.size()); + verifyFlowEntites(client, uri, 1, new int[] {3}, + new String[] {"flow1"}); long firstFlowActivity = HBaseTimelineStorageUtils.getTopOfTheDayTimestamp(1425016501000L); DateFormat fmt = TimelineReaderWebServices.DATE_FORMAT.get(); - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/flows?daterange=" + fmt.format(firstFlowActivity) + "-" + fmt.format(dayTs)); - resp = getResponse(client, uri); - entities = resp.getEntity(new GenericType>(){}); - assertNotNull(entities); - assertEquals(2, entities.size()); - for (FlowActivityEntity entity : entities) { - assertTrue((entity.getId().endsWith("@flow_name") && - entity.getFlowRuns().size() == 2) || - (entity.getId().endsWith("@flow_name2") && - entity.getFlowRuns().size() == 1)); - } + verifyFlowEntites(client, uri, 3, new int[] {3, 2, 1}, + new String[] {"flow1", "flow_name", "flow_name2"}); - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/flows?daterange=" + fmt.format(dayTs + (4*86400000L))); - resp = getResponse(client, uri); - entities = resp.getEntity(new GenericType>(){}); - assertNotNull(entities); - assertEquals(0, entities.size()); + verifyFlowEntites(client, uri, 0, new int[] {}, new String[] {}); - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/flows?daterange=-" + fmt.format(dayTs)); - resp = getResponse(client, uri); - entities = resp.getEntity(new GenericType>(){}); - assertNotNull(entities); - assertEquals(2, entities.size()); + verifyFlowEntites(client, uri, 3, new int[] {3, 2, 1}, + new String[] {"flow1", "flow_name", "flow_name2"}); - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/flows?daterange=" + fmt.format(firstFlowActivity) + "-"); - resp = getResponse(client, uri); - entities = resp.getEntity(new GenericType>(){}); - assertNotNull(entities); - assertEquals(2, entities.size()); + verifyFlowEntites(client, uri, 3, new int[] {3, 2, 1}, + new String[] {"flow1", "flow_name", "flow_name2"}); - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/flows?daterange=20150711:20150714"); verifyHttpResponse(client, uri, Status.BAD_REQUEST); - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/flows?daterange=20150714-20150711"); verifyHttpResponse(client, uri, Status.BAD_REQUEST); - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/flows?daterange=2015071129-20150712"); verifyHttpResponse(client, uri, Status.BAD_REQUEST); - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/flows?daterange=20150711-2015071243"); verifyHttpResponse(client, uri, Status.BAD_REQUEST); } finally { @@ -1046,11 +976,48 @@ public class TestTimelineReaderWebServicesHBaseStorage { } } + @Test + public void testGetFlowsForPagination() throws Exception { + Client client = createClient(); + int noOfEntities = 3; + int limit = 2; + try { + String flowURI = "http://localhost:" + getServerPort() + "/ws/v2/" + + "timeline/clusters/cluster1/flows"; + URI uri = URI.create(flowURI); + List flowEntites = + verifyFlowEntites(client, uri, 3, new int[] {3, 2, 1}, + new String[] {"flow1", "flow_name", "flow_name2"}); + FlowActivityEntity fEntity1 = flowEntites.get(0); + FlowActivityEntity fEntity3 = flowEntites.get(noOfEntities - 1); + + uri = URI.create(flowURI + "?limit=" + limit); + flowEntites = verifyFlowEntites(client, uri, limit); + assertEquals(fEntity1, flowEntites.get(0)); + FlowActivityEntity fEntity2 = flowEntites.get(limit - 1); + + uri = URI + .create(flowURI + "?limit=" + limit + "&fromid=" + + fEntity2.getInfo().get(TimelineReaderUtils.FROMID_KEY)); + flowEntites = verifyFlowEntites(client, uri, noOfEntities - limit + 1); + assertEquals(fEntity2, flowEntites.get(0)); + assertEquals(fEntity3, flowEntites.get(noOfEntities - limit)); + + uri = URI + .create(flowURI + "?limit=" + limit + "&fromid=" + + fEntity3.getInfo().get(TimelineReaderUtils.FROMID_KEY)); + flowEntites = verifyFlowEntites(client, uri, 1); + assertEquals(fEntity3, flowEntites.get(0)); + } finally { + client.destroy(); + } + } + @Test public void testGetApp() throws Exception { Client client = createClient(); try { - URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + URI uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1111?" + "userid=user1&fields=ALL&flowname=flow_name&flowrunid=1002345678919"); ClientResponse resp = getResponse(client, uri); @@ -1068,7 +1035,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { assertTrue(verifyMetrics(metric, m1, m2, m3)); } - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/apps/application_1111111111_2222?userid=user1" + "&fields=metrics&flowname=flow_name&flowrunid=1002345678919"); resp = getResponse(client, uri); @@ -1090,7 +1057,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { public void testGetAppWithoutFlowInfo() throws Exception { Client client = createClient(); try { - URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + URI uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1111?" + "fields=ALL"); ClientResponse resp = getResponse(client, uri); @@ -1109,7 +1076,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { assertTrue(verifyMetrics(metric, m1, m2, m3)); } - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1111?" + "fields=ALL&metricslimit=10"); resp = getResponse(client, uri); @@ -1139,7 +1106,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { public void testGetEntityWithoutFlowInfo() throws Exception { Client client = createClient(); try { - URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + URI uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + "entities/type1/entity1"); ClientResponse resp = getResponse(client, uri); @@ -1156,7 +1123,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { public void testGetEntitiesWithoutFlowInfo() throws Exception { Client client = createClient(); try { - URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + URI uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + "entities/type1"); ClientResponse resp = getResponse(client, uri); @@ -1180,7 +1147,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { public void testGetEntitiesDataToRetrieve() throws Exception { Client client = createClient(); try { - URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + URI uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + "entities/type1?confstoretrieve=cfg_"); ClientResponse resp = getResponse(client, uri); @@ -1197,7 +1164,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { } assertEquals(2, cfgCnt); - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + "entities/type1?confstoretrieve=cfg_,config_"); resp = getResponse(client, uri); @@ -1214,7 +1181,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { } assertEquals(5, cfgCnt); - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + "entities/type1?confstoretrieve=!(cfg_,config_)"); resp = getResponse(client, uri); @@ -1230,7 +1197,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { } assertEquals(1, cfgCnt); - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + "entities/type1?metricstoretrieve=MAP_"); resp = getResponse(client, uri); @@ -1246,7 +1213,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { } assertEquals(1, metricCnt); - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + "entities/type1?metricstoretrieve=MAP1_,HDFS_"); resp = getResponse(client, uri); @@ -1263,7 +1230,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { } assertEquals(3, metricCnt); - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + "entities/type1?metricstoretrieve=!(MAP1_,HDFS_)"); resp = getResponse(client, uri); @@ -1288,7 +1255,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { public void testGetEntitiesConfigFilters() throws Exception { Client client = createClient(); try { - URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + URI uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + "entities/type1?conffilters=config_param1%20eq%20value1%20OR%20" + "config_param1%20eq%20value3"); @@ -1302,7 +1269,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { entity.getId().equals("entity2")); } - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + "entities/type1?conffilters=config_param1%20eq%20value1%20AND" + "%20configuration_param2%20eq%20value2"); @@ -1313,7 +1280,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { // conffilters=(config_param1 eq value1 AND configuration_param2 eq // value2) OR (config_param1 eq value3 AND cfg_param3 eq value1) - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + "entities/type1?conffilters=(config_param1%20eq%20value1%20AND" + "%20configuration_param2%20eq%20value2)%20OR%20(config_param1%20eq" + @@ -1331,7 +1298,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { // conffilters=(config_param1 eq value1 AND configuration_param2 eq // value2) OR (config_param1 eq value3 AND cfg_param3 eq value1) - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + "entities/type1?conffilters=(config_param1%20eq%20value1%20AND" + "%20configuration_param2%20eq%20value2)%20OR%20(config_param1%20eq" + @@ -1347,7 +1314,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { } assertEquals(3, cfgCnt); - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + "entities/type1?conffilters=(config_param1%20eq%20value1%20AND" + "%20configuration_param2%20eq%20value2)%20OR%20(config_param1%20eq" + @@ -1373,7 +1340,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { // entity1. For ne, both entity1 and entity2 will be returned. For ene, // only entity2 will be returned as we are checking for existence too. // conffilters=configuration_param2 ne value3 - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + "entities/type1?conffilters=configuration_param2%20ne%20value3"); resp = getResponse(client, uri); @@ -1385,7 +1352,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { entity.getId().equals("entity2")); } // conffilters=configuration_param2 ene value3 - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + "entities/type1?conffilters=configuration_param2%20ene%20value3"); resp = getResponse(client, uri); @@ -1405,7 +1372,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { Client client = createClient(); try { // infofilters=info1 eq cluster1 OR info1 eq cluster2 - URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + URI uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + "entities/type1?infofilters=info1%20eq%20cluster1%20OR%20info1%20eq" + "%20cluster2"); @@ -1420,7 +1387,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { } // infofilters=info1 eq cluster1 AND info4 eq 35000 - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + "entities/type1?infofilters=info1%20eq%20cluster1%20AND%20info4%20" + "eq%2035000"); @@ -1430,7 +1397,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { assertEquals(0, entities.size()); // infofilters=info4 eq 35000 OR info4 eq 36000 - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + "entities/type1?infofilters=info4%20eq%2035000%20OR%20info4%20eq" + "%2036000"); @@ -1445,7 +1412,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { // infofilters=(info1 eq cluster1 AND info4 eq 35000) OR // (info1 eq cluster2 AND info2 eq 2.0) - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + "entities/type1?infofilters=(info1%20eq%20cluster1%20AND%20info4%20" + "eq%2035000)%20OR%20(info1%20eq%20cluster2%20AND%20info2%20eq%202.0" + @@ -1459,12 +1426,13 @@ public class TestTimelineReaderWebServicesHBaseStorage { infoCnt += entity.getInfo().size(); assertEquals("entity2", entity.getId()); } - // Includes UID in info field even if fields not specified as INFO. - assertEquals(1, infoCnt); + // Includes UID and FROM_ID in info field even if fields not specified as + // INFO. + assertEquals(2, infoCnt); // infofilters=(info1 eq cluster1 AND info4 eq 35000) OR // (info1 eq cluster2 AND info2 eq 2.0) - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + "entities/type1?infofilters=(info1%20eq%20cluster1%20AND%20info4%20" + "eq%2035000)%20OR%20(info1%20eq%20cluster2%20AND%20info2%20eq%20" + @@ -1478,15 +1446,15 @@ public class TestTimelineReaderWebServicesHBaseStorage { infoCnt += entity.getInfo().size(); assertEquals("entity2", entity.getId()); } - // Includes UID in info field. - assertEquals(4, infoCnt); + // Includes UID and FROM_ID in info field. + assertEquals(5, infoCnt); // Test for behavior when compare op is ne(not equals) vs ene // (exists and not equals). info3 does not exist for entity2. For ne, // both entity1 and entity2 will be returned. For ene, only entity2 will // be returned as we are checking for existence too. // infofilters=info3 ne 39000 - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + "entities/type1?infofilters=info3%20ne%2039000"); resp = getResponse(client, uri); @@ -1498,7 +1466,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { entity.getId().equals("entity2")); } // infofilters=info3 ene 39000 - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + "entities/type1?infofilters=info3%20ene%2039000"); resp = getResponse(client, uri); @@ -1518,7 +1486,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { Client client = createClient(); try { // metricfilters=HDFS_BYTES_READ lt 60 OR HDFS_BYTES_READ eq 157 - URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + URI uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + "entities/type1?metricfilters=HDFS_BYTES_READ%20lt%2060%20OR%20" + "HDFS_BYTES_READ%20eq%20157"); @@ -1533,7 +1501,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { } // metricfilters=HDFS_BYTES_READ lt 60 AND MAP_SLOT_MILLIS gt 40 - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + "entities/type1?metricfilters=HDFS_BYTES_READ%20lt%2060%20AND%20" + "MAP_SLOT_MILLIS%20gt%2040"); @@ -1544,7 +1512,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { // metricfilters=(HDFS_BYTES_READ lt 60 AND MAP_SLOT_MILLIS gt 40) OR // (MAP1_SLOT_MILLIS ge 140 AND MAP11_SLOT_MILLIS le 122) - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + "entities/type1?metricfilters=(HDFS_BYTES_READ%20lt%2060%20AND%20" + "MAP_SLOT_MILLIS%20gt%2040)%20OR%20(MAP1_SLOT_MILLIS%20ge" + @@ -1562,7 +1530,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { // metricfilters=(HDFS_BYTES_READ lt 60 AND MAP_SLOT_MILLIS gt 40) OR // (MAP1_SLOT_MILLIS ge 140 AND MAP11_SLOT_MILLIS le 122) - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + "entities/type1?metricfilters=(HDFS_BYTES_READ%20lt%2060%20AND%20" + "MAP_SLOT_MILLIS%20gt%2040)%20OR%20(MAP1_SLOT_MILLIS%20ge" + @@ -1580,7 +1548,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { // metricfilters=(HDFS_BYTES_READ lt 60 AND MAP_SLOT_MILLIS gt 40) OR // (MAP1_SLOT_MILLIS ge 140 AND MAP11_SLOT_MILLIS le 122) - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + "entities/type1?metricfilters=(HDFS_BYTES_READ%20lt%2060%20AND%20" + "MAP_SLOT_MILLIS%20gt%2040)%20OR%20(MAP1_SLOT_MILLIS%20ge" + @@ -1601,7 +1569,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { } assertEquals(2, metricCnt); - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + "entities/type1?metricfilters=(HDFS_BYTES_READ%20lt%2060%20AND%20" + "MAP_SLOT_MILLIS%20gt%2040)%20OR%20(MAP1_SLOT_MILLIS%20ge" + @@ -1634,7 +1602,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { // entity1. For ne, both entity1 and entity2 will be returned. For ene, // only entity2 will be returned as we are checking for existence too. // metricfilters=MAP11_SLOT_MILLIS ne 100 - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + "entities/type1?metricfilters=MAP11_SLOT_MILLIS%20ne%20100"); resp = getResponse(client, uri); @@ -1646,7 +1614,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { entity.getId().equals("entity2")); } // metricfilters=MAP11_SLOT_MILLIS ene 100 - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + "entities/type1?metricfilters=MAP11_SLOT_MILLIS%20ene%20100"); resp = getResponse(client, uri); @@ -1665,7 +1633,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { public void testGetEntitiesEventFilters() throws Exception { Client client = createClient(); try { - URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + URI uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + "entities/type1?eventfilters=event1,event3"); ClientResponse resp = getResponse(client, uri); @@ -1678,7 +1646,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { entity.getId().equals("entity2")); } - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + "entities/type1?eventfilters=!(event1,event3)"); resp = getResponse(client, uri); @@ -1687,7 +1655,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { assertEquals(0, entities.size()); // eventfilters=!(event1,event3) OR event5,event6 - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + "entities/type1?eventfilters=!(event1,event3)%20OR%20event5,event6"); resp = getResponse(client, uri); @@ -1700,7 +1668,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { // eventfilters=(!(event1,event3) OR event5,event6) OR // (event1,event2 AND (event3,event4)) - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + "entities/type1?eventfilters=(!(event1,event3)%20OR%20event5," + "event6)%20OR%20(event1,event2%20AND%20(event3,event4))"); @@ -1721,7 +1689,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { public void testGetEntitiesRelationFilters() throws Exception { Client client = createClient(); try { - URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + URI uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + "entities/type1?isrelatedto=type3:entity31,type2:entity21:entity22"); ClientResponse resp = getResponse(client, uri); @@ -1734,7 +1702,8 @@ public class TestTimelineReaderWebServicesHBaseStorage { entity.getId().equals("entity2")); } - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" + + uri = URI.create("http://localhost:" + getServerPort() + + "/ws/v2/timeline/" + "clusters/cluster1/apps/application_1111111111_1111/entities/type1" + "?isrelatedto=!(type3:entity31,type2:entity21:entity22)"); resp = getResponse(client, uri); @@ -1744,7 +1713,8 @@ public class TestTimelineReaderWebServicesHBaseStorage { // isrelatedto=!(type3:entity31,type2:entity21:entity22)OR type5:entity51, // type6:entity61:entity66 - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" + + uri = URI.create("http://localhost:" + getServerPort() + + "/ws/v2/timeline/" + "clusters/cluster1/apps/application_1111111111_1111/entities/type1" + "?isrelatedto=!(type3:entity31,type2:entity21:entity22)%20OR%20" + "type5:entity51,type6:entity61:entity66"); @@ -1759,7 +1729,8 @@ public class TestTimelineReaderWebServicesHBaseStorage { // isrelatedto=(!(type3:entity31,type2:entity21:entity22)OR type5: // entity51,type6:entity61:entity66) OR (type1:entity14,type2:entity21: // entity22 AND (type3:entity32:entity35,type4:entity42)) - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" + + uri = URI.create("http://localhost:" + getServerPort() + + "/ws/v2/timeline/" + "clusters/cluster1/apps/application_1111111111_1111/entities/type1" + "?isrelatedto=(!(type3:entity31,type2:entity21:entity22)%20OR%20" + "type5:entity51,type6:entity61:entity66)%20OR%20(type1:entity14," + @@ -1776,7 +1747,8 @@ public class TestTimelineReaderWebServicesHBaseStorage { // relatesto=!(type3:entity31,type2:entity21:entity22)OR type5:entity51, // type6:entity61:entity66 - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" + + uri = URI.create("http://localhost:" + getServerPort() + + "/ws/v2/timeline/" + "clusters/cluster1/apps/application_1111111111_1111/entities/type1" + "?relatesto=!%20(type3:entity31,type2:entity21:entity22%20)%20OR%20" + "type5:entity51,type6:entity61:entity66"); @@ -1791,7 +1763,9 @@ public class TestTimelineReaderWebServicesHBaseStorage { // relatesto=(!(type3:entity31,type2:entity21:entity22)OR type5:entity51, // type6:entity61:entity66) OR (type1:entity14,type2:entity21:entity22 AND // (type3:entity32:entity35 , type4:entity42)) - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" + + uri = + URI.create("http://localhost:" + getServerPort() + + "/ws/v2/timeline/" + "clusters/cluster1/apps/application_1111111111_1111/entities/type1" + "?relatesto=(!(%20type3:entity31,type2:entity21:entity22)%20OR%20" + "type5:entity51,type6:entity61:entity66%20)%20OR%20(type1:entity14," + @@ -1810,6 +1784,113 @@ public class TestTimelineReaderWebServicesHBaseStorage { } } + private static void verifyMetricCount(TimelineEntity entity, + int expectedMetricsCnt, int expectedMeticsValCnt) { + int metricsValCnt = 0; + for (TimelineMetric m : entity.getMetrics()) { + metricsValCnt += m.getValues().size(); + } + assertEquals(expectedMetricsCnt, entity.getMetrics().size()); + assertEquals(expectedMeticsValCnt, metricsValCnt); + } + + private static void verifyMetricsCount(Set entities, + int expectedMetricsCnt, int expectedMeticsValCnt) { + int metricsCnt = 0; + int metricsValCnt = 0; + for (TimelineEntity entity : entities) { + metricsCnt += entity.getMetrics().size(); + for (TimelineMetric m : entity.getMetrics()) { + metricsValCnt += m.getValues().size(); + } + } + assertEquals(expectedMetricsCnt, metricsCnt); + assertEquals(expectedMeticsValCnt, metricsValCnt); + } + + @Test + public void testGetEntitiesMetricsTimeRange() throws Exception { + Client client = createClient(); + try { + URI uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + + "entities/type1?fields=ALL&metricslimit=100&metricstimestart=" + + (ts - 90000) + "&metricstimeend=" + (ts - 80000)); + ClientResponse resp = getResponse(client, uri); + Set entities = + resp.getEntity(new GenericType>(){}); + assertNotNull(entities); + assertEquals(2, entities.size()); + verifyMetricsCount(entities, 4, 4); + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + + "entities/type1?fields=ALL&metricslimit=100&metricstimestart=" + + (ts - 100000) + "&metricstimeend=" + (ts - 80000)); + resp = getResponse(client, uri); + entities = resp.getEntity(new GenericType>(){}); + assertNotNull(entities); + assertEquals(2, entities.size()); + verifyMetricsCount(entities, 5, 9); + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + + "entities/type1?fields=ALL&metricslimit=100&metricstimestart=" + + (ts - 100000)); + resp = getResponse(client, uri); + entities = resp.getEntity(new GenericType>(){}); + assertNotNull(entities); + assertEquals(2, entities.size()); + verifyMetricsCount(entities, 5, 9); + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + + "entities/type1?fields=ALL&metricslimit=100&metricstimeend=" + + (ts - 90000)); + resp = getResponse(client, uri); + entities = resp.getEntity(new GenericType>(){}); + assertNotNull(entities); + assertEquals(2, entities.size()); + verifyMetricsCount(entities, 5, 5); + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + + "entities/type1?fields=ALL&metricstimestart=" + + (ts - 100000)); + resp = getResponse(client, uri); + entities = resp.getEntity(new GenericType>(){}); + assertNotNull(entities); + assertEquals(2, entities.size()); + verifyMetricsCount(entities, 5, 5); + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + + "entities/type1/entity2?fields=ALL&metricstimestart=" + + (ts - 100000) + "&metricstimeend=" + (ts - 80000)); + resp = getResponse(client, uri); + TimelineEntity entity = resp.getEntity(TimelineEntity.class); + assertNotNull(entity); + verifyMetricCount(entity, 3, 3); + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + + "entities/type1/entity2?fields=ALL&metricslimit=5&metricstimestart=" + + (ts - 100000) + "&metricstimeend=" + (ts - 80000)); + resp = getResponse(client, uri); + entity = resp.getEntity(TimelineEntity.class); + assertNotNull(entity); + verifyMetricCount(entity, 3, 5); + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + + "entities/type1?fields=ALL&metricslimit=100&metricstimestart=" + + (ts - 80000) + "&metricstimeend=" + (ts - 90000)); + verifyHttpResponse(client, uri, Status.BAD_REQUEST); + } finally { + client.destroy(); + } + } + /** * Tests if specific configs and metrics are retrieve for getEntity call. */ @@ -1817,7 +1898,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { public void testGetEntityDataToRetrieve() throws Exception { Client client = createClient(); try { - URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + URI uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + "entities/type1/entity2?confstoretrieve=cfg_,configuration_"); ClientResponse resp = getResponse(client, uri); @@ -1831,7 +1912,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { configKey.startsWith("cfg_")); } - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + "entities/type1/entity2?confstoretrieve=!(cfg_,configuration_)"); resp = getResponse(client, uri); @@ -1844,7 +1925,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { assertTrue(configKey.startsWith("config_")); } - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + "entities/type1/entity2?metricstoretrieve=MAP1_,HDFS_"); resp = getResponse(client, uri); @@ -1858,7 +1939,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { metric.getId().startsWith("HDFS_")); } - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + "entities/type1/entity2?metricstoretrieve=!(MAP1_,HDFS_)"); resp = getResponse(client, uri); @@ -1873,7 +1954,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { assertEquals(1, metric.getValues().size()); } - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + "entities/type1/entity2?metricstoretrieve=!(MAP1_,HDFS_)&" + "metricslimit=5"); @@ -1896,7 +1977,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { public void testGetFlowRunApps() throws Exception { Client client = createClient(); try { - URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + URI uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/users/user1/flows/flow_name/runs/" + "1002345678919/apps?fields=ALL"); ClientResponse resp = getResponse(client, uri); @@ -1916,7 +1997,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { } } - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/users/user1/flows/flow_name/runs/" + "1002345678919/apps?fields=ALL&metricslimit=2"); resp = getResponse(client, uri); @@ -1936,14 +2017,14 @@ public class TestTimelineReaderWebServicesHBaseStorage { } // Query without specifying cluster ID. - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/users/user1/flows/flow_name/runs/1002345678919/apps"); resp = getResponse(client, uri); entities = resp.getEntity(new GenericType>(){}); assertNotNull(entities); assertEquals(2, entities.size()); - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/users/user1/flows/flow_name/runs/1002345678919/" + "apps?limit=1"); resp = getResponse(client, uri); @@ -1959,7 +2040,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { public void testGetFlowApps() throws Exception { Client client = createClient(); try { - URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + URI uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/users/user1/flows/flow_name/apps?" + "fields=ALL"); ClientResponse resp = getResponse(client, uri); @@ -1998,7 +2079,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { } } - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/users/user1/flows/flow_name/apps?" + "fields=ALL&metricslimit=6"); resp = getResponse(client, uri); @@ -2042,14 +2123,14 @@ public class TestTimelineReaderWebServicesHBaseStorage { } // Query without specifying cluster ID. - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/users/user1/flows/flow_name/apps"); resp = getResponse(client, uri); entities = resp.getEntity(new GenericType>(){}); assertNotNull(entities); assertEquals(3, entities.size()); - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/users/user1/flows/flow_name/apps?limit=1"); resp = getResponse(client, uri); entities = resp.getEntity(new GenericType>(){}); @@ -2065,7 +2146,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { Client client = createClient(); try { String entityType = TimelineEntityType.YARN_APPLICATION.toString(); - URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + URI uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/users/user1/flows/flow_name/apps?" + "eventfilters=" + ApplicationMetricsConstants.FINISHED_EVENT_TYPE); ClientResponse resp = getResponse(client, uri); @@ -2076,7 +2157,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { assertTrue("Unexpected app in result", entities.contains( newEntity(entityType, "application_1111111111_1111"))); - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/users/user1/flows/flow_name/apps?" + "metricfilters=HDFS_BYTES_READ%20ge%200"); resp = getResponse(client, uri); @@ -2086,7 +2167,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { assertTrue("Unexpected app in result", entities.contains( newEntity(entityType, "application_1111111111_1111"))); - uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/users/user1/flows/flow_name/apps?" + "conffilters=cfg1%20eq%20value1"); resp = getResponse(client, uri); @@ -2104,7 +2185,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { public void testGetFlowRunNotPresent() throws Exception { Client client = createClient(); try { - URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + URI uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/users/user1/flows/flow_name/runs/" + "1002345678929"); verifyHttpResponse(client, uri, Status.NOT_FOUND); @@ -2117,7 +2198,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { public void testGetFlowsNotPresent() throws Exception { Client client = createClient(); try { - URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + URI uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster2/flows"); ClientResponse resp = getResponse(client, uri); Set entities = @@ -2135,7 +2216,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { public void testGetAppNotPresent() throws Exception { Client client = createClient(); try { - URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + URI uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1378"); verifyHttpResponse(client, uri, Status.NOT_FOUND); } finally { @@ -2147,7 +2228,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { public void testGetFlowRunAppsNotPresent() throws Exception { Client client = createClient(); try { - URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + URI uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster2/users/user1/flows/flow_name/runs/" + "1002345678919/apps"); ClientResponse resp = getResponse(client, uri); @@ -2166,7 +2247,7 @@ public class TestTimelineReaderWebServicesHBaseStorage { public void testGetFlowAppsNotPresent() throws Exception { Client client = createClient(); try { - URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + URI uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + "timeline/clusters/cluster2/users/user1/flows/flow_name55/apps"); ClientResponse resp = getResponse(client, uri); Set entities = @@ -2180,11 +2261,329 @@ public class TestTimelineReaderWebServicesHBaseStorage { } } - @After - public void stop() throws Exception { - if (server != null) { - server.stop(); - server = null; + @Test + public void testGenericEntitiesForPagination() throws Exception { + Client client = createClient(); + try { + String resourceUri = "http://localhost:" + getServerPort() + "/ws/v2/" + + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + + "entities/entitytype"; + verifyEntitiesForPagination(client, resourceUri); + resourceUri = "http://localhost:" + getServerPort() + "/ws/v2/" + + "timeline/clusters/cluster1/users/" + doAsUser + + "/entities/entitytype"; + verifyEntitiesForPagination(client, resourceUri); + } finally { + client.destroy(); + } + } + + private void verifyEntitiesForPagination(Client client, String resourceUri) + throws Exception { + int limit = 10; + String queryParam = "?limit=" + limit; + URI uri = URI.create(resourceUri + queryParam); + + ClientResponse resp = getResponse(client, uri); + List entities = + resp.getEntity(new GenericType>() { + }); + // verify for entity-10 to entity-1 in descending order. + verifyPaginatedEntites(entities, limit, limit); + + limit = 4; + queryParam = "?limit=" + limit; + uri = URI.create(resourceUri + queryParam); + resp = getResponse(client, uri); + entities = resp.getEntity(new GenericType>() { + }); + // verify for entity-10 to entity-7 in descending order. + TimelineEntity entity = verifyPaginatedEntites(entities, limit, 10); + + queryParam = "?limit=" + limit + "&fromid=" + + entity.getInfo().get(TimelineReaderUtils.FROMID_KEY); + uri = URI.create(resourceUri + queryParam); + resp = getResponse(client, uri); + entities = resp.getEntity(new GenericType>() { + }); + // verify for entity-7 to entity-4 in descending order. + entity = verifyPaginatedEntites(entities, limit, 7); + + queryParam = "?limit=" + limit + "&fromid=" + + entity.getInfo().get(TimelineReaderUtils.FROMID_KEY); + uri = URI.create(resourceUri + queryParam); + resp = getResponse(client, uri); + entities = resp.getEntity(new GenericType>() { + }); + // verify for entity-4 to entity-1 in descending order. + entity = verifyPaginatedEntites(entities, limit, 4); + + queryParam = "?limit=" + limit + "&fromid=" + + entity.getInfo().get(TimelineReaderUtils.FROMID_KEY); + uri = URI.create(resourceUri + queryParam); + resp = getResponse(client, uri); + entities = resp.getEntity(new GenericType>() { + }); + // always entity-1 will be retrieved + entity = verifyPaginatedEntites(entities, 1, 1); + } + + private TimelineEntity verifyPaginatedEntites(List entities, + int limit, int startFrom) { + assertNotNull(entities); + assertEquals(limit, entities.size()); + TimelineEntity entity = null; + for (TimelineEntity timelineEntity : entities) { + assertEquals("entitytype", timelineEntity.getType()); + assertEquals("entityid-" + startFrom, timelineEntity.getId()); + assertEquals(11 - startFrom--, timelineEntity.getIdPrefix()); + entity = timelineEntity; + } + return entity; + } + + private List verifyFlowEntites(Client client, URI uri, + int noOfEntities, + int[] a, String[] flowsInSequence) throws Exception { + ClientResponse resp = getResponse(client, uri); + List entities = + resp.getEntity(new GenericType>() { + }); + assertNotNull(entities); + assertEquals(noOfEntities, entities.size()); + assertEquals(noOfEntities, flowsInSequence.length); + assertEquals(noOfEntities, a.length); + int count = 0; + for (FlowActivityEntity timelineEntity : entities) { + assertEquals(flowsInSequence[count], + timelineEntity.getInfo().get("SYSTEM_INFO_FLOW_NAME")); + assertEquals(a[count++], timelineEntity.getFlowRuns().size()); + } + return entities; + } + + @Test + public void testForFlowAppsPagination() throws Exception { + Client client = createClient(); + try { + // app entities stored is 15 during initialization. + int totalAppEntities = 15; + String resourceUri = "http://localhost:" + getServerPort() + "/ws/v2/" + + "timeline/clusters/cluster1/users/user1/flows/flow1/apps"; + URI uri = URI.create(resourceUri); + ClientResponse resp = getResponse(client, uri); + List entities = + resp.getEntity(new GenericType>() { + }); + assertNotNull(entities); + assertEquals(totalAppEntities, entities.size()); + TimelineEntity entity1 = entities.get(0); + TimelineEntity entity15 = entities.get(totalAppEntities - 1); + + int limit = 10; + String queryParam = "?limit=" + limit; + uri = URI.create(resourceUri + queryParam); + resp = getResponse(client, uri); + entities = resp.getEntity(new GenericType>() { + }); + assertNotNull(entities); + assertEquals(limit, entities.size()); + assertEquals(entity1, entities.get(0)); + TimelineEntity entity10 = entities.get(limit - 1); + + uri = + URI.create(resourceUri + queryParam + "&fromid=" + + entity10.getInfo().get(TimelineReaderUtils.FROMID_KEY)); + resp = getResponse(client, uri); + entities = resp.getEntity(new GenericType>() { + }); + assertNotNull(entities); + assertEquals(6, entities.size()); + assertEquals(entity10, entities.get(0)); + assertEquals(entity15, entities.get(5)); + + } finally { + client.destroy(); + } + } + + @Test + public void testForFlowRunAppsPagination() throws Exception { + Client client = createClient(); + try { + // app entities stored is 15 during initialization. + int totalAppEntities = 5; + String resourceUri = "http://localhost:" + getServerPort() + "/ws/v2/" + + "timeline/clusters/cluster1/users/user1/flows/flow1/runs/1/apps"; + URI uri = URI.create(resourceUri); + ClientResponse resp = getResponse(client, uri); + List entities = + resp.getEntity(new GenericType>() { + }); + assertNotNull(entities); + assertEquals(totalAppEntities, entities.size()); + TimelineEntity entity1 = entities.get(0); + TimelineEntity entity5 = entities.get(totalAppEntities - 1); + + int limit = 3; + String queryParam = "?limit=" + limit; + uri = URI.create(resourceUri + queryParam); + resp = getResponse(client, uri); + entities = resp.getEntity(new GenericType>() { + }); + assertNotNull(entities); + assertEquals(limit, entities.size()); + assertEquals(entity1, entities.get(0)); + TimelineEntity entity3 = entities.get(limit - 1); + + uri = + URI.create(resourceUri + queryParam + "&fromid=" + + entity3.getInfo().get(TimelineReaderUtils.FROMID_KEY)); + resp = getResponse(client, uri); + entities = resp.getEntity(new GenericType>() { + }); + assertNotNull(entities); + assertEquals(3, entities.size()); + assertEquals(entity3, entities.get(0)); + assertEquals(entity5, entities.get(2)); + + } finally { + client.destroy(); + } + } + + @Test + public void testForFlowRunsPagination() throws Exception { + Client client = createClient(); + try { + // app entities stored is 15 during initialization. + int totalRuns = 3; + String resourceUri = "http://localhost:" + getServerPort() + "/ws/v2/" + + "timeline/clusters/cluster1/users/user1/flows/flow1/runs"; + URI uri = URI.create(resourceUri); + ClientResponse resp = getResponse(client, uri); + List entities = + resp.getEntity(new GenericType>() { + }); + assertNotNull(entities); + assertEquals(totalRuns, entities.size()); + TimelineEntity entity1 = entities.get(0); + TimelineEntity entity3 = entities.get(totalRuns - 1); + + int limit = 2; + String queryParam = "?limit=" + limit; + uri = URI.create(resourceUri + queryParam); + resp = getResponse(client, uri); + entities = resp.getEntity(new GenericType>() { + }); + assertNotNull(entities); + assertEquals(limit, entities.size()); + assertEquals(entity1, entities.get(0)); + TimelineEntity entity2 = entities.get(limit - 1); + + uri = URI.create(resourceUri + queryParam + "&fromid=" + + entity2.getInfo().get(TimelineReaderUtils.FROMID_KEY)); + resp = getResponse(client, uri); + entities = resp.getEntity(new GenericType>() { + }); + assertNotNull(entities); + assertEquals(limit, entities.size()); + assertEquals(entity2, entities.get(0)); + assertEquals(entity3, entities.get(1)); + + uri = URI.create(resourceUri + queryParam + "&fromid=" + + entity3.getInfo().get(TimelineReaderUtils.FROMID_KEY)); + resp = getResponse(client, uri); + entities = resp.getEntity(new GenericType>() { + }); + assertNotNull(entities); + assertEquals(1, entities.size()); + assertEquals(entity3, entities.get(0)); + } finally { + client.destroy(); + } + } + + @Test + public void testGetAppsMetricsRange() throws Exception { + Client client = createClient(); + try { + URI uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + + "timeline/clusters/cluster1/users/user1/flows/flow_name/runs/" + + "1002345678919/apps?fields=ALL&metricslimit=100&metricstimestart=" + + (ts - 200000) + "&metricstimeend=" + (ts - 100000)); + ClientResponse resp = getResponse(client, uri); + Set entities = + resp.getEntity(new GenericType>(){}); + assertNotNull(entities); + assertEquals(2, entities.size()); + verifyMetricsCount(entities, 4, 4); + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + + "timeline/clusters/cluster1/users/user1/flows/flow_name/runs/" + + "1002345678919/apps?fields=ALL&metricslimit=100"); + resp = getResponse(client, uri); + entities = resp.getEntity(new GenericType>(){}); + assertNotNull(entities); + assertEquals(2, entities.size()); + verifyMetricsCount(entities, 4, 10); + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + + "timeline/clusters/cluster1/users/user1/flows/flow_name/" + + "apps?fields=ALL&metricslimit=100&metricstimestart=" + + (ts - 200000) + "&metricstimeend=" + (ts - 100000)); + resp = getResponse(client, uri); + entities = resp.getEntity(new GenericType>(){}); + assertNotNull(entities); + assertEquals(3, entities.size()); + verifyMetricsCount(entities, 5, 5); + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + + "timeline/clusters/cluster1/users/user1/flows/flow_name/" + + "apps?fields=ALL&metricslimit=100"); + resp = getResponse(client, uri); + entities = resp.getEntity(new GenericType>(){}); + assertNotNull(entities); + assertEquals(3, entities.size()); + verifyMetricsCount(entities, 5, 12); + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + + "timeline/clusters/cluster1/users/user1/flows/flow_name/runs/" + + "1002345678919/apps?fields=ALL&metricslimit=100&metricstimestart=" + + (ts - 200000)); + resp = getResponse(client, uri); + entities = resp.getEntity(new GenericType>(){}); + assertNotNull(entities); + assertEquals(2, entities.size()); + verifyMetricsCount(entities, 4, 10); + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + + "timeline/clusters/cluster1/users/user1/flows/flow_name/runs/" + + "1002345678919/apps?fields=ALL&metricslimit=100&metricstimeend=" + + (ts - 100000)); + resp = getResponse(client, uri); + entities = resp.getEntity(new GenericType>(){}); + assertNotNull(entities); + assertEquals(2, entities.size()); + verifyMetricsCount(entities, 4, 4); + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + + "timeline/apps/application_1111111111_1111?userid=user1&fields=ALL" + + "&flowname=flow_name&flowrunid=1002345678919&metricslimit=100" + + "&metricstimestart=" +(ts - 200000) + "&metricstimeend=" + + (ts - 100000)); + resp = getResponse(client, uri); + TimelineEntity entity = resp.getEntity(TimelineEntity.class); + assertNotNull(entity); + verifyMetricCount(entity, 3, 3); + + uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" + + "timeline/clusters/cluster1/users/user1/flows/flow_name/" + + "apps?fields=ALL&metricslimit=100&metricstimestart=" + + (ts - 100000) + "&metricstimeend=" + (ts - 200000)); + verifyHttpResponse(client, uri, Status.BAD_REQUEST); + } finally { + client.destroy(); } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/DataGeneratorForTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/DataGeneratorForTest.java index 5cbb78145d3..cf6a8544241 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/DataGeneratorForTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/DataGeneratorForTest.java @@ -24,16 +24,44 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric.Type; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants; +import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorContext; -final class DataGeneratorForTest { - static void loadApps(HBaseTestingUtility util) throws IOException { +/** + * Utility class that creates the schema and generates test data. + */ +public final class DataGeneratorForTest { + + // private constructor for utility class + private DataGeneratorForTest() { + } + + /** + * Creates the schema for timeline service. + * @param conf + * @throws IOException + */ + public static void createSchema(final Configuration conf) + throws IOException { + // set the jar location to null so that + // the coprocessor class is loaded from classpath + conf.set(YarnConfiguration.FLOW_RUN_COPROCESSOR_JAR_HDFS_LOCATION, " "); + // now create all tables + TimelineSchemaCreator.createAllTables(conf, false); + } + + public static void loadApps(HBaseTestingUtility util, long ts) + throws IOException { TimelineEntities te = new TimelineEntities(); TimelineEntity entity = new TimelineEntity(); String id = "application_1111111111_2222"; @@ -42,11 +70,7 @@ final class DataGeneratorForTest { Long cTime = 1425016502000L; entity.setCreatedTime(cTime); // add the info map in Timeline Entity - Map infoMap = new HashMap<>(); - infoMap.put("infoMapKey1", "infoMapValue2"); - infoMap.put("infoMapKey2", 20); - infoMap.put("infoMapKey3", 85.85); - entity.addInfo(infoMap); + entity.addInfo(getInfoMap3()); // add the isRelatedToEntity info Set isRelatedToSet = new HashSet<>(); isRelatedToSet.add("relatedto1"); @@ -71,29 +95,14 @@ final class DataGeneratorForTest { entity.addConfigs(conf); // add metrics Set metrics = new HashSet<>(); - TimelineMetric m1 = new TimelineMetric(); - m1.setId("MAP_SLOT_MILLIS"); - Map metricValues = new HashMap<>(); - long ts = System.currentTimeMillis(); - metricValues.put(ts - 120000, 100000000); - metricValues.put(ts - 100000, 200000000); - metricValues.put(ts - 80000, 300000000); - metricValues.put(ts - 60000, 400000000); - metricValues.put(ts - 40000, 50000000000L); - metricValues.put(ts - 20000, 60000000000L); - m1.setType(Type.TIME_SERIES); - m1.setValues(metricValues); - metrics.add(m1); + metrics.add(getMetric4(ts)); TimelineMetric m12 = new TimelineMetric(); m12.setId("MAP1_BYTES"); m12.addValue(ts, 50); metrics.add(m12); entity.addMetrics(metrics); - TimelineEvent event = new TimelineEvent(); - event.setId("start_event"); - event.setTimestamp(ts); - entity.addEvent(event); + entity.addEvent(addStartEvent(ts)); te.addEntity(entity); TimelineEntities te1 = new TimelineEntities(); TimelineEntity entity1 = new TimelineEntity(); @@ -102,10 +111,7 @@ final class DataGeneratorForTest { entity1.setType(TimelineEntityType.YARN_APPLICATION.toString()); entity1.setCreatedTime(cTime + 20L); // add the info map in Timeline Entity - Map infoMap1 = new HashMap<>(); - infoMap1.put("infoMapKey1", "infoMapValue1"); - infoMap1.put("infoMapKey2", 10); - entity1.addInfo(infoMap1); + entity1.addInfo(getInfoMap4()); // add the isRelatedToEntity info Set isRelatedToSet1 = new HashSet<>(); @@ -133,21 +139,7 @@ final class DataGeneratorForTest { entity1.addConfigs(conf1); // add metrics - Set metrics1 = new HashSet<>(); - TimelineMetric m2 = new TimelineMetric(); - m2.setId("MAP1_SLOT_MILLIS"); - Map metricValues1 = new HashMap<>(); - long ts1 = System.currentTimeMillis(); - metricValues1.put(ts1 - 120000, 100000000); - metricValues1.put(ts1 - 100000, 200000000); - metricValues1.put(ts1 - 80000, 300000000); - metricValues1.put(ts1 - 60000, 400000000); - metricValues1.put(ts1 - 40000, 50000000000L); - metricValues1.put(ts1 - 20000, 60000000000L); - m2.setType(Type.TIME_SERIES); - m2.setValues(metricValues1); - metrics1.add(m2); - entity1.addMetrics(metrics1); + entity1.addMetrics(getMetrics4(ts)); TimelineEvent event11 = new TimelineEvent(); event11.setId("end_event"); event11.setTimestamp(ts); @@ -159,6 +151,53 @@ final class DataGeneratorForTest { te1.addEntity(entity1); TimelineEntities te2 = new TimelineEntities(); + te2.addEntity(getEntity4(cTime, ts)); + HBaseTimelineWriterImpl hbi = null; + try { + hbi = new HBaseTimelineWriterImpl(); + hbi.init(util.getConfiguration()); + hbi.start(); + UserGroupInformation remoteUser = + UserGroupInformation.createRemoteUser("user1"); + hbi.write( + new TimelineCollectorContext("cluster1", "user1", "some_flow_name", + "AB7822C10F1111", 1002345678919L, "application_1111111111_2222"), + te, remoteUser); + hbi.write( + new TimelineCollectorContext("cluster1", "user1", "some_flow_name", + "AB7822C10F1111", 1002345678919L, "application_1111111111_3333"), + te1, remoteUser); + hbi.write( + new TimelineCollectorContext("cluster1", "user1", "some_flow_name", + "AB7822C10F1111", 1002345678919L, "application_1111111111_4444"), + te2, remoteUser); + hbi.stop(); + } finally { + if (hbi != null) { + hbi.stop(); + hbi.close(); + } + } + } + + private static Set getMetrics4(long ts) { + Set metrics1 = new HashSet<>(); + TimelineMetric m2 = new TimelineMetric(); + m2.setId("MAP1_SLOT_MILLIS"); + Map metricValues1 = new HashMap<>(); + metricValues1.put(ts - 120000, 100000000); + metricValues1.put(ts - 100000, 200000000); + metricValues1.put(ts - 80000, 300000000); + metricValues1.put(ts - 60000, 400000000); + metricValues1.put(ts - 40000, 50000000000L); + metricValues1.put(ts - 20000, 60000000000L); + m2.setType(Type.TIME_SERIES); + m2.setValues(metricValues1); + metrics1.add(m2); + return metrics1; + } + + private static TimelineEntity getEntity4(long cTime, long ts) { TimelineEntity entity2 = new TimelineEntity(); String id2 = "application_1111111111_4444"; entity2.setId(id2); @@ -178,34 +217,102 @@ final class DataGeneratorForTest { relatesToSet14.add("relatesto7"); relatesTo3.put("container2", relatesToSet14); entity2.setRelatesToEntities(relatesTo3); - - te2.addEntity(entity2); - HBaseTimelineWriterImpl hbi = null; - try { - hbi = new HBaseTimelineWriterImpl(); - hbi.init(util.getConfiguration()); - hbi.start(); - String cluster = "cluster1"; - String user = "user1"; - String flow = "some_flow_name"; - String flowVersion = "AB7822C10F1111"; - long runid = 1002345678919L; - String appName = "application_1111111111_2222"; - hbi.write(cluster, user, flow, flowVersion, runid, appName, te); - appName = "application_1111111111_3333"; - hbi.write(cluster, user, flow, flowVersion, runid, appName, te1); - appName = "application_1111111111_4444"; - hbi.write(cluster, user, flow, flowVersion, runid, appName, te2); - hbi.stop(); - } finally { - if (hbi != null) { - hbi.stop(); - hbi.close(); - } - } + return entity2; } - static void loadEntities(HBaseTestingUtility util) throws IOException { + private static Map getInfoMap4() { + Map infoMap1 = new HashMap<>(); + infoMap1.put("infoMapKey1", "infoMapValue1"); + infoMap1.put("infoMapKey2", 10); + return infoMap1; + } + + private static TimelineMetric getMetric4(long ts) { + TimelineMetric m1 = new TimelineMetric(); + m1.setId("MAP_SLOT_MILLIS"); + Map metricValues = new HashMap<>(); + metricValues.put(ts - 120000, 100000000); + metricValues.put(ts - 100000, 200000000); + metricValues.put(ts - 80000, 300000000); + metricValues.put(ts - 60000, 400000000); + metricValues.put(ts - 40000, 50000000000L); + metricValues.put(ts - 20000, 60000000000L); + m1.setType(Type.TIME_SERIES); + m1.setValues(metricValues); + return m1; + } + + private static Map getInfoMap3() { + Map infoMap = new HashMap<>(); + infoMap.put("infoMapKey1", "infoMapValue2"); + infoMap.put("infoMapKey2", 20); + infoMap.put("infoMapKey3", 85.85); + return infoMap; + } + + private static Map getInfoMap1() { + Map infoMap = new HashMap<>(); + infoMap.put("infoMapKey1", "infoMapValue2"); + infoMap.put("infoMapKey2", 20); + infoMap.put("infoMapKey3", 71.4); + return infoMap; + } + + private static Map> getRelatesTo1() { + Set relatesToSet = new HashSet(); + relatesToSet.add("relatesto1"); + relatesToSet.add("relatesto3"); + Map> relatesTo = new HashMap<>(); + relatesTo.put("container", relatesToSet); + Set relatesToSet11 = new HashSet<>(); + relatesToSet11.add("relatesto4"); + relatesTo.put("container1", relatesToSet11); + return relatesTo; + } + + private static Map getConfig1() { + Map conf = new HashMap<>(); + conf.put("config_param1", "value1"); + conf.put("config_param2", "value2"); + conf.put("cfg_param1", "value3"); + return conf; + } + + private static Map getConfig2() { + Map conf1 = new HashMap<>(); + conf1.put("cfg_param1", "value1"); + conf1.put("cfg_param2", "value2"); + return conf1; + } + + private static Map getInfoMap2() { + Map infoMap1 = new HashMap<>(); + infoMap1.put("infoMapKey1", "infoMapValue1"); + infoMap1.put("infoMapKey2", 10); + return infoMap1; + } + + private static Map> getIsRelatedTo1() { + Set isRelatedToSet = new HashSet<>(); + isRelatedToSet.add("relatedto1"); + Map> isRelatedTo = new HashMap<>(); + isRelatedTo.put("task", isRelatedToSet); + return isRelatedTo; + } + + private static Map getMetricValues1(long ts) { + Map metricValues = new HashMap<>(); + metricValues.put(ts - 120000, 100000000); + metricValues.put(ts - 100000, 200000000); + metricValues.put(ts - 80000, 300000000); + metricValues.put(ts - 60000, 400000000); + metricValues.put(ts - 40000, 50000000000L); + metricValues.put(ts - 20000, 70000000000L); + return metricValues; + } + + public static void loadEntities(HBaseTestingUtility util, long ts) + throws IOException { TimelineEntities te = new TimelineEntities(); TimelineEntity entity = new TimelineEntity(); String id = "hello"; @@ -215,50 +322,22 @@ final class DataGeneratorForTest { Long cTime = 1425016502000L; entity.setCreatedTime(cTime); // add the info map in Timeline Entity - Map infoMap = new HashMap<>(); - infoMap.put("infoMapKey1", "infoMapValue2"); - infoMap.put("infoMapKey2", 20); - infoMap.put("infoMapKey3", 71.4); - entity.addInfo(infoMap); + entity.addInfo(getInfoMap1()); // add the isRelatedToEntity info - Set isRelatedToSet = new HashSet<>(); - isRelatedToSet.add("relatedto1"); - Map> isRelatedTo = new HashMap<>(); - isRelatedTo.put("task", isRelatedToSet); - entity.setIsRelatedToEntities(isRelatedTo); + entity.setIsRelatedToEntities(getIsRelatedTo1()); // add the relatesTo info - Set relatesToSet = new HashSet(); - relatesToSet.add("relatesto1"); - relatesToSet.add("relatesto3"); - Map> relatesTo = new HashMap<>(); - relatesTo.put("container", relatesToSet); - Set relatesToSet11 = new HashSet<>(); - relatesToSet11.add("relatesto4"); - relatesTo.put("container1", relatesToSet11); - entity.setRelatesToEntities(relatesTo); + entity.setRelatesToEntities(getRelatesTo1()); // add some config entries - Map conf = new HashMap<>(); - conf.put("config_param1", "value1"); - conf.put("config_param2", "value2"); - conf.put("cfg_param1", "value3"); - entity.addConfigs(conf); + entity.addConfigs(getConfig1()); // add metrics Set metrics = new HashSet<>(); TimelineMetric m1 = new TimelineMetric(); m1.setId("MAP_SLOT_MILLIS"); - Map metricValues = new HashMap<>(); - long ts = System.currentTimeMillis(); - metricValues.put(ts - 120000, 100000000); - metricValues.put(ts - 100000, 200000000); - metricValues.put(ts - 80000, 300000000); - metricValues.put(ts - 60000, 400000000); - metricValues.put(ts - 40000, 50000000000L); - metricValues.put(ts - 20000, 70000000000L); m1.setType(Type.TIME_SERIES); - m1.setValues(metricValues); + m1.setValues(getMetricValues1(ts)); metrics.add(m1); TimelineMetric m12 = new TimelineMetric(); @@ -266,10 +345,7 @@ final class DataGeneratorForTest { m12.addValue(ts, 50); metrics.add(m12); entity.addMetrics(metrics); - TimelineEvent event = new TimelineEvent(); - event.setId("start_event"); - event.setTimestamp(ts); - entity.addEvent(event); + entity.addEvent(addStartEvent(ts)); te.addEntity(entity); TimelineEntity entity1 = new TimelineEntity(); @@ -279,10 +355,7 @@ final class DataGeneratorForTest { entity1.setCreatedTime(cTime + 20L); // add the info map in Timeline Entity - Map infoMap1 = new HashMap<>(); - infoMap1.put("infoMapKey1", "infoMapValue1"); - infoMap1.put("infoMapKey2", 10); - entity1.addInfo(infoMap1); + entity1.addInfo(getInfoMap2()); // add event. TimelineEvent event11 = new TimelineEvent(); @@ -296,15 +369,7 @@ final class DataGeneratorForTest { // add the isRelatedToEntity info - Set isRelatedToSet1 = new HashSet<>(); - isRelatedToSet1.add("relatedto3"); - isRelatedToSet1.add("relatedto5"); - Map> isRelatedTo1 = new HashMap<>(); - isRelatedTo1.put("task1", isRelatedToSet1); - Set isRelatedToSet11 = new HashSet<>(); - isRelatedToSet11.add("relatedto4"); - isRelatedTo1.put("task2", isRelatedToSet11); - entity1.setIsRelatedToEntities(isRelatedTo1); + entity1.setIsRelatedToEntities(getIsRelatedTo2()); // add the relatesTo info Set relatesToSet1 = new HashSet(); @@ -315,29 +380,88 @@ final class DataGeneratorForTest { entity1.setRelatesToEntities(relatesTo1); // add some config entries - Map conf1 = new HashMap<>(); - conf1.put("cfg_param1", "value1"); - conf1.put("cfg_param2", "value2"); - entity1.addConfigs(conf1); + entity1.addConfigs(getConfig2()); // add metrics Set metrics1 = new HashSet<>(); TimelineMetric m2 = new TimelineMetric(); m2.setId("MAP1_SLOT_MILLIS"); - Map metricValues1 = new HashMap<>(); - long ts1 = System.currentTimeMillis(); - metricValues1.put(ts1 - 120000, 100000000); - metricValues1.put(ts1 - 100000, 200000000); - metricValues1.put(ts1 - 80000, 300000000); - metricValues1.put(ts1 - 60000, 400000000); - metricValues1.put(ts1 - 40000, 50000000000L); - metricValues1.put(ts1 - 20000, 60000000000L); m2.setType(Type.TIME_SERIES); - m2.setValues(metricValues1); + m2.setValues(getMetricValues2(ts)); metrics1.add(m2); entity1.addMetrics(metrics1); te.addEntity(entity1); + te.addEntity(getEntity2(type, cTime, ts)); + + // For listing types + for (int i = 0; i < 10; i++) { + TimelineEntity entity3 = new TimelineEntity(); + String id3 = "typeTest" + i; + entity3.setId(id3); + StringBuilder typeName = new StringBuilder("newType"); + for (int j = 0; j < (i % 3); j++) { + typeName.append(" ").append(j); + } + entity3.setType(typeName.toString()); + entity3.setCreatedTime(cTime + 80L + i); + te.addEntity(entity3); + } + + // Create app entity for app to flow table + TimelineEntities appTe1 = new TimelineEntities(); + TimelineEntity entityApp1 = new TimelineEntity(); + String appName1 = "application_1231111111_1111"; + entityApp1.setId(appName1); + entityApp1.setType(TimelineEntityType.YARN_APPLICATION.toString()); + entityApp1.setCreatedTime(cTime + 40L); + TimelineEvent appCreationEvent1 = new TimelineEvent(); + appCreationEvent1.setId(ApplicationMetricsConstants.CREATED_EVENT_TYPE); + appCreationEvent1.setTimestamp(cTime); + entityApp1.addEvent(appCreationEvent1); + appTe1.addEntity(entityApp1); + + TimelineEntities appTe2 = new TimelineEntities(); + TimelineEntity entityApp2 = new TimelineEntity(); + String appName2 = "application_1231111111_1112"; + entityApp2.setId(appName2); + entityApp2.setType(TimelineEntityType.YARN_APPLICATION.toString()); + entityApp2.setCreatedTime(cTime + 50L); + TimelineEvent appCreationEvent2 = new TimelineEvent(); + appCreationEvent2.setId(ApplicationMetricsConstants.CREATED_EVENT_TYPE); + appCreationEvent2.setTimestamp(cTime); + entityApp2.addEvent(appCreationEvent2); + appTe2.addEntity(entityApp2); + + HBaseTimelineWriterImpl hbi = null; + try { + hbi = new HBaseTimelineWriterImpl(); + hbi.init(util.getConfiguration()); + hbi.start(); + + UserGroupInformation user = + UserGroupInformation.createRemoteUser("user1"); + TimelineCollectorContext context = + new TimelineCollectorContext("cluster1", "user1", "some_flow_name", + "AB7822C10F1111", 1002345678919L, appName1); + hbi.write(context, te, user); + hbi.write(context, appTe1, user); + + context = new TimelineCollectorContext("cluster1", "user1", + "some_flow_name", "AB7822C10F1111", 1002345678919L, appName2); + hbi.write(context, te, user); + hbi.write(context, appTe2, user); + hbi.stop(); + } finally { + if (hbi != null) { + hbi.stop(); + hbi.close(); + } + } + } + + private static TimelineEntity getEntity2(String type, long cTime, + long ts) { TimelineEntity entity2 = new TimelineEntity(); String id2 = "hello2"; entity2.setId(id2); @@ -357,25 +481,36 @@ final class DataGeneratorForTest { relatesToSet14.add("relatesto7"); relatesTo3.put("container2", relatesToSet14); entity2.setRelatesToEntities(relatesTo3); - te.addEntity(entity2); - HBaseTimelineWriterImpl hbi = null; - try { - hbi = new HBaseTimelineWriterImpl(); - hbi.init(util.getConfiguration()); - hbi.start(); - String cluster = "cluster1"; - String user = "user1"; - String flow = "some_flow_name"; - String flowVersion = "AB7822C10F1111"; - long runid = 1002345678919L; - String appName = "application_1231111111_1111"; - hbi.write(cluster, user, flow, flowVersion, runid, appName, te); - hbi.stop(); - } finally { - if (hbi != null) { - hbi.stop(); - hbi.close(); - } - } + return entity2; + } + + private static TimelineEvent addStartEvent(long ts) { + TimelineEvent event = new TimelineEvent(); + event.setId("start_event"); + event.setTimestamp(ts); + return event; + } + + private static Map getMetricValues2(long ts1) { + Map metricValues1 = new HashMap<>(); + metricValues1.put(ts1 - 120000, 100000000); + metricValues1.put(ts1 - 100000, 200000000); + metricValues1.put(ts1 - 80000, 300000000); + metricValues1.put(ts1 - 60000, 400000000); + metricValues1.put(ts1 - 40000, 50000000000L); + metricValues1.put(ts1 - 20000, 60000000000L); + return metricValues1; + } + + private static Map> getIsRelatedTo2() { + Set isRelatedToSet1 = new HashSet<>(); + isRelatedToSet1.add("relatedto3"); + isRelatedToSet1.add("relatedto5"); + Map> isRelatedTo1 = new HashMap<>(); + isRelatedTo1.put("task1", isRelatedToSet1); + Set isRelatedToSet11 = new HashSet<>(); + isRelatedToSet11.add("relatedto4"); + isRelatedTo1.put("task2", isRelatedToSet11); + return isRelatedTo1; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageApps.java index 3948d236741..111008a5d4c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageApps.java @@ -41,6 +41,7 @@ import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.timelineservice.ApplicationEntity; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities; @@ -51,6 +52,7 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetricOperation; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric.Type; import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants; +import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorContext; import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve; import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters; import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext; @@ -87,17 +89,14 @@ public class TestHBaseTimelineStorageApps { private static HBaseTestingUtility util; private HBaseTimelineReaderImpl reader; + private static final long CURRENT_TIME = System.currentTimeMillis(); @BeforeClass public static void setupBeforeClass() throws Exception { util = new HBaseTestingUtility(); util.startMiniCluster(); - createSchema(); - DataGeneratorForTest.loadApps(util); - } - - private static void createSchema() throws IOException { - TimelineSchemaCreator.createAllTables(util.getConfiguration(), false); + DataGeneratorForTest.createSchema(util.getConfiguration()); + DataGeneratorForTest.loadApps(util, CURRENT_TIME); } @Before @@ -165,7 +164,8 @@ public class TestHBaseTimelineStorageApps { String flow = null; String flowVersion = "AB7822C10F1111"; long runid = 1002345678919L; - hbi.write(cluster, user, flow, flowVersion, runid, appId, te); + hbi.write(new TimelineCollectorContext(cluster, user, flow, flowVersion, + runid, appId), te, UserGroupInformation.createRemoteUser(user)); hbi.stop(); // retrieve the row @@ -240,13 +240,12 @@ public class TestHBaseTimelineStorageApps { TimelineMetric m1 = new TimelineMetric(); m1.setId("MAP_SLOT_MILLIS"); Map metricValues = new HashMap(); - long ts = System.currentTimeMillis(); - metricValues.put(ts - 120000, 100000000); - metricValues.put(ts - 100000, 200000000); - metricValues.put(ts - 80000, 300000000); - metricValues.put(ts - 60000, 400000000); - metricValues.put(ts - 40000, 50000000000L); - metricValues.put(ts - 20000, 60000000000L); + metricValues.put(CURRENT_TIME - 120000, 100000000); + metricValues.put(CURRENT_TIME - 100000, 200000000); + metricValues.put(CURRENT_TIME - 80000, 300000000); + metricValues.put(CURRENT_TIME - 60000, 400000000); + metricValues.put(CURRENT_TIME - 40000, 50000000000L); + metricValues.put(CURRENT_TIME - 20000, 60000000000L); m1.setType(Type.TIME_SERIES); m1.setValues(metricValues); metrics.add(m1); @@ -263,7 +262,7 @@ public class TestHBaseTimelineStorageApps { TimelineMetric aggMetric = new TimelineMetric(); aggMetric.setId("MEM_USAGE"); Map aggMetricValues = new HashMap(); - long aggTs = ts; + long aggTs = CURRENT_TIME; aggMetricValues.put(aggTs - 120000, 102400000L); aggMetric.setType(Type.SINGLE_VALUE); aggMetric.setRealtimeAggregationOp(TimelineMetricOperation.SUM); @@ -284,7 +283,8 @@ public class TestHBaseTimelineStorageApps { String flow = "s!ome_f\tlow _n am!e"; String flowVersion = "AB7822C10F1111"; long runid = 1002345678919L; - hbi.write(cluster, user, flow, flowVersion, runid, appId, te); + hbi.write(new TimelineCollectorContext(cluster, user, flow, flowVersion, + runid, appId), te, UserGroupInformation.createRemoteUser(user)); // Write entity again, this time without created time. entity = new ApplicationEntity(); @@ -296,7 +296,8 @@ public class TestHBaseTimelineStorageApps { entity.addInfo(infoMap1); te = new TimelineEntities(); te.addEntity(entity); - hbi.write(cluster, user, flow, flowVersion, runid, appId, te); + hbi.write(new TimelineCollectorContext(cluster, user, flow, flowVersion, + runid, appId), te, UserGroupInformation.createRemoteUser(user)); hbi.stop(); infoMap.putAll(infoMap1); @@ -384,7 +385,7 @@ public class TestHBaseTimelineStorageApps { new TimelineReaderContext(cluster, user, flow, runid, appId, entity.getType(), entity.getId()), new TimelineDataToRetrieve(null, null, - EnumSet.of(TimelineReader.Field.ALL), Integer.MAX_VALUE)); + EnumSet.of(TimelineReader.Field.ALL), Integer.MAX_VALUE, null, null)); assertNotNull(e1); // verify attributes @@ -393,6 +394,8 @@ public class TestHBaseTimelineStorageApps { e1.getType()); assertEquals(cTime, e1.getCreatedTime()); Map infoMap2 = e1.getInfo(); + // fromid key is added by storage. Remove it for comparision. + infoMap2.remove("FROM_ID"); assertEquals(infoMap, infoMap2); Map> isRelatedTo2 = e1.getIsRelatedToEntities(); @@ -425,7 +428,7 @@ public class TestHBaseTimelineStorageApps { e1 = reader.getEntity(new TimelineReaderContext(cluster, user, flow, runid, appId, entity.getType(), entity.getId()), new TimelineDataToRetrieve(null, null, - EnumSet.of(TimelineReader.Field.ALL), 3)); + EnumSet.of(TimelineReader.Field.ALL), 3, null, null)); assertNotNull(e1); assertEquals(appId, e1.getId()); assertEquals(TimelineEntityType.YARN_APPLICATION.toString(), @@ -446,12 +449,15 @@ public class TestHBaseTimelineStorageApps { e1 = reader.getEntity( new TimelineReaderContext(cluster, user, flow, runid, appId, entity.getType(), entity.getId()), new TimelineDataToRetrieve( - null, null, EnumSet.of(TimelineReader.Field.ALL), null)); + null, null, EnumSet.of(TimelineReader.Field.ALL), null, null, null)); assertNotNull(e1); assertEquals(appId, e1.getId()); assertEquals(TimelineEntityType.YARN_APPLICATION.toString(), e1.getType()); assertEquals(cTime, e1.getCreatedTime()); + infoMap2 = e1.getInfo(); + // fromid key is added by storage. Remove it for comparison. + infoMap2.remove("FROM_ID"); assertEquals(infoMap, e1.getInfo()); assertEquals(isRelatedTo, e1.getIsRelatedToEntities()); assertEquals(relatesTo, e1.getRelatesToEntities()); @@ -464,9 +470,9 @@ public class TestHBaseTimelineStorageApps { metric.getId().equals("MEM_USAGE")); assertEquals(1, metric.getValues().size()); if (metric.getId().equals("MAP_SLOT_MILLIS")) { - assertTrue(metric.getValues().containsKey(ts - 20000)); - assertEquals(metricValues.get(ts - 20000), - metric.getValues().get(ts - 20000)); + assertTrue(metric.getValues().containsKey(CURRENT_TIME - 20000)); + assertEquals(metricValues.get(CURRENT_TIME - 20000), + metric.getValues().get(CURRENT_TIME - 20000)); } if (metric.getId().equals("MEM_USAGE")) { assertTrue(metric.getValues().containsKey(aggTs - 120000)); @@ -513,7 +519,9 @@ public class TestHBaseTimelineStorageApps { String flowVersion = "1111F01C2287BA"; long runid = 1009876543218L; String appName = "application_123465899910_1001"; - hbi.write(cluster, user, flow, flowVersion, runid, appName, entities); + hbi.write(new TimelineCollectorContext(cluster, user, flow, flowVersion, + runid, appName), entities, + UserGroupInformation.createRemoteUser(user)); hbi.stop(); // retrieve the row @@ -553,11 +561,13 @@ public class TestHBaseTimelineStorageApps { TimelineEntity e1 = reader.getEntity( new TimelineReaderContext(cluster, user, flow, runid, appName, entity.getType(), entity.getId()), - new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null)); + new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null, + null, null)); TimelineEntity e2 = reader.getEntity( new TimelineReaderContext(cluster, user, null, null, appName, entity.getType(), entity.getId()), - new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null)); + new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null, + null, null)); assertNotNull(e1); assertNotNull(e2); assertEquals(e1, e2); @@ -626,14 +636,17 @@ public class TestHBaseTimelineStorageApps { hbi.init(c1); hbi.start(); // Writing application entity. + TimelineCollectorContext context = new TimelineCollectorContext("c1", + "u1", "f1", "v1", 1002345678919L, appId); + UserGroupInformation user = UserGroupInformation.createRemoteUser("u1"); try { - hbi.write("c1", "u1", "f1", "v1", 1002345678919L, appId, teApp); + hbi.write(context, teApp, user); Assert.fail("Expected an exception as metric values are non integral"); } catch (IOException e) {} // Writing generic entity. try { - hbi.write("c1", "u1", "f1", "v1", 1002345678919L, appId, teEntity); + hbi.write(context, teEntity, user); Assert.fail("Expected an exception as metric values are non integral"); } catch (IOException e) {} hbi.stop(); @@ -651,7 +664,8 @@ public class TestHBaseTimelineStorageApps { new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1111111111_2222", TimelineEntityType.YARN_APPLICATION.toString(), null), - new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null)); + new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null, + null, null)); assertNotNull(entity); assertEquals(3, entity.getConfigs().size()); assertEquals(1, entity.getIsRelatedToEntities().size()); @@ -659,8 +673,9 @@ public class TestHBaseTimelineStorageApps { new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), - new TimelineEntityFilters(), - new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null)); + new TimelineEntityFilters.Builder().build(), + new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null, + null, null)); assertEquals(3, entities.size()); int cfgCnt = 0; int metricCnt = 0; @@ -684,7 +699,7 @@ public class TestHBaseTimelineStorageApps { } assertEquals(5, cfgCnt); assertEquals(3, metricCnt); - assertEquals(5, infoCnt); + assertEquals(8, infoCnt); assertEquals(4, eventCnt); assertEquals(4, relatesToCnt); assertEquals(4, isRelatedToCnt); @@ -696,8 +711,8 @@ public class TestHBaseTimelineStorageApps { new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), - new TimelineEntityFilters(null, 1425016502000L, 1425016502040L, null, - null, null, null, null, null), + new TimelineEntityFilters.Builder().createdTimeBegin(1425016502000L) + .createTimeEnd(1425016502040L).build(), new TimelineDataToRetrieve()); assertEquals(3, entities.size()); for (TimelineEntity entity : entities) { @@ -713,8 +728,8 @@ public class TestHBaseTimelineStorageApps { new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), - new TimelineEntityFilters(null, 1425016502015L, null, null, null, null, - null, null, null), + new TimelineEntityFilters.Builder().createdTimeBegin(1425016502015L) + .build(), new TimelineDataToRetrieve()); assertEquals(2, entities.size()); for (TimelineEntity entity : entities) { @@ -728,8 +743,8 @@ public class TestHBaseTimelineStorageApps { new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), - new TimelineEntityFilters(null, null, 1425016502015L, null, null, null, - null, null, null), + new TimelineEntityFilters.Builder().createTimeEnd(1425016502015L) + .build(), new TimelineDataToRetrieve()); assertEquals(1, entities.size()); for (TimelineEntity entity : entities) { @@ -748,18 +763,20 @@ public class TestHBaseTimelineStorageApps { TimelineEntityType.YARN_APPLICATION.toString(), null), new TimelineDataToRetrieve()); assertNotNull(e1); - assertTrue(e1.getInfo().isEmpty() && e1.getConfigs().isEmpty() && + assertEquals(1, e1.getInfo().size()); + assertTrue(e1.getConfigs().isEmpty() && e1.getMetrics().isEmpty() && e1.getIsRelatedToEntities().isEmpty() && e1.getRelatesToEntities().isEmpty()); Set es1 = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), - new TimelineEntityFilters(), + new TimelineEntityFilters.Builder().build(), new TimelineDataToRetrieve()); assertEquals(3, es1.size()); for (TimelineEntity e : es1) { - assertTrue(e.getInfo().isEmpty() && e.getConfigs().isEmpty() && + assertEquals(1, e1.getInfo().size()); + assertTrue(e.getConfigs().isEmpty() && e.getMetrics().isEmpty() && e.getIsRelatedToEntities().isEmpty() && e.getRelatesToEntities().isEmpty()); } @@ -772,17 +789,17 @@ public class TestHBaseTimelineStorageApps { 1002345678919L, "application_1111111111_2222", TimelineEntityType.YARN_APPLICATION.toString(), null), new TimelineDataToRetrieve( - null, null, EnumSet.of(Field.INFO, Field.CONFIGS), null)); + null, null, EnumSet.of(Field.INFO, Field.CONFIGS), null, null, null)); assertNotNull(e1); assertEquals(3, e1.getConfigs().size()); assertEquals(0, e1.getIsRelatedToEntities().size()); Set es1 = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), - null), - new TimelineEntityFilters(), + null), new TimelineEntityFilters.Builder().build(), new TimelineDataToRetrieve( - null, null, EnumSet.of(Field.IS_RELATED_TO, Field.METRICS), null)); + null, null, EnumSet.of(Field.IS_RELATED_TO, Field.METRICS), null, + null, null)); assertEquals(3, es1.size()); int metricsCnt = 0; int isRelatedToCnt = 0; @@ -792,7 +809,7 @@ public class TestHBaseTimelineStorageApps { isRelatedToCnt += entity.getIsRelatedToEntities().size(); infoCnt += entity.getInfo().size(); } - assertEquals(0, infoCnt); + assertEquals(3, infoCnt); assertEquals(4, isRelatedToCnt); assertEquals(3, metricsCnt); } @@ -810,9 +827,9 @@ public class TestHBaseTimelineStorageApps { new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), - new TimelineEntityFilters(null, null, null, null, irt, null, null, null, - null), - new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null)); + new TimelineEntityFilters.Builder().isRelatedTo(irt).build(), + new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null, + null, null)); assertEquals(2, entities.size()); int isRelatedToCnt = 0; for (TimelineEntity timelineEntity : entities) { @@ -836,8 +853,7 @@ public class TestHBaseTimelineStorageApps { new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), - new TimelineEntityFilters(null, null, null, null, irt1, null, null, - null, null), + new TimelineEntityFilters.Builder().isRelatedTo(irt1).build(), new TimelineDataToRetrieve()); assertEquals(1, entities.size()); isRelatedToCnt = 0; @@ -860,8 +876,7 @@ public class TestHBaseTimelineStorageApps { new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), - new TimelineEntityFilters(null, null, null, null, irt2, null, null, - null, null), + new TimelineEntityFilters.Builder().isRelatedTo(irt2).build(), new TimelineDataToRetrieve()); assertEquals(2, entities.size()); isRelatedToCnt = 0; @@ -883,8 +898,7 @@ public class TestHBaseTimelineStorageApps { new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), - new TimelineEntityFilters(null, null, null, null, irt3, null, null, - null, null), + new TimelineEntityFilters.Builder().isRelatedTo(irt3).build(), new TimelineDataToRetrieve()); assertEquals(1, entities.size()); isRelatedToCnt = 0; @@ -907,8 +921,7 @@ public class TestHBaseTimelineStorageApps { new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), - new TimelineEntityFilters(null, null, null, null, irt4, null, null, - null, null), + new TimelineEntityFilters.Builder().isRelatedTo(irt4).build(), new TimelineDataToRetrieve()); assertEquals(0, entities.size()); @@ -920,8 +933,7 @@ public class TestHBaseTimelineStorageApps { new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), - new TimelineEntityFilters(null, null, null, null, irt5, null, null, - null, null), + new TimelineEntityFilters.Builder().isRelatedTo(irt5).build(), new TimelineDataToRetrieve()); assertEquals(0, entities.size()); @@ -941,8 +953,7 @@ public class TestHBaseTimelineStorageApps { new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), - new TimelineEntityFilters(null, null, null, null, irt6, null, null, - null, null), + new TimelineEntityFilters.Builder().isRelatedTo(irt6).build(), new TimelineDataToRetrieve()); assertEquals(1, entities.size()); isRelatedToCnt = 0; @@ -969,9 +980,9 @@ public class TestHBaseTimelineStorageApps { new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), - new TimelineEntityFilters(null, null, null, rt, null, null, null, null, - null), - new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null)); + new TimelineEntityFilters.Builder().relatesTo(rt).build(), + new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null, + null, null)); assertEquals(2, entities.size()); int relatesToCnt = 0; for (TimelineEntity timelineEntity : entities) { @@ -995,8 +1006,7 @@ public class TestHBaseTimelineStorageApps { new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), - new TimelineEntityFilters(null, null, null, rt1, null, null, null, null, - null), + new TimelineEntityFilters.Builder().relatesTo(rt1).build(), new TimelineDataToRetrieve()); assertEquals(1, entities.size()); relatesToCnt = 0; @@ -1019,8 +1029,7 @@ public class TestHBaseTimelineStorageApps { new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), - new TimelineEntityFilters(null, null, null, rt2, null, null, null, null, - null), + new TimelineEntityFilters.Builder().relatesTo(rt2).build(), new TimelineDataToRetrieve()); assertEquals(2, entities.size()); relatesToCnt = 0; @@ -1042,8 +1051,7 @@ public class TestHBaseTimelineStorageApps { new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), - new TimelineEntityFilters(null, null, null, rt3, null, null, null, null, - null), + new TimelineEntityFilters.Builder().relatesTo(rt3).build(), new TimelineDataToRetrieve()); assertEquals(1, entities.size()); relatesToCnt = 0; @@ -1066,8 +1074,7 @@ public class TestHBaseTimelineStorageApps { new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), - new TimelineEntityFilters(null, null, null, rt4, null, null, null, null, - null), + new TimelineEntityFilters.Builder().relatesTo(rt4).build(), new TimelineDataToRetrieve()); assertEquals(0, entities.size()); @@ -1079,8 +1086,7 @@ public class TestHBaseTimelineStorageApps { new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), - new TimelineEntityFilters(null, null, null, rt5, null, null, null, null, - null), + new TimelineEntityFilters.Builder().relatesTo(rt5).build(), new TimelineDataToRetrieve()); assertEquals(0, entities.size()); @@ -1100,8 +1106,7 @@ public class TestHBaseTimelineStorageApps { new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), - new TimelineEntityFilters(null, null, null, rt6, null, null, null, null, - null), + new TimelineEntityFilters.Builder().relatesTo(rt6).build(), new TimelineDataToRetrieve()); assertEquals(1, entities.size()); relatesToCnt = 0; @@ -1137,8 +1142,7 @@ public class TestHBaseTimelineStorageApps { new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), - new TimelineEntityFilters(null, null, null, rt7, null, null, null, null, - null), + new TimelineEntityFilters.Builder().relatesTo(rt7).build(), new TimelineDataToRetrieve()); assertEquals(1, entities.size()); relatesToCnt = 0; @@ -1175,8 +1179,8 @@ public class TestHBaseTimelineStorageApps { new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), - new TimelineEntityFilters(null, null, null, relatesTo, isRelatedTo, - null, null, null, eventFilter), + new TimelineEntityFilters.Builder().relatesTo(relatesTo) + .isRelatedTo(isRelatedTo).eventFilters(eventFilter).build(), new TimelineDataToRetrieve()); assertEquals(1, entities.size()); int eventCnt = 0; @@ -1213,10 +1217,10 @@ public class TestHBaseTimelineStorageApps { new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), - new TimelineEntityFilters(null, null, null, null, null, null, - confFilterList, null, null), + new TimelineEntityFilters.Builder().configFilters(confFilterList) + .build(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.CONFIGS), - null)); + null, null, null)); assertEquals(2, entities.size()); int cfgCnt = 0; for (TimelineEntity entity : entities) { @@ -1228,9 +1232,10 @@ public class TestHBaseTimelineStorageApps { new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), - new TimelineEntityFilters(null, null, null, null, null, null, - confFilterList, null, null), - new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null)); + new TimelineEntityFilters.Builder().configFilters(confFilterList) + .build(), + new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null, + null, null)); assertEquals(2, entities.size()); cfgCnt = 0; for (TimelineEntity entity : entities) { @@ -1245,10 +1250,10 @@ public class TestHBaseTimelineStorageApps { new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), - new TimelineEntityFilters(null, null, null, null, null, null, - confFilterList1, null, null), + new TimelineEntityFilters.Builder().configFilters(confFilterList1) + .build(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.CONFIGS), - null)); + null, null, null)); assertEquals(1, entities.size()); cfgCnt = 0; for (TimelineEntity entity : entities) { @@ -1265,10 +1270,10 @@ public class TestHBaseTimelineStorageApps { new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), - new TimelineEntityFilters(null, null, null, null, null, null, - confFilterList2, null, null), + new TimelineEntityFilters.Builder().configFilters(confFilterList2) + .build(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.CONFIGS), - null)); + null, null, null)); assertEquals(0, entities.size()); TimelineFilterList confFilterList3 = new TimelineFilterList( @@ -1278,10 +1283,10 @@ public class TestHBaseTimelineStorageApps { new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), - new TimelineEntityFilters(null, null, null, null, null, null, - confFilterList3, null, null), + new TimelineEntityFilters.Builder().configFilters(confFilterList3) + .build(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.CONFIGS), - null)); + null, null, null)); assertEquals(0, entities.size()); TimelineFilterList confFilterList4 = new TimelineFilterList( @@ -1291,10 +1296,10 @@ public class TestHBaseTimelineStorageApps { new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), - new TimelineEntityFilters(null, null, null, null, null, null, - confFilterList4, null, null), + new TimelineEntityFilters.Builder().configFilters(confFilterList4) + .build(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.CONFIGS), - null)); + null, null, null)); assertEquals(0, entities.size()); TimelineFilterList confFilterList5 = new TimelineFilterList( @@ -1304,10 +1309,10 @@ public class TestHBaseTimelineStorageApps { new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), - new TimelineEntityFilters(null, null, null, null, null, null, - confFilterList5, null, null), + new TimelineEntityFilters.Builder().configFilters(confFilterList5) + .build(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.CONFIGS), - null)); + null, null, null)); assertEquals(3, entities.size()); } @@ -1322,9 +1327,9 @@ public class TestHBaseTimelineStorageApps { new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), - new TimelineEntityFilters(null, null, null, null, null, null, null, - null, ef), - new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null)); + new TimelineEntityFilters.Builder().eventFilters(ef).build(), + new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null, + null, null)); assertEquals(1, entities.size()); int eventCnt = 0; for (TimelineEntity timelineEntity : entities) { @@ -1344,8 +1349,8 @@ public class TestHBaseTimelineStorageApps { new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), - new TimelineEntityFilters(null, null, null, null, null, null, null, - null, ef1), new TimelineDataToRetrieve()); + new TimelineEntityFilters.Builder().eventFilters(ef1).build(), + new TimelineDataToRetrieve()); assertEquals(1, entities.size()); eventCnt = 0; for (TimelineEntity timelineEntity : entities) { @@ -1363,8 +1368,7 @@ public class TestHBaseTimelineStorageApps { new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), - new TimelineEntityFilters(null, null, null, null, null, null, null, - null, ef2), + new TimelineEntityFilters.Builder().eventFilters(ef2).build(), new TimelineDataToRetrieve()); assertEquals(2, entities.size()); eventCnt = 0; @@ -1387,8 +1391,7 @@ public class TestHBaseTimelineStorageApps { new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), - new TimelineEntityFilters(null, null, null, null, null, null, null, - null, ef3), + new TimelineEntityFilters.Builder().eventFilters(ef3).build(), new TimelineDataToRetrieve()); assertEquals(0, entities.size()); @@ -1405,8 +1408,7 @@ public class TestHBaseTimelineStorageApps { new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), - new TimelineEntityFilters(null, null, null, null, null, null, null, - null, ef4), + new TimelineEntityFilters.Builder().eventFilters(ef4).build(), new TimelineDataToRetrieve()); assertEquals(1, entities.size()); eventCnt = 0; @@ -1427,8 +1429,7 @@ public class TestHBaseTimelineStorageApps { new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), - new TimelineEntityFilters(null, null, null, null, null, null, null, - null, ef5), + new TimelineEntityFilters.Builder().eventFilters(ef5).build(), new TimelineDataToRetrieve()); assertEquals(1, entities.size()); eventCnt = 0; @@ -1450,15 +1451,15 @@ public class TestHBaseTimelineStorageApps { new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1111111111_2222", TimelineEntityType.YARN_APPLICATION.toString(), null), - new TimelineDataToRetrieve(list, null, null, null)); + new TimelineDataToRetrieve(list, null, null, null, null, null)); assertNotNull(e1); assertEquals(1, e1.getConfigs().size()); Set es1 = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null) , - new TimelineEntityFilters(), - new TimelineDataToRetrieve(list, null, null, null)); + new TimelineEntityFilters.Builder().build(), + new TimelineDataToRetrieve(list, null, null, null, null, null)); int cfgCnt = 0; for (TimelineEntity entity : es1) { cfgCnt += entity.getConfigs().size(); @@ -1482,9 +1483,9 @@ public class TestHBaseTimelineStorageApps { new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), - new TimelineEntityFilters(null, null, null, null, null, null, - confFilterList, null, null), - new TimelineDataToRetrieve(list, null, null, null)); + new TimelineEntityFilters.Builder().configFilters(confFilterList) + .build(), + new TimelineDataToRetrieve(list, null, null, null, null, null)); assertEquals(1, entities.size()); int cfgCnt = 0; for (TimelineEntity entity : entities) { @@ -1515,9 +1516,10 @@ public class TestHBaseTimelineStorageApps { new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), - new TimelineEntityFilters(null, null, null, null, null, null, - confFilterList1, null, null), - new TimelineDataToRetrieve(confsToRetrieve, null, null, null)); + new TimelineEntityFilters.Builder().configFilters(confFilterList1) + .build(), + new TimelineDataToRetrieve(confsToRetrieve, null, null, null, null, + null)); assertEquals(2, entities.size()); cfgCnt = 0; for (TimelineEntity entity : entities) { @@ -1546,10 +1548,10 @@ public class TestHBaseTimelineStorageApps { new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), - new TimelineEntityFilters(null, null, null, null, null, null, null, - metricFilterList, null), + new TimelineEntityFilters.Builder().metricFilters(metricFilterList) + .build(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS), - null)); + null, null, null)); assertEquals(2, entities.size()); int metricCnt = 0; for (TimelineEntity entity : entities) { @@ -1561,9 +1563,10 @@ public class TestHBaseTimelineStorageApps { new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), - new TimelineEntityFilters(null, null, null, null, null, null, null, - metricFilterList, null), - new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null)); + new TimelineEntityFilters.Builder().metricFilters(metricFilterList) + .build(), + new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null, + null, null)); assertEquals(2, entities.size()); metricCnt = 0; for (TimelineEntity entity : entities) { @@ -1580,10 +1583,10 @@ public class TestHBaseTimelineStorageApps { new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), - new TimelineEntityFilters(null, null, null, null, null, null, null, - metricFilterList1, null), + new TimelineEntityFilters.Builder().metricFilters(metricFilterList1) + .build(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS), - null)); + null, null, null)); assertEquals(1, entities.size()); metricCnt = 0; for (TimelineEntity entity : entities) { @@ -1600,10 +1603,10 @@ public class TestHBaseTimelineStorageApps { new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), - new TimelineEntityFilters(null, null, null, null, null, null, null, - metricFilterList2, null), + new TimelineEntityFilters.Builder().metricFilters(metricFilterList2) + .build(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS), - null)); + null, null, null)); assertEquals(0, entities.size()); TimelineFilterList metricFilterList3 = new TimelineFilterList( @@ -1613,10 +1616,10 @@ public class TestHBaseTimelineStorageApps { new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), - new TimelineEntityFilters(null, null, null, null, null, null, null, - metricFilterList3, null), + new TimelineEntityFilters.Builder().metricFilters(metricFilterList3) + .build(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS), - null)); + null, null, null)); assertEquals(0, entities.size()); TimelineFilterList metricFilterList4 = new TimelineFilterList( @@ -1626,10 +1629,10 @@ public class TestHBaseTimelineStorageApps { new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), - new TimelineEntityFilters(null, null, null, null, null, null, null, - metricFilterList4, null), + new TimelineEntityFilters.Builder().metricFilters(metricFilterList4) + .build(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS), - null)); + null, null, null)); assertEquals(0, entities.size()); TimelineFilterList metricFilterList5 = new TimelineFilterList( @@ -1639,10 +1642,10 @@ public class TestHBaseTimelineStorageApps { new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), - new TimelineEntityFilters(null, null, null, null, null, null, null, - metricFilterList5, null), + new TimelineEntityFilters.Builder().metricFilters(metricFilterList5) + .build(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS), - null)); + null, null, null)); assertEquals(3, entities.size()); } @@ -1655,15 +1658,15 @@ public class TestHBaseTimelineStorageApps { new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1111111111_2222", TimelineEntityType.YARN_APPLICATION.toString(), null), - new TimelineDataToRetrieve(null, list, null, null)); + new TimelineDataToRetrieve(null, list, null, null, null, null)); assertNotNull(e1); assertEquals(1, e1.getMetrics().size()); Set es1 = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), - new TimelineEntityFilters(), - new TimelineDataToRetrieve(null, list, null, null)); + new TimelineEntityFilters.Builder().build(), + new TimelineDataToRetrieve(null, list, null, null, null, null)); int metricCnt = 0; for (TimelineEntity entity : es1) { metricCnt += entity.getMetrics().size(); @@ -1687,9 +1690,9 @@ public class TestHBaseTimelineStorageApps { new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), - new TimelineEntityFilters(null, null, null, null, null, null, null, - metricFilterList, null), - new TimelineDataToRetrieve(null, list, null, null)); + new TimelineEntityFilters.Builder().metricFilters(metricFilterList) + .build(), + new TimelineDataToRetrieve(null, list, null, null, null, null)); int metricCnt = 0; assertEquals(1, entities.size()); for (TimelineEntity entity : entities) { @@ -1713,9 +1716,10 @@ public class TestHBaseTimelineStorageApps { new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), - new TimelineEntityFilters(null, null, null, null, null, null, null, - metricFilterList1, null), - new TimelineDataToRetrieve(null, metricsToRetrieve, null, null)); + new TimelineEntityFilters.Builder().metricFilters(metricFilterList1) + .build(), + new TimelineDataToRetrieve(null, metricsToRetrieve, null, null, null, + null)); metricCnt = 0; assertEquals(2, entities.size()); for (TimelineEntity entity : entities) { @@ -1730,9 +1734,10 @@ public class TestHBaseTimelineStorageApps { entities = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), - new TimelineEntityFilters(null, null, null, null, null, null, null, - metricFilterList1, null), new TimelineDataToRetrieve(null, - metricsToRetrieve, EnumSet.of(Field.METRICS), Integer.MAX_VALUE)); + new TimelineEntityFilters.Builder().metricFilters(metricFilterList1) + .build(), + new TimelineDataToRetrieve(null, metricsToRetrieve, + EnumSet.of(Field.METRICS), Integer.MAX_VALUE, null, null)); metricCnt = 0; int metricValCnt = 0; assertEquals(2, entities.size()); @@ -1748,6 +1753,86 @@ public class TestHBaseTimelineStorageApps { assertEquals(7, metricValCnt); } + @Test + public void testReadAppsMetricTimeRange() throws Exception { + Set entities = reader.getEntities( + new TimelineReaderContext("cluster1", "user1", "some_flow_name", + 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), + null), new TimelineEntityFilters.Builder().build(), + new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS), + 100, null, null)); + assertEquals(3, entities.size()); + int metricTimeSeriesCnt = 0; + int metricCnt = 0; + for (TimelineEntity entity : entities) { + metricCnt += entity.getMetrics().size(); + for (TimelineMetric m : entity.getMetrics()) { + metricTimeSeriesCnt += m.getValues().size(); + } + } + assertEquals(3, metricCnt); + assertEquals(13, metricTimeSeriesCnt); + + entities = reader.getEntities( + new TimelineReaderContext("cluster1", "user1", "some_flow_name", + 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), + null), new TimelineEntityFilters.Builder().build(), + new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS), + 100, CURRENT_TIME - 40000, CURRENT_TIME)); + assertEquals(3, entities.size()); + metricCnt = 0; + metricTimeSeriesCnt = 0; + for (TimelineEntity entity : entities) { + metricCnt += entity.getMetrics().size(); + for (TimelineMetric m : entity.getMetrics()) { + for (Long ts : m.getValues().keySet()) { + assertTrue(ts >= CURRENT_TIME - 40000 && ts <= CURRENT_TIME); + } + metricTimeSeriesCnt += m.getValues().size(); + } + } + assertEquals(3, metricCnt); + assertEquals(5, metricTimeSeriesCnt); + + entities = reader.getEntities( + new TimelineReaderContext("cluster1", "user1", "some_flow_name", + 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), + null), new TimelineEntityFilters.Builder().build(), + new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS), + null, CURRENT_TIME - 40000, CURRENT_TIME)); + assertEquals(3, entities.size()); + metricCnt = 0; + metricTimeSeriesCnt = 0; + for (TimelineEntity entity : entities) { + metricCnt += entity.getMetrics().size(); + for (TimelineMetric m : entity.getMetrics()) { + for (Long ts : m.getValues().keySet()) { + assertTrue(ts >= CURRENT_TIME - 40000 && ts <= CURRENT_TIME); + } + metricTimeSeriesCnt += m.getValues().size(); + } + } + assertEquals(3, metricCnt); + assertEquals(3, metricTimeSeriesCnt); + + TimelineEntity entity = reader.getEntity(new TimelineReaderContext( + "cluster1", "user1", "some_flow_name", 1002345678919L, + "application_1111111111_2222", + TimelineEntityType.YARN_APPLICATION.toString(), null), + new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS), 100, + CURRENT_TIME - 40000, CURRENT_TIME)); + assertNotNull(entity); + assertEquals(2, entity.getMetrics().size()); + metricTimeSeriesCnt = 0; + for (TimelineMetric m : entity.getMetrics()) { + for (Long ts : m.getValues().keySet()) { + assertTrue(ts >= CURRENT_TIME - 40000 && ts <= CURRENT_TIME); + } + metricTimeSeriesCnt += m.getValues().size(); + } + assertEquals(3, metricTimeSeriesCnt); + } + @Test public void testReadAppsInfoFilters() throws Exception { TimelineFilterList list1 = new TimelineFilterList(); @@ -1766,15 +1851,15 @@ public class TestHBaseTimelineStorageApps { new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), - new TimelineEntityFilters(null, null, null, null, null, infoFilterList, - null, null, null), - new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null)); + new TimelineEntityFilters.Builder().infoFilters(infoFilterList).build(), + new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null, + null, null)); assertEquals(2, entities.size()); int infoCnt = 0; for (TimelineEntity entity : entities) { infoCnt += entity.getInfo().size(); } - assertEquals(5, infoCnt); + assertEquals(7, infoCnt); TimelineFilterList infoFilterList1 = new TimelineFilterList( new TimelineKeyValueFilter( @@ -1783,15 +1868,16 @@ public class TestHBaseTimelineStorageApps { new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), - new TimelineEntityFilters(null, null, null, null, null, infoFilterList1, - null, null, null), - new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null)); + new TimelineEntityFilters.Builder().infoFilters(infoFilterList1) + .build(), + new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null, + null, null)); assertEquals(1, entities.size()); infoCnt = 0; for (TimelineEntity entity : entities) { infoCnt += entity.getInfo().size(); } - assertEquals(3, infoCnt); + assertEquals(4, infoCnt); TimelineFilterList infoFilterList2 = new TimelineFilterList( new TimelineKeyValueFilter( @@ -1802,9 +1888,10 @@ public class TestHBaseTimelineStorageApps { new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), - new TimelineEntityFilters(null, null, null, null, null, infoFilterList2, - null, null, null), - new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null)); + new TimelineEntityFilters.Builder().infoFilters(infoFilterList2) + .build(), + new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null, + null, null)); assertEquals(0, entities.size()); TimelineFilterList infoFilterList3 = new TimelineFilterList( @@ -1814,9 +1901,10 @@ public class TestHBaseTimelineStorageApps { new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), - new TimelineEntityFilters(null, null, null, null, null, infoFilterList3, - null, null, null), - new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null)); + new TimelineEntityFilters.Builder().infoFilters(infoFilterList3) + .build(), + new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null, + null, null)); assertEquals(0, entities.size()); TimelineFilterList infoFilterList4 = new TimelineFilterList( @@ -1826,9 +1914,10 @@ public class TestHBaseTimelineStorageApps { new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), - new TimelineEntityFilters(null, null, null, null, null, infoFilterList4, - null, null, null), - new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null)); + new TimelineEntityFilters.Builder().infoFilters(infoFilterList4) + .build(), + new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null, + null, null)); assertEquals(0, entities.size()); TimelineFilterList infoFilterList5 = new TimelineFilterList( @@ -1838,9 +1927,10 @@ public class TestHBaseTimelineStorageApps { new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), - new TimelineEntityFilters(null, null, null, null, null, infoFilterList5, - null, null, null), - new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null)); + new TimelineEntityFilters.Builder().infoFilters(infoFilterList5) + .build(), + new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null, + null, null)); assertEquals(3, entities.size()); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageEntities.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageEntities.java index e18d0d065b4..5e089992ba6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageEntities.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageEntities.java @@ -40,6 +40,7 @@ import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.timelineservice.ApplicationEntity; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities; @@ -48,6 +49,7 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric.Type; import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants; +import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorContext; import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve; import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters; import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext; @@ -72,6 +74,11 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumn import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKey; import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKeyPrefix; import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTable; +import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationColumn; +import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationColumnPrefix; +import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationRowKey; +import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationRowKeyPrefix; +import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationTable; import org.junit.After; import org.junit.AfterClass; import org.junit.Assert; @@ -94,17 +101,14 @@ public class TestHBaseTimelineStorageEntities { private static HBaseTestingUtility util; private HBaseTimelineReaderImpl reader; + private static final long CURRENT_TIME = System.currentTimeMillis(); @BeforeClass public static void setupBeforeClass() throws Exception { util = new HBaseTestingUtility(); util.startMiniCluster(); - createSchema(); - DataGeneratorForTest.loadEntities(util); - } - - private static void createSchema() throws IOException { - TimelineSchemaCreator.createAllTables(util.getConfiguration(), false); + DataGeneratorForTest.createSchema(util.getConfiguration()); + DataGeneratorForTest.loadEntities(util, CURRENT_TIME); } @Before @@ -200,13 +204,16 @@ public class TestHBaseTimelineStorageEntities { hbi.start(); String cluster = "cluster_test_write_entity"; String user = "user1"; + String subAppUser = "subAppUser1"; String flow = "some_flow_name"; String flowVersion = "AB7822C10F1111"; long runid = 1002345678919L; String appName = HBaseTimelineStorageUtils.convertApplicationIdToString( ApplicationId.newInstance(System.currentTimeMillis() + 9000000L, 1) ); - hbi.write(cluster, user, flow, flowVersion, runid, appName, te); + hbi.write(new TimelineCollectorContext(cluster, user, flow, flowVersion, + runid, appName), te, + UserGroupInformation.createRemoteUser(subAppUser)); hbi.stop(); // scan the table and see that entity exists @@ -300,13 +307,13 @@ public class TestHBaseTimelineStorageEntities { new TimelineReaderContext(cluster, user, flow, runid, appName, entity.getType(), entity.getId()), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), - Integer.MAX_VALUE)); + Integer.MAX_VALUE, null, null)); Set es1 = reader.getEntities( new TimelineReaderContext(cluster, user, flow, runid, appName, entity.getType(), null), - new TimelineEntityFilters(), + new TimelineEntityFilters.Builder().build(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), - Integer.MAX_VALUE)); + Integer.MAX_VALUE, null, null)); assertNotNull(e1); assertEquals(1, es1.size()); @@ -315,6 +322,8 @@ public class TestHBaseTimelineStorageEntities { assertEquals(type, e1.getType()); assertEquals(cTime, e1.getCreatedTime()); Map infoMap2 = e1.getInfo(); + // fromid key is added by storage. Remove it for comparison. + infoMap2.remove("FROM_ID"); assertEquals(infoMap, infoMap2); Map> isRelatedTo2 = e1.getIsRelatedToEntities(); @@ -335,12 +344,16 @@ public class TestHBaseTimelineStorageEntities { e1 = reader.getEntity(new TimelineReaderContext(cluster, user, flow, runid, appName, entity.getType(), entity.getId()), - new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null)); + new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null, + null, null)); assertNotNull(e1); assertEquals(id, e1.getId()); assertEquals(type, e1.getType()); assertEquals(cTime, e1.getCreatedTime()); - assertEquals(infoMap, e1.getInfo()); + infoMap2 = e1.getInfo(); + // fromid key is added by storage. Remove it for comparision. + infoMap2.remove("FROM_ID"); + assertEquals(infoMap, infoMap2); assertEquals(isRelatedTo, e1.getIsRelatedToEntities()); assertEquals(relatesTo, e1.getRelatesToEntities()); assertEquals(conf, e1.getConfigs()); @@ -351,6 +364,11 @@ public class TestHBaseTimelineStorageEntities { assertEquals(metricValues.get(ts - 20000), metric.getValues().get(ts - 20000)); } + + // verify for sub application table entities. + verifySubApplicationTableEntities(cluster, user, flow, flowVersion, runid, + appName, subAppUser, c1, entity, id, type, infoMap, isRelatedTo, + relatesTo, conf, metricValues, metrics, cTime, m1); } finally { if (hbi != null) { hbi.stop(); @@ -359,6 +377,98 @@ public class TestHBaseTimelineStorageEntities { } } + private void verifySubApplicationTableEntities(String cluster, String user, + String flow, String flowVersion, Long runid, String appName, + String subAppUser, Configuration c1, TimelineEntity entity, String id, + String type, Map infoMap, + Map> isRelatedTo, Map> relatesTo, + Map conf, Map metricValues, + Set metrics, Long cTime, TimelineMetric m1) + throws IOException { + Scan s = new Scan(); + // read from SubApplicationTable + byte[] startRow = new SubApplicationRowKeyPrefix(cluster, subAppUser, null, + null, null, null).getRowKeyPrefix(); + s.setStartRow(startRow); + s.setMaxVersions(Integer.MAX_VALUE); + Connection conn = ConnectionFactory.createConnection(c1); + ResultScanner scanner = + new SubApplicationTable().getResultScanner(c1, conn, s); + + int rowCount = 0; + int colCount = 0; + KeyConverter stringKeyConverter = new StringKeyConverter(); + for (Result result : scanner) { + if (result != null && !result.isEmpty()) { + rowCount++; + colCount += result.size(); + byte[] row1 = result.getRow(); + assertTrue(verifyRowKeyForSubApplication(row1, subAppUser, cluster, + user, entity)); + + // check info column family + String id1 = SubApplicationColumn.ID.readResult(result).toString(); + assertEquals(id, id1); + + String type1 = SubApplicationColumn.TYPE.readResult(result).toString(); + assertEquals(type, type1); + + Long cTime1 = + (Long) SubApplicationColumn.CREATED_TIME.readResult(result); + assertEquals(cTime1, cTime); + + Map infoColumns = SubApplicationColumnPrefix.INFO + .readResults(result, new StringKeyConverter()); + assertEquals(infoMap, infoColumns); + + // Remember isRelatedTo is of type Map> + for (Map.Entry> isRelatedToEntry : isRelatedTo + .entrySet()) { + Object isRelatedToValue = SubApplicationColumnPrefix.IS_RELATED_TO + .readResult(result, isRelatedToEntry.getKey()); + String compoundValue = isRelatedToValue.toString(); + // id7?id9?id6 + Set isRelatedToValues = + new HashSet(Separator.VALUES.splitEncoded(compoundValue)); + assertEquals(isRelatedTo.get(isRelatedToEntry.getKey()).size(), + isRelatedToValues.size()); + for (String v : isRelatedToEntry.getValue()) { + assertTrue(isRelatedToValues.contains(v)); + } + } + + // RelatesTo + for (Map.Entry> relatesToEntry : relatesTo + .entrySet()) { + String compoundValue = SubApplicationColumnPrefix.RELATES_TO + .readResult(result, relatesToEntry.getKey()).toString(); + // id3?id4?id5 + Set relatesToValues = + new HashSet(Separator.VALUES.splitEncoded(compoundValue)); + assertEquals(relatesTo.get(relatesToEntry.getKey()).size(), + relatesToValues.size()); + for (String v : relatesToEntry.getValue()) { + assertTrue(relatesToValues.contains(v)); + } + } + + // Configuration + Map configColumns = SubApplicationColumnPrefix.CONFIG + .readResults(result, stringKeyConverter); + assertEquals(conf, configColumns); + + NavigableMap> metricsResult = + SubApplicationColumnPrefix.METRIC.readResultsWithTimestamps(result, + stringKeyConverter); + + NavigableMap metricMap = metricsResult.get(m1.getId()); + matchMetrics(metricValues, metricMap); + } + } + assertEquals(1, rowCount); + assertEquals(16, colCount); + } + private boolean isRowKeyCorrect(byte[] rowKey, String cluster, String user, String flow, Long runid, String appName, TimelineEntity te) { @@ -406,7 +516,9 @@ public class TestHBaseTimelineStorageEntities { byte[] startRow = new EntityRowKeyPrefix(cluster, user, flow, runid, appName) .getRowKeyPrefix(); - hbi.write(cluster, user, flow, flowVersion, runid, appName, entities); + hbi.write(new TimelineCollectorContext(cluster, user, flow, flowVersion, + runid, appName), entities, + UserGroupInformation.createRemoteUser(user)); hbi.stop(); // scan the table and see that entity exists Scan s = new Scan(); @@ -450,12 +562,14 @@ public class TestHBaseTimelineStorageEntities { TimelineEntity e1 = reader.getEntity( new TimelineReaderContext(cluster, user, flow, runid, appName, entity.getType(), entity.getId()), - new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null)); + new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null, + null, null)); Set es1 = reader.getEntities( new TimelineReaderContext(cluster, user, flow, runid, appName, entity.getType(), null), - new TimelineEntityFilters(), - new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null)); + new TimelineEntityFilters.Builder().build(), + new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null, + null, null)); assertNotNull(e1); assertEquals(1, es1.size()); @@ -509,14 +623,17 @@ public class TestHBaseTimelineStorageEntities { String flowVersion = "1111F01C2287BA"; long runid = 1009876543218L; String appName = "application_123465899910_2001"; - hbi.write(cluster, user, flow, flowVersion, runid, appName, entities); + hbi.write(new TimelineCollectorContext(cluster, user, flow, flowVersion, + runid, appName), entities, + UserGroupInformation.createRemoteUser(user)); hbi.stop(); // read the timeline entity using the reader this time TimelineEntity e1 = reader.getEntity( new TimelineReaderContext(cluster, user, flow, runid, appName, entity.getType(), entity.getId()), - new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null)); + new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null, + null, null)); assertNotNull(e1); // check the events NavigableSet events = e1.getEvents(); @@ -545,15 +662,17 @@ public class TestHBaseTimelineStorageEntities { TimelineEntity entity = reader.getEntity( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", "hello"), - new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null)); + new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null, + null, null)); assertNotNull(entity); assertEquals(3, entity.getConfigs().size()); assertEquals(1, entity.getIsRelatedToEntities().size()); Set entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", - null), new TimelineEntityFilters(), - new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null)); + null), new TimelineEntityFilters.Builder().build(), + new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null, + null, null)); assertEquals(3, entities.size()); int cfgCnt = 0; int metricCnt = 0; @@ -577,7 +696,7 @@ public class TestHBaseTimelineStorageEntities { } assertEquals(5, cfgCnt); assertEquals(3, metricCnt); - assertEquals(5, infoCnt); + assertEquals(8, infoCnt); assertEquals(4, eventCnt); assertEquals(4, relatesToCnt); assertEquals(4, isRelatedToCnt); @@ -588,8 +707,9 @@ public class TestHBaseTimelineStorageEntities { Set entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", null), - new TimelineEntityFilters(null, 1425016502000L, 1425016502040L, null, - null, null, null, null, null), new TimelineDataToRetrieve()); + new TimelineEntityFilters.Builder().createdTimeBegin(1425016502000L) + .createTimeEnd(1425016502040L).build(), + new TimelineDataToRetrieve()); assertEquals(3, entities.size()); for (TimelineEntity entity : entities) { if (!entity.getId().equals("hello") && !entity.getId().equals("hello1") && @@ -601,8 +721,9 @@ public class TestHBaseTimelineStorageEntities { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", null), - new TimelineEntityFilters(null, 1425016502015L, null, null, null, null, - null, null, null), new TimelineDataToRetrieve()); + new TimelineEntityFilters.Builder().createdTimeBegin(1425016502015L) + .build(), + new TimelineDataToRetrieve()); assertEquals(2, entities.size()); for (TimelineEntity entity : entities) { if (!entity.getId().equals("hello1") && @@ -613,8 +734,9 @@ public class TestHBaseTimelineStorageEntities { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", null), - new TimelineEntityFilters(null, null, 1425016502015L, null, null, null, - null, null, null), new TimelineDataToRetrieve()); + new TimelineEntityFilters.Builder().createTimeEnd(1425016502015L) + .build(), + new TimelineDataToRetrieve()); assertEquals(1, entities.size()); for (TimelineEntity entity : entities) { if (!entity.getId().equals("hello")) { @@ -646,8 +768,9 @@ public class TestHBaseTimelineStorageEntities { Set entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", null), - new TimelineEntityFilters(null, null, null, relatesTo, isRelatedTo, - null, null, null, eventFilter), new TimelineDataToRetrieve()); + new TimelineEntityFilters.Builder().relatesTo(relatesTo) + .isRelatedTo(isRelatedTo).eventFilters(eventFilter).build(), + new TimelineDataToRetrieve()); assertEquals(1, entities.size()); int eventCnt = 0; int isRelatedToCnt = 0; @@ -675,9 +798,9 @@ public class TestHBaseTimelineStorageEntities { Set entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", null), - new TimelineEntityFilters(null, null, null, null, null, null, null, - null, ef), - new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null)); + new TimelineEntityFilters.Builder().eventFilters(ef).build(), + new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null, + null, null)); assertEquals(1, entities.size()); int eventCnt = 0; for (TimelineEntity timelineEntity : entities) { @@ -696,8 +819,7 @@ public class TestHBaseTimelineStorageEntities { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", null), - new TimelineEntityFilters(null, null, null, null, null, null, null, - null, ef1), + new TimelineEntityFilters.Builder().eventFilters(ef1).build(), new TimelineDataToRetrieve()); assertEquals(1, entities.size()); eventCnt = 0; @@ -715,8 +837,7 @@ public class TestHBaseTimelineStorageEntities { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", null), - new TimelineEntityFilters(null, null, null, null, null, null, null, - null, ef2), + new TimelineEntityFilters.Builder().eventFilters(ef2).build(), new TimelineDataToRetrieve()); assertEquals(2, entities.size()); eventCnt = 0; @@ -737,8 +858,7 @@ public class TestHBaseTimelineStorageEntities { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", null), - new TimelineEntityFilters(null, null, null, null, null, null, null, - null, ef3), + new TimelineEntityFilters.Builder().eventFilters(ef3).build(), new TimelineDataToRetrieve()); assertEquals(0, entities.size()); @@ -754,8 +874,7 @@ public class TestHBaseTimelineStorageEntities { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", null), - new TimelineEntityFilters(null, null, null, null, null, null, null, - null, ef4), + new TimelineEntityFilters.Builder().eventFilters(ef4).build(), new TimelineDataToRetrieve()); assertEquals(1, entities.size()); eventCnt = 0; @@ -775,8 +894,7 @@ public class TestHBaseTimelineStorageEntities { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", null), - new TimelineEntityFilters(null, null, null, null, null, null, null, - null, ef5), + new TimelineEntityFilters.Builder().eventFilters(ef5).build(), new TimelineDataToRetrieve()); assertEquals(1, entities.size()); eventCnt = 0; @@ -801,9 +919,9 @@ public class TestHBaseTimelineStorageEntities { Set entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", null), - new TimelineEntityFilters(null, null, null, null, irt, null, null, null, - null), - new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null)); + new TimelineEntityFilters.Builder().isRelatedTo(irt).build(), + new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null, + null, null)); assertEquals(2, entities.size()); int isRelatedToCnt = 0; for (TimelineEntity timelineEntity : entities) { @@ -825,8 +943,7 @@ public class TestHBaseTimelineStorageEntities { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", null), - new TimelineEntityFilters(null, null, null, null, irt1, null, null, - null, null), + new TimelineEntityFilters.Builder().isRelatedTo(irt1).build(), new TimelineDataToRetrieve()); assertEquals(1, entities.size()); isRelatedToCnt = 0; @@ -848,8 +965,7 @@ public class TestHBaseTimelineStorageEntities { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", null), - new TimelineEntityFilters(null, null, null, null, irt2, null, null, - null, null), + new TimelineEntityFilters.Builder().isRelatedTo(irt2).build(), new TimelineDataToRetrieve()); assertEquals(2, entities.size()); isRelatedToCnt = 0; @@ -869,8 +985,7 @@ public class TestHBaseTimelineStorageEntities { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", null), - new TimelineEntityFilters(null, null, null, null, irt3, null, null, - null, null), + new TimelineEntityFilters.Builder().isRelatedTo(irt3).build(), new TimelineDataToRetrieve()); assertEquals(1, entities.size()); isRelatedToCnt = 0; @@ -892,8 +1007,7 @@ public class TestHBaseTimelineStorageEntities { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", null), - new TimelineEntityFilters(null, null, null, null, irt4, null, null, - null, null), + new TimelineEntityFilters.Builder().isRelatedTo(irt4).build(), new TimelineDataToRetrieve()); assertEquals(0, entities.size()); @@ -904,8 +1018,7 @@ public class TestHBaseTimelineStorageEntities { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", null), - new TimelineEntityFilters(null, null, null, null, irt5, null, null, - null, null), + new TimelineEntityFilters.Builder().isRelatedTo(irt5).build(), new TimelineDataToRetrieve()); assertEquals(0, entities.size()); @@ -924,8 +1037,7 @@ public class TestHBaseTimelineStorageEntities { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", null), - new TimelineEntityFilters(null, null, null, null, irt6, null, null, - null, null), + new TimelineEntityFilters.Builder().isRelatedTo(irt6).build(), new TimelineDataToRetrieve()); assertEquals(1, entities.size()); isRelatedToCnt = 0; @@ -950,9 +1062,9 @@ public class TestHBaseTimelineStorageEntities { Set entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", null), - new TimelineEntityFilters(null, null, null, rt, null, null, null, null, - null), - new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null)); + new TimelineEntityFilters.Builder().relatesTo(rt).build(), + new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null, + null, null)); assertEquals(2, entities.size()); int relatesToCnt = 0; for (TimelineEntity timelineEntity : entities) { @@ -974,8 +1086,7 @@ public class TestHBaseTimelineStorageEntities { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", null), - new TimelineEntityFilters(null, null, null, rt1, null, null, null, null, - null), + new TimelineEntityFilters.Builder().relatesTo(rt1).build(), new TimelineDataToRetrieve()); assertEquals(1, entities.size()); relatesToCnt = 0; @@ -997,8 +1108,7 @@ public class TestHBaseTimelineStorageEntities { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", null), - new TimelineEntityFilters(null, null, null, rt2, null, null, null, null, - null), + new TimelineEntityFilters.Builder().relatesTo(rt2).build(), new TimelineDataToRetrieve()); assertEquals(2, entities.size()); relatesToCnt = 0; @@ -1018,8 +1128,7 @@ public class TestHBaseTimelineStorageEntities { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", null), - new TimelineEntityFilters(null, null, null, rt3, null, null, null, null, - null), + new TimelineEntityFilters.Builder().relatesTo(rt3).build(), new TimelineDataToRetrieve()); assertEquals(1, entities.size()); relatesToCnt = 0; @@ -1041,8 +1150,7 @@ public class TestHBaseTimelineStorageEntities { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", null), - new TimelineEntityFilters(null, null, null, rt4, null, null, null, null, - null), + new TimelineEntityFilters.Builder().relatesTo(rt4).build(), new TimelineDataToRetrieve()); assertEquals(0, entities.size()); @@ -1053,8 +1161,7 @@ public class TestHBaseTimelineStorageEntities { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", null), - new TimelineEntityFilters(null, null, null, rt5, null, null, null, null, - null), + new TimelineEntityFilters.Builder().relatesTo(rt5).build(), new TimelineDataToRetrieve()); assertEquals(0, entities.size()); @@ -1073,8 +1180,7 @@ public class TestHBaseTimelineStorageEntities { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", null), - new TimelineEntityFilters(null, null, null, rt6, null, null, null, null, - null), + new TimelineEntityFilters.Builder().relatesTo(rt6).build(), new TimelineDataToRetrieve()); assertEquals(1, entities.size()); relatesToCnt = 0; @@ -1109,8 +1215,7 @@ public class TestHBaseTimelineStorageEntities { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", null), - new TimelineEntityFilters(null, null, null, rt7, null, null, null, null, - null), + new TimelineEntityFilters.Builder().relatesTo(rt7).build(), new TimelineDataToRetrieve()); assertEquals(1, entities.size()); relatesToCnt = 0; @@ -1130,19 +1235,21 @@ public class TestHBaseTimelineStorageEntities { 1002345678919L, "application_1231111111_1111", "world", "hello"), new TimelineDataToRetrieve()); assertNotNull(e1); - assertTrue(e1.getInfo().isEmpty() && e1.getConfigs().isEmpty() && + assertEquals(1, e1.getInfo().size()); + assertTrue(e1.getConfigs().isEmpty() && e1.getMetrics().isEmpty() && e1.getIsRelatedToEntities().isEmpty() && e1.getRelatesToEntities().isEmpty()); Set es1 = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", null), - new TimelineEntityFilters(), + new TimelineEntityFilters.Builder().build(), new TimelineDataToRetrieve()); assertEquals(3, es1.size()); for (TimelineEntity e : es1) { - assertTrue(e.getInfo().isEmpty() && e.getConfigs().isEmpty() && + assertTrue(e.getConfigs().isEmpty() && e.getMetrics().isEmpty() && e.getIsRelatedToEntities().isEmpty() && e.getRelatesToEntities().isEmpty()); + assertEquals(1, e.getInfo().size()); } } @@ -1152,16 +1259,16 @@ public class TestHBaseTimelineStorageEntities { new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", "hello"), new TimelineDataToRetrieve( - null, null, EnumSet.of(Field.INFO, Field.CONFIGS), null)); + null, null, EnumSet.of(Field.INFO, Field.CONFIGS), null, null, null)); assertNotNull(e1); assertEquals(3, e1.getConfigs().size()); assertEquals(0, e1.getIsRelatedToEntities().size()); Set es1 = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", null), - new TimelineEntityFilters(), - new TimelineDataToRetrieve( - null, null, EnumSet.of(Field.IS_RELATED_TO, Field.METRICS), null)); + new TimelineEntityFilters.Builder().build(), + new TimelineDataToRetrieve(null, null, EnumSet.of(Field.IS_RELATED_TO, + Field.METRICS), null, null, null)); assertEquals(3, es1.size()); int metricsCnt = 0; int isRelatedToCnt = 0; @@ -1171,7 +1278,7 @@ public class TestHBaseTimelineStorageEntities { isRelatedToCnt += entity.getIsRelatedToEntities().size(); infoCnt += entity.getInfo().size(); } - assertEquals(0, infoCnt); + assertEquals(3, infoCnt); assertEquals(4, isRelatedToCnt); assertEquals(3, metricsCnt); } @@ -1184,14 +1291,14 @@ public class TestHBaseTimelineStorageEntities { TimelineEntity e1 = reader.getEntity( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", "hello"), - new TimelineDataToRetrieve(list, null, null, null)); + new TimelineDataToRetrieve(list, null, null, null, null, null)); assertNotNull(e1); assertEquals(1, e1.getConfigs().size()); Set es1 = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", null), - new TimelineEntityFilters(), - new TimelineDataToRetrieve(list, null, null, null)); + new TimelineEntityFilters.Builder().build(), + new TimelineDataToRetrieve(list, null, null, null, null, null)); int cfgCnt = 0; for (TimelineEntity entity : es1) { cfgCnt += entity.getConfigs().size(); @@ -1220,10 +1327,10 @@ public class TestHBaseTimelineStorageEntities { Set entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", null), - new TimelineEntityFilters(null, null, null, null, null, null, - confFilterList, null, null), + new TimelineEntityFilters.Builder().configFilters(confFilterList) + .build(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.CONFIGS), - null)); + null, null, null)); assertEquals(2, entities.size()); int cfgCnt = 0; for (TimelineEntity entity : entities) { @@ -1234,9 +1341,10 @@ public class TestHBaseTimelineStorageEntities { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", null), - new TimelineEntityFilters(null, null, null, null, null, null, - confFilterList, null, null), - new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null)); + new TimelineEntityFilters.Builder().configFilters(confFilterList) + .build(), + new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null, + null, null)); assertEquals(2, entities.size()); cfgCnt = 0; for (TimelineEntity entity : entities) { @@ -1250,10 +1358,10 @@ public class TestHBaseTimelineStorageEntities { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", null), - new TimelineEntityFilters(null, null, null, null, null, null, - confFilterList1, null, null), + new TimelineEntityFilters.Builder().configFilters(confFilterList1) + .build(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.CONFIGS), - null)); + null, null, null)); assertEquals(1, entities.size()); cfgCnt = 0; for (TimelineEntity entity : entities) { @@ -1269,10 +1377,10 @@ public class TestHBaseTimelineStorageEntities { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", null), - new TimelineEntityFilters(null, null, null, null, null, null, - confFilterList2, null, null), + new TimelineEntityFilters.Builder().configFilters(confFilterList2) + .build(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.CONFIGS), - null)); + null, null, null)); assertEquals(0, entities.size()); TimelineFilterList confFilterList3 = new TimelineFilterList( @@ -1281,22 +1389,22 @@ public class TestHBaseTimelineStorageEntities { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", null), - new TimelineEntityFilters(null, null, null, null, null, null, - confFilterList3, null, null), + new TimelineEntityFilters.Builder().configFilters(confFilterList3) + .build(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.CONFIGS), - null)); + null, null, null)); assertEquals(0, entities.size()); TimelineFilterList confFilterList4 = new TimelineFilterList( new TimelineKeyValueFilter( TimelineCompareOp.NOT_EQUAL, "dummy_config", "value1")); entities = reader.getEntities( - new TimelineReaderContext("cluster1", "user1", "some_flow_name", + new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", null), - new TimelineEntityFilters(null, null, null, null, null, null, - confFilterList4, null, null), - new TimelineDataToRetrieve(null, null, EnumSet.of(Field.CONFIGS), - null)); + new TimelineEntityFilters.Builder().configFilters(confFilterList4) + .build(), + new TimelineDataToRetrieve(null, null, EnumSet.of(Field.CONFIGS), + null, null, null)); assertEquals(0, entities.size()); TimelineFilterList confFilterList5 = new TimelineFilterList( @@ -1305,10 +1413,10 @@ public class TestHBaseTimelineStorageEntities { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", null), - new TimelineEntityFilters(null, null, null, null, null, null, - confFilterList5, null, null), + new TimelineEntityFilters.Builder().configFilters(confFilterList5) + .build(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.CONFIGS), - null)); + null, null, null)); assertEquals(3, entities.size()); } @@ -1323,9 +1431,9 @@ public class TestHBaseTimelineStorageEntities { Set entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", null), - new TimelineEntityFilters(null, null, null, null, null, null, - confFilterList, null, null), - new TimelineDataToRetrieve(list, null, null, null)); + new TimelineEntityFilters.Builder().configFilters(confFilterList) + .build(), + new TimelineDataToRetrieve(list, null, null, null, null, null)); assertEquals(1, entities.size()); int cfgCnt = 0; for (TimelineEntity entity : entities) { @@ -1354,9 +1462,10 @@ public class TestHBaseTimelineStorageEntities { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", null), - new TimelineEntityFilters(null, null, null, null, null, null, - confFilterList1, null, null), - new TimelineDataToRetrieve(confsToRetrieve, null, null, null)); + new TimelineEntityFilters.Builder().configFilters(confFilterList1) + .build(), + new TimelineDataToRetrieve(confsToRetrieve, null, null, null, null, + null)); assertEquals(2, entities.size()); cfgCnt = 0; for (TimelineEntity entity : entities) { @@ -1377,14 +1486,14 @@ public class TestHBaseTimelineStorageEntities { TimelineEntity e1 = reader.getEntity( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", "hello"), - new TimelineDataToRetrieve(null, list, null, null)); + new TimelineDataToRetrieve(null, list, null, null, null, null)); assertNotNull(e1); assertEquals(1, e1.getMetrics().size()); Set es1 = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", null), - new TimelineEntityFilters(), - new TimelineDataToRetrieve(null, list, null, null)); + new TimelineEntityFilters.Builder().build(), + new TimelineDataToRetrieve(null, list, null, null, null, null)); int metricCnt = 0; for (TimelineEntity entity : es1) { metricCnt += entity.getMetrics().size(); @@ -1396,6 +1505,63 @@ public class TestHBaseTimelineStorageEntities { assertEquals(2, metricCnt); } + @Test + public void testReadEntitiesMetricTimeRange() throws Exception { + Set entities = reader.getEntities( + new TimelineReaderContext("cluster1", "user1", "some_flow_name", + 1002345678919L, "application_1231111111_1111", "world", null), + new TimelineEntityFilters.Builder().build(), + new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS), + 100, null, null)); + assertEquals(3, entities.size()); + int metricTimeSeriesCnt = 0; + int metricCnt = 0; + for (TimelineEntity entity : entities) { + metricCnt += entity.getMetrics().size(); + for (TimelineMetric m : entity.getMetrics()) { + metricTimeSeriesCnt += m.getValues().size(); + } + } + assertEquals(3, metricCnt); + assertEquals(13, metricTimeSeriesCnt); + + entities = reader.getEntities(new TimelineReaderContext("cluster1", "user1", + "some_flow_name", 1002345678919L, "application_1231111111_1111", + "world", null), new TimelineEntityFilters.Builder().build(), + new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS), + 100, CURRENT_TIME - 40000, CURRENT_TIME)); + assertEquals(3, entities.size()); + metricCnt = 0; + metricTimeSeriesCnt = 0; + for (TimelineEntity entity : entities) { + metricCnt += entity.getMetrics().size(); + for (TimelineMetric m : entity.getMetrics()) { + for (Long ts : m.getValues().keySet()) { + assertTrue(ts >= CURRENT_TIME - 40000 && ts <= CURRENT_TIME); + } + metricTimeSeriesCnt += m.getValues().size(); + } + } + assertEquals(3, metricCnt); + assertEquals(5, metricTimeSeriesCnt); + + TimelineEntity entity = reader.getEntity(new TimelineReaderContext( + "cluster1", "user1", "some_flow_name", 1002345678919L, + "application_1231111111_1111", "world", "hello"), + new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS), 100, + CURRENT_TIME - 40000, CURRENT_TIME)); + assertNotNull(entity); + assertEquals(2, entity.getMetrics().size()); + metricTimeSeriesCnt = 0; + for (TimelineMetric m : entity.getMetrics()) { + for (Long ts : m.getValues().keySet()) { + assertTrue(ts >= CURRENT_TIME - 40000 && ts <= CURRENT_TIME); + } + metricTimeSeriesCnt += m.getValues().size(); + } + assertEquals(3, metricTimeSeriesCnt); + } + @Test public void testReadEntitiesMetricFilters() throws Exception { TimelineFilterList list1 = new TimelineFilterList(); @@ -1411,10 +1577,10 @@ public class TestHBaseTimelineStorageEntities { Set entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", null), - new TimelineEntityFilters(null, null, null, null, null, null, null, - metricFilterList, null), + new TimelineEntityFilters.Builder().metricFilters(metricFilterList) + .build(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS), - null)); + null, null, null)); assertEquals(2, entities.size()); int metricCnt = 0; for (TimelineEntity entity : entities) { @@ -1425,9 +1591,10 @@ public class TestHBaseTimelineStorageEntities { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", null), - new TimelineEntityFilters(null, null, null, null, null, null, null, - metricFilterList, null), - new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null)); + new TimelineEntityFilters.Builder().metricFilters(metricFilterList) + .build(), + new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null, + null, null)); assertEquals(2, entities.size()); metricCnt = 0; for (TimelineEntity entity : entities) { @@ -1443,10 +1610,10 @@ public class TestHBaseTimelineStorageEntities { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", null), - new TimelineEntityFilters(null, null, null, null, null, null, null, - metricFilterList1, null), + new TimelineEntityFilters.Builder().metricFilters(metricFilterList1) + .build(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS), - null)); + null, null, null)); assertEquals(1, entities.size()); metricCnt = 0; for (TimelineEntity entity : entities) { @@ -1462,10 +1629,10 @@ public class TestHBaseTimelineStorageEntities { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", null), - new TimelineEntityFilters(null, null, null, null, null, null, null, - metricFilterList2, null), + new TimelineEntityFilters.Builder().metricFilters(metricFilterList2) + .build(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS), - null)); + null, null, null)); assertEquals(0, entities.size()); TimelineFilterList metricFilterList3 = new TimelineFilterList( @@ -1474,10 +1641,10 @@ public class TestHBaseTimelineStorageEntities { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", null), - new TimelineEntityFilters(null, null, null, null, null, null, null, - metricFilterList3, null), + new TimelineEntityFilters.Builder().metricFilters(metricFilterList3) + .build(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS), - null)); + null, null, null)); assertEquals(0, entities.size()); TimelineFilterList metricFilterList4 = new TimelineFilterList( @@ -1486,10 +1653,10 @@ public class TestHBaseTimelineStorageEntities { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", null), - new TimelineEntityFilters(null, null, null, null, null, null, null, - metricFilterList4, null), + new TimelineEntityFilters.Builder().metricFilters(metricFilterList4) + .build(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS), - null)); + null, null, null)); assertEquals(0, entities.size()); TimelineFilterList metricFilterList5 = new TimelineFilterList( @@ -1498,10 +1665,10 @@ public class TestHBaseTimelineStorageEntities { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", null), - new TimelineEntityFilters(null, null, null, null, null, null, null, - metricFilterList5, null), + new TimelineEntityFilters.Builder().metricFilters(metricFilterList5) + .build(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS), - null)); + null, null, null)); assertEquals(3, entities.size()); } @@ -1516,9 +1683,9 @@ public class TestHBaseTimelineStorageEntities { Set entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", null), - new TimelineEntityFilters(null, null, null, null, null, null, null, - metricFilterList, null), - new TimelineDataToRetrieve(null, list, null, null)); + new TimelineEntityFilters.Builder().metricFilters(metricFilterList) + .build(), + new TimelineDataToRetrieve(null, list, null, null, null, null)); assertEquals(1, entities.size()); int metricCnt = 0; for (TimelineEntity entity : entities) { @@ -1545,10 +1712,10 @@ public class TestHBaseTimelineStorageEntities { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", null), - new TimelineEntityFilters(null, null, null, null, null, null, null, - metricFilterList1, null), + new TimelineEntityFilters.Builder().metricFilters(metricFilterList1) + .build(), new TimelineDataToRetrieve( - null, metricsToRetrieve, EnumSet.of(Field.METRICS), null)); + null, metricsToRetrieve, EnumSet.of(Field.METRICS), null, null, null)); assertEquals(2, entities.size()); metricCnt = 0; for (TimelineEntity entity : entities) { @@ -1564,9 +1731,11 @@ public class TestHBaseTimelineStorageEntities { entities = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", - "world", null), new TimelineEntityFilters(null, null, null, null, null, - null, null, metricFilterList1, null), new TimelineDataToRetrieve(null, - metricsToRetrieve, EnumSet.of(Field.METRICS), Integer.MAX_VALUE)); + "world", null), + new TimelineEntityFilters.Builder().metricFilters(metricFilterList1) + .build(), + new TimelineDataToRetrieve(null, metricsToRetrieve, + EnumSet.of(Field.METRICS), Integer.MAX_VALUE, null, null)); assertEquals(2, entities.size()); metricCnt = 0; int metricValCnt = 0; @@ -1599,15 +1768,15 @@ public class TestHBaseTimelineStorageEntities { Set entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", null), - new TimelineEntityFilters(null, null, null, null, null, infoFilterList, - null, null, null), - new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null)); + new TimelineEntityFilters.Builder().infoFilters(infoFilterList).build(), + new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null, + null, null)); assertEquals(2, entities.size()); int infoCnt = 0; for (TimelineEntity entity : entities) { infoCnt += entity.getInfo().size(); } - assertEquals(5, infoCnt); + assertEquals(7, infoCnt); TimelineFilterList infoFilterList1 = new TimelineFilterList( new TimelineKeyValueFilter( @@ -1615,15 +1784,16 @@ public class TestHBaseTimelineStorageEntities { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", null), - new TimelineEntityFilters(null, null, null, null, null, infoFilterList1, - null, null, null), - new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null)); + new TimelineEntityFilters.Builder().infoFilters(infoFilterList1) + .build(), + new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null, + null, null)); assertEquals(1, entities.size()); infoCnt = 0; for (TimelineEntity entity : entities) { infoCnt += entity.getInfo().size(); } - assertEquals(3, infoCnt); + assertEquals(4, infoCnt); TimelineFilterList infoFilterList2 = new TimelineFilterList( new TimelineKeyValueFilter( @@ -1633,9 +1803,10 @@ public class TestHBaseTimelineStorageEntities { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", null), - new TimelineEntityFilters(null, null, null, null, null, infoFilterList2, - null, null, null), - new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null)); + new TimelineEntityFilters.Builder().infoFilters(infoFilterList2) + .build(), + new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null, + null, null)); assertEquals(0, entities.size()); TimelineFilterList infoFilterList3 = new TimelineFilterList( @@ -1644,9 +1815,10 @@ public class TestHBaseTimelineStorageEntities { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", null), - new TimelineEntityFilters(null, null, null, null, null, infoFilterList3, - null, null, null), - new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null)); + new TimelineEntityFilters.Builder().infoFilters(infoFilterList3) + .build(), + new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null, + null, null)); assertEquals(0, entities.size()); TimelineFilterList infoFilterList4 = new TimelineFilterList( @@ -1655,9 +1827,10 @@ public class TestHBaseTimelineStorageEntities { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", null), - new TimelineEntityFilters(null, null, null, null, null, infoFilterList4, - null, null, null), - new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null)); + new TimelineEntityFilters.Builder().infoFilters(infoFilterList4) + .build(), + new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null, + null, null)); assertEquals(0, entities.size()); TimelineFilterList infoFilterList5 = new TimelineFilterList( @@ -1666,14 +1839,49 @@ public class TestHBaseTimelineStorageEntities { entities = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", null), - new TimelineEntityFilters(null, null, null, null, null, infoFilterList5, - null, null, null), - new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null)); + new TimelineEntityFilters.Builder().infoFilters(infoFilterList5) + .build(), + new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null, + null, null)); assertEquals(3, entities.size()); } + @Test(timeout = 90000) + public void testListTypesInApp() throws Exception { + Set types = reader.getEntityTypes( + new TimelineReaderContext("cluster1", "user1", "some_flow_name", + 1002345678919L, "application_1231111111_1111", null, null)); + assertEquals(4, types.size()); + + types = reader.getEntityTypes( + new TimelineReaderContext("cluster1", null, null, + null, "application_1231111111_1111", null, null)); + assertEquals(4, types.size()); + + types = reader.getEntityTypes( + new TimelineReaderContext("cluster1", null, null, + null, "application_1231111111_1112", null, null)); + assertEquals(4, types.size()); + + types = reader.getEntityTypes( + new TimelineReaderContext("cluster1", "user1", "some_flow_name", + 1002345678919L, "application_1231111111_1113", null, null)); + assertEquals(0, types.size()); + } + @AfterClass public static void tearDownAfterClass() throws Exception { util.shutdownMiniCluster(); } + + private boolean verifyRowKeyForSubApplication(byte[] rowKey, String suAppUser, + String cluster, String user, TimelineEntity te) { + SubApplicationRowKey key = SubApplicationRowKey.parseRowKey(rowKey); + assertEquals(suAppUser, key.getSubAppUserId()); + assertEquals(cluster, key.getClusterId()); + assertEquals(te.getType(), key.getEntityType()); + assertEquals(te.getId(), key.getEntityId()); + assertEquals(user, key.getUserId()); + return true; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageSchema.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageSchema.java new file mode 100644 index 00000000000..0dcd171ec52 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageSchema.java @@ -0,0 +1,135 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.timelineservice.storage; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.client.Table; + +import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable; +import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTable; +import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunTable; + +/** + * Unit tests for checking different schema prefixes. + */ +public class TestHBaseTimelineStorageSchema { + private static HBaseTestingUtility util; + + @BeforeClass + public static void setupBeforeClass() throws Exception { + util = new HBaseTestingUtility(); + Configuration conf = util.getConfiguration(); + conf.setInt("hfile.format.version", 3); + util.startMiniCluster(); + } + + @Test + public void createWithDefaultPrefix() throws IOException { + Configuration hbaseConf = util.getConfiguration(); + DataGeneratorForTest.createSchema(hbaseConf); + Connection conn = null; + conn = ConnectionFactory.createConnection(hbaseConf); + Admin admin = conn.getAdmin(); + + TableName entityTableName = BaseTable.getTableName(hbaseConf, + EntityTable.TABLE_NAME_CONF_NAME, EntityTable.DEFAULT_TABLE_NAME); + assertTrue(admin.tableExists(entityTableName)); + assertTrue(entityTableName.getNameAsString().startsWith( + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_HBASE_SCHEMA_PREFIX)); + Table entityTable = conn.getTable(BaseTable.getTableName(hbaseConf, + EntityTable.TABLE_NAME_CONF_NAME, EntityTable.DEFAULT_TABLE_NAME)); + assertNotNull(entityTable); + + TableName flowRunTableName = BaseTable.getTableName(hbaseConf, + FlowRunTable.TABLE_NAME_CONF_NAME, FlowRunTable.DEFAULT_TABLE_NAME); + assertTrue(admin.tableExists(flowRunTableName)); + assertTrue(flowRunTableName.getNameAsString().startsWith( + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_HBASE_SCHEMA_PREFIX)); + Table flowRunTable = conn.getTable(BaseTable.getTableName(hbaseConf, + FlowRunTable.TABLE_NAME_CONF_NAME, FlowRunTable.DEFAULT_TABLE_NAME)); + assertNotNull(flowRunTable); + } + + @Test + public void createWithSetPrefix() throws IOException { + Configuration hbaseConf = util.getConfiguration(); + String prefix = "unit-test."; + hbaseConf.set(YarnConfiguration.TIMELINE_SERVICE_HBASE_SCHEMA_PREFIX_NAME, + prefix); + DataGeneratorForTest.createSchema(hbaseConf); + Connection conn = null; + conn = ConnectionFactory.createConnection(hbaseConf); + Admin admin = conn.getAdmin(); + + TableName entityTableName = BaseTable.getTableName(hbaseConf, + EntityTable.TABLE_NAME_CONF_NAME, EntityTable.DEFAULT_TABLE_NAME); + assertTrue(admin.tableExists(entityTableName)); + assertTrue(entityTableName.getNameAsString().startsWith(prefix)); + Table entityTable = conn.getTable(BaseTable.getTableName(hbaseConf, + EntityTable.TABLE_NAME_CONF_NAME, EntityTable.DEFAULT_TABLE_NAME)); + assertNotNull(entityTable); + + TableName flowRunTableName = BaseTable.getTableName(hbaseConf, + FlowRunTable.TABLE_NAME_CONF_NAME, FlowRunTable.DEFAULT_TABLE_NAME); + assertTrue(admin.tableExists(flowRunTableName)); + assertTrue(flowRunTableName.getNameAsString().startsWith(prefix)); + Table flowRunTable = conn.getTable(BaseTable.getTableName(hbaseConf, + FlowRunTable.TABLE_NAME_CONF_NAME, FlowRunTable.DEFAULT_TABLE_NAME)); + assertNotNull(flowRunTable); + + // create another set with a diff prefix + hbaseConf + .unset(YarnConfiguration.TIMELINE_SERVICE_HBASE_SCHEMA_PREFIX_NAME); + prefix = "yet-another-unit-test."; + hbaseConf.set(YarnConfiguration.TIMELINE_SERVICE_HBASE_SCHEMA_PREFIX_NAME, + prefix); + DataGeneratorForTest.createSchema(hbaseConf); + entityTableName = BaseTable.getTableName(hbaseConf, + EntityTable.TABLE_NAME_CONF_NAME, EntityTable.DEFAULT_TABLE_NAME); + assertTrue(admin.tableExists(entityTableName)); + assertTrue(entityTableName.getNameAsString().startsWith(prefix)); + entityTable = conn.getTable(BaseTable.getTableName(hbaseConf, + EntityTable.TABLE_NAME_CONF_NAME, EntityTable.DEFAULT_TABLE_NAME)); + assertNotNull(entityTable); + + flowRunTableName = BaseTable.getTableName(hbaseConf, + FlowRunTable.TABLE_NAME_CONF_NAME, FlowRunTable.DEFAULT_TABLE_NAME); + assertTrue(admin.tableExists(flowRunTableName)); + assertTrue(flowRunTableName.getNameAsString().startsWith(prefix)); + flowRunTable = conn.getTable(BaseTable.getTableName(hbaseConf, + FlowRunTable.TABLE_NAME_CONF_NAME, FlowRunTable.DEFAULT_TABLE_NAME)); + assertNotNull(flowRunTable); + hbaseConf + .unset(YarnConfiguration.TIMELINE_SERVICE_HBASE_SCHEMA_PREFIX_NAME); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestFlowDataGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestFlowDataGenerator.java index b6089873f4f..8ab32dfff31 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestFlowDataGenerator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestFlowDataGenerator.java @@ -33,7 +33,7 @@ import org.apache.hadoop.conf.Configuration; /** * Generates the data/entities for the FlowRun and FlowActivity Tables. */ -final class TestFlowDataGenerator { +public final class TestFlowDataGenerator { private TestFlowDataGenerator() { } @@ -41,7 +41,8 @@ final class TestFlowDataGenerator { private static final String METRIC_2 = "HDFS_BYTES_READ"; public static final long END_TS_INCR = 10000L; - static TimelineEntity getEntityMetricsApp1(long insertTs, Configuration c1) { + public static TimelineEntity getEntityMetricsApp1(long insertTs, + Configuration c1) { TimelineEntity entity = new TimelineEntity(); String id = "flowRunMetrics_test"; String type = TimelineEntityType.YARN_APPLICATION.toString(); @@ -83,7 +84,7 @@ final class TestFlowDataGenerator { } - static TimelineEntity getEntityMetricsApp1Complete(long insertTs, + public static TimelineEntity getEntityMetricsApp1Complete(long insertTs, Configuration c1) { TimelineEntity entity = new TimelineEntity(); String id = "flowRunMetrics_test"; @@ -125,7 +126,7 @@ final class TestFlowDataGenerator { } - static TimelineEntity getEntityMetricsApp1(long insertTs) { + public static TimelineEntity getEntityMetricsApp1(long insertTs) { TimelineEntity entity = new TimelineEntity(); String id = "flowRunMetrics_test"; String type = TimelineEntityType.YARN_APPLICATION.toString(); @@ -168,8 +169,7 @@ final class TestFlowDataGenerator { return entity; } - - static TimelineEntity getEntityMetricsApp2(long insertTs) { + public static TimelineEntity getEntityMetricsApp2(long insertTs) { TimelineEntity entity = new TimelineEntity(); String id = "flowRunMetrics_test"; String type = TimelineEntityType.YARN_APPLICATION.toString(); @@ -200,7 +200,7 @@ final class TestFlowDataGenerator { return entity; } - static TimelineEntity getEntity1() { + public static TimelineEntity getEntity1() { TimelineEntity entity = new TimelineEntity(); String id = "flowRunHello"; String type = TimelineEntityType.YARN_APPLICATION.toString(); @@ -243,7 +243,7 @@ final class TestFlowDataGenerator { return entity; } - static TimelineEntity getAFullEntity(long ts, long endTs) { + public static TimelineEntity getAFullEntity(long ts, long endTs) { TimelineEntity entity = new TimelineEntity(); String id = "flowRunFullEntity"; String type = TimelineEntityType.YARN_APPLICATION.toString(); @@ -292,7 +292,7 @@ final class TestFlowDataGenerator { return entity; } - static TimelineEntity getEntityGreaterStartTime(long startTs) { + public static TimelineEntity getEntityGreaterStartTime(long startTs) { TimelineEntity entity = new TimelineEntity(); entity.setCreatedTime(startTs); entity.setId("flowRunHello with greater start time"); @@ -308,7 +308,7 @@ final class TestFlowDataGenerator { return entity; } - static TimelineEntity getEntityMaxEndTime(long endTs) { + public static TimelineEntity getEntityMaxEndTime(long endTs) { TimelineEntity entity = new TimelineEntity(); entity.setId("flowRunHello Max End time"); entity.setType(TimelineEntityType.YARN_APPLICATION.toString()); @@ -322,7 +322,7 @@ final class TestFlowDataGenerator { return entity; } - static TimelineEntity getEntityMinStartTime(long startTs) { + public static TimelineEntity getEntityMinStartTime(long startTs) { TimelineEntity entity = new TimelineEntity(); String id = "flowRunHelloMInStartTime"; String type = TimelineEntityType.YARN_APPLICATION.toString(); @@ -336,7 +336,7 @@ final class TestFlowDataGenerator { return entity; } - static TimelineEntity getMinFlushEntity(long startTs) { + public static TimelineEntity getMinFlushEntity(long startTs) { TimelineEntity entity = new TimelineEntity(); String id = "flowRunHelloFlushEntityMin"; String type = TimelineEntityType.YARN_APPLICATION.toString(); @@ -350,7 +350,7 @@ final class TestFlowDataGenerator { return entity; } - static TimelineEntity getMaxFlushEntity(long startTs) { + public static TimelineEntity getMaxFlushEntity(long startTs) { TimelineEntity entity = new TimelineEntity(); String id = "flowRunHelloFlushEntityMax"; String type = TimelineEntityType.YARN_APPLICATION.toString(); @@ -365,7 +365,7 @@ final class TestFlowDataGenerator { return entity; } - static TimelineEntity getFlowApp1(long appCreatedTime) { + public static TimelineEntity getFlowApp1(long appCreatedTime) { TimelineEntity entity = new TimelineEntity(); String id = "flowActivity_test"; String type = TimelineEntityType.YARN_APPLICATION.toString(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java index 2778f50df73..4bf221e2ad6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java @@ -30,7 +30,6 @@ import java.util.Set; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.Get; @@ -39,18 +38,21 @@ import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.timelineservice.FlowActivityEntity; import org.apache.hadoop.yarn.api.records.timelineservice.FlowRunEntity; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType; import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper; +import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorContext; import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve; import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters; import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext; +import org.apache.hadoop.yarn.server.timelineservice.storage.DataGeneratorForTest; import org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineReaderImpl; import org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl; -import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineSchemaCreator; +import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable; import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper; import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineStorageUtils; import org.junit.AfterClass; @@ -70,11 +72,7 @@ public class TestHBaseStorageFlowActivity { Configuration conf = util.getConfiguration(); conf.setInt("hfile.format.version", 3); util.startMiniCluster(); - createSchema(); - } - - private static void createSchema() throws IOException { - TimelineSchemaCreator.createAllTables(util.getConfiguration(), false); + DataGeneratorForTest.createSchema(util.getConfiguration()); } /** @@ -121,13 +119,18 @@ public class TestHBaseStorageFlowActivity { try { hbi = new HBaseTimelineWriterImpl(); hbi.init(c1); - hbi.write(cluster, user, flow, flowVersion, runid, appName, te); + + UserGroupInformation remoteUser = + UserGroupInformation.createRemoteUser(user); + hbi.write(new TimelineCollectorContext(cluster, user, flow, flowVersion, + runid, appName), te, remoteUser); // write another entity with the right min start time te = new TimelineEntities(); te.addEntity(entityMinStartTime); appName = "application_100000000000_3333"; - hbi.write(cluster, user, flow, flowVersion, runid, appName, te); + hbi.write(new TimelineCollectorContext(cluster, user, flow, flowVersion, + runid, appName), te, remoteUser); // writer another entity for max end time TimelineEntity entityMaxEndTime = TestFlowDataGenerator @@ -135,7 +138,8 @@ public class TestHBaseStorageFlowActivity { te = new TimelineEntities(); te.addEntity(entityMaxEndTime); appName = "application_100000000000_4444"; - hbi.write(cluster, user, flow, flowVersion, runid, appName, te); + hbi.write(new TimelineCollectorContext(cluster, user, flow, flowVersion, + runid, appName), te, remoteUser); // writer another entity with greater start time TimelineEntity entityGreaterStartTime = TestFlowDataGenerator @@ -143,7 +147,8 @@ public class TestHBaseStorageFlowActivity { te = new TimelineEntities(); te.addEntity(entityGreaterStartTime); appName = "application_1000000000000000_2222"; - hbi.write(cluster, user, flow, flowVersion, runid, appName, te); + hbi.write(new TimelineCollectorContext(cluster, user, flow, flowVersion, + runid, appName), te, remoteUser); // flush everything to hbase hbi.flush(); @@ -155,8 +160,9 @@ public class TestHBaseStorageFlowActivity { Connection conn = ConnectionFactory.createConnection(c1); // check in flow activity table - Table table1 = conn.getTable(TableName - .valueOf(FlowActivityTable.DEFAULT_TABLE_NAME)); + Table table1 = conn.getTable( + BaseTable.getTableName(c1, FlowActivityTable.TABLE_NAME_CONF_NAME, + FlowActivityTable.DEFAULT_TABLE_NAME)); byte[] startRow = new FlowActivityRowKey(cluster, minStartTs, user, flow).getRowKey(); Get g = new Get(startRow); @@ -187,8 +193,7 @@ public class TestHBaseStorageFlowActivity { Set entities = hbr.getEntities( new TimelineReaderContext(cluster, null, null, null, null, TimelineEntityType.YARN_FLOW_ACTIVITY.toString(), null), - new TimelineEntityFilters(10L, null, null, null, null, null, - null, null, null), + new TimelineEntityFilters.Builder().entityLimit(10L).build(), new TimelineDataToRetrieve()); assertEquals(1, entities.size()); for (TimelineEntity e : entities) { @@ -231,7 +236,8 @@ public class TestHBaseStorageFlowActivity { hbi = new HBaseTimelineWriterImpl(); hbi.init(c1); String appName = "application_1111999999_1234"; - hbi.write(cluster, user, flow, flowVersion, runid, appName, te); + hbi.write(new TimelineCollectorContext(cluster, user, flow, flowVersion, + runid, appName), te, UserGroupInformation.createRemoteUser(user)); hbi.flush(); } finally { if (hbi != null) { @@ -252,8 +258,7 @@ public class TestHBaseStorageFlowActivity { Set entities = hbr.getEntities( new TimelineReaderContext(cluster, user, flow, null, null, TimelineEntityType.YARN_FLOW_ACTIVITY.toString(), null), - new TimelineEntityFilters(10L, null, null, null, null, null, - null, null, null), + new TimelineEntityFilters.Builder().entityLimit(10L).build(), new TimelineDataToRetrieve()); assertEquals(1, entities.size()); for (TimelineEntity e : entities) { @@ -286,8 +291,9 @@ public class TestHBaseStorageFlowActivity { .getRowKey(); s.setStopRow(stopRow); Connection conn = ConnectionFactory.createConnection(c1); - Table table1 = conn.getTable(TableName - .valueOf(FlowActivityTable.DEFAULT_TABLE_NAME)); + Table table1 = conn.getTable( + BaseTable.getTableName(c1, FlowActivityTable.TABLE_NAME_CONF_NAME, + FlowActivityTable.DEFAULT_TABLE_NAME)); ResultScanner scanner = table1.getScanner(s); int rowCount = 0; for (Result result : scanner) { @@ -344,20 +350,27 @@ public class TestHBaseStorageFlowActivity { try { hbi = new HBaseTimelineWriterImpl(); hbi.init(c1); + + UserGroupInformation remoteUser = + UserGroupInformation.createRemoteUser(user); + String appName = "application_11888888888_1111"; - hbi.write(cluster, user, flow, flowVersion1, runid1, appName, te); + hbi.write(new TimelineCollectorContext(cluster, user, flow, flowVersion1, + runid1, appName), te, remoteUser); // write an application with to this flow but a different runid/ version te = new TimelineEntities(); te.addEntity(entityApp1); appName = "application_11888888888_2222"; - hbi.write(cluster, user, flow, flowVersion2, runid2, appName, te); + hbi.write(new TimelineCollectorContext(cluster, user, flow, flowVersion2, + runid2, appName), te, remoteUser); // write an application with to this flow but a different runid/ version te = new TimelineEntities(); te.addEntity(entityApp1); appName = "application_11888888888_3333"; - hbi.write(cluster, user, flow, flowVersion3, runid3, appName, te); + hbi.write(new TimelineCollectorContext(cluster, user, flow, flowVersion3, + runid3, appName), te, remoteUser); hbi.flush(); } finally { @@ -379,8 +392,7 @@ public class TestHBaseStorageFlowActivity { Set entities = hbr.getEntities( new TimelineReaderContext(cluster, null, null, null, null, TimelineEntityType.YARN_FLOW_ACTIVITY.toString(), null), - new TimelineEntityFilters(10L, null, null, null, null, null, - null, null, null), + new TimelineEntityFilters.Builder().entityLimit(10L).build(), new TimelineDataToRetrieve()); assertEquals(1, entities.size()); for (TimelineEntity e : entities) { @@ -425,13 +437,13 @@ public class TestHBaseStorageFlowActivity { new FlowActivityRowKey(cluster, appCreatedTime, user, flow).getRowKey(); s.setStartRow(startRow); String clusterStop = cluster + "1"; - byte[] stopRow = - new FlowActivityRowKey(clusterStop, appCreatedTime, user, flow) - .getRowKey(); + byte[] stopRow = new FlowActivityRowKey(clusterStop, appCreatedTime, user, + flow).getRowKey(); s.setStopRow(stopRow); Connection conn = ConnectionFactory.createConnection(c1); - Table table1 = conn.getTable(TableName - .valueOf(FlowActivityTable.DEFAULT_TABLE_NAME)); + Table table1 = conn.getTable( + BaseTable.getTableName(c1, FlowActivityTable.TABLE_NAME_CONF_NAME, + FlowActivityTable.DEFAULT_TABLE_NAME)); ResultScanner scanner = table1.getScanner(s); int rowCount = 0; for (Result result : scanner) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java index 7f46a5a0fc8..1ad02e1886c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java @@ -19,7 +19,6 @@ package org.apache.hadoop.yarn.server.timelineservice.storage.flow; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -41,14 +40,16 @@ import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; -import org.apache.hadoop.hbase.regionserver.Region; import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.regionserver.Region; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.timelineservice.FlowRunEntity; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric; +import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorContext; import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve; import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters; import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext; @@ -57,12 +58,12 @@ import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineCompa import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList; import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList.Operator; import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelinePrefixFilter; +import org.apache.hadoop.yarn.server.timelineservice.storage.DataGeneratorForTest; import org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineReaderImpl; import org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl; import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field; -import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineSchemaCreator; +import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable; import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper; -import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineStorageUtils; import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTable; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -84,18 +85,14 @@ public class TestHBaseStorageFlowRun { Configuration conf = util.getConfiguration(); conf.setInt("hfile.format.version", 3); util.startMiniCluster(); - createSchema(); - } - - private static void createSchema() throws IOException { - TimelineSchemaCreator.createAllTables(util.getConfiguration(), false); + DataGeneratorForTest.createSchema(util.getConfiguration()); } @Test public void checkCoProcessorOff() throws IOException, InterruptedException { Configuration hbaseConf = util.getConfiguration(); - TableName table = TableName.valueOf(hbaseConf.get( - FlowRunTable.TABLE_NAME_CONF_NAME, FlowRunTable.DEFAULT_TABLE_NAME)); + TableName table = BaseTable.getTableName(hbaseConf, + FlowRunTable.TABLE_NAME_CONF_NAME, FlowRunTable.DEFAULT_TABLE_NAME); Connection conn = null; conn = ConnectionFactory.createConnection(hbaseConf); Admin admin = conn.getAdmin(); @@ -106,42 +103,42 @@ public class TestHBaseStorageFlowRun { // check the regions. // check in flow run table util.waitUntilAllRegionsAssigned(table); - HRegionServer server = util.getRSForFirstRegionInTable(table); - List regions = server.getOnlineRegions(table); - for (Region region : regions) { - assertTrue(HBaseTimelineStorageUtils.isFlowRunTable( - region.getRegionInfo(), hbaseConf)); - } + checkCoprocessorExists(table, true); } - table = TableName.valueOf(hbaseConf.get( + table = BaseTable.getTableName(hbaseConf, FlowActivityTable.TABLE_NAME_CONF_NAME, - FlowActivityTable.DEFAULT_TABLE_NAME)); + FlowActivityTable.DEFAULT_TABLE_NAME); if (admin.tableExists(table)) { // check the regions. // check in flow activity table util.waitUntilAllRegionsAssigned(table); - HRegionServer server = util.getRSForFirstRegionInTable(table); - List regions = server.getOnlineRegions(table); - for (Region region : regions) { - assertFalse(HBaseTimelineStorageUtils.isFlowRunTable( - region.getRegionInfo(), hbaseConf)); - } + checkCoprocessorExists(table, false); } - table = TableName.valueOf(hbaseConf.get( - EntityTable.TABLE_NAME_CONF_NAME, - EntityTable.DEFAULT_TABLE_NAME)); + table = BaseTable.getTableName(hbaseConf, EntityTable.TABLE_NAME_CONF_NAME, + EntityTable.DEFAULT_TABLE_NAME); if (admin.tableExists(table)) { // check the regions. // check in entity run table util.waitUntilAllRegionsAssigned(table); - HRegionServer server = util.getRSForFirstRegionInTable(table); - List regions = server.getOnlineRegions(table); - for (Region region : regions) { - assertFalse(HBaseTimelineStorageUtils.isFlowRunTable( - region.getRegionInfo(), hbaseConf)); + checkCoprocessorExists(table, false); + } + } + + private void checkCoprocessorExists(TableName table, boolean exists) + throws IOException, InterruptedException { + HRegionServer server = util.getRSForFirstRegionInTable(table); + List regions = server.getOnlineRegions(table); + for (Region region : regions) { + boolean found = false; + Set coprocs = region.getCoprocessorHost().getCoprocessors(); + for (String coprocName : coprocs) { + if (coprocName.contains("FlowRunCoprocessor")) { + found = true; + } } + assertEquals(found, exists); } } @@ -186,13 +183,18 @@ public class TestHBaseStorageFlowRun { try { hbi = new HBaseTimelineWriterImpl(); hbi.init(c1); - hbi.write(cluster, user, flow, flowVersion, runid, appName, te); + + UserGroupInformation remoteUser = + UserGroupInformation.createRemoteUser(user); + hbi.write(new TimelineCollectorContext(cluster, user, flow, flowVersion, + runid, appName), te, remoteUser); // write another entity with the right min start time te = new TimelineEntities(); te.addEntity(entityMinStartTime); appName = "application_100000000000_3333"; - hbi.write(cluster, user, flow, flowVersion, runid, appName, te); + hbi.write(new TimelineCollectorContext(cluster, user, flow, flowVersion, + runid, appName), te, remoteUser); // writer another entity for max end time TimelineEntity entityMaxEndTime = TestFlowDataGenerator @@ -200,7 +202,8 @@ public class TestHBaseStorageFlowRun { te = new TimelineEntities(); te.addEntity(entityMaxEndTime); appName = "application_100000000000_4444"; - hbi.write(cluster, user, flow, flowVersion, runid, appName, te); + hbi.write(new TimelineCollectorContext(cluster, user, flow, flowVersion, + runid, appName), te, remoteUser); // writer another entity with greater start time TimelineEntity entityGreaterStartTime = TestFlowDataGenerator @@ -208,7 +211,8 @@ public class TestHBaseStorageFlowRun { te = new TimelineEntities(); te.addEntity(entityGreaterStartTime); appName = "application_1000000000000000_2222"; - hbi.write(cluster, user, flow, flowVersion, runid, appName, te); + hbi.write(new TimelineCollectorContext(cluster, user, flow, flowVersion, + runid, appName), te, remoteUser); // flush everything to hbase hbi.flush(); @@ -220,8 +224,8 @@ public class TestHBaseStorageFlowRun { Connection conn = ConnectionFactory.createConnection(c1); // check in flow run table - Table table1 = conn.getTable(TableName - .valueOf(FlowRunTable.DEFAULT_TABLE_NAME)); + Table table1 = conn.getTable(BaseTable.getTableName(c1, + FlowRunTable.TABLE_NAME_CONF_NAME, FlowRunTable.DEFAULT_TABLE_NAME)); // scan the table and see that we get back the right min and max // timestamps byte[] startRow = new FlowRunRowKey(cluster, user, flow, runid).getRowKey(); @@ -292,15 +296,19 @@ public class TestHBaseStorageFlowRun { try { hbi = new HBaseTimelineWriterImpl(); hbi.init(c1); + UserGroupInformation remoteUser = + UserGroupInformation.createRemoteUser(user); String appName = "application_11111111111111_1111"; - hbi.write(cluster, user, flow, flowVersion, runid, appName, te); + hbi.write(new TimelineCollectorContext(cluster, user, flow, flowVersion, + runid, appName), te, remoteUser); // write another application with same metric to this flow te = new TimelineEntities(); TimelineEntity entityApp2 = TestFlowDataGenerator .getEntityMetricsApp2(System.currentTimeMillis()); te.addEntity(entityApp2); appName = "application_11111111111111_2222"; - hbi.write(cluster, user, flow, flowVersion, runid, appName, te); + hbi.write(new TimelineCollectorContext(cluster, user, flow, flowVersion, + runid, appName), te, remoteUser); hbi.flush(); } finally { if (hbi != null) { @@ -356,24 +364,24 @@ public class TestHBaseStorageFlowRun { /* * checks the batch limits on a scan */ - void checkFlowRunTableBatchLimit(String cluster, String user, - String flow, long runid, Configuration c1) throws IOException { + void checkFlowRunTableBatchLimit(String cluster, String user, String flow, + long runid, Configuration c1) throws IOException { Scan s = new Scan(); s.addFamily(FlowRunColumnFamily.INFO.getBytes()); - byte[] startRow = - new FlowRunRowKey(cluster, user, flow, runid).getRowKey(); + byte[] startRow = new FlowRunRowKey(cluster, user, flow, runid) + .getRowKey(); s.setStartRow(startRow); // set a batch limit int batchLimit = 2; s.setBatch(batchLimit); String clusterStop = cluster + "1"; - byte[] stopRow = - new FlowRunRowKey(clusterStop, user, flow, runid).getRowKey(); + byte[] stopRow = new FlowRunRowKey(clusterStop, user, flow, runid) + .getRowKey(); s.setStopRow(stopRow); Connection conn = ConnectionFactory.createConnection(c1); - Table table1 = conn - .getTable(TableName.valueOf(FlowRunTable.DEFAULT_TABLE_NAME)); + Table table1 = conn.getTable(BaseTable.getTableName(c1, + FlowRunTable.TABLE_NAME_CONF_NAME, FlowRunTable.DEFAULT_TABLE_NAME)); ResultScanner scanner = table1.getScanner(s); int loopCount = 0; @@ -517,8 +525,8 @@ public class TestHBaseStorageFlowRun { new FlowRunRowKey(clusterStop, user, flow, runid).getRowKey(); s.setStopRow(stopRow); Connection conn = ConnectionFactory.createConnection(c1); - Table table1 = conn.getTable(TableName - .valueOf(FlowRunTable.DEFAULT_TABLE_NAME)); + Table table1 = conn.getTable(BaseTable.getTableName(c1, + FlowRunTable.TABLE_NAME_CONF_NAME, FlowRunTable.DEFAULT_TABLE_NAME)); ResultScanner scanner = table1.getScanner(s); int rowCount = 0; @@ -561,15 +569,22 @@ public class TestHBaseStorageFlowRun { try { hbi = new HBaseTimelineWriterImpl(); hbi.init(c1); + UserGroupInformation remoteUser = + UserGroupInformation.createRemoteUser(user); + String appName = "application_11111111111111_1111"; - hbi.write(cluster, user, flow, flowVersion, 1002345678919L, appName, te); + hbi.write(new TimelineCollectorContext(cluster, user, flow, flowVersion, + 1002345678919L, appName), te, + remoteUser); // write another application with same metric to this flow te = new TimelineEntities(); TimelineEntity entityApp2 = TestFlowDataGenerator .getEntityMetricsApp2(System.currentTimeMillis()); te.addEntity(entityApp2); appName = "application_11111111111111_2222"; - hbi.write(cluster, user, flow, flowVersion, 1002345678918L, appName, te); + hbi.write(new TimelineCollectorContext(cluster, user, flow, flowVersion, + 1002345678918L, appName), te, + remoteUser); hbi.flush(); } finally { if (hbi != null) { @@ -589,7 +604,8 @@ public class TestHBaseStorageFlowRun { TimelineEntity entity = hbr.getEntity( new TimelineReaderContext(cluster, user, flow, 1002345678919L, null, TimelineEntityType.YARN_FLOW_RUN.toString(), null), - new TimelineDataToRetrieve(null, metricsToRetrieve, null, null)); + new TimelineDataToRetrieve(null, metricsToRetrieve, null, null, null, + null)); assertTrue(TimelineEntityType.YARN_FLOW_RUN.matches(entity.getType())); Set metrics = entity.getMetrics(); assertEquals(1, metrics.size()); @@ -613,8 +629,9 @@ public class TestHBaseStorageFlowRun { Set entities = hbr.getEntities( new TimelineReaderContext(cluster, user, flow, null, null, TimelineEntityType.YARN_FLOW_RUN.toString(), null), - new TimelineEntityFilters(), - new TimelineDataToRetrieve(null, metricsToRetrieve, null, null)); + new TimelineEntityFilters.Builder().build(), + new TimelineDataToRetrieve(null, metricsToRetrieve, null, null, null, + null)); assertEquals(2, entities.size()); int metricCnt = 0; for (TimelineEntity timelineEntity : entities) { @@ -646,15 +663,20 @@ public class TestHBaseStorageFlowRun { try { hbi = new HBaseTimelineWriterImpl(); hbi.init(c1); + UserGroupInformation remoteUser = + UserGroupInformation.createRemoteUser(user); + String appName = "application_11111111111111_1111"; - hbi.write(cluster, user, flow, flowVersion, runid, appName, te); + hbi.write(new TimelineCollectorContext(cluster, user, flow, flowVersion, + runid, appName), te, remoteUser); // write another application with same metric to this flow te = new TimelineEntities(); TimelineEntity entityApp2 = TestFlowDataGenerator .getEntityMetricsApp2(System.currentTimeMillis()); te.addEntity(entityApp2); appName = "application_11111111111111_2222"; - hbi.write(cluster, user, flow, flowVersion, runid, appName, te); + hbi.write(new TimelineCollectorContext(cluster, user, flow, flowVersion, + runid, appName), te, remoteUser); hbi.flush(); } finally { if (hbi != null) { @@ -674,7 +696,7 @@ public class TestHBaseStorageFlowRun { Set entities = hbr.getEntities( new TimelineReaderContext(cluster, user, flow, runid, null, TimelineEntityType.YARN_FLOW_RUN.toString(), null), - new TimelineEntityFilters(), + new TimelineEntityFilters.Builder().build(), new TimelineDataToRetrieve()); assertEquals(1, entities.size()); for (TimelineEntity timelineEntity : entities) { @@ -684,8 +706,9 @@ public class TestHBaseStorageFlowRun { entities = hbr.getEntities( new TimelineReaderContext(cluster, user, flow, runid, null, TimelineEntityType.YARN_FLOW_RUN.toString(), null), - new TimelineEntityFilters(), new TimelineDataToRetrieve(null, null, - EnumSet.of(Field.METRICS), null)); + new TimelineEntityFilters.Builder().build(), + new TimelineDataToRetrieve(null, null, + EnumSet.of(Field.METRICS), null, null, null)); assertEquals(1, entities.size()); for (TimelineEntity timelineEntity : entities) { Set timelineMetrics = timelineEntity.getMetrics(); @@ -739,6 +762,8 @@ public class TestHBaseStorageFlowRun { try { hbi = new HBaseTimelineWriterImpl(); hbi.init(c1); + UserGroupInformation remoteUser = + UserGroupInformation.createRemoteUser(user); for (int i = start; i < count; i++) { String appName = "application_1060350000000_" + appIdSuffix; @@ -748,7 +773,8 @@ public class TestHBaseStorageFlowRun { te1.addEntity(entityApp1); entityApp2 = TestFlowDataGenerator.getMaxFlushEntity(insertTs); te1.addEntity(entityApp2); - hbi.write(cluster, user, flow, flowVersion, runid, appName, te1); + hbi.write(new TimelineCollectorContext(cluster, user, flow, flowVersion, + runid, appName), te1, remoteUser); Thread.sleep(1); appName = "application_1001199480000_7" + appIdSuffix; @@ -760,7 +786,9 @@ public class TestHBaseStorageFlowRun { entityApp2 = TestFlowDataGenerator.getMaxFlushEntity(insertTs); te1.addEntity(entityApp2); - hbi.write(cluster, user, flow, flowVersion, runid, appName, te1); + hbi.write(new TimelineCollectorContext(cluster, user, flow, flowVersion, + runid, appName), te1, + remoteUser); if (i % 1000 == 0) { hbi.flush(); checkMinMaxFlush(c1, minTS, startTs, count, cluster, user, flow, @@ -782,8 +810,8 @@ public class TestHBaseStorageFlowRun { boolean checkMax) throws IOException { Connection conn = ConnectionFactory.createConnection(c1); // check in flow run table - Table table1 = conn.getTable(TableName - .valueOf(FlowRunTable.DEFAULT_TABLE_NAME)); + Table table1 = conn.getTable(BaseTable.getTableName(c1, + FlowRunTable.TABLE_NAME_CONF_NAME, FlowRunTable.DEFAULT_TABLE_NAME)); // scan the table and see that we get back the right min and max // timestamps byte[] startRow = new FlowRunRowKey(cluster, user, flow, runid).getRowKey(); @@ -828,16 +856,23 @@ public class TestHBaseStorageFlowRun { try { hbi = new HBaseTimelineWriterImpl(); hbi.init(c1); - hbi.write(cluster, user, flow, "CF7022C10F1354", 1002345678919L, - "application_11111111111111_1111", te); + UserGroupInformation remoteUser = + UserGroupInformation.createRemoteUser(user); + + hbi.write( + new TimelineCollectorContext(cluster, user, flow, "CF7022C10F1354", + 1002345678919L, "application_11111111111111_1111"), + te, remoteUser); // write another application with same metric to this flow te = new TimelineEntities(); TimelineEntity entityApp2 = TestFlowDataGenerator.getEntityMetricsApp2( System.currentTimeMillis()); entityApp2.setCreatedTime(1425016502000L); te.addEntity(entityApp2); - hbi.write(cluster, user, flow, "CF7022C10F1354", 1002345678918L, - "application_11111111111111_2222", te); + hbi.write( + new TimelineCollectorContext(cluster, user, flow, "CF7022C10F1354", + 1002345678918L, "application_11111111111111_2222"), + te, remoteUser); hbi.flush(); } finally { if (hbi != null) { @@ -855,8 +890,9 @@ public class TestHBaseStorageFlowRun { Set entities = hbr.getEntities( new TimelineReaderContext(cluster, user, flow, null, null, TimelineEntityType.YARN_FLOW_RUN.toString(), null), - new TimelineEntityFilters(null, 1425016501000L, 1425016502001L, null, - null, null, null, null, null), new TimelineDataToRetrieve()); + new TimelineEntityFilters.Builder().createdTimeBegin(1425016501000L) + .createTimeEnd(1425016502001L).build(), + new TimelineDataToRetrieve()); assertEquals(2, entities.size()); for (TimelineEntity entity : entities) { if (!entity.getId().equals("user2@flow_name2/1002345678918") && @@ -868,8 +904,9 @@ public class TestHBaseStorageFlowRun { entities = hbr.getEntities( new TimelineReaderContext(cluster, user, flow, null, null, TimelineEntityType.YARN_FLOW_RUN.toString(), null), - new TimelineEntityFilters(null, 1425016501050L, null, null, null, - null, null, null, null), new TimelineDataToRetrieve()); + new TimelineEntityFilters.Builder().createdTimeBegin(1425016501050L) + .build(), + new TimelineDataToRetrieve()); assertEquals(1, entities.size()); for (TimelineEntity entity : entities) { if (!entity.getId().equals("user2@flow_name2/1002345678918")) { @@ -879,8 +916,9 @@ public class TestHBaseStorageFlowRun { entities = hbr.getEntities( new TimelineReaderContext(cluster, user, flow, null, null, TimelineEntityType.YARN_FLOW_RUN.toString(), null), - new TimelineEntityFilters(null, null, 1425016501050L, null, null, - null, null, null, null), new TimelineDataToRetrieve()); + new TimelineEntityFilters.Builder().createTimeEnd(1425016501050L) + .build(), + new TimelineDataToRetrieve()); assertEquals(1, entities.size()); for (TimelineEntity entity : entities) { if (!entity.getId().equals("user2@flow_name2/1002345678919")) { @@ -910,15 +948,22 @@ public class TestHBaseStorageFlowRun { try { hbi = new HBaseTimelineWriterImpl(); hbi.init(c1); - hbi.write(cluster, user, flow, "CF7022C10F1354", 1002345678919L, - "application_11111111111111_1111", te); + UserGroupInformation remoteUser = + UserGroupInformation.createRemoteUser(user); + + hbi.write( + new TimelineCollectorContext(cluster, user, flow, "CF7022C10F1354", + 1002345678919L, "application_11111111111111_1111"), + te, remoteUser); // write another application with same metric to this flow te = new TimelineEntities(); TimelineEntity entityApp2 = TestFlowDataGenerator.getEntityMetricsApp2( System.currentTimeMillis()); te.addEntity(entityApp2); - hbi.write(cluster, user, flow, "CF7022C10F1354", 1002345678918L, - "application_11111111111111_2222", te); + hbi.write( + new TimelineCollectorContext(cluster, user, flow, "CF7022C10F1354", + 1002345678918L, "application_11111111111111_2222"), + te, remoteUser); hbi.flush(); } finally { if (hbi != null) { @@ -946,9 +991,10 @@ public class TestHBaseStorageFlowRun { Set entities = hbr.getEntities( new TimelineReaderContext(cluster, user, flow, null, null, TimelineEntityType.YARN_FLOW_RUN.toString(), null), - new TimelineEntityFilters(null, null, null, null, null, null, null, - metricFilterList, null), new TimelineDataToRetrieve(null, null, - EnumSet.of(Field.METRICS), null)); + new TimelineEntityFilters.Builder().metricFilters(metricFilterList) + .build(), + new TimelineDataToRetrieve(null, null, + EnumSet.of(Field.METRICS), null, null, null)); assertEquals(2, entities.size()); int metricCnt = 0; for (TimelineEntity entity : entities) { @@ -963,9 +1009,10 @@ public class TestHBaseStorageFlowRun { entities = hbr.getEntities( new TimelineReaderContext(cluster, user, flow, null, null, TimelineEntityType.YARN_FLOW_RUN.toString(), null), - new TimelineEntityFilters(null, null, null, null, null, null, null, - metricFilterList1, null), new TimelineDataToRetrieve(null, null, - EnumSet.of(Field.METRICS), null)); + new TimelineEntityFilters.Builder().metricFilters(metricFilterList1) + .build(), + new TimelineDataToRetrieve(null, null, + EnumSet.of(Field.METRICS), null, null, null)); assertEquals(1, entities.size()); metricCnt = 0; for (TimelineEntity entity : entities) { @@ -979,9 +1026,10 @@ public class TestHBaseStorageFlowRun { entities = hbr.getEntities( new TimelineReaderContext(cluster, user, flow, null, null, TimelineEntityType.YARN_FLOW_RUN.toString(), null), - new TimelineEntityFilters(null, null, null, null, null, null, null, - metricFilterList2, null), new TimelineDataToRetrieve(null, null, - EnumSet.of(Field.METRICS), null)); + new TimelineEntityFilters.Builder().metricFilters(metricFilterList2) + .build(), + new TimelineDataToRetrieve(null, null, + EnumSet.of(Field.METRICS), null, null, null)); assertEquals(0, entities.size()); TimelineFilterList metricFilterList3 = new TimelineFilterList( @@ -989,9 +1037,10 @@ public class TestHBaseStorageFlowRun { entities = hbr.getEntities( new TimelineReaderContext(cluster, user, flow, null, null, TimelineEntityType.YARN_FLOW_RUN.toString(), null), - new TimelineEntityFilters(null, null, null, null, null, null, null, - metricFilterList3, null), new TimelineDataToRetrieve(null, null, - EnumSet.of(Field.METRICS), null)); + new TimelineEntityFilters.Builder().metricFilters(metricFilterList3) + .build(), + new TimelineDataToRetrieve(null, null, + EnumSet.of(Field.METRICS), null, null, null)); assertEquals(0, entities.size()); TimelineFilterList list3 = new TimelineFilterList(); @@ -1010,10 +1059,10 @@ public class TestHBaseStorageFlowRun { entities = hbr.getEntities( new TimelineReaderContext(cluster, user, flow, null, null, TimelineEntityType.YARN_FLOW_RUN.toString(), null), - new TimelineEntityFilters(null, null, null, null, null, null, null, - metricFilterList4, null), + new TimelineEntityFilters.Builder().metricFilters(metricFilterList4) + .build(), new TimelineDataToRetrieve(null, metricsToRetrieve, - EnumSet.of(Field.ALL), null)); + EnumSet.of(Field.ALL), null, null, null)); assertEquals(2, entities.size()); metricCnt = 0; for (TimelineEntity entity : entities) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRunCompaction.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRunCompaction.java index eb18e28243d..0ef8260dee4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRunCompaction.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRunCompaction.java @@ -35,7 +35,6 @@ import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.Tag; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; @@ -49,10 +48,13 @@ import org.apache.hadoop.hbase.regionserver.Region; import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; +import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorContext; +import org.apache.hadoop.yarn.server.timelineservice.storage.DataGeneratorForTest; import org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl; -import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineSchemaCreator; +import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable; import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper; import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineStorageUtils; import org.apache.hadoop.yarn.server.timelineservice.storage.common.LongConverter; @@ -69,8 +71,8 @@ public class TestHBaseStorageFlowRunCompaction { private static HBaseTestingUtility util; - private static final String METRIC_1 = "MAP_SLOT_MILLIS"; - private static final String METRIC_2 = "HDFS_BYTES_READ"; + private static final String METRIC1 = "MAP_SLOT_MILLIS"; + private static final String METRIC2 = "HDFS_BYTES_READ"; private final byte[] aRowKey = Bytes.toBytes("a"); private final byte[] aFamily = Bytes.toBytes("family"); @@ -82,15 +84,12 @@ public class TestHBaseStorageFlowRunCompaction { Configuration conf = util.getConfiguration(); conf.setInt("hfile.format.version", 3); util.startMiniCluster(); - createSchema(); + DataGeneratorForTest.createSchema(util.getConfiguration()); } - private static void createSchema() throws IOException { - TimelineSchemaCreator.createAllTables(util.getConfiguration(), false); - } - - /** Writes non numeric data into flow run table - * reads it back. + /** + * writes non numeric data into flow run table. + * reads it back * * @throws Exception */ @@ -106,11 +105,10 @@ public class TestHBaseStorageFlowRunCompaction { p.addColumn(FlowRunColumnFamily.INFO.getBytes(), columnNameBytes, valueBytes); Configuration hbaseConf = util.getConfiguration(); - TableName table = TableName.valueOf(hbaseConf.get( - FlowRunTable.TABLE_NAME_CONF_NAME, FlowRunTable.DEFAULT_TABLE_NAME)); Connection conn = null; conn = ConnectionFactory.createConnection(hbaseConf); - Table flowRunTable = conn.getTable(table); + Table flowRunTable = conn.getTable(BaseTable.getTableName(hbaseConf, + FlowRunTable.TABLE_NAME_CONF_NAME, FlowRunTable.DEFAULT_TABLE_NAME)); flowRunTable.put(p); Get g = new Get(rowKeyBytes); @@ -156,11 +154,10 @@ public class TestHBaseStorageFlowRunCompaction { value4Bytes); Configuration hbaseConf = util.getConfiguration(); - TableName table = TableName.valueOf(hbaseConf.get( - FlowRunTable.TABLE_NAME_CONF_NAME, FlowRunTable.DEFAULT_TABLE_NAME)); Connection conn = null; conn = ConnectionFactory.createConnection(hbaseConf); - Table flowRunTable = conn.getTable(table); + Table flowRunTable = conn.getTable(BaseTable.getTableName(hbaseConf, + FlowRunTable.TABLE_NAME_CONF_NAME, FlowRunTable.DEFAULT_TABLE_NAME)); flowRunTable.put(p); String rowKey2 = "nonNumericRowKey2"; @@ -262,7 +259,6 @@ public class TestHBaseStorageFlowRunCompaction { .getFamilyMap(FlowRunColumnFamily.INFO.getBytes()); // we expect all back in one next call assertEquals(4, values.size()); - System.out.println(" values size " + values.size() + " " + batchLimit); rowCount++; } // should get back 1 row with each invocation @@ -286,9 +282,12 @@ public class TestHBaseStorageFlowRunCompaction { Configuration c1 = util.getConfiguration(); TimelineEntities te1 = null; TimelineEntity entityApp1 = null; + UserGroupInformation remoteUser = + UserGroupInformation.createRemoteUser(user); try { hbi = new HBaseTimelineWriterImpl(); hbi.init(c1); + // now insert count * ( 100 + 100) metrics // each call to getEntityMetricsApp1 brings back 100 values // of metric1 and 100 of metric2 @@ -298,14 +297,16 @@ public class TestHBaseStorageFlowRunCompaction { te1 = new TimelineEntities(); entityApp1 = TestFlowDataGenerator.getEntityMetricsApp1(insertTs, c1); te1.addEntity(entityApp1); - hbi.write(cluster, user, flow, flowVersion, runid, appName, te1); + hbi.write(new TimelineCollectorContext(cluster, user, flow, flowVersion, + runid, appName), te1, remoteUser); appName = "application_2048000000000_7" + appIdSuffix; insertTs++; te1 = new TimelineEntities(); entityApp1 = TestFlowDataGenerator.getEntityMetricsApp2(insertTs); te1.addEntity(entityApp1); - hbi.write(cluster, user, flow, flowVersion, runid, appName, te1); + hbi.write(new TimelineCollectorContext(cluster, user, flow, flowVersion, + runid, appName), te1, remoteUser); } } finally { String appName = "application_10240000000000_" + appIdSuffix; @@ -314,17 +315,19 @@ public class TestHBaseStorageFlowRunCompaction { insertTs + 1, c1); te1.addEntity(entityApp1); if (hbi != null) { - hbi.write(cluster, user, flow, flowVersion, runid, appName, te1); + hbi.write(new TimelineCollectorContext(cluster, user, flow, flowVersion, + runid, appName), te1, remoteUser); hbi.flush(); hbi.close(); } } // check in flow run table - HRegionServer server = util.getRSForFirstRegionInTable(TableName - .valueOf(FlowRunTable.DEFAULT_TABLE_NAME)); - List regions = server.getOnlineRegions(TableName - .valueOf(FlowRunTable.DEFAULT_TABLE_NAME)); + HRegionServer server = util.getRSForFirstRegionInTable( + BaseTable.getTableName(c1, FlowRunTable.TABLE_NAME_CONF_NAME, + FlowRunTable.DEFAULT_TABLE_NAME)); + List regions = server.getOnlineRegions(BaseTable.getTableName(c1, + FlowRunTable.TABLE_NAME_CONF_NAME, FlowRunTable.DEFAULT_TABLE_NAME)); assertTrue("Didn't find any regions for primary table!", regions.size() > 0); // flush and compact all the regions of the primary table @@ -349,8 +352,8 @@ public class TestHBaseStorageFlowRunCompaction { new FlowRunRowKey(clusterStop, user, flow, runid).getRowKey(); s.setStopRow(stopRow); Connection conn = ConnectionFactory.createConnection(c1); - Table table1 = conn.getTable(TableName - .valueOf(FlowRunTable.DEFAULT_TABLE_NAME)); + Table table1 = conn.getTable(BaseTable.getTableName(c1, + FlowRunTable.TABLE_NAME_CONF_NAME, FlowRunTable.DEFAULT_TABLE_NAME)); ResultScanner scanner = table1.getScanner(s); int rowCount = 0; @@ -364,13 +367,13 @@ public class TestHBaseStorageFlowRunCompaction { rowCount++; // check metric1 byte[] q = ColumnHelper.getColumnQualifier( - FlowRunColumnPrefix.METRIC.getColumnPrefixBytes(), METRIC_1); + FlowRunColumnPrefix.METRIC.getColumnPrefixBytes(), METRIC1); assertTrue(values.containsKey(q)); assertEquals(141, Bytes.toLong(values.get(q))); // check metric2 q = ColumnHelper.getColumnQualifier( - FlowRunColumnPrefix.METRIC.getColumnPrefixBytes(), METRIC_2); + FlowRunColumnPrefix.METRIC.getColumnPrefixBytes(), METRIC2); assertTrue(values.containsKey(q)); assertEquals(57, Bytes.toLong(values.get(q))); } @@ -587,9 +590,9 @@ public class TestHBaseStorageFlowRunCompaction { long cellTsFinalStart = 10001120L; long cellTsFinal = cellTsFinalStart; - long cellTsFinalStartNotExpire = - TimestampGenerator.getSupplementedTimestamp( - System.currentTimeMillis(), "application_10266666661166_118821"); + long cellTsFinalStartNotExpire = TimestampGenerator + .getSupplementedTimestamp(System.currentTimeMillis(), + "application_10266666661166_118821"); long cellTsFinalNotExpire = cellTsFinalStartNotExpire; long cellTsNotFinalStart = currentTimestamp - 5; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineFilterUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineFilterUtils.java index a934a3dbf19..8b46d324135 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineFilterUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineFilterUtils.java @@ -181,6 +181,23 @@ public final class TimelineFilterUtils { return list; } + /** + * Creates a HBase {@link SingleColumnValueFilter} with specified column. + * @param Describes the type of column prefix. + * @param column Column which value to be filtered. + * @param value Value to be filtered. + * @param op Compare operator + * @return a SingleColumnValue Filter + * @throws IOException if any exception. + */ + public static Filter createHBaseSingleColValueFilter(Column column, + Object value, CompareOp op) throws IOException { + Filter singleColValFilter = createHBaseSingleColValueFilter( + column.getColumnFamilyBytes(), column.getColumnQualifierBytes(), + column.getValueConverter().encodeValue(value), op, true); + return singleColValFilter; + } + /** * Creates a HBase {@link SingleColumnValueFilter}. * diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java index dc50f42396f..1ebfab29cad 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java @@ -22,7 +22,6 @@ import java.io.IOException; import java.util.Set; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.service.AbstractService; @@ -30,6 +29,8 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve; import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters; import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext; +import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineStorageUtils; +import org.apache.hadoop.yarn.server.timelineservice.storage.reader.EntityTypeReader; import org.apache.hadoop.yarn.server.timelineservice.storage.reader.TimelineEntityReader; import org.apache.hadoop.yarn.server.timelineservice.storage.reader.TimelineEntityReaderFactory; import org.slf4j.Logger; @@ -54,7 +55,7 @@ public class HBaseTimelineReaderImpl @Override public void serviceInit(Configuration conf) throws Exception { super.serviceInit(conf); - hbaseConf = HBaseConfiguration.create(conf); + hbaseConf = HBaseTimelineStorageUtils.getTimelineServiceHBaseConf(conf); conn = ConnectionFactory.createConnection(hbaseConf); } @@ -85,4 +86,11 @@ public class HBaseTimelineReaderImpl filters, dataToRetrieve); return reader.readEntities(hbaseConf, conn); } + + @Override + public Set getEntityTypes(TimelineReaderContext context) + throws IOException { + EntityTypeReader reader = new EntityTypeReader(context); + return reader.readEntityTypes(hbaseConf, conn); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java index afa58cb88a6..9e9134cf3eb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java @@ -24,26 +24,28 @@ import java.util.Set; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.yarn.api.records.timelineservice.ApplicationEntity; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; -import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse; import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants; +import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorContext; import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumn; import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumnPrefix; import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationRowKey; import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationTable; -import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowColumn; +import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowColumnPrefix; import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowRowKey; import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowTable; import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix; import org.apache.hadoop.yarn.server.timelineservice.storage.common.EventColumnName; +import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineStorageUtils; import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter; import org.apache.hadoop.yarn.server.timelineservice.storage.common.LongKeyConverter; import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator; @@ -63,6 +65,10 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunColumn; import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunColumnPrefix; import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunRowKey; import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunTable; +import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationColumn; +import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationColumnPrefix; +import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationRowKey; +import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationTable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -85,6 +91,7 @@ public class HBaseTimelineWriterImpl extends AbstractService implements private TypedBufferedMutator applicationTable; private TypedBufferedMutator flowActivityTable; private TypedBufferedMutator flowRunTable; + private TypedBufferedMutator subApplicationTable; /** * Used to convert strings key components to and from storage format. @@ -97,6 +104,10 @@ public class HBaseTimelineWriterImpl extends AbstractService implements */ private final KeyConverter longKeyConverter = new LongKeyConverter(); + private enum Tables { + APPLICATION_TABLE, ENTITY_TABLE, SUBAPPLICATION_TABLE + }; + public HBaseTimelineWriterImpl() { super(HBaseTimelineWriterImpl.class.getName()); } @@ -107,7 +118,8 @@ public class HBaseTimelineWriterImpl extends AbstractService implements @Override protected void serviceInit(Configuration conf) throws Exception { super.serviceInit(conf); - Configuration hbaseConf = HBaseConfiguration.create(conf); + Configuration hbaseConf = + HBaseTimelineStorageUtils.getTimelineServiceHBaseConf(conf); conn = ConnectionFactory.createConnection(hbaseConf); entityTable = new EntityTable().getTableMutator(hbaseConf, conn); appToFlowTable = new AppToFlowTable().getTableMutator(hbaseConf, conn); @@ -115,17 +127,28 @@ public class HBaseTimelineWriterImpl extends AbstractService implements flowRunTable = new FlowRunTable().getTableMutator(hbaseConf, conn); flowActivityTable = new FlowActivityTable().getTableMutator(hbaseConf, conn); + subApplicationTable = + new SubApplicationTable().getTableMutator(hbaseConf, conn); } /** * Stores the entire information in TimelineEntities to the timeline store. */ @Override - public TimelineWriteResponse write(String clusterId, String userId, - String flowName, String flowVersion, long flowRunId, String appId, - TimelineEntities data) throws IOException { + public TimelineWriteResponse write(TimelineCollectorContext context, + TimelineEntities data, UserGroupInformation callerUgi) + throws IOException { TimelineWriteResponse putStatus = new TimelineWriteResponse(); + + String clusterId = context.getClusterId(); + String userId = context.getUserId(); + String flowName = context.getFlowName(); + String flowVersion = context.getFlowVersion(); + long flowRunId = context.getFlowRunId(); + String appId = context.getAppId(); + String subApplicationUser = callerUgi.getShortUserName(); + // defensive coding to avoid NPE during row key construction if ((flowName == null) || (appId == null) || (clusterId == null) || (userId == null)) { @@ -144,43 +167,45 @@ public class HBaseTimelineWriterImpl extends AbstractService implements // if the entity is the application, the destination is the application // table - boolean isApplication = isApplicationEntity(te); + boolean isApplication = ApplicationEntity.isApplicationEntity(te); byte[] rowKey; if (isApplication) { ApplicationRowKey applicationRowKey = new ApplicationRowKey(clusterId, userId, flowName, flowRunId, appId); rowKey = applicationRowKey.getRowKey(); + store(rowKey, te, flowVersion, Tables.APPLICATION_TABLE); } else { EntityRowKey entityRowKey = new EntityRowKey(clusterId, userId, flowName, flowRunId, appId, - te.getType(), te.getId()); + te.getType(), te.getIdPrefix(), te.getId()); rowKey = entityRowKey.getRowKey(); + store(rowKey, te, flowVersion, Tables.ENTITY_TABLE); } - storeInfo(rowKey, te, flowVersion, isApplication); - storeEvents(rowKey, te.getEvents(), isApplication); - storeConfig(rowKey, te.getConfigs(), isApplication); - storeMetrics(rowKey, te.getMetrics(), isApplication); - storeRelations(rowKey, te, isApplication); + if (!isApplication && !userId.equals(subApplicationUser)) { + SubApplicationRowKey subApplicationRowKey = + new SubApplicationRowKey(subApplicationUser, clusterId, + te.getType(), te.getIdPrefix(), te.getId(), userId); + rowKey = subApplicationRowKey.getRowKey(); + store(rowKey, te, flowVersion, Tables.SUBAPPLICATION_TABLE); + } if (isApplication) { TimelineEvent event = - getApplicationEvent(te, + ApplicationEntity.getApplicationEvent(te, ApplicationMetricsConstants.CREATED_EVENT_TYPE); FlowRunRowKey flowRunRowKey = new FlowRunRowKey(clusterId, userId, flowName, flowRunId); if (event != null) { - AppToFlowRowKey appToFlowRowKey = - new AppToFlowRowKey(clusterId, appId); - onApplicationCreated(flowRunRowKey, appToFlowRowKey, appId, userId, + onApplicationCreated(flowRunRowKey, clusterId, appId, userId, flowVersion, te, event.getTimestamp()); } // if it's an application entity, store metrics storeFlowMetricsAppRunning(flowRunRowKey, appId, te); // if application has finished, store it's finish time and write final // values of all metrics - event = getApplicationEvent(te, + event = ApplicationEntity.getApplicationEvent(te, ApplicationMetricsConstants.FINISHED_EVENT_TYPE); if (event != null) { onApplicationFinished(flowRunRowKey, flowVersion, appId, te, @@ -192,18 +217,22 @@ public class HBaseTimelineWriterImpl extends AbstractService implements } private void onApplicationCreated(FlowRunRowKey flowRunRowKey, - AppToFlowRowKey appToFlowRowKey, String appId, String userId, - String flowVersion, TimelineEntity te, long appCreatedTimeStamp) + String clusterId, String appId, String userId, String flowVersion, + TimelineEntity te, long appCreatedTimeStamp) throws IOException { String flowName = flowRunRowKey.getFlowName(); Long flowRunId = flowRunRowKey.getFlowRunId(); // store in App to flow table + AppToFlowRowKey appToFlowRowKey = new AppToFlowRowKey(appId); byte[] rowKey = appToFlowRowKey.getRowKey(); - AppToFlowColumn.FLOW_ID.store(rowKey, appToFlowTable, null, flowName); - AppToFlowColumn.FLOW_RUN_ID.store(rowKey, appToFlowTable, null, flowRunId); - AppToFlowColumn.USER_ID.store(rowKey, appToFlowTable, null, userId); + AppToFlowColumnPrefix.FLOW_NAME.store(rowKey, appToFlowTable, clusterId, + null, flowName); + AppToFlowColumnPrefix.FLOW_RUN_ID.store(rowKey, appToFlowTable, clusterId, + null, flowRunId); + AppToFlowColumnPrefix.USER_ID.store(rowKey, appToFlowTable, clusterId, null, + userId); // store in flow run table storeAppCreatedInFlowRunTable(flowRunRowKey, appId, te); @@ -301,72 +330,108 @@ public class HBaseTimelineWriterImpl extends AbstractService implements } } - private void storeRelations(byte[] rowKey, TimelineEntity te, - boolean isApplication) throws IOException { - if (isApplication) { - storeRelations(rowKey, te.getIsRelatedToEntities(), - ApplicationColumnPrefix.IS_RELATED_TO, applicationTable); - storeRelations(rowKey, te.getRelatesToEntities(), - ApplicationColumnPrefix.RELATES_TO, applicationTable); - } else { - storeRelations(rowKey, te.getIsRelatedToEntities(), - EntityColumnPrefix.IS_RELATED_TO, entityTable); - storeRelations(rowKey, te.getRelatesToEntities(), - EntityColumnPrefix.RELATES_TO, entityTable); - } - } - /** * Stores the Relations from the {@linkplain TimelineEntity} object. */ private void storeRelations(byte[] rowKey, - Map> connectedEntities, - ColumnPrefix columnPrefix, TypedBufferedMutator table) - throws IOException { - for (Map.Entry> connectedEntity : connectedEntities - .entrySet()) { - // id3?id4?id5 - String compoundValue = - Separator.VALUES.joinEncoded(connectedEntity.getValue()); - columnPrefix.store(rowKey, table, - stringKeyConverter.encode(connectedEntity.getKey()), null, - compoundValue); + Map> connectedEntities, ColumnPrefix columnPrefix, + TypedBufferedMutator table) throws IOException { + if (connectedEntities != null) { + for (Map.Entry> connectedEntity : connectedEntities + .entrySet()) { + // id3?id4?id5 + String compoundValue = + Separator.VALUES.joinEncoded(connectedEntity.getValue()); + columnPrefix.store(rowKey, table, + stringKeyConverter.encode(connectedEntity.getKey()), null, + compoundValue); + } } } /** * Stores information from the {@linkplain TimelineEntity} object. */ - private void storeInfo(byte[] rowKey, TimelineEntity te, String flowVersion, - boolean isApplication) throws IOException { - - if (isApplication) { + private void store(byte[] rowKey, TimelineEntity te, + String flowVersion, + Tables table) throws IOException { + switch (table) { + case APPLICATION_TABLE: ApplicationColumn.ID.store(rowKey, applicationTable, null, te.getId()); ApplicationColumn.CREATED_TIME.store(rowKey, applicationTable, null, te.getCreatedTime()); ApplicationColumn.FLOW_VERSION.store(rowKey, applicationTable, null, flowVersion); - Map info = te.getInfo(); - if (info != null) { - for (Map.Entry entry : info.entrySet()) { - ApplicationColumnPrefix.INFO.store(rowKey, applicationTable, - stringKeyConverter.encode(entry.getKey()), null, - entry.getValue()); - } - } - } else { + storeInfo(rowKey, te.getInfo(), flowVersion, ApplicationColumnPrefix.INFO, + applicationTable); + storeMetrics(rowKey, te.getMetrics(), ApplicationColumnPrefix.METRIC, + applicationTable); + storeEvents(rowKey, te.getEvents(), ApplicationColumnPrefix.EVENT, + applicationTable); + storeConfig(rowKey, te.getConfigs(), ApplicationColumnPrefix.CONFIG, + applicationTable); + storeRelations(rowKey, te.getIsRelatedToEntities(), + ApplicationColumnPrefix.IS_RELATED_TO, applicationTable); + storeRelations(rowKey, te.getRelatesToEntities(), + ApplicationColumnPrefix.RELATES_TO, applicationTable); + break; + case ENTITY_TABLE: EntityColumn.ID.store(rowKey, entityTable, null, te.getId()); EntityColumn.TYPE.store(rowKey, entityTable, null, te.getType()); EntityColumn.CREATED_TIME.store(rowKey, entityTable, null, te.getCreatedTime()); EntityColumn.FLOW_VERSION.store(rowKey, entityTable, null, flowVersion); - Map info = te.getInfo(); - if (info != null) { - for (Map.Entry entry : info.entrySet()) { - EntityColumnPrefix.INFO.store(rowKey, entityTable, - stringKeyConverter.encode(entry.getKey()), null, - entry.getValue()); - } + storeInfo(rowKey, te.getInfo(), flowVersion, EntityColumnPrefix.INFO, + entityTable); + storeMetrics(rowKey, te.getMetrics(), EntityColumnPrefix.METRIC, + entityTable); + storeEvents(rowKey, te.getEvents(), EntityColumnPrefix.EVENT, + entityTable); + storeConfig(rowKey, te.getConfigs(), EntityColumnPrefix.CONFIG, + entityTable); + storeRelations(rowKey, te.getIsRelatedToEntities(), + EntityColumnPrefix.IS_RELATED_TO, entityTable); + storeRelations(rowKey, te.getRelatesToEntities(), + EntityColumnPrefix.RELATES_TO, entityTable); + break; + case SUBAPPLICATION_TABLE: + SubApplicationColumn.ID.store(rowKey, subApplicationTable, null, + te.getId()); + SubApplicationColumn.TYPE.store(rowKey, subApplicationTable, null, + te.getType()); + SubApplicationColumn.CREATED_TIME.store(rowKey, subApplicationTable, null, + te.getCreatedTime()); + SubApplicationColumn.FLOW_VERSION.store(rowKey, subApplicationTable, null, + flowVersion); + storeInfo(rowKey, te.getInfo(), flowVersion, + SubApplicationColumnPrefix.INFO, subApplicationTable); + storeMetrics(rowKey, te.getMetrics(), SubApplicationColumnPrefix.METRIC, + subApplicationTable); + storeEvents(rowKey, te.getEvents(), SubApplicationColumnPrefix.EVENT, + subApplicationTable); + storeConfig(rowKey, te.getConfigs(), SubApplicationColumnPrefix.CONFIG, + subApplicationTable); + storeRelations(rowKey, te.getIsRelatedToEntities(), + SubApplicationColumnPrefix.IS_RELATED_TO, subApplicationTable); + storeRelations(rowKey, te.getRelatesToEntities(), + SubApplicationColumnPrefix.RELATES_TO, subApplicationTable); + break; + default: + LOG.info("Invalid table name provided."); + break; + } + } + + /** + * stores the info information from {@linkplain TimelineEntity}. + */ + private void storeInfo(byte[] rowKey, Map info, + String flowVersion, ColumnPrefix columnPrefix, + TypedBufferedMutator table) throws IOException { + if (info != null) { + for (Map.Entry entry : info.entrySet()) { + columnPrefix.store(rowKey, table, + stringKeyConverter.encode(entry.getKey()), null, entry.getValue()); } } } @@ -374,19 +439,13 @@ public class HBaseTimelineWriterImpl extends AbstractService implements /** * stores the config information from {@linkplain TimelineEntity}. */ - private void storeConfig(byte[] rowKey, Map config, - boolean isApplication) throws IOException { - if (config == null) { - return; - } - for (Map.Entry entry : config.entrySet()) { - byte[] configKey = stringKeyConverter.encode(entry.getKey()); - if (isApplication) { - ApplicationColumnPrefix.CONFIG.store(rowKey, applicationTable, - configKey, null, entry.getValue()); - } else { - EntityColumnPrefix.CONFIG.store(rowKey, entityTable, configKey, - null, entry.getValue()); + private void storeConfig(byte[] rowKey, Map config, + ColumnPrefix columnPrefix, TypedBufferedMutator table) + throws IOException { + if (config != null) { + for (Map.Entry entry : config.entrySet()) { + byte[] configKey = stringKeyConverter.encode(entry.getKey()); + columnPrefix.store(rowKey, table, configKey, null, entry.getValue()); } } } @@ -395,8 +454,9 @@ public class HBaseTimelineWriterImpl extends AbstractService implements * stores the {@linkplain TimelineMetric} information from the * {@linkplain TimelineEvent} object. */ - private void storeMetrics(byte[] rowKey, Set metrics, - boolean isApplication) throws IOException { + private void storeMetrics(byte[] rowKey, Set metrics, + ColumnPrefix columnPrefix, TypedBufferedMutator table) + throws IOException { if (metrics != null) { for (TimelineMetric metric : metrics) { byte[] metricColumnQualifier = @@ -404,13 +464,8 @@ public class HBaseTimelineWriterImpl extends AbstractService implements Map timeseries = metric.getValues(); for (Map.Entry timeseriesEntry : timeseries.entrySet()) { Long timestamp = timeseriesEntry.getKey(); - if (isApplication) { - ApplicationColumnPrefix.METRIC.store(rowKey, applicationTable, - metricColumnQualifier, timestamp, timeseriesEntry.getValue()); - } else { - EntityColumnPrefix.METRIC.store(rowKey, entityTable, - metricColumnQualifier, timestamp, timeseriesEntry.getValue()); - } + columnPrefix.store(rowKey, table, metricColumnQualifier, timestamp, + timeseriesEntry.getValue()); } } } @@ -419,8 +474,9 @@ public class HBaseTimelineWriterImpl extends AbstractService implements /** * Stores the events from the {@linkplain TimelineEvent} object. */ - private void storeEvents(byte[] rowKey, Set events, - boolean isApplication) throws IOException { + private void storeEvents(byte[] rowKey, Set events, + ColumnPrefix columnPrefix, TypedBufferedMutator table) + throws IOException { if (events != null) { for (TimelineEvent event : events) { if (event != null) { @@ -438,26 +494,16 @@ public class HBaseTimelineWriterImpl extends AbstractService implements byte[] columnQualifierBytes = new EventColumnName(eventId, eventTimestamp, null) .getColumnQualifier(); - if (isApplication) { - ApplicationColumnPrefix.EVENT.store(rowKey, applicationTable, - columnQualifierBytes, null, Separator.EMPTY_BYTES); - } else { - EntityColumnPrefix.EVENT.store(rowKey, entityTable, - columnQualifierBytes, null, Separator.EMPTY_BYTES); - } + columnPrefix.store(rowKey, table, columnQualifierBytes, null, + Separator.EMPTY_BYTES); } else { for (Map.Entry info : eventInfo.entrySet()) { // eventId=infoKey byte[] columnQualifierBytes = new EventColumnName(eventId, eventTimestamp, info.getKey()) .getColumnQualifier(); - if (isApplication) { - ApplicationColumnPrefix.EVENT.store(rowKey, applicationTable, - columnQualifierBytes, null, info.getValue()); - } else { - EntityColumnPrefix.EVENT.store(rowKey, entityTable, - columnQualifierBytes, null, info.getValue()); - } + columnPrefix.store(rowKey, table, columnQualifierBytes, null, + info.getValue()); } // for info: eventInfo } } @@ -466,33 +512,6 @@ public class HBaseTimelineWriterImpl extends AbstractService implements } } - /** - * Checks if the input TimelineEntity object is an ApplicationEntity. - * - * @param te TimelineEntity object. - * @return true if input is an ApplicationEntity, false otherwise - */ - static boolean isApplicationEntity(TimelineEntity te) { - return te.getType().equals(TimelineEntityType.YARN_APPLICATION.toString()); - } - - /** - * @param te TimelineEntity object. - * @param eventId event with this id needs to be fetched - * @return TimelineEvent if TimelineEntity contains the desired event. - */ - private static TimelineEvent getApplicationEvent(TimelineEntity te, - String eventId) { - if (isApplicationEntity(te)) { - for (TimelineEvent event : te.getEvents()) { - if (event.getId().equals(eventId)) { - return event; - } - } - } - return null; - } - /* * (non-Javadoc) * @@ -524,6 +543,7 @@ public class HBaseTimelineWriterImpl extends AbstractService implements applicationTable.flush(); flowRunTable.flush(); flowActivityTable.flush(); + subApplicationTable.flush(); } /** @@ -556,11 +576,13 @@ public class HBaseTimelineWriterImpl extends AbstractService implements // The close API performs flushing and releases any resources held flowActivityTable.close(); } + if (subApplicationTable != null) { + subApplicationTable.close(); + } if (conn != null) { LOG.info("closing the hbase Connection"); conn.close(); } super.serviceStop(); } - } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java index dbed05d5e19..210fd850460 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java @@ -32,16 +32,18 @@ import org.apache.commons.lang.StringUtils; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.util.GenericOptionsParser; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationTable; import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowTable; +import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineStorageUtils; import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTable; import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityTable; import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunTable; +import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationTable; import com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; @@ -62,7 +64,9 @@ public final class TimelineSchemaCreator { LoggerFactory.getLogger(TimelineSchemaCreator.class); private static final String SKIP_EXISTING_TABLE_OPTION_SHORT = "s"; private static final String APP_METRICS_TTL_OPTION_SHORT = "ma"; + private static final String SUB_APP_METRICS_TTL_OPTION_SHORT = "msa"; private static final String APP_TABLE_NAME_SHORT = "a"; + private static final String SUB_APP_TABLE_NAME_SHORT = "sa"; private static final String APP_TO_FLOW_TABLE_NAME_SHORT = "a2f"; private static final String ENTITY_METRICS_TTL_OPTION_SHORT = "me"; private static final String ENTITY_TABLE_NAME_SHORT = "e"; @@ -71,7 +75,10 @@ public final class TimelineSchemaCreator { public static void main(String[] args) throws Exception { - Configuration hbaseConf = HBaseConfiguration.create(); + LOG.info("Starting the schema creation"); + Configuration hbaseConf = + HBaseTimelineStorageUtils.getTimelineServiceHBaseConf( + new YarnConfiguration()); // Grab input args and allow for -Dxyz style arguments String[] otherArgs = new GenericOptionsParser(hbaseConf, args) .getRemainingArgs(); @@ -117,6 +124,21 @@ public final class TimelineSchemaCreator { new ApplicationTable().setMetricsTTL(appMetricsTTL, hbaseConf); } + // Grab the subApplicationTableName argument + String subApplicationTableName = commandLine.getOptionValue( + SUB_APP_TABLE_NAME_SHORT); + if (StringUtils.isNotBlank(subApplicationTableName)) { + hbaseConf.set(SubApplicationTable.TABLE_NAME_CONF_NAME, + subApplicationTableName); + } + // Grab the subApplication metrics TTL + String subApplicationTableMetricsTTL = commandLine + .getOptionValue(SUB_APP_METRICS_TTL_OPTION_SHORT); + if (StringUtils.isNotBlank(subApplicationTableMetricsTTL)) { + int subAppMetricsTTL = Integer.parseInt(subApplicationTableMetricsTTL); + new SubApplicationTable().setMetricsTTL(subAppMetricsTTL, hbaseConf); + } + // create all table schemas in hbase final boolean skipExisting = commandLine.hasOption( SKIP_EXISTING_TABLE_OPTION_SHORT); @@ -178,6 +200,18 @@ public final class TimelineSchemaCreator { o.setRequired(false); options.addOption(o); + o = new Option(SUB_APP_TABLE_NAME_SHORT, "subApplicationTableName", true, + "subApplication table name"); + o.setArgName("subApplicationTableName"); + o.setRequired(false); + options.addOption(o); + + o = new Option(SUB_APP_METRICS_TTL_OPTION_SHORT, "subApplicationMetricsTTL", + true, "TTL for metrics column family"); + o.setArgName("subApplicationMetricsTTL"); + o.setRequired(false); + options.addOption(o); + // Options without an argument // No need to set arg name since we do not need an argument here o = new Option(SKIP_EXISTING_TABLE_OPTION_SHORT, "skipExistingTable", @@ -216,6 +250,11 @@ public final class TimelineSchemaCreator { " The name of the Application table\n"); usage.append("[-applicationMetricsTTL ]" + " TTL for metrics in the Application table\n"); + usage.append("[-subApplicationTableName ]" + + " The name of the SubApplication table\n"); + usage.append("[-subApplicationMetricsTTL " + + " ]" + + " TTL for metrics in the SubApplication table\n"); usage.append("[-skipExistingTable] Whether to skip existing" + " hbase tables\n"); System.out.println(usage.toString()); @@ -308,6 +347,15 @@ public final class TimelineSchemaCreator { throw e; } } + try { + new SubApplicationTable().createTable(admin, hbaseConf); + } catch (IOException e) { + if (skipExisting) { + LOG.warn("Skip and continue on: " + e.getMessage()); + } else { + throw e; + } + } } finally { if (conn != null) { conn.close(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumn.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumn.java index dde3911549a..00eaa7eb3fc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumn.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumn.java @@ -105,52 +105,4 @@ public enum ApplicationColumn implements Column { return column.getValueConverter(); } - /** - * Retrieve an {@link ApplicationColumn} given a name, or null if there is no - * match. The following holds true: {@code columnFor(x) == columnFor(y)} if - * and only if {@code x.equals(y)} or {@code (x == y == null)}. - * - * @param columnQualifier Name of the column to retrieve - * @return the corresponding {@link ApplicationColumn} or null - */ - public static final ApplicationColumn columnFor(String columnQualifier) { - - // Match column based on value, assume column family matches. - for (ApplicationColumn ac : ApplicationColumn.values()) { - // Find a match based only on name. - if (ac.getColumnQualifier().equals(columnQualifier)) { - return ac; - } - } - - // Default to null - return null; - } - - /** - * Retrieve an {@link ApplicationColumn} given a name, or null if there is no - * match. The following holds true: {@code columnFor(a,x) == columnFor(b,y)} - * if and only if {@code a.equals(b) & x.equals(y)} or - * {@code (x == y == null)} - * - * @param columnFamily The columnFamily for which to retrieve the column. - * @param name Name of the column to retrieve - * @return the corresponding {@link ApplicationColumn} or null if both - * arguments don't match. - */ - public static final ApplicationColumn columnFor( - ApplicationColumnFamily columnFamily, String name) { - - for (ApplicationColumn ac : ApplicationColumn.values()) { - // Find a match based column family and on name. - if (ac.columnFamily.equals(columnFamily) - && ac.getColumnQualifier().equals(name)) { - return ac; - } - } - - // Default to null - return null; - } - } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnPrefix.java index 42488f4dc1e..8297dc5b762 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnPrefix.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnPrefix.java @@ -233,56 +233,4 @@ public enum ApplicationColumnPrefix implements ColumnPrefix { keyConverter); } - /** - * Retrieve an {@link ApplicationColumnPrefix} given a name, or null if there - * is no match. The following holds true: {@code columnFor(x) == columnFor(y)} - * if and only if {@code x.equals(y)} or {@code (x == y == null)} - * - * @param columnPrefix Name of the column to retrieve - * @return the corresponding {@link ApplicationColumnPrefix} or null - */ - public static final ApplicationColumnPrefix columnFor(String columnPrefix) { - - // Match column based on value, assume column family matches. - for (ApplicationColumnPrefix acp : ApplicationColumnPrefix.values()) { - // Find a match based only on name. - if (acp.getColumnPrefix().equals(columnPrefix)) { - return acp; - } - } - - // Default to null - return null; - } - - /** - * Retrieve an {@link ApplicationColumnPrefix} given a name, or null if there - * is no match. The following holds true: - * {@code columnFor(a,x) == columnFor(b,y)} if and only if - * {@code (x == y == null)} or {@code a.equals(b) & x.equals(y)} - * - * @param columnFamily The columnFamily for which to retrieve the column. - * @param columnPrefix Name of the column to retrieve - * @return the corresponding {@link ApplicationColumnPrefix} or null if both - * arguments don't match. - */ - public static final ApplicationColumnPrefix columnFor( - ApplicationColumnFamily columnFamily, String columnPrefix) { - - // TODO: needs unit test to confirm and need to update javadoc to explain - // null prefix case. - - for (ApplicationColumnPrefix acp : ApplicationColumnPrefix.values()) { - // Find a match based column family and on name. - if (acp.columnFamily.equals(columnFamily) - && (((columnPrefix == null) && (acp.getColumnPrefix() == null)) || - (acp.getColumnPrefix().equals(columnPrefix)))) { - return acp; - } - } - - // Default to null - return null; - } - } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationRowKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationRowKey.java index da62fdfc588..e89a6a7433d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationRowKey.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationRowKey.java @@ -18,9 +18,13 @@ package org.apache.hadoop.yarn.server.timelineservice.storage.application; +import java.util.List; + import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderUtils; import org.apache.hadoop.yarn.server.timelineservice.storage.common.AppIdKeyConverter; import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter; +import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverterToString; import org.apache.hadoop.yarn.server.timelineservice.storage.common.LongConverter; import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator; @@ -33,7 +37,7 @@ public class ApplicationRowKey { private final String flowName; private final Long flowRunId; private final String appId; - private final KeyConverter appRowKeyConverter = + private final ApplicationRowKeyConverter appRowKeyConverter = new ApplicationRowKeyConverter(); public ApplicationRowKey(String clusterId, String userId, String flowName, @@ -85,6 +89,24 @@ public class ApplicationRowKey { return new ApplicationRowKeyConverter().decode(rowKey); } + /** + * Constructs a row key for the application table as follows: + * {@code clusterId!userName!flowName!flowRunId!AppId}. + * @return String representation of row key. + */ + public String getRowKeyAsString() { + return appRowKeyConverter.encodeAsString(this); + } + + /** + * Given the encoded row key as string, returns the row key as an object. + * @param encodedRowKey String representation of row key. + * @return A ApplicationRowKey object. + */ + public static ApplicationRowKey parseRowKeyFromString(String encodedRowKey) { + return new ApplicationRowKeyConverter().decodeFromString(encodedRowKey); + } + /** * Encodes and decodes row key for application table. The row key is of the * form: clusterId!userName!flowName!flowRunId!appId. flowRunId is a long, @@ -93,7 +115,7 @@ public class ApplicationRowKey { *

*/ final private static class ApplicationRowKeyConverter implements - KeyConverter { + KeyConverter, KeyConverterToString { private final KeyConverter appIDKeyConverter = new AppIdKeyConverter(); @@ -201,6 +223,29 @@ public class ApplicationRowKey { return new ApplicationRowKey(clusterId, userId, flowName, flowRunId, appId); } + + @Override + public String encodeAsString(ApplicationRowKey key) { + if (key.clusterId == null || key.userId == null || key.flowName == null + || key.flowRunId == null || key.appId == null) { + throw new IllegalArgumentException(); + } + return TimelineReaderUtils + .joinAndEscapeStrings(new String[] {key.clusterId, key.userId, + key.flowName, key.flowRunId.toString(), key.appId}); + } + + @Override + public ApplicationRowKey decodeFromString(String encodedRowKey) { + List split = TimelineReaderUtils.split(encodedRowKey); + if (split == null || split.size() != 5) { + throw new IllegalArgumentException( + "Invalid row key for application table."); + } + Long flowRunId = Long.valueOf(split.get(3)); + return new ApplicationRowKey(split.get(0), split.get(1), split.get(2), + flowRunId, split.get(4)); + } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java index d3bdd3949c4..4da720e7a54 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java @@ -70,7 +70,7 @@ import org.slf4j.LoggerFactory; public class ApplicationTable extends BaseTable { /** application prefix. */ private static final String PREFIX = - YarnConfiguration.TIMELINE_SERVICE_PREFIX + ".application"; + YarnConfiguration.TIMELINE_SERVICE_PREFIX + "application"; /** config param name that specifies the application table name. */ public static final String TABLE_NAME_CONF_NAME = PREFIX + ".table.name"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumn.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumn.java index ff616336e17..67497fc8c97 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumn.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumn.java @@ -98,51 +98,4 @@ public enum AppToFlowColumn implements Column { return column.readResult(result, columnQualifierBytes); } - /** - * Retrieve an {@link AppToFlowColumn} given a name, or null if there is no - * match. The following holds true: {@code columnFor(x) == columnFor(y)} if - * and only if {@code x.equals(y)} or {@code (x == y == null)} - * - * @param columnQualifier Name of the column to retrieve - * @return the corresponding {@link AppToFlowColumn} or null - */ - public static final AppToFlowColumn columnFor(String columnQualifier) { - - // Match column based on value, assume column family matches. - for (AppToFlowColumn ec : AppToFlowColumn.values()) { - // Find a match based only on name. - if (ec.getColumnQualifier().equals(columnQualifier)) { - return ec; - } - } - - // Default to null - return null; - } - - /** - * Retrieve an {@link AppToFlowColumn} given a name, or null if there is no - * match. The following holds true: {@code columnFor(a,x) == columnFor(b,y)} - * if and only if {@code a.equals(b) & x.equals(y)} or - * {@code (x == y == null)} - * - * @param columnFamily The columnFamily for which to retrieve the column. - * @param name Name of the column to retrieve - * @return the corresponding {@link AppToFlowColumn} or null if both arguments - * don't match. - */ - public static final AppToFlowColumn columnFor( - AppToFlowColumnFamily columnFamily, String name) { - - for (AppToFlowColumn ec : AppToFlowColumn.values()) { - // Find a match based column family and on name. - if (ec.columnFamily.equals(columnFamily) - && ec.getColumnQualifier().equals(name)) { - return ec; - } - } - - // Default to null - return null; - } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumnPrefix.java new file mode 100644 index 00000000000..752a3803fe3 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumnPrefix.java @@ -0,0 +1,206 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow; + +import java.io.IOException; +import java.util.Map; +import java.util.NavigableMap; + +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily; +import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper; +import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix; +import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter; +import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator; +import org.apache.hadoop.yarn.server.timelineservice.storage.common.TypedBufferedMutator; +import org.apache.hadoop.yarn.server.timelineservice.storage.common.ValueConverter; +import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute; + +/** + * Identifies partially qualified columns for the app-to-flow table. + */ +public enum AppToFlowColumnPrefix implements ColumnPrefix { + + /** + * The flow name. + */ + FLOW_NAME(AppToFlowColumnFamily.MAPPING, "flow_name"), + + /** + * The flow run ID. + */ + FLOW_RUN_ID(AppToFlowColumnFamily.MAPPING, "flow_run_id"), + + /** + * The user. + */ + USER_ID(AppToFlowColumnFamily.MAPPING, "user_id"); + + private final ColumnHelper column; + private final ColumnFamily columnFamily; + private final String columnPrefix; + private final byte[] columnPrefixBytes; + + AppToFlowColumnPrefix(ColumnFamily columnFamily, + String columnPrefix) { + this.columnFamily = columnFamily; + this.columnPrefix = columnPrefix; + if (columnPrefix == null) { + this.columnPrefixBytes = null; + } else { + // Future-proof by ensuring the right column prefix hygiene. + this.columnPrefixBytes = + Bytes.toBytes(Separator.SPACE.encode(columnPrefix)); + } + this.column = new ColumnHelper(columnFamily); + } + + @Override + public byte[] getColumnPrefixBytes(String qualifierPrefix) { + return ColumnHelper.getColumnQualifier( + columnPrefixBytes, qualifierPrefix); + } + + @Override + public byte[] getColumnPrefixBytes(byte[] qualifierPrefix) { + return ColumnHelper.getColumnQualifier( + columnPrefixBytes, qualifierPrefix); + } + + @Override + public byte[] getColumnFamilyBytes() { + return columnFamily.getBytes(); + } + + @Override + public void store(byte[] rowKey, + TypedBufferedMutator tableMutator, byte[] qualifier, + Long timestamp, Object inputValue, Attribute... attributes) + throws IOException { + + // Null check + if (qualifier == null) { + throw new IOException("Cannot store column with null qualifier in " + + tableMutator.getName().getNameAsString()); + } + + byte[] columnQualifier = getColumnPrefixBytes(qualifier); + + column.store(rowKey, tableMutator, columnQualifier, timestamp, inputValue, + attributes); + } + + @Override + public void store(byte[] rowKey, + TypedBufferedMutator tableMutator, String qualifier, + Long timestamp, Object inputValue, Attribute... attributes) + throws IOException { + + // Null check + if (qualifier == null) { + throw new IOException("Cannot store column with null qualifier in " + + tableMutator.getName().getNameAsString()); + } + + byte[] columnQualifier = getColumnPrefixBytes(qualifier); + + column.store(rowKey, tableMutator, columnQualifier, timestamp, inputValue, + attributes); + } + + @Override + public ValueConverter getValueConverter() { + return column.getValueConverter(); + } + + @Override + public Object readResult(Result result, String qualifier) throws IOException { + byte[] columnQualifier = + ColumnHelper.getColumnQualifier(columnPrefixBytes, qualifier); + return column.readResult(result, columnQualifier); + } + + @Override + public Map readResults(Result result, + KeyConverter keyConverter) + throws IOException { + return column.readResults(result, columnPrefixBytes, keyConverter); + } + + @Override + public NavigableMap> + readResultsWithTimestamps(Result result, + KeyConverter keyConverter) throws IOException { + return column.readResultsWithTimestamps(result, columnPrefixBytes, + keyConverter); + } + + /** + * Retrieve an {@link AppToFlowColumnPrefix} given a name, or null if there + * is no match. The following holds true: {@code columnFor(x) == columnFor(y)} + * if and only if {@code x.equals(y)} or {@code (x == y == null)} + * + * @param columnPrefix Name of the column to retrieve + * @return the corresponding {@link AppToFlowColumnPrefix} or null + */ + public static final AppToFlowColumnPrefix columnFor(String columnPrefix) { + + // Match column based on value, assume column family matches. + for (AppToFlowColumnPrefix afcp : AppToFlowColumnPrefix.values()) { + // Find a match based only on name. + if (afcp.columnPrefix.equals(columnPrefix)) { + return afcp; + } + } + + // Default to null + return null; + } + + /** + * Retrieve an {@link AppToFlowColumnPrefix} given a name, or null if there + * is no match. The following holds true: + * {@code columnFor(a,x) == columnFor(b,y)} if and only if + * {@code (x == y == null)} or {@code a.equals(b) & x.equals(y)} + * + * @param columnFamily The columnFamily for which to retrieve the column. + * @param columnPrefix Name of the column to retrieve + * @return the corresponding {@link AppToFlowColumnPrefix} or null if both + * arguments don't match. + */ + public static final AppToFlowColumnPrefix columnFor( + AppToFlowColumnFamily columnFamily, String columnPrefix) { + + // TODO: needs unit test to confirm and need to update javadoc to explain + // null prefix case. + + for (AppToFlowColumnPrefix afcp : AppToFlowColumnPrefix.values()) { + // Find a match based column family and on name. + if (afcp.columnFamily.equals(columnFamily) + && (((columnPrefix == null) && (afcp.columnPrefix == null)) || + (afcp.columnPrefix.equals(columnPrefix)))) { + return afcp; + } + } + + // Default to null + return null; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowRowKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowRowKey.java index 8df440788de..146c47520d4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowRowKey.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowRowKey.java @@ -17,41 +17,32 @@ */ package org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow; -import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.yarn.server.timelineservice.storage.common.AppIdKeyConverter; import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter; -import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator; /** - * Represents a rowkey for the app_flow table. + * Represents a row key for the app_flow table, which is the app id. */ public class AppToFlowRowKey { - private final String clusterId; private final String appId; - private final KeyConverter appToFlowRowKeyConverter = - new AppToFlowRowKeyConverter(); + private final KeyConverter appIdKeyConverter = + new AppIdKeyConverter(); - public AppToFlowRowKey(String clusterId, String appId) { - this.clusterId = clusterId; + public AppToFlowRowKey(String appId) { this.appId = appId; } - public String getClusterId() { - return clusterId; - } - public String getAppId() { return appId; } /** - * Constructs a row key prefix for the app_flow table as follows: - * {@code clusterId!AppId}. + * Constructs a row key prefix for the app_flow table. * * @return byte array with the row key */ public byte[] getRowKey() { - return appToFlowRowKeyConverter.encode(this); + return appIdKeyConverter.encode(appId); } /** @@ -61,83 +52,7 @@ public class AppToFlowRowKey { * @return an AppToFlowRowKey object. */ public static AppToFlowRowKey parseRowKey(byte[] rowKey) { - return new AppToFlowRowKeyConverter().decode(rowKey); - } - - /** - * Encodes and decodes row key for app_flow table. The row key is of the form - * clusterId!appId. clusterId is a string and appId is encoded/decoded using - * {@link AppIdKeyConverter}. - *

- */ - final private static class AppToFlowRowKeyConverter implements - KeyConverter { - - private final KeyConverter appIDKeyConverter = - new AppIdKeyConverter(); - - /** - * Intended for use in AppToFlowRowKey only. - */ - private AppToFlowRowKeyConverter() { - } - - - /** - * App to flow row key is of the form clusterId!appId with the 2 segments - * separated by !. The sizes below indicate sizes of both of these segments - * in sequence. clusterId is a string. appId is represented as 12 bytes w. - * cluster Timestamp part of appid taking 8 bytes(long) and seq id taking 4 - * bytes(int). Strings are variable in size (i.e. end whenever separator is - * encountered). This is used while decoding and helps in determining where - * to split. - */ - private static final int[] SEGMENT_SIZES = {Separator.VARIABLE_SIZE, - Bytes.SIZEOF_LONG + Bytes.SIZEOF_INT }; - - /* - * (non-Javadoc) - * - * Encodes AppToFlowRowKey object into a byte array with each - * component/field in AppToFlowRowKey separated by Separator#QUALIFIERS. - * This leads to an app to flow table row key of the form clusterId!appId - * - * @see - * org.apache.hadoop.yarn.server.timelineservice.storage.common - * .KeyConverter#encode(java.lang.Object) - */ - @Override - public byte[] encode(AppToFlowRowKey rowKey) { - byte[] first = - Separator.encode(rowKey.getClusterId(), Separator.SPACE, - Separator.TAB, Separator.QUALIFIERS); - byte[] second = appIDKeyConverter.encode(rowKey.getAppId()); - return Separator.QUALIFIERS.join(first, second); - } - - /* - * (non-Javadoc) - * - * Decodes an app to flow row key of the form clusterId!appId represented - * in byte format and converts it into an AppToFlowRowKey object. - * - * @see - * org.apache.hadoop.yarn.server.timelineservice.storage.common - * .KeyConverter#decode(byte[]) - */ - @Override - public AppToFlowRowKey decode(byte[] rowKey) { - byte[][] rowKeyComponents = - Separator.QUALIFIERS.split(rowKey, SEGMENT_SIZES); - if (rowKeyComponents.length != 2) { - throw new IllegalArgumentException("the row key is not valid for " - + "the app-to-flow table"); - } - String clusterId = - Separator.decode(Bytes.toString(rowKeyComponents[0]), - Separator.QUALIFIERS, Separator.TAB, Separator.SPACE); - String appId = appIDKeyConverter.decode(rowKeyComponents[1]); - return new AppToFlowRowKey(clusterId, appId); - } + String appId = new AppIdKeyConverter().decode(rowKey); + return new AppToFlowRowKey(appId); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowTable.java index 40d95a4ced7..04da5c70d40 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowTable.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowTable.java @@ -41,21 +41,32 @@ import java.io.IOException; * 

  * |--------------------------------------|
  * |  Row       | Column Family           |
- * |  key       | info                    |
+ * |  key       | mapping                 |
  * |--------------------------------------|
- * | clusterId! | flowName:               |
- * | AppId      | foo@daily_hive_report   |
+ * | appId      | flow_name!cluster1:     |
+ * |            | foo@daily_hive_report   |
  * |            |                         |
- * |            | flowRunId:              |
+ * |            | flow_run_id!cluster1:   |
  * |            | 1452828720457           |
  * |            |                         |
- * |            | user_id:                |
+ * |            | user_id!cluster1:       |
  * |            | admin                   |
  * |            |                         |
+ * |            | flow_name!cluster2:     |
+ * |            | bar@ad_hoc_query        |
  * |            |                         |
+ * |            | flow_run_id!cluster2:   |
+ * |            | 1452828498752           |
+ * |            |                         |
+ * |            | user_id!cluster2:       |
+ * |            | joe                     |
  * |            |                         |
  * |--------------------------------------|
  *
+ * + * It is possible (although unlikely) in a multi-cluster environment that there + * may be more than one applications for a given app id. Different clusters are + * recorded as different sets of columns. */ public class AppToFlowTable extends BaseTable { /** app_flow prefix. */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BaseTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BaseTable.java index 8581aa45f28..93d809c003e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BaseTable.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BaseTable.java @@ -29,6 +29,7 @@ import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.yarn.conf.YarnConfiguration; /** * Implements behavior common to tables used in the timeline service storage. It @@ -114,16 +115,42 @@ public abstract class BaseTable { } /** - * Get the table name for this table. + * Get the table name for the input table. * - * @param hbaseConf HBase configuration from which table name will be fetched. + * @param conf HBase configuration from which table name will be fetched. + * @param tableName name of the table to be fetched * @return A {@link TableName} object. */ - public TableName getTableName(Configuration hbaseConf) { - TableName table = - TableName.valueOf(hbaseConf.get(tableNameConfName, defaultTableName)); - return table; + public static TableName getTableName(Configuration conf, String tableName) { + String tableSchemaPrefix = conf.get( + YarnConfiguration.TIMELINE_SERVICE_HBASE_SCHEMA_PREFIX_NAME, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_HBASE_SCHEMA_PREFIX); + return TableName.valueOf(tableSchemaPrefix + tableName); + } + /** + * Get the table name for this table. + * + * @param conf HBase configuration from which table name will be fetched. + * @return A {@link TableName} object. + */ + public TableName getTableName(Configuration conf) { + String tableName = conf.get(tableNameConfName, defaultTableName); + return getTableName(conf, tableName); + } + + /** + * Get the table name based on the input config parameters. + * + * @param conf HBase configuration from which table name will be fetched. + * @param tableNameInConf the table name parameter in conf. + * @param defaultTableName the default table name. + * @return A {@link TableName} object. + */ + public static TableName getTableName(Configuration conf, + String tableNameInConf, String defaultTableName) { + String tableName = conf.get(tableNameInConf, defaultTableName); + return getTableName(conf, tableName); } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java index a9c2148ebc4..9f95d445271 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java @@ -52,11 +52,28 @@ public class ColumnHelper { private final ValueConverter converter; + private final boolean supplementTs; + public ColumnHelper(ColumnFamily columnFamily) { this(columnFamily, GenericConverter.getInstance()); } public ColumnHelper(ColumnFamily columnFamily, ValueConverter converter) { + this(columnFamily, converter, false); + } + + /** + * @param columnFamily column family implementation. + * @param converter converter use to encode/decode values stored in the column + * or column prefix. + * @param needSupplementTs flag to indicate if cell timestamp needs to be + * modified for this column by calling + * {@link TimestampGenerator#getSupplementedTimestamp(long, String)}. This + * would be required for columns(such as metrics in flow run table) where + * potential collisions can occur due to same timestamp. + */ + public ColumnHelper(ColumnFamily columnFamily, ValueConverter converter, + boolean needSupplementTs) { this.columnFamily = columnFamily; columnFamilyBytes = columnFamily.getBytes(); if (converter == null) { @@ -64,6 +81,7 @@ public class ColumnHelper { } else { this.converter = converter; } + this.supplementTs = needSupplementTs; } /** @@ -106,18 +124,24 @@ public class ColumnHelper { } /* - * Figures out the cell timestamp used in the Put For storing into flow run - * table. We would like to left shift the timestamp and supplement it with the - * AppId id so that there are no collisions in the flow run table's cells + * Figures out the cell timestamp used in the Put For storing. + * Will supplement the timestamp if required. Typically done for flow run + * table.If we supplement the timestamp, we left shift the timestamp and + * supplement it with the AppId id so that there are no collisions in the flow + * run table's cells. */ private long getPutTimestamp(Long timestamp, Attribute[] attributes) { if (timestamp == null) { timestamp = System.currentTimeMillis(); } - String appId = getAppIdFromAttributes(attributes); - long supplementedTS = TimestampGenerator.getSupplementedTimestamp( - timestamp, appId); - return supplementedTS; + if (!this.supplementTs) { + return timestamp; + } else { + String appId = getAppIdFromAttributes(attributes); + long supplementedTS = TimestampGenerator.getSupplementedTimestamp( + timestamp, appId); + return supplementedTS; + } } private String getAppIdFromAttributes(Attribute[] attributes) { @@ -192,7 +216,6 @@ public class ColumnHelper { NavigableMap> columnCellMap = resultMap.get(columnFamilyBytes); - // could be that there is no such column family. if (columnCellMap != null) { for (Entry> entry : columnCellMap @@ -235,9 +258,9 @@ public class ColumnHelper { for (Entry cell : cells.entrySet()) { V value = (V) converter.decodeValue(cell.getValue()); - cellResults.put( - TimestampGenerator.getTruncatedTimestamp(cell.getKey()), - value); + Long ts = supplementTs ? TimestampGenerator. + getTruncatedTimestamp(cell.getKey()) : cell.getKey(); + cellResults.put(ts, value); } } results.put(converterColumnKey, cellResults); @@ -318,8 +341,9 @@ public class ColumnHelper { /** * @param columnPrefixBytes The byte representation for the column prefix. * Should not contain {@link Separator#QUALIFIERS}. - * @param qualifier for the remainder of the column. Any - * {@link Separator#QUALIFIERS} will be encoded in the qualifier. + * @param qualifier for the remainder of the column. + * {@link Separator#QUALIFIERS} is permissible in the qualifier + * as it is joined only with the column prefix bytes. * @return fully sanitized column qualifier that is a combination of prefix * and qualifier. If prefix is null, the result is simply the encoded * qualifier without any separator. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineStorageUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineStorageUtils.java index b8c70291793..b0d85274e60 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineStorageUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineStorageUtils.java @@ -17,25 +17,31 @@ package org.apache.hadoop.yarn.server.timelineservice.storage.common; +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; -import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.Tag; +import org.apache.hadoop.hbase.client.Query; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.timelineservice.storage.flow.AggregationCompactionDimension; import org.apache.hadoop.yarn.server.timelineservice.storage.flow.AggregationOperation; import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute; -import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunTable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; import java.text.NumberFormat; -import java.util.List; -import java.util.Map; /** * A bunch of utility functions used in HBase TimelineService backend. @@ -200,24 +206,6 @@ public final class HBaseTimelineStorageUtils { return appId; } - public static boolean isFlowRunTable(HRegionInfo hRegionInfo, - Configuration conf) { - String regionTableName = hRegionInfo.getTable().getNameAsString(); - String flowRunTableName = conf.get(FlowRunTable.TABLE_NAME_CONF_NAME, - FlowRunTable.DEFAULT_TABLE_NAME); - if (HBaseTimelineStorageUtils.LOG.isDebugEnabled()) { - HBaseTimelineStorageUtils.LOG.debug("regionTableName=" + regionTableName); - } - if (flowRunTableName.equalsIgnoreCase(regionTableName)) { - if (HBaseTimelineStorageUtils.LOG.isDebugEnabled()) { - HBaseTimelineStorageUtils.LOG.debug( - "table is the flow run table!! " + flowRunTableName); - } - return true; - } - return false; - } - /** * Converts an int into it's inverse int to be used in (row) keys * where we want to have the largest int value in the top of the table @@ -273,4 +261,94 @@ public final class HBaseTimelineStorageUtils { sb.append(APP_ID_FORMAT.get().format(appId.getId())); return sb.toString(); } + + /** + * @param conf Yarn configuration. Used to see if there is an explicit config + * pointing to the HBase config file to read. It should not be null + * or a NullPointerException will be thrown. + * @return a configuration with the HBase configuration from the classpath, + * optionally overwritten by the timeline service configuration URL if + * specified. + * @throws MalformedURLException if a timeline service HBase configuration URL + * is specified but is a malformed URL. + */ + public static Configuration getTimelineServiceHBaseConf(Configuration conf) + throws MalformedURLException { + if (conf == null) { + throw new NullPointerException(); + } + + Configuration hbaseConf; + String timelineServiceHBaseConfFileURL = + conf.get(YarnConfiguration.TIMELINE_SERVICE_HBASE_CONFIGURATION_FILE); + if (timelineServiceHBaseConfFileURL != null + && timelineServiceHBaseConfFileURL.length() > 0) { + LOG.info("Using hbase configuration at " + + timelineServiceHBaseConfFileURL); + // create a clone so that we don't mess with out input one + hbaseConf = new Configuration(conf); + Configuration plainHBaseConf = new Configuration(false); + URL hbaseSiteXML = new URL(timelineServiceHBaseConfFileURL); + plainHBaseConf.addResource(hbaseSiteXML); + HBaseConfiguration.merge(hbaseConf, plainHBaseConf); + } else { + // default to what is on the classpath + hbaseConf = HBaseConfiguration.create(conf); + } + return hbaseConf; + } + + /** + * Given a row key prefix stored in a byte array, return a byte array for its + * immediate next row key. + * + * @param rowKeyPrefix The provided row key prefix, represented in an array. + * @return the closest next row key of the provided row key. + */ + public static byte[] calculateTheClosestNextRowKeyForPrefix( + byte[] rowKeyPrefix) { + // Essentially we are treating it like an 'unsigned very very long' and + // doing +1 manually. + // Search for the place where the trailing 0xFFs start + int offset = rowKeyPrefix.length; + while (offset > 0) { + if (rowKeyPrefix[offset - 1] != (byte) 0xFF) { + break; + } + offset--; + } + + if (offset == 0) { + // We got an 0xFFFF... (only FFs) stopRow value which is + // the last possible prefix before the end of the table. + // So set it to stop at the 'end of the table' + return HConstants.EMPTY_END_ROW; + } + + // Copy the right length of the original + byte[] newStopRow = Arrays.copyOfRange(rowKeyPrefix, 0, offset); + // And increment the last one + newStopRow[newStopRow.length - 1]++; + return newStopRow; + } + + /** + * Checks if passed object is of integral type(Short/Integer/Long). + * + * @param obj Object to be checked. + * @return true if object passed is of type Short or Integer or Long, false + * otherwise. + */ + public static boolean isIntegralValue(Object obj) { + return (obj instanceof Short) || (obj instanceof Integer) || + (obj instanceof Long); + } + + public static void setMetricsTimeRange(Query query, byte[] metricsCf, + long tsBegin, long tsEnd) { + if (tsBegin != 0 || tsEnd != Long.MAX_VALUE) { + query.setColumnFamilyTimeRange(metricsCf, + tsBegin, ((tsEnd == Long.MAX_VALUE) ? Long.MAX_VALUE : (tsEnd + 1))); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/KeyConverterToString.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/KeyConverterToString.java new file mode 100644 index 00000000000..1f52a7b6f89 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/KeyConverterToString.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.timelineservice.storage.common; + +/** + * Interface which has to be implemented for encoding and decoding row keys or + * column qualifiers as string. + */ +public interface KeyConverterToString { + /** + * Encode key as string. + * @param key of type T to be encoded as string. + * @return encoded value as string. + */ + String encodeAsString(T key); + + /** + * Decode row key from string to a key of type T. + * @param encodedKey string representation of row key + * @return type T which has been constructed after decoding string. + */ + T decodeFromString(String encodedKey); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/LongConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/LongConverter.java index 600601a4489..6ab69f7de15 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/LongConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/LongConverter.java @@ -40,7 +40,7 @@ public final class LongConverter implements NumericValueConverter, @Override public byte[] encodeValue(Object value) throws IOException { - if (!TimelineStorageUtils.isIntegralValue(value)) { + if (!HBaseTimelineStorageUtils.isIntegralValue(value)) { throw new IOException("Expected integral value"); } return Bytes.toBytes(((Number)value).longValue()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java index 93b4b36033b..b228d848064 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java @@ -94,28 +94,6 @@ public enum EntityColumn implements Column { return column.readResult(result, columnQualifierBytes); } - /** - * Retrieve an {@link EntityColumn} given a name, or null if there is no - * match. The following holds true: {@code columnFor(x) == columnFor(y)} if - * and only if {@code x.equals(y)} or {@code (x == y == null)} - * - * @param columnQualifier Name of the column to retrieve - * @return the corresponding {@link EntityColumn} or null - */ - public static final EntityColumn columnFor(String columnQualifier) { - - // Match column based on value, assume column family matches. - for (EntityColumn ec : EntityColumn.values()) { - // Find a match based only on name. - if (ec.getColumnQualifier().equals(columnQualifier)) { - return ec; - } - } - - // Default to null - return null; - } - @Override public byte[] getColumnQualifierBytes() { return columnQualifierBytes.clone(); @@ -131,30 +109,4 @@ public enum EntityColumn implements Column { return column.getValueConverter(); } - /** - * Retrieve an {@link EntityColumn} given a name, or null if there is no - * match. The following holds true: {@code columnFor(a,x) == columnFor(b,y)} - * if and only if {@code a.equals(b) & x.equals(y)} or - * {@code (x == y == null)} - * - * @param columnFamily The columnFamily for which to retrieve the column. - * @param name Name of the column to retrieve - * @return the corresponding {@link EntityColumn} or null if both arguments - * don't match. - */ - public static final EntityColumn columnFor(EntityColumnFamily columnFamily, - String name) { - - for (EntityColumn ec : EntityColumn.values()) { - // Find a match based column family and on name. - if (ec.columnFamily.equals(columnFamily) - && ec.getColumnQualifier().equals(name)) { - return ec; - } - } - - // Default to null - return null; - } - } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java index e410549295e..d3851089c8f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java @@ -246,55 +246,4 @@ public enum EntityColumnPrefix implements ColumnPrefix { keyConverter); } - /** - * Retrieve an {@link EntityColumnPrefix} given a name, or null if there is no - * match. The following holds true: {@code columnFor(x) == columnFor(y)} if - * and only if {@code x.equals(y)} or {@code (x == y == null)} - * - * @param columnPrefix Name of the column to retrieve - * @return the corresponding {@link EntityColumnPrefix} or null - */ - public static final EntityColumnPrefix columnFor(String columnPrefix) { - - // Match column based on value, assume column family matches. - for (EntityColumnPrefix ecp : EntityColumnPrefix.values()) { - // Find a match based only on name. - if (ecp.getColumnPrefix().equals(columnPrefix)) { - return ecp; - } - } - - // Default to null - return null; - } - - /** - * Retrieve an {@link EntityColumnPrefix} given a name, or null if there is no - * match. The following holds true: {@code columnFor(a,x) == columnFor(b,y)} - * if and only if {@code (x == y == null)} or - * {@code a.equals(b) & x.equals(y)} - * - * @param columnFamily The columnFamily for which to retrieve the column. - * @param columnPrefix Name of the column to retrieve - * @return the corresponding {@link EntityColumnPrefix} or null if both - * arguments don't match. - */ - public static final EntityColumnPrefix columnFor( - EntityColumnFamily columnFamily, String columnPrefix) { - - // TODO: needs unit test to confirm and need to update javadoc to explain - // null prefix case. - - for (EntityColumnPrefix ecp : EntityColumnPrefix.values()) { - // Find a match based column family and on name. - if (ecp.columnFamily.equals(columnFamily) - && (((columnPrefix == null) && (ecp.getColumnPrefix() == null)) || - (ecp.getColumnPrefix().equals(columnPrefix)))) { - return ecp; - } - } - - // Default to null - return null; - } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java index ff22178fd5c..b85a9b09c0a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java @@ -17,9 +17,13 @@ */ package org.apache.hadoop.yarn.server.timelineservice.storage.entity; +import java.util.List; + import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderUtils; import org.apache.hadoop.yarn.server.timelineservice.storage.common.AppIdKeyConverter; import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter; +import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverterToString; import org.apache.hadoop.yarn.server.timelineservice.storage.common.LongConverter; import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator; @@ -33,18 +37,21 @@ public class EntityRowKey { private final Long flowRunId; private final String appId; private final String entityType; + private final Long entityIdPrefix; private final String entityId; - private final KeyConverter entityRowKeyConverter = + private final EntityRowKeyConverter entityRowKeyConverter = new EntityRowKeyConverter(); public EntityRowKey(String clusterId, String userId, String flowName, - Long flowRunId, String appId, String entityType, String entityId) { + Long flowRunId, String appId, String entityType, Long entityIdPrefix, + String entityId) { this.clusterId = clusterId; this.userId = userId; this.flowName = flowName; this.flowRunId = flowRunId; this.appId = appId; this.entityType = entityType; + this.entityIdPrefix = entityIdPrefix; this.entityId = entityId; } @@ -76,6 +83,10 @@ public class EntityRowKey { return entityId; } + public Long getEntityIdPrefix() { + return entityIdPrefix; + } + /** * Constructs a row key for the entity table as follows: * {@code userName!clusterId!flowName!flowRunId!AppId!entityType!entityId}. @@ -89,7 +100,6 @@ public class EntityRowKey { /** * Given the raw row key as bytes, returns the row key as an object. - * * @param rowKey byte representation of row key. * @return An EntityRowKey object. */ @@ -97,6 +107,27 @@ public class EntityRowKey { return new EntityRowKeyConverter().decode(rowKey); } + /** + * Constructs a row key for the entity table as follows: + *

+ * {@code userName!clusterId!flowName!flowRunId!AppId! + * entityType!entityIdPrefix!entityId}. + *

+ * @return String representation of row key. + */ + public String getRowKeyAsString() { + return entityRowKeyConverter.encodeAsString(this); + } + + /** + * Given the encoded row key as string, returns the row key as an object. + * @param encodedRowKey String representation of row key. + * @return A EntityRowKey object. + */ + public static EntityRowKey parseRowKeyFromString(String encodedRowKey) { + return new EntityRowKeyConverter().decodeFromString(encodedRowKey); + } + /** * Encodes and decodes row key for entity table. The row key is of the form : * userName!clusterId!flowName!flowRunId!appId!entityType!entityId. flowRunId @@ -105,7 +136,7 @@ public class EntityRowKey { *

*/ final private static class EntityRowKeyConverter implements - KeyConverter { + KeyConverter, KeyConverterToString { private final AppIdKeyConverter appIDKeyConverter = new AppIdKeyConverter(); @@ -126,7 +157,7 @@ public class EntityRowKey { private static final int[] SEGMENT_SIZES = {Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE, Bytes.SIZEOF_LONG, AppIdKeyConverter.getKeySize(), Separator.VARIABLE_SIZE, - Separator.VARIABLE_SIZE }; + Bytes.SIZEOF_LONG, Separator.VARIABLE_SIZE }; /* * (non-Javadoc) @@ -172,11 +203,25 @@ public class EntityRowKey { byte[] entityType = Separator.encode(rowKey.getEntityType(), Separator.SPACE, Separator.TAB, Separator.QUALIFIERS); - byte[] entityId = - rowKey.getEntityId() == null ? Separator.EMPTY_BYTES : Separator - .encode(rowKey.getEntityId(), Separator.SPACE, Separator.TAB, - Separator.QUALIFIERS); - byte[] fourth = Separator.QUALIFIERS.join(entityType, entityId); + + if (rowKey.getEntityIdPrefix() == null) { + return Separator.QUALIFIERS.join(first, second, third, entityType, + Separator.EMPTY_BYTES); + } + + byte[] entityIdPrefix = Bytes.toBytes(rowKey.getEntityIdPrefix()); + + if (rowKey.getEntityId() == null) { + return Separator.QUALIFIERS.join(first, second, third, entityType, + entityIdPrefix, Separator.EMPTY_BYTES); + } + + byte[] entityId = Separator.encode(rowKey.getEntityId(), Separator.SPACE, + Separator.TAB, Separator.QUALIFIERS); + + byte[] fourth = + Separator.QUALIFIERS.join(entityType, entityIdPrefix, entityId); + return Separator.QUALIFIERS.join(first, second, third, fourth); } @@ -196,7 +241,7 @@ public class EntityRowKey { public EntityRowKey decode(byte[] rowKey) { byte[][] rowKeyComponents = Separator.QUALIFIERS.split(rowKey, SEGMENT_SIZES); - if (rowKeyComponents.length != 7) { + if (rowKeyComponents.length != 8) { throw new IllegalArgumentException("the row key is not valid for " + "an entity"); } @@ -215,11 +260,40 @@ public class EntityRowKey { String entityType = Separator.decode(Bytes.toString(rowKeyComponents[5]), Separator.QUALIFIERS, Separator.TAB, Separator.SPACE); + + Long entityPrefixId = Bytes.toLong(rowKeyComponents[6]); + String entityId = - Separator.decode(Bytes.toString(rowKeyComponents[6]), + Separator.decode(Bytes.toString(rowKeyComponents[7]), Separator.QUALIFIERS, Separator.TAB, Separator.SPACE); return new EntityRowKey(clusterId, userId, flowName, flowRunId, appId, - entityType, entityId); + entityType, entityPrefixId, entityId); + } + + @Override + public String encodeAsString(EntityRowKey key) { + if (key.clusterId == null || key.userId == null || key.flowName == null + || key.flowRunId == null || key.appId == null + || key.entityType == null || key.entityIdPrefix == null + || key.entityId == null) { + throw new IllegalArgumentException(); + } + return TimelineReaderUtils + .joinAndEscapeStrings(new String[] {key.clusterId, key.userId, + key.flowName, key.flowRunId.toString(), key.appId, key.entityType, + key.entityIdPrefix.toString(), key.entityId}); + } + + @Override + public EntityRowKey decodeFromString(String encodedRowKey) { + List split = TimelineReaderUtils.split(encodedRowKey); + if (split == null || split.size() != 8) { + throw new IllegalArgumentException("Invalid row key for entity table."); + } + Long flowRunId = Long.valueOf(split.get(3)); + Long entityIdPrefix = Long.valueOf(split.get(6)); + return new EntityRowKey(split.get(0), split.get(1), split.get(2), + flowRunId, split.get(4), split.get(5), entityIdPrefix, split.get(7)); } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKeyPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKeyPrefix.java index 91461800ef3..47a1789bf95 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKeyPrefix.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKeyPrefix.java @@ -31,17 +31,20 @@ public class EntityRowKeyPrefix extends EntityRowKey implements * Creates a prefix which generates the following rowKeyPrefixes for the * entity table: * {@code userName!clusterId!flowName!flowRunId!AppId!entityType!}. - * * @param clusterId identifying the cluster * @param userId identifying the user * @param flowName identifying the flow * @param flowRunId identifying the individual run of this flow * @param appId identifying the application * @param entityType which entity type + * @param entityIdPrefix for entityId + * @param entityId for an entity */ public EntityRowKeyPrefix(String clusterId, String userId, String flowName, - Long flowRunId, String appId, String entityType) { - super(clusterId, userId, flowName, flowRunId, appId, entityType, null); + Long flowRunId, String appId, String entityType, Long entityIdPrefix, + String entityId) { + super(clusterId, userId, flowName, flowRunId, appId, entityType, + entityIdPrefix, entityId); } /** @@ -57,7 +60,7 @@ public class EntityRowKeyPrefix extends EntityRowKey implements */ public EntityRowKeyPrefix(String clusterId, String userId, String flowName, Long flowRunId, String appId) { - super(clusterId, userId, flowName, flowRunId, appId, null, null); + this(clusterId, userId, flowName, flowRunId, appId, null, null, null); } /* diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java index df5ce69a271..988bba2852f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java @@ -50,8 +50,8 @@ import org.slf4j.LoggerFactory; * | flowRunId! | | | configKey2: | * | AppId! | created_time: | metricId1: | configValue2 | * | entityType!| 1392993084018 | metricValue2 | | - * | entityId | | @timestamp2 | | - * | | i!infoKey: | | | + * | idPrefix! | | @timestamp2 | | + * | entityId | i!infoKey: | | | * | | infoValue | metricId1: | | * | | | metricValue1 | | * | | r!relatesToKey: | @timestamp2 | | diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityColumnPrefix.java index 439e0c8feec..706b002f93a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityColumnPrefix.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityColumnPrefix.java @@ -191,62 +191,6 @@ public enum FlowActivityColumnPrefix keyConverter); } - /** - * Retrieve an {@link FlowActivityColumnPrefix} given a name, or null if there - * is no match. The following holds true: {@code columnFor(x) == columnFor(y)} - * if and only if {@code x.equals(y)} or {@code (x == y == null)} - * - * @param columnPrefix - * Name of the column to retrieve - * @return the corresponding {@link FlowActivityColumnPrefix} or null - */ - public static final FlowActivityColumnPrefix columnFor(String columnPrefix) { - - // Match column based on value, assume column family matches. - for (FlowActivityColumnPrefix flowActivityColPrefix : - FlowActivityColumnPrefix.values()) { - // Find a match based only on name. - if (flowActivityColPrefix.getColumnPrefix().equals(columnPrefix)) { - return flowActivityColPrefix; - } - } - // Default to null - return null; - } - - /** - * Retrieve an {@link FlowActivityColumnPrefix} given a name, or null if there - * is no match. The following holds true: - * {@code columnFor(a,x) == columnFor(b,y)} if and only if - * {@code (x == y == null)} or {@code a.equals(b) & x.equals(y)} - * - * @param columnFamily - * The columnFamily for which to retrieve the column. - * @param columnPrefix - * Name of the column to retrieve - * @return the corresponding {@link FlowActivityColumnPrefix} or null if both - * arguments don't match. - */ - public static final FlowActivityColumnPrefix columnFor( - FlowActivityColumnFamily columnFamily, String columnPrefix) { - - // TODO: needs unit test to confirm and need to update javadoc to explain - // null prefix case. - - for (FlowActivityColumnPrefix flowActivityColumnPrefix : - FlowActivityColumnPrefix.values()) { - // Find a match based column family and on name. - if (flowActivityColumnPrefix.columnFamily.equals(columnFamily) - && (((columnPrefix == null) && (flowActivityColumnPrefix - .getColumnPrefix() == null)) || (flowActivityColumnPrefix - .getColumnPrefix().equals(columnPrefix)))) { - return flowActivityColumnPrefix; - } - } - // Default to null - return null; - } - /* * (non-Javadoc) * @@ -271,7 +215,7 @@ public enum FlowActivityColumnPrefix byte[] columnQualifier = getColumnPrefixBytes(qualifier); Attribute[] combinedAttributes = HBaseTimelineStorageUtils.combineAttributes(attributes, this.aggOp); - column.store(rowKey, tableMutator, columnQualifier, null, inputValue, + column.store(rowKey, tableMutator, columnQualifier, timestamp, inputValue, combinedAttributes); } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityRowKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityRowKey.java index bb77e36d4df..b8a5dba6cd6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityRowKey.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityRowKey.java @@ -17,10 +17,14 @@ */ package org.apache.hadoop.yarn.server.timelineservice.storage.flow; +import java.util.List; + import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderUtils; import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineStorageUtils; import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter; import org.apache.hadoop.yarn.server.timelineservice.storage.common.LongConverter; +import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverterToString; import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator; /** @@ -32,8 +36,8 @@ public class FlowActivityRowKey { private final Long dayTs; private final String userId; private final String flowName; - private final KeyConverter flowActivityRowKeyConverter = - new FlowActivityRowKeyConverter(); + private final FlowActivityRowKeyConverter + flowActivityRowKeyConverter = new FlowActivityRowKeyConverter(); /** * @param clusterId identifying the cluster @@ -103,14 +107,33 @@ public class FlowActivityRowKey { return new FlowActivityRowKeyConverter().decode(rowKey); } + /** + * Constructs a row key for the flow activity table as follows: + * {@code clusterId!dayTimestamp!user!flowName}. + * @return String representation of row key + */ + public String getRowKeyAsString() { + return flowActivityRowKeyConverter.encodeAsString(this); + } + + /** + * Given the raw row key as string, returns the row key as an object. + * @param encodedRowKey String representation of row key. + * @return A FlowActivityRowKey object. + */ + public static FlowActivityRowKey parseRowKeyFromString(String encodedRowKey) { + return new FlowActivityRowKeyConverter().decodeFromString(encodedRowKey); + } + /** * Encodes and decodes row key for flow activity table. The row key is of the * form : clusterId!dayTimestamp!user!flowName. dayTimestamp(top of the day * timestamp) is a long and rest are strings. *

*/ - final private static class FlowActivityRowKeyConverter implements - KeyConverter { + final private static class FlowActivityRowKeyConverter + implements KeyConverter, + KeyConverterToString { private FlowActivityRowKeyConverter() { } @@ -192,5 +215,33 @@ public class FlowActivityRowKey { Separator.QUALIFIERS, Separator.TAB, Separator.SPACE); return new FlowActivityRowKey(clusterId, dayTs, userId, flowName); } + + @Override + public String encodeAsString(FlowActivityRowKey key) { + if (key.getDayTimestamp() == null) { + return TimelineReaderUtils + .joinAndEscapeStrings(new String[] {key.clusterId}); + } else if (key.getUserId() == null) { + return TimelineReaderUtils.joinAndEscapeStrings( + new String[] {key.clusterId, key.dayTs.toString()}); + } else if (key.getFlowName() == null) { + return TimelineReaderUtils.joinAndEscapeStrings( + new String[] {key.clusterId, key.dayTs.toString(), key.userId}); + } + return TimelineReaderUtils.joinAndEscapeStrings(new String[] { + key.clusterId, key.dayTs.toString(), key.userId, key.flowName}); + } + + @Override + public FlowActivityRowKey decodeFromString(String encodedRowKey) { + List split = TimelineReaderUtils.split(encodedRowKey); + if (split == null || split.size() != 4) { + throw new IllegalArgumentException( + "Invalid row key for flow activity."); + } + Long dayTs = Long.valueOf(split.get(1)); + return new FlowActivityRowKey(split.get(0), dayTs, split.get(2), + split.get(3)); + } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumn.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumn.java index 90dd3456caa..3797fafedc8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumn.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumn.java @@ -76,7 +76,7 @@ public enum FlowRunColumn implements Column { // Future-proof by ensuring the right column prefix hygiene. this.columnQualifierBytes = Bytes.toBytes(Separator.SPACE .encode(columnQualifier)); - this.column = new ColumnHelper(columnFamily, converter); + this.column = new ColumnHelper(columnFamily, converter, true); } /** @@ -123,60 +123,9 @@ public enum FlowRunColumn implements Column { return column.readResult(result, columnQualifierBytes); } - /** - * Retrieve an {@link FlowRunColumn} given a name, or null if there is no - * match. The following holds true: {@code columnFor(x) == columnFor(y)} if - * and only if {@code x.equals(y)} or {@code (x == y == null)} - * - * @param columnQualifier - * Name of the column to retrieve - * @return the corresponding {@link FlowRunColumn} or null - */ - public static final FlowRunColumn columnFor(String columnQualifier) { - - // Match column based on value, assume column family matches. - for (FlowRunColumn ec : FlowRunColumn.values()) { - // Find a match based only on name. - if (ec.getColumnQualifier().equals(columnQualifier)) { - return ec; - } - } - - // Default to null - return null; - } - @Override public ValueConverter getValueConverter() { return column.getValueConverter(); } - /** - * Retrieve an {@link FlowRunColumn} given a name, or null if there is no - * match. The following holds true: {@code columnFor(a,x) == columnFor(b,y)} - * if and only if {@code a.equals(b) & x.equals(y)} or - * {@code (x == y == null)} - * - * @param columnFamily - * The columnFamily for which to retrieve the column. - * @param name - * Name of the column to retrieve - * @return the corresponding {@link FlowRunColumn} or null if both arguments - * don't match. - */ - public static final FlowRunColumn columnFor(FlowRunColumnFamily columnFamily, - String name) { - - for (FlowRunColumn ec : FlowRunColumn.values()) { - // Find a match based column family and on name. - if (ec.columnFamily.equals(columnFamily) - && ec.getColumnQualifier().equals(name)) { - return ec; - } - } - - // Default to null - return null; - } - } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumnPrefix.java index 278d18eb442..f521cd71844 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumnPrefix.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumnPrefix.java @@ -69,7 +69,7 @@ public enum FlowRunColumnPrefix implements ColumnPrefix { private FlowRunColumnPrefix(ColumnFamily columnFamily, String columnPrefix, AggregationOperation fra, ValueConverter converter, boolean compoundColQual) { - column = new ColumnHelper(columnFamily, converter); + column = new ColumnHelper(columnFamily, converter, true); this.columnFamily = columnFamily; this.columnPrefix = columnPrefix; if (columnPrefix == null) { @@ -209,60 +209,9 @@ public enum FlowRunColumnPrefix implements ColumnPrefix { keyConverter); } - /** - * Retrieve an {@link FlowRunColumnPrefix} given a name, or null if there is - * no match. The following holds true: {@code columnFor(x) == columnFor(y)} if - * and only if {@code x.equals(y)} or {@code (x == y == null)} - * - * @param columnPrefix Name of the column to retrieve - * @return the corresponding {@link FlowRunColumnPrefix} or null - */ - public static final FlowRunColumnPrefix columnFor(String columnPrefix) { - - // Match column based on value, assume column family matches. - for (FlowRunColumnPrefix frcp : FlowRunColumnPrefix.values()) { - // Find a match based only on name. - if (frcp.getColumnPrefix().equals(columnPrefix)) { - return frcp; - } - } - - // Default to null - return null; - } - @Override public ValueConverter getValueConverter() { return column.getValueConverter(); } - /** - * Retrieve an {@link FlowRunColumnPrefix} given a name, or null if there is - * no match. The following holds true: - * {@code columnFor(a,x) == columnFor(b,y)} if and only if - * {@code (x == y == null)} or {@code a.equals(b) & x.equals(y)} - * - * @param columnFamily The columnFamily for which to retrieve the column. - * @param columnPrefix Name of the column to retrieve - * @return the corresponding {@link FlowRunColumnPrefix} or null if both - * arguments don't match. - */ - public static final FlowRunColumnPrefix columnFor( - FlowRunColumnFamily columnFamily, String columnPrefix) { - - // TODO: needs unit test to confirm and need to update javadoc to explain - // null prefix case. - - for (FlowRunColumnPrefix frcp : FlowRunColumnPrefix.values()) { - // Find a match based column family and on name. - if (frcp.columnFamily.equals(columnFamily) - && (((columnPrefix == null) && (frcp.getColumnPrefix() == null)) || - (frcp.getColumnPrefix().equals(columnPrefix)))) { - return frcp; - } - } - - // Default to null - return null; - } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunCoprocessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunCoprocessor.java index 221420eb503..359eec98fa2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunCoprocessor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunCoprocessor.java @@ -58,7 +58,6 @@ public class FlowRunCoprocessor extends BaseRegionObserver { private static final Logger LOG = LoggerFactory.getLogger(FlowRunCoprocessor.class); - private boolean isFlowRunRegion = false; private Region region; /** @@ -72,15 +71,9 @@ public class FlowRunCoprocessor extends BaseRegionObserver { if (e instanceof RegionCoprocessorEnvironment) { RegionCoprocessorEnvironment env = (RegionCoprocessorEnvironment) e; this.region = env.getRegion(); - isFlowRunRegion = HBaseTimelineStorageUtils.isFlowRunTable( - region.getRegionInfo(), env.getConfiguration()); } } - public boolean isFlowRunRegion() { - return isFlowRunRegion; - } - /* * (non-Javadoc) * @@ -100,10 +93,6 @@ public class FlowRunCoprocessor extends BaseRegionObserver { public void prePut(ObserverContext e, Put put, WALEdit edit, Durability durability) throws IOException { Map attributes = put.getAttributesMap(); - - if (!isFlowRunRegion) { - return; - } // Assumption is that all the cells in a put are the same operation. List tags = new ArrayList<>(); if ((attributes != null) && (attributes.size() > 0)) { @@ -171,10 +160,6 @@ public class FlowRunCoprocessor extends BaseRegionObserver { @Override public void preGetOp(ObserverContext e, Get get, List results) throws IOException { - if (!isFlowRunRegion) { - return; - } - Scan scan = new Scan(get); scan.setMaxVersions(); RegionScanner scanner = null; @@ -206,12 +191,9 @@ public class FlowRunCoprocessor extends BaseRegionObserver { public RegionScanner preScannerOpen( ObserverContext e, Scan scan, RegionScanner scanner) throws IOException { - - if (isFlowRunRegion) { - // set max versions for scan to see all - // versions to aggregate for metrics - scan.setMaxVersions(); - } + // set max versions for scan to see all + // versions to aggregate for metrics + scan.setMaxVersions(); return scanner; } @@ -231,9 +213,6 @@ public class FlowRunCoprocessor extends BaseRegionObserver { public RegionScanner postScannerOpen( ObserverContext e, Scan scan, RegionScanner scanner) throws IOException { - if (!isFlowRunRegion) { - return scanner; - } return new FlowScanner(e.getEnvironment(), scan, scanner, FlowScannerOperation.READ); } @@ -242,9 +221,6 @@ public class FlowRunCoprocessor extends BaseRegionObserver { public InternalScanner preFlush( ObserverContext c, Store store, InternalScanner scanner) throws IOException { - if (!isFlowRunRegion) { - return scanner; - } if (LOG.isDebugEnabled()) { if (store != null) { LOG.debug("preFlush store = " + store.getColumnFamilyName() @@ -265,9 +241,6 @@ public class FlowRunCoprocessor extends BaseRegionObserver { @Override public void postFlush(ObserverContext c, Store store, StoreFile resultFile) { - if (!isFlowRunRegion) { - return; - } if (LOG.isDebugEnabled()) { if (store != null) { LOG.debug("postFlush store = " + store.getColumnFamilyName() @@ -289,9 +262,6 @@ public class FlowRunCoprocessor extends BaseRegionObserver { InternalScanner scanner, ScanType scanType, CompactionRequest request) throws IOException { - if (!isFlowRunRegion) { - return scanner; - } FlowScannerOperation requestOp = FlowScannerOperation.MINOR_COMPACTION; if (request != null) { requestOp = (request.isMajor() ? FlowScannerOperation.MAJOR_COMPACTION diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunRowKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunRowKey.java index 8fda9a861b7..7ce91cfed5c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunRowKey.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunRowKey.java @@ -17,8 +17,12 @@ */ package org.apache.hadoop.yarn.server.timelineservice.storage.flow; +import java.util.List; + import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderUtils; import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter; +import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverterToString; import org.apache.hadoop.yarn.server.timelineservice.storage.common.LongConverter; import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator; @@ -70,7 +74,6 @@ public class FlowRunRowKey { /** * Given the raw row key as bytes, returns the row key as an object. - * * @param rowKey Byte representation of row key. * @return A FlowRunRowKey object. */ @@ -78,6 +81,24 @@ public class FlowRunRowKey { return new FlowRunRowKeyConverter().decode(rowKey); } + /** + * Constructs a row key for the flow run table as follows: + * {@code clusterId!userId!flowName!Flow Run Id}. + * @return String representation of row key + */ + public String getRowKeyAsString() { + return flowRunRowKeyConverter.encodeAsString(this); + } + + /** + * Given the encoded row key as string, returns the row key as an object. + * @param encodedRowKey String representation of row key. + * @return A FlowRunRowKey object. + */ + public static FlowRunRowKey parseRowKeyFromString(String encodedRowKey) { + return new FlowRunRowKeyConverter().decodeFromString(encodedRowKey); + } + /** * returns the Flow Key as a verbose String output. * @return String @@ -101,7 +122,7 @@ public class FlowRunRowKey { *

*/ final private static class FlowRunRowKeyConverter implements - KeyConverter { + KeyConverter, KeyConverterToString { private FlowRunRowKeyConverter() { } @@ -186,5 +207,27 @@ public class FlowRunRowKey { LongConverter.invertLong(Bytes.toLong(rowKeyComponents[3])); return new FlowRunRowKey(clusterId, userId, flowName, flowRunId); } + + @Override + public String encodeAsString(FlowRunRowKey key) { + if (key.clusterId == null || key.userId == null || key.flowName == null + || key.flowRunId == null) { + throw new IllegalArgumentException(); + } + return TimelineReaderUtils.joinAndEscapeStrings(new String[] { + key.clusterId, key.userId, key.flowName, key.flowRunId.toString()}); + } + + @Override + public FlowRunRowKey decodeFromString(String encodedRowKey) { + List split = TimelineReaderUtils.split(encodedRowKey); + if (split == null || split.size() != 4) { + throw new IllegalArgumentException( + "Invalid row key for flow run table."); + } + Long flowRunId = Long.valueOf(split.get(3)); + return new FlowRunRowKey(split.get(0), split.get(1), split.get(2), + flowRunId); + } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunTable.java index 9c6549ffc12..a1d32ee78cd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunTable.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunTable.java @@ -29,6 +29,8 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.Coprocessor; /** * The flow run table has column family info @@ -133,8 +135,15 @@ public class FlowRunTable extends BaseTable { infoCF.setMaxVersions(DEFAULT_METRICS_MAX_VERSIONS); // TODO: figure the split policy - flowRunTableDescp.addCoprocessor(FlowRunCoprocessor.class - .getCanonicalName()); + String coprocessorJarPathStr = hbaseConf.get( + YarnConfiguration.FLOW_RUN_COPROCESSOR_JAR_HDFS_LOCATION, + YarnConfiguration.DEFAULT_HDFS_LOCATION_FLOW_RUN_COPROCESSOR_JAR); + + Path coprocessorJarPath = new Path(coprocessorJarPathStr); + LOG.info("CoprocessorJarPath=" + coprocessorJarPath.toString()); + flowRunTableDescp.addCoprocessor( + FlowRunCoprocessor.class.getCanonicalName(), coprocessorJarPath, + Coprocessor.PRIORITY_USER, null); admin.createTable(flowRunTableDescp); LOG.info("Status of table creation for " + table.getNameAsString() + "=" + admin.tableExists(table)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/package-info.java index d0bc366b9ff..e78db2a1ef5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/package-info.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/package-info.java @@ -17,12 +17,12 @@ */ /** - * Package org.apache.hadoop.server.timelineservice contains classes to be used - * across timeline reader and collector. + * Package org.apache.hadoop.yarn.server.timelineservice.storage contains + * classes which define and implement reading and writing to backend storage. */ @InterfaceAudience.Private @InterfaceStability.Unstable package org.apache.hadoop.yarn.server.timelineservice.storage; import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; \ No newline at end of file +import org.apache.hadoop.classification.InterfaceStability; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/AbstractTimelineStorageReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/AbstractTimelineStorageReader.java new file mode 100644 index 00000000000..5bacf66fb45 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/AbstractTimelineStorageReader.java @@ -0,0 +1,158 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.timelineservice.storage.reader; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext; +import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowColumnPrefix; +import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowRowKey; +import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowTable; +import org.apache.hadoop.yarn.webapp.NotFoundException; + +/** + * The base class for reading timeline data from the HBase storage. This class + * provides basic support to validate and augment reader context. + */ +public abstract class AbstractTimelineStorageReader { + + private final TimelineReaderContext context; + /** + * Used to look up the flow context. + */ + private final AppToFlowTable appToFlowTable = new AppToFlowTable(); + + public AbstractTimelineStorageReader(TimelineReaderContext ctxt) { + context = ctxt; + } + + protected TimelineReaderContext getContext() { + return context; + } + + /** + * Looks up flow context from AppToFlow table. + * + * @param appToFlowRowKey to identify Cluster and App Ids. + * @param clusterId the cluster id. + * @param hbaseConf HBase configuration. + * @param conn HBase Connection. + * @return flow context information. + * @throws IOException if any problem occurs while fetching flow information. + */ + protected FlowContext lookupFlowContext(AppToFlowRowKey appToFlowRowKey, + String clusterId, Configuration hbaseConf, Connection conn) + throws IOException { + byte[] rowKey = appToFlowRowKey.getRowKey(); + Get get = new Get(rowKey); + Result result = appToFlowTable.getResult(hbaseConf, conn, get); + if (result != null && !result.isEmpty()) { + Object flowName = + AppToFlowColumnPrefix.FLOW_NAME.readResult(result, clusterId); + Object flowRunId = + AppToFlowColumnPrefix.FLOW_RUN_ID.readResult(result, clusterId); + Object userId = + AppToFlowColumnPrefix.USER_ID.readResult(result, clusterId); + if (flowName == null || userId == null || flowRunId == null) { + throw new NotFoundException( + "Unable to find the context flow name, and flow run id, " + + "and user id for clusterId=" + clusterId + + ", appId=" + appToFlowRowKey.getAppId()); + } + return new FlowContext((String)userId, (String)flowName, + ((Number)flowRunId).longValue()); + } else { + throw new NotFoundException( + "Unable to find the context flow name, and flow run id, " + + "and user id for clusterId=" + clusterId + + ", appId=" + appToFlowRowKey.getAppId()); + } + } + + /** + * Sets certain parameters to defaults if the values are not provided. + * + * @param hbaseConf HBase Configuration. + * @param conn HBase Connection. + * @throws IOException if any exception is encountered while setting params. + */ + protected void augmentParams(Configuration hbaseConf, Connection conn) + throws IOException { + defaultAugmentParams(hbaseConf, conn); + } + + /** + * Default behavior for all timeline readers to augment parameters. + * + * @param hbaseConf HBase Configuration. + * @param conn HBase Connection. + * @throws IOException if any exception is encountered while setting params. + */ + final protected void defaultAugmentParams(Configuration hbaseConf, + Connection conn) throws IOException { + // In reality all three should be null or neither should be null + if (context.getFlowName() == null || context.getFlowRunId() == null + || context.getUserId() == null) { + // Get flow context information from AppToFlow table. + AppToFlowRowKey appToFlowRowKey = + new AppToFlowRowKey(context.getAppId()); + FlowContext flowContext = + lookupFlowContext(appToFlowRowKey, context.getClusterId(), hbaseConf, + conn); + context.setFlowName(flowContext.flowName); + context.setFlowRunId(flowContext.flowRunId); + context.setUserId(flowContext.userId); + } + } + + /** + * Validates the required parameters to read the entities. + */ + protected abstract void validateParams(); + + /** + * Encapsulates flow context information. + */ + protected static class FlowContext { + private final String userId; + private final String flowName; + private final Long flowRunId; + + public FlowContext(String user, String flowName, Long flowRunId) { + this.userId = user; + this.flowName = flowName; + this.flowRunId = flowRunId; + } + + protected String getUserId() { + return userId; + } + + protected String getFlowName() { + return flowName; + } + + protected Long getFlowRunId() { + return flowRunId; + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java index aa2bfdae658..0edd6a52680 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java @@ -24,6 +24,7 @@ import java.util.Set; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Query; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; @@ -39,6 +40,7 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType; import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve; import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters; import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext; +import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderUtils; import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList; import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterUtils; import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field; @@ -48,10 +50,11 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.application.Applica import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationRowKey; import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationRowKeyPrefix; import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationTable; -import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowRowKey; import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable; +import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineStorageUtils; import org.apache.hadoop.yarn.server.timelineservice.storage.common.RowKeyPrefix; import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineStorageUtils; +import org.apache.hadoop.yarn.webapp.BadRequestException; import com.google.common.base.Preconditions; @@ -65,7 +68,7 @@ class ApplicationEntityReader extends GenericEntityReader { public ApplicationEntityReader(TimelineReaderContext ctxt, TimelineEntityFilters entityFilters, TimelineDataToRetrieve toRetrieve) { - super(ctxt, entityFilters, toRetrieve, true); + super(ctxt, entityFilters, toRetrieve); } public ApplicationEntityReader(TimelineReaderContext ctxt, @@ -313,6 +316,8 @@ class ApplicationEntityReader extends GenericEntityReader { context.getFlowName(), context.getFlowRunId(), context.getAppId()); byte[] rowKey = applicationRowKey.getRowKey(); Get get = new Get(rowKey); + // Set time range for metric values. + setMetricsTimeRange(get); get.setMaxVersions(getDataToRetrieve().getMetricsLimit()); if (filterList != null && !filterList.getFilters().isEmpty()) { get.setFilter(filterList); @@ -343,20 +348,9 @@ class ApplicationEntityReader extends GenericEntityReader { @Override protected void augmentParams(Configuration hbaseConf, Connection conn) throws IOException { - TimelineReaderContext context = getContext(); if (isSingleEntityRead()) { // Get flow context information from AppToFlow table. - if (context.getFlowName() == null || context.getFlowRunId() == null - || context.getUserId() == null) { - AppToFlowRowKey appToFlowRowKey = - new AppToFlowRowKey(context.getClusterId(), context.getAppId()); - FlowContext flowContext = - lookupFlowContext(appToFlowRowKey, - hbaseConf, conn); - context.setFlowName(flowContext.getFlowName()); - context.setFlowRunId(flowContext.getFlowRunId()); - context.setUserId(flowContext.getUserId()); - } + defaultAugmentParams(hbaseConf, conn); } // Add configs/metrics to fields to retrieve if confsToRetrieve and/or // metricsToRetrieve are specified. @@ -366,24 +360,65 @@ class ApplicationEntityReader extends GenericEntityReader { } } + private void setMetricsTimeRange(Query query) { + // Set time range for metric values. + HBaseTimelineStorageUtils.setMetricsTimeRange( + query, ApplicationColumnFamily.METRICS.getBytes(), + getDataToRetrieve().getMetricsTimeBegin(), + getDataToRetrieve().getMetricsTimeEnd()); + } + @Override protected ResultScanner getResults(Configuration hbaseConf, Connection conn, FilterList filterList) throws IOException { Scan scan = new Scan(); TimelineReaderContext context = getContext(); + RowKeyPrefix applicationRowKeyPrefix = null; + // Whether or not flowRunID is null doesn't matter, the // ApplicationRowKeyPrefix will do the right thing. - RowKeyPrefix applicationRowKeyPrefix = - new ApplicationRowKeyPrefix(context.getClusterId(), - context.getUserId(), context.getFlowName(), - context.getFlowRunId()); - scan.setRowPrefixFilter(applicationRowKeyPrefix.getRowKeyPrefix()); + // default mode, will always scans from beginning of entity type. + if (getFilters().getFromId() == null) { + applicationRowKeyPrefix = new ApplicationRowKeyPrefix( + context.getClusterId(), context.getUserId(), context.getFlowName(), + context.getFlowRunId()); + scan.setRowPrefixFilter(applicationRowKeyPrefix.getRowKeyPrefix()); + } else { + ApplicationRowKey applicationRowKey = null; + try { + applicationRowKey = + ApplicationRowKey.parseRowKeyFromString(getFilters().getFromId()); + } catch (IllegalArgumentException e) { + throw new BadRequestException("Invalid filter fromid is provided."); + } + if (!context.getClusterId().equals(applicationRowKey.getClusterId())) { + throw new BadRequestException( + "fromid doesn't belong to clusterId=" + context.getClusterId()); + } + + // set start row + scan.setStartRow(applicationRowKey.getRowKey()); + + // get the bytes for stop row + applicationRowKeyPrefix = new ApplicationRowKeyPrefix( + context.getClusterId(), context.getUserId(), context.getFlowName(), + context.getFlowRunId()); + + // set stop row + scan.setStopRow( + HBaseTimelineStorageUtils.calculateTheClosestNextRowKeyForPrefix( + applicationRowKeyPrefix.getRowKeyPrefix())); + } + FilterList newList = new FilterList(); newList.addFilter(new PageFilter(getFilters().getLimit())); if (filterList != null && !filterList.getFilters().isEmpty()) { newList.addFilter(filterList); } scan.setFilter(newList); + + // Set time range for metric values. + setMetricsTimeRange(scan); scan.setMaxVersions(getDataToRetrieve().getMetricsLimit()); return getTable().getResultScanner(hbaseConf, conn, scan); } @@ -476,6 +511,10 @@ class ApplicationEntityReader extends GenericEntityReader { if (hasField(fieldsToRetrieve, Field.METRICS)) { readMetrics(entity, result, ApplicationColumnPrefix.METRIC); } + + ApplicationRowKey rowKey = ApplicationRowKey.parseRowKey(result.getRow()); + entity.getInfo().put(TimelineReaderUtils.FROMID_KEY, + rowKey.getRowKeyAsString()); return entity; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/EntityTypeReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/EntityTypeReader.java new file mode 100644 index 00000000000..05570f1f530 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/EntityTypeReader.java @@ -0,0 +1,179 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.timelineservice.storage.reader; + +import com.google.common.base.Preconditions; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.filter.FilterList; +import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter; +import org.apache.hadoop.hbase.filter.KeyOnlyFilter; +import org.apache.hadoop.hbase.filter.PageFilter; +import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; +import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext; +import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineStorageUtils; +import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator; +import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKey; +import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKeyPrefix; +import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTable; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Set; +import java.util.TreeSet; + +/** + * Timeline entity reader for listing all available entity types given one + * reader context. Right now only supports listing all entity types within one + * YARN application. + */ +public final class EntityTypeReader extends AbstractTimelineStorageReader { + + private static final Log LOG = LogFactory.getLog(EntityTypeReader.class); + private static final EntityTable ENTITY_TABLE = new EntityTable(); + + public EntityTypeReader(TimelineReaderContext context) { + super(context); + } + + /** + * Reads a set of timeline entity types from the HBase storage for the given + * context. + * + * @param hbaseConf HBase Configuration. + * @param conn HBase Connection. + * @return a set of TimelineEntity objects, with only type field + * set. + * @throws IOException if any exception is encountered while reading entities. + */ + public Set readEntityTypes(Configuration hbaseConf, + Connection conn) throws IOException { + + validateParams(); + augmentParams(hbaseConf, conn); + + Set types = new TreeSet<>(); + TimelineReaderContext context = getContext(); + EntityRowKeyPrefix prefix = new EntityRowKeyPrefix(context.getClusterId(), + context.getUserId(), context.getFlowName(), context.getFlowRunId(), + context.getAppId()); + byte[] currRowKey = prefix.getRowKeyPrefix(); + byte[] nextRowKey = prefix.getRowKeyPrefix(); + nextRowKey[nextRowKey.length - 1]++; + + FilterList typeFilterList = new FilterList(); + typeFilterList.addFilter(new FirstKeyOnlyFilter()); + typeFilterList.addFilter(new KeyOnlyFilter()); + typeFilterList.addFilter(new PageFilter(1)); + if (LOG.isDebugEnabled()) { + LOG.debug("FilterList created for scan is - " + typeFilterList); + } + + int counter = 0; + while (true) { + try (ResultScanner results = + getResult(hbaseConf, conn, typeFilterList, currRowKey, nextRowKey)) { + TimelineEntity entity = parseEntityForType(results.next()); + if (entity == null) { + break; + } + ++counter; + if (!types.add(entity.getType())) { + LOG.warn("Failed to add type " + entity.getType() + + " to the result set because there is a duplicated copy. "); + } + String currType = entity.getType(); + if (LOG.isDebugEnabled()) { + LOG.debug("Current row key: " + Arrays.toString(currRowKey)); + LOG.debug("New entity type discovered: " + currType); + } + currRowKey = getNextRowKey(prefix.getRowKeyPrefix(), currType); + } + } + if (LOG.isDebugEnabled()) { + LOG.debug("Scanned " + counter + "records for " + + types.size() + "types"); + } + return types; + } + + @Override + protected void validateParams() { + Preconditions.checkNotNull(getContext(), "context shouldn't be null"); + Preconditions.checkNotNull(getContext().getClusterId(), + "clusterId shouldn't be null"); + Preconditions.checkNotNull(getContext().getAppId(), + "appId shouldn't be null"); + } + + /** + * Gets the possibly next row key prefix given current prefix and type. + * + * @param currRowKeyPrefix The current prefix that contains user, cluster, + * flow, run, and application id. + * @param entityType Current entity type. + * @return A new prefix for the possibly immediately next row key. + */ + private static byte[] getNextRowKey(byte[] currRowKeyPrefix, + String entityType) { + if (currRowKeyPrefix == null || entityType == null) { + return null; + } + + byte[] entityTypeEncoded = Separator.QUALIFIERS.join( + Separator.encode(entityType, Separator.SPACE, Separator.TAB, + Separator.QUALIFIERS), + Separator.EMPTY_BYTES); + + byte[] currRowKey + = new byte[currRowKeyPrefix.length + entityTypeEncoded.length]; + System.arraycopy(currRowKeyPrefix, 0, currRowKey, 0, + currRowKeyPrefix.length); + System.arraycopy(entityTypeEncoded, 0, currRowKey, currRowKeyPrefix.length, + entityTypeEncoded.length); + + return HBaseTimelineStorageUtils.calculateTheClosestNextRowKeyForPrefix( + currRowKey); + } + + private ResultScanner getResult(Configuration hbaseConf, Connection conn, + FilterList filterList, byte[] startPrefix, byte[] endPrefix) + throws IOException { + Scan scan = new Scan(startPrefix, endPrefix); + scan.setFilter(filterList); + scan.setSmall(true); + return ENTITY_TABLE.getResultScanner(hbaseConf, conn, scan); + } + + private TimelineEntity parseEntityForType(Result result) + throws IOException { + if (result == null || result.isEmpty()) { + return null; + } + TimelineEntity entity = new TimelineEntity(); + EntityRowKey newRowKey = EntityRowKey.parseRowKey(result.getRow()); + entity.setType(newRowKey.getEntityType()); + return entity; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowActivityEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowActivityEntityReader.java index 9ba5e38e37a..a1cdb29b124 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowActivityEntityReader.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowActivityEntityReader.java @@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve; import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters; import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext; +import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderUtils; import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable; import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter; import org.apache.hadoop.yarn.server.timelineservice.storage.common.LongKeyConverter; @@ -41,6 +42,7 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityCo import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityRowKey; import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityRowKeyPrefix; import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityTable; +import org.apache.hadoop.yarn.webapp.BadRequestException; import com.google.common.base.Preconditions; @@ -60,7 +62,7 @@ class FlowActivityEntityReader extends TimelineEntityReader { public FlowActivityEntityReader(TimelineReaderContext ctxt, TimelineEntityFilters entityFilters, TimelineDataToRetrieve toRetrieve) { - super(ctxt, entityFilters, toRetrieve, true); + super(ctxt, entityFilters, toRetrieve); } public FlowActivityEntityReader(TimelineReaderContext ctxt, @@ -110,11 +112,30 @@ class FlowActivityEntityReader extends TimelineEntityReader { Connection conn, FilterList filterList) throws IOException { Scan scan = new Scan(); String clusterId = getContext().getClusterId(); - if (getFilters().getCreatedTimeBegin() == 0L && - getFilters().getCreatedTimeEnd() == Long.MAX_VALUE) { + if (getFilters().getFromId() == null + && getFilters().getCreatedTimeBegin() == 0L + && getFilters().getCreatedTimeEnd() == Long.MAX_VALUE) { // All records have to be chosen. scan.setRowPrefixFilter(new FlowActivityRowKeyPrefix(clusterId) .getRowKeyPrefix()); + } else if (getFilters().getFromId() != null) { + FlowActivityRowKey key = null; + try { + key = + FlowActivityRowKey.parseRowKeyFromString(getFilters().getFromId()); + } catch (IllegalArgumentException e) { + throw new BadRequestException("Invalid filter fromid is provided."); + } + if (!clusterId.equals(key.getClusterId())) { + throw new BadRequestException( + "fromid doesn't belong to clusterId=" + clusterId); + } + scan.setStartRow(key.getRowKey()); + scan.setStopRow( + new FlowActivityRowKeyPrefix(clusterId, + (getFilters().getCreatedTimeBegin() <= 0 ? 0 + : (getFilters().getCreatedTimeBegin() - 1))) + .getRowKeyPrefix()); } else { scan.setStartRow(new FlowActivityRowKeyPrefix(clusterId, getFilters() .getCreatedTimeEnd()).getRowKeyPrefix()); @@ -157,7 +178,8 @@ class FlowActivityEntityReader extends TimelineEntityReader { flowRun.setId(flowRun.getId()); flowActivity.addFlowRun(flowRun); } - + flowActivity.getInfo().put(TimelineReaderUtils.FROMID_KEY, + rowKey.getRowKeyAsString()); return flowActivity; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowRunEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowRunEntityReader.java index 986a28f43ac..af043b3bc7a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowRunEntityReader.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowRunEntityReader.java @@ -39,10 +39,12 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve; import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters; import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext; +import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderUtils; import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList; import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterUtils; import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field; import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable; +import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineStorageUtils; import org.apache.hadoop.yarn.server.timelineservice.storage.common.RowKeyPrefix; import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunColumn; import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunColumnFamily; @@ -63,7 +65,7 @@ class FlowRunEntityReader extends TimelineEntityReader { public FlowRunEntityReader(TimelineReaderContext ctxt, TimelineEntityFilters entityFilters, TimelineDataToRetrieve toRetrieve) { - super(ctxt, entityFilters, toRetrieve, true); + super(ctxt, entityFilters, toRetrieve); } public FlowRunEntityReader(TimelineReaderContext ctxt, @@ -210,10 +212,36 @@ class FlowRunEntityReader extends TimelineEntityReader { FilterList filterList) throws IOException { Scan scan = new Scan(); TimelineReaderContext context = getContext(); - RowKeyPrefix flowRunRowKeyPrefix = - new FlowRunRowKeyPrefix(context.getClusterId(), context.getUserId(), - context.getFlowName()); - scan.setRowPrefixFilter(flowRunRowKeyPrefix.getRowKeyPrefix()); + RowKeyPrefix flowRunRowKeyPrefix = null; + if (getFilters().getFromId() == null) { + flowRunRowKeyPrefix = new FlowRunRowKeyPrefix(context.getClusterId(), + context.getUserId(), context.getFlowName()); + scan.setRowPrefixFilter(flowRunRowKeyPrefix.getRowKeyPrefix()); + } else { + FlowRunRowKey flowRunRowKey = null; + try { + flowRunRowKey = + FlowRunRowKey.parseRowKeyFromString(getFilters().getFromId()); + } catch (IllegalArgumentException e) { + throw new BadRequestException("Invalid filter fromid is provided."); + } + if (!context.getClusterId().equals(flowRunRowKey.getClusterId())) { + throw new BadRequestException( + "fromid doesn't belong to clusterId=" + context.getClusterId()); + } + // set start row + scan.setStartRow(flowRunRowKey.getRowKey()); + + // get the bytes for stop row + flowRunRowKeyPrefix = new FlowRunRowKeyPrefix(context.getClusterId(), + context.getUserId(), context.getFlowName()); + + // set stop row + scan.setStopRow( + HBaseTimelineStorageUtils.calculateTheClosestNextRowKeyForPrefix( + flowRunRowKeyPrefix.getRowKeyPrefix())); + } + FilterList newList = new FilterList(); newList.addFilter(new PageFilter(getFilters().getLimit())); if (filterList != null && !filterList.getFilters().isEmpty()) { @@ -226,16 +254,11 @@ class FlowRunEntityReader extends TimelineEntityReader { @Override protected TimelineEntity parseEntity(Result result) throws IOException { - TimelineReaderContext context = getContext(); FlowRunEntity flowRun = new FlowRunEntity(); - flowRun.setUser(context.getUserId()); - flowRun.setName(context.getFlowName()); - if (isSingleEntityRead()) { - flowRun.setRunId(context.getFlowRunId()); - } else { - FlowRunRowKey rowKey = FlowRunRowKey.parseRowKey(result.getRow()); - flowRun.setRunId(rowKey.getFlowRunId()); - } + FlowRunRowKey rowKey = FlowRunRowKey.parseRowKey(result.getRow()); + flowRun.setRunId(rowKey.getFlowRunId()); + flowRun.setUser(rowKey.getUserId()); + flowRun.setName(rowKey.getFlowName()); // read the start time Long startTime = (Long) FlowRunColumn.MIN_START_TIME.readResult(result); @@ -264,6 +287,8 @@ class FlowRunEntityReader extends TimelineEntityReader { // set the id flowRun.setId(flowRun.getId()); + flowRun.getInfo().put(TimelineReaderUtils.FROMID_KEY, + rowKey.getRowKeyAsString()); return flowRun; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java index 4e1ab8a378c..3a444455bd9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java @@ -19,12 +19,14 @@ package org.apache.hadoop.yarn.server.timelineservice.storage.reader; import java.io.IOException; import java.util.EnumSet; +import java.util.Iterator; import java.util.Map; import java.util.Set; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Query; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; @@ -32,20 +34,20 @@ import org.apache.hadoop.hbase.filter.BinaryComparator; import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; import org.apache.hadoop.hbase.filter.FamilyFilter; import org.apache.hadoop.hbase.filter.FilterList; +import org.apache.hadoop.hbase.filter.PageFilter; import org.apache.hadoop.hbase.filter.FilterList.Operator; import org.apache.hadoop.hbase.filter.QualifierFilter; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve; import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters; import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext; +import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderUtils; import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList; import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterUtils; import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field; -import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowColumn; -import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowRowKey; -import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowTable; import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable; import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix; +import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineStorageUtils; import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter; import org.apache.hadoop.yarn.server.timelineservice.storage.common.RowKeyPrefix; import org.apache.hadoop.yarn.server.timelineservice.storage.common.StringKeyConverter; @@ -56,7 +58,7 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumn import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKey; import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKeyPrefix; import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTable; -import org.apache.hadoop.yarn.webapp.NotFoundException; +import org.apache.hadoop.yarn.webapp.BadRequestException; import com.google.common.base.Preconditions; @@ -67,11 +69,6 @@ import com.google.common.base.Preconditions; class GenericEntityReader extends TimelineEntityReader { private static final EntityTable ENTITY_TABLE = new EntityTable(); - /** - * Used to look up the flow context. - */ - private final AppToFlowTable appToFlowTable = new AppToFlowTable(); - /** * Used to convert strings key components to and from storage format. */ @@ -79,9 +76,8 @@ class GenericEntityReader extends TimelineEntityReader { new StringKeyConverter(); public GenericEntityReader(TimelineReaderContext ctxt, - TimelineEntityFilters entityFilters, TimelineDataToRetrieve toRetrieve, - boolean sortedKeys) { - super(ctxt, entityFilters, toRetrieve, sortedKeys); + TimelineEntityFilters entityFilters, TimelineDataToRetrieve toRetrieve) { + super(ctxt, entityFilters, toRetrieve); } public GenericEntityReader(TimelineReaderContext ctxt, @@ -139,7 +135,7 @@ class GenericEntityReader extends TimelineEntityReader { * * @return true if we need to fetch some of the columns, false otherwise. */ - private boolean fetchPartialEventCols(TimelineFilterList eventFilters, + protected boolean fetchPartialEventCols(TimelineFilterList eventFilters, EnumSet fieldsToRetrieve) { return (eventFilters != null && !eventFilters.getFilterList().isEmpty() && !hasField(fieldsToRetrieve, Field.EVENTS)); @@ -150,7 +146,7 @@ class GenericEntityReader extends TimelineEntityReader { * * @return true if we need to fetch some of the columns, false otherwise. */ - private boolean fetchPartialRelatesToCols(TimelineFilterList relatesTo, + protected boolean fetchPartialRelatesToCols(TimelineFilterList relatesTo, EnumSet fieldsToRetrieve) { return (relatesTo != null && !relatesTo.getFilterList().isEmpty() && !hasField(fieldsToRetrieve, Field.RELATES_TO)); @@ -400,60 +396,6 @@ class GenericEntityReader extends TimelineEntityReader { return listBasedOnFields; } - /** - * Looks up flow context from AppToFlow table. - * - * @param appToFlowRowKey to identify Cluster and App Ids. - * @param hbaseConf HBase configuration. - * @param conn HBase Connection. - * @return flow context information. - * @throws IOException if any problem occurs while fetching flow information. - */ - protected FlowContext lookupFlowContext(AppToFlowRowKey appToFlowRowKey, - Configuration hbaseConf, Connection conn) throws IOException { - byte[] rowKey = appToFlowRowKey.getRowKey(); - Get get = new Get(rowKey); - Result result = appToFlowTable.getResult(hbaseConf, conn, get); - if (result != null && !result.isEmpty()) { - return new FlowContext(AppToFlowColumn.USER_ID.readResult(result) - .toString(), AppToFlowColumn.FLOW_ID.readResult(result).toString(), - ((Number) AppToFlowColumn.FLOW_RUN_ID.readResult(result)) - .longValue()); - } else { - throw new NotFoundException( - "Unable to find the context flow ID and flow run ID for clusterId=" - + appToFlowRowKey.getClusterId() + ", appId=" - + appToFlowRowKey.getAppId()); - } - } - - /** - * Encapsulates flow context information. - */ - protected static class FlowContext { - private final String userId; - private final String flowName; - private final Long flowRunId; - - public FlowContext(String user, String flowName, Long flowRunId) { - this.userId = user; - this.flowName = flowName; - this.flowRunId = flowRunId; - } - - protected String getUserId() { - return userId; - } - - protected String getFlowName() { - return flowName; - } - - protected Long getFlowRunId() { - return flowRunId; - } - } - @Override protected void validateParams() { Preconditions.checkNotNull(getContext(), "context shouldn't be null"); @@ -474,19 +416,7 @@ class GenericEntityReader extends TimelineEntityReader { @Override protected void augmentParams(Configuration hbaseConf, Connection conn) throws IOException { - TimelineReaderContext context = getContext(); - // In reality all three should be null or neither should be null - if (context.getFlowName() == null || context.getFlowRunId() == null - || context.getUserId() == null) { - // Get flow context information from AppToFlow table. - AppToFlowRowKey appToFlowRowKey = - new AppToFlowRowKey(context.getClusterId(), context.getAppId()); - FlowContext flowContext = - lookupFlowContext(appToFlowRowKey, hbaseConf, conn); - context.setFlowName(flowContext.flowName); - context.setFlowRunId(flowContext.flowRunId); - context.setUserId(flowContext.userId); - } + defaultAugmentParams(hbaseConf, conn); // Add configs/metrics to fields to retrieve if confsToRetrieve and/or // metricsToRetrieve are specified. getDataToRetrieve().addFieldsBasedOnConfsAndMetricsToRetrieve(); @@ -499,16 +429,52 @@ class GenericEntityReader extends TimelineEntityReader { protected Result getResult(Configuration hbaseConf, Connection conn, FilterList filterList) throws IOException { TimelineReaderContext context = getContext(); - byte[] rowKey = - new EntityRowKey(context.getClusterId(), context.getUserId(), - context.getFlowName(), context.getFlowRunId(), context.getAppId(), - context.getEntityType(), context.getEntityId()).getRowKey(); - Get get = new Get(rowKey); - get.setMaxVersions(getDataToRetrieve().getMetricsLimit()); - if (filterList != null && !filterList.getFilters().isEmpty()) { - get.setFilter(filterList); + Result result = null; + if (context.getEntityIdPrefix() != null) { + byte[] rowKey = new EntityRowKey(context.getClusterId(), + context.getUserId(), context.getFlowName(), context.getFlowRunId(), + context.getAppId(), context.getEntityType(), + context.getEntityIdPrefix(), context.getEntityId()).getRowKey(); + Get get = new Get(rowKey); + setMetricsTimeRange(get); + get.setMaxVersions(getDataToRetrieve().getMetricsLimit()); + if (filterList != null && !filterList.getFilters().isEmpty()) { + get.setFilter(filterList); + } + result = getTable().getResult(hbaseConf, conn, get); + + } else { + // Prepare for range scan + // create single SingleColumnValueFilter and add to existing filters. + FilterList filter = new FilterList(Operator.MUST_PASS_ALL); + if (filterList != null && !filterList.getFilters().isEmpty()) { + filter.addFilter(filterList); + } + FilterList newFilter = new FilterList(); + newFilter.addFilter(TimelineFilterUtils.createHBaseSingleColValueFilter( + EntityColumn.ID, context.getEntityId(), CompareOp.EQUAL)); + newFilter.addFilter(new PageFilter(1)); + filter.addFilter(newFilter); + + ResultScanner results = getResults(hbaseConf, conn, filter); + try { + Iterator iterator = results.iterator(); + if (iterator.hasNext()) { + result = iterator.next(); + } + } finally { + results.close(); + } } - return getTable().getResult(hbaseConf, conn, get); + return result; + } + + private void setMetricsTimeRange(Query query) { + // Set time range for metric values. + HBaseTimelineStorageUtils.setMetricsTimeRange( + query, EntityColumnFamily.METRICS.getBytes(), + getDataToRetrieve().getMetricsTimeBegin(), + getDataToRetrieve().getMetricsTimeEnd()); } @Override @@ -518,11 +484,45 @@ class GenericEntityReader extends TimelineEntityReader { // and one type Scan scan = new Scan(); TimelineReaderContext context = getContext(); - RowKeyPrefix entityRowKeyPrefix = - new EntityRowKeyPrefix(context.getClusterId(), context.getUserId(), - context.getFlowName(), context.getFlowRunId(), context.getAppId(), - context.getEntityType()); - scan.setRowPrefixFilter(entityRowKeyPrefix.getRowKeyPrefix()); + RowKeyPrefix entityRowKeyPrefix = null; + // default mode, will always scans from beginning of entity type. + if (getFilters() == null || getFilters().getFromId() == null) { + entityRowKeyPrefix = new EntityRowKeyPrefix(context.getClusterId(), + context.getUserId(), context.getFlowName(), context.getFlowRunId(), + context.getAppId(), context.getEntityType(), null, null); + scan.setRowPrefixFilter(entityRowKeyPrefix.getRowKeyPrefix()); + } else { // pagination mode, will scan from given entityIdPrefix!enitityId + + EntityRowKey entityRowKey = null; + try { + entityRowKey = + EntityRowKey.parseRowKeyFromString(getFilters().getFromId()); + } catch (IllegalArgumentException e) { + throw new BadRequestException("Invalid filter fromid is provided."); + } + if (!context.getClusterId().equals(entityRowKey.getClusterId())) { + throw new BadRequestException( + "fromid doesn't belong to clusterId=" + context.getClusterId()); + } + + // set start row + scan.setStartRow(entityRowKey.getRowKey()); + + // get the bytes for stop row + entityRowKeyPrefix = new EntityRowKeyPrefix(context.getClusterId(), + context.getUserId(), context.getFlowName(), context.getFlowRunId(), + context.getAppId(), context.getEntityType(), null, null); + + // set stop row + scan.setStopRow( + HBaseTimelineStorageUtils.calculateTheClosestNextRowKeyForPrefix( + entityRowKeyPrefix.getRowKeyPrefix())); + + // set page filter to limit. This filter has to set only in pagination + // mode. + filterList.addFilter(new PageFilter(getFilters().getLimit())); + } + setMetricsTimeRange(scan); scan.setMaxVersions(getDataToRetrieve().getMetricsLimit()); if (filterList != null && !filterList.getFilters().isEmpty()) { scan.setFilter(filterList); @@ -536,10 +536,10 @@ class GenericEntityReader extends TimelineEntityReader { return null; } TimelineEntity entity = new TimelineEntity(); - String entityType = EntityColumn.TYPE.readResult(result).toString(); - entity.setType(entityType); - String entityId = EntityColumn.ID.readResult(result).toString(); - entity.setId(entityId); + EntityRowKey parseRowKey = EntityRowKey.parseRowKey(result.getRow()); + entity.setType(parseRowKey.getEntityType()); + entity.setId(parseRowKey.getEntityId()); + entity.setIdPrefix(parseRowKey.getEntityIdPrefix().longValue()); TimelineEntityFilters filters = getFilters(); // fetch created time @@ -619,6 +619,9 @@ class GenericEntityReader extends TimelineEntityReader { if (hasField(fieldsToRetrieve, Field.METRICS)) { readMetrics(entity, result, EntityColumnPrefix.METRIC); } + + entity.getInfo().put(TimelineReaderUtils.FROMID_KEY, + parseRowKey.getRowKeyAsString()); return entity; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/SubApplicationEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/SubApplicationEntityReader.java new file mode 100644 index 00000000000..e780dccd3a8 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/SubApplicationEntityReader.java @@ -0,0 +1,488 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.timelineservice.storage.reader; + +import java.io.IOException; +import java.util.EnumSet; +import java.util.Set; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.Query; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.filter.BinaryComparator; +import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; +import org.apache.hadoop.hbase.filter.FamilyFilter; +import org.apache.hadoop.hbase.filter.FilterList; +import org.apache.hadoop.hbase.filter.FilterList.Operator; +import org.apache.hadoop.hbase.filter.PageFilter; +import org.apache.hadoop.hbase.filter.QualifierFilter; +import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; +import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve; +import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters; +import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext; +import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderUtils; +import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList; +import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterUtils; +import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field; +import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable; +import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineStorageUtils; +import org.apache.hadoop.yarn.server.timelineservice.storage.common.RowKeyPrefix; +import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineStorageUtils; +import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationColumn; +import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationColumnFamily; +import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationColumnPrefix; +import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationRowKey; +import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationRowKeyPrefix; +import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationTable; +import org.apache.hadoop.yarn.webapp.BadRequestException; + +import com.google.common.base.Preconditions; + +class SubApplicationEntityReader extends GenericEntityReader { + private static final SubApplicationTable SUB_APPLICATION_TABLE = + new SubApplicationTable(); + + SubApplicationEntityReader(TimelineReaderContext ctxt, + TimelineEntityFilters entityFilters, TimelineDataToRetrieve toRetrieve) { + super(ctxt, entityFilters, toRetrieve); + } + + SubApplicationEntityReader(TimelineReaderContext ctxt, + TimelineDataToRetrieve toRetrieve) { + super(ctxt, toRetrieve); + } + + /** + * Uses the {@link SubApplicationTable}. + */ + protected BaseTable getTable() { + return SUB_APPLICATION_TABLE; + } + + @Override + protected FilterList constructFilterListBasedOnFilters() throws IOException { + // Filters here cannot be null for multiple entity reads as they are set in + // augmentParams if null. + FilterList listBasedOnFilters = new FilterList(); + TimelineEntityFilters filters = getFilters(); + // Create filter list based on created time range and add it to + // listBasedOnFilters. + long createdTimeBegin = filters.getCreatedTimeBegin(); + long createdTimeEnd = filters.getCreatedTimeEnd(); + if (createdTimeBegin != 0 || createdTimeEnd != Long.MAX_VALUE) { + listBasedOnFilters.addFilter(TimelineFilterUtils + .createSingleColValueFiltersByRange(SubApplicationColumn.CREATED_TIME, + createdTimeBegin, createdTimeEnd)); + } + // Create filter list based on metric filters and add it to + // listBasedOnFilters. + TimelineFilterList metricFilters = filters.getMetricFilters(); + if (metricFilters != null && !metricFilters.getFilterList().isEmpty()) { + listBasedOnFilters.addFilter(TimelineFilterUtils.createHBaseFilterList( + SubApplicationColumnPrefix.METRIC, metricFilters)); + } + // Create filter list based on config filters and add it to + // listBasedOnFilters. + TimelineFilterList configFilters = filters.getConfigFilters(); + if (configFilters != null && !configFilters.getFilterList().isEmpty()) { + listBasedOnFilters.addFilter(TimelineFilterUtils.createHBaseFilterList( + SubApplicationColumnPrefix.CONFIG, configFilters)); + } + // Create filter list based on info filters and add it to listBasedOnFilters + TimelineFilterList infoFilters = filters.getInfoFilters(); + if (infoFilters != null && !infoFilters.getFilterList().isEmpty()) { + listBasedOnFilters.addFilter(TimelineFilterUtils + .createHBaseFilterList(SubApplicationColumnPrefix.INFO, infoFilters)); + } + return listBasedOnFilters; + } + + /** + * Add {@link QualifierFilter} filters to filter list for each column of + * entity table. + * + * @param list filter list to which qualifier filters have to be added. + */ + protected void updateFixedColumns(FilterList list) { + for (SubApplicationColumn column : SubApplicationColumn.values()) { + list.addFilter(new QualifierFilter(CompareOp.EQUAL, + new BinaryComparator(column.getColumnQualifierBytes()))); + } + } + + /** + * Creates a filter list which indicates that only some of the column + * qualifiers in the info column family will be returned in result. + * + * @param isApplication If true, it means operations are to be performed for + * application table, otherwise for entity table. + * @return filter list. + * @throws IOException if any problem occurs while creating filter list. + */ + private FilterList createFilterListForColsOfInfoFamily() throws IOException { + FilterList infoFamilyColsFilter = new FilterList(Operator.MUST_PASS_ONE); + // Add filters for each column in entity table. + updateFixedColumns(infoFamilyColsFilter); + EnumSet fieldsToRetrieve = getDataToRetrieve().getFieldsToRetrieve(); + // If INFO field has to be retrieved, add a filter for fetching columns + // with INFO column prefix. + if (hasField(fieldsToRetrieve, Field.INFO)) { + infoFamilyColsFilter.addFilter( + TimelineFilterUtils.createHBaseQualifierFilter(CompareOp.EQUAL, + SubApplicationColumnPrefix.INFO)); + } + TimelineFilterList relatesTo = getFilters().getRelatesTo(); + if (hasField(fieldsToRetrieve, Field.RELATES_TO)) { + // If RELATES_TO field has to be retrieved, add a filter for fetching + // columns with RELATES_TO column prefix. + infoFamilyColsFilter.addFilter( + TimelineFilterUtils.createHBaseQualifierFilter(CompareOp.EQUAL, + SubApplicationColumnPrefix.RELATES_TO)); + } else if (relatesTo != null && !relatesTo.getFilterList().isEmpty()) { + // Even if fields to retrieve does not contain RELATES_TO, we still + // need to have a filter to fetch some of the column qualifiers if + // relatesTo filters are specified. relatesTo filters will then be + // matched after fetching rows from HBase. + Set relatesToCols = + TimelineFilterUtils.fetchColumnsFromFilterList(relatesTo); + infoFamilyColsFilter.addFilter(createFiltersFromColumnQualifiers( + SubApplicationColumnPrefix.RELATES_TO, relatesToCols)); + } + TimelineFilterList isRelatedTo = getFilters().getIsRelatedTo(); + if (hasField(fieldsToRetrieve, Field.IS_RELATED_TO)) { + // If IS_RELATED_TO field has to be retrieved, add a filter for fetching + // columns with IS_RELATED_TO column prefix. + infoFamilyColsFilter.addFilter( + TimelineFilterUtils.createHBaseQualifierFilter(CompareOp.EQUAL, + SubApplicationColumnPrefix.IS_RELATED_TO)); + } else if (isRelatedTo != null && !isRelatedTo.getFilterList().isEmpty()) { + // Even if fields to retrieve does not contain IS_RELATED_TO, we still + // need to have a filter to fetch some of the column qualifiers if + // isRelatedTo filters are specified. isRelatedTo filters will then be + // matched after fetching rows from HBase. + Set isRelatedToCols = + TimelineFilterUtils.fetchColumnsFromFilterList(isRelatedTo); + infoFamilyColsFilter.addFilter(createFiltersFromColumnQualifiers( + SubApplicationColumnPrefix.IS_RELATED_TO, isRelatedToCols)); + } + TimelineFilterList eventFilters = getFilters().getEventFilters(); + if (hasField(fieldsToRetrieve, Field.EVENTS)) { + // If EVENTS field has to be retrieved, add a filter for fetching columns + // with EVENT column prefix. + infoFamilyColsFilter.addFilter( + TimelineFilterUtils.createHBaseQualifierFilter(CompareOp.EQUAL, + SubApplicationColumnPrefix.EVENT)); + } else if (eventFilters != null + && !eventFilters.getFilterList().isEmpty()) { + // Even if fields to retrieve does not contain EVENTS, we still need to + // have a filter to fetch some of the column qualifiers on the basis of + // event filters specified. Event filters will then be matched after + // fetching rows from HBase. + Set eventCols = + TimelineFilterUtils.fetchColumnsFromFilterList(eventFilters); + infoFamilyColsFilter.addFilter(createFiltersFromColumnQualifiers( + SubApplicationColumnPrefix.EVENT, eventCols)); + } + return infoFamilyColsFilter; + } + + /** + * Exclude column prefixes via filters which are not required(based on fields + * to retrieve) from info column family. These filters are added to filter + * list which contains a filter for getting info column family. + * + * @param infoColFamilyList filter list for info column family. + */ + private void excludeFieldsFromInfoColFamily(FilterList infoColFamilyList) { + EnumSet fieldsToRetrieve = getDataToRetrieve().getFieldsToRetrieve(); + // Events not required. + if (!hasField(fieldsToRetrieve, Field.EVENTS)) { + infoColFamilyList.addFilter( + TimelineFilterUtils.createHBaseQualifierFilter(CompareOp.NOT_EQUAL, + SubApplicationColumnPrefix.EVENT)); + } + // info not required. + if (!hasField(fieldsToRetrieve, Field.INFO)) { + infoColFamilyList.addFilter( + TimelineFilterUtils.createHBaseQualifierFilter(CompareOp.NOT_EQUAL, + SubApplicationColumnPrefix.INFO)); + } + // is related to not required. + if (!hasField(fieldsToRetrieve, Field.IS_RELATED_TO)) { + infoColFamilyList.addFilter( + TimelineFilterUtils.createHBaseQualifierFilter(CompareOp.NOT_EQUAL, + SubApplicationColumnPrefix.IS_RELATED_TO)); + } + // relates to not required. + if (!hasField(fieldsToRetrieve, Field.RELATES_TO)) { + infoColFamilyList.addFilter( + TimelineFilterUtils.createHBaseQualifierFilter(CompareOp.NOT_EQUAL, + SubApplicationColumnPrefix.RELATES_TO)); + } + } + + /** + * Updates filter list based on fields for confs and metrics to retrieve. + * + * @param listBasedOnFields filter list based on fields. + * @throws IOException if any problem occurs while updating filter list. + */ + private void updateFilterForConfsAndMetricsToRetrieve( + FilterList listBasedOnFields) throws IOException { + TimelineDataToRetrieve dataToRetrieve = getDataToRetrieve(); + // Please note that if confsToRetrieve is specified, we would have added + // CONFS to fields to retrieve in augmentParams() even if not specified. + if (dataToRetrieve.getFieldsToRetrieve().contains(Field.CONFIGS)) { + // Create a filter list for configs. + listBasedOnFields.addFilter( + TimelineFilterUtils.createFilterForConfsOrMetricsToRetrieve( + dataToRetrieve.getConfsToRetrieve(), + SubApplicationColumnFamily.CONFIGS, + SubApplicationColumnPrefix.CONFIG)); + } + + // Please note that if metricsToRetrieve is specified, we would have added + // METRICS to fields to retrieve in augmentParams() even if not specified. + if (dataToRetrieve.getFieldsToRetrieve().contains(Field.METRICS)) { + // Create a filter list for metrics. + listBasedOnFields.addFilter( + TimelineFilterUtils.createFilterForConfsOrMetricsToRetrieve( + dataToRetrieve.getMetricsToRetrieve(), + SubApplicationColumnFamily.METRICS, + SubApplicationColumnPrefix.METRIC)); + } + } + + @Override + protected FilterList constructFilterListBasedOnFields() throws IOException { + if (!needCreateFilterListBasedOnFields()) { + // Fetch all the columns. No need of a filter. + return null; + } + FilterList listBasedOnFields = new FilterList(Operator.MUST_PASS_ONE); + FilterList infoColFamilyList = new FilterList(); + // By default fetch everything in INFO column family. + FamilyFilter infoColumnFamily = new FamilyFilter(CompareOp.EQUAL, + new BinaryComparator(SubApplicationColumnFamily.INFO.getBytes())); + infoColFamilyList.addFilter(infoColumnFamily); + if (fetchPartialColsFromInfoFamily()) { + // We can fetch only some of the columns from info family. + infoColFamilyList.addFilter(createFilterListForColsOfInfoFamily()); + } else { + // Exclude column prefixes in info column family which are not required + // based on fields to retrieve. + excludeFieldsFromInfoColFamily(infoColFamilyList); + } + listBasedOnFields.addFilter(infoColFamilyList); + updateFilterForConfsAndMetricsToRetrieve(listBasedOnFields); + return listBasedOnFields; + } + + @Override + protected void validateParams() { + Preconditions.checkNotNull(getContext(), "context shouldn't be null"); + Preconditions.checkNotNull(getDataToRetrieve(), + "data to retrieve shouldn't be null"); + Preconditions.checkNotNull(getContext().getClusterId(), + "clusterId shouldn't be null"); + Preconditions.checkNotNull(getContext().getDoAsUser(), + "DoAsUser shouldn't be null"); + Preconditions.checkNotNull(getContext().getEntityType(), + "entityType shouldn't be null"); + } + + @Override + protected void augmentParams(Configuration hbaseConf, Connection conn) + throws IOException { + getDataToRetrieve().addFieldsBasedOnConfsAndMetricsToRetrieve(); + createFiltersIfNull(); + } + + private void setMetricsTimeRange(Query query) { + // Set time range for metric values. + HBaseTimelineStorageUtils.setMetricsTimeRange(query, + SubApplicationColumnFamily.METRICS.getBytes(), + getDataToRetrieve().getMetricsTimeBegin(), + getDataToRetrieve().getMetricsTimeEnd()); + } + + @Override + protected ResultScanner getResults(Configuration hbaseConf, Connection conn, + FilterList filterList) throws IOException { + + // Scan through part of the table to find the entities belong to one app + // and one type + Scan scan = new Scan(); + TimelineReaderContext context = getContext(); + if (context.getDoAsUser() == null) { + throw new BadRequestException("Invalid user!"); + } + + RowKeyPrefix subApplicationRowKeyPrefix = null; + // default mode, will always scans from beginning of entity type. + if (getFilters() == null || getFilters().getFromId() == null) { + subApplicationRowKeyPrefix = new SubApplicationRowKeyPrefix( + context.getDoAsUser(), context.getClusterId(), + context.getEntityType(), null, null, null); + scan.setRowPrefixFilter(subApplicationRowKeyPrefix.getRowKeyPrefix()); + } else { // pagination mode, will scan from given entityIdPrefix!enitityId + + SubApplicationRowKey entityRowKey = null; + try { + entityRowKey = SubApplicationRowKey + .parseRowKeyFromString(getFilters().getFromId()); + } catch (IllegalArgumentException e) { + throw new BadRequestException("Invalid filter fromid is provided."); + } + if (!context.getClusterId().equals(entityRowKey.getClusterId())) { + throw new BadRequestException( + "fromid doesn't belong to clusterId=" + context.getClusterId()); + } + + // set start row + scan.setStartRow(entityRowKey.getRowKey()); + + // get the bytes for stop row + subApplicationRowKeyPrefix = new SubApplicationRowKeyPrefix( + context.getDoAsUser(), context.getClusterId(), + context.getEntityType(), null, null, null); + + // set stop row + scan.setStopRow( + HBaseTimelineStorageUtils.calculateTheClosestNextRowKeyForPrefix( + subApplicationRowKeyPrefix.getRowKeyPrefix())); + + // set page filter to limit. This filter has to set only in pagination + // mode. + filterList.addFilter(new PageFilter(getFilters().getLimit())); + } + setMetricsTimeRange(scan); + scan.setMaxVersions(getDataToRetrieve().getMetricsLimit()); + if (filterList != null && !filterList.getFilters().isEmpty()) { + scan.setFilter(filterList); + } + return getTable().getResultScanner(hbaseConf, conn, scan); + } + + @Override + protected Result getResult(Configuration hbaseConf, Connection conn, + FilterList filterList) throws IOException { + throw new UnsupportedOperationException( + "we don't support a single entity query"); + } + + @Override + protected TimelineEntity parseEntity(Result result) throws IOException { + if (result == null || result.isEmpty()) { + return null; + } + TimelineEntity entity = new TimelineEntity(); + SubApplicationRowKey parseRowKey = + SubApplicationRowKey.parseRowKey(result.getRow()); + entity.setType(parseRowKey.getEntityType()); + entity.setId(parseRowKey.getEntityId()); + entity.setIdPrefix(parseRowKey.getEntityIdPrefix().longValue()); + + TimelineEntityFilters filters = getFilters(); + // fetch created time + Long createdTime = + (Long) SubApplicationColumn.CREATED_TIME.readResult(result); + entity.setCreatedTime(createdTime); + + EnumSet fieldsToRetrieve = getDataToRetrieve().getFieldsToRetrieve(); + // fetch is related to entities and match isRelatedTo filter. If isRelatedTo + // filters do not match, entity would be dropped. We have to match filters + // locally as relevant HBase filters to filter out rows on the basis of + // isRelatedTo are not set in HBase scan. + boolean checkIsRelatedTo = + filters.getIsRelatedTo() != null + && filters.getIsRelatedTo().getFilterList().size() > 0; + if (hasField(fieldsToRetrieve, Field.IS_RELATED_TO) || checkIsRelatedTo) { + readRelationship(entity, result, SubApplicationColumnPrefix.IS_RELATED_TO, + true); + if (checkIsRelatedTo && !TimelineStorageUtils.matchIsRelatedTo(entity, + filters.getIsRelatedTo())) { + return null; + } + if (!hasField(fieldsToRetrieve, Field.IS_RELATED_TO)) { + entity.getIsRelatedToEntities().clear(); + } + } + + // fetch relates to entities and match relatesTo filter. If relatesTo + // filters do not match, entity would be dropped. We have to match filters + // locally as relevant HBase filters to filter out rows on the basis of + // relatesTo are not set in HBase scan. + boolean checkRelatesTo = + !isSingleEntityRead() && filters.getRelatesTo() != null + && filters.getRelatesTo().getFilterList().size() > 0; + if (hasField(fieldsToRetrieve, Field.RELATES_TO) || checkRelatesTo) { + readRelationship(entity, result, SubApplicationColumnPrefix.RELATES_TO, + false); + if (checkRelatesTo && !TimelineStorageUtils.matchRelatesTo(entity, + filters.getRelatesTo())) { + return null; + } + if (!hasField(fieldsToRetrieve, Field.RELATES_TO)) { + entity.getRelatesToEntities().clear(); + } + } + + // fetch info if fieldsToRetrieve contains INFO or ALL. + if (hasField(fieldsToRetrieve, Field.INFO)) { + readKeyValuePairs(entity, result, SubApplicationColumnPrefix.INFO, false); + } + + // fetch configs if fieldsToRetrieve contains CONFIGS or ALL. + if (hasField(fieldsToRetrieve, Field.CONFIGS)) { + readKeyValuePairs(entity, result, SubApplicationColumnPrefix.CONFIG, + true); + } + + // fetch events and match event filters if they exist. If event filters do + // not match, entity would be dropped. We have to match filters locally + // as relevant HBase filters to filter out rows on the basis of events + // are not set in HBase scan. + boolean checkEvents = + !isSingleEntityRead() && filters.getEventFilters() != null + && filters.getEventFilters().getFilterList().size() > 0; + if (hasField(fieldsToRetrieve, Field.EVENTS) || checkEvents) { + readEvents(entity, result, SubApplicationColumnPrefix.EVENT); + if (checkEvents && !TimelineStorageUtils.matchEventFilters(entity, + filters.getEventFilters())) { + return null; + } + if (!hasField(fieldsToRetrieve, Field.EVENTS)) { + entity.getEvents().clear(); + } + } + + // fetch metrics if fieldsToRetrieve contains METRICS or ALL. + if (hasField(fieldsToRetrieve, Field.METRICS)) { + readMetrics(entity, result, SubApplicationColumnPrefix.METRIC); + } + + entity.getInfo().put(TimelineReaderUtils.FROMID_KEY, + parseRowKey.getRowKeyAsString()); + return entity; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReader.java index 424d14118b2..07e84234254 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReader.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReader.java @@ -21,11 +21,10 @@ import java.io.IOException; import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.Map; import java.util.NavigableMap; -import java.util.NavigableSet; import java.util.Set; -import java.util.TreeSet; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.Connection; @@ -60,12 +59,12 @@ import org.slf4j.LoggerFactory; * HBase storage. Different types can be defined for different types of the * entities that are being requested. */ -public abstract class TimelineEntityReader { +public abstract class TimelineEntityReader extends + AbstractTimelineStorageReader { private static final Logger LOG = LoggerFactory.getLogger(TimelineEntityReader.class); private final boolean singleEntityRead; - private TimelineReaderContext context; private TimelineDataToRetrieve dataToRetrieve; // used only for multiple entity read mode private TimelineEntityFilters filters; @@ -75,14 +74,6 @@ public abstract class TimelineEntityReader { */ private BaseTable table; - /** - * Specifies whether keys for this table are sorted in a manner where entities - * can be retrieved by created time. If true, it will be sufficient to collect - * the first results as specified by the limit. Otherwise all matched entities - * will be fetched and then limit applied. - */ - private boolean sortedKeys = false; - /** * Used to convert strings key components to and from storage format. */ @@ -96,15 +87,11 @@ public abstract class TimelineEntityReader { * made. * @param entityFilters Filters which limit the entities returned. * @param toRetrieve Data to retrieve for each entity. - * @param sortedKeys Specifies whether key for this table are sorted or not. - * If sorted, entities can be retrieved by created time. */ protected TimelineEntityReader(TimelineReaderContext ctxt, - TimelineEntityFilters entityFilters, TimelineDataToRetrieve toRetrieve, - boolean sortedKeys) { + TimelineEntityFilters entityFilters, TimelineDataToRetrieve toRetrieve) { + super(ctxt); this.singleEntityRead = false; - this.sortedKeys = sortedKeys; - this.context = ctxt; this.dataToRetrieve = toRetrieve; this.filters = entityFilters; @@ -120,8 +107,8 @@ public abstract class TimelineEntityReader { */ protected TimelineEntityReader(TimelineReaderContext ctxt, TimelineDataToRetrieve toRetrieve) { + super(ctxt); this.singleEntityRead = true; - this.context = ctxt; this.dataToRetrieve = toRetrieve; this.setTable(getTable()); @@ -185,10 +172,6 @@ public abstract class TimelineEntityReader { return null; } - protected TimelineReaderContext getContext() { - return context; - } - protected TimelineDataToRetrieve getDataToRetrieve() { return dataToRetrieve; } @@ -203,7 +186,7 @@ public abstract class TimelineEntityReader { */ protected void createFiltersIfNull() { if (filters == null) { - filters = new TimelineEntityFilters(); + filters = new TimelineEntityFilters.Builder().build(); } } @@ -229,7 +212,7 @@ public abstract class TimelineEntityReader { if (result == null || result.isEmpty()) { // Could not find a matching row. LOG.info("Cannot find matching entity of type " + - context.getEntityType()); + getContext().getEntityType()); return null; } return parseEntity(result); @@ -250,7 +233,7 @@ public abstract class TimelineEntityReader { validateParams(); augmentParams(hbaseConf, conn); - NavigableSet entities = new TreeSet<>(); + Set entities = new LinkedHashSet<>(); FilterList filterList = createFilterList(); if (LOG.isDebugEnabled() && filterList != null) { LOG.debug("FilterList created for scan is - " + filterList); @@ -263,14 +246,8 @@ public abstract class TimelineEntityReader { continue; } entities.add(entity); - if (!sortedKeys) { - if (entities.size() > filters.getLimit()) { - entities.pollLast(); - } - } else { - if (entities.size() == filters.getLimit()) { - break; - } + if (entities.size() == filters.getLimit()) { + break; } } return entities; @@ -288,21 +265,6 @@ public abstract class TimelineEntityReader { return table; } - /** - * Validates the required parameters to read the entities. - */ - protected abstract void validateParams(); - - /** - * Sets certain parameters to defaults if the values are not provided. - * - * @param hbaseConf HBase Configuration. - * @param conn HBase Connection. - * @throws IOException if any exception is encountered while setting params. - */ - protected abstract void augmentParams(Configuration hbaseConf, - Connection conn) throws IOException; - /** * Fetches a {@link Result} instance for a single-entity read. * diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReaderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReaderFactory.java index b2a947648a2..fa16077c2ed 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReaderFactory.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReaderFactory.java @@ -82,8 +82,24 @@ public final class TimelineEntityReaderFactory { YARN_FLOW_RUN.matches(context.getEntityType())) { return new FlowRunEntityReader(context, filters, dataToRetrieve); } else { + if (context.getDoAsUser() != null) { + return new SubApplicationEntityReader(context, filters, dataToRetrieve); + } // assume we're dealing with a generic entity read - return new GenericEntityReader(context, filters, dataToRetrieve, false); + return new GenericEntityReader(context, filters, dataToRetrieve); } } + + /** + * Creates a timeline entity type reader that will read all available entity + * types within the specified context. + * + * @param context Reader context which defines the scope in which query has to + * be made. Limited to application level only. + * @return an EntityTypeReader object + */ + public static EntityTypeReader createEntityTypeReader( + TimelineReaderContext context) { + return new EntityTypeReader(context); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationColumn.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationColumn.java new file mode 100644 index 00000000000..46b0cc90e43 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationColumn.java @@ -0,0 +1,108 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.timelineservice.storage.subapplication; + +import java.io.IOException; + +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.yarn.server.timelineservice.storage.common.Column; +import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily; +import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper; +import org.apache.hadoop.yarn.server.timelineservice.storage.common.GenericConverter; +import org.apache.hadoop.yarn.server.timelineservice.storage.common.LongConverter; +import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator; +import org.apache.hadoop.yarn.server.timelineservice.storage.common.TypedBufferedMutator; +import org.apache.hadoop.yarn.server.timelineservice.storage.common.ValueConverter; +import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute; + +/** + * Identifies fully qualified columns for the {@link SubApplicationTable}. + */ +public enum SubApplicationColumn implements Column { + + /** + * Identifier for the sub application. + */ + ID(SubApplicationColumnFamily.INFO, "id"), + + /** + * The type of sub application. + */ + TYPE(SubApplicationColumnFamily.INFO, "type"), + + /** + * When the sub application was created. + */ + CREATED_TIME(SubApplicationColumnFamily.INFO, "created_time", + new LongConverter()), + + /** + * The version of the flow that this sub application belongs to. + */ + FLOW_VERSION(SubApplicationColumnFamily.INFO, "flow_version"); + + private final ColumnHelper column; + private final ColumnFamily columnFamily; + private final String columnQualifier; + private final byte[] columnQualifierBytes; + + SubApplicationColumn(ColumnFamily columnFamily, + String columnQualifier) { + this(columnFamily, columnQualifier, GenericConverter.getInstance()); + } + + SubApplicationColumn(ColumnFamily columnFamily, + String columnQualifier, ValueConverter converter) { + this.columnFamily = columnFamily; + this.columnQualifier = columnQualifier; + // Future-proof by ensuring the right column prefix hygiene. + this.columnQualifierBytes = + Bytes.toBytes(Separator.SPACE.encode(columnQualifier)); + this.column = new ColumnHelper(columnFamily, + converter); + } + + + public void store(byte[] rowKey, + TypedBufferedMutator tableMutator, Long timestamp, + Object inputValue, Attribute... attributes) throws IOException { + column.store(rowKey, tableMutator, columnQualifierBytes, timestamp, + inputValue, attributes); + } + + public Object readResult(Result result) throws IOException { + return column.readResult(result, columnQualifierBytes); + } + + @Override + public byte[] getColumnQualifierBytes() { + return columnQualifierBytes.clone(); + } + + @Override + public byte[] getColumnFamilyBytes() { + return columnFamily.getBytes(); + } + + @Override + public ValueConverter getValueConverter() { + return column.getValueConverter(); + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationColumnFamily.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationColumnFamily.java new file mode 100644 index 00000000000..1d7f8fdf3da --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationColumnFamily.java @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.timelineservice.storage.subapplication; + +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily; +import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator; + +/** + * Represents the sub application table column families. + */ +public enum SubApplicationColumnFamily + implements ColumnFamily { + + /** + * Info column family houses known columns, specifically ones included in + * columnfamily filters. + */ + INFO("i"), + + /** + * Configurations are in a separate column family for two reasons: + * a) the size of the config values can be very large and + * b) we expect that config values + * are often separately accessed from other metrics and info columns. + */ + CONFIGS("c"), + + /** + * Metrics have a separate column family, because they have a separate TTL. + */ + METRICS("m"); + + /** + * Byte representation of this column family. + */ + private final byte[] bytes; + + /** + * @param value + * create a column family with this name. Must be lower case and + * without spaces. + */ + SubApplicationColumnFamily(String value) { + // column families should be lower case and not contain any spaces. + this.bytes = Bytes.toBytes(Separator.SPACE.encode(value)); + } + + public byte[] getBytes() { + return Bytes.copy(bytes); + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationColumnPrefix.java new file mode 100644 index 00000000000..06eccedf3a5 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationColumnPrefix.java @@ -0,0 +1,250 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.timelineservice.storage.subapplication; + +import java.io.IOException; +import java.util.Map; +import java.util.NavigableMap; + +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily; +import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper; +import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix; +import org.apache.hadoop.yarn.server.timelineservice.storage.common.GenericConverter; +import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter; +import org.apache.hadoop.yarn.server.timelineservice.storage.common.LongConverter; +import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator; +import org.apache.hadoop.yarn.server.timelineservice.storage.common.TypedBufferedMutator; +import org.apache.hadoop.yarn.server.timelineservice.storage.common.ValueConverter; +import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute; + +/** + * Identifies partially qualified columns for the sub app table. + */ +public enum SubApplicationColumnPrefix + implements ColumnPrefix { + + /** + * To store TimelineEntity getIsRelatedToEntities values. + */ + IS_RELATED_TO(SubApplicationColumnFamily.INFO, "s"), + + /** + * To store TimelineEntity getRelatesToEntities values. + */ + RELATES_TO(SubApplicationColumnFamily.INFO, "r"), + + /** + * To store TimelineEntity info values. + */ + INFO(SubApplicationColumnFamily.INFO, "i"), + + /** + * Lifecycle events for an entity. + */ + EVENT(SubApplicationColumnFamily.INFO, "e", true), + + /** + * Config column stores configuration with config key as the column name. + */ + CONFIG(SubApplicationColumnFamily.CONFIGS, null), + + /** + * Metrics are stored with the metric name as the column name. + */ + METRIC(SubApplicationColumnFamily.METRICS, null, new LongConverter()); + + private final ColumnHelper column; + private final ColumnFamily columnFamily; + + /** + * Can be null for those cases where the provided column qualifier is the + * entire column name. + */ + private final String columnPrefix; + private final byte[] columnPrefixBytes; + + /** + * Private constructor, meant to be used by the enum definition. + * + * @param columnFamily that this column is stored in. + * @param columnPrefix for this column. + */ + SubApplicationColumnPrefix(ColumnFamily columnFamily, + String columnPrefix) { + this(columnFamily, columnPrefix, false, GenericConverter.getInstance()); + } + + SubApplicationColumnPrefix(ColumnFamily columnFamily, + String columnPrefix, boolean compondColQual) { + this(columnFamily, columnPrefix, compondColQual, + GenericConverter.getInstance()); + } + + SubApplicationColumnPrefix(ColumnFamily columnFamily, + String columnPrefix, ValueConverter converter) { + this(columnFamily, columnPrefix, false, converter); + } + + /** + * Private constructor, meant to be used by the enum definition. + * + * @param columnFamily that this column is stored in. + * @param columnPrefix for this column. + * @param converter used to encode/decode values to be stored in HBase for + * this column prefix. + */ + SubApplicationColumnPrefix(ColumnFamily columnFamily, + String columnPrefix, boolean compondColQual, ValueConverter converter) { + column = new ColumnHelper(columnFamily, converter); + this.columnFamily = columnFamily; + this.columnPrefix = columnPrefix; + if (columnPrefix == null) { + this.columnPrefixBytes = null; + } else { + // Future-proof by ensuring the right column prefix hygiene. + this.columnPrefixBytes = + Bytes.toBytes(Separator.SPACE.encode(columnPrefix)); + } + } + + /** + * @return the column name value + */ + public String getColumnPrefix() { + return columnPrefix; + } + + @Override + public byte[] getColumnPrefixBytes(byte[] qualifierPrefix) { + return ColumnHelper.getColumnQualifier( + this.columnPrefixBytes, qualifierPrefix); + } + + @Override + public byte[] getColumnPrefixBytes(String qualifierPrefix) { + return ColumnHelper.getColumnQualifier( + this.columnPrefixBytes, qualifierPrefix); + } + + @Override + public byte[] getColumnFamilyBytes() { + return columnFamily.getBytes(); + } + + @Override + public ValueConverter getValueConverter() { + return column.getValueConverter(); + } + + /* + * (non-Javadoc) + * + * @see + * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix + * #store(byte[], + * org.apache.hadoop.yarn.server.timelineservice.storage.common. + * TypedBufferedMutator, java.lang.String, java.lang.Long, java.lang.Object, + * org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute[]) + */ + public void store(byte[] rowKey, + TypedBufferedMutator tableMutator, String qualifier, + Long timestamp, Object inputValue, Attribute... attributes) + throws IOException { + + // Null check + if (qualifier == null) { + throw new IOException("Cannot store column with null qualifier in " + + tableMutator.getName().getNameAsString()); + } + + byte[] columnQualifier = getColumnPrefixBytes(qualifier); + + column.store(rowKey, tableMutator, columnQualifier, timestamp, inputValue, + attributes); + } + + /* + * (non-Javadoc) + * + * @see + * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix + * #store(byte[], + * org.apache.hadoop.yarn.server.timelineservice.storage.common. + * TypedBufferedMutator, java.lang.String, java.lang.Long, java.lang.Object) + */ + public void store(byte[] rowKey, + TypedBufferedMutator tableMutator, byte[] qualifier, + Long timestamp, Object inputValue, Attribute... attributes) + throws IOException { + + // Null check + if (qualifier == null) { + throw new IOException("Cannot store column with null qualifier in " + + tableMutator.getName().getNameAsString()); + } + + byte[] columnQualifier = getColumnPrefixBytes(qualifier); + + column.store(rowKey, tableMutator, columnQualifier, timestamp, inputValue, + attributes); + } + + /* + * (non-Javadoc) + * + * @see + * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix + * #readResult(org.apache.hadoop.hbase.client.Result, java.lang.String) + */ + public Object readResult(Result result, String qualifier) throws IOException { + byte[] columnQualifier = + ColumnHelper.getColumnQualifier(this.columnPrefixBytes, qualifier); + return column.readResult(result, columnQualifier); + } + + /* + * (non-Javadoc) + * + * @see + * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix + * #readResults(org.apache.hadoop.hbase.client.Result, + * org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter) + */ + public Map readResults(Result result, + KeyConverter keyConverter) throws IOException { + return column.readResults(result, columnPrefixBytes, keyConverter); + } + + /* + * (non-Javadoc) + * + * @see + * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix + * #readResultsWithTimestamps(org.apache.hadoop.hbase.client.Result, + * org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter) + */ + public NavigableMap> + readResultsWithTimestamps(Result result, KeyConverter keyConverter) + throws IOException { + return column.readResultsWithTimestamps(result, columnPrefixBytes, + keyConverter); + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationRowKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationRowKey.java new file mode 100644 index 00000000000..fb1f774eaa0 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationRowKey.java @@ -0,0 +1,290 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.timelineservice.storage.subapplication; + +import java.util.List; + +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderUtils; +import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter; +import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverterToString; +import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator; + +/** + * Represents a rowkey for the sub app table. + */ +public class SubApplicationRowKey { + private final String subAppUserId; + private final String clusterId; + private final String entityType; + private final Long entityIdPrefix; + private final String entityId; + private final String userId; + private final SubApplicationRowKeyConverter subAppRowKeyConverter = + new SubApplicationRowKeyConverter(); + + public SubApplicationRowKey(String subAppUserId, String clusterId, + String entityType, Long entityIdPrefix, String entityId, String userId) { + this.subAppUserId = subAppUserId; + this.clusterId = clusterId; + this.entityType = entityType; + this.entityIdPrefix = entityIdPrefix; + this.entityId = entityId; + this.userId = userId; + } + + public String getClusterId() { + return clusterId; + } + + public String getSubAppUserId() { + return subAppUserId; + } + + public String getEntityType() { + return entityType; + } + + public String getEntityId() { + return entityId; + } + + public Long getEntityIdPrefix() { + return entityIdPrefix; + } + + public String getUserId() { + return userId; + } + + /** + * Constructs a row key for the sub app table as follows: + * {@code subAppUserId!clusterId!entityType + * !entityPrefix!entityId!userId}. + * Typically used while querying a specific sub app. + * + * subAppUserId is usually the doAsUser. + * userId is the yarn user that the AM runs as. + * + * @return byte array with the row key. + */ + public byte[] getRowKey() { + return subAppRowKeyConverter.encode(this); + } + + /** + * Given the raw row key as bytes, returns the row key as an object. + * + * @param rowKey byte representation of row key. + * @return An SubApplicationRowKey object. + */ + public static SubApplicationRowKey parseRowKey(byte[] rowKey) { + return new SubApplicationRowKeyConverter().decode(rowKey); + } + + /** + * Constructs a row key for the sub app table as follows: + *

+ * {@code subAppUserId!clusterId! + * entityType!entityIdPrefix!entityId!userId}. + * + * subAppUserId is usually the doAsUser. + * userId is the yarn user that that the AM runs as. + * + *

+ * + * @return String representation of row key. + */ + public String getRowKeyAsString() { + return subAppRowKeyConverter.encodeAsString(this); + } + + /** + * Given the encoded row key as string, returns the row key as an object. + * + * @param encodedRowKey String representation of row key. + * @return A SubApplicationRowKey object. + */ + public static SubApplicationRowKey parseRowKeyFromString( + String encodedRowKey) { + return new SubApplicationRowKeyConverter().decodeFromString(encodedRowKey); + } + + /** + * Encodes and decodes row key for sub app table. + * The row key is of the form : + * subAppUserId!clusterId!flowRunId!appId!entityType!entityId!userId + * + * subAppUserId is usually the doAsUser. + * userId is the yarn user that the AM runs as. + * + *

+ */ + final private static class SubApplicationRowKeyConverter + implements KeyConverter, + KeyConverterToString { + + private SubApplicationRowKeyConverter() { + } + + /** + * sub app row key is of the form + * subAppUserId!clusterId!entityType!entityPrefix!entityId!userId + * w. each segment separated by !. + * + * subAppUserId is usually the doAsUser. + * userId is the yarn user that the AM runs as. + * + * The sizes below indicate sizes of each one of these + * segments in sequence. clusterId, subAppUserId, entityType, + * entityId and userId are strings. + * entity prefix is a long hence 8 bytes in size. Strings are + * variable in size (i.e. end whenever separator is encountered). + * This is used while decoding and helps in determining where to split. + */ + private static final int[] SEGMENT_SIZES = {Separator.VARIABLE_SIZE, + Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE, Bytes.SIZEOF_LONG, + Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE}; + + /* + * (non-Javadoc) + * + * Encodes SubApplicationRowKey object into a byte array with each + * component/field in SubApplicationRowKey separated by + * Separator#QUALIFIERS. + * This leads to an sub app table row key of the form + * subAppUserId!clusterId!entityType!entityPrefix!entityId!userId + * + * subAppUserId is usually the doAsUser. + * userId is the yarn user that the AM runs as. + * + * If entityType in passed SubApplicationRowKey object is null (and the + * fields preceding it are not null i.e. clusterId, subAppUserId), this + * returns a row key prefix of the form subAppUserId!clusterId! + * If entityId in SubApplicationRowKey is null + * (other components are not null), this returns a row key prefix + * of the form subAppUserId!clusterId!entityType! + * + * @see org.apache.hadoop.yarn.server.timelineservice.storage.common + * .KeyConverter#encode(java.lang.Object) + */ + @Override + public byte[] encode(SubApplicationRowKey rowKey) { + byte[] subAppUser = Separator.encode(rowKey.getSubAppUserId(), + Separator.SPACE, Separator.TAB, Separator.QUALIFIERS); + byte[] cluster = Separator.encode(rowKey.getClusterId(), Separator.SPACE, + Separator.TAB, Separator.QUALIFIERS); + byte[] first = Separator.QUALIFIERS.join(subAppUser, cluster); + if (rowKey.getEntityType() == null) { + return first; + } + byte[] entityType = Separator.encode(rowKey.getEntityType(), + Separator.SPACE, Separator.TAB, Separator.QUALIFIERS); + + if (rowKey.getEntityIdPrefix() == null) { + return Separator.QUALIFIERS.join(first, entityType, + Separator.EMPTY_BYTES); + } + + byte[] entityIdPrefix = Bytes.toBytes(rowKey.getEntityIdPrefix()); + + if (rowKey.getEntityId() == null) { + return Separator.QUALIFIERS.join(first, entityType, entityIdPrefix, + Separator.EMPTY_BYTES); + } + + byte[] entityId = Separator.encode(rowKey.getEntityId(), Separator.SPACE, + Separator.TAB, Separator.QUALIFIERS); + + byte[] userId = Separator.encode(rowKey.getUserId(), + Separator.SPACE, Separator.TAB, Separator.QUALIFIERS); + + byte[] second = Separator.QUALIFIERS.join(entityType, entityIdPrefix, + entityId, userId); + + return Separator.QUALIFIERS.join(first, second); + } + + /* + * (non-Javadoc) + * + * Decodes a sub application row key of the form + * subAppUserId!clusterId!entityType!entityPrefix!entityId!userId + * + * subAppUserId is usually the doAsUser. + * userId is the yarn user that the AM runs as. + * + * represented in byte format + * and converts it into an SubApplicationRowKey object. + * + * @see org.apache.hadoop.yarn.server.timelineservice.storage.common + * .KeyConverter#decode(byte[]) + */ + @Override + public SubApplicationRowKey decode(byte[] rowKey) { + byte[][] rowKeyComponents = + Separator.QUALIFIERS.split(rowKey, SEGMENT_SIZES); + if (rowKeyComponents.length != 6) { + throw new IllegalArgumentException( + "the row key is not valid for " + "a sub app"); + } + String subAppUserId = + Separator.decode(Bytes.toString(rowKeyComponents[0]), + Separator.QUALIFIERS, Separator.TAB, Separator.SPACE); + String clusterId = Separator.decode(Bytes.toString(rowKeyComponents[1]), + Separator.QUALIFIERS, Separator.TAB, Separator.SPACE); + String entityType = Separator.decode(Bytes.toString(rowKeyComponents[2]), + Separator.QUALIFIERS, Separator.TAB, Separator.SPACE); + + Long entityPrefixId = Bytes.toLong(rowKeyComponents[3]); + + String entityId = Separator.decode(Bytes.toString(rowKeyComponents[4]), + Separator.QUALIFIERS, Separator.TAB, Separator.SPACE); + String userId = + Separator.decode(Bytes.toString(rowKeyComponents[5]), + Separator.QUALIFIERS, Separator.TAB, Separator.SPACE); + + return new SubApplicationRowKey(subAppUserId, clusterId, entityType, + entityPrefixId, entityId, userId); + } + + @Override + public String encodeAsString(SubApplicationRowKey key) { + if (key.subAppUserId == null || key.clusterId == null + || key.entityType == null || key.entityIdPrefix == null + || key.entityId == null || key.userId == null) { + throw new IllegalArgumentException(); + } + return TimelineReaderUtils.joinAndEscapeStrings( + new String[] {key.subAppUserId, key.clusterId, key.entityType, + key.entityIdPrefix.toString(), key.entityId, key.userId}); + } + + @Override + public SubApplicationRowKey decodeFromString(String encodedRowKey) { + List split = TimelineReaderUtils.split(encodedRowKey); + if (split == null || split.size() != 6) { + throw new IllegalArgumentException( + "Invalid row key for sub app table."); + } + Long entityIdPrefix = Long.valueOf(split.get(3)); + return new SubApplicationRowKey(split.get(0), split.get(1), + split.get(2), entityIdPrefix, split.get(4), split.get(5)); + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationRowKeyPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationRowKeyPrefix.java new file mode 100644 index 00000000000..0c049596583 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationRowKeyPrefix.java @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.timelineservice.storage.subapplication; + +import org.apache.hadoop.yarn.server.timelineservice.storage.common.RowKeyPrefix; + +/** + * Represents a partial rowkey without the entityId or without entityType and + * entityId for the sub application table. + * + */ +public class SubApplicationRowKeyPrefix extends SubApplicationRowKey + implements RowKeyPrefix { + + /** + * Creates a prefix which generates the following rowKeyPrefixes for the sub + * application table: + * {@code subAppUserId!clusterId!entityType!entityPrefix!userId}. + * + * @param subAppUserId + * identifying the subApp User + * @param clusterId + * identifying the cluster + * @param entityType + * which entity type + * @param entityIdPrefix + * for entityId + * @param entityId + * for an entity + * @param userId + * for the user who runs the AM + * + * subAppUserId is usually the doAsUser. + * userId is the yarn user that the AM runs as. + * + */ + public SubApplicationRowKeyPrefix(String subAppUserId, String clusterId, + String entityType, Long entityIdPrefix, String entityId, + String userId) { + super(subAppUserId, clusterId, entityType, entityIdPrefix, entityId, + userId); + } + + /* + * (non-Javadoc) + * + * @see org.apache.hadoop.yarn.server.timelineservice.storage.subapplication. + * RowKeyPrefix#getRowKeyPrefix() + */ + public byte[] getRowKeyPrefix() { + return super.getRowKey(); + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationTable.java new file mode 100644 index 00000000000..334bab65f1e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationTable.java @@ -0,0 +1,174 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.timelineservice.storage.subapplication; + +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.regionserver.BloomType; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable; +import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineHBaseSchemaConstants; + +/** + * The sub application table has column families: + * info, config and metrics. + * Info stores information about a timeline entity object + * config stores configuration data of a timeline entity object + * metrics stores the metrics of a timeline entity object + * + * Example sub application table record: + * + * 

+ * |-------------------------------------------------------------------------|
+ * |  Row          | Column Family             | Column Family| Column Family|
+ * |  key          | info                      | metrics      | config       |
+ * |-------------------------------------------------------------------------|
+ * | subAppUserId! | id:entityId               | metricId1:   | configKey1:  |
+ * | clusterId!    | type:entityType           | metricValue1 | configValue1 |
+ * | entityType!   |                           | @timestamp1  |              |
+ * | idPrefix!|    |                           |              | configKey2:  |
+ * | entityId!     | created_time:             | metricId1:   | configValue2 |
+ * | userId        | 1392993084018             | metricValue2 |              |
+ * |               |                           | @timestamp2  |              |
+ * |               | i!infoKey:                |              |              |
+ * |               | infoValue                 | metricId1:   |              |
+ * |               |                           | metricValue1 |              |
+ * |               |                           | @timestamp2  |              |
+ * |               | e!eventId=timestamp=      |              |              |
+ * |               | infoKey:                  |              |              |
+ * |               | eventInfoValue            |              |              |
+ * |               |                           |              |              |
+ * |               | r!relatesToKey:           |              |              |
+ * |               | id3=id4=id5               |              |              |
+ * |               |                           |              |              |
+ * |               | s!isRelatedToKey          |              |              |
+ * |               | id7=id9=id6               |              |              |
+ * |               |                           |              |              |
+ * |               | flowVersion:              |              |              |
+ * |               | versionValue              |              |              |
+ * |-------------------------------------------------------------------------|
+ *
+ */ +public class SubApplicationTable extends BaseTable { + /** sub app prefix. */ + private static final String PREFIX = + YarnConfiguration.TIMELINE_SERVICE_PREFIX + "subapplication"; + + /** config param name that specifies the subapplication table name. */ + public static final String TABLE_NAME_CONF_NAME = PREFIX + ".table.name"; + + /** + * config param name that specifies the TTL for metrics column family in + * subapplication table. + */ + private static final String METRICS_TTL_CONF_NAME = PREFIX + + ".table.metrics.ttl"; + + /** + * config param name that specifies max-versions for + * metrics column family in subapplication table. + */ + private static final String METRICS_MAX_VERSIONS = + PREFIX + ".table.metrics.max-versions"; + + /** default value for subapplication table name. */ + public static final String DEFAULT_TABLE_NAME = + "timelineservice.subapplication"; + + /** default TTL is 30 days for metrics timeseries. */ + private static final int DEFAULT_METRICS_TTL = 2592000; + + /** default max number of versions. */ + private static final int DEFAULT_METRICS_MAX_VERSIONS = 10000; + + private static final Log LOG = LogFactory.getLog( + SubApplicationTable.class); + + public SubApplicationTable() { + super(TABLE_NAME_CONF_NAME, DEFAULT_TABLE_NAME); + } + + /* + * (non-Javadoc) + * + * @see + * org.apache.hadoop.yarn.server.timelineservice.storage.BaseTable#createTable + * (org.apache.hadoop.hbase.client.Admin, + * org.apache.hadoop.conf.Configuration) + */ + public void createTable(Admin admin, Configuration hbaseConf) + throws IOException { + + TableName table = getTableName(hbaseConf); + if (admin.tableExists(table)) { + // do not disable / delete existing table + // similar to the approach taken by map-reduce jobs when + // output directory exists + throw new IOException("Table " + table.getNameAsString() + + " already exists."); + } + + HTableDescriptor subAppTableDescp = new HTableDescriptor(table); + HColumnDescriptor infoCF = + new HColumnDescriptor(SubApplicationColumnFamily.INFO.getBytes()); + infoCF.setBloomFilterType(BloomType.ROWCOL); + subAppTableDescp.addFamily(infoCF); + + HColumnDescriptor configCF = + new HColumnDescriptor(SubApplicationColumnFamily.CONFIGS.getBytes()); + configCF.setBloomFilterType(BloomType.ROWCOL); + configCF.setBlockCacheEnabled(true); + subAppTableDescp.addFamily(configCF); + + HColumnDescriptor metricsCF = + new HColumnDescriptor(SubApplicationColumnFamily.METRICS.getBytes()); + subAppTableDescp.addFamily(metricsCF); + metricsCF.setBlockCacheEnabled(true); + // always keep 1 version (the latest) + metricsCF.setMinVersions(1); + metricsCF.setMaxVersions( + hbaseConf.getInt(METRICS_MAX_VERSIONS, DEFAULT_METRICS_MAX_VERSIONS)); + metricsCF.setTimeToLive(hbaseConf.getInt(METRICS_TTL_CONF_NAME, + DEFAULT_METRICS_TTL)); + subAppTableDescp.setRegionSplitPolicyClassName( + "org.apache.hadoop.hbase.regionserver.KeyPrefixRegionSplitPolicy"); + subAppTableDescp.setValue("KeyPrefixRegionSplitPolicy.prefix_length", + TimelineHBaseSchemaConstants.USERNAME_SPLIT_KEY_PREFIX_LENGTH); + admin.createTable(subAppTableDescp, + TimelineHBaseSchemaConstants.getUsernameSplits()); + LOG.info("Status of table creation for " + table.getNameAsString() + "=" + + admin.tableExists(table)); + } + + /** + * @param metricsTTL time to live parameter for the metricss in this table. + * @param hbaseConf configururation in which to set the metrics TTL config + * variable. + */ + public void setMetricsTTL(int metricsTTL, Configuration hbaseConf) { + hbaseConf.setInt(METRICS_TTL_CONF_NAME, metricsTTL); + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/package-info.java new file mode 100644 index 00000000000..52cc39964d5 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/package-info.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Package org.apache.hadoop.yarn.server.timelineservice.storage.subapplication + * contains classes related to implementation for subapplication table. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +package org.apache.hadoop.yarn.server.timelineservice.storage.subapplication; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestHBaseTimelineStorageUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestHBaseTimelineStorageUtils.java new file mode 100644 index 00000000000..402a89bf380 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestHBaseTimelineStorageUtils.java @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.timelineservice.storage.common; + +import org.junit.Test; + +/** + * Unit tests for HBaseTimelineStorageUtils static methos. + */ +public class TestHBaseTimelineStorageUtils { + + @Test(expected=NullPointerException.class) + public void testGetTimelineServiceHBaseConfNullArgument() throws Exception { + HBaseTimelineStorageUtils.getTimelineServiceHBaseConf(null); + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestKeyConverters.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestKeyConverters.java index 58df9708a9f..1bd363fd73a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestKeyConverters.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestKeyConverters.java @@ -26,6 +26,10 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.junit.Test; +/** + * Unit tests for key converters for various tables' row keys. + * + */ public class TestKeyConverters { @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestRowKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestRowKeys.java index 5beb18941c3..47702383ee2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestRowKeys.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestRowKeys.java @@ -22,6 +22,7 @@ import static org.junit.Assert.assertTrue; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationRowKey; import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationRowKeyPrefix; import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowRowKey; @@ -30,9 +31,14 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKey import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityRowKey; import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityRowKeyPrefix; import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunRowKey; +import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationRowKey; import org.junit.Test; +/** + * Class to test the row key structures for various tables. + * + */ public class TestRowKeys { private final static String QUALIFIER_SEP = Separator.QUALIFIERS.getValue(); @@ -40,6 +46,7 @@ public class TestRowKeys { .toBytes(QUALIFIER_SEP); private final static String CLUSTER = "cl" + QUALIFIER_SEP + "uster"; private final static String USER = QUALIFIER_SEP + "user"; + private final static String SUB_APP_USER = QUALIFIER_SEP + "subAppUser"; private final static String FLOW_NAME = "dummy_" + QUALIFIER_SEP + "flow" + QUALIFIER_SEP; private final static Long FLOW_RUN_ID; @@ -126,43 +133,46 @@ public class TestRowKeys { */ @Test public void testAppToFlowRowKey() { - byte[] byteRowKey = new AppToFlowRowKey(CLUSTER, - APPLICATION_ID).getRowKey(); + byte[] byteRowKey = new AppToFlowRowKey(APPLICATION_ID).getRowKey(); AppToFlowRowKey rowKey = AppToFlowRowKey.parseRowKey(byteRowKey); - assertEquals(CLUSTER, rowKey.getClusterId()); assertEquals(APPLICATION_ID, rowKey.getAppId()); } @Test public void testEntityRowKey() { - String entityId = "!ent!ity!!id!"; - String entityType = "entity!Type"; + TimelineEntity entity = new TimelineEntity(); + entity.setId("!ent!ity!!id!"); + entity.setType("entity!Type"); + entity.setIdPrefix(54321); + byte[] byteRowKey = new EntityRowKey(CLUSTER, USER, FLOW_NAME, FLOW_RUN_ID, APPLICATION_ID, - entityType, entityId).getRowKey(); + entity.getType(), entity.getIdPrefix(), + entity.getId()).getRowKey(); EntityRowKey rowKey = EntityRowKey.parseRowKey(byteRowKey); assertEquals(CLUSTER, rowKey.getClusterId()); assertEquals(USER, rowKey.getUserId()); assertEquals(FLOW_NAME, rowKey.getFlowName()); assertEquals(FLOW_RUN_ID, rowKey.getFlowRunId()); assertEquals(APPLICATION_ID, rowKey.getAppId()); - assertEquals(entityType, rowKey.getEntityType()); - assertEquals(entityId, rowKey.getEntityId()); + assertEquals(entity.getType(), rowKey.getEntityType()); + assertEquals(entity.getIdPrefix(), rowKey.getEntityIdPrefix().longValue()); + assertEquals(entity.getId(), rowKey.getEntityId()); byte[] byteRowKeyPrefix = new EntityRowKeyPrefix(CLUSTER, USER, FLOW_NAME, FLOW_RUN_ID, - APPLICATION_ID, entityType).getRowKeyPrefix(); + APPLICATION_ID, entity.getType(), null, null) + .getRowKeyPrefix(); byte[][] splits = Separator.QUALIFIERS.split( byteRowKeyPrefix, new int[] {Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE, Bytes.SIZEOF_LONG, AppIdKeyConverter.getKeySize(), Separator.VARIABLE_SIZE, - Separator.VARIABLE_SIZE}); + Bytes.SIZEOF_LONG, Separator.VARIABLE_SIZE }); assertEquals(7, splits.length); - assertEquals(0, splits[6].length); assertEquals(APPLICATION_ID, new AppIdKeyConverter().decode(splits[4])); - assertEquals(entityType, + assertEquals(entity.getType(), Separator.QUALIFIERS.decode(Bytes.toString(splits[5]))); verifyRowPrefixBytes(byteRowKeyPrefix); @@ -243,4 +253,24 @@ public class TestRowKeys { verifyRowPrefixBytes(byteRowKeyPrefix); } + @Test + public void testSubAppRowKey() { + TimelineEntity entity = new TimelineEntity(); + entity.setId("entity1"); + entity.setType("DAG"); + entity.setIdPrefix(54321); + + byte[] byteRowKey = + new SubApplicationRowKey(SUB_APP_USER, CLUSTER, + entity.getType(), entity.getIdPrefix(), + entity.getId(), USER).getRowKey(); + SubApplicationRowKey rowKey = SubApplicationRowKey.parseRowKey(byteRowKey); + assertEquals(CLUSTER, rowKey.getClusterId()); + assertEquals(SUB_APP_USER, rowKey.getSubAppUserId()); + assertEquals(entity.getType(), rowKey.getEntityType()); + assertEquals(entity.getIdPrefix(), rowKey.getEntityIdPrefix().longValue()); + assertEquals(entity.getId(), rowKey.getEntityId()); + assertEquals(USER, rowKey.getUserId()); + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestRowKeysAsString.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestRowKeysAsString.java new file mode 100644 index 00000000000..148cf567c45 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestRowKeysAsString.java @@ -0,0 +1,144 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.timelineservice.storage.common; + +import static org.junit.Assert.assertEquals; + +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; +import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderUtils; +import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationRowKey; +import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKey; +import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityRowKey; +import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunRowKey; +import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationRowKey; +import org.junit.Test; + +/** + * Test for row key as string. + */ +public class TestRowKeysAsString { + + private final static String CLUSTER = + "cl" + TimelineReaderUtils.DEFAULT_DELIMITER_CHAR + "uster" + + TimelineReaderUtils.DEFAULT_ESCAPE_CHAR; + private final static String USER = + TimelineReaderUtils.DEFAULT_ESCAPE_CHAR + "user"; + private final static String SUB_APP_USER = + TimelineReaderUtils.DEFAULT_ESCAPE_CHAR + "subAppUser"; + + private final static String FLOW_NAME = + "dummy_" + TimelineReaderUtils.DEFAULT_DELIMITER_CHAR + + TimelineReaderUtils.DEFAULT_ESCAPE_CHAR + "flow" + + TimelineReaderUtils.DEFAULT_DELIMITER_CHAR; + private final static Long FLOW_RUN_ID = System.currentTimeMillis(); + private final static String APPLICATION_ID = + ApplicationId.newInstance(System.currentTimeMillis(), 1).toString(); + + @Test(timeout = 10000) + public void testApplicationRow() { + String rowKeyAsString = new ApplicationRowKey(CLUSTER, USER, FLOW_NAME, + FLOW_RUN_ID, APPLICATION_ID).getRowKeyAsString(); + ApplicationRowKey rowKey = + ApplicationRowKey.parseRowKeyFromString(rowKeyAsString); + assertEquals(CLUSTER, rowKey.getClusterId()); + assertEquals(USER, rowKey.getUserId()); + assertEquals(FLOW_NAME, rowKey.getFlowName()); + assertEquals(FLOW_RUN_ID, rowKey.getFlowRunId()); + assertEquals(APPLICATION_ID, rowKey.getAppId()); + } + + @Test(timeout = 10000) + public void testEntityRowKey() { + char del = TimelineReaderUtils.DEFAULT_DELIMITER_CHAR; + char esc = TimelineReaderUtils.DEFAULT_ESCAPE_CHAR; + String id = del + esc + "ent" + esc + del + "ity" + esc + del + esc + "id" + + esc + del + esc; + String type = "entity" + esc + del + esc + "Type"; + TimelineEntity entity = new TimelineEntity(); + entity.setId(id); + entity.setType(type); + entity.setIdPrefix(54321); + + String rowKeyAsString = + new EntityRowKey(CLUSTER, USER, FLOW_NAME, FLOW_RUN_ID, APPLICATION_ID, + entity.getType(), entity.getIdPrefix(), entity.getId()) + .getRowKeyAsString(); + EntityRowKey rowKey = EntityRowKey.parseRowKeyFromString(rowKeyAsString); + assertEquals(CLUSTER, rowKey.getClusterId()); + assertEquals(USER, rowKey.getUserId()); + assertEquals(FLOW_NAME, rowKey.getFlowName()); + assertEquals(FLOW_RUN_ID, rowKey.getFlowRunId()); + assertEquals(APPLICATION_ID, rowKey.getAppId()); + assertEquals(entity.getType(), rowKey.getEntityType()); + assertEquals(entity.getIdPrefix(), rowKey.getEntityIdPrefix().longValue()); + assertEquals(entity.getId(), rowKey.getEntityId()); + + } + + @Test(timeout = 10000) + public void testFlowActivityRowKey() { + Long ts = 1459900830000L; + Long dayTimestamp = HBaseTimelineStorageUtils.getTopOfTheDayTimestamp(ts); + String rowKeyAsString = new FlowActivityRowKey(CLUSTER, ts, USER, FLOW_NAME) + .getRowKeyAsString(); + FlowActivityRowKey rowKey = + FlowActivityRowKey.parseRowKeyFromString(rowKeyAsString); + assertEquals(CLUSTER, rowKey.getClusterId()); + assertEquals(dayTimestamp, rowKey.getDayTimestamp()); + assertEquals(USER, rowKey.getUserId()); + assertEquals(FLOW_NAME, rowKey.getFlowName()); + } + + @Test(timeout = 10000) + public void testFlowRunRowKey() { + String rowKeyAsString = + new FlowRunRowKey(CLUSTER, USER, FLOW_NAME, FLOW_RUN_ID) + .getRowKeyAsString(); + FlowRunRowKey rowKey = FlowRunRowKey.parseRowKeyFromString(rowKeyAsString); + assertEquals(CLUSTER, rowKey.getClusterId()); + assertEquals(USER, rowKey.getUserId()); + assertEquals(FLOW_NAME, rowKey.getFlowName()); + assertEquals(FLOW_RUN_ID, rowKey.getFlowRunId()); + } + + @Test(timeout = 10000) + public void testSubApplicationRowKey() { + char del = TimelineReaderUtils.DEFAULT_DELIMITER_CHAR; + char esc = TimelineReaderUtils.DEFAULT_ESCAPE_CHAR; + String id = del + esc + "ent" + esc + del + "ity" + esc + del + esc + "id" + + esc + del + esc; + String type = "entity" + esc + del + esc + "Type"; + TimelineEntity entity = new TimelineEntity(); + entity.setId(id); + entity.setType(type); + entity.setIdPrefix(54321); + + String rowKeyAsString = new SubApplicationRowKey(SUB_APP_USER, CLUSTER, + entity.getType(), entity.getIdPrefix(), entity.getId(), USER) + .getRowKeyAsString(); + SubApplicationRowKey rowKey = SubApplicationRowKey + .parseRowKeyFromString(rowKeyAsString); + assertEquals(SUB_APP_USER, rowKey.getSubAppUserId()); + assertEquals(CLUSTER, rowKey.getClusterId()); + assertEquals(entity.getType(), rowKey.getEntityType()); + assertEquals(entity.getIdPrefix(), rowKey.getEntityIdPrefix().longValue()); + assertEquals(entity.getId(), rowKey.getEntityId()); + assertEquals(USER, rowKey.getUserId()); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollector.java index 56f7b2b8c97..38221fe98a8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollector.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollector.java @@ -18,27 +18,22 @@ package org.apache.hadoop.yarn.server.timelineservice.collector; -import com.google.common.util.concurrent.ThreadFactoryBuilder; +import java.util.concurrent.Future; + import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.Token; import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities; -import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; -import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; - /** * Service that handles writes to the timeline service and writes them to the * backing storage for a given YARN application. @@ -51,30 +46,68 @@ public class AppLevelTimelineCollector extends TimelineCollector { private static final Logger LOG = LoggerFactory.getLogger(TimelineCollector.class); - private final static int AGGREGATION_EXECUTOR_NUM_THREADS = 1; - private final static int AGGREGATION_EXECUTOR_EXEC_INTERVAL_SECS = 15; - private static Set entityTypesSkipAggregation - = initializeSkipSet(); - private final ApplicationId appId; + private final String appUser; private final TimelineCollectorContext context; - private ScheduledThreadPoolExecutor appAggregationExecutor; - private AppLevelAggregator appAggregator; private UserGroupInformation currentUser; + private Token delegationTokenForApp; + private long tokenMaxDate = 0; + private String tokenRenewer; + private Future renewalOrRegenerationFuture; public AppLevelTimelineCollector(ApplicationId appId) { + this(appId, null); + } + + public AppLevelTimelineCollector(ApplicationId appId, String user) { super(AppLevelTimelineCollector.class.getName() + " - " + appId.toString()); Preconditions.checkNotNull(appId, "AppId shouldn't be null"); this.appId = appId; + this.appUser = user; context = new TimelineCollectorContext(); } - private static Set initializeSkipSet() { - Set result = new HashSet<>(); - result.add(TimelineEntityType.YARN_APPLICATION.toString()); - result.add(TimelineEntityType.YARN_FLOW_RUN.toString()); - result.add(TimelineEntityType.YARN_FLOW_ACTIVITY.toString()); - return result; + public UserGroupInformation getCurrentUser() { + return currentUser; + } + + public String getAppUser() { + return appUser; + } + + void setDelegationTokenAndFutureForApp( + Token token, + Future appRenewalOrRegenerationFuture, long tknMaxDate, + String renewer) { + this.delegationTokenForApp = token; + this.tokenMaxDate = tknMaxDate; + this.tokenRenewer = renewer; + this.renewalOrRegenerationFuture = appRenewalOrRegenerationFuture; + } + + void setRenewalOrRegenerationFutureForApp( + Future appRenewalOrRegenerationFuture) { + this.renewalOrRegenerationFuture = appRenewalOrRegenerationFuture; + } + + void cancelRenewalOrRegenerationFutureForApp() { + if (renewalOrRegenerationFuture != null && + !renewalOrRegenerationFuture.isDone()) { + renewalOrRegenerationFuture.cancel(true); + } + } + + long getAppDelegationTokenMaxDate() { + return tokenMaxDate; + } + + String getAppDelegationTokenRenewer() { + return tokenRenewer; + } + + @VisibleForTesting + public Token getDelegationTokenForApp() { + return this.delegationTokenForApp; } @Override @@ -92,29 +125,12 @@ public class AppLevelTimelineCollector extends TimelineCollector { @Override protected void serviceStart() throws Exception { - // Launch the aggregation thread - appAggregationExecutor = new ScheduledThreadPoolExecutor( - AppLevelTimelineCollector.AGGREGATION_EXECUTOR_NUM_THREADS, - new ThreadFactoryBuilder() - .setNameFormat("TimelineCollector Aggregation thread #%d") - .build()); - appAggregator = new AppLevelAggregator(); - appAggregationExecutor.scheduleAtFixedRate(appAggregator, - AppLevelTimelineCollector.AGGREGATION_EXECUTOR_EXEC_INTERVAL_SECS, - AppLevelTimelineCollector.AGGREGATION_EXECUTOR_EXEC_INTERVAL_SECS, - TimeUnit.SECONDS); super.serviceStart(); } @Override protected void serviceStop() throws Exception { - appAggregationExecutor.shutdown(); - if (!appAggregationExecutor.awaitTermination(10, TimeUnit.SECONDS)) { - LOG.info("App-level aggregator shutdown timed out, shutdown now. "); - appAggregationExecutor.shutdownNow(); - } - // Perform one round of aggregation after the aggregation executor is done. - appAggregator.aggregate(); + cancelRenewalOrRegenerationFutureForApp(); super.serviceStop(); } @@ -122,49 +138,4 @@ public class AppLevelTimelineCollector extends TimelineCollector { public TimelineCollectorContext getTimelineEntityContext() { return context; } - - @Override - protected Set getEntityTypesSkipAggregation() { - return entityTypesSkipAggregation; - } - - private class AppLevelAggregator implements Runnable { - - private void aggregate() { - if (LOG.isDebugEnabled()) { - LOG.debug("App-level real-time aggregating"); - } - if (!isReadyToAggregate()) { - LOG.warn("App-level collector is not ready, skip aggregation. "); - return; - } - try { - TimelineCollectorContext currContext = getTimelineEntityContext(); - Map aggregationGroups - = getAggregationGroups(); - if (aggregationGroups == null - || aggregationGroups.isEmpty()) { - LOG.debug("App-level collector is empty, skip aggregation. "); - return; - } - TimelineEntity resultEntity = TimelineCollector.aggregateWithoutGroupId( - aggregationGroups, currContext.getAppId(), - TimelineEntityType.YARN_APPLICATION.toString()); - TimelineEntities entities = new TimelineEntities(); - entities.addEntity(resultEntity); - putEntitiesAsync(entities, currentUser); - } catch (Exception e) { - LOG.error("Error aggregating timeline metrics", e); - } - if (LOG.isDebugEnabled()) { - LOG.debug("App-level real-time aggregation complete"); - } - } - - @Override - public void run() { - aggregate(); - } - } - } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollectorWithAgg.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollectorWithAgg.java new file mode 100644 index 00000000000..6c0d693f008 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollectorWithAgg.java @@ -0,0 +1,150 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.timelineservice.collector; + +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities; +import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; +import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType; +import org.apache.hadoop.conf.Configuration; + +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +/** + * Service that handles aggregations for applications + * and makes use of {@link AppLevelTimelineCollector} class for + * writes to Timeline Service. + * + * App-related lifecycle management is handled by this service. + */ +@Private +@Unstable +public class AppLevelTimelineCollectorWithAgg + extends AppLevelTimelineCollector { + private static final Log LOG = LogFactory.getLog(TimelineCollector.class); + + private final static int AGGREGATION_EXECUTOR_NUM_THREADS = 1; + private final static int AGGREGATION_EXECUTOR_EXEC_INTERVAL_SECS = 15; + private static Set entityTypesSkipAggregation + = initializeSkipSet(); + + private ScheduledThreadPoolExecutor appAggregationExecutor; + private AppLevelAggregator appAggregator; + + public AppLevelTimelineCollectorWithAgg(ApplicationId appId, String user) { + super(appId, user); + } + + private static Set initializeSkipSet() { + Set result = new HashSet<>(); + result.add(TimelineEntityType.YARN_APPLICATION.toString()); + result.add(TimelineEntityType.YARN_FLOW_RUN.toString()); + result.add(TimelineEntityType.YARN_FLOW_ACTIVITY.toString()); + return result; + } + + @Override + protected void serviceInit(Configuration conf) throws Exception { + super.serviceInit(conf); + } + + @Override + protected void serviceStart() throws Exception { + // Launch the aggregation thread + appAggregationExecutor = new ScheduledThreadPoolExecutor( + AppLevelTimelineCollectorWithAgg.AGGREGATION_EXECUTOR_NUM_THREADS, + new ThreadFactoryBuilder() + .setNameFormat("TimelineCollector Aggregation thread #%d") + .build()); + appAggregator = new AppLevelAggregator(); + appAggregationExecutor.scheduleAtFixedRate(appAggregator, + AppLevelTimelineCollectorWithAgg. + AGGREGATION_EXECUTOR_EXEC_INTERVAL_SECS, + AppLevelTimelineCollectorWithAgg. + AGGREGATION_EXECUTOR_EXEC_INTERVAL_SECS, + TimeUnit.SECONDS); + super.serviceStart(); + } + + @Override + protected void serviceStop() throws Exception { + appAggregationExecutor.shutdown(); + if (!appAggregationExecutor.awaitTermination(10, TimeUnit.SECONDS)) { + LOG.info("App-level aggregator shutdown timed out, shutdown now. "); + appAggregationExecutor.shutdownNow(); + } + // Perform one round of aggregation after the aggregation executor is done. + appAggregator.aggregate(); + super.serviceStop(); + } + + @Override + protected Set getEntityTypesSkipAggregation() { + return entityTypesSkipAggregation; + } + + private class AppLevelAggregator implements Runnable { + + private void aggregate() { + if (LOG.isDebugEnabled()) { + LOG.debug("App-level real-time aggregating"); + } + if (!isReadyToAggregate()) { + LOG.warn("App-level collector is not ready, skip aggregation. "); + return; + } + try { + TimelineCollectorContext currContext = getTimelineEntityContext(); + Map aggregationGroups + = getAggregationGroups(); + if (aggregationGroups == null + || aggregationGroups.isEmpty()) { + LOG.debug("App-level collector is empty, skip aggregation. "); + return; + } + TimelineEntity resultEntity = TimelineCollector.aggregateWithoutGroupId( + aggregationGroups, currContext.getAppId(), + TimelineEntityType.YARN_APPLICATION.toString()); + TimelineEntities entities = new TimelineEntities(); + entities.addEntity(resultEntity); + putEntitiesAsync(entities, getCurrentUser()); + } catch (Exception e) { + LOG.error("Error aggregating timeline metrics", e); + } + if (LOG.isDebugEnabled()) { + LOG.debug("App-level real-time aggregation complete"); + } + } + + @Override + public void run() { + aggregate(); + } + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/NodeTimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/NodeTimelineCollectorManager.java index 171978287c2..68a68f0ded5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/NodeTimelineCollectorManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/NodeTimelineCollectorManager.java @@ -18,33 +18,43 @@ package org.apache.hadoop.yarn.server.timelineservice.collector; -import static org.apache.hadoop.fs.CommonConfigurationKeys.DEFAULT_HADOOP_HTTP_STATIC_USER; -import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_HTTP_STATIC_USER; - import java.io.IOException; import java.net.InetSocketAddress; import java.net.URI; -import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.Set; +import java.util.concurrent.Future; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.http.HttpServer2; -import org.apache.hadoop.http.lib.StaticUserWebFilter; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.util.Time; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.ipc.YarnRPC; +import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier; import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocol; import org.apache.hadoop.yarn.server.api.protocolrecords.GetTimelineCollectorContextRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.GetTimelineCollectorContextResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewCollectorInfoRequest; +import org.apache.hadoop.yarn.server.timelineservice.security.TimelineV2DelegationTokenSecretManagerService; +import org.apache.hadoop.yarn.server.util.timeline.TimelineServerUtils; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import com.google.common.annotations.VisibleForTesting; +import com.google.common.util.concurrent.ThreadFactoryBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -65,17 +75,82 @@ public class NodeTimelineCollectorManager extends TimelineCollectorManager { private volatile CollectorNodemanagerProtocol nmCollectorService; + private TimelineV2DelegationTokenSecretManagerService tokenMgrService; + + private final boolean runningAsAuxService; + + private UserGroupInformation loginUGI; + + private ScheduledThreadPoolExecutor tokenRenewalExecutor; + + private long tokenRenewInterval; + + private static final long TIME_BEFORE_RENEW_DATE = 10 * 1000; // 10 seconds. + + private static final long TIME_BEFORE_EXPIRY = 5 * 60 * 1000; // 5 minutes. + static final String COLLECTOR_MANAGER_ATTR_KEY = "collector.manager"; @VisibleForTesting protected NodeTimelineCollectorManager() { + this(true); + } + + protected NodeTimelineCollectorManager(boolean asAuxService) { super(NodeTimelineCollectorManager.class.getName()); + this.runningAsAuxService = asAuxService; + } + + @Override + protected void serviceInit(Configuration conf) throws Exception { + tokenMgrService = createTokenManagerService(); + addService(tokenMgrService); + this.loginUGI = UserGroupInformation.getCurrentUser(); + tokenRenewInterval = conf.getLong( + YarnConfiguration.TIMELINE_DELEGATION_TOKEN_RENEW_INTERVAL, + YarnConfiguration.DEFAULT_TIMELINE_DELEGATION_TOKEN_RENEW_INTERVAL); + super.serviceInit(conf); } @Override protected void serviceStart() throws Exception { - startWebApp(); + if (UserGroupInformation.isSecurityEnabled()) { + // Do security login for cases where collector is running outside NM. + if (!runningAsAuxService) { + try { + doSecureLogin(); + } catch(IOException ie) { + throw new YarnRuntimeException("Failed to login", ie); + } + } + this.loginUGI = UserGroupInformation.getLoginUser(); + } + tokenRenewalExecutor = new ScheduledThreadPoolExecutor( + 1, new ThreadFactoryBuilder().setNameFormat( + "App Collector Token Renewal thread").build()); super.serviceStart(); + startWebApp(); + } + + protected TimelineV2DelegationTokenSecretManagerService + createTokenManagerService() { + return new TimelineV2DelegationTokenSecretManagerService(); + } + + @VisibleForTesting + public TimelineV2DelegationTokenSecretManagerService + getTokenManagerService() { + return tokenMgrService; + } + + private void doSecureLogin() throws IOException { + Configuration conf = getConfig(); + InetSocketAddress addr = NetUtils.createSocketAddr(conf.getTrimmed( + YarnConfiguration.TIMELINE_SERVICE_BIND_HOST, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_BIND_HOST), 0, + YarnConfiguration.TIMELINE_SERVICE_BIND_HOST); + SecurityUtil.login(conf, YarnConfiguration.TIMELINE_SERVICE_KEYTAB, + YarnConfiguration.TIMELINE_SERVICE_PRINCIPAL, addr.getHostName()); } @Override @@ -83,16 +158,95 @@ public class NodeTimelineCollectorManager extends TimelineCollectorManager { if (timelineRestServer != null) { timelineRestServer.stop(); } + if (tokenRenewalExecutor != null) { + tokenRenewalExecutor.shutdownNow(); + } super.serviceStop(); } + @VisibleForTesting + public Token generateTokenForAppCollector( + String user) { + Token token = tokenMgrService. + generateToken(UserGroupInformation.createRemoteUser(user), + loginUGI.getShortUserName()); + token.setService(new Text(timelineRestServerBindAddress)); + return token; + } + + @VisibleForTesting + public long renewTokenForAppCollector( + AppLevelTimelineCollector appCollector) throws IOException { + if (appCollector.getDelegationTokenForApp() != null) { + return tokenMgrService.renewToken(appCollector.getDelegationTokenForApp(), + appCollector.getAppDelegationTokenRenewer()); + } else { + LOG.info("Delegation token not available for renewal for app " + + appCollector.getTimelineEntityContext().getAppId()); + return -1; + } + } + + @VisibleForTesting + public void cancelTokenForAppCollector( + AppLevelTimelineCollector appCollector) throws IOException { + if (appCollector.getDelegationTokenForApp() != null) { + tokenMgrService.cancelToken(appCollector.getDelegationTokenForApp(), + appCollector.getAppUser()); + } + } + + private long getRenewalDelay(long renewInterval) { + return ((renewInterval > TIME_BEFORE_RENEW_DATE) ? + renewInterval - TIME_BEFORE_RENEW_DATE : renewInterval); + } + + private long getRegenerationDelay(long tokenMaxDate) { + long regenerateTime = tokenMaxDate - Time.now(); + return ((regenerateTime > TIME_BEFORE_EXPIRY) ? + regenerateTime - TIME_BEFORE_EXPIRY : regenerateTime); + } + + private org.apache.hadoop.yarn.api.records.Token generateTokenAndSetTimer( + ApplicationId appId, AppLevelTimelineCollector appCollector) + throws IOException { + Token timelineToken = + generateTokenForAppCollector(appCollector.getAppUser()); + TimelineDelegationTokenIdentifier tokenId = + timelineToken.decodeIdentifier(); + long renewalDelay = getRenewalDelay(tokenRenewInterval); + long regenerationDelay = getRegenerationDelay(tokenId.getMaxDate()); + if (renewalDelay > 0 || regenerationDelay > 0) { + boolean isTimerForRenewal = renewalDelay < regenerationDelay; + Future renewalOrRegenerationFuture = tokenRenewalExecutor.schedule( + new CollectorTokenRenewer(appId, isTimerForRenewal), + isTimerForRenewal? renewalDelay : regenerationDelay, + TimeUnit.MILLISECONDS); + appCollector.setDelegationTokenAndFutureForApp(timelineToken, + renewalOrRegenerationFuture, tokenId.getMaxDate(), + tokenId.getRenewer().toString()); + } + LOG.info("Generated a new token " + timelineToken + " for app " + appId); + return org.apache.hadoop.yarn.api.records.Token.newInstance( + timelineToken.getIdentifier(), timelineToken.getKind().toString(), + timelineToken.getPassword(), timelineToken.getService().toString()); + } + @Override protected void doPostPut(ApplicationId appId, TimelineCollector collector) { try { // Get context info from NM updateTimelineCollectorContext(appId, collector); + // Generate token for app collector. + org.apache.hadoop.yarn.api.records.Token token = null; + if (UserGroupInformation.isSecurityEnabled() && + collector instanceof AppLevelTimelineCollector) { + AppLevelTimelineCollector appCollector = + (AppLevelTimelineCollector) collector; + token = generateTokenAndSetTimer(appId, appCollector); + } // Report to NM if a new collector is added. - reportNewCollectorToNM(appId); + reportNewCollectorInfoToNM(appId, token); } catch (YarnException | IOException e) { // throw exception here as it cannot be used if failed communicate with NM LOG.error("Failed to communicate with NM Collector Service for " + appId); @@ -100,11 +254,29 @@ public class NodeTimelineCollectorManager extends TimelineCollectorManager { } } + @Override + protected void postRemove(ApplicationId appId, TimelineCollector collector) { + if (collector instanceof AppLevelTimelineCollector) { + try { + cancelTokenForAppCollector((AppLevelTimelineCollector) collector); + } catch (IOException e) { + LOG.warn("Failed to cancel token for app collector with appId " + + appId, e); + } + } + } + /** * Launch the REST web server for this collector manager. */ private void startWebApp() { Configuration conf = getConfig(); + String initializers = conf.get("hadoop.http.filter.initializers", ""); + Set defaultInitializers = new LinkedHashSet(); + TimelineServerUtils.addTimelineAuthFilter( + initializers, defaultInitializers, tokenMgrService); + TimelineServerUtils.setTimelineFilters( + conf, initializers, defaultInitializers); String bindAddress = conf.get(YarnConfiguration.TIMELINE_SERVICE_BIND_HOST, YarnConfiguration.DEFAULT_TIMELINE_SERVICE_BIND_HOST) + ":0"; try { @@ -114,16 +286,10 @@ public class NodeTimelineCollectorManager extends TimelineCollectorManager { .addEndpoint(URI.create( (YarnConfiguration.useHttps(conf) ? "https://" : "http://") + bindAddress)); + if (YarnConfiguration.useHttps(conf)) { + builder = WebAppUtils.loadSslConfiguration(builder, conf); + } timelineRestServer = builder.build(); - // TODO: replace this by an authentication filter in future. - HashMap options = new HashMap<>(); - String username = conf.get(HADOOP_HTTP_STATIC_USER, - DEFAULT_HADOOP_HTTP_STATIC_USER); - options.put(HADOOP_HTTP_STATIC_USER, username); - HttpServer2.defineFilter(timelineRestServer.getWebAppContext(), - "static_user_filter_timeline", - StaticUserWebFilter.StaticUserFilter.class.getName(), - options, new String[] {"/*"}); timelineRestServer.addJerseyResourcePackage( TimelineCollectorWebService.class.getPackage().getName() + ";" @@ -144,11 +310,12 @@ public class NodeTimelineCollectorManager extends TimelineCollectorManager { timelineRestServerBindAddress); } - private void reportNewCollectorToNM(ApplicationId appId) + private void reportNewCollectorInfoToNM(ApplicationId appId, + org.apache.hadoop.yarn.api.records.Token token) throws YarnException, IOException { ReportNewCollectorInfoRequest request = ReportNewCollectorInfoRequest.newInstance(appId, - this.timelineRestServerBindAddress); + this.timelineRestServerBindAddress, token); LOG.info("Report a new collector for application: " + appId + " to the NM Collector Service."); getNMCollectorService().reportNewCollectorInfo(request); @@ -220,4 +387,76 @@ public class NodeTimelineCollectorManager extends TimelineCollectorManager { public String getRestServerBindAddress() { return timelineRestServerBindAddress; } + + private final class CollectorTokenRenewer implements Runnable { + private ApplicationId appId; + // Indicates whether timer is for renewal or regeneration of token. + private boolean timerForRenewal = true; + private CollectorTokenRenewer(ApplicationId applicationId, + boolean forRenewal) { + appId = applicationId; + timerForRenewal = forRenewal; + } + + private void renewToken(AppLevelTimelineCollector appCollector) + throws IOException { + long newExpirationTime = renewTokenForAppCollector(appCollector); + // Set renewal or regeneration timer based on delay. + long renewalDelay = 0; + if (newExpirationTime > 0) { + LOG.info("Renewed token for " + appId + " with new expiration " + + "timestamp = " + newExpirationTime); + renewalDelay = getRenewalDelay(newExpirationTime - Time.now()); + } + long regenerationDelay = + getRegenerationDelay(appCollector.getAppDelegationTokenMaxDate()); + if (renewalDelay > 0 || regenerationDelay > 0) { + this.timerForRenewal = renewalDelay < regenerationDelay; + Future renewalOrRegenerationFuture = tokenRenewalExecutor.schedule( + this, timerForRenewal ? renewalDelay : regenerationDelay, + TimeUnit.MILLISECONDS); + appCollector.setRenewalOrRegenerationFutureForApp( + renewalOrRegenerationFuture); + } + } + + private void regenerateToken(AppLevelTimelineCollector appCollector) + throws IOException { + org.apache.hadoop.yarn.api.records.Token token = + generateTokenAndSetTimer(appId, appCollector); + // Report to NM if a new collector is added. + try { + reportNewCollectorInfoToNM(appId, token); + } catch (YarnException e) { + LOG.warn("Unable to report regenerated token to NM for " + appId); + } + } + + @Override + public void run() { + TimelineCollector collector = get(appId); + if (collector == null) { + LOG.info("Cannot find active collector while " + (timerForRenewal ? + "renewing" : "regenerating") + " token for " + appId); + return; + } + AppLevelTimelineCollector appCollector = + (AppLevelTimelineCollector) collector; + + synchronized (collector) { + if (!collector.isStopped()) { + try { + if (timerForRenewal) { + renewToken(appCollector); + } else { + regenerateToken(appCollector); + } + } catch (Exception e) { + LOG.warn("Unable to " + (timerForRenewal ? "renew" : "regenerate") + + " token for " + appId, e); + } + } + } + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java index e4e6421d108..66f9aab034b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java @@ -61,7 +61,7 @@ public class PerNodeTimelineCollectorsAuxService extends AuxiliaryService { private ScheduledExecutorService scheduler; public PerNodeTimelineCollectorsAuxService() { - this(new NodeTimelineCollectorManager()); + this(new NodeTimelineCollectorManager(true)); } @VisibleForTesting PerNodeTimelineCollectorsAuxService( @@ -114,11 +114,12 @@ public class PerNodeTimelineCollectorsAuxService extends AuxiliaryService { * exists, no new service is created. * * @param appId Application Id to be added. + * @param user Application Master container user. * @return whether it was added successfully */ - public boolean addApplication(ApplicationId appId) { + public boolean addApplication(ApplicationId appId, String user) { AppLevelTimelineCollector collector = - new AppLevelTimelineCollector(appId); + new AppLevelTimelineCollectorWithAgg(appId, user); return (collectorManager.putIfAbsent(appId, collector) == collector); } @@ -147,7 +148,7 @@ public class PerNodeTimelineCollectorsAuxService extends AuxiliaryService { if (context.getContainerType() == ContainerType.APPLICATION_MASTER) { ApplicationId appId = context.getContainerId(). getApplicationAttemptId().getApplicationId(); - addApplication(appId); + addApplication(appId, context.getUser()); } } @@ -202,7 +203,8 @@ public class PerNodeTimelineCollectorsAuxService extends AuxiliaryService { PerNodeTimelineCollectorsAuxService auxService = null; try { auxService = collectorManager == null ? - new PerNodeTimelineCollectorsAuxService() : + new PerNodeTimelineCollectorsAuxService( + new NodeTimelineCollectorManager(false)) : new PerNodeTimelineCollectorsAuxService(collectorManager); ShutdownHookManager.get().addShutdownHook(new ShutdownHook(auxService), SHUTDOWN_HOOK_PRIORITY); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java index 37387f1f767..8202431459d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java @@ -63,6 +63,8 @@ public abstract class TimelineCollector extends CompositeService { private volatile boolean readyToAggregate = false; + private volatile boolean isStopped = false; + public TimelineCollector(String name) { super(name); } @@ -79,9 +81,14 @@ public abstract class TimelineCollector extends CompositeService { @Override protected void serviceStop() throws Exception { + isStopped = true; super.serviceStop(); } + boolean isStopped() { + return isStopped; + } + protected void setWriter(TimelineWriter w) { this.writer = w; } @@ -139,7 +146,7 @@ public abstract class TimelineCollector extends CompositeService { // flush the writer buffer concurrently and swallow any exception // caused by the timeline enitites that are being put here. synchronized (writer) { - response = writeTimelineEntities(entities); + response = writeTimelineEntities(entities, callerUgi); flushBufferedTimelineEntities(); } @@ -147,15 +154,14 @@ public abstract class TimelineCollector extends CompositeService { } private TimelineWriteResponse writeTimelineEntities( - TimelineEntities entities) throws IOException { + TimelineEntities entities, UserGroupInformation callerUgi) + throws IOException { // Update application metrics for aggregation updateAggregateStatus(entities, aggregationGroups, getEntityTypesSkipAggregation()); final TimelineCollectorContext context = getTimelineEntityContext(); - return writer.write(context.getClusterId(), context.getUserId(), - context.getFlowName(), context.getFlowVersion(), - context.getFlowRunId(), context.getAppId(), entities); + return writer.write(context, entities, callerUgi); } /** @@ -187,7 +193,7 @@ public abstract class TimelineCollector extends CompositeService { callerUgi + ")"); } - writeTimelineEntities(entities); + writeTimelineEntities(entities, callerUgi); } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java index 94b95ad3c09..7909a2e82f6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java @@ -29,7 +29,7 @@ import java.util.concurrent.TimeUnit; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.service.CompositeService; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -47,7 +47,7 @@ import org.slf4j.LoggerFactory; */ @InterfaceAudience.Private @InterfaceStability.Unstable -public class TimelineCollectorManager extends AbstractService { +public class TimelineCollectorManager extends CompositeService { private static final Logger LOG = LoggerFactory.getLogger(TimelineCollectorManager.class); @@ -57,7 +57,7 @@ public class TimelineCollectorManager extends AbstractService { private boolean writerFlusherRunning; @Override - public void serviceInit(Configuration conf) throws Exception { + protected void serviceInit(Configuration conf) throws Exception { writer = createTimelineWriter(conf); writer.init(conf); // create a single dedicated thread for flushing the writer on a periodic @@ -184,9 +184,11 @@ public class TimelineCollectorManager extends AbstractService { if (collector == null) { LOG.error("the collector for " + appId + " does not exist!"); } else { - postRemove(appId, collector); - // stop the service to do clean up - collector.stop(); + synchronized (collector) { + postRemove(appId, collector); + // stop the service to do clean up + collector.stop(); + } LOG.info("The collector service for " + appId + " was removed"); } return collector != null; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineDataToRetrieve.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineDataToRetrieve.java index 325050ad2a6..8d09c007eba 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineDataToRetrieve.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineDataToRetrieve.java @@ -57,6 +57,11 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Fiel * metricsToRetrieve is specified, this limit defines an upper limit to the * number of metrics to return. This parameter is ignored if METRICS are not to * be fetched. + *
  • metricsTimeStart - Metric values before this timestamp would not + * be retrieved. If null or {@literal <0}, defaults to 0.
    • + *
  • metricsTimeEnd - Metric values after this timestamp would not + * be retrieved. If null or {@literal <0}, defaults to {@link Long#MAX_VALUE}. + *
    • * */ @Private @@ -66,6 +71,10 @@ public class TimelineDataToRetrieve { private TimelineFilterList metricsToRetrieve; private EnumSet fieldsToRetrieve; private Integer metricsLimit; + private Long metricsTimeBegin; + private Long metricsTimeEnd; + private static final long DEFAULT_METRICS_BEGIN_TIME = 0L; + private static final long DEFAULT_METRICS_END_TIME = Long.MAX_VALUE; /** * Default limit of number of metrics to return. @@ -73,12 +82,12 @@ public class TimelineDataToRetrieve { public static final Integer DEFAULT_METRICS_LIMIT = 1; public TimelineDataToRetrieve() { - this(null, null, null, null); + this(null, null, null, null, null, null); } public TimelineDataToRetrieve(TimelineFilterList confs, TimelineFilterList metrics, EnumSet fields, - Integer limitForMetrics) { + Integer limitForMetrics, Long metricTimeBegin, Long metricTimeEnd) { this.confsToRetrieve = confs; this.metricsToRetrieve = metrics; this.fieldsToRetrieve = fields; @@ -91,6 +100,20 @@ public class TimelineDataToRetrieve { if (this.fieldsToRetrieve == null) { this.fieldsToRetrieve = EnumSet.noneOf(Field.class); } + if (metricTimeBegin == null || metricTimeBegin < 0) { + this.metricsTimeBegin = DEFAULT_METRICS_BEGIN_TIME; + } else { + this.metricsTimeBegin = metricTimeBegin; + } + if (metricTimeEnd == null || metricTimeEnd < 0) { + this.metricsTimeEnd = DEFAULT_METRICS_END_TIME; + } else { + this.metricsTimeEnd = metricTimeEnd; + } + if (this.metricsTimeBegin > this.metricsTimeEnd) { + throw new IllegalArgumentException("metricstimebegin should not be " + + "greater than metricstimeend"); + } } public TimelineFilterList getConfsToRetrieve() { @@ -137,6 +160,14 @@ public class TimelineDataToRetrieve { return metricsLimit; } + public Long getMetricsTimeBegin() { + return this.metricsTimeBegin; + } + + public Long getMetricsTimeEnd() { + return metricsTimeEnd; + } + public void setMetricsLimit(Integer limit) { if (limit == null || limit < 1) { this.metricsLimit = DEFAULT_METRICS_LIMIT; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineEntityFilters.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineEntityFilters.java index 8f2b7252146..a415d3467cb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineEntityFilters.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineEntityFilters.java @@ -35,10 +35,10 @@ import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineKeyVa *
  • limit - A limit on the number of entities to return. If null or * {@literal < 0}, defaults to {@link #DEFAULT_LIMIT}. The maximum possible * value for limit can be {@link Long#MAX_VALUE}.
    • - *
  • createdTimeBegin - Matched entities should not be created - * before this timestamp. If null or {@literal <=0}, defaults to 0.
    • - *
  • createdTimeEnd - Matched entities should not be created after - * this timestamp. If null or {@literal <=0}, defaults to + *
  • createdTimeBegin - Matched entities should not be created before + * this timestamp. If null or {@literal <=0}, defaults to 0.
    • + *
  • createdTimeEnd - Matched entities should not be created after this + * timestamp. If null or {@literal <=0}, defaults to * {@link Long#MAX_VALUE}.
    • *
  • relatesTo - Matched entities should or should not relate to given * entities depending on what's specified in the filter. The entities in @@ -99,40 +99,42 @@ import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineKeyVa * filter list, event filters can be evaluated with logical AND/OR and we can * create a hierarchy of these {@link TimelineExistsFilter} objects. If null or * empty, the filter is not applied.
    • + *
  • fromId - If specified, retrieve the next set of entities from the + * given fromId. The set of entities retrieved is inclusive of specified fromId. + * fromId should be taken from the value associated with FROM_ID info key in + * entity response which was sent earlier.
    • * */ @Private @Unstable -public class TimelineEntityFilters { - private long limit; +public final class TimelineEntityFilters { + private final long limit; private long createdTimeBegin; private long createdTimeEnd; - private TimelineFilterList relatesTo; - private TimelineFilterList isRelatedTo; - private TimelineFilterList infoFilters; - private TimelineFilterList configFilters; - private TimelineFilterList metricFilters; - private TimelineFilterList eventFilters; + private final TimelineFilterList relatesTo; + private final TimelineFilterList isRelatedTo; + private final TimelineFilterList infoFilters; + private final TimelineFilterList configFilters; + private final TimelineFilterList metricFilters; + private final TimelineFilterList eventFilters; + private final String fromId; private static final long DEFAULT_BEGIN_TIME = 0L; private static final long DEFAULT_END_TIME = Long.MAX_VALUE; + /** * Default limit of number of entities to return for getEntities API. */ public static final long DEFAULT_LIMIT = 100; - public TimelineEntityFilters() { - this(null, null, null, null, null, null, null, null, null); - } - - public TimelineEntityFilters( + private TimelineEntityFilters( Long entityLimit, Long timeBegin, Long timeEnd, TimelineFilterList entityRelatesTo, TimelineFilterList entityIsRelatedTo, TimelineFilterList entityInfoFilters, TimelineFilterList entityConfigFilters, TimelineFilterList entityMetricFilters, - TimelineFilterList entityEventFilters) { + TimelineFilterList entityEventFilters, String fromId) { if (entityLimit == null || entityLimit < 0) { this.limit = DEFAULT_LIMIT; } else { @@ -154,89 +156,119 @@ public class TimelineEntityFilters { this.configFilters = entityConfigFilters; this.metricFilters = entityMetricFilters; this.eventFilters = entityEventFilters; + this.fromId = fromId; } public long getLimit() { return limit; } - public void setLimit(Long entityLimit) { - if (entityLimit == null || entityLimit < 0) { - this.limit = DEFAULT_LIMIT; - } else { - this.limit = entityLimit; - } - } - public long getCreatedTimeBegin() { return createdTimeBegin; } - public void setCreatedTimeBegin(Long timeBegin) { - if (timeBegin == null || timeBegin < 0) { - this.createdTimeBegin = DEFAULT_BEGIN_TIME; - } else { - this.createdTimeBegin = timeBegin; - } - } - public long getCreatedTimeEnd() { return createdTimeEnd; } - public void setCreatedTimeEnd(Long timeEnd) { - if (timeEnd == null || timeEnd < 0) { - this.createdTimeEnd = DEFAULT_END_TIME; - } else { - this.createdTimeEnd = timeEnd; - } - } - public TimelineFilterList getRelatesTo() { return relatesTo; } - public void setRelatesTo(TimelineFilterList relations) { - this.relatesTo = relations; - } - public TimelineFilterList getIsRelatedTo() { return isRelatedTo; } - public void setIsRelatedTo(TimelineFilterList relations) { - this.isRelatedTo = relations; - } - public TimelineFilterList getInfoFilters() { return infoFilters; } - public void setInfoFilters(TimelineFilterList filters) { - this.infoFilters = filters; - } - public TimelineFilterList getConfigFilters() { return configFilters; } - public void setConfigFilters(TimelineFilterList filters) { - this.configFilters = filters; - } - public TimelineFilterList getMetricFilters() { return metricFilters; } - public void setMetricFilters(TimelineFilterList filters) { - this.metricFilters = filters; - } - public TimelineFilterList getEventFilters() { return eventFilters; } - public void setEventFilters(TimelineFilterList filters) { - this.eventFilters = filters; + public String getFromId() { + return fromId; + } + + /** + * A builder class to build an instance of TimelineEntityFilters. + */ + public static class Builder { + private Long entityLimit; + private Long createdTimeBegin; + private Long createdTimeEnd; + private TimelineFilterList relatesToFilters; + private TimelineFilterList isRelatedToFilters; + private TimelineFilterList entityInfoFilters; + private TimelineFilterList entityConfigFilters; + private TimelineFilterList entityMetricFilters; + private TimelineFilterList entityEventFilters; + private String entityFromId; + + public Builder entityLimit(Long limit) { + this.entityLimit = limit; + return this; + } + + public Builder createdTimeBegin(Long timeBegin) { + this.createdTimeBegin = timeBegin; + return this; + } + + public Builder createTimeEnd(Long timeEnd) { + this.createdTimeEnd = timeEnd; + return this; + } + + public Builder relatesTo(TimelineFilterList relatesTo) { + this.relatesToFilters = relatesTo; + return this; + } + + public Builder isRelatedTo(TimelineFilterList isRelatedTo) { + this.isRelatedToFilters = isRelatedTo; + return this; + } + + public Builder infoFilters(TimelineFilterList infoFilters) { + this.entityInfoFilters = infoFilters; + return this; + } + + public Builder configFilters(TimelineFilterList configFilters) { + this.entityConfigFilters = configFilters; + return this; + } + + public Builder metricFilters(TimelineFilterList metricFilters) { + this.entityMetricFilters = metricFilters; + return this; + } + + public Builder eventFilters(TimelineFilterList eventFilters) { + this.entityEventFilters = eventFilters; + return this; + } + + public Builder fromId(String fromId) { + this.entityFromId = fromId; + return this; + } + + public TimelineEntityFilters build() { + return new TimelineEntityFilters(entityLimit, createdTimeBegin, + createdTimeEnd, relatesToFilters, isRelatedToFilters, + entityInfoFilters, entityConfigFilters, entityMetricFilters, + entityEventFilters, entityFromId); + } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderContext.java index 633bb232e4f..67c3d297e11 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderContext.java @@ -31,6 +31,8 @@ public class TimelineReaderContext extends TimelineContext { private String entityType; private String entityId; + private Long entityIdPrefix; + private String doAsUser; public TimelineReaderContext(String clusterId, String userId, String flowName, Long flowRunId, String appId, String entityType, String entityId) { super(clusterId, userId, flowName, flowRunId, appId); @@ -38,16 +40,33 @@ public class TimelineReaderContext extends TimelineContext { this.entityId = entityId; } + public TimelineReaderContext(String clusterId, String userId, String flowName, + Long flowRunId, String appId, String entityType, Long entityIdPrefix, + String entityId) { + this(clusterId, userId, flowName, flowRunId, appId, entityType, entityId); + this.entityIdPrefix = entityIdPrefix; + } + + public TimelineReaderContext(String clusterId, String userId, String flowName, + Long flowRunId, String appId, String entityType, Long entityIdPrefix, + String entityId, String doasUser) { + this(clusterId, userId, flowName, flowRunId, appId, entityType, entityId); + this.entityIdPrefix = entityIdPrefix; + this.doAsUser = doasUser; + } + public TimelineReaderContext(TimelineReaderContext other) { this(other.getClusterId(), other.getUserId(), other.getFlowName(), other.getFlowRunId(), other.getAppId(), other.getEntityType(), - other.getEntityId()); + other.getEntityIdPrefix(), other.getEntityId(), other.getDoAsUser()); } @Override public int hashCode() { final int prime = 31; int result = super.hashCode(); + result = prime * result + + ((entityIdPrefix == null) ? 0 : entityIdPrefix.hashCode()); result = prime * result + ((entityId == null) ? 0 : entityId.hashCode()); result = prime * result + ((entityType == null) ? 0 : entityType.hashCode()); @@ -95,4 +114,20 @@ public class TimelineReaderContext extends TimelineContext { public void setEntityId(String id) { this.entityId = id; } + + public Long getEntityIdPrefix() { + return entityIdPrefix; + } + + public void setEntityIdPrefix(Long entityIdPrefix) { + this.entityIdPrefix = entityIdPrefix; + } + + public String getDoAsUser() { + return doAsUser; + } + + public void setDoAsUser(String doAsUser) { + this.doAsUser = doAsUser; + } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java index 4cff3bc2e4c..67e5849ed6a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java @@ -32,8 +32,6 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader; -import com.google.common.annotations.VisibleForTesting; - /** * This class wraps over the timeline reader store implementation. It does some * non trivial manipulation of the timeline data before or after getting @@ -43,8 +41,6 @@ import com.google.common.annotations.VisibleForTesting; @Unstable public class TimelineReaderManager extends AbstractService { - @VisibleForTesting - public static final String UID_KEY = "UID"; private TimelineReader reader; public TimelineReaderManager(TimelineReader timelineReader) { @@ -94,18 +90,18 @@ public class TimelineReaderManager extends AbstractService { FlowActivityEntity activityEntity = (FlowActivityEntity)entity; context.setUserId(activityEntity.getUser()); context.setFlowName(activityEntity.getFlowName()); - entity.setUID(UID_KEY, + entity.setUID(TimelineReaderUtils.UID_KEY, TimelineUIDConverter.FLOW_UID.encodeUID(context)); return; case YARN_FLOW_RUN: FlowRunEntity runEntity = (FlowRunEntity)entity; context.setFlowRunId(runEntity.getRunId()); - entity.setUID(UID_KEY, + entity.setUID(TimelineReaderUtils.UID_KEY, TimelineUIDConverter.FLOWRUN_UID.encodeUID(context)); return; case YARN_APPLICATION: context.setAppId(entity.getId()); - entity.setUID(UID_KEY, + entity.setUID(TimelineReaderUtils.UID_KEY, TimelineUIDConverter.APPLICATION_UID.encodeUID(context)); return; default: @@ -113,9 +109,15 @@ public class TimelineReaderManager extends AbstractService { } } context.setEntityType(entity.getType()); + context.setEntityIdPrefix(entity.getIdPrefix()); context.setEntityId(entity.getId()); - entity.setUID(UID_KEY, - TimelineUIDConverter.GENERIC_ENTITY_UID.encodeUID(context)); + if (context.getDoAsUser() != null) { + entity.setUID(TimelineReaderUtils.UID_KEY, + TimelineUIDConverter.SUB_APPLICATION_ENTITY_UID.encodeUID(context)); + } else { + entity.setUID(TimelineReaderUtils.UID_KEY, + TimelineUIDConverter.GENERIC_ENTITY_UID.encodeUID(context)); + } } /** @@ -176,4 +178,24 @@ public class TimelineReaderManager extends AbstractService { } return entity; } + + /** + * Gets a list of available timeline entity types for an application. This can + * be done by making a call to the backend storage implementation. The meaning + * of each argument in detail is the same as {@link TimelineReader#getEntity}. + * If cluster ID has not been supplied by the client, fills the cluster id + * from config before making a call to backend storage. + * + * @param context Timeline context within the scope of which entity types + * have to be fetched. Entity type field of this context should + * be null. + * @return A set which contains available timeline entity types, represented + * as strings if found, empty otherwise. + * @throws IOException if any problem occurs while getting entity types. + */ + public Set getEntityTypes(TimelineReaderContext context) + throws IOException{ + context.setClusterId(getClusterID(context.getClusterId(), getConfig())); + return reader.getEntityTypes(new TimelineReaderContext(context)); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java index d7eff328686..5c049eaac90 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java @@ -18,19 +18,18 @@ package org.apache.hadoop.yarn.server.timelineservice.reader; -import static org.apache.hadoop.fs.CommonConfigurationKeys.DEFAULT_HADOOP_HTTP_STATIC_USER; -import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_HTTP_STATIC_USER; - +import java.io.IOException; +import java.net.InetSocketAddress; import java.net.URI; -import java.util.HashMap; -import java.util.Map; +import java.util.LinkedHashSet; +import java.util.Set; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.http.HttpServer2; -import org.apache.hadoop.http.lib.StaticUserWebFilter; import org.apache.hadoop.security.HttpCrossOriginFilterInitializer; +import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.service.CompositeService; import org.apache.hadoop.util.ExitUtil; import org.apache.hadoop.util.ReflectionUtils; @@ -40,7 +39,10 @@ import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.server.timelineservice.reader.security.TimelineReaderAuthenticationFilterInitializer; +import org.apache.hadoop.yarn.server.timelineservice.reader.security.TimelineReaderWhitelistAuthorizationFilterInitializer; import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader; +import org.apache.hadoop.yarn.server.util.timeline.TimelineServerUtils; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; @@ -71,6 +73,17 @@ public class TimelineReaderServer extends CompositeService { if (!YarnConfiguration.timelineServiceV2Enabled(conf)) { throw new YarnException("timeline service v.2 is not enabled"); } + InetSocketAddress bindAddr = conf.getSocketAddr( + YarnConfiguration.TIMELINE_SERVICE_ADDRESS, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ADDRESS, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_PORT); + // Login from keytab if security is enabled. + try { + SecurityUtil.login(conf, YarnConfiguration.TIMELINE_SERVICE_KEYTAB, + YarnConfiguration.TIMELINE_SERVICE_PRINCIPAL, bindAddr.getHostName()); + } catch(IOException e) { + throw new YarnRuntimeException("Failed to login from keytab", e); + } TimelineReader timelineReaderStore = createTimelineReaderStore(conf); timelineReaderStore.init(conf); @@ -130,29 +143,43 @@ public class TimelineReaderServer extends CompositeService { super.serviceStop(); } - private void startTimelineReaderWebApp() { - Configuration conf = getConfig(); - String bindAddress = WebAppUtils.getWebAppBindURL(conf, - YarnConfiguration.TIMELINE_SERVICE_BIND_HOST, - WebAppUtils.getTimelineReaderWebAppURL(conf)); - LOG.info("Instantiating TimelineReaderWebApp at " + bindAddress); + protected void addFilters(Configuration conf) { boolean enableCorsFilter = conf.getBoolean( YarnConfiguration.TIMELINE_SERVICE_HTTP_CROSS_ORIGIN_ENABLED, YarnConfiguration.TIMELINE_SERVICE_HTTP_CROSS_ORIGIN_ENABLED_DEFAULT); - // setup CORS + // Setup CORS if (enableCorsFilter) { conf.setBoolean(HttpCrossOriginFilterInitializer.PREFIX + HttpCrossOriginFilterInitializer.ENABLED_SUFFIX, true); } + String initializers = conf.get("hadoop.http.filter.initializers", ""); + Set defaultInitializers = new LinkedHashSet(); + if (!initializers.contains( + TimelineReaderAuthenticationFilterInitializer.class.getName())) { + defaultInitializers.add( + TimelineReaderAuthenticationFilterInitializer.class.getName()); + } + + defaultInitializers.add( + TimelineReaderWhitelistAuthorizationFilterInitializer.class.getName()); + + TimelineServerUtils.setTimelineFilters( + conf, initializers, defaultInitializers); + } + + private void startTimelineReaderWebApp() { + Configuration conf = getConfig(); + addFilters(conf); + String bindAddress = WebAppUtils.getWebAppBindURL(conf, + YarnConfiguration.TIMELINE_SERVICE_BIND_HOST, + WebAppUtils.getTimelineReaderWebAppURL(conf)); + LOG.info("Instantiating TimelineReaderWebApp at " + bindAddress); try { HttpServer2.Builder builder = new HttpServer2.Builder() .setName("timeline") .setConf(conf) .addEndpoint(URI.create("http://" + bindAddress)); readerWebServer = builder.build(); - - setupOptions(conf); - readerWebServer.addJerseyResourcePackage( TimelineReaderWebServices.class.getPackage().getName() + ";" + GenericExceptionHandler.class.getPackage().getName() + ";" @@ -168,24 +195,8 @@ public class TimelineReaderServer extends CompositeService { } } - /** - * Sets up some options and filters. - * - * @param conf Configuration - */ - protected void setupOptions(Configuration conf) { - Map options = new HashMap<>(); - String username = conf.get(HADOOP_HTTP_STATIC_USER, - DEFAULT_HADOOP_HTTP_STATIC_USER); - options.put(HADOOP_HTTP_STATIC_USER, username); - HttpServer2.defineFilter(readerWebServer.getWebAppContext(), - "static_user_filter_timeline", - StaticUserWebFilter.StaticUserFilter.class.getName(), - options, new String[] {"/*"}); - } - @VisibleForTesting - int getWebServerPort() { + public int getWebServerPort() { return readerWebServer.getConnectorAddress(0).getPort(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderUtils.java index c93c631b364..4fd846857e8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderUtils.java @@ -24,13 +24,32 @@ import java.util.List; import org.apache.commons.lang.StringUtils; +import com.google.common.annotations.VisibleForTesting; + /** * Set of utility methods to be used across timeline reader. */ -final class TimelineReaderUtils { +public final class TimelineReaderUtils { private TimelineReaderUtils() { } + /** + * Default delimiter for joining strings. + */ + @VisibleForTesting + public static final char DEFAULT_DELIMITER_CHAR = '!'; + + /** + * Default escape character used for joining strings. + */ + @VisibleForTesting + public static final char DEFAULT_ESCAPE_CHAR = '*'; + + public static final String FROMID_KEY = "FROM_ID"; + + @VisibleForTesting + public static final String UID_KEY = "UID"; + /** * Split the passed string along the passed delimiter character while looking * for escape char to interpret the splitted parts correctly. For delimiter or @@ -168,4 +187,14 @@ final class TimelineReaderUtils { // Join the strings after they have been escaped. return StringUtils.join(strs, delimiterChar); } + + public static List split(final String str) + throws IllegalArgumentException { + return split(str, DEFAULT_DELIMITER_CHAR, DEFAULT_ESCAPE_CHAR); + } + + public static String joinAndEscapeStrings(final String[] strs) { + return joinAndEscapeStrings(strs, DEFAULT_DELIMITER_CHAR, + DEFAULT_ESCAPE_CHAR); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java index b3e3cdc5fee..dfe04f9fa09 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java @@ -265,6 +265,15 @@ public class TimelineReaderWebServices { * or has a value less than 1, and metrics have to be retrieved, then * metricsLimit will be considered as 1 i.e. latest single value of * metric(s) will be returned. (Optional query param). + * @param metricsTimeStart If specified, returned metrics for the entities + * would not contain metric values before this timestamp(Optional query + * param). + * @param metricsTimeEnd If specified, returned metrics for the entities would + * not contain metric values after this timestamp(Optional query param). + * @param fromId If specified, retrieve the next set of entities from the + * given fromId. The set of entities retrieved is inclusive of specified + * fromId. fromId should be taken from the value associated with FROM_ID + * info key in entity response which was sent earlier. * * @return If successful, a HTTP 200(OK) response having a JSON representing * a set of TimelineEntity instances of the given entity type @@ -295,7 +304,10 @@ public class TimelineReaderWebServices { @QueryParam("confstoretrieve") String confsToRetrieve, @QueryParam("metricstoretrieve") String metricsToRetrieve, @QueryParam("fields") String fields, - @QueryParam("metricslimit") String metricsLimit) { + @QueryParam("metricslimit") String metricsLimit, + @QueryParam("metricstimestart") String metricsTimeStart, + @QueryParam("metricstimeend") String metricsTimeEnd, + @QueryParam("fromid") String fromId) { String url = req.getRequestURI() + (req.getQueryString() == null ? "" : QUERY_STRING_SEP + req.getQueryString()); @@ -318,9 +330,11 @@ public class TimelineReaderWebServices { entities = timelineReaderManager.getEntities(context, TimelineReaderWebServicesUtils.createTimelineEntityFilters( limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo, - infofilters, conffilters, metricfilters, eventfilters), + infofilters, conffilters, metricfilters, eventfilters, + fromId), TimelineReaderWebServicesUtils.createTimelineDataToRetrieve( - confsToRetrieve, metricsToRetrieve, fields, metricsLimit)); + confsToRetrieve, metricsToRetrieve, fields, metricsLimit, + metricsTimeStart, metricsTimeEnd)); } catch (Exception e) { handleException(e, url, startTime, "createdTime start/end or limit or flowrunid"); @@ -401,6 +415,15 @@ public class TimelineReaderWebServices { * or has a value less than 1, and metrics have to be retrieved, then * metricsLimit will be considered as 1 i.e. latest single value of * metric(s) will be returned. (Optional query param). + * @param metricsTimeStart If specified, returned metrics for the entities + * would not contain metric values before this timestamp(Optional query + * param). + * @param metricsTimeEnd If specified, returned metrics for the entities would + * not contain metric values after this timestamp(Optional query param). + * @param fromId If specified, retrieve the next set of entities from the + * given fromId. The set of entities retrieved is inclusive of specified + * fromId. fromId should be taken from the value associated with FROM_ID + * info key in entity response which was sent earlier. * * @return If successful, a HTTP 200(OK) response having a JSON representing * a set of TimelineEntity instances of the given entity type @@ -436,11 +459,15 @@ public class TimelineReaderWebServices { @QueryParam("confstoretrieve") String confsToRetrieve, @QueryParam("metricstoretrieve") String metricsToRetrieve, @QueryParam("fields") String fields, - @QueryParam("metricslimit") String metricsLimit) { + @QueryParam("metricslimit") String metricsLimit, + @QueryParam("metricstimestart") String metricsTimeStart, + @QueryParam("metricstimeend") String metricsTimeEnd, + @QueryParam("fromid") String fromId) { return getEntities(req, res, null, appId, entityType, userId, flowName, flowRunId, limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo, infofilters, conffilters, metricfilters, eventfilters, - confsToRetrieve, metricsToRetrieve, fields, metricsLimit); + confsToRetrieve, metricsToRetrieve, fields, metricsLimit, + metricsTimeStart, metricsTimeEnd, fromId); } /** @@ -511,6 +538,15 @@ public class TimelineReaderWebServices { * or has a value less than 1, and metrics have to be retrieved, then * metricsLimit will be considered as 1 i.e. latest single value of * metric(s) will be returned. (Optional query param). + * @param metricsTimeStart If specified, returned metrics for the entities + * would not contain metric values before this timestamp(Optional query + * param). + * @param metricsTimeEnd If specified, returned metrics for the entities would + * not contain metric values after this timestamp(Optional query param). + * @param fromId If specified, retrieve the next set of entities from the + * given fromId. The set of entities retrieved is inclusive of specified + * fromId. fromId should be taken from the value associated with FROM_ID + * info key in entity response which was sent earlier. * * @return If successful, a HTTP 200(OK) response having a JSON representing * a set of TimelineEntity instances of the given entity type @@ -547,7 +583,10 @@ public class TimelineReaderWebServices { @QueryParam("confstoretrieve") String confsToRetrieve, @QueryParam("metricstoretrieve") String metricsToRetrieve, @QueryParam("fields") String fields, - @QueryParam("metricslimit") String metricsLimit) { + @QueryParam("metricslimit") String metricsLimit, + @QueryParam("metricstimestart") String metricsTimeStart, + @QueryParam("metricstimeend") String metricsTimeEnd, + @QueryParam("fromid") String fromId) { String url = req.getRequestURI() + (req.getQueryString() == null ? "" : QUERY_STRING_SEP + req.getQueryString()); @@ -560,14 +599,17 @@ public class TimelineReaderWebServices { TimelineReaderManager timelineReaderManager = getTimelineReaderManager(); Set entities = null; try { - entities = timelineReaderManager.getEntities( - TimelineReaderWebServicesUtils.createTimelineReaderContext( - clusterId, userId, flowName, flowRunId, appId, entityType, null), + TimelineReaderContext context = TimelineReaderWebServicesUtils + .createTimelineReaderContext(clusterId, userId, flowName, flowRunId, + appId, entityType, null, null); + entities = timelineReaderManager.getEntities(context, TimelineReaderWebServicesUtils.createTimelineEntityFilters( limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo, - infofilters, conffilters, metricfilters, eventfilters), + infofilters, conffilters, metricfilters, eventfilters, + fromId), TimelineReaderWebServicesUtils.createTimelineDataToRetrieve( - confsToRetrieve, metricsToRetrieve, fields, metricsLimit)); + confsToRetrieve, metricsToRetrieve, fields, metricsLimit, + metricsTimeStart, metricsTimeEnd)); } catch (Exception e) { handleException(e, url, startTime, "createdTime start/end or limit or flowrunid"); @@ -609,6 +651,10 @@ public class TimelineReaderWebServices { * or has a value less than 1, and metrics have to be retrieved, then * metricsLimit will be considered as 1 i.e. latest single value of * metric(s) will be returned. (Optional query param). + * @param metricsTimeStart If specified, returned metrics for the entity would + * not contain metric values before this timestamp(Optional query param). + * @param metricsTimeEnd If specified, returned metrics for the entity would + * not contain metric values after this timestamp(Optional query param). * * @return If successful, a HTTP 200(OK) response having a JSON representing a * TimelineEntity instance is returned.
    @@ -630,7 +676,9 @@ public class TimelineReaderWebServices { @QueryParam("confstoretrieve") String confsToRetrieve, @QueryParam("metricstoretrieve") String metricsToRetrieve, @QueryParam("fields") String fields, - @QueryParam("metricslimit") String metricsLimit) { + @QueryParam("metricslimit") String metricsLimit, + @QueryParam("metricstimestart") String metricsTimeStart, + @QueryParam("metricstimeend") String metricsTimeEnd) { String url = req.getRequestURI() + (req.getQueryString() == null ? "" : QUERY_STRING_SEP + req.getQueryString()); @@ -650,7 +698,8 @@ public class TimelineReaderWebServices { } entity = timelineReaderManager.getEntity(context, TimelineReaderWebServicesUtils.createTimelineDataToRetrieve( - confsToRetrieve, metricsToRetrieve, fields, metricsLimit)); + confsToRetrieve, metricsToRetrieve, fields, metricsLimit, + metricsTimeStart, metricsTimeEnd)); } catch (Exception e) { handleException(e, url, startTime, "flowrunid"); } @@ -704,6 +753,12 @@ public class TimelineReaderWebServices { * or has a value less than 1, and metrics have to be retrieved, then * metricsLimit will be considered as 1 i.e. latest single value of * metric(s) will be returned. (Optional query param). + * @param metricsTimeStart If specified, returned metrics for the entity would + * not contain metric values before this timestamp(Optional query param). + * @param metricsTimeEnd If specified, returned metrics for the entity would + * not contain metric values after this timestamp(Optional query param). + * @param entityIdPrefix Defines the id prefix for the entity to be fetched. + * If specified, then entity retrieval will be faster. * * @return If successful, a HTTP 200(OK) response having a JSON representing a * TimelineEntity instance is returned.
    @@ -730,10 +785,13 @@ public class TimelineReaderWebServices { @QueryParam("confstoretrieve") String confsToRetrieve, @QueryParam("metricstoretrieve") String metricsToRetrieve, @QueryParam("fields") String fields, - @QueryParam("metricslimit") String metricsLimit) { + @QueryParam("metricslimit") String metricsLimit, + @QueryParam("metricstimestart") String metricsTimeStart, + @QueryParam("metricstimeend") String metricsTimeEnd, + @QueryParam("entityidprefix") String entityIdPrefix) { return getEntity(req, res, null, appId, entityType, entityId, userId, flowName, flowRunId, confsToRetrieve, metricsToRetrieve, fields, - metricsLimit); + metricsLimit, metricsTimeStart, metricsTimeEnd, entityIdPrefix); } /** @@ -775,6 +833,12 @@ public class TimelineReaderWebServices { * or has a value less than 1, and metrics have to be retrieved, then * metricsLimit will be considered as 1 i.e. latest single value of * metric(s) will be returned. (Optional query param). + * @param metricsTimeStart If specified, returned metrics for the entity would + * not contain metric values before this timestamp(Optional query param). + * @param metricsTimeEnd If specified, returned metrics for the entity would + * not contain metric values after this timestamp(Optional query param). + * @param entityIdPrefix Defines the id prefix for the entity to be fetched. + * If specified, then entity retrieval will be faster. * * @return If successful, a HTTP 200(OK) response having a JSON representing a * TimelineEntity instance is returned.
    @@ -802,7 +866,10 @@ public class TimelineReaderWebServices { @QueryParam("confstoretrieve") String confsToRetrieve, @QueryParam("metricstoretrieve") String metricsToRetrieve, @QueryParam("fields") String fields, - @QueryParam("metricslimit") String metricsLimit) { + @QueryParam("metricslimit") String metricsLimit, + @QueryParam("metricstimestart") String metricsTimeStart, + @QueryParam("metricstimeend") String metricsTimeEnd, + @QueryParam("entityidprefix") String entityIdPrefix) { String url = req.getRequestURI() + (req.getQueryString() == null ? "" : QUERY_STRING_SEP + req.getQueryString()); @@ -817,9 +884,11 @@ public class TimelineReaderWebServices { try { entity = timelineReaderManager.getEntity( TimelineReaderWebServicesUtils.createTimelineReaderContext( - clusterId, userId, flowName, flowRunId, appId, entityType, entityId), + clusterId, userId, flowName, flowRunId, appId, entityType, + entityIdPrefix, entityId), TimelineReaderWebServicesUtils.createTimelineDataToRetrieve( - confsToRetrieve, metricsToRetrieve, fields, metricsLimit)); + confsToRetrieve, metricsToRetrieve, fields, metricsLimit, + metricsTimeStart, metricsTimeEnd)); } catch (Exception e) { handleException(e, url, startTime, "flowrunid"); } @@ -886,7 +955,7 @@ public class TimelineReaderWebServices { context.setEntityType(TimelineEntityType.YARN_FLOW_RUN.toString()); entity = timelineReaderManager.getEntity(context, TimelineReaderWebServicesUtils.createTimelineDataToRetrieve( - null, metricsToRetrieve, null, null)); + null, metricsToRetrieve, null, null, null, null)); } catch (Exception e) { handleException(e, url, startTime, "flowrunid"); } @@ -993,9 +1062,9 @@ public class TimelineReaderWebServices { entity = timelineReaderManager.getEntity( TimelineReaderWebServicesUtils.createTimelineReaderContext( clusterId, userId, flowName, flowRunId, null, - TimelineEntityType.YARN_FLOW_RUN.toString(), null), + TimelineEntityType.YARN_FLOW_RUN.toString(), null, null), TimelineReaderWebServicesUtils.createTimelineDataToRetrieve( - null, metricsToRetrieve, null, null)); + null, metricsToRetrieve, null, null, null, null)); } catch (Exception e) { handleException(e, url, startTime, "flowrunid"); } @@ -1039,6 +1108,10 @@ public class TimelineReaderWebServices { * METRICS makes sense for flow runs hence only ALL or METRICS are * supported as fields for fetching flow runs. Other fields will lead to * HTTP 400 (Bad Request) response. (Optional query param). + * @param fromId If specified, retrieve the next set of flow run entities + * from the given fromId. The set of entities retrieved is inclusive of + * specified fromId. fromId should be taken from the value associated + * with FROM_ID info key in entity response which was sent earlier. * * @return If successful, a HTTP 200(OK) response having a JSON representing a * set of FlowRunEntity instances for the given flow are @@ -1060,7 +1133,8 @@ public class TimelineReaderWebServices { @QueryParam("createdtimestart") String createdTimeStart, @QueryParam("createdtimeend") String createdTimeEnd, @QueryParam("metricstoretrieve") String metricsToRetrieve, - @QueryParam("fields") String fields) { + @QueryParam("fields") String fields, + @QueryParam("fromid") String fromId) { String url = req.getRequestURI() + (req.getQueryString() == null ? "" : QUERY_STRING_SEP + req.getQueryString()); @@ -1082,11 +1156,12 @@ public class TimelineReaderWebServices { entities = timelineReaderManager.getEntities(context, TimelineReaderWebServicesUtils.createTimelineEntityFilters( limit, createdTimeStart, createdTimeEnd, null, null, null, - null, null, null), + null, null, null, fromId), TimelineReaderWebServicesUtils.createTimelineDataToRetrieve( - null, metricsToRetrieve, fields, null)); + null, metricsToRetrieve, fields, null, null, null)); } catch (Exception e) { - handleException(e, url, startTime, "createdTime start/end or limit"); + handleException(e, url, startTime, + "createdTime start/end or limit or fromId"); } long endTime = Time.monotonicNow(); if (entities == null) { @@ -1124,6 +1199,10 @@ public class TimelineReaderWebServices { * METRICS makes sense for flow runs hence only ALL or METRICS are * supported as fields for fetching flow runs. Other fields will lead to * HTTP 400 (Bad Request) response. (Optional query param). + * @param fromId If specified, retrieve the next set of flow run entities + * from the given fromId. The set of entities retrieved is inclusive of + * specified fromId. fromId should be taken from the value associated + * with FROM_ID info key in entity response which was sent earlier. * * @return If successful, a HTTP 200(OK) response having a JSON representing a * set of FlowRunEntity instances for the given flow are @@ -1146,9 +1225,10 @@ public class TimelineReaderWebServices { @QueryParam("createdtimestart") String createdTimeStart, @QueryParam("createdtimeend") String createdTimeEnd, @QueryParam("metricstoretrieve") String metricsToRetrieve, - @QueryParam("fields") String fields) { + @QueryParam("fields") String fields, + @QueryParam("fromid") String fromId) { return getFlowRuns(req, res, null, userId, flowName, limit, - createdTimeStart, createdTimeEnd, metricsToRetrieve, fields); + createdTimeStart, createdTimeEnd, metricsToRetrieve, fields, fromId); } /** @@ -1179,6 +1259,10 @@ public class TimelineReaderWebServices { * METRICS makes sense for flow runs hence only ALL or METRICS are * supported as fields for fetching flow runs. Other fields will lead to * HTTP 400 (Bad Request) response. (Optional query param). + * @param fromId If specified, retrieve the next set of flow run entities + * from the given fromId. The set of entities retrieved is inclusive of + * specified fromId. fromId should be taken from the value associated + * with FROM_ID info key in entity response which was sent earlier. * * @return If successful, a HTTP 200(OK) response having a JSON representing a * set of FlowRunEntity instances for the given flow are @@ -1202,7 +1286,8 @@ public class TimelineReaderWebServices { @QueryParam("createdtimestart") String createdTimeStart, @QueryParam("createdtimeend") String createdTimeEnd, @QueryParam("metricstoretrieve") String metricsToRetrieve, - @QueryParam("fields") String fields) { + @QueryParam("fields") String fields, + @QueryParam("fromid") String fromId) { String url = req.getRequestURI() + (req.getQueryString() == null ? "" : QUERY_STRING_SEP + req.getQueryString()); @@ -1218,14 +1303,15 @@ public class TimelineReaderWebServices { entities = timelineReaderManager.getEntities( TimelineReaderWebServicesUtils.createTimelineReaderContext( clusterId, userId, flowName, null, null, - TimelineEntityType.YARN_FLOW_RUN.toString(), null), + TimelineEntityType.YARN_FLOW_RUN.toString(), null, null), TimelineReaderWebServicesUtils.createTimelineEntityFilters( limit, createdTimeStart, createdTimeEnd, null, null, null, - null, null, null), + null, null, null, fromId), TimelineReaderWebServicesUtils.createTimelineDataToRetrieve( - null, metricsToRetrieve, fields, null)); + null, metricsToRetrieve, fields, null, null, null)); } catch (Exception e) { - handleException(e, url, startTime, "createdTime start/end or limit"); + handleException(e, url, startTime, + "createdTime start/end or limit or fromId"); } long endTime = Time.monotonicNow(); if (entities == null) { @@ -1261,6 +1347,10 @@ public class TimelineReaderWebServices { * 2 dates. * "daterange=20150711-" returns flows active on and after 20150711. * "daterange=-20150711" returns flows active on and before 20150711. + * @param fromId If specified, retrieve the next set of flows from the given + * fromId. The set of flows retrieved is inclusive of specified fromId. + * fromId should be taken from the value associated with FROM_ID info key + * in flow entity response which was sent earlier. * * @return If successful, a HTTP 200(OK) response having a JSON representing a * set of FlowActivityEntity instances are returned.
    @@ -1277,8 +1367,9 @@ public class TimelineReaderWebServices { @Context HttpServletRequest req, @Context HttpServletResponse res, @QueryParam("limit") String limit, - @QueryParam("daterange") String dateRange) { - return getFlows(req, res, null, limit, dateRange); + @QueryParam("daterange") String dateRange, + @QueryParam("fromid") String fromId) { + return getFlows(req, res, null, limit, dateRange, fromId); } /** @@ -1307,6 +1398,10 @@ public class TimelineReaderWebServices { * 2 dates. * "daterange=20150711-" returns flows active on and after 20150711. * "daterange=-20150711" returns flows active on and before 20150711. + * @param fromId If specified, retrieve the next set of flows from the given + * fromId. The set of flows retrieved is inclusive of specified fromId. + * fromId should be taken from the value associated with FROM_ID info key + * in flow entity response which was sent earlier. * * @return If successful, a HTTP 200(OK) response having a JSON representing a * set of FlowActivityEntity instances are returned.
    @@ -1324,7 +1419,8 @@ public class TimelineReaderWebServices { @Context HttpServletResponse res, @PathParam("clusterid") String clusterId, @QueryParam("limit") String limit, - @QueryParam("daterange") String dateRange) { + @QueryParam("daterange") String dateRange, + @QueryParam("fromid") String fromId) { String url = req.getRequestURI() + (req.getQueryString() == null ? "" : QUERY_STRING_SEP + req.getQueryString()); @@ -1340,15 +1436,14 @@ public class TimelineReaderWebServices { DateRange range = parseDateRange(dateRange); TimelineEntityFilters entityFilters = TimelineReaderWebServicesUtils.createTimelineEntityFilters( - limit, null, null, null, null, null, null, null, null); - entityFilters.setCreatedTimeBegin(range.dateStart); - entityFilters.setCreatedTimeEnd(range.dateEnd); + limit, range.dateStart, range.dateEnd, + null, null, null, null, null, null, fromId); entities = timelineReaderManager.getEntities( TimelineReaderWebServicesUtils.createTimelineReaderContext( clusterId, null, null, null, null, - TimelineEntityType.YARN_FLOW_ACTIVITY.toString(), null), + TimelineEntityType.YARN_FLOW_ACTIVITY.toString(), null, null), entityFilters, TimelineReaderWebServicesUtils. - createTimelineDataToRetrieve(null, null, null, null)); + createTimelineDataToRetrieve(null, null, null, null, null, null)); } catch (Exception e) { handleException(e, url, startTime, "limit"); } @@ -1389,6 +1484,10 @@ public class TimelineReaderWebServices { * or has a value less than 1, and metrics have to be retrieved, then * metricsLimit will be considered as 1 i.e. latest single value of * metric(s) will be returned. (Optional query param). + * @param metricsTimeStart If specified, returned metrics for the apps would + * not contain metric values before this timestamp(Optional query param). + * @param metricsTimeEnd If specified, returned metrics for the apps would + * not contain metric values after this timestamp(Optional query param). * * @return If successful, a HTTP 200(OK) response having a JSON representing a * TimelineEntity instance is returned.
    @@ -1410,7 +1509,9 @@ public class TimelineReaderWebServices { @QueryParam("confstoretrieve") String confsToRetrieve, @QueryParam("metricstoretrieve") String metricsToRetrieve, @QueryParam("fields") String fields, - @QueryParam("metricslimit") String metricsLimit) { + @QueryParam("metricslimit") String metricsLimit, + @QueryParam("metricstimestart") String metricsTimeStart, + @QueryParam("metricstimeend") String metricsTimeEnd) { String url = req.getRequestURI() + (req.getQueryString() == null ? "" : QUERY_STRING_SEP + req.getQueryString()); @@ -1431,7 +1532,8 @@ public class TimelineReaderWebServices { context.setEntityType(TimelineEntityType.YARN_APPLICATION.toString()); entity = timelineReaderManager.getEntity(context, TimelineReaderWebServicesUtils.createTimelineDataToRetrieve( - confsToRetrieve, metricsToRetrieve, fields, metricsLimit)); + confsToRetrieve, metricsToRetrieve, fields, metricsLimit, + metricsTimeStart, metricsTimeEnd)); } catch (Exception e) { handleException(e, url, startTime, "flowrunid"); } @@ -1480,6 +1582,10 @@ public class TimelineReaderWebServices { * or has a value less than 1, and metrics have to be retrieved, then * metricsLimit will be considered as 1 i.e. latest single value of * metric(s) will be returned. (Optional query param). + * @param metricsTimeStart If specified, returned metrics for the app would + * not contain metric values before this timestamp(Optional query param). + * @param metricsTimeEnd If specified, returned metrics for the app would + * not contain metric values after this timestamp(Optional query param). * * @return If successful, a HTTP 200(OK) response having a JSON representing a * TimelineEntity instance is returned.
    @@ -1504,9 +1610,12 @@ public class TimelineReaderWebServices { @QueryParam("confstoretrieve") String confsToRetrieve, @QueryParam("metricstoretrieve") String metricsToRetrieve, @QueryParam("fields") String fields, - @QueryParam("metricslimit") String metricsLimit) { + @QueryParam("metricslimit") String metricsLimit, + @QueryParam("metricstimestart") String metricsTimeStart, + @QueryParam("metricstimeend") String metricsTimeEnd) { return getApp(req, res, null, appId, flowName, flowRunId, userId, - confsToRetrieve, metricsToRetrieve, fields, metricsLimit); + confsToRetrieve, metricsToRetrieve, fields, metricsLimit, + metricsTimeStart, metricsTimeEnd); } /** @@ -1544,6 +1653,10 @@ public class TimelineReaderWebServices { * or has a value less than 1, and metrics have to be retrieved, then * metricsLimit will be considered as 1 i.e. latest single value of * metric(s) will be returned. (Optional query param). + * @param metricsTimeStart If specified, returned metrics for the app would + * not contain metric values before this timestamp(Optional query param). + * @param metricsTimeEnd If specified, returned metrics for the app would + * not contain metric values after this timestamp(Optional query param). * * @return If successful, a HTTP 200(OK) response having a JSON representing a * TimelineEntity instance is returned.
    @@ -1569,7 +1682,9 @@ public class TimelineReaderWebServices { @QueryParam("confstoretrieve") String confsToRetrieve, @QueryParam("metricstoretrieve") String metricsToRetrieve, @QueryParam("fields") String fields, - @QueryParam("metricslimit") String metricsLimit) { + @QueryParam("metricslimit") String metricsLimit, + @QueryParam("metricstimestart") String metricsTimeStart, + @QueryParam("metricstimeend") String metricsTimeEnd) { String url = req.getRequestURI() + (req.getQueryString() == null ? "" : QUERY_STRING_SEP + req.getQueryString()); @@ -1585,9 +1700,10 @@ public class TimelineReaderWebServices { entity = timelineReaderManager.getEntity( TimelineReaderWebServicesUtils.createTimelineReaderContext( clusterId, userId, flowName, flowRunId, appId, - TimelineEntityType.YARN_APPLICATION.toString(), null), + TimelineEntityType.YARN_APPLICATION.toString(), null, null), TimelineReaderWebServicesUtils.createTimelineDataToRetrieve( - confsToRetrieve, metricsToRetrieve, fields, metricsLimit)); + confsToRetrieve, metricsToRetrieve, fields, metricsLimit, + metricsTimeStart, metricsTimeEnd)); } catch (Exception e) { handleException(e, url, startTime, "flowrunid"); } @@ -1660,6 +1776,14 @@ public class TimelineReaderWebServices { * or has a value less than 1, and metrics have to be retrieved, then * metricsLimit will be considered as 1 i.e. latest single value of * metric(s) will be returned. (Optional query param). + * @param metricsTimeStart If specified, returned metrics for the apps would + * not contain metric values before this timestamp(Optional query param). + * @param metricsTimeEnd If specified, returned metrics for the apps would + * not contain metric values after this timestamp(Optional query param). + * @param fromId If specified, retrieve the next set of applications + * from the given fromId. The set of applications retrieved is inclusive + * of specified fromId. fromId should be taken from the value associated + * with FROM_ID info key in entity response which was sent earlier. * * @return If successful, a HTTP 200(OK) response having a JSON representing * a set of TimelineEntity instances representing apps is @@ -1689,7 +1813,10 @@ public class TimelineReaderWebServices { @QueryParam("confstoretrieve") String confsToRetrieve, @QueryParam("metricstoretrieve") String metricsToRetrieve, @QueryParam("fields") String fields, - @QueryParam("metricslimit") String metricsLimit) { + @QueryParam("metricslimit") String metricsLimit, + @QueryParam("metricstimestart") String metricsTimeStart, + @QueryParam("metricstimeend") String metricsTimeEnd, + @QueryParam("fromid") String fromId) { String url = req.getRequestURI() + (req.getQueryString() == null ? "" : QUERY_STRING_SEP + req.getQueryString()); @@ -1711,9 +1838,11 @@ public class TimelineReaderWebServices { entities = timelineReaderManager.getEntities(context, TimelineReaderWebServicesUtils.createTimelineEntityFilters( limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo, - infofilters, conffilters, metricfilters, eventfilters), + infofilters, conffilters, metricfilters, eventfilters, + fromId), TimelineReaderWebServicesUtils.createTimelineDataToRetrieve( - confsToRetrieve, metricsToRetrieve, fields, metricsLimit)); + confsToRetrieve, metricsToRetrieve, fields, metricsLimit, + metricsTimeStart, metricsTimeEnd)); } catch (Exception e) { handleException(e, url, startTime, "createdTime start/end or limit or flowrunid"); @@ -1787,6 +1916,14 @@ public class TimelineReaderWebServices { * or has a value less than 1, and metrics have to be retrieved, then * metricsLimit will be considered as 1 i.e. latest single value of * metric(s) will be returned. (Optional query param). + * @param metricsTimeStart If specified, returned metrics for the apps would + * not contain metric values before this timestamp(Optional query param). + * @param metricsTimeEnd If specified, returned metrics for the apps would + * not contain metric values after this timestamp(Optional query param). + * @param fromId If specified, retrieve the next set of applications + * from the given fromId. The set of applications retrieved is inclusive + * of specified fromId. fromId should be taken from the value associated + * with FROM_ID info key in entity response which was sent earlier. * * @return If successful, a HTTP 200(OK) response having a JSON representing * a set of TimelineEntity instances representing apps is @@ -1818,12 +1955,16 @@ public class TimelineReaderWebServices { @QueryParam("confstoretrieve") String confsToRetrieve, @QueryParam("metricstoretrieve") String metricsToRetrieve, @QueryParam("fields") String fields, - @QueryParam("metricslimit") String metricsLimit) { + @QueryParam("metricslimit") String metricsLimit, + @QueryParam("metricstimestart") String metricsTimeStart, + @QueryParam("metricstimeend") String metricsTimeEnd, + @QueryParam("fromid") String fromId) { return getEntities(req, res, null, null, TimelineEntityType.YARN_APPLICATION.toString(), userId, flowName, flowRunId, limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo, infofilters, conffilters, metricfilters, eventfilters, - confsToRetrieve, metricsToRetrieve, fields, metricsLimit); + confsToRetrieve, metricsToRetrieve, fields, metricsLimit, + metricsTimeStart, metricsTimeEnd, fromId); } /** @@ -1887,6 +2028,14 @@ public class TimelineReaderWebServices { * or has a value less than 1, and metrics have to be retrieved, then * metricsLimit will be considered as 1 i.e. latest single value of * metric(s) will be returned. (Optional query param). + * @param metricsTimeStart If specified, returned metrics for the apps would + * not contain metric values before this timestamp(Optional query param). + * @param metricsTimeEnd If specified, returned metrics for the apps would + * not contain metric values after this timestamp(Optional query param). + * @param fromId If specified, retrieve the next set of applications + * from the given fromId. The set of applications retrieved is inclusive + * of specified fromId. fromId should be taken from the value associated + * with FROM_ID info key in entity response which was sent earlier. * * @return If successful, a HTTP 200(OK) response having a JSON representing * a set of TimelineEntity instances representing apps is @@ -1920,12 +2069,16 @@ public class TimelineReaderWebServices { @QueryParam("confstoretrieve") String confsToRetrieve, @QueryParam("metricstoretrieve") String metricsToRetrieve, @QueryParam("fields") String fields, - @QueryParam("metricslimit") String metricsLimit) { + @QueryParam("metricslimit") String metricsLimit, + @QueryParam("metricstimestart") String metricsTimeStart, + @QueryParam("metricstimeend") String metricsTimeEnd, + @QueryParam("fromid") String fromId) { return getEntities(req, res, clusterId, null, TimelineEntityType.YARN_APPLICATION.toString(), userId, flowName, flowRunId, limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo, infofilters, conffilters, metricfilters, eventfilters, - confsToRetrieve, metricsToRetrieve, fields, metricsLimit); + confsToRetrieve, metricsToRetrieve, fields, metricsLimit, + metricsTimeStart, metricsTimeEnd, fromId); } /** @@ -1986,6 +2139,14 @@ public class TimelineReaderWebServices { * or has a value less than 1, and metrics have to be retrieved, then * metricsLimit will be considered as 1 i.e. latest single value of * metric(s) will be returned. (Optional query param). + * @param metricsTimeStart If specified, returned metrics for the apps would + * not contain metric values before this timestamp(Optional query param). + * @param metricsTimeEnd If specified, returned metrics for the apps would + * not contain metric values after this timestamp(Optional query param). + * @param fromId If specified, retrieve the next set of applications + * from the given fromId. The set of applications retrieved is inclusive + * of specified fromId. fromId should be taken from the value associated + * with FROM_ID info key in entity response which was sent earlier. * * @return If successful, a HTTP 200(OK) response having a JSON representing * a set of TimelineEntity instances representing apps is @@ -2016,12 +2177,16 @@ public class TimelineReaderWebServices { @QueryParam("confstoretrieve") String confsToRetrieve, @QueryParam("metricstoretrieve") String metricsToRetrieve, @QueryParam("fields") String fields, - @QueryParam("metricslimit") String metricsLimit) { + @QueryParam("metricslimit") String metricsLimit, + @QueryParam("metricstimestart") String metricsTimeStart, + @QueryParam("metricstimeend") String metricsTimeEnd, + @QueryParam("fromid") String fromId) { return getEntities(req, res, null, null, TimelineEntityType.YARN_APPLICATION.toString(), userId, flowName, null, limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo, infofilters, conffilters, metricfilters, eventfilters, - confsToRetrieve, metricsToRetrieve, fields, metricsLimit); + confsToRetrieve, metricsToRetrieve, fields, metricsLimit, + metricsTimeStart, metricsTimeEnd, fromId); } /** @@ -2083,6 +2248,14 @@ public class TimelineReaderWebServices { * or has a value less than 1, and metrics have to be retrieved, then * metricsLimit will be considered as 1 i.e. latest single value of * metric(s) will be returned. (Optional query param). + * @param metricsTimeStart If specified, returned metrics for the apps would + * not contain metric values before this timestamp(Optional query param). + * @param metricsTimeEnd If specified, returned metrics for the apps would + * not contain metric values after this timestamp(Optional query param). + * @param fromId If specified, retrieve the next set of applications + * from the given fromId. The set of applications retrieved is inclusive + * of specified fromId. fromId should be taken from the value associated + * with FROM_ID info key in entity response which was sent earlier. * * @return If successful, a HTTP 200(OK) response having a JSON representing * a set of TimelineEntity instances representing apps is @@ -2114,12 +2287,16 @@ public class TimelineReaderWebServices { @QueryParam("confstoretrieve") String confsToRetrieve, @QueryParam("metricstoretrieve") String metricsToRetrieve, @QueryParam("fields") String fields, - @QueryParam("metricslimit") String metricsLimit) { + @QueryParam("metricslimit") String metricsLimit, + @QueryParam("metricstimestart") String metricsTimeStart, + @QueryParam("metricstimeend") String metricsTimeEnd, + @QueryParam("fromid") String fromId) { return getEntities(req, res, clusterId, null, TimelineEntityType.YARN_APPLICATION.toString(), userId, flowName, null, limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo, infofilters, conffilters, metricfilters, eventfilters, - confsToRetrieve, metricsToRetrieve, fields, metricsLimit); + confsToRetrieve, metricsToRetrieve, fields, metricsLimit, + metricsTimeStart, metricsTimeEnd, fromId); } /** @@ -2190,6 +2367,17 @@ public class TimelineReaderWebServices { * have to be retrieved, then metricsLimit will be considered as 1 * i.e. latest single value of metric(s) will be returned. (Optional * query param). + * @param metricsTimeStart If specified, returned metrics for the app attempts + * would not contain metric values before this timestamp(Optional + * query param). + * @param metricsTimeEnd If specified, returned metrics for the app attempts + * would not contain metric values after this timestamp(Optional + * query param). + * @param fromId If specified, retrieve the next set of application-attempt + * entities from the given fromId. The set of application-attempt + * entities retrieved is inclusive of specified fromId. fromId should + * be taken from the value associated with FROM_ID info key in + * entity response which was sent earlier. * * @return If successful, a HTTP 200(OK) response having a JSON representing a * set of TimelineEntity instances of the app-attempt @@ -2222,12 +2410,16 @@ public class TimelineReaderWebServices { @QueryParam("confstoretrieve") String confsToRetrieve, @QueryParam("metricstoretrieve") String metricsToRetrieve, @QueryParam("fields") String fields, - @QueryParam("metricslimit") String metricsLimit) { + @QueryParam("metricslimit") String metricsLimit, + @QueryParam("metricstimestart") String metricsTimeStart, + @QueryParam("metricstimeend") String metricsTimeEnd, + @QueryParam("fromid") String fromId) { return getAppAttempts(req, res, null, appId, userId, flowName, flowRunId, limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo, infofilters, conffilters, metricfilters, eventfilters, confsToRetrieve, - metricsToRetrieve, fields, metricsLimit); + metricsToRetrieve, fields, metricsLimit, metricsTimeStart, + metricsTimeEnd, fromId); } /** @@ -2299,6 +2491,17 @@ public class TimelineReaderWebServices { * have to be retrieved, then metricsLimit will be considered as 1 * i.e. latest single value of metric(s) will be returned. (Optional * query param). + * @param metricsTimeStart If specified, returned metrics for the app attempts + * would not contain metric values before this timestamp(Optional + * query param). + * @param metricsTimeEnd If specified, returned metrics for the app attempts + * would not contain metric values after this timestamp(Optional + * query param). + * @param fromId If specified, retrieve the next set of application-attempt + * entities from the given fromId. The set of application-attempt + * entities retrieved is inclusive of specified fromId. fromId should + * be taken from the value associated with FROM_ID info key in + * entity response which was sent earlier. * * @return If successful, a HTTP 200(OK) response having a JSON representing a * set of TimelineEntity instances of the app-attempts @@ -2332,13 +2535,17 @@ public class TimelineReaderWebServices { @QueryParam("confstoretrieve") String confsToRetrieve, @QueryParam("metricstoretrieve") String metricsToRetrieve, @QueryParam("fields") String fields, - @QueryParam("metricslimit") String metricsLimit) { + @QueryParam("metricslimit") String metricsLimit, + @QueryParam("metricstimestart") String metricsTimeStart, + @QueryParam("metricstimeend") String metricsTimeEnd, + @QueryParam("fromid") String fromId) { return getEntities(req, res, clusterId, appId, TimelineEntityType.YARN_APPLICATION_ATTEMPT.toString(), userId, flowName, flowRunId, limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo, infofilters, conffilters, metricfilters, eventfilters, - confsToRetrieve, metricsToRetrieve, fields, metricsLimit); + confsToRetrieve, metricsToRetrieve, fields, metricsLimit, + metricsTimeStart, metricsTimeEnd, fromId); } /** @@ -2381,6 +2588,14 @@ public class TimelineReaderWebServices { * have to be retrieved, then metricsLimit will be considered as 1 * i.e. latest single value of metric(s) will be returned. (Optional * query param). + * @param metricsTimeStart If specified, returned metrics for the app attempt + * would not contain metric values before this timestamp(Optional + * query param). + * @param metricsTimeEnd If specified, returned metrics for the app attempt + * would not contain metric values after this timestamp(Optional + * query param). + * @param entityIdPrefix Defines the id prefix for the entity to be fetched. + * If specified, then entity retrieval will be faster. * * @return If successful, a HTTP 200(OK) response having a JSON representing a * TimelineEntity instance is returned.
    @@ -2405,9 +2620,13 @@ public class TimelineReaderWebServices { @QueryParam("confstoretrieve") String confsToRetrieve, @QueryParam("metricstoretrieve") String metricsToRetrieve, @QueryParam("fields") String fields, - @QueryParam("metricslimit") String metricsLimit) { + @QueryParam("metricslimit") String metricsLimit, + @QueryParam("metricstimestart") String metricsTimeStart, + @QueryParam("metricstimeend") String metricsTimeEnd, + @QueryParam("entityidprefix") String entityIdPrefix) { return getAppAttempt(req, res, null, appId, appAttemptId, userId, flowName, - flowRunId, confsToRetrieve, metricsToRetrieve, fields, metricsLimit); + flowRunId, confsToRetrieve, metricsToRetrieve, fields, metricsLimit, + metricsTimeStart, metricsTimeEnd, entityIdPrefix); } /** @@ -2450,6 +2669,14 @@ public class TimelineReaderWebServices { * have to be retrieved, then metricsLimit will be considered as 1 * i.e. latest single value of metric(s) will be returned. (Optional * query param). + * @param metricsTimeStart If specified, returned metrics for the app attempt + * would not contain metric values before this timestamp(Optional + * query param). + * @param metricsTimeEnd If specified, returned metrics for the app attempt + * would not contain metric values after this timestamp(Optional + * query param). + * @param entityIdPrefix Defines the id prefix for the entity to be fetched. + * If specified, then entity retrieval will be faster. * * @return If successful, a HTTP 200(OK) response having a JSON representing a * TimelineEntity instance is returned.
    @@ -2476,11 +2703,14 @@ public class TimelineReaderWebServices { @QueryParam("confstoretrieve") String confsToRetrieve, @QueryParam("metricstoretrieve") String metricsToRetrieve, @QueryParam("fields") String fields, - @QueryParam("metricslimit") String metricsLimit) { + @QueryParam("metricslimit") String metricsLimit, + @QueryParam("metricstimestart") String metricsTimeStart, + @QueryParam("metricstimeend") String metricsTimeEnd, + @QueryParam("entityidprefix") String entityIdPrefix) { return getEntity(req, res, clusterId, appId, TimelineEntityType.YARN_APPLICATION_ATTEMPT.toString(), appAttemptId, userId, flowName, flowRunId, confsToRetrieve, metricsToRetrieve, fields, - metricsLimit); + metricsLimit, metricsTimeStart, metricsTimeEnd, entityIdPrefix); } /** @@ -2553,6 +2783,17 @@ public class TimelineReaderWebServices { * have to be retrieved, then metricsLimit will be considered as 1 * i.e. latest single value of metric(s) will be returned. (Optional * query param). + * @param metricsTimeStart If specified, returned metrics for the containers + * would not contain metric values before this timestamp(Optional + * query param). + * @param metricsTimeEnd If specified, returned metrics for the containers + * would not contain metric values after this timestamp(Optional + * query param). + * @param fromId If specified, retrieve the next set of container + * entities from the given fromId. The set of container + * entities retrieved is inclusive of specified fromId. fromId should + * be taken from the value associated with FROM_ID info key in + * entity response which was sent earlier. * * @return If successful, a HTTP 200(OK) response having a JSON representing a * set of TimelineEntity instances of the containers @@ -2586,11 +2827,15 @@ public class TimelineReaderWebServices { @QueryParam("confstoretrieve") String confsToRetrieve, @QueryParam("metricstoretrieve") String metricsToRetrieve, @QueryParam("fields") String fields, - @QueryParam("metricslimit") String metricsLimit) { + @QueryParam("metricslimit") String metricsLimit, + @QueryParam("metricstimestart") String metricsTimeStart, + @QueryParam("metricstimeend") String metricsTimeEnd, + @QueryParam("fromid") String fromId) { return getContainers(req, res, null, appId, appattemptId, userId, flowName, flowRunId, limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo, infofilters, conffilters, metricfilters, eventfilters, - confsToRetrieve, metricsToRetrieve, fields, metricsLimit); + confsToRetrieve, metricsToRetrieve, fields, metricsLimit, + metricsTimeStart, metricsTimeEnd, fromId); } /** @@ -2664,6 +2909,17 @@ public class TimelineReaderWebServices { * have to be retrieved, then metricsLimit will be considered as 1 * i.e. latest single value of metric(s) will be returned. (Optional * query param). + * @param metricsTimeStart If specified, returned metrics for the containers + * would not contain metric values before this timestamp(Optional + * query param). + * @param metricsTimeEnd If specified, returned metrics for the containers + * would not contain metric values after this timestamp(Optional + * query param). + * @param fromId If specified, retrieve the next set of container + * entities from the given fromId. The set of container + * entities retrieved is inclusive of specified fromId. fromId should + * be taken from the value associated with FROM_ID info key in + * entity response which was sent earlier. * * @return If successful, a HTTP 200(OK) response having a JSON representing a * set of TimelineEntity instances of the containers @@ -2699,7 +2955,10 @@ public class TimelineReaderWebServices { @QueryParam("confstoretrieve") String confsToRetrieve, @QueryParam("metricstoretrieve") String metricsToRetrieve, @QueryParam("fields") String fields, - @QueryParam("metricslimit") String metricsLimit) { + @QueryParam("metricslimit") String metricsLimit, + @QueryParam("metricstimestart") String metricsTimeStart, + @QueryParam("metricstimeend") String metricsTimeEnd, + @QueryParam("fromid") String fromId) { String entityType = TimelineEntityType.YARN_CONTAINER.toString(); String parentEntityType = @@ -2717,7 +2976,8 @@ public class TimelineReaderWebServices { return getEntities(req, res, clusterId, appId, entityType, userId, flowName, flowRunId, limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo, infofilter, conffilters, metricfilters, eventfilters, - confsToRetrieve, metricsToRetrieve, fields, metricsLimit); + confsToRetrieve, metricsToRetrieve, fields, metricsLimit, + metricsTimeStart, metricsTimeEnd, fromId); } /** @@ -2759,6 +3019,14 @@ public class TimelineReaderWebServices { * have to be retrieved, then metricsLimit will be considered as 1 * i.e. latest single value of metric(s) will be returned. (Optional * query param). + * @param metricsTimeStart If specified, returned metrics for the container + * would not contain metric values before this timestamp(Optional + * query param). + * @param metricsTimeEnd If specified, returned metrics for the container + * would not contain metric values after this timestamp(Optional + * query param). + * @param entityIdPrefix Defines the id prefix for the entity to be fetched. + * If specified, then entity retrieval will be faster. * * @return If successful, a HTTP 200(OK) response having a JSON representing * TimelineEntity instance is returned.
    @@ -2783,9 +3051,13 @@ public class TimelineReaderWebServices { @QueryParam("confstoretrieve") String confsToRetrieve, @QueryParam("metricstoretrieve") String metricsToRetrieve, @QueryParam("fields") String fields, - @QueryParam("metricslimit") String metricsLimit) { + @QueryParam("metricslimit") String metricsLimit, + @QueryParam("metricstimestart") String metricsTimeStart, + @QueryParam("metricstimeend") String metricsTimeEnd, + @QueryParam("entityidprefix") String entityIdPrefix) { return getContainer(req, res, null, appId, containerId, userId, flowName, - flowRunId, confsToRetrieve, metricsToRetrieve, fields, metricsLimit); + flowRunId, confsToRetrieve, metricsToRetrieve, fields, metricsLimit, + entityIdPrefix, metricsTimeStart, metricsTimeEnd); } /** @@ -2828,6 +3100,14 @@ public class TimelineReaderWebServices { * have to be retrieved, then metricsLimit will be considered as 1 * i.e. latest single value of metric(s) will be returned. (Optional * query param). + * @param metricsTimeStart If specified, returned metrics for the container + * would not contain metric values before this timestamp(Optional + * query param). + * @param metricsTimeEnd If specified, returned metrics for the container + * would not contain metric values after this timestamp(Optional + * query param). + * @param entityIdPrefix Defines the id prefix for the entity to be fetched. + * If specified, then entity retrieval will be faster. * * @return If successful, a HTTP 200(OK) response having a JSON representing a * TimelineEntity instance is returned.
    @@ -2854,10 +3134,273 @@ public class TimelineReaderWebServices { @QueryParam("confstoretrieve") String confsToRetrieve, @QueryParam("metricstoretrieve") String metricsToRetrieve, @QueryParam("fields") String fields, - @QueryParam("metricslimit") String metricsLimit) { + @QueryParam("metricslimit") String metricsLimit, + @QueryParam("metricstimestart") String metricsTimeStart, + @QueryParam("metricstimeend") String metricsTimeEnd, + @QueryParam("entityidprefix") String entityIdPrefix) { return getEntity(req, res, clusterId, appId, TimelineEntityType.YARN_CONTAINER.toString(), containerId, userId, flowName, flowRunId, confsToRetrieve, metricsToRetrieve, fields, - metricsLimit); + metricsLimit, metricsTimeStart, metricsTimeEnd, entityIdPrefix); } -} \ No newline at end of file + + /** + * Returns a set of available entity types for a given app id. Cluster ID is + * not provided by client so default cluster ID has to be taken. If userid, + * flow name and flow run id which are optional query parameters are not + * specified, they will be queried based on app id and cluster id from the + * flow context information stored in underlying storage implementation. + * + * @param req Servlet request. + * @param res Servlet response. + * @param appId Application id to be queried(Mandatory path param). + * @param flowName Flow name which should match for the app(Optional query + * param). + * @param flowRunId Run id which should match for the app(Optional query + * param). + * @param userId User id which should match for the app(Optional query param). + * + * @return If successful, a HTTP 200(OK) response having a JSON representing a + * list contains all timeline entity types is returned.
    + * On failures,
    + * If any problem occurs in parsing request, HTTP 400(Bad Request) is + * returned.
    + * If flow context information cannot be retrieved or app for the given + * app id cannot be found, HTTP 404(Not Found) is returned.
    + * For all other errors while retrieving data, HTTP 500(Internal Server + * Error) is returned. + */ + @GET + @Path("/apps/{appid}/entity-types") + @Produces(MediaType.APPLICATION_JSON) + public Set getEntityTypes( + @Context HttpServletRequest req, + @Context HttpServletResponse res, + @PathParam("appid") String appId, + @QueryParam("flowname") String flowName, + @QueryParam("flowrunid") String flowRunId, + @QueryParam("userid") String userId) { + return getEntityTypes(req, res, null, appId, flowName, flowRunId, userId); + } + + /** + * Returns a set of available entity types for a given app id. If userid, + * flow name and flow run id which are optional query parameters are not + * specified, they will be queried based on app id and cluster id from the + * flow context information stored in underlying storage implementation. + * + * @param req Servlet request. + * @param res Servlet response. + * @param clusterId Cluster id to which the app to be queried belong to( + * Mandatory path param). + * @param appId Application id to be queried(Mandatory path param). + * @param flowName Flow name which should match for the app(Optional query + * param). + * @param flowRunId Run id which should match for the app(Optional query + * param). + * @param userId User id which should match for the app(Optional query param). + * + * @return If successful, a HTTP 200(OK) response having a JSON representing a + * list contains all timeline entity types is returned.
    + * On failures,
    + * If any problem occurs in parsing request, HTTP 400(Bad Request) is + * returned.
    + * If flow context information cannot be retrieved or app for the given + * app id cannot be found, HTTP 404(Not Found) is returned.
    + * For all other errors while retrieving data, HTTP 500(Internal Server + * Error) is returned. + */ + @GET + @Path("/clusters/{clusterid}/apps/{appid}/entity-types") + @Produces(MediaType.APPLICATION_JSON) + public Set getEntityTypes( + @Context HttpServletRequest req, + @Context HttpServletResponse res, + @PathParam("clusterid") String clusterId, + @PathParam("appid") String appId, + @QueryParam("flowname") String flowName, + @QueryParam("flowrunid") String flowRunId, + @QueryParam("userid") String userId) { + String url = req.getRequestURI() + + (req.getQueryString() == null ? "" : + QUERY_STRING_SEP + req.getQueryString()); + UserGroupInformation callerUGI = + TimelineReaderWebServicesUtils.getUser(req); + LOG.info("Received URL " + url + " from user " + + TimelineReaderWebServicesUtils.getUserName(callerUGI)); + long startTime = Time.monotonicNow(); + init(res); + TimelineReaderManager timelineReaderManager = getTimelineReaderManager(); + Set results = null; + try { + results = timelineReaderManager.getEntityTypes( + TimelineReaderWebServicesUtils.createTimelineReaderContext( + clusterId, userId, flowName, flowRunId, appId, + null, null, null)); + } catch (Exception e) { + handleException(e, url, startTime, "flowrunid"); + } + long endTime = Time.monotonicNow(); + LOG.info("Processed URL " + url + + " (Took " + (endTime - startTime) + " ms.)"); + return results; + } + + @GET + @Path("/users/{userid}/entities/{entitytype}") + @Produces(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8) + public Set getSubAppEntities( + @Context HttpServletRequest req, + @Context HttpServletResponse res, + @PathParam("userid") String userId, + @PathParam("entitytype") String entityType, + @QueryParam("limit") String limit, + @QueryParam("createdtimestart") String createdTimeStart, + @QueryParam("createdtimeend") String createdTimeEnd, + @QueryParam("relatesto") String relatesTo, + @QueryParam("isrelatedto") String isRelatedTo, + @QueryParam("infofilters") String infofilters, + @QueryParam("conffilters") String conffilters, + @QueryParam("metricfilters") String metricfilters, + @QueryParam("eventfilters") String eventfilters, + @QueryParam("confstoretrieve") String confsToRetrieve, + @QueryParam("metricstoretrieve") String metricsToRetrieve, + @QueryParam("fields") String fields, + @QueryParam("metricslimit") String metricsLimit, + @QueryParam("metricstimestart") String metricsTimeStart, + @QueryParam("metricstimeend") String metricsTimeEnd, + @QueryParam("fromid") String fromId) { + return getSubAppEntities(req, res, null, userId, entityType, limit, + createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo, infofilters, + conffilters, metricfilters, eventfilters, confsToRetrieve, + metricsToRetrieve, fields, metricsLimit, metricsTimeStart, + metricsTimeEnd, fromId); + } + + @GET + @Path("/clusters/{clusterid}/users/{userid}/entities/{entitytype}") + @Produces(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8) + public Set getSubAppEntities( + @Context HttpServletRequest req, + @Context HttpServletResponse res, + @PathParam("clusterid") String clusterId, + @PathParam("userid") String userId, + @PathParam("entitytype") String entityType, + @QueryParam("limit") String limit, + @QueryParam("createdtimestart") String createdTimeStart, + @QueryParam("createdtimeend") String createdTimeEnd, + @QueryParam("relatesto") String relatesTo, + @QueryParam("isrelatedto") String isRelatedTo, + @QueryParam("infofilters") String infofilters, + @QueryParam("conffilters") String conffilters, + @QueryParam("metricfilters") String metricfilters, + @QueryParam("eventfilters") String eventfilters, + @QueryParam("confstoretrieve") String confsToRetrieve, + @QueryParam("metricstoretrieve") String metricsToRetrieve, + @QueryParam("fields") String fields, + @QueryParam("metricslimit") String metricsLimit, + @QueryParam("metricstimestart") String metricsTimeStart, + @QueryParam("metricstimeend") String metricsTimeEnd, + @QueryParam("fromid") String fromId) { + String url = req.getRequestURI() + + (req.getQueryString() == null ? "" : + QUERY_STRING_SEP + req.getQueryString()); + UserGroupInformation callerUGI = + TimelineReaderWebServicesUtils.getUser(req); + LOG.info("Received URL " + url + " from user " + + TimelineReaderWebServicesUtils.getUserName(callerUGI)); + long startTime = Time.monotonicNow(); + init(res); + TimelineReaderManager timelineReaderManager = getTimelineReaderManager(); + Set entities = null; + try { + TimelineReaderContext context = + TimelineReaderWebServicesUtils.createTimelineReaderContext(clusterId, + null, null, null, null, entityType, null, null, userId); + entities = timelineReaderManager.getEntities(context, + TimelineReaderWebServicesUtils.createTimelineEntityFilters( + limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo, + infofilters, conffilters, metricfilters, eventfilters, + fromId), + TimelineReaderWebServicesUtils.createTimelineDataToRetrieve( + confsToRetrieve, metricsToRetrieve, fields, metricsLimit, + metricsTimeStart, metricsTimeEnd)); + } catch (Exception e) { + handleException(e, url, startTime, + "createdTime start/end or limit"); + } + long endTime = Time.monotonicNow(); + if (entities == null) { + entities = Collections.emptySet(); + } + LOG.info("Processed URL " + url + + " (Took " + (endTime - startTime) + " ms.)"); + return entities; + } + + @GET + @Path("/users/{userid}/entities/{entitytype}/{entityid}") + @Produces(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8) + public Set getSubAppEntities(@Context HttpServletRequest req, + @Context HttpServletResponse res, @PathParam("userid") String userId, + @PathParam("entitytype") String entityType, + @PathParam("entityid") String entityId, + @QueryParam("confstoretrieve") String confsToRetrieve, + @QueryParam("metricstoretrieve") String metricsToRetrieve, + @QueryParam("fields") String fields, + @QueryParam("metricslimit") String metricsLimit, + @QueryParam("metricstimestart") String metricsTimeStart, + @QueryParam("metricstimeend") String metricsTimeEnd, + @QueryParam("entityidprefix") String entityIdPrefix) { + return getSubAppEntities(req, res, null, userId, entityType, entityId, + confsToRetrieve, metricsToRetrieve, fields, metricsLimit, + metricsTimeStart, metricsTimeEnd, entityIdPrefix); + } + + @GET + @Path("/clusters/{clusterid}/users/{userid}/entities/{entitytype}/{entityid}") + @Produces(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8) + public Set getSubAppEntities(@Context HttpServletRequest req, + @Context HttpServletResponse res, + @PathParam("clusterid") String clusterId, + @PathParam("userid") String userId, + @PathParam("entitytype") String entityType, + @PathParam("entityid") String entityId, + @QueryParam("confstoretrieve") String confsToRetrieve, + @QueryParam("metricstoretrieve") String metricsToRetrieve, + @QueryParam("fields") String fields, + @QueryParam("metricslimit") String metricsLimit, + @QueryParam("metricstimestart") String metricsTimeStart, + @QueryParam("metricstimeend") String metricsTimeEnd, + @QueryParam("entityidprefix") String entityIdPrefix) { + String url = req.getRequestURI() + (req.getQueryString() == null ? "" + : QUERY_STRING_SEP + req.getQueryString()); + UserGroupInformation callerUGI = + TimelineReaderWebServicesUtils.getUser(req); + LOG.info("Received URL " + url + " from user " + + TimelineReaderWebServicesUtils.getUserName(callerUGI)); + long startTime = Time.monotonicNow(); + init(res); + TimelineReaderManager timelineReaderManager = getTimelineReaderManager(); + Set entities = null; + try { + TimelineReaderContext context = TimelineReaderWebServicesUtils + .createTimelineReaderContext(clusterId, null, null, null, null, + entityType, entityIdPrefix, entityId, userId); + entities = timelineReaderManager.getEntities(context, + new TimelineEntityFilters.Builder().build(), + TimelineReaderWebServicesUtils.createTimelineDataToRetrieve( + confsToRetrieve, metricsToRetrieve, fields, metricsLimit, + metricsTimeStart, metricsTimeEnd)); + } catch (Exception e) { + handleException(e, url, startTime, ""); + } + long endTime = Time.monotonicNow(); + if (entities == null) { + entities = Collections.emptySet(); + } + LOG.info( + "Processed URL " + url + " (Took " + (endTime - startTime) + " ms.)"); + return entities; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java index 7fc8cb8483c..f83c1ac346f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java @@ -18,11 +18,13 @@ package org.apache.hadoop.yarn.server.timelineservice.reader; +import java.security.Principal; import java.util.EnumSet; import javax.servlet.http.HttpServletRequest; import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList; import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field; @@ -30,7 +32,7 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Fiel /** * Set of utility methods to be used by timeline reader web services. */ -final class TimelineReaderWebServicesUtils { +public final class TimelineReaderWebServicesUtils { private TimelineReaderWebServicesUtils() { } @@ -49,10 +51,20 @@ final class TimelineReaderWebServicesUtils { */ static TimelineReaderContext createTimelineReaderContext(String clusterId, String userId, String flowName, String flowRunId, String appId, - String entityType, String entityId) { + String entityType, String entityIdPrefix, String entityId) { return new TimelineReaderContext(parseStr(clusterId), parseStr(userId), parseStr(flowName), parseLongStr(flowRunId), parseStr(appId), - parseStr(entityType), parseStr(entityId)); + parseStr(entityType), parseLongStr(entityIdPrefix), parseStr(entityId)); + } + + static TimelineReaderContext createTimelineReaderContext(String clusterId, + String userId, String flowName, String flowRunId, String appId, + String entityType, String entityIdPrefix, String entityId, + String doAsUser) { + return new TimelineReaderContext(parseStr(clusterId), parseStr(userId), + parseStr(flowName), parseLongStr(flowRunId), parseStr(appId), + parseStr(entityType), parseLongStr(entityIdPrefix), parseStr(entityId), + parseStr(doAsUser)); } /** @@ -73,12 +85,46 @@ final class TimelineReaderWebServicesUtils { static TimelineEntityFilters createTimelineEntityFilters(String limit, String createdTimeStart, String createdTimeEnd, String relatesTo, String isRelatedTo, String infofilters, String conffilters, - String metricfilters, String eventfilters) throws TimelineParseException { - return new TimelineEntityFilters(parseLongStr(limit), - parseLongStr(createdTimeStart), parseLongStr(createdTimeEnd), - parseRelationFilters(relatesTo), parseRelationFilters(isRelatedTo), - parseKVFilters(infofilters, false), parseKVFilters(conffilters, true), - parseMetricFilters(metricfilters), parseEventFilters(eventfilters)); + String metricfilters, String eventfilters, + String fromid) throws TimelineParseException { + return createTimelineEntityFilters( + limit, parseLongStr(createdTimeStart), + parseLongStr(createdTimeEnd), + relatesTo, isRelatedTo, infofilters, + conffilters, metricfilters, eventfilters, fromid); + } + + /** + * Parse the passed filters represented as strings and convert them into a + * {@link TimelineEntityFilters} object. + * @param limit Limit to number of entities to return. + * @param createdTimeStart Created time start for the entities to return. + * @param createdTimeEnd Created time end for the entities to return. + * @param relatesTo Entities to return must match relatesTo. + * @param isRelatedTo Entities to return must match isRelatedTo. + * @param infofilters Entities to return must match these info filters. + * @param conffilters Entities to return must match these metric filters. + * @param metricfilters Entities to return must match these metric filters. + * @param eventfilters Entities to return must match these event filters. + * @return a {@link TimelineEntityFilters} object. + * @throws TimelineParseException if any problem occurs during parsing. + */ + static TimelineEntityFilters createTimelineEntityFilters(String limit, + Long createdTimeStart, Long createdTimeEnd, String relatesTo, + String isRelatedTo, String infofilters, String conffilters, + String metricfilters, String eventfilters, + String fromid) throws TimelineParseException { + return new TimelineEntityFilters.Builder() + .entityLimit(parseLongStr(limit)) + .createdTimeBegin(createdTimeStart) + .createTimeEnd(createdTimeEnd) + .relatesTo(parseRelationFilters(relatesTo)) + .isRelatedTo(parseRelationFilters(isRelatedTo)) + .infoFilters(parseKVFilters(infofilters, false)) + .configFilters(parseKVFilters(conffilters, true)) + .metricFilters(parseMetricFilters(metricfilters)) + .eventFilters(parseEventFilters(eventfilters)) + .fromId(parseStr(fromid)).build(); } /** @@ -92,11 +138,13 @@ final class TimelineReaderWebServicesUtils { * @throws TimelineParseException if any problem occurs during parsing. */ static TimelineDataToRetrieve createTimelineDataToRetrieve(String confs, - String metrics, String fields, String metricsLimit) + String metrics, String fields, String metricsLimit, + String metricsTimeBegin, String metricsTimeEnd) throws TimelineParseException { return new TimelineDataToRetrieve(parseDataToRetrieve(confs), parseDataToRetrieve(metrics), parseFieldsStr(fields, - TimelineParseConstants.COMMA_DELIMITER), parseIntStr(metricsLimit)); + TimelineParseConstants.COMMA_DELIMITER), parseIntStr(metricsLimit), + parseLongStr(metricsTimeBegin), parseLongStr(metricsTimeEnd)); } /** @@ -207,20 +255,40 @@ final class TimelineReaderWebServicesUtils { * @return trimmed string if string is not null, null otherwise. */ static String parseStr(String str) { - return str == null ? null : str.trim(); + return StringUtils.trimToNull(str); } /** - * Get UGI from HTTP request. + * Get UGI based on the remote user in the HTTP request. + * * @param req HTTP request. * @return UGI. */ - static UserGroupInformation getUser(HttpServletRequest req) { - String remoteUser = req.getRemoteUser(); + public static UserGroupInformation getUser(HttpServletRequest req) { + return getCallerUserGroupInformation(req, false); + } + + /** + * Get UGI from the HTTP request. + * + * @param hsr HTTP request. + * @param usePrincipal if true, use principal name else use remote user name + * @return UGI. + */ + public static UserGroupInformation getCallerUserGroupInformation( + HttpServletRequest hsr, boolean usePrincipal) { + + String remoteUser = hsr.getRemoteUser(); + if (usePrincipal) { + Principal princ = hsr.getUserPrincipal(); + remoteUser = princ == null ? null : princ.getName(); + } + UserGroupInformation callerUGI = null; if (remoteUser != null) { callerUGI = UserGroupInformation.createRemoteUser(remoteUser); } + return callerUGI; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineUIDConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineUIDConverter.java index 08e5405ff52..b875828719b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineUIDConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineUIDConverter.java @@ -137,6 +137,41 @@ enum TimelineUIDConverter { } }, + // Sub Application Entity UID should contain cluster, user, entity type and + // entity id + SUB_APPLICATION_ENTITY_UID { + @Override + String encodeUID(TimelineReaderContext context) { + if (context == null) { + return null; + } + if (context.getClusterId() == null || context.getDoAsUser() == null + || context.getEntityType() == null || context.getEntityId() == null) { + return null; + } + String[] entityTupleArr = {context.getClusterId(), context.getDoAsUser(), + context.getEntityType(), context.getEntityIdPrefix().toString(), + context.getEntityId()}; + return joinAndEscapeUIDParts(entityTupleArr); + } + + @Override + TimelineReaderContext decodeUID(String uId) throws Exception { + if (uId == null) { + return null; + } + List entityTupleList = splitUID(uId); + if (entityTupleList.size() == 5) { + // Flow information exists. + return new TimelineReaderContext(entityTupleList.get(0), null, null, + null, null, entityTupleList.get(2), + Long.parseLong(entityTupleList.get(3)), entityTupleList.get(4), + entityTupleList.get(1)); + } + return null; + } + }, + // Generic Entity UID should contain cluster, user, flow name, flowrun id, // app id, entity type and entity id OR should contain cluster, appid, entity // type and entity id(i.e.without flow context info). @@ -155,12 +190,14 @@ enum TimelineUIDConverter { // Flow information exists. String[] entityTupleArr = {context.getClusterId(), context.getUserId(), context.getFlowName(), context.getFlowRunId().toString(), - context.getAppId(), context.getEntityType(), context.getEntityId()}; + context.getAppId(), context.getEntityType(), + context.getEntityIdPrefix().toString(), context.getEntityId() }; return joinAndEscapeUIDParts(entityTupleArr); } else { // Only entity and app information exists. Flow info does not exist. String[] entityTupleArr = {context.getClusterId(), context.getAppId(), - context.getEntityType(), context.getEntityId()}; + context.getEntityType(), context.getEntityIdPrefix().toString(), + context.getEntityId() }; return joinAndEscapeUIDParts(entityTupleArr); } } @@ -171,20 +208,21 @@ enum TimelineUIDConverter { return null; } List entityTupleList = splitUID(uId); - // Should have 7 parts i.e. cluster, user, flow name, flowrun id, app id, - // entity type and entity id OR should have 4 parts i.e. cluster, app id, + // Should have 8 parts i.e. cluster, user, flow name, flowrun id, app id, + // entity type and entity id OR should have 5 parts i.e. cluster, app id, // entity type and entity id. - if (entityTupleList.size() == 7) { + if (entityTupleList.size() == 8) { // Flow information exists. return new TimelineReaderContext(entityTupleList.get(0), entityTupleList.get(1), entityTupleList.get(2), Long.parseLong(entityTupleList.get(3)), entityTupleList.get(4), - entityTupleList.get(5), entityTupleList.get(6)); - } else if (entityTupleList.size() == 4) { + entityTupleList.get(5), Long.parseLong(entityTupleList.get(6)), + entityTupleList.get(7)); + } else if (entityTupleList.size() == 5) { // Flow information does not exist. return new TimelineReaderContext(entityTupleList.get(0), null, null, null, entityTupleList.get(1), entityTupleList.get(2), - entityTupleList.get(3)); + Long.parseLong(entityTupleList.get(3)), entityTupleList.get(4)); } else { return null; } @@ -192,39 +230,29 @@ enum TimelineUIDConverter { }; /** - * Delimiter used for UID. - */ - public static final char UID_DELIMITER_CHAR = '!'; - - /** - * Escape Character used if delimiter or escape character itself is part of - * different components of UID. - */ - public static final char UID_ESCAPE_CHAR = '*'; - - /** - * Split UID using {@link #UID_DELIMITER_CHAR} and {@link #UID_ESCAPE_CHAR}. + * Split UID using {@link TimelineReaderUtils#DEFAULT_DELIMITER_CHAR} and + * {@link TimelineReaderUtils#DEFAULT_ESCAPE_CHAR}. * @param uid UID to be splitted. * @return a list of different parts of UID split across delimiter. * @throws IllegalArgumentException if UID is not properly escaped. */ private static List splitUID(String uid) throws IllegalArgumentException { - return TimelineReaderUtils.split(uid, UID_DELIMITER_CHAR, UID_ESCAPE_CHAR); + return TimelineReaderUtils.split(uid); } /** - * Join different parts of UID delimited by {@link #UID_DELIMITER_CHAR} with - * delimiter and escape character escaped using {@link #UID_ESCAPE_CHAR} if - * UID parts contain them. + * Join different parts of UID delimited by + * {@link TimelineReaderUtils#DEFAULT_DELIMITER_CHAR} with delimiter and + * escape character escaped using + * {@link TimelineReaderUtils#DEFAULT_ESCAPE_CHAR} if UID parts contain them. * @param parts an array of UID parts to be joined. * @return a string joined using the delimiter with escape and delimiter - * characters escaped if they are part of the string parts to be joined. - * Returns null if one of the parts is null. + * characters escaped if they are part of the string parts to be + * joined. Returns null if one of the parts is null. */ private static String joinAndEscapeUIDParts(String[] parts) { - return TimelineReaderUtils.joinAndEscapeStrings(parts, UID_DELIMITER_CHAR, - UID_ESCAPE_CHAR); + return TimelineReaderUtils.joinAndEscapeStrings(parts); } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/security/TimelineReaderAuthenticationFilterInitializer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/security/TimelineReaderAuthenticationFilterInitializer.java new file mode 100644 index 00000000000..e0e1f4d9a13 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/security/TimelineReaderAuthenticationFilterInitializer.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.timelineservice.reader.security; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.http.FilterContainer; +import org.apache.hadoop.security.AuthenticationWithProxyUserFilter; +import org.apache.hadoop.yarn.server.timeline.security.TimelineAuthenticationFilterInitializer; + +/** + * Filter initializer to initialize {@link AuthenticationWithProxyUserFilter} + * for ATSv2 timeline reader server with timeline service specific + * configurations. + */ +public class TimelineReaderAuthenticationFilterInitializer extends + TimelineAuthenticationFilterInitializer{ + + /** + * Initializes {@link AuthenticationWithProxyUserFilter} + *

    + * Propagates to {@link AuthenticationWithProxyUserFilter} configuration all + * YARN configuration properties prefixed with + * {@value TimelineAuthenticationFilterInitializer#PREFIX}. + * + * @param container + * The filter container + * @param conf + * Configuration for run-time parameters + */ + @Override + public void initFilter(FilterContainer container, Configuration conf) { + setAuthFilterConfig(conf); + container.addGlobalFilter("Timeline Reader Authentication Filter", + AuthenticationWithProxyUserFilter.class.getName(), + getFilterConfig()); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/security/TimelineReaderWhitelistAuthorizationFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/security/TimelineReaderWhitelistAuthorizationFilter.java new file mode 100644 index 00000000000..b22ea3faa04 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/security/TimelineReaderWhitelistAuthorizationFilter.java @@ -0,0 +1,123 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.timelineservice.reader.security; + +import java.io.IOException; + +import javax.servlet.Filter; +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.Response.Status; + +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authorize.AccessControlList; +import org.apache.hadoop.security.authorize.AuthorizationException; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.webapp.ForbiddenException; +import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderWebServicesUtils; + +/** + * Filter to check if a particular user is allowed to read ATSv2 data. + */ + +public class TimelineReaderWhitelistAuthorizationFilter implements Filter { + + public static final String EMPTY_STRING = ""; + + private static final Log LOG = + LogFactory.getLog(TimelineReaderWhitelistAuthorizationFilter.class); + + private boolean isWhitelistReadAuthEnabled = false; + + private AccessControlList allowedUsersAclList; + private AccessControlList adminAclList; + + @Override + public void destroy() { + // NOTHING + } + + @Override + public void doFilter(ServletRequest request, ServletResponse response, + FilterChain chain) throws IOException, ServletException { + if (isWhitelistReadAuthEnabled) { + UserGroupInformation callerUGI = TimelineReaderWebServicesUtils + .getCallerUserGroupInformation((HttpServletRequest) request, true); + if (callerUGI == null) { + String msg = "Unable to obtain user name, user not authenticated"; + throw new AuthorizationException(msg); + } + if (!(adminAclList.isUserAllowed(callerUGI) + || allowedUsersAclList.isUserAllowed(callerUGI))) { + String userName = callerUGI.getShortUserName(); + String msg = "User " + userName + + " is not allowed to read TimelineService V2 data."; + Response.status(Status.FORBIDDEN).entity(msg).build(); + throw new ForbiddenException("user " + userName + + " is not allowed to read TimelineService V2 data"); + } + } + if (chain != null) { + chain.doFilter(request, response); + } + } + + @Override + public void init(FilterConfig conf) throws ServletException { + String isWhitelistReadAuthEnabledStr = conf + .getInitParameter(YarnConfiguration.TIMELINE_SERVICE_READ_AUTH_ENABLED); + if (isWhitelistReadAuthEnabledStr == null) { + isWhitelistReadAuthEnabled = + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_READ_AUTH_ENABLED; + } else { + isWhitelistReadAuthEnabled = + Boolean.valueOf(isWhitelistReadAuthEnabledStr); + } + + if (isWhitelistReadAuthEnabled) { + String listAllowedUsers = conf.getInitParameter( + YarnConfiguration.TIMELINE_SERVICE_READ_ALLOWED_USERS); + if (StringUtils.isEmpty(listAllowedUsers)) { + listAllowedUsers = + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_READ_ALLOWED_USERS; + } + LOG.info("listAllowedUsers=" + listAllowedUsers); + allowedUsersAclList = new AccessControlList(listAllowedUsers); + LOG.info("allowedUsersAclList=" + allowedUsersAclList.getUsers()); + // also allow admins + String adminAclListStr = + conf.getInitParameter(YarnConfiguration.YARN_ADMIN_ACL); + if (StringUtils.isEmpty(adminAclListStr)) { + adminAclListStr = + TimelineReaderWhitelistAuthorizationFilter.EMPTY_STRING; + LOG.info("adminAclList not set, hence setting it to \"\""); + } + adminAclList = new AccessControlList(adminAclListStr); + LOG.info("adminAclList=" + adminAclList.getUsers()); + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/security/TimelineReaderWhitelistAuthorizationFilterInitializer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/security/TimelineReaderWhitelistAuthorizationFilterInitializer.java new file mode 100644 index 00000000000..a970731272a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/security/TimelineReaderWhitelistAuthorizationFilterInitializer.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.timelineservice.reader.security; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.http.FilterContainer; +import org.apache.hadoop.http.FilterInitializer; +import org.apache.hadoop.yarn.conf.YarnConfiguration; + +/** + * Filter initializer to initialize + * {@link TimelineReaderWhitelistAuthorizationFilter} for ATSv2 timeline reader + * with timeline service specific configurations. + */ +public class TimelineReaderWhitelistAuthorizationFilterInitializer + extends FilterInitializer { + + /** + * Initializes {@link TimelineReaderWhitelistAuthorizationFilter}. + * + * @param container The filter container + * @param conf Configuration for run-time parameters + */ + @Override + public void initFilter(FilterContainer container, Configuration conf) { + Map params = new HashMap(); + String isWhitelistReadAuthEnabled = Boolean.toString( + conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_READ_AUTH_ENABLED, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_READ_AUTH_ENABLED)); + params.put(YarnConfiguration.TIMELINE_SERVICE_READ_AUTH_ENABLED, + isWhitelistReadAuthEnabled); + params.put(YarnConfiguration.TIMELINE_SERVICE_READ_ALLOWED_USERS, + conf.get(YarnConfiguration.TIMELINE_SERVICE_READ_ALLOWED_USERS, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_READ_ALLOWED_USERS)); + + params.put(YarnConfiguration.YARN_ADMIN_ACL, + conf.get(YarnConfiguration.YARN_ADMIN_ACL, + // using a default of "" + // instead of DEFAULT_YARN_ADMIN_ACL + // The reason being, DEFAULT_YARN_ADMIN_ACL is set to all users + // and we do not wish to allow everyone by default if + // read auth is enabled and YARN_ADMIN_ACL is unset + TimelineReaderWhitelistAuthorizationFilter.EMPTY_STRING)); + container.addGlobalFilter("Timeline Reader Whitelist Authorization Filter", + TimelineReaderWhitelistAuthorizationFilter.class.getName(), params); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/security/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/security/package-info.java new file mode 100644 index 00000000000..5888c989548 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/security/package-info.java @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Package org.apache.hadoop.server.timelineservice.reader.security contains + * classes to be used to support SPNEGO authentication for timeline reader. + */ +@InterfaceAudience.Private +package org.apache.hadoop.yarn.server.timelineservice.reader.security; +import org.apache.hadoop.classification.InterfaceAudience; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/security/CollectorNodemanagerSecurityInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/security/CollectorNodemanagerSecurityInfo.java new file mode 100644 index 00000000000..0eb5ee5aa52 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/security/CollectorNodemanagerSecurityInfo.java @@ -0,0 +1,69 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.server.timelineservice.security; + +import java.lang.annotation.Annotation; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Evolving; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.KerberosInfo; +import org.apache.hadoop.security.SecurityInfo; +import org.apache.hadoop.security.token.TokenInfo; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocolPB; + +/** + * SecurityInfo implementation for CollectorNodemanager protocol. + */ +@Public +@Evolving +public class CollectorNodemanagerSecurityInfo extends SecurityInfo { + + @Override + public KerberosInfo getKerberosInfo(Class protocol, Configuration conf) { + if (!protocol + .equals(CollectorNodemanagerProtocolPB.class)) { + return null; + } + return new KerberosInfo() { + + @Override + public Class annotationType() { + return null; + } + + @Override + public String serverPrincipal() { + return YarnConfiguration.NM_PRINCIPAL; + } + + @Override + public String clientPrincipal() { + return null; + } + }; + } + + @Override + public TokenInfo getTokenInfo(Class protocol, Configuration conf) { + return null; + } +} + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/security/TimelineV2DelegationTokenSecretManagerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/security/TimelineV2DelegationTokenSecretManagerService.java new file mode 100644 index 00000000000..de5ccdc89d4 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/security/TimelineV2DelegationTokenSecretManagerService.java @@ -0,0 +1,126 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.timelineservice.security; + +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager; +import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier; +import org.apache.hadoop.yarn.server.timeline.security.TimelineDelgationTokenSecretManagerService; + +/** + * The service wrapper of {@link TimelineV2DelegationTokenSecretManager}. + */ +public class TimelineV2DelegationTokenSecretManagerService extends + TimelineDelgationTokenSecretManagerService { + + public TimelineV2DelegationTokenSecretManagerService() { + super(TimelineV2DelegationTokenSecretManagerService.class.getName()); + } + + @Override + protected AbstractDelegationTokenSecretManager + + createTimelineDelegationTokenSecretManager(long secretKeyInterval, + long tokenMaxLifetime, long tokenRenewInterval, + long tokenRemovalScanInterval) { + return new TimelineV2DelegationTokenSecretManager(secretKeyInterval, + tokenMaxLifetime, tokenRenewInterval, tokenRemovalScanInterval); + } + + public Token generateToken( + UserGroupInformation ugi, String renewer) { + return ((TimelineV2DelegationTokenSecretManager) + getTimelineDelegationTokenSecretManager()).generateToken(ugi, renewer); + } + + public long renewToken(Token token, + String renewer) throws IOException { + return getTimelineDelegationTokenSecretManager().renewToken(token, renewer); + } + + public void cancelToken(Token token, + String canceller) throws IOException { + getTimelineDelegationTokenSecretManager().cancelToken(token, canceller); + } + + /** + * Delegation token secret manager for ATSv2. + */ + @Private + @Unstable + public static class TimelineV2DelegationTokenSecretManager extends + AbstractDelegationTokenSecretManager { + + private static final Log LOG = + LogFactory.getLog(TimelineV2DelegationTokenSecretManager.class); + + /** + * Create a timeline v2 secret manager. + * @param delegationKeyUpdateInterval the number of milliseconds for rolling + * new secret keys. + * @param delegationTokenMaxLifetime the maximum lifetime of the delegation + * tokens in milliseconds + * @param delegationTokenRenewInterval how often the tokens must be renewed + * in milliseconds + * @param delegationTokenRemoverScanInterval how often the tokens are + * scanned for expired tokens in milliseconds + */ + public TimelineV2DelegationTokenSecretManager( + long delegationKeyUpdateInterval, long delegationTokenMaxLifetime, + long delegationTokenRenewInterval, + long delegationTokenRemoverScanInterval) { + super(delegationKeyUpdateInterval, delegationTokenMaxLifetime, + delegationTokenRenewInterval, delegationTokenRemoverScanInterval); + } + + public Token generateToken( + UserGroupInformation ugi, String renewer) { + Text realUser = null; + if (ugi.getRealUser() != null) { + realUser = new Text(ugi.getRealUser().getUserName()); + } + TimelineDelegationTokenIdentifier identifier = createIdentifier(); + identifier.setOwner(new Text(ugi.getUserName())); + identifier.setRenewer(new Text(renewer)); + identifier.setRealUser(realUser); + byte[] password = createPassword(identifier); + return new Token(identifier.getBytes(), + password, identifier.getKind(), null); + } + + @Override + public TimelineDelegationTokenIdentifier createIdentifier() { + return new TimelineDelegationTokenIdentifier(); + } + + @Override + protected void logExpireToken(TimelineDelegationTokenIdentifier ident) + throws IOException { + LOG.info("Token " + ident + " expired."); + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/security/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/security/package-info.java new file mode 100644 index 00000000000..825009204c0 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/security/package-info.java @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Package org.apache.hadoop.server.timelineservice.security contains classes + * to be used to generate delegation tokens for ATSv2. + */ +@InterfaceAudience.Private +package org.apache.hadoop.yarn.server.timelineservice.security; +import org.apache.hadoop.classification.InterfaceAudience; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java index b4e792b0ea5..adb8821c3f0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java @@ -32,6 +32,7 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Set; import java.util.TreeMap; +import java.util.TreeSet; import com.fasterxml.jackson.core.JsonGenerationException; import com.fasterxml.jackson.databind.JsonMappingException; @@ -408,4 +409,24 @@ public class FileSystemTimelineReaderImpl extends AbstractService context.getEntityType()); return getEntities(dir, context.getEntityType(), filters, dataToRetrieve); } + + @Override public Set getEntityTypes(TimelineReaderContext context) + throws IOException { + Set result = new TreeSet<>(); + String flowRunPath = getFlowRunPath(context.getUserId(), + context.getClusterId(), context.getFlowName(), context.getFlowRunId(), + context.getAppId()); + File dir = new File(new File(rootPath, ENTITIES_DIR), + context.getClusterId() + File.separator + flowRunPath + + File.separator + context.getAppId()); + File[] fileList = dir.listFiles(); + if (fileList != null) { + for (File f : fileList) { + if (f.isDirectory()) { + result.add(f.getName()); + } + } + } + return result; + } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java index 1f527f24508..ee4197000bf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java @@ -28,12 +28,14 @@ import java.io.PrintWriter; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse.TimelineWriteError; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorContext; import org.apache.hadoop.yarn.util.timeline.TimelineUtils; import com.google.common.annotations.VisibleForTesting; @@ -68,10 +70,17 @@ public class FileSystemTimelineWriterImpl extends AbstractService } @Override - public TimelineWriteResponse write(String clusterId, String userId, - String flowName, String flowVersion, long flowRunId, String appId, - TimelineEntities entities) throws IOException { + public TimelineWriteResponse write(TimelineCollectorContext context, + TimelineEntities entities, UserGroupInformation callerUgi) + throws IOException { TimelineWriteResponse response = new TimelineWriteResponse(); + String clusterId = context.getClusterId(); + String userId = context.getUserId(); + String flowName = context.getFlowName(); + String flowVersion = context.getFlowVersion(); + long flowRunId = context.getFlowRunId(); + String appId = context.getAppId(); + for (TimelineEntity entity : entities.getEntities()) { write(clusterId, userId, flowName, flowVersion, flowRunId, appId, entity, response); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java index e8eabf161a9..16d623ab46f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java @@ -125,8 +125,8 @@ public interface TimelineReader extends Service { *

  • flowRunId - Context flow run id.
    • *
  • appId - Context app id.
    • * - * Although entityId is also part of context, it has no meaning for - * getEntities.
    + * Although entityIdPrefix and entityId are also part of context, + * it has no meaning for getEntities.
    * Fields in context which are mandatory depends on entity type. Entity * type is always mandatory. In addition to entity type, below is the list * of context fields which are mandatory, based on entity type.
    @@ -161,8 +161,10 @@ public interface TimelineReader extends Service { * {@link TimelineDataToRetrieve} for details. * @return A set of TimelineEntity instances of the given entity * type in the given context scope which matches the given predicates - * ordered by created time, descending. Each entity will only contain the - * metadata(id, type and created time) plus the given fields to retrieve. + * ordered by enitityIdPrefix(for generic entities only). + * Each entity will only contain + * the metadata(id, type , idPrefix and created time) plus the given + * fields to retrieve. *
    * If entityType is YARN_FLOW_ACTIVITY, entities returned are of type * FlowActivityEntity.
    @@ -177,4 +179,17 @@ public interface TimelineReader extends Service { TimelineReaderContext context, TimelineEntityFilters filters, TimelineDataToRetrieve dataToRetrieve) throws IOException; + + /** + * The API to list all available entity types of the given context. + * + * @param context A context defines the scope of this query. The incoming + * context should contain at least the cluster id and application id. + * + * @return A set of entity types available in the given context. + * + * @throws IOException if an exception occurred while listing from backend + * storage. + */ + Set getEntityTypes(TimelineReaderContext context) throws IOException; } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriter.java index 663a18a495b..12bc1cb3f0e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriter.java @@ -21,10 +21,12 @@ import java.io.IOException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.service.Service; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse; +import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorContext; /** * This interface is for storing application timeline information. @@ -34,25 +36,19 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse; public interface TimelineWriter extends Service { /** - * Stores the entire information in {@link TimelineEntities} to the - * timeline store. Any errors occurring for individual write request objects - * will be reported in the response. + * Stores the entire information in {@link TimelineEntities} to the timeline + * store. Any errors occurring for individual write request objects will be + * reported in the response. * - * @param clusterId context cluster ID - * @param userId context user ID - * @param flowName context flow name - * @param flowVersion context flow version - * @param flowRunId run id for the flow. - * @param appId context app ID. - * @param data - * a {@link TimelineEntities} object. + * @param context a {@link TimelineCollectorContext} + * @param data a {@link TimelineEntities} object. + * @param callerUgi {@link UserGroupInformation}. * @return a {@link TimelineWriteResponse} object. - * @throws IOException if there is any exception encountered while storing - * or writing entities to the backend storage. + * @throws IOException if there is any exception encountered while storing or + * writing entities to the back end storage. */ - TimelineWriteResponse write(String clusterId, String userId, - String flowName, String flowVersion, long flowRunId, String appId, - TimelineEntities data) throws IOException; + TimelineWriteResponse write(TimelineCollectorContext context, + TimelineEntities data, UserGroupInformation callerUgi) throws IOException; /** * Aggregates the entity information to the timeline store based on which diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineStorageUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineStorageUtils.java index 7f7d6405ae9..203d950fa81 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineStorageUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineStorageUtils.java @@ -371,5 +371,4 @@ public final class TimelineStorageUtils { return (obj instanceof Short) || (obj instanceof Integer) || (obj instanceof Long); } - } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo new file mode 100644 index 00000000000..4389219ab74 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo @@ -0,0 +1,14 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +org.apache.hadoop.yarn.server.timelineservice.security.CollectorNodemanagerSecurityInfo diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestNMTimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestNMTimelineCollectorManager.java index 7bc89c582c1..af9acce265e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestNMTimelineCollectorManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestNMTimelineCollectorManager.java @@ -95,7 +95,7 @@ public class TestNMTimelineCollectorManager { Callable task = new Callable() { public Boolean call() { AppLevelTimelineCollector collector = - new AppLevelTimelineCollector(appId); + new AppLevelTimelineCollectorWithAgg(appId, "user"); return (collectorManager.putIfAbsent(appId, collector) == collector); } }; @@ -126,7 +126,7 @@ public class TestNMTimelineCollectorManager { Callable task = new Callable() { public Boolean call() { AppLevelTimelineCollector collector = - new AppLevelTimelineCollector(appId); + new AppLevelTimelineCollectorWithAgg(appId, "user"); boolean successPut = (collectorManager.putIfAbsent(appId, collector) == collector); return successPut && collectorManager.remove(appId); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollector.java index 0f175537b23..ec454284dab 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollector.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollector.java @@ -41,8 +41,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyLong; -import static org.mockito.Matchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; @@ -156,9 +154,8 @@ public class TestTimelineCollector { collector.putEntities( entities, UserGroupInformation.createRemoteUser("test-user")); - verify(writer, times(1)).write( - anyString(), anyString(), anyString(), anyString(), anyLong(), - anyString(), any(TimelineEntities.class)); + verify(writer, times(1)).write(any(TimelineCollectorContext.class), + any(TimelineEntities.class), any(UserGroupInformation.class)); verify(writer, times(1)).flush(); } @@ -175,9 +172,8 @@ public class TestTimelineCollector { collector.putEntitiesAsync( entities, UserGroupInformation.createRemoteUser("test-user")); - verify(writer, times(1)).write( - anyString(), anyString(), anyString(), anyString(), anyLong(), - anyString(), any(TimelineEntities.class)); + verify(writer, times(1)).write(any(TimelineCollectorContext.class), + any(TimelineEntities.class), any(UserGroupInformation.class)); verify(writer, never()).flush(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java index 4bd37f8321a..f760834bd57 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java @@ -238,7 +238,7 @@ public class TestTimelineReaderWebServices { assertEquals(3, entity.getConfigs().size()); assertEquals(3, entity.getMetrics().size()); assertTrue("UID should be present", - entity.getInfo().containsKey(TimelineReaderManager.UID_KEY)); + entity.getInfo().containsKey(TimelineReaderUtils.UID_KEY)); // Includes UID. assertEquals(3, entity.getInfo().size()); // No events will be returned as events are not part of fields. @@ -265,7 +265,7 @@ public class TestTimelineReaderWebServices { assertEquals(3, entity.getConfigs().size()); assertEquals(3, entity.getMetrics().size()); assertTrue("UID should be present", - entity.getInfo().containsKey(TimelineReaderManager.UID_KEY)); + entity.getInfo().containsKey(TimelineReaderUtils.UID_KEY)); // Includes UID. assertEquals(3, entity.getInfo().size()); assertEquals(2, entity.getEvents().size()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWhitelistAuthorizationFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWhitelistAuthorizationFilter.java new file mode 100644 index 00000000000..bd4f0c2ebf9 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWhitelistAuthorizationFilter.java @@ -0,0 +1,380 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.timelineservice.reader; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.security.Principal; +import java.security.PrivilegedExceptionAction; +import java.util.Collections; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.Map; + +import javax.servlet.FilterConfig; +import javax.servlet.ServletContext; +import javax.servlet.ServletException; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletRequest; + +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.timelineservice.reader.security.TimelineReaderWhitelistAuthorizationFilter; +import org.apache.hadoop.yarn.webapp.ForbiddenException; +import org.junit.Test; +import org.mockito.Mockito; + +/** + * Unit tests for {@link TimelineReaderWhitelistAuthorizationFilter}. + * + */ +public class TestTimelineReaderWhitelistAuthorizationFilter { + + final private static String GROUP1_NAME = "group1"; + final private static String GROUP2_NAME = "group2"; + final private static String GROUP3_NAME = "group3"; + final private static String[] GROUP_NAMES = + new String[] {GROUP1_NAME, GROUP2_NAME, GROUP3_NAME}; + + private static class DummyFilterConfig implements FilterConfig { + final private Map map; + + DummyFilterConfig(Map map) { + this.map = map; + } + + @Override + public String getFilterName() { + return "dummy"; + } + + @Override + public String getInitParameter(String arg0) { + return map.get(arg0); + } + + @Override + public Enumeration getInitParameterNames() { + return Collections.enumeration(map.keySet()); + } + + @Override + public ServletContext getServletContext() { + return null; + } + } + + @Test + public void checkFilterAllowedUser() throws ServletException, IOException { + Map map = new HashMap(); + map.put(YarnConfiguration.TIMELINE_SERVICE_READ_AUTH_ENABLED, "true"); + map.put(YarnConfiguration.TIMELINE_SERVICE_READ_ALLOWED_USERS, + "user1,user2"); + TimelineReaderWhitelistAuthorizationFilter f = + new TimelineReaderWhitelistAuthorizationFilter(); + FilterConfig fc = new DummyFilterConfig(map); + f.init(fc); + HttpServletRequest mockHsr = Mockito.mock(HttpServletRequest.class); + Mockito.when(mockHsr.getUserPrincipal()).thenReturn(new Principal() { + @Override + public String getName() { + return "user1"; + } + }); + + ServletResponse r = Mockito.mock(ServletResponse.class); + f.doFilter(mockHsr, r, null); + } + + @Test(expected = ForbiddenException.class) + public void checkFilterNotAllowedUser() throws ServletException, IOException { + Map map = new HashMap(); + map.put(YarnConfiguration.TIMELINE_SERVICE_READ_AUTH_ENABLED, "true"); + map.put(YarnConfiguration.TIMELINE_SERVICE_READ_ALLOWED_USERS, + "user1,user2"); + TimelineReaderWhitelistAuthorizationFilter f = + new TimelineReaderWhitelistAuthorizationFilter(); + FilterConfig fc = new DummyFilterConfig(map); + f.init(fc); + HttpServletRequest mockHsr = Mockito.mock(HttpServletRequest.class); + Mockito.when(mockHsr.getUserPrincipal()).thenReturn(new Principal() { + @Override + public String getName() { + return "testuser1"; + } + }); + ServletResponse r = Mockito.mock(ServletResponse.class); + f.doFilter(mockHsr, r, null); + } + + @Test + public void checkFilterAllowedUserGroup() + throws ServletException, IOException, InterruptedException { + Map map = new HashMap(); + map.put(YarnConfiguration.TIMELINE_SERVICE_READ_AUTH_ENABLED, "true"); + map.put(YarnConfiguration.TIMELINE_SERVICE_READ_ALLOWED_USERS, + "user2 group1,group2"); + TimelineReaderWhitelistAuthorizationFilter f = + new TimelineReaderWhitelistAuthorizationFilter(); + FilterConfig fc = new DummyFilterConfig(map); + f.init(fc); + HttpServletRequest mockHsr = Mockito.mock(HttpServletRequest.class); + Mockito.when(mockHsr.getUserPrincipal()).thenReturn(new Principal() { + @Override + public String getName() { + return "user1"; + } + }); + ServletResponse r = Mockito.mock(ServletResponse.class); + UserGroupInformation user1 = + UserGroupInformation.createUserForTesting("user1", GROUP_NAMES); + user1.doAs(new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + f.doFilter(mockHsr, r, null); + return null; + } + }); + } + + @Test(expected = ForbiddenException.class) + public void checkFilterNotAlloweGroup() + throws ServletException, IOException, InterruptedException { + Map map = new HashMap(); + map.put(YarnConfiguration.TIMELINE_SERVICE_READ_AUTH_ENABLED, "true"); + map.put(YarnConfiguration.TIMELINE_SERVICE_READ_ALLOWED_USERS, + " group5,group6"); + TimelineReaderWhitelistAuthorizationFilter f = + new TimelineReaderWhitelistAuthorizationFilter(); + FilterConfig fc = new DummyFilterConfig(map); + f.init(fc); + HttpServletRequest mockHsr = Mockito.mock(HttpServletRequest.class); + Mockito.when(mockHsr.getUserPrincipal()).thenReturn(new Principal() { + @Override + public String getName() { + return "user200"; + } + }); + ServletResponse r = Mockito.mock(ServletResponse.class); + UserGroupInformation user1 = + UserGroupInformation.createUserForTesting("user200", GROUP_NAMES); + user1.doAs(new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + f.doFilter(mockHsr, r, null); + return null; + } + }); + } + + @Test + public void checkFilterAllowAdmins() + throws ServletException, IOException, InterruptedException { + // check that users in admin acl list are allowed to read + Map map = new HashMap(); + map.put(YarnConfiguration.TIMELINE_SERVICE_READ_AUTH_ENABLED, "true"); + map.put(YarnConfiguration.TIMELINE_SERVICE_READ_ALLOWED_USERS, + "user3 group5,group6"); + map.put(YarnConfiguration.YARN_ADMIN_ACL, " group1,group2"); + TimelineReaderWhitelistAuthorizationFilter f = + new TimelineReaderWhitelistAuthorizationFilter(); + FilterConfig fc = new DummyFilterConfig(map); + f.init(fc); + HttpServletRequest mockHsr = Mockito.mock(HttpServletRequest.class); + Mockito.when(mockHsr.getUserPrincipal()).thenReturn(new Principal() { + @Override + public String getName() { + return "user90"; + } + }); + ServletResponse r = Mockito.mock(ServletResponse.class); + UserGroupInformation user1 = + UserGroupInformation.createUserForTesting("user90", GROUP_NAMES); + user1.doAs(new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + f.doFilter(mockHsr, r, null); + return null; + } + }); + } + + @Test + public void checkFilterAllowAdminsWhenNoUsersSet() + throws ServletException, IOException, InterruptedException { + // check that users in admin acl list are allowed to read + Map map = new HashMap(); + map.put(YarnConfiguration.TIMELINE_SERVICE_READ_AUTH_ENABLED, "true"); + map.put(YarnConfiguration.YARN_ADMIN_ACL, " group1,group2"); + TimelineReaderWhitelistAuthorizationFilter f = + new TimelineReaderWhitelistAuthorizationFilter(); + FilterConfig fc = new DummyFilterConfig(map); + f.init(fc); + HttpServletRequest mockHsr = Mockito.mock(HttpServletRequest.class); + Mockito.when(mockHsr.getUserPrincipal()).thenReturn(new Principal() { + @Override + public String getName() { + return "user90"; + } + }); + ServletResponse r = Mockito.mock(ServletResponse.class); + UserGroupInformation user1 = + UserGroupInformation.createUserForTesting("user90", GROUP_NAMES); + user1.doAs(new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + f.doFilter(mockHsr, r, null); + return null; + } + }); + } + + @Test(expected = ForbiddenException.class) + public void checkFilterAllowNoOneWhenAdminAclsEmptyAndUserAclsEmpty() + throws ServletException, IOException, InterruptedException { + // check that users in admin acl list are allowed to read + Map map = new HashMap(); + map.put(YarnConfiguration.TIMELINE_SERVICE_READ_AUTH_ENABLED, "true"); + TimelineReaderWhitelistAuthorizationFilter f = + new TimelineReaderWhitelistAuthorizationFilter(); + FilterConfig fc = new DummyFilterConfig(map); + f.init(fc); + HttpServletRequest mockHsr = Mockito.mock(HttpServletRequest.class); + Mockito.when(mockHsr.getUserPrincipal()).thenReturn(new Principal() { + @Override + public String getName() { + return "user88"; + } + }); + ServletResponse r = Mockito.mock(ServletResponse.class); + UserGroupInformation user1 = + UserGroupInformation.createUserForTesting("user88", GROUP_NAMES); + user1.doAs(new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + f.doFilter(mockHsr, r, null); + return null; + } + }); + } + + @Test + public void checkFilterReadAuthDisabledNoAclSettings() + throws ServletException, IOException, InterruptedException { + // Default settings for Read Auth Enabled (false) + // No values in admin acls or allowed read user list + Map map = new HashMap(); + TimelineReaderWhitelistAuthorizationFilter f = + new TimelineReaderWhitelistAuthorizationFilter(); + FilterConfig fc = new DummyFilterConfig(map); + f.init(fc); + HttpServletRequest mockHsr = Mockito.mock(HttpServletRequest.class); + Mockito.when(mockHsr.getUserPrincipal()).thenReturn(new Principal() { + @Override + public String getName() { + return "user437"; + } + }); + ServletResponse r = Mockito.mock(ServletResponse.class); + UserGroupInformation user1 = + UserGroupInformation.createUserForTesting("user437", GROUP_NAMES); + user1.doAs(new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + f.doFilter(mockHsr, r, null); + return null; + } + }); + } + + @Test + public void checkFilterReadAuthDisabledButAclSettingsPopulated() + throws ServletException, IOException, InterruptedException { + Map map = new HashMap(); + // Default settings for Read Auth Enabled (false) + // But some values in admin acls and allowed read user list + map.put(YarnConfiguration.YARN_ADMIN_ACL, "user1,user2 group9,group21"); + map.put(YarnConfiguration.TIMELINE_SERVICE_READ_ALLOWED_USERS, + "user27,user36 group5,group6"); + TimelineReaderWhitelistAuthorizationFilter f = + new TimelineReaderWhitelistAuthorizationFilter(); + FilterConfig fc = new DummyFilterConfig(map); + f.init(fc); + + HttpServletRequest mockHsr = mock(HttpServletRequest.class); + when(mockHsr.getUserPrincipal()).thenReturn(new Principal() { + @Override + public String getName() { + return "user37"; + } + }); + + ServletResponse r = Mockito.mock(ServletResponse.class); + UserGroupInformation user1 = + // both username and group name are not part of admin and + // read allowed users + // but read auth is turned off + UserGroupInformation.createUserForTesting("user37", GROUP_NAMES); + user1.doAs(new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + f.doFilter(mockHsr, r, null); + return null; + } + }); + + // test with username in read allowed users + Mockito.when(mockHsr.getUserPrincipal()).thenReturn(new Principal() { + @Override + public String getName() { + return "user27"; + } + }); + ServletResponse r2 = Mockito.mock(ServletResponse.class); + UserGroupInformation user2 = + UserGroupInformation.createUserForTesting("user27", GROUP_NAMES); + user2.doAs(new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + f.doFilter(mockHsr, r2, null); + return null; + } + }); + + // test with username in admin users + Mockito.when(mockHsr.getUserPrincipal()).thenReturn(new Principal() { + @Override + public String getName() { + return "user2"; + } + }); + ServletResponse r3 = Mockito.mock(ServletResponse.class); + UserGroupInformation user3 = + UserGroupInformation.createUserForTesting("user2", GROUP_NAMES); + user3.doAs(new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + f.doFilter(mockHsr, r3, null); + return null; + } + }); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineUIDConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineUIDConverter.java index d5e791b4e3f..12b3fc0140f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineUIDConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineUIDConverter.java @@ -53,21 +53,30 @@ public class TestTimelineUIDConverter { assertEquals(context, TimelineUIDConverter.APPLICATION_UID.decodeUID(uid)); context = new TimelineReaderContext("yarn_cluster", "root", "hive_join", - 1234L, "application_1111111111_1111", "YARN_CONTAINER", + 1234L, "application_1111111111_1111", "YARN_CONTAINER", 12345L, "container_1111111111_1111_01_000001"); uid = TimelineUIDConverter.GENERIC_ENTITY_UID.encodeUID(context); assertEquals("yarn_cluster!root!hive_join!1234!application_1111111111_1111!" - + "YARN_CONTAINER!container_1111111111_1111_01_000001", uid); + + "YARN_CONTAINER!12345!container_1111111111_1111_01_000001", uid); assertEquals( context, TimelineUIDConverter.GENERIC_ENTITY_UID.decodeUID(uid)); context = new TimelineReaderContext("yarn_cluster", null, null, null, - "application_1111111111_1111", "YARN_CONTAINER", + "application_1111111111_1111", "YARN_CONTAINER", 54321L, "container_1111111111_1111_01_000001"); uid = TimelineUIDConverter.GENERIC_ENTITY_UID.encodeUID(context); assertEquals("yarn_cluster!application_1111111111_1111!YARN_CONTAINER!" + - "container_1111111111_1111_01_000001", uid); + "54321!container_1111111111_1111_01_000001", uid); assertEquals( context, TimelineUIDConverter.GENERIC_ENTITY_UID.decodeUID(uid)); + + context = new TimelineReaderContext("yarn_cluster", null, null, null, null, + "YARN_CONTAINER", 54321L, "container_1111111111_1111_01_000001", + "user1"); + uid = TimelineUIDConverter.SUB_APPLICATION_ENTITY_UID.encodeUID(context); + assertEquals("yarn_cluster!user1!YARN_CONTAINER!" + + "54321!container_1111111111_1111_01_000001", uid); + assertEquals(context, + TimelineUIDConverter.SUB_APPLICATION_ENTITY_UID.decodeUID(uid)); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java index 35af16970b8..46873ab9904 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java @@ -319,7 +319,7 @@ public class TestFileSystemTimelineReaderImpl { TimelineEntity result = reader.getEntity( new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1", "app", "id_1"), - new TimelineDataToRetrieve(null, null, null, null)); + new TimelineDataToRetrieve(null, null, null, null, null, null)); Assert.assertEquals( (new TimelineEntity.Identifier("app", "id_1")).toString(), result.getIdentifier().toString()); @@ -334,7 +334,7 @@ public class TestFileSystemTimelineReaderImpl { TimelineEntity result = reader.getEntity( new TimelineReaderContext("cluster1", null, null, null, "app1", "app", "id_1"), - new TimelineDataToRetrieve(null, null, null, null)); + new TimelineDataToRetrieve(null, null, null, null, null, null)); Assert.assertEquals( (new TimelineEntity.Identifier("app", "id_1")).toString(), result.getIdentifier().toString()); @@ -351,7 +351,7 @@ public class TestFileSystemTimelineReaderImpl { TimelineEntity result = reader.getEntity( new TimelineReaderContext("cluster1", null, null, null, "app2", "app", "id_5"), - new TimelineDataToRetrieve(null, null, null, null)); + new TimelineDataToRetrieve(null, null, null, null, null, null)); Assert.assertEquals( (new TimelineEntity.Identifier("app", "id_5")).toString(), result.getIdentifier().toString()); @@ -365,7 +365,8 @@ public class TestFileSystemTimelineReaderImpl { new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1", "app", "id_1"), new TimelineDataToRetrieve(null, null, - EnumSet.of(Field.INFO, Field.CONFIGS, Field.METRICS), null)); + EnumSet.of(Field.INFO, Field.CONFIGS, Field.METRICS), null, null, + null)); Assert.assertEquals( (new TimelineEntity.Identifier("app", "id_1")).toString(), result.getIdentifier().toString()); @@ -383,7 +384,8 @@ public class TestFileSystemTimelineReaderImpl { TimelineEntity result = reader.getEntity( new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1", "app", "id_1"), - new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null)); + new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null, + null, null)); Assert.assertEquals( (new TimelineEntity.Identifier("app", "id_1")).toString(), result.getIdentifier().toString()); @@ -398,8 +400,9 @@ public class TestFileSystemTimelineReaderImpl { public void testGetAllEntities() throws Exception { Set result = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1", - "app", null), new TimelineEntityFilters(), - new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null)); + "app", null), new TimelineEntityFilters.Builder().build(), + new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null, + null, null)); // All 4 entities will be returned Assert.assertEquals(4, result.size()); } @@ -409,8 +412,8 @@ public class TestFileSystemTimelineReaderImpl { Set result = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1", "app", null), - new TimelineEntityFilters(2L, null, null, null, null, null, null, - null, null), new TimelineDataToRetrieve()); + new TimelineEntityFilters.Builder().entityLimit(2L).build(), + new TimelineDataToRetrieve()); Assert.assertEquals(2, result.size()); // Needs to be rewritten once hashcode and equals for // TimelineEntity is implemented @@ -424,8 +427,8 @@ public class TestFileSystemTimelineReaderImpl { result = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1", "app", null), - new TimelineEntityFilters(3L, null, null, null, null, null, null, - null, null), new TimelineDataToRetrieve()); + new TimelineEntityFilters.Builder().entityLimit(3L).build(), + new TimelineDataToRetrieve()); // Even though 2 entities out of 4 have same created time, one entity // is left out due to limit Assert.assertEquals(3, result.size()); @@ -437,8 +440,8 @@ public class TestFileSystemTimelineReaderImpl { Set result = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1", "app", null), - new TimelineEntityFilters(null, 1425016502030L, 1425016502060L, null, - null, null, null, null, null), + new TimelineEntityFilters.Builder().createdTimeBegin(1425016502030L) + .createTimeEnd(1425016502060L).build(), new TimelineDataToRetrieve()); Assert.assertEquals(1, result.size()); // Only one entity with ID id_4 should be returned. @@ -452,9 +455,9 @@ public class TestFileSystemTimelineReaderImpl { result = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1", "app", null), - new TimelineEntityFilters(null, null, 1425016502010L, null, null, - null, null, null, null), - new TimelineDataToRetrieve()); + new TimelineEntityFilters.Builder().createTimeEnd(1425016502010L) + .build(), + new TimelineDataToRetrieve()); Assert.assertEquals(3, result.size()); for (TimelineEntity entity : result) { if (entity.getId().equals("id_4")) { @@ -466,8 +469,8 @@ public class TestFileSystemTimelineReaderImpl { result = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1", "app", null), - new TimelineEntityFilters(null, 1425016502010L, null, null, null, - null, null, null, null), + new TimelineEntityFilters.Builder().createdTimeBegin(1425016502010L) + .build(), new TimelineDataToRetrieve()); Assert.assertEquals(1, result.size()); for (TimelineEntity entity : result) { @@ -486,8 +489,7 @@ public class TestFileSystemTimelineReaderImpl { Set result = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1", "app", null), - new TimelineEntityFilters(null, null, null, null, null, infoFilterList, - null, null, null), + new TimelineEntityFilters.Builder().infoFilters(infoFilterList).build(), new TimelineDataToRetrieve()); Assert.assertEquals(1, result.size()); // Only one entity with ID id_3 should be returned. @@ -506,8 +508,8 @@ public class TestFileSystemTimelineReaderImpl { result = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1", "app", null), - new TimelineEntityFilters(null, null, null, null, null, null, - confFilterList, null, null), + new TimelineEntityFilters.Builder().configFilters(confFilterList) + .build(), new TimelineDataToRetrieve()); Assert.assertEquals(1, result.size()); for (TimelineEntity entity : result) { @@ -525,8 +527,7 @@ public class TestFileSystemTimelineReaderImpl { result = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1", "app", null), - new TimelineEntityFilters(null, null, null, null, null, null, null, - null, eventFilters), + new TimelineEntityFilters.Builder().eventFilters(eventFilters).build(), new TimelineDataToRetrieve()); Assert.assertEquals(1, result.size()); for (TimelineEntity entity : result) { @@ -542,8 +543,8 @@ public class TestFileSystemTimelineReaderImpl { result = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1", "app", null), - new TimelineEntityFilters(null, null, null, null, null, null, null, - metricFilterList, null), + new TimelineEntityFilters.Builder().metricFilters(metricFilterList) + .build(), new TimelineDataToRetrieve()); Assert.assertEquals(2, result.size()); // Two entities with IDs' id_1 and id_2 should be returned. @@ -569,8 +570,8 @@ public class TestFileSystemTimelineReaderImpl { result = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1", "app", null), - new TimelineEntityFilters(null, null, null, null, null, null, - confFilterList1, null, null), + new TimelineEntityFilters.Builder().configFilters(confFilterList1) + .build(), new TimelineDataToRetrieve()); Assert.assertEquals(2, result.size()); for (TimelineEntity entity : result) { @@ -592,8 +593,8 @@ public class TestFileSystemTimelineReaderImpl { result = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1", "app", null), - new TimelineEntityFilters(null, null, null, null, null, null, - confFilterList2, null, null), + new TimelineEntityFilters.Builder().configFilters(confFilterList2) + .build(), new TimelineDataToRetrieve()); Assert.assertEquals(2, result.size()); for (TimelineEntity entity : result) { @@ -610,8 +611,8 @@ public class TestFileSystemTimelineReaderImpl { result = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1", "app", null), - new TimelineEntityFilters(null, null, null, null, null, null, - confFilterList3, null, null), + new TimelineEntityFilters.Builder().configFilters(confFilterList3) + .build(), new TimelineDataToRetrieve()); Assert.assertEquals(1, result.size()); for(TimelineEntity entity : result) { @@ -628,8 +629,8 @@ public class TestFileSystemTimelineReaderImpl { result = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1", "app", null), - new TimelineEntityFilters(null, null, null, null, null, null, - confFilterList4, null, null), + new TimelineEntityFilters.Builder().configFilters(confFilterList4) + .build(), new TimelineDataToRetrieve()); Assert.assertEquals(0, result.size()); @@ -641,8 +642,8 @@ public class TestFileSystemTimelineReaderImpl { result = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1", "app", null), - new TimelineEntityFilters(null, null, null, null, null, null, - confFilterList5, null, null), + new TimelineEntityFilters.Builder().configFilters(confFilterList5) + .build(), new TimelineDataToRetrieve()); Assert.assertEquals(1, result.size()); for (TimelineEntity entity : result) { @@ -665,8 +666,8 @@ public class TestFileSystemTimelineReaderImpl { result = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1", "app", null), - new TimelineEntityFilters(null, null, null, null, null, null, null, - metricFilterList1, null), + new TimelineEntityFilters.Builder().metricFilters(metricFilterList1) + .build(), new TimelineDataToRetrieve()); Assert.assertEquals(2, result.size()); // Two entities with IDs' id_2 and id_3 should be returned. @@ -684,8 +685,8 @@ public class TestFileSystemTimelineReaderImpl { result = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1", "app", null), - new TimelineEntityFilters(null, null, null, null, null, null, null, - metricFilterList2, null), + new TimelineEntityFilters.Builder().metricFilters(metricFilterList2) + .build(), new TimelineDataToRetrieve()); Assert.assertEquals(1, result.size()); for (TimelineEntity entity : result) { @@ -702,8 +703,8 @@ public class TestFileSystemTimelineReaderImpl { result = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1", "app", null), - new TimelineEntityFilters(null, null, null, null, null, null, null, - metricFilterList3, null), + new TimelineEntityFilters.Builder().metricFilters(metricFilterList3) + .build(), new TimelineDataToRetrieve()); Assert.assertEquals(0, result.size()); @@ -715,8 +716,8 @@ public class TestFileSystemTimelineReaderImpl { result = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1", "app", null), - new TimelineEntityFilters(null, null, null, null, null, null, null, - metricFilterList4, null), + new TimelineEntityFilters.Builder().metricFilters(metricFilterList4) + .build(), new TimelineDataToRetrieve()); Assert.assertEquals(2, result.size()); for (TimelineEntity entity : result) { @@ -731,8 +732,8 @@ public class TestFileSystemTimelineReaderImpl { result = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1", "app", null), - new TimelineEntityFilters(null, null, null, null, null, null, null, - metricFilterList5, null), + new TimelineEntityFilters.Builder().metricFilters(metricFilterList5) + .build(), new TimelineDataToRetrieve()); Assert.assertEquals(2, result.size()); for (TimelineEntity entity : result) { @@ -749,8 +750,8 @@ public class TestFileSystemTimelineReaderImpl { result = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1", "app", null), - new TimelineEntityFilters(null, null, null, null, null, infoFilterList1, - null, null, null), + new TimelineEntityFilters.Builder().infoFilters(infoFilterList1) + .build(), new TimelineDataToRetrieve()); Assert.assertEquals(0, result.size()); @@ -762,8 +763,8 @@ public class TestFileSystemTimelineReaderImpl { result = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1", "app", null), - new TimelineEntityFilters(null, null, null, null, null, infoFilterList2, - null, null, null), + new TimelineEntityFilters.Builder().infoFilters(infoFilterList2) + .build(), new TimelineDataToRetrieve()); Assert.assertEquals(2, result.size()); for (TimelineEntity entity : result) { @@ -780,8 +781,8 @@ public class TestFileSystemTimelineReaderImpl { result = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1", "app", null), - new TimelineEntityFilters(null, null, null, null, null, infoFilterList3, - null, null, null), + new TimelineEntityFilters.Builder().infoFilters(infoFilterList3) + .build(), new TimelineDataToRetrieve()); Assert.assertEquals(0, result.size()); @@ -793,8 +794,8 @@ public class TestFileSystemTimelineReaderImpl { result = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1", "app", null), - new TimelineEntityFilters(null, null, null, null, null, infoFilterList4, - null, null, null), + new TimelineEntityFilters.Builder().infoFilters(infoFilterList4) + .build(), new TimelineDataToRetrieve()); Assert.assertEquals(1, result.size()); for (TimelineEntity entity : result) { @@ -815,8 +816,7 @@ public class TestFileSystemTimelineReaderImpl { Set result = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1", "app", null), - new TimelineEntityFilters(null, null, null, relatesTo, null, null, - null, null, null), + new TimelineEntityFilters.Builder().relatesTo(relatesTo).build(), new TimelineDataToRetrieve()); Assert.assertEquals(1, result.size()); // Only one entity with ID id_1 should be returned. @@ -835,8 +835,7 @@ public class TestFileSystemTimelineReaderImpl { result = reader.getEntities( new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1", "app", null), - new TimelineEntityFilters(null, null, null, null, isRelatedTo, null, - null, null, null), + new TimelineEntityFilters.Builder().isRelatedTo(isRelatedTo).build(), new TimelineDataToRetrieve()); Assert.assertEquals(2, result.size()); // Two entities with IDs' id_1 and id_3 should be returned. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineWriterImpl.java index 4f12c57f7bc..bb9f54f9379 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineWriterImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineWriterImpl.java @@ -30,11 +30,13 @@ import java.util.List; import java.util.Map; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetricOperation; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorContext; import org.apache.hadoop.yarn.util.timeline.TimelineUtils; import org.junit.Rule; import org.junit.Test; @@ -89,8 +91,10 @@ public class TestFileSystemTimelineWriterImpl { outputRoot); fsi.init(conf); fsi.start(); - fsi.write("cluster_id", "user_id", "flow_name", "flow_version", 12345678L, - "app_id", te); + fsi.write( + new TimelineCollectorContext("cluster_id", "user_id", "flow_name", + "flow_version", 12345678L, "app_id"), + te, UserGroupInformation.createRemoteUser("user_id")); String fileName = fsi.getOutputRoot() + File.separator + "entities" + File.separator + "cluster_id" + File.separator + "user_id" + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md index 04822c9fb06..2de305d1cd3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md @@ -73,12 +73,8 @@ The following diagram illustrates the design at a high level. ### Current Status and Future Plans -YARN Timeline Service v.2 is currently in alpha ("alpha 1"). It is very much work in progress, and -many things can and will change rapidly. Users must enable Timeline Service v.2 only on a test or -experimental cluster to test the feature. - -Most importantly, **security is not enabled**. Do not set up or use Timeline Service v.2 until -security is implemented if security is a requirement. +YARN Timeline Service v.2 is currently in alpha ("alpha 2"). It is a work in progress, and +many things can and will change rapidly. A complete end-to-end flow of writes and reads is functional, with Apache HBase as the backend. You should be able to start generating data. When enabled, all YARN-generic events are @@ -95,16 +91,19 @@ resource manager also has its dedicated in-process collector. The reader is curr instance. Currently, it is not possible to write to Timeline Service outside the context of a YARN application (i.e. no off-cluster client). +Starting from alpha2, Timeline Service v.2 supports simple authorization in terms of a +configurable whitelist of users and groups who can read timeline data. Cluster admins are +allowed by default to read timeline data. + When YARN Timeline Service v.2 is disabled, one can expect no functional or performance impact on any other existing functionality. The work to make it truly production-ready continues. Some key items include * More robust storage fault tolerance -* Security * Support for off-cluster clients -* More complete and integrated web UI * Better support for long-running apps +* Support for ACLs * Offline (time-based periodic) aggregation for flows, users, and queues for reporting and analysis * Timeline collectors as separate instances from node managers @@ -127,6 +126,7 @@ New configuration parameters that are introduced with v.2 are marked bold. | **`yarn.timeline-service.writer.class`** | The class for the backend storage writer. Defaults to HBase storage writer. | | **`yarn.timeline-service.reader.class`** | The class for the backend storage reader. Defaults to HBase storage reader. | | **`yarn.system-metrics-publisher.enabled`** | The setting that controls whether yarn system metrics is published on the Timeline service or not by RM And NM. Defaults to `false`. | +| **`yarn.timeline-service.schema.prefix`** | The schema prefix for hbase tables. Defaults to "prod.". | #### Advanced configuration @@ -136,12 +136,34 @@ New configuration parameters that are introduced with v.2 are marked bold. | `yarn.timeline-service.address` | Address for the Timeline server to start the RPC server. Defaults to `${yarn.timeline-service.hostname}:10200`. | | `yarn.timeline-service.webapp.address` | The http address of the Timeline service web application. Defaults to `${yarn.timeline-service.hostname}:8188`. | | `yarn.timeline-service.webapp.https.address` | The https address of the Timeline service web application. Defaults to `${yarn.timeline-service.hostname}:8190`. | +| **`yarn.timeline-service.hbase.configuration.file`** | Optional URL to an hbase-site.xml configuration file to be used to connect to the timeline-service hbase cluster. If empty or not specified, then the HBase configuration will be loaded from the classpath. When specified the values in the specified configuration file will override those from the ones that are present on the classpath. Defaults to `null`. | | **`yarn.timeline-service.writer.flush-interval-seconds`** | The setting that controls how often the timeline collector flushes the timeline writer. Defaults to `60`. | | **`yarn.timeline-service.app-collector.linger-period.ms`** | Time period till which the application collector will be alive in NM, after the application master container finishes. Defaults to `1000` (1 second). | | **`yarn.timeline-service.timeline-client.number-of-async-entities-to-merge`** | Time line V2 client tries to merge these many number of async entities (if available) and then call the REST ATS V2 API to submit. Defaults to `10`. | | **`yarn.timeline-service.hbase.coprocessor.app-final-value-retention-milliseconds`** | The setting that controls how long the final value of a metric of a completed app is retained before merging into the flow sum. Defaults to `259200000` (3 days). This should be set in the HBase cluster. | | **`yarn.rm.system-metrics-publisher.emit-container-events`** | The setting that controls whether yarn container metrics is published to the timeline server or not by RM. This configuration setting is for ATS V2. Defaults to `false`. | +#### Security Configuration + + +Security can be enabled by setting `yarn.timeline-service.http-authentication.type` +to `kerberos`, after which the following configuration options are available: + + +| Configuration Property | Description | +|:---- |:---- | +| `yarn.timeline-service.http-authentication.type` | Defines authentication used for the timeline server(collector/reader) HTTP endpoint. Supported values are: `simple` / `kerberos` / #AUTHENTICATION_HANDLER_CLASSNAME#. Defaults to `simple`. | +| `yarn.timeline-service.http-authentication.simple.anonymous.allowed` | Indicates if anonymous requests are allowed by the timeline server when using 'simple' authentication. Defaults to `true`. | +| `yarn.timeline-service.http-authentication.kerberos.principal` | The Kerberos principal to be used for the Timeline Server(Collector/Reader) HTTP endpoint. | +| `yarn.timeline-service.http-authentication.kerberos.keytab` | The Kerberos keytab to be used for the Timeline Server(Collector/Reader) HTTP endpoint.. | +| `yarn.timeline-service.principal` | The Kerberos principal for the timeline reader. NM principal would be used for timeline collector as it runs as an auxiliary service inside NM. | +| `yarn.timeline-service.keytab` | The Kerberos keytab for the timeline reader. NM keytab would be used for timeline collector as it runs as an auxiliary service inside NM. | +| `yarn.timeline-service.delegation.key.update-interval` | Defaults to `86400000` (1 day). | +| `yarn.timeline-service.delegation.token.renew-interval` | Defaults to `86400000` (1 day). | +| `yarn.timeline-service.delegation.token.max-lifetime` | Defaults to `604800000` (7 days). | +| `yarn.timeline-service.read.authentication.enabled` | Enables or disables authorization checks for reading timeline service v2 data. Default is `false` which is disabled. | +| `yarn.timeline-service.read.allowed.users` | Comma separated list of user, followed by space, then comma separated list of groups. It will allow this list of users and groups to read the data and reject everyone else. Default value is set to none. If authorization is enabled, then this configuration is mandatory. | + #### Enabling CORS support To enable cross-origin support (CORS) for the Timeline Service v.2, please set the following configuration parameters: @@ -156,39 +178,92 @@ For more configurations used for cross-origin support, refer to [HttpAuthenticat ### Enabling Timeline Service v.2 #### Preparing Apache HBase cluster for storage +There are a few steps to be done for preparing the storage for Timeline Service v.2: + +Step 1) [Set up the HBase cluster](#Set_up_the_HBase_cluster) + +Step 2) [Enable the coprocessor](#Enable_the_coprocessor) + +Step 3) [Create the schema for Timeline Service v.2](#Create_schema) + +Each step is explained in more detail below. + +##### Step 1) Set up the HBase cluster The first part is to set up or pick an Apache HBase cluster to use as the storage cluster. The -version of Apache HBase that is supported with Timeline Service v.2 is 1.1.x. The 1.0.x versions -do not work with Timeline Service v.2. The 1.2.x versions have not been tested. +version of Apache HBase that is supported with Timeline Service v.2 is 1.2.6. The 1.0.x versions +do not work with Timeline Service v.2. Later versions of HBase have not been tested with +Timeline Service. -Once you have an Apache HBase cluster ready to use for this purpose, perform the following steps. +HBase has different deployment modes. Refer to the HBase book for understanding them and pick a +mode that is suitable for your setup. +(http://hbase.apache.org/book.html#standalone_dist) -First, add the timeline service jar to the HBase classpath in all HBase machines in the cluster. It -is needed for the coprocessor as well as the schema creator. For example, +##### Simple deployment for HBase +If you are intent on a simple deploy profile for the Apache HBase cluster +where the data loading is light but the data needs to persist across node +comings and goings, you could consider the "Standalone HBase over HDFS" deploy mode. - cp hadoop-yarn-server-timelineservice-hbase-3.0.0-alpha1-SNAPSHOT.jar /usr/hbase/lib/ +This is a useful variation on the standalone HBase setup and has all HBase daemons running inside +one JVM but rather than persisting to the local filesystem, it persists to an HDFS instance. +Writing to HDFS where data is replicated ensures that data is persisted across node +comings and goings. To configure this standalone variant, edit your `hbase-site.xml` setting +the `hbase.rootdir` to point at a directory in your HDFS instance but then set +`hbase.cluster.distributed` to false. For example: -Then, enable the coprocessor that handles the aggregation. To enable it, add the following entry in -region servers' `hbase-site.xml` file (generally located in the `conf` directory) as follows: +``` + + + hbase.rootdir + hdfs://namenode.example.org:8020/hbase + + + hbase.cluster.distributed + false + + +``` + +For more details on this mode, refer to +http://hbase.apache.org/book.html#standalone.over.hdfs . + +Once you have an Apache HBase cluster ready to use, perform the following steps. + +##### Step 2) Enable the coprocessor +In this version, the coprocessor is loaded dynamically (table coprocessor for the `flowrun` table). + +Copy the timeline service jar to HDFS from where HBase can load it. It +is needed for the `flowrun` table creation in the schema creator. The default HDFS location is `/hbase/coprocessor`. +For example, + + hadoop fs -mkdir /hbase/coprocessor + hadoop fs -put hadoop-yarn-server-timelineservice-hbase-3.0.0-alpha1-SNAPSHOT.jar + /hbase/coprocessor/hadoop-yarn-server-timelineservice.jar + + +If you want to place the jar at a different location on hdfs, there also exists a yarn +configuration setting called `yarn.timeline-service.hbase.coprocessor.jar.hdfs.location`. +For example, ``` - hbase.coprocessor.region.classes - org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunCoprocessor + yarn.timeline-service.hbase.coprocessor.jar.hdfs.location + /custom/hdfs/path/jarName ``` -Restart the region servers and the master to pick up the timeline service jar as well as the config -change. In this version, the coprocessor is loaded statically (i.e. system coprocessor) as opposed -to a dynamically (table coprocessor). - +##### Step 3) Create the timeline service schema Finally, run the schema creator tool to create the necessary tables: bin/hadoop org.apache.hadoop.yarn.server.timelineservice.storage.TimelineSchemaCreator -create The `TimelineSchemaCreator` tool supports a few options that may come handy especially when you are testing. For example, you can use `-skipExistingTable` (`-s` for short) to skip existing tables +and continue to create other tables rather than failing the schema creation. By default, the tables +will have a schema prefix of "prod.". When no option or '-help' ('-h' for short) is provided, the +command usage is printed. and continue to create other tables rather than failing the schema creation. When no option or '-help' -('-h' for short) is provided, the command usage is printed. +('-h' for short) is provided, the command usage is printed. By default, the tables +will have a schema prefix of "prod." #### Enabling Timeline Service v.2 Following are the basic configurations to start Timeline service v.2: @@ -241,7 +316,22 @@ are using multiple clusters to store data in the same Apache HBase storage: ``` Also, add the `hbase-site.xml` configuration file to the client Hadoop cluster configuration so -that it can write data to the Apache HBase cluster you are using. +that it can write data to the Apache HBase cluster you are using, or set +`yarn.timeline-service.hbase.configuration.file` to the file URL pointing to +`hbase-site.xml` for the same. For example: + +``` + + Optional URL to an hbase-site.xml configuration file to be + used to connect to the timeline-service hbase cluster. If empty or not + specified, then the HBase configuration will be loaded from the classpath. + When specified the values in the specified configuration file will override + those from the ones that are present on the classpath. + + yarn.timeline-service.hbase.configuration.file + file:/etc/hbase/hbase-ats-dc1/hbase-site.xml + +``` #### Running Timeline Service v.2 Restart the resource manager as well as the node managers to pick up the new configuration. The @@ -263,6 +353,15 @@ To write MapReduce framework data to Timeline Service v.2, enable the following ``` +### Upgrade from alpha1 to alpha2 +If you are currently running Timeline Service v2 alpha1 version, we recommend the following: + +- Clear existing data in tables (truncate tables) since the row key for AppToFlow has changed. + +- The coprocessor is now a dynamically loaded table level coprocessor in alpha2. We +recommend dropping the table, replacing the coprocessor jar on hdfs with the alpha2 one, +restarting the Region servers and recreating the `flowrun` table. + ### Publishing application specific data This section is for YARN application developers that want to integrate with Timeline Service v.2. @@ -447,6 +546,8 @@ predicates, an empty list will be returned. "daterange=20150711-20150714" returns flows active between these 2 dates.
    "daterange=20150711-" returns flows active on and after 20150711.
    "daterange=-20150711" returns flows active on and before 20150711.
    +1. `fromid` - If specified, retrieve the next set of flows from the given fromid. The set of entities retrieved is inclusive of specified fromid. + fromid should be taken from the value associated with FROM_ID info key in flow entity response which was sent earlier. #### Example JSON Response: @@ -492,6 +593,7 @@ predicates, an empty list will be returned. "info": { "SYSTEM_INFO_CLUSTER": "test-cluster", "UID": "test-cluster!sjlee!ds-date", + "FROM_ID": "test-cluster!1460419200000!sjlee!ds-date", "SYSTEM_INFO_FLOW_NAME": "ds-date", "SYSTEM_INFO_DATE": 1460419200000, "SYSTEM_INFO_USER": "sjlee" @@ -543,6 +645,8 @@ predicates, an empty list will be returned. 1. `fields` - Specifies which fields to retrieve. For querying flow runs, only `ALL` or `METRICS` are valid fields. Other fields will lead to HTTP 400 (Bad Request) response. If not specified, in response, id, type, createdtime and info fields will be returned. +1. `fromid` - If specified, retrieve the next set of flow run entities from the given fromid. The set of entities retrieved is inclusive of specified fromid. + fromid should be taken from the value associated with FROM_ID info key in flow entity response which was sent earlier. #### Example JSON Response: @@ -555,6 +659,7 @@ predicates, an empty list will be returned. "createdtime": 1460420587974, "info": { "UID": "test-cluster!sjlee!ds-date!1460420587974", + "FROM_ID": "test-cluster!sjlee!ds-date!1460420587974", "SYSTEM_INFO_FLOW_RUN_ID": 1460420587974, "SYSTEM_INFO_FLOW_NAME": "ds-date", "SYSTEM_INFO_FLOW_RUN_END_TIME": 1460420595198, @@ -571,6 +676,7 @@ predicates, an empty list will be returned. "createdtime": 1460420305659, "info": { "UID": "test-cluster!sjlee!ds-date!1460420305659", + "FROM_ID": "test-cluster!sjlee!ds-date!1460420305659", "SYSTEM_INFO_FLOW_RUN_ID": 1460420305659, "SYSTEM_INFO_FLOW_NAME": "ds-date", "SYSTEM_INFO_FLOW_RUN_END_TIME": 1460420311966, @@ -642,6 +748,7 @@ while querying individual flow runs. "isrelatedto": {}, "info": { "UID":"yarn-cluster!varun!QuasiMonteCarlo!1465246348599", + "FROM_ID":"yarn-cluster!varun!QuasiMonteCarlo!1465246348599", "SYSTEM_INFO_FLOW_RUN_END_TIME":1465246378051, "SYSTEM_INFO_FLOW_NAME":"QuasiMonteCarlo", "SYSTEM_INFO_USER":"varun", @@ -746,6 +853,10 @@ none of the apps match the predicates, an empty list will be returned. or metricstoretrieve is specified. Ignored otherwise. The maximum possible value for metricslimit can be maximum value of Integer. If it is not specified or has a value less than 1, and metrics have to be retrieved, then metricslimit will be considered as 1 i.e. latest single value of metric(s) will be returned. +1. `metricsTimeStart` - If specified, then metrics for the entity after this timestamp are returned. +1. `metricsTimeEnd` - If specified, then metrics for the entity before this timestamp are returned. +1. `fromid` - If specified, retrieve the next set of application entities from the given fromid. The set of entities retrieved is inclusive of specified fromid. + fromid should be taken from the value associated with FROM_ID info key in flow entity response which was sent earlier. #### Example JSON Response: @@ -760,6 +871,7 @@ none of the apps match the predicates, an empty list will be returned. "configs": { }, "info": { "UID": "yarn-cluster!application_1465246237936_0001" + "FROM_ID": "yarn-cluster!varun!QuasiMonteCarlo!1465246348599!application_1465246237936_0001", }, "relatesto": { } }, @@ -773,6 +885,7 @@ none of the apps match the predicates, an empty list will be returned. "configs": { }, "info": { "UID": "yarn-cluster!application_1464983628730_0005" + "FROM_ID": "yarn-cluster!varun!QuasiMonteCarlo!1465246348599!application_1464983628730_0005", }, "relatesto": { } } @@ -873,6 +986,10 @@ match the predicates, an empty list will be returned. or metricstoretrieve is specified. Ignored otherwise. The maximum possible value for metricslimit can be maximum value of Integer. If it is not specified or has a value less than 1, and metrics have to be retrieved, then metricslimit will be considered as 1 i.e. latest single value of metric(s) will be returned. +1. `metricsTimeStart` - If specified, then metrics for the entity after this timestamp are returned. +1. `metricsTimeEnd` - If specified, then metrics for the entity before this timestamp are returned. +1. `fromid` - If specified, retrieve the next set of application entities from the given fromid. The set of entities retrieved is inclusive of specified fromid. + fromid should be taken from the value associated with FROM_ID info key in flow entity response which was sent earlier. #### Example JSON Response: @@ -885,6 +1002,7 @@ match the predicates, an empty list will be returned. "createdtime": 1460419580171, "info": { "UID": "test-cluster!sjlee!ds-date!1460419580171!application_1460419579913_0002" + "FROM_ID": "test-cluster!sjlee!ds-date!1460419580171!application_1460419579913_0002", }, "configs": {}, "isrelatedto": {}, @@ -949,6 +1067,8 @@ and app id. or metricstoretrieve is specified. Ignored otherwise. The maximum possible value for metricslimit can be maximum value of Integer. If it is not specified or has a value less than 1, and metrics have to be retrieved, then metricslimit will be considered as 1 i.e. latest single value of metric(s) will be returned. +1. `metricsTimeStart` - If specified, then metrics for the entity after this timestamp are returned. +1. `metricsTimeEnd` - If specified, then metrics for the entity before this timestamp are returned. #### Example JSON Response: @@ -973,7 +1093,7 @@ and app id. 1. If flow context information cannot be retrieved or application for the given app id cannot be found, HTTP 404 (Not Found) is returned. 1. For non-recoverable errors while retrieving data, HTTP 500 (Internal Server Error) is returned. -### Query generic entities +### Query generic entities with in the scope of Application With this API, you can query generic entities identified by cluster ID, application ID and per-framework entity type. If the REST endpoint without the cluster name is used, the cluster @@ -1073,6 +1193,10 @@ If none of the entities match the predicates, an empty list will be returned. or metricstoretrieve is specified. Ignored otherwise. The maximum possible value for metricslimit can be maximum value of Integer. If it is not specified or has a value less than 1, and metrics have to be retrieved, then metricslimit will be considered as 1 i.e. latest single value of metric(s) will be returned. +1. `metricsTimeStart` - If specified, then metrics for the entity after this timestamp are returned. +1. `metricsTimeEnd` - If specified, then metrics for the entity before this timestamp are returned. +1. `fromid` - If specified, retrieve the next set of generic entities from the given fromid. The set of entities retrieved is inclusive of specified fromid. + fromid should be taken from the value associated with FROM_ID info key in flow entity response which was sent earlier. #### Example JSON Response: @@ -1087,6 +1211,7 @@ If none of the entities match the predicates, an empty list will be returned. "configs": { }, "info": { "UID": "yarn-cluster!application_1465246237936_0001!YARN_APPLICATION_ATTEMPT!appattempt_1465246237936_0001_000001" + "FROM_ID": "yarn-cluster!sjlee!ds-date!1460419580171!application_1465246237936_0001!YARN_APPLICATION_ATTEMPT!0!appattempt_1465246237936_0001_000001" }, "relatesto": { } }, @@ -1100,6 +1225,7 @@ If none of the entities match the predicates, an empty list will be returned. "configs": { }, "info": { "UID": "yarn-cluster!application_1465246237936_0001!YARN_APPLICATION_ATTEMPT!appattempt_1465246237936_0001_000002" + "FROM_ID": "yarn-cluster!sjlee!ds-date!1460419580171!application_1465246237936_0001!YARN_APPLICATION_ATTEMPT!0!appattempt_1465246237936_0001_000002" }, "relatesto": { } } @@ -1112,7 +1238,142 @@ If none of the entities match the predicates, an empty list will be returned. 1. If flow context information cannot be retrieved, HTTP 404 (Not Found) is returned. 1. For non-recoverable errors while retrieving data, HTTP 500 (Internal Server Error) is returned. -### Query generic entity +### Query generic entities. + +With this API, you can query generic entities per user identified by cluster ID, doAsUser and +entity type. If the REST endpoint without the cluster name is used, the cluster +specified by the configuration `yarn.resourcemanager.cluster-id` in `yarn-site.xml` is taken. +If number of matching entities are more than the limit, the most recent +entities up to the limit will be returned. This endpoint can be used to query generic entity which +clients put into the backend. For instance, we can query user entities by specifying entity type as `TEZ_DAG_ID`. +If none of the entities match the predicates, an empty list will be returned. +**Note** : As of today, we can query only those entities which are published with doAsUser which is different from application owner. + +#### HTTP request: + + GET /ws/v2/timeline/clusters/{cluster name}/users/{userid}/entities/{entitytype} + + or + + GET /ws/v2/timeline/users/{userid}/entities/{entitytype} + +#### Query Parameters Supported: + +1. `limit` - If specified, defines the number of entities to return. The maximum possible value for limit is maximum value of Long. If it is not specified + or has a value less than 0, then limit will be considered as 100. +1. `createdtimestart` - If specified, then only entities created after this timestamp are returned. +1. `createdtimeend` - If specified, then only entities created before this timestamp are returned. +1. `relatesto` - If specified, matched entities must relate to or not relate to given entities associated with a entity type. + relatesto is represented as an expression of the form :
    + "(<entitytype>:<entityid>:<entityid>...,<entitytype>:<entityid>:<entityid>...) <op> !(<entitytype>:<entityid>:<entityid>...,<entitytype>:<entityid>:<entityid>...)".
    + If relatesto expression has entity type - entity id(s) relations specified within enclosing brackets proceeding "!", this means entities with + these relations in its relatesto field, will not be returned. For expressions or subexpressions without "!", all entities which have the specified + relations in its relatesto field, will be returned. "op" is a logical operator and can be either AND or OR. entity type can be followed by any number + of entity id(s). And we can combine any number of ANDs' and ORs' to create complex expressions. Brackets can be used to club expressions together.
    + _For example_ : relatesto can be "(((type1:id1:id2:id3,type3:id9) AND !(type2:id7:id8)) OR (type1:id4))".
    + Please note that URL unsafe characters such as spaces will have to be suitably encoded. +1. `isrelatedto` - If specified, matched entities must be related to or not related to given entities associated with a entity type. isrelatedto is + represented in the same form as relatesto. +1. `infofilters` - If specified, matched entities must have exact matches to the given info key and must be either equal or not equal to + given value. The info key is a string but value can be any object. infofilters are represented as an expression of the form :
    + "(<key> <compareop> <value>) <op> (<key> <compareop> <value>)".
    + Here op can be either of AND or OR. And compareop can be either of "eq", "ne" or "ene".
    + "eq" means equals, "ne" means not equals and existence of key is not required for a match and "ene" means not equals but existence of key is + required. We can combine any number of ANDs' and ORs' to create complex expressions. Brackets can be used to club expressions together.
    + _For example_ : infofilters can be "(((infokey1 eq value1) AND (infokey2 ne value1)) OR (infokey1 ene value3))".
    + Note : If value is an object then value can be given in the form of JSON format without any space.
    + _For example_ : infofilters can be (infokey1 eq {"<key>":"<value>","<key>":"<value>"...}).
    + Please note that URL unsafe characters such as spaces will have to be suitably encoded. +1. `conffilters` - If specified, matched entities must have exact matches to the given config name and must be either equal or not equal + to the given config value. Both the config name and value must be strings. conffilters are represented in the same form as infofilters. +1. `metricfilters` - If specified, matched entities must have exact matches to the given metric and satisfy the specified relation with the + metric value. Metric id must be a string and and metric value must be an integral value. metricfilters are represented as an expression of the form :
    + "(<metricid> <compareop> <metricvalue>) <op> (<metricid> <compareop> <metricvalue>)"
    + Here op can be either of AND or OR. And compareop can be either of "eq", "ne", "ene", "gt", "ge", "lt" and "le".
    + "eq" means equals, "ne" means not equals and existence of metric is not required for a match, "ene" means not equals but existence of metric is + required, "gt" means greater than, "ge" means greater than or equals, "lt" means less than and "le" means less than or equals. We can combine + any number of ANDs' and ORs' to create complex expressions. Brackets can be used to club expressions together.
    + _For example_ : metricfilters can be "(((metric1 eq 50) AND (metric2 gt 40)) OR (metric1 lt 20))".
    + This in essence is an expression equivalent to "(metric1 == 50 AND metric2 > 40) OR (metric1 < 20)"
    + Please note that URL unsafe characters such as spaces will have to be suitably encoded. +1. `eventfilters` - If specified, matched entities must contain or not contain the given events depending on the expression. eventfilters is + represented as an expression of the form :
    + "(<eventid>,<eventid>) <op> !(<eventid>,<eventid>,<eventid>)".
    + Here, "!" means none of the comma-separated list of events within the enclosed brackets proceeding "!" must exist for a match to occur. + If "!" is not specified, the specified events within the enclosed brackets must exist. op is a logical operator and can be either AND or OR. + We can combine any number of ANDs' and ORs' to create complex expressions. Brackets can be used to club expressions together.
    + _For example_ : eventfilters can be "(((event1,event2) AND !(event4)) OR (event3,event7,event5))".
    + Please note that URL unsafe characters such as spaces will have to be suitably encoded. +1. `metricstoretrieve` - If specified, defines which metrics to retrieve or which ones not to retrieve and send back in response. + metricstoretrieve can be an expression of the form :
    + (<metricprefix>,<metricprefix>,<metricprefix>,<metricprefix>...)
    + This specifies a comma separated list of metric id prefixes. Only metrics matching any of the prefixes will be retrieved. Brackets are optional for + the simple expression. Alternatively, expressions can be of the form:
    + !(<metricprefix>,<metricprefix>,<metricprefix>,<metricprefix>...)
    + This specifies a comma separated list of metric id prefixes. Only metrics not matching any of the prefixes will be retrieved.
    + If metricstoretrieve is specified, metrics will be retrieved irrespective of whether `METRICS` is specified in fields query param + or not. Please note that URL unsafe characters such as spaces will have to be suitably encoded. +1. `confstoretrieve` - If specified, defines which configs to retrieve or which ones not to retrieve and send back in response. + confstoretrieve can be an expression of the form :
    + (<config\_name\_prefix>,<config\_name\_prefix>,<config\_name\_prefix>,<config\_name\_prefix>...)
    + This specifies a comma separated list of config name prefixes. Only configs matching any of the prefixes will be retrieved. Brackets are optional for + the simple expression. Alternatively, expressions can be of the form:
    + !(<config\_name\_prefix>,<config\_name\_prefix>,<config\_name\_prefix>,<config\_name\_prefix>...)
    + This specifies a comma separated list of config name prefixes. Only configs not matching any of the prefixes will be retrieved.
    + If confstoretrieve is specified, configs will be retrieved irrespective of whether `CONFIGS` is specified in fields query param + or not. Please note that URL unsafe characters such as spaces will have to be suitably encoded. +1. `fields` - Specifies which fields to retrieve. Possible values for fields can be `EVENTS`, `INFO`, `CONFIGS`, `METRICS`, `RELATES_TO`, + `IS_RELATED_TO` and `ALL`. All fields will be retrieved if `ALL` is specified. Multiple fields can be specified as a comma-separated list. + If fields is not specified, in response, entity id, entity type, createdtime and UID in info field will be returned. +1. `metricslimit` - If specified, defines the number of metrics to return. Considered only if fields contains METRICS/ALL + or metricstoretrieve is specified. Ignored otherwise. The maximum possible value for metricslimit can be maximum value of + Integer. If it is not specified or has a value less than 1, and metrics have to be retrieved, then metricslimit will be + considered as 1 i.e. latest single value of metric(s) will be returned. +1. `metricsTimeStart` - If specified, then metrics for the entity after this timestamp are returned. +1. `metricsTimeEnd` - If specified, then metrics for the entity before this timestamp are returned. +1. `fromid` - If specified, retrieve the next set of generic entities from the given fromid. The set of entities retrieved is inclusive of specified fromid. + fromid should be taken from the value associated with FROM_ID info key in flow entity response which was sent earlier. + +#### Example JSON Response: + + [ + { + "metrics": [ ], + "events": [ ], + "type": "TEZ_DAG_ID", + "id": "dag_1465246237936_0001_000001", + "createdtime": 1465246358873, + "isrelatedto": { }, + "configs": { }, + "info": { + "UID": "yarn-cluster!sjlee!TEZ_DAG_ID!0!dag_1465246237936_0001_000001" + "FROM_ID": "sjlee!yarn-cluster!TEZ_DAG_ID!0!dag_1465246237936_0001_000001" + }, + "relatesto": { } + }, + { + "metrics": [ ], + "events": [ ], + "type": "TEZ_DAG_ID", + "id": "dag_1465246237936_0001_000002", + "createdtime": 1465246359045, + "isrelatedto": { }, + "configs": { }, + "info": { + "UID": "yarn-cluster!sjlee!TEZ_DAG_ID!0!dag_1465246237936_0001_000002!userX" + "FROM_ID": "sjlee!yarn-cluster!TEZ_DAG_ID!0!dag_1465246237936_0001_000002!userX" + }, + "relatesto": { } + } + ] + +#### Response Codes + +1. If successful, a HTTP 200(OK) response is returned. +1. If any problem occurs in parsing request, HTTP 400 (Bad Request) is returned. +1. For non-recoverable errors while retrieving data, HTTP 500 (Internal Server Error) is returned. + +### Query generic entity with in the scope of Application With this API, you can query a specific generic entity identified by cluster ID, application ID, per-framework entity type and entity ID. If the REST endpoint without the cluster name is used, the @@ -1166,6 +1427,9 @@ container ID. Similarly, application attempt can be queried by specifying entity or metricstoretrieve is specified. Ignored otherwise. The maximum possible value for metricslimit can be maximum value of Integer. If it is not specified or has a value less than 1, and metrics have to be retrieved, then metricslimit will be considered as 1 i.e. latest single value of metric(s) will be returned. +1. `metricsTimeStart` - If specified, then metrics for the entity after this timestamp are returned. +1. `metricsTimeEnd` - If specified, then metrics for the entity before this timestamp are returned. +1. `entityidprefix` Defines the id prefix for the entity to be fetched. If specified, then entity retrieval will be faster. #### Example JSON Response: @@ -1178,7 +1442,8 @@ container ID. Similarly, application attempt can be queried by specifying entity "isrelatedto": { }, "configs": { }, "info": { - "UID": "yarn-cluster!application_1465246237936_0001!YARN_APPLICATION_ATTEMPT!appattempt_1465246237936_0001_000001" + "UID": "yarn-cluster!application_1465246237936_0001!YARN_APPLICATION_ATTEMPT!0!appattempt_1465246237936_0001_000001" + "FROM_ID": "yarn-cluster!sjlee!ds-date!1460419580171!application_1465246237936_0001!YARN_APPLICATION_ATTEMPT!0!appattempt_1465246237936_0001_000001" }, "relatesto": { } } @@ -1189,3 +1454,117 @@ container ID. Similarly, application attempt can be queried by specifying entity 1. If any problem occurs in parsing request, HTTP 400 (Bad Request) is returned. 1. If flow context information cannot be retrieved or entity for the given entity id cannot be found, HTTP 404 (Not Found) is returned. 1. For non-recoverable errors while retrieving data, HTTP 500 (Internal Server Error) is returned. + +### Query generic entity. + +With this API, you can query generic entity per user identified by cluster ID, doAsUser and +entity type and entity ID. If the REST endpoint without the cluster name is used, the cluster +specified by the configuration `yarn.resourcemanager.cluster-id` in `yarn-site.xml` is taken. +If number of matching entities are more than the limit, the most recent +entities up to the limit will be returned. This endpoint can be used to query generic entity which +clients put into the backend. For instance, we can query user entities by specifying entity type as `TEZ_DAG_ID`. +If none of the entities match the predicates, an empty list will be returned. +**Note** : As of today, we can query only those entities which are published with doAsUser which is different from application owner. + +#### HTTP request: + + GET /ws/v2/timeline/clusters/{cluster name}/users/{userid}/entities/{entitytype}/{entityid} + + or + + GET /ws/v2/timeline/users/{userid}/entities/{entitytype}/{entityid} + +#### Query Parameters Supported: + +1. `metricstoretrieve` - If specified, defines which metrics to retrieve or which ones not to retrieve and send back in response. + metricstoretrieve can be an expression of the form :
    + (<metricprefix>,<metricprefix>,<metricprefix>,<metricprefix>...)
    + This specifies a comma separated list of metric id prefixes. Only metrics matching any of the prefixes will be retrieved. Brackets are optional for + the simple expression. Alternatively, expressions can be of the form:
    + !(<metricprefix>,<metricprefix>,<metricprefix>,<metricprefix>...)
    + This specifies a comma separated list of metric id prefixes. Only metrics not matching any of the prefixes will be retrieved.
    + If metricstoretrieve is specified, metrics will be retrieved irrespective of whether `METRICS` is specified in fields query param + or not. Please note that URL unsafe characters such as spaces will have to be suitably encoded. +1. `confstoretrieve` - If specified, defines which configs to retrieve or which ones not to retrieve and send back in response. + confstoretrieve can be an expression of the form :
    + (<config\_name\_prefix>,<config\_name\_prefix>,<config\_name\_prefix>,<config\_name\_prefix>...)
    + This specifies a comma separated list of config name prefixes. Only configs matching any of the prefixes will be retrieved. Brackets are optional for + the simple expression. Alternatively, expressions can be of the form:
    + !(<config\_name\_prefix>,<config\_name\_prefix>,<config\_name\_prefix>,<config\_name\_prefix>...)
    + This specifies a comma separated list of config name prefixes. Only configs not matching any of the prefixes will be retrieved.
    + If confstoretrieve is specified, configs will be retrieved irrespective of whether `CONFIGS` is specified in fields query param + or not. Please note that URL unsafe characters such as spaces will have to be suitably encoded. +1. `fields` - Specifies which fields to retrieve. Possible values for fields can be `EVENTS`, `INFO`, `CONFIGS`, `METRICS`, `RELATES_TO`, + `IS_RELATED_TO` and `ALL`. All fields will be retrieved if `ALL` is specified. Multiple fields can be specified as a comma-separated list. + If fields is not specified, in response, entity id, entity type, createdtime and UID in info field will be returned. +1. `metricslimit` - If specified, defines the number of metrics to return. Considered only if fields contains METRICS/ALL + or metricstoretrieve is specified. Ignored otherwise. The maximum possible value for metricslimit can be maximum value of + Integer. If it is not specified or has a value less than 1, and metrics have to be retrieved, then metricslimit will be + considered as 1 i.e. latest single value of metric(s) will be returned. +1. `metricsTimeStart` - If specified, then metrics for the entity after this timestamp are returned. +1. `metricsTimeEnd` - If specified, then metrics for the entity before this timestamp are returned. +1. `fromid` - If specified, retrieve the next set of generic entities from the given fromid. The set of entities retrieved is inclusive of specified fromid. + fromid should be taken from the value associated with FROM_ID info key in flow entity response which was sent earlier. + +#### Example JSON Response: + + [ + { + "metrics": [ ], + "events": [ ], + "type": "TEZ_DAG_ID", + "id": "dag_1465246237936_0001_000001", + "createdtime": 1465246358873, + "isrelatedto": { }, + "configs": { }, + "info": { + "UID": "yarn-cluster!sjlee!TEZ_DAG_ID!0!dag_1465246237936_0001_000001!userX" + "FROM_ID": "sjlee!yarn-cluster!TEZ_DAG_ID!0!dag_1465246237936_0001_000001!userX" + }, + "relatesto": { } + } + ] + +#### Response Codes + +1. If successful, a HTTP 200(OK) response is returned. +1. If any problem occurs in parsing request, HTTP 400 (Bad Request) is returned. +1. For non-recoverable errors while retrieving data, HTTP 500 (Internal Server Error) is returned. + +### Query generic entity types + +With this API, you can query set of available entity types for a given app id. If the REST endpoint without the cluster name is used, the cluster specified by the configuration yarn.resourcemanager.cluster-id in yarn-site.xml is taken. If userid, flow name and flow run id which are optional query parameters are not specified, they will be queried based on app id and cluster id from the flow context information stored in underlying storage implementation. + +#### HTTP request: + + GET /ws/v2/timeline/apps/{appid}/entity-types + + or + + GET /ws/v2/timeline/clusters/{clusterid}/apps/{appid}/entity-types + +#### Query Parameters Supported: + +1. `userid` - If specified, entity must belong to this user. This query param must be specified along with flowname and flowrunid query params, otherwise it will be ignored. + If userid, flowname and flowrunid are not specified then timeline reader will fetch flow context information based on cluster and appid while executing the query. +1. `flowname` - If specified, entity must belong to this flow name. This query param must be specified along with userid and flowrunid query params, otherwise it will be ignored. + If userid, flowname and flowrunid are not specified, we would have to fetch flow context information based on cluster and appid while executing the query. +1. `flowrunid` - If specified, entity must belong to this flow run id. This query param must be specified along with userid and flowname query params, otherwise it will be ignored. + If userid, flowname and flowrunid are not specified, we would have to fetch flow context information based on cluster and appid while executing the query. + +#### Example JSON Response: + + { + YARN_APPLICATION_ATTEMPT, + YARN_CONTAINER, + MAPREDUCE_JOB, + MAPREDUCE_TASK, + MAPREDUCE_TASK_ATTEMPT + } + +#### Response Codes + +1. If successful, a HTTP 200 (OK) response is returned. +1. If any problem occurs in parsing request, HTTP 400 (Bad Request) is returned. +1. If flow context information cannot be retrieved or entity for the given entity id cannot be found, HTTP 404 (Not Found) is returned. +1. For non-recoverable errors while retrieving data, HTTP 500 (Internal Server Error) is returned.