From 070ae0d73e7726c4687514d4ddaec4e3b16a6d62 Mon Sep 17 00:00:00 2001 From: Robert Joseph Evans Date: Wed, 17 Oct 2012 15:01:10 +0000 Subject: [PATCH] svn merge -c 1399289 FIXES: MAPREDUCE-4721. Task startup time in JHS is same as job startup time. (Ravi Prakash via bobby) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1399291 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 + .../hadoop/mapreduce/v2/hs/CompletedTask.java | 7 +- .../mapreduce/v2/hs/TestCompletedTask.java | 66 +++++++++++++++++++ .../src/site/apt/HistoryServerRest.apt.vm | 4 +- 4 files changed, 77 insertions(+), 3 deletions(-) create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestCompletedTask.java diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index bc7d1847852..120a978e228 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -440,6 +440,9 @@ Release 0.23.5 - UNRELEASED MAPREDUCE-4521. mapreduce.user.classpath.first incompatibility with 0.20/1.x (Ravi Prakash via bobby) + MAPREDUCE-4721. Task startup time in JHS is same as job startup time. + (Ravi Prakash via bobby) + Release 0.23.4 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedTask.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedTask.java index 830b64f1ad3..8469b27dd9d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedTask.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedTask.java @@ -122,7 +122,12 @@ public class CompletedTask implements Task { loadAllTaskAttempts(); this.report = Records.newRecord(TaskReport.class); report.setTaskId(taskId); - report.setStartTime(taskInfo.getStartTime()); + long minLaunchTime = Long.MAX_VALUE; + for(TaskAttempt attempt: attempts.values()) { + minLaunchTime = Math.min(minLaunchTime, attempt.getLaunchTime()); + } + minLaunchTime = minLaunchTime == Long.MAX_VALUE ? -1 : minLaunchTime; + report.setStartTime(minLaunchTime); report.setFinishTime(taskInfo.getFinishTime()); report.setTaskState(getState()); report.setProgress(getProgress()); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestCompletedTask.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestCompletedTask.java new file mode 100644 index 00000000000..23e636d0dfc --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestCompletedTask.java @@ -0,0 +1,66 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.mapreduce.v2.hs; + +import java.util.Map; +import java.util.TreeMap; + +import org.apache.hadoop.mapreduce.TaskAttemptID; +import org.apache.hadoop.mapreduce.TaskType; +import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskAttemptInfo; +import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskInfo; +import org.apache.hadoop.mapreduce.v2.api.records.TaskId; +import org.apache.hadoop.mapreduce.v2.api.records.TaskReport; +import org.apache.hadoop.mapreduce.v2.hs.CompletedTask; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; + +public class TestCompletedTask{ + + @Test + public void testTaskStartTimes() { + + TaskId taskId = Mockito.mock(TaskId.class); + TaskInfo taskInfo = Mockito.mock(TaskInfo.class); + Map taskAttempts + = new TreeMap(); + + TaskAttemptID id = new TaskAttemptID("0", 0, TaskType.MAP, 0, 0); + TaskAttemptInfo info = Mockito.mock(TaskAttemptInfo.class); + Mockito.when(info.getAttemptId()).thenReturn(id); + Mockito.when(info.getStartTime()).thenReturn(10l); + taskAttempts.put(id, info); + + id = new TaskAttemptID("1", 0, TaskType.MAP, 1, 1); + info = Mockito.mock(TaskAttemptInfo.class); + Mockito.when(info.getAttemptId()).thenReturn(id); + Mockito.when(info.getStartTime()).thenReturn(20l); + taskAttempts.put(id, info); + + + Mockito.when(taskInfo.getAllTaskAttempts()).thenReturn(taskAttempts); + CompletedTask task = new CompletedTask(taskId, taskInfo); + TaskReport report = task.getReport(); + + // Make sure the startTime returned by report is the lesser of the + // attempy launch times + Assert.assertTrue(report.getStartTime() == 10); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm index 5b11736b8fe..358f8ca0230 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm @@ -1527,7 +1527,7 @@ History Server REST API's. *---------------+--------------+--------------------------------+ | progress | float | The progress of the task as a percent| *---------------+--------------+--------------------------------+ -| startTime | long | The time in which the task started (in ms since epoch)| +| startTime | long | The time in which the task started (in ms since epoch) or -1 if it was never started | *---------------+--------------+--------------------------------+ | finishTime | long | The time in which the task finished (in ms since epoch)| *---------------+--------------+--------------------------------+ @@ -1607,7 +1607,7 @@ History Server REST API's. ** Task Counters API - With the task counters API, you can object a collection of resources that represent al the counters for that task. + With the task counters API, you can object a collection of resources that represent all the counters for that task. *** URI