From 74ffc7a74dc6cdd5a615cd2267400873f3c65ceb Mon Sep 17 00:00:00 2001 From: Siddharth Seth Date: Tue, 22 Jan 2013 19:07:08 +0000 Subject: [PATCH] MAPREDUCE-4946. Fix a performance problem for large jobs by reducing the number of map completion event type conversions. Contributed by Jason Lowe. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1437103 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 + .../mapred/TaskAttemptListenerImpl.java | 5 +- .../hadoop/mapreduce/v2/app/job/Job.java | 3 +- .../mapreduce/v2/app/job/impl/JobImpl.java | 75 +++++++++++++------ .../mapred/TestTaskAttemptListenerImpl.java | 10 ++- .../hadoop/mapreduce/v2/app/MockJobs.java | 3 +- .../mapreduce/v2/app/TestFetchFailure.java | 18 +++-- .../v2/app/TestRuntimeEstimators.java | 3 +- .../hadoop/mapreduce/v2/hs/CompletedJob.java | 7 +- .../hadoop/mapreduce/v2/hs/PartialJob.java | 3 +- .../mapreduce/v2/hs/MockHistoryJobs.java | 3 +- 11 files changed, 90 insertions(+), 43 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index f4c6ce947c9..be38a08089f 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -654,6 +654,9 @@ Release 0.23.7 - UNRELEASED OPTIMIZATIONS + MAPREDUCE-4946. Fix a performance problem for large jobs by reducing the + number of map completion event type conversions. (Jason Lowe via sseth) + BUG FIXES MAPREDUCE-4458. Warn if java.library.path is used for AM or Task diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java index f32b5d59b7c..38a43454ee0 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java @@ -275,14 +275,13 @@ public class TaskAttemptListenerImpl extends CompositeService boolean shouldReset = false; org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId attemptID = TypeConverter.toYarn(taskAttemptID); - org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEvent[] events = + TaskCompletionEvent[] events = context.getJob(attemptID.getTaskId().getJobId()).getMapAttemptCompletionEvents( startIndex, maxEvents); taskHeartbeatHandler.progressing(attemptID); - return new MapTaskCompletionEventsUpdate( - TypeConverter.fromYarn(events), shouldReset); + return new MapTaskCompletionEventsUpdate(events, shouldReset); } @Override diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/Job.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/Job.java index ffa245bfb40..b14abcc6d55 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/Job.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/Job.java @@ -24,6 +24,7 @@ import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.mapred.TaskCompletionEvent; import org.apache.hadoop.mapreduce.Counters; import org.apache.hadoop.mapreduce.JobACL; import org.apache.hadoop.mapreduce.v2.api.records.AMInfo; @@ -88,7 +89,7 @@ public interface Job { TaskAttemptCompletionEvent[] getTaskAttemptCompletionEvents(int fromEventId, int maxEvents); - TaskAttemptCompletionEvent[] + TaskCompletionEvent[] getMapAttemptCompletionEvents(int startIndex, int maxEvents); /** diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java index bf5e96ce1c2..fa8764a412f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java @@ -43,6 +43,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapred.JobACLsManager; import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.TaskCompletionEvent; import org.apache.hadoop.mapreduce.Counters; import org.apache.hadoop.mapreduce.JobACL; import org.apache.hadoop.mapreduce.JobContext; @@ -130,6 +131,9 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, private static final TaskAttemptCompletionEvent[] EMPTY_TASK_ATTEMPT_COMPLETION_EVENTS = new TaskAttemptCompletionEvent[0]; + private static final TaskCompletionEvent[] + EMPTY_TASK_COMPLETION_EVENTS = new TaskCompletionEvent[0]; + private static final Log LOG = LogFactory.getLog(JobImpl.class); //The maximum fraction of fetch failures allowed for a map @@ -196,7 +200,8 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, private int allowedMapFailuresPercent = 0; private int allowedReduceFailuresPercent = 0; private List taskAttemptCompletionEvents; - private List mapAttemptCompletionEvents; + private List mapAttemptCompletionEvents; + private List taskCompletionIdxToMapCompletionIdx; private final List diagnostics = new ArrayList(); //task/attempt related datastructures @@ -684,27 +689,31 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, @Override public TaskAttemptCompletionEvent[] getTaskAttemptCompletionEvents( int fromEventId, int maxEvents) { - return getAttemptCompletionEvents(taskAttemptCompletionEvents, - fromEventId, maxEvents); - } - - @Override - public TaskAttemptCompletionEvent[] getMapAttemptCompletionEvents( - int startIndex, int maxEvents) { - return getAttemptCompletionEvents(mapAttemptCompletionEvents, - startIndex, maxEvents); - } - - private TaskAttemptCompletionEvent[] getAttemptCompletionEvents( - List eventList, - int startIndex, int maxEvents) { TaskAttemptCompletionEvent[] events = EMPTY_TASK_ATTEMPT_COMPLETION_EVENTS; readLock.lock(); try { - if (eventList.size() > startIndex) { + if (taskAttemptCompletionEvents.size() > fromEventId) { int actualMax = Math.min(maxEvents, - (eventList.size() - startIndex)); - events = eventList.subList(startIndex, + (taskAttemptCompletionEvents.size() - fromEventId)); + events = taskAttemptCompletionEvents.subList(fromEventId, + actualMax + fromEventId).toArray(events); + } + return events; + } finally { + readLock.unlock(); + } + } + + @Override + public TaskCompletionEvent[] getMapAttemptCompletionEvents( + int startIndex, int maxEvents) { + TaskCompletionEvent[] events = EMPTY_TASK_COMPLETION_EVENTS; + readLock.lock(); + try { + if (mapAttemptCompletionEvents.size() > startIndex) { + int actualMax = Math.min(maxEvents, + (mapAttemptCompletionEvents.size() - startIndex)); + events = mapAttemptCompletionEvents.subList(startIndex, actualMax + startIndex).toArray(events); } return events; @@ -1247,7 +1256,9 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, new ArrayList( job.numMapTasks + job.numReduceTasks + 10); job.mapAttemptCompletionEvents = - new ArrayList(job.numMapTasks + 10); + new ArrayList(job.numMapTasks + 10); + job.taskCompletionIdxToMapCompletionIdx = new ArrayList( + job.numMapTasks + job.numReduceTasks + 10); job.allowedMapFailuresPercent = job.conf.getInt(MRJobConfig.MAP_FAILURES_MAX_PERCENT, 0); @@ -1562,19 +1573,37 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, //eventId is equal to index in the arraylist tce.setEventId(job.taskAttemptCompletionEvents.size()); job.taskAttemptCompletionEvents.add(tce); + int mapEventIdx = -1; if (TaskType.MAP.equals(tce.getAttemptId().getTaskId().getTaskType())) { - job.mapAttemptCompletionEvents.add(tce); + // we track map completions separately from task completions because + // - getMapAttemptCompletionEvents uses index ranges specific to maps + // - type converting the same events over and over is expensive + mapEventIdx = job.mapAttemptCompletionEvents.size(); + job.mapAttemptCompletionEvents.add(TypeConverter.fromYarn(tce)); } + job.taskCompletionIdxToMapCompletionIdx.add(mapEventIdx); TaskAttemptId attemptId = tce.getAttemptId(); TaskId taskId = attemptId.getTaskId(); //make the previous completion event as obsolete if it exists - Object successEventNo = - job.successAttemptCompletionEventNoMap.remove(taskId); + Integer successEventNo = + job.successAttemptCompletionEventNoMap.remove(taskId); if (successEventNo != null) { TaskAttemptCompletionEvent successEvent = - job.taskAttemptCompletionEvents.get((Integer) successEventNo); + job.taskAttemptCompletionEvents.get(successEventNo); successEvent.setStatus(TaskAttemptCompletionEventStatus.OBSOLETE); + int mapCompletionIdx = + job.taskCompletionIdxToMapCompletionIdx.get(successEventNo); + if (mapCompletionIdx >= 0) { + // update the corresponding TaskCompletionEvent for the map + TaskCompletionEvent mapEvent = + job.mapAttemptCompletionEvents.get(mapCompletionIdx); + job.mapAttemptCompletionEvents.set(mapCompletionIdx, + new TaskCompletionEvent(mapEvent.getEventId(), + mapEvent.getTaskAttemptId(), mapEvent.idWithinJob(), + mapEvent.isMapTask(), TaskCompletionEvent.Status.OBSOLETE, + mapEvent.getTaskTrackerHttp())); + } } // if this attempt is not successful then why is the previous successful diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestTaskAttemptListenerImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestTaskAttemptListenerImpl.java index dfeed7f3f49..b58ad347e67 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestTaskAttemptListenerImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestTaskAttemptListenerImpl.java @@ -34,6 +34,7 @@ import java.util.Arrays; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapreduce.TaskType; +import org.apache.hadoop.mapreduce.TypeConverter; import org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager; import org.apache.hadoop.mapreduce.v2.api.records.JobId; import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEvent; @@ -153,9 +154,12 @@ public class TestTaskAttemptListenerImpl { .thenReturn(Arrays.copyOfRange(taskEvents, 0, 2)); when(mockJob.getTaskAttemptCompletionEvents(2, 100)) .thenReturn(Arrays.copyOfRange(taskEvents, 2, 4)); - when(mockJob.getMapAttemptCompletionEvents(0, 100)).thenReturn(mapEvents); - when(mockJob.getMapAttemptCompletionEvents(0, 2)).thenReturn(mapEvents); - when(mockJob.getMapAttemptCompletionEvents(2, 100)).thenReturn(empty); + when(mockJob.getMapAttemptCompletionEvents(0, 100)).thenReturn( + TypeConverter.fromYarn(mapEvents)); + when(mockJob.getMapAttemptCompletionEvents(0, 2)).thenReturn( + TypeConverter.fromYarn(mapEvents)); + when(mockJob.getMapAttemptCompletionEvents(2, 100)).thenReturn( + TypeConverter.fromYarn(empty)); AppContext appCtx = mock(AppContext.class); when(appCtx.getJob(any(JobId.class))).thenReturn(mockJob); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java index 638a8da86ca..5bab5cd3518 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java @@ -33,6 +33,7 @@ import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapred.JobACLsManager; import org.apache.hadoop.mapred.ShuffleHandler; +import org.apache.hadoop.mapred.TaskCompletionEvent; import org.apache.hadoop.mapreduce.Counters; import org.apache.hadoop.mapreduce.FileSystemCounter; import org.apache.hadoop.mapreduce.JobACL; @@ -556,7 +557,7 @@ public class MockJobs extends MockApps { } @Override - public TaskAttemptCompletionEvent[] getMapAttemptCompletionEvents( + public TaskCompletionEvent[] getMapAttemptCompletionEvents( int startIndex, int maxEvents) { return null; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFetchFailure.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFetchFailure.java index 0c9832477a8..8edd07c2774 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFetchFailure.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFetchFailure.java @@ -25,8 +25,10 @@ import java.util.Arrays; import java.util.Iterator; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.mapred.TaskCompletionEvent; import org.apache.hadoop.mapreduce.Counters; import org.apache.hadoop.mapreduce.MRJobConfig; +import org.apache.hadoop.mapreduce.TypeConverter; import org.apache.hadoop.mapreduce.jobhistory.JobHistoryEvent; import org.apache.hadoop.mapreduce.jobhistory.JobHistoryEventHandler; import org.apache.hadoop.mapreduce.v2.api.records.JobState; @@ -150,14 +152,16 @@ public class TestFetchFailure { Assert.assertEquals("Event status not correct for reduce attempt1", TaskAttemptCompletionEventStatus.SUCCEEDED, events[3].getStatus()); - TaskAttemptCompletionEvent mapEvents[] = + TaskCompletionEvent mapEvents[] = job.getMapAttemptCompletionEvents(0, 2); + TaskCompletionEvent convertedEvents[] = TypeConverter.fromYarn(events); Assert.assertEquals("Incorrect number of map events", 2, mapEvents.length); Assert.assertArrayEquals("Unexpected map events", - Arrays.copyOfRange(events, 0, 2), mapEvents); + Arrays.copyOfRange(convertedEvents, 0, 2), mapEvents); mapEvents = job.getMapAttemptCompletionEvents(2, 200); Assert.assertEquals("Incorrect number of map events", 1, mapEvents.length); - Assert.assertEquals("Unexpected map event", events[2], mapEvents[0]); + Assert.assertEquals("Unexpected map event", convertedEvents[2], + mapEvents[0]); } /** @@ -395,14 +399,16 @@ public class TestFetchFailure { Assert.assertEquals("Event status not correct for reduce attempt1", TaskAttemptCompletionEventStatus.SUCCEEDED, events[3].getStatus()); - TaskAttemptCompletionEvent mapEvents[] = + TaskCompletionEvent mapEvents[] = job.getMapAttemptCompletionEvents(0, 2); + TaskCompletionEvent convertedEvents[] = TypeConverter.fromYarn(events); Assert.assertEquals("Incorrect number of map events", 2, mapEvents.length); Assert.assertArrayEquals("Unexpected map events", - Arrays.copyOfRange(events, 0, 2), mapEvents); + Arrays.copyOfRange(convertedEvents, 0, 2), mapEvents); mapEvents = job.getMapAttemptCompletionEvents(2, 200); Assert.assertEquals("Incorrect number of map events", 1, mapEvents.length); - Assert.assertEquals("Unexpected map event", events[2], mapEvents[0]); + Assert.assertEquals("Unexpected map event", convertedEvents[2], + mapEvents[0]); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java index be897fa37db..2ddab831275 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java @@ -32,6 +32,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.mapred.TaskCompletionEvent; import org.apache.hadoop.mapreduce.Counters; import org.apache.hadoop.mapreduce.JobACL; import org.apache.hadoop.mapreduce.v2.api.records.AMInfo; @@ -441,7 +442,7 @@ public class TestRuntimeEstimators { } @Override - public TaskAttemptCompletionEvent[] + public TaskCompletionEvent[] getMapAttemptCompletionEvents(int startIndex, int maxEvents) { throw new UnsupportedOperationException("Not supported yet."); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java index cfa7e290595..2f3c57d6b8c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java @@ -36,6 +36,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapred.JobACLsManager; +import org.apache.hadoop.mapred.TaskCompletionEvent; import org.apache.hadoop.mapreduce.Counters; import org.apache.hadoop.mapreduce.JobACL; import org.apache.hadoop.mapreduce.TaskID; @@ -183,13 +184,13 @@ public class CompletedJob implements org.apache.hadoop.mapreduce.v2.app.job.Job } @Override - public synchronized TaskAttemptCompletionEvent[] getMapAttemptCompletionEvents( + public synchronized TaskCompletionEvent[] getMapAttemptCompletionEvents( int startIndex, int maxEvents) { if (mapCompletionEvents == null) { constructTaskAttemptCompletionEvents(); } - return getAttemptCompletionEvents(mapCompletionEvents, - startIndex, maxEvents); + return TypeConverter.fromYarn(getAttemptCompletionEvents( + mapCompletionEvents, startIndex, maxEvents)); } private static TaskAttemptCompletionEvent[] getAttemptCompletionEvents( diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/PartialJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/PartialJob.java index 0bfffac1b07..ce51c390b15 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/PartialJob.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/PartialJob.java @@ -25,6 +25,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.mapred.TaskCompletionEvent; import org.apache.hadoop.mapreduce.Counters; import org.apache.hadoop.mapreduce.JobACL; import org.apache.hadoop.mapreduce.v2.api.records.AMInfo; @@ -154,7 +155,7 @@ public class PartialJob implements org.apache.hadoop.mapreduce.v2.app.job.Job { } @Override - public TaskAttemptCompletionEvent[] getMapAttemptCompletionEvents( + public TaskCompletionEvent[] getMapAttemptCompletionEvents( int startIndex, int maxEvents) { return null; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/MockHistoryJobs.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/MockHistoryJobs.java index ac37bbce7ff..00b62a79734 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/MockHistoryJobs.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/MockHistoryJobs.java @@ -23,6 +23,7 @@ import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.mapred.TaskCompletionEvent; import org.apache.hadoop.mapreduce.JobACL; import org.apache.hadoop.mapreduce.v2.api.records.AMInfo; import org.apache.hadoop.mapreduce.v2.api.records.JobId; @@ -143,7 +144,7 @@ public class MockHistoryJobs extends MockJobs { } @Override - public TaskAttemptCompletionEvent[] getMapAttemptCompletionEvents( + public TaskCompletionEvent[] getMapAttemptCompletionEvents( int startIndex, int maxEvents) { return job.getMapAttemptCompletionEvents(startIndex, maxEvents); }