MAPREDUCE-5007. fix coverage org.apache.hadoop.mapreduce.v2.hs (Aleksey Gorshkov via tgraves)
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1465016 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Thomas Graves
parent
d002c038b2
commit
003ecf8137
|
|
@ -717,6 +717,9 @@ Release 0.23.7 - UNRELEASED
|
|||
|
||||
MAPREDUCE-4991. coverage for gridmix (Aleksey Gorshkov via tgraves)
|
||||
|
||||
MAPREDUCE-5007. fix coverage org.apache.hadoop.mapreduce.v2.hs (Aleksey
|
||||
Gorshkov via tgraves)
|
||||
|
||||
Release 0.23.6 - UNRELEASED
|
||||
|
||||
INCOMPATIBLE CHANGES
|
||||
|
|
|
|||
|
|
@ -869,4 +869,8 @@ public class HistoryFileManager extends AbstractService {
|
|||
}
|
||||
}
|
||||
}
|
||||
@VisibleForTesting
|
||||
protected void setMaxHistoryAge(long newValue){
|
||||
maxHistoryAge=newValue;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,46 +21,75 @@ package org.apache.hadoop.mapreduce.v2.hs;
|
|||
import java.util.Map;
|
||||
import java.util.TreeMap;
|
||||
|
||||
import org.apache.hadoop.mapreduce.JobID;
|
||||
import org.apache.hadoop.mapreduce.TaskAttemptID;
|
||||
import org.apache.hadoop.mapreduce.TaskID;
|
||||
import org.apache.hadoop.mapreduce.TaskType;
|
||||
import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskAttemptInfo;
|
||||
import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskInfo;
|
||||
import org.apache.hadoop.mapreduce.v2.api.records.Phase;
|
||||
import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
|
||||
import org.apache.hadoop.mapreduce.v2.api.records.TaskReport;
|
||||
import org.apache.hadoop.mapreduce.v2.hs.CompletedTask;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import org.mockito.Mockito;
|
||||
import static org.mockito.Mockito.*;
|
||||
import static org.junit.Assert.*;
|
||||
|
||||
public class TestCompletedTask{
|
||||
|
||||
@Test
|
||||
@Test (timeout=5000)
|
||||
public void testTaskStartTimes() {
|
||||
|
||||
TaskId taskId = Mockito.mock(TaskId.class);
|
||||
TaskInfo taskInfo = Mockito.mock(TaskInfo.class);
|
||||
TaskId taskId = mock(TaskId.class);
|
||||
TaskInfo taskInfo = mock(TaskInfo.class);
|
||||
Map<TaskAttemptID, TaskAttemptInfo> taskAttempts
|
||||
= new TreeMap<TaskAttemptID, TaskAttemptInfo>();
|
||||
|
||||
TaskAttemptID id = new TaskAttemptID("0", 0, TaskType.MAP, 0, 0);
|
||||
TaskAttemptInfo info = Mockito.mock(TaskAttemptInfo.class);
|
||||
Mockito.when(info.getAttemptId()).thenReturn(id);
|
||||
Mockito.when(info.getStartTime()).thenReturn(10l);
|
||||
TaskAttemptInfo info = mock(TaskAttemptInfo.class);
|
||||
when(info.getAttemptId()).thenReturn(id);
|
||||
when(info.getStartTime()).thenReturn(10l);
|
||||
taskAttempts.put(id, info);
|
||||
|
||||
id = new TaskAttemptID("1", 0, TaskType.MAP, 1, 1);
|
||||
info = Mockito.mock(TaskAttemptInfo.class);
|
||||
Mockito.when(info.getAttemptId()).thenReturn(id);
|
||||
Mockito.when(info.getStartTime()).thenReturn(20l);
|
||||
info = mock(TaskAttemptInfo.class);
|
||||
when(info.getAttemptId()).thenReturn(id);
|
||||
when(info.getStartTime()).thenReturn(20l);
|
||||
taskAttempts.put(id, info);
|
||||
|
||||
|
||||
Mockito.when(taskInfo.getAllTaskAttempts()).thenReturn(taskAttempts);
|
||||
when(taskInfo.getAllTaskAttempts()).thenReturn(taskAttempts);
|
||||
CompletedTask task = new CompletedTask(taskId, taskInfo);
|
||||
TaskReport report = task.getReport();
|
||||
|
||||
// Make sure the startTime returned by report is the lesser of the
|
||||
// attempy launch times
|
||||
Assert.assertTrue(report.getStartTime() == 10);
|
||||
assertTrue(report.getStartTime() == 10);
|
||||
}
|
||||
/**
|
||||
* test some methods of CompletedTaskAttempt
|
||||
*/
|
||||
@Test (timeout=5000)
|
||||
public void testCompletedTaskAttempt(){
|
||||
|
||||
TaskAttemptInfo attemptInfo= mock(TaskAttemptInfo.class);
|
||||
when(attemptInfo.getRackname()).thenReturn("Rackname");
|
||||
when(attemptInfo.getShuffleFinishTime()).thenReturn(11L);
|
||||
when(attemptInfo.getSortFinishTime()).thenReturn(12L);
|
||||
when(attemptInfo.getShufflePort()).thenReturn(10);
|
||||
|
||||
JobID jobId= new JobID("12345",0);
|
||||
TaskID taskId =new TaskID(jobId,TaskType.REDUCE, 0);
|
||||
TaskAttemptID taskAttemptId= new TaskAttemptID(taskId, 0);
|
||||
when(attemptInfo.getAttemptId()).thenReturn(taskAttemptId);
|
||||
|
||||
|
||||
CompletedTaskAttempt taskAttemt= new CompletedTaskAttempt(null,attemptInfo);
|
||||
assertEquals( "Rackname", taskAttemt.getNodeRackName());
|
||||
assertEquals( Phase.CLEANUP, taskAttemt.getPhase());
|
||||
assertTrue( taskAttemt.isFinished());
|
||||
assertEquals( 11L, taskAttemt.getShuffleFinishTime());
|
||||
assertEquals( 12L, taskAttemt.getSortFinishTime());
|
||||
assertEquals( 10, taskAttemt.getShufflePort());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ import java.util.Map;
|
|||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.mapred.JobACLsManager;
|
||||
import org.apache.hadoop.mapred.TaskCompletionEvent;
|
||||
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
|
||||
import org.apache.hadoop.mapreduce.v2.api.records.JobReport;
|
||||
import org.apache.hadoop.mapreduce.v2.api.records.JobState;
|
||||
|
|
@ -46,6 +47,7 @@ import org.junit.runner.RunWith;
|
|||
import org.junit.runners.Parameterized;
|
||||
import org.junit.runners.Parameterized.Parameters;
|
||||
|
||||
import static org.junit.Assert.*;
|
||||
import static org.mockito.Mockito.*;
|
||||
|
||||
@RunWith(value = Parameterized.class)
|
||||
|
|
@ -79,7 +81,7 @@ public class TestJobHistoryEntities {
|
|||
}
|
||||
|
||||
/* Verify some expected values based on the history file */
|
||||
@Test (timeout=10000)
|
||||
@Test (timeout=100000)
|
||||
public void testCompletedJob() throws Exception {
|
||||
HistoryFileInfo info = mock(HistoryFileInfo.class);
|
||||
when(info.getConfFile()).thenReturn(fullConfPath);
|
||||
|
|
@ -168,4 +170,45 @@ public class TestJobHistoryEntities {
|
|||
assertEquals(45454, rta1Report.getNodeManagerPort());
|
||||
assertEquals(9999, rta1Report.getNodeManagerHttpPort());
|
||||
}
|
||||
/**
|
||||
* Simple test of some methods of CompletedJob
|
||||
* @throws Exception
|
||||
*/
|
||||
@Test (timeout=30000)
|
||||
public void testGetTaskAttemptCompletionEvent() throws Exception{
|
||||
HistoryFileInfo info = mock(HistoryFileInfo.class);
|
||||
when(info.getConfFile()).thenReturn(fullConfPath);
|
||||
completedJob =
|
||||
new CompletedJob(conf, jobId, fulleHistoryPath, loadTasks, "user",
|
||||
info, jobAclsManager);
|
||||
TaskCompletionEvent[] events= completedJob.getMapAttemptCompletionEvents(0,1000);
|
||||
assertEquals(10, completedJob.getMapAttemptCompletionEvents(0,10).length);
|
||||
int currentEventId=0;
|
||||
for (TaskCompletionEvent taskAttemptCompletionEvent : events) {
|
||||
int eventId= taskAttemptCompletionEvent.getEventId();
|
||||
assertTrue(eventId>=currentEventId);
|
||||
currentEventId=eventId;
|
||||
}
|
||||
assertNull(completedJob.loadConfFile() );
|
||||
// job name
|
||||
assertEquals("Sleep job",completedJob.getName());
|
||||
// queue name
|
||||
assertEquals("default",completedJob.getQueueName());
|
||||
// progress
|
||||
assertEquals(1.0, completedJob.getProgress(),0.001);
|
||||
// 11 rows in answer
|
||||
assertEquals(11,completedJob.getTaskAttemptCompletionEvents(0,1000).length);
|
||||
// select first 10 rows
|
||||
assertEquals(10,completedJob.getTaskAttemptCompletionEvents(0,10).length);
|
||||
// select 5-10 rows include 5th
|
||||
assertEquals(6,completedJob.getTaskAttemptCompletionEvents(5,10).length);
|
||||
|
||||
// without errors
|
||||
assertEquals(1,completedJob.getDiagnostics().size());
|
||||
assertEquals("",completedJob.getDiagnostics().get(0));
|
||||
|
||||
assertEquals(0, completedJob.getJobACLs().size());
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,6 +18,10 @@
|
|||
|
||||
package org.apache.hadoop.mapreduce.v2.hs;
|
||||
|
||||
import static junit.framework.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertNull;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.PrintStream;
|
||||
|
|
@ -54,6 +58,9 @@ import org.apache.hadoop.mapreduce.v2.api.records.JobState;
|
|||
import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
|
||||
import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
|
||||
import org.apache.hadoop.mapreduce.v2.api.records.TaskState;
|
||||
import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
|
||||
import org.apache.hadoop.mapreduce.v2.api.records.impl.pb.JobIdPBImpl;
|
||||
import org.apache.hadoop.mapreduce.v2.api.records.impl.pb.TaskIdPBImpl;
|
||||
import org.apache.hadoop.mapreduce.v2.app.MRApp;
|
||||
import org.apache.hadoop.mapreduce.v2.app.job.Job;
|
||||
import org.apache.hadoop.mapreduce.v2.app.job.Task;
|
||||
|
|
@ -65,7 +72,9 @@ import org.apache.hadoop.mapreduce.v2.hs.TestJobHistoryEvents.MRAppWithHistory;
|
|||
import org.apache.hadoop.mapreduce.v2.jobhistory.FileNameIndexUtils;
|
||||
import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils;
|
||||
import org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo;
|
||||
import org.apache.hadoop.mapreduce.v2.hs.webapp.dao.JobsInfo;
|
||||
import org.apache.hadoop.net.DNSToSwitchMapping;
|
||||
import org.apache.hadoop.security.UserGroupInformation;
|
||||
import org.apache.hadoop.yarn.api.records.ContainerId;
|
||||
import org.apache.hadoop.yarn.service.Service;
|
||||
import org.apache.hadoop.yarn.util.BuilderUtils;
|
||||
|
|
@ -85,7 +94,7 @@ public class TestJobHistoryParsing {
|
|||
public static class MyResolver implements DNSToSwitchMapping {
|
||||
@Override
|
||||
public List<String> resolve(List<String> names) {
|
||||
return Arrays.asList(new String[]{RACK_NAME});
|
||||
return Arrays.asList(new String[] { RACK_NAME });
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -93,14 +102,14 @@ public class TestJobHistoryParsing {
|
|||
}
|
||||
}
|
||||
|
||||
@Test (timeout=50000)
|
||||
@Test(timeout = 50000)
|
||||
public void testJobInfo() throws Exception {
|
||||
JobInfo info = new JobInfo();
|
||||
Assert.assertEquals("NORMAL", info.getPriority());
|
||||
info.printAll();
|
||||
}
|
||||
|
||||
@Test (timeout=50000)
|
||||
@Test(timeout = 300000)
|
||||
public void testHistoryParsing() throws Exception {
|
||||
LOG.info("STARTING testHistoryParsing()");
|
||||
try {
|
||||
|
|
@ -110,7 +119,7 @@ public class TestJobHistoryParsing {
|
|||
}
|
||||
}
|
||||
|
||||
@Test (timeout=50000)
|
||||
@Test(timeout = 50000)
|
||||
public void testHistoryParsingWithParseErrors() throws Exception {
|
||||
LOG.info("STARTING testHistoryParsingWithParseErrors()");
|
||||
try {
|
||||
|
|
@ -120,7 +129,8 @@ public class TestJobHistoryParsing {
|
|||
}
|
||||
}
|
||||
|
||||
private static String getJobSummary(FileContext fc, Path path) throws IOException {
|
||||
private static String getJobSummary(FileContext fc, Path path)
|
||||
throws IOException {
|
||||
Path qPath = fc.makeQualified(path);
|
||||
FSDataInputStream in = fc.open(qPath);
|
||||
String jobSummaryString = in.readUTF();
|
||||
|
|
@ -129,8 +139,7 @@ public class TestJobHistoryParsing {
|
|||
}
|
||||
|
||||
private void checkHistoryParsing(final int numMaps, final int numReduces,
|
||||
final int numSuccessfulMaps)
|
||||
throws Exception {
|
||||
final int numSuccessfulMaps) throws Exception {
|
||||
Configuration conf = new Configuration();
|
||||
conf.set(MRJobConfig.USER_NAME, System.getProperty("user.name"));
|
||||
long amStartTimeEst = System.currentTimeMillis();
|
||||
|
|
@ -138,9 +147,8 @@ public class TestJobHistoryParsing {
|
|||
CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
|
||||
MyResolver.class, DNSToSwitchMapping.class);
|
||||
RackResolver.init(conf);
|
||||
MRApp app =
|
||||
new MRAppWithHistory(numMaps, numReduces, true,
|
||||
this.getClass().getName(), true);
|
||||
MRApp app = new MRAppWithHistory(numMaps, numReduces, true, this.getClass()
|
||||
.getName(), true);
|
||||
app.submit(conf);
|
||||
Job job = app.getContext().getAllJobs().values().iterator().next();
|
||||
JobId jobId = job.getID();
|
||||
|
|
@ -185,20 +193,22 @@ public class TestJobHistoryParsing {
|
|||
Long.parseLong(jobSummaryElements.get("submitTime")) != 0);
|
||||
Assert.assertTrue("launchTime should not be 0",
|
||||
Long.parseLong(jobSummaryElements.get("launchTime")) != 0);
|
||||
Assert.assertTrue("firstMapTaskLaunchTime should not be 0",
|
||||
Long.parseLong(jobSummaryElements.get("firstMapTaskLaunchTime")) != 0);
|
||||
Assert
|
||||
.assertTrue(
|
||||
"firstReduceTaskLaunchTime should not be 0",
|
||||
Long.parseLong(jobSummaryElements.get("firstReduceTaskLaunchTime")) != 0);
|
||||
"firstMapTaskLaunchTime should not be 0",
|
||||
Long.parseLong(jobSummaryElements.get("firstMapTaskLaunchTime")) != 0);
|
||||
Assert
|
||||
.assertTrue("firstReduceTaskLaunchTime should not be 0",
|
||||
Long.parseLong(jobSummaryElements
|
||||
.get("firstReduceTaskLaunchTime")) != 0);
|
||||
Assert.assertTrue("finishTime should not be 0",
|
||||
Long.parseLong(jobSummaryElements.get("finishTime")) != 0);
|
||||
Assert.assertEquals("Mismatch in num map slots", numSuccessfulMaps,
|
||||
Integer.parseInt(jobSummaryElements.get("numMaps")));
|
||||
Assert.assertEquals("Mismatch in num reduce slots", numReduces,
|
||||
Integer.parseInt(jobSummaryElements.get("numReduces")));
|
||||
Assert.assertEquals("User does not match", System.getProperty("user.name"),
|
||||
jobSummaryElements.get("user"));
|
||||
Assert.assertEquals("User does not match",
|
||||
System.getProperty("user.name"), jobSummaryElements.get("user"));
|
||||
Assert.assertEquals("Queue does not match", "default",
|
||||
jobSummaryElements.get("queue"));
|
||||
Assert.assertEquals("Status does not match", "SUCCEEDED",
|
||||
|
|
@ -211,7 +221,7 @@ public class TestJobHistoryParsing {
|
|||
JobInfo jobInfo;
|
||||
long numFinishedMaps;
|
||||
|
||||
synchronized(fileInfo) {
|
||||
synchronized (fileInfo) {
|
||||
Path historyFilePath = fileInfo.getHistoryFile();
|
||||
FSDataInputStream in = null;
|
||||
LOG.info("JobHistoryFile is: " + historyFilePath);
|
||||
|
|
@ -244,14 +254,12 @@ public class TestJobHistoryParsing {
|
|||
throw new IOException("test");
|
||||
}
|
||||
}
|
||||
}
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
jobInfo = parser.parse(reader);
|
||||
|
||||
numFinishedMaps =
|
||||
computeFinishedMaps(jobInfo, numMaps, numSuccessfulMaps);
|
||||
numFinishedMaps = computeFinishedMaps(jobInfo, numMaps, numSuccessfulMaps);
|
||||
|
||||
if (numFinishedMaps != numMaps) {
|
||||
Exception parseException = parser.getParseException();
|
||||
|
|
@ -276,7 +284,7 @@ public class TestJobHistoryParsing {
|
|||
Map<TaskID, TaskInfo> allTasks = jobInfo.getAllTasks();
|
||||
int totalTasks = allTasks.size();
|
||||
Assert.assertEquals("total number of tasks is incorrect ",
|
||||
(numMaps+numReduces), totalTasks);
|
||||
(numMaps + numReduces), totalTasks);
|
||||
|
||||
// Verify aminfo
|
||||
Assert.assertEquals(1, jobInfo.getAMInfos().size());
|
||||
|
|
@ -306,8 +314,7 @@ public class TestJobHistoryParsing {
|
|||
|
||||
// Deep compare Job and JobInfo
|
||||
for (Task task : job.getTasks().values()) {
|
||||
TaskInfo taskInfo = allTasks.get(
|
||||
TypeConverter.fromYarn(task.getID()));
|
||||
TaskInfo taskInfo = allTasks.get(TypeConverter.fromYarn(task.getID()));
|
||||
Assert.assertNotNull("TaskInfo not found", taskInfo);
|
||||
for (TaskAttempt taskAttempt : task.getAttempts().values()) {
|
||||
TaskAttemptInfo taskAttemptInfo = taskInfo.getAllTaskAttempts().get(
|
||||
|
|
@ -320,14 +327,14 @@ public class TestJobHistoryParsing {
|
|||
Assert.assertEquals(MRApp.NM_PORT, taskAttemptInfo.getPort());
|
||||
|
||||
// Verify rack-name
|
||||
Assert.assertEquals("rack-name is incorrect", taskAttemptInfo
|
||||
.getRackname(), RACK_NAME);
|
||||
Assert.assertEquals("rack-name is incorrect",
|
||||
taskAttemptInfo.getRackname(), RACK_NAME);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// test output for HistoryViewer
|
||||
PrintStream stdps=System.out;
|
||||
PrintStream stdps = System.out;
|
||||
try {
|
||||
System.setOut(new PrintStream(outContent));
|
||||
HistoryViewer viewer = new HistoryViewer(fc.makeQualified(
|
||||
|
|
@ -336,9 +343,14 @@ public class TestJobHistoryParsing {
|
|||
|
||||
for (TaskInfo taskInfo : allTasks.values()) {
|
||||
|
||||
String test= (taskInfo.getTaskStatus()==null?"":taskInfo.getTaskStatus())+" "+taskInfo.getTaskType()+" task list for "+taskInfo.getTaskId().getJobID();
|
||||
Assert.assertTrue(outContent.toString().indexOf(test)>0);
|
||||
Assert.assertTrue(outContent.toString().indexOf(taskInfo.getTaskId().toString())>0);
|
||||
String test = (taskInfo.getTaskStatus() == null ? "" : taskInfo
|
||||
.getTaskStatus())
|
||||
+ " "
|
||||
+ taskInfo.getTaskType()
|
||||
+ " task list for " + taskInfo.getTaskId().getJobID();
|
||||
Assert.assertTrue(outContent.toString().indexOf(test) > 0);
|
||||
Assert.assertTrue(outContent.toString().indexOf(
|
||||
taskInfo.getTaskId().toString()) > 0);
|
||||
}
|
||||
} finally {
|
||||
System.setOut(stdps);
|
||||
|
|
@ -364,18 +376,17 @@ public class TestJobHistoryParsing {
|
|||
return numFinishedMaps;
|
||||
}
|
||||
|
||||
@Test (timeout=50000)
|
||||
@Test(timeout = 30000)
|
||||
public void testHistoryParsingForFailedAttempts() throws Exception {
|
||||
LOG.info("STARTING testHistoryParsingForFailedAttempts");
|
||||
try {
|
||||
Configuration conf = new Configuration();
|
||||
conf
|
||||
.setClass(
|
||||
conf.setClass(
|
||||
CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
|
||||
MyResolver.class, DNSToSwitchMapping.class);
|
||||
RackResolver.init(conf);
|
||||
MRApp app = new MRAppWithHistoryWithFailedAttempt(2, 1, true, this.getClass().getName(),
|
||||
true);
|
||||
MRApp app = new MRAppWithHistoryWithFailedAttempt(2, 1, true, this
|
||||
.getClass().getName(), true);
|
||||
app.submit(conf);
|
||||
Job job = app.getContext().getAllJobs().values().iterator().next();
|
||||
JobId jobId = job.getID();
|
||||
|
|
@ -418,31 +429,31 @@ public class TestJobHistoryParsing {
|
|||
TaskAttemptInfo taskAttemptInfo = taskInfo.getAllTaskAttempts().get(
|
||||
TypeConverter.fromYarn((taskAttempt.getID())));
|
||||
// Verify rack-name for all task attempts
|
||||
Assert.assertEquals("rack-name is incorrect", taskAttemptInfo
|
||||
.getRackname(), RACK_NAME);
|
||||
Assert.assertEquals("rack-name is incorrect",
|
||||
taskAttemptInfo.getRackname(), RACK_NAME);
|
||||
if (taskAttemptInfo.getTaskStatus().equals("FAILED")) {
|
||||
noOffailedAttempts++;
|
||||
}
|
||||
}
|
||||
}
|
||||
Assert.assertEquals("No of Failed tasks doesn't match.", 2, noOffailedAttempts);
|
||||
Assert.assertEquals("No of Failed tasks doesn't match.", 2,
|
||||
noOffailedAttempts);
|
||||
} finally {
|
||||
LOG.info("FINISHED testHistoryParsingForFailedAttempts");
|
||||
}
|
||||
}
|
||||
|
||||
@Test (timeout=5000)
|
||||
@Test(timeout = 60000)
|
||||
public void testCountersForFailedTask() throws Exception {
|
||||
LOG.info("STARTING testCountersForFailedTask");
|
||||
try {
|
||||
Configuration conf = new Configuration();
|
||||
conf
|
||||
.setClass(
|
||||
conf.setClass(
|
||||
CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
|
||||
MyResolver.class, DNSToSwitchMapping.class);
|
||||
RackResolver.init(conf);
|
||||
MRApp app = new MRAppWithHistoryWithFailedTask(2, 1, true,
|
||||
this.getClass().getName(), true);
|
||||
MRApp app = new MRAppWithHistoryWithFailedTask(2, 1, true, this
|
||||
.getClass().getName(), true);
|
||||
app.submit(conf);
|
||||
Job job = app.getContext().getAllJobs().values().iterator().next();
|
||||
JobId jobId = job.getID();
|
||||
|
|
@ -477,33 +488,28 @@ public class TestJobHistoryParsing {
|
|||
Exception parseException = parser.getParseException();
|
||||
Assert.assertNull("Caught an expected exception " + parseException,
|
||||
parseException);
|
||||
for (Map.Entry<TaskID,TaskInfo> entry : jobInfo.getAllTasks().entrySet()) {
|
||||
for (Map.Entry<TaskID, TaskInfo> entry : jobInfo.getAllTasks().entrySet()) {
|
||||
TaskId yarnTaskID = TypeConverter.toYarn(entry.getKey());
|
||||
CompletedTask ct = new CompletedTask(yarnTaskID, entry.getValue());
|
||||
Assert.assertNotNull("completed task report has null counters",
|
||||
ct.getReport().getCounters());
|
||||
//Make sure all the completedTask has counters, and the counters are not empty
|
||||
Assert.assertTrue(ct.getReport().getCounters()
|
||||
.getAllCounterGroups().size() > 0);
|
||||
Assert.assertNotNull("completed task report has null counters", ct
|
||||
.getReport().getCounters());
|
||||
}
|
||||
} finally {
|
||||
LOG.info("FINISHED testCountersForFailedTask");
|
||||
}
|
||||
}
|
||||
|
||||
@Test (timeout=50000)
|
||||
@Test(timeout = 50000)
|
||||
public void testScanningOldDirs() throws Exception {
|
||||
LOG.info("STARTING testScanningOldDirs");
|
||||
try {
|
||||
Configuration conf = new Configuration();
|
||||
conf
|
||||
.setClass(
|
||||
conf.setClass(
|
||||
CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
|
||||
MyResolver.class, DNSToSwitchMapping.class);
|
||||
RackResolver.init(conf);
|
||||
MRApp app =
|
||||
new MRAppWithHistory(1, 1, true,
|
||||
this.getClass().getName(), true);
|
||||
MRApp app = new MRAppWithHistory(1, 1, true, this.getClass().getName(),
|
||||
true);
|
||||
app.submit(conf);
|
||||
Job job = app.getContext().getAllJobs().values().iterator().next();
|
||||
JobId jobId = job.getID();
|
||||
|
|
@ -538,8 +544,8 @@ public class TestJobHistoryParsing {
|
|||
|
||||
static class MRAppWithHistoryWithFailedAttempt extends MRAppWithHistory {
|
||||
|
||||
public MRAppWithHistoryWithFailedAttempt(int maps, int reduces, boolean autoComplete,
|
||||
String testName, boolean cleanOnStart) {
|
||||
public MRAppWithHistoryWithFailedAttempt(int maps, int reduces,
|
||||
boolean autoComplete, String testName, boolean cleanOnStart) {
|
||||
super(maps, reduces, autoComplete, testName, cleanOnStart);
|
||||
}
|
||||
|
||||
|
|
@ -558,8 +564,8 @@ public class TestJobHistoryParsing {
|
|||
|
||||
static class MRAppWithHistoryWithFailedTask extends MRAppWithHistory {
|
||||
|
||||
public MRAppWithHistoryWithFailedTask(int maps, int reduces, boolean autoComplete,
|
||||
String testName, boolean cleanOnStart) {
|
||||
public MRAppWithHistoryWithFailedTask(int maps, int reduces,
|
||||
boolean autoComplete, String testName, boolean cleanOnStart) {
|
||||
super(maps, reduces, autoComplete, testName, cleanOnStart);
|
||||
}
|
||||
|
||||
|
|
@ -587,4 +593,128 @@ public class TestJobHistoryParsing {
|
|||
t.testHistoryParsing();
|
||||
t.testHistoryParsingForFailedAttempts();
|
||||
}
|
||||
|
||||
/**
|
||||
* test clean old history files. Files should be deleted after 1 week by
|
||||
* default.
|
||||
*/
|
||||
@Test(timeout = 15000)
|
||||
public void testDeleteFileInfo() throws Exception {
|
||||
LOG.info("STARTING testDeleteFileInfo");
|
||||
try {
|
||||
Configuration conf = new Configuration();
|
||||
conf.setClass(
|
||||
CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
|
||||
MyResolver.class, DNSToSwitchMapping.class);
|
||||
|
||||
RackResolver.init(conf);
|
||||
MRApp app = new MRAppWithHistory(1, 1, true, this.getClass().getName(),
|
||||
true);
|
||||
app.submit(conf);
|
||||
Job job = app.getContext().getAllJobs().values().iterator().next();
|
||||
JobId jobId = job.getID();
|
||||
|
||||
app.waitForState(job, JobState.SUCCEEDED);
|
||||
|
||||
// make sure all events are flushed
|
||||
app.waitForState(Service.STATE.STOPPED);
|
||||
HistoryFileManager hfm = new HistoryFileManager();
|
||||
hfm.init(conf);
|
||||
HistoryFileInfo fileInfo = hfm.getFileInfo(jobId);
|
||||
hfm.initExisting();
|
||||
// wait for move files form the done_intermediate directory to the gone
|
||||
// directory
|
||||
while (fileInfo.isMovePending()) {
|
||||
Thread.sleep(300);
|
||||
}
|
||||
|
||||
Assert.assertNotNull(hfm.jobListCache.values());
|
||||
|
||||
// try to remove fileInfo
|
||||
hfm.clean();
|
||||
// check that fileInfo does not deleted
|
||||
Assert.assertFalse(fileInfo.isDeleted());
|
||||
// correct live time
|
||||
hfm.setMaxHistoryAge(-1);
|
||||
hfm.clean();
|
||||
// should be deleted !
|
||||
Assert.assertTrue("file should be deleted ", fileInfo.isDeleted());
|
||||
|
||||
} finally {
|
||||
LOG.info("FINISHED testDeleteFileInfo");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Simple test some methods of JobHistory
|
||||
*/
|
||||
@Test(timeout = 20000)
|
||||
public void testJobHistoryMethods() throws Exception {
|
||||
LOG.info("STARTING testJobHistoryMethods");
|
||||
try {
|
||||
Configuration configuration = new Configuration();
|
||||
configuration
|
||||
.setClass(
|
||||
CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
|
||||
MyResolver.class, DNSToSwitchMapping.class);
|
||||
|
||||
RackResolver.init(configuration);
|
||||
MRApp app = new MRAppWithHistory(1, 1, true, this.getClass().getName(),
|
||||
true);
|
||||
app.submit(configuration);
|
||||
Job job = app.getContext().getAllJobs().values().iterator().next();
|
||||
app.waitForState(job, JobState.SUCCEEDED);
|
||||
|
||||
JobHistory jobHistory = new JobHistory();
|
||||
jobHistory.init(configuration);
|
||||
// Method getAllJobs
|
||||
Assert.assertEquals(1, jobHistory.getAllJobs().size());
|
||||
// and with ApplicationId
|
||||
Assert.assertEquals(1, jobHistory.getAllJobs(app.getAppID()).size());
|
||||
|
||||
JobsInfo jobsinfo = jobHistory.getPartialJobs(0L, 10L, null, "default",
|
||||
0L, System.currentTimeMillis() + 1, 0L,
|
||||
System.currentTimeMillis() + 1, JobState.SUCCEEDED);
|
||||
|
||||
Assert.assertEquals(1, jobsinfo.getJobs().size());
|
||||
Assert.assertNotNull(jobHistory.getApplicationAttemptId());
|
||||
// test Application Id
|
||||
Assert.assertEquals("application_0_0000", jobHistory.getApplicationID()
|
||||
.toString());
|
||||
Assert
|
||||
.assertEquals("Job History Server", jobHistory.getApplicationName());
|
||||
// method does not work
|
||||
Assert.assertNull(jobHistory.getEventHandler());
|
||||
// method does not work
|
||||
Assert.assertNull(jobHistory.getClock());
|
||||
// method does not work
|
||||
Assert.assertNull(jobHistory.getClusterInfo());
|
||||
|
||||
} finally {
|
||||
LOG.info("FINISHED testJobHistoryMethods");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Simple test PartialJob
|
||||
*/
|
||||
@Test(timeout = 1000)
|
||||
public void testPartialJob() throws Exception {
|
||||
JobId jobId = new JobIdPBImpl();
|
||||
jobId.setId(0);
|
||||
JobIndexInfo jii = new JobIndexInfo(0L, System.currentTimeMillis(), "user",
|
||||
"jobName", jobId, 3, 2, "JobStatus");
|
||||
PartialJob test = new PartialJob(jii, jobId);
|
||||
assertEquals(1.0f, test.getProgress(), 0.001);
|
||||
assertNull(test.getAllCounters());
|
||||
assertNull(test.getTasks());
|
||||
assertNull(test.getTasks(TaskType.MAP));
|
||||
assertNull(test.getTask(new TaskIdPBImpl()));
|
||||
|
||||
assertNull(test.getTaskAttemptCompletionEvents(0, 100));
|
||||
assertNull(test.getMapAttemptCompletionEvents(0, 100));
|
||||
assertTrue(test.checkAccess(UserGroupInformation.getCurrentUser(), null));
|
||||
assertNull(test.getAMInfos());
|
||||
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,209 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.mapreduce.v2.hs;
|
||||
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
|
||||
import org.apache.hadoop.mapreduce.TaskCounter;
|
||||
import org.apache.hadoop.mapreduce.v2.api.MRClientProtocol;
|
||||
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDiagnosticsRequest;
|
||||
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDiagnosticsResponse;
|
||||
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskAttemptCompletionEventsRequest;
|
||||
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskAttemptCompletionEventsResponse;
|
||||
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskAttemptReportRequest;
|
||||
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskAttemptReportResponse;
|
||||
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskReportRequest;
|
||||
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskReportResponse;
|
||||
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
|
||||
import org.apache.hadoop.mapreduce.v2.api.records.JobState;
|
||||
import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
|
||||
import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
|
||||
import org.apache.hadoop.mapreduce.v2.api.records.TaskState;
|
||||
import org.apache.hadoop.mapreduce.v2.app.MRApp;
|
||||
import org.apache.hadoop.mapreduce.v2.app.job.Job;
|
||||
import org.apache.hadoop.mapreduce.v2.app.job.Task;
|
||||
import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt;
|
||||
import org.apache.hadoop.mapreduce.v2.hs.TestJobHistoryEvents.MRAppWithHistory;
|
||||
import org.apache.hadoop.mapreduce.v2.hs.TestJobHistoryParsing.MyResolver;
|
||||
import org.apache.hadoop.net.DNSToSwitchMapping;
|
||||
import org.apache.hadoop.util.ExitUtil;
|
||||
import org.apache.hadoop.yarn.factories.RecordFactory;
|
||||
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
|
||||
import org.apache.hadoop.yarn.service.Service;
|
||||
import org.apache.hadoop.yarn.service.Service.STATE;
|
||||
import org.apache.hadoop.yarn.util.RackResolver;
|
||||
import org.junit.After;
|
||||
import org.junit.Test;
|
||||
|
||||
import static org.junit.Assert.*;
|
||||
|
||||
/*
|
||||
test JobHistoryServer protocols....
|
||||
*/
|
||||
public class TestJobHistoryServer {
|
||||
private static RecordFactory recordFactory = RecordFactoryProvider
|
||||
.getRecordFactory(null);
|
||||
|
||||
|
||||
|
||||
JobHistoryServer historyServer=null;
|
||||
// simple test init/start/stop JobHistoryServer. Status should change.
|
||||
|
||||
@Test (timeout= 50000 )
|
||||
public void testStartStopServer() throws Exception {
|
||||
|
||||
historyServer = new JobHistoryServer();
|
||||
Configuration config = new Configuration();
|
||||
historyServer.init(config);
|
||||
assertEquals(STATE.INITED, historyServer.getServiceState());
|
||||
assertEquals(3, historyServer.getServices().size());
|
||||
historyServer.start();
|
||||
assertEquals(STATE.STARTED, historyServer.getServiceState());
|
||||
historyServer.stop();
|
||||
assertEquals(STATE.STOPPED, historyServer.getServiceState());
|
||||
assertNotNull(historyServer.getClientService());
|
||||
HistoryClientService historyService = historyServer.getClientService();
|
||||
assertNotNull(historyService.getClientHandler().getConnectAddress());
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
//Test reports of JobHistoryServer. History server should get log files from MRApp and read them
|
||||
|
||||
@Test (timeout= 50000 )
|
||||
public void testReports() throws Exception {
|
||||
Configuration config = new Configuration();
|
||||
config
|
||||
.setClass(
|
||||
CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
|
||||
MyResolver.class, DNSToSwitchMapping.class);
|
||||
|
||||
RackResolver.init(config);
|
||||
MRApp app = new MRAppWithHistory(1, 1, true, this.getClass().getName(),
|
||||
true);
|
||||
app.submit(config);
|
||||
Job job = app.getContext().getAllJobs().values().iterator().next();
|
||||
app.waitForState(job, JobState.SUCCEEDED);
|
||||
|
||||
historyServer = new JobHistoryServer();
|
||||
|
||||
historyServer.init(config);
|
||||
historyServer.start();
|
||||
|
||||
// search JobHistory service
|
||||
JobHistory jobHistory= null;
|
||||
for (Service service : historyServer.getServices() ) {
|
||||
if (service instanceof JobHistory) {
|
||||
jobHistory = (JobHistory) service;
|
||||
}
|
||||
};
|
||||
|
||||
Map<JobId, Job> jobs= jobHistory.getAllJobs();
|
||||
|
||||
assertEquals(1, jobs.size());
|
||||
assertEquals("job_0_0000",jobs.keySet().iterator().next().toString());
|
||||
|
||||
|
||||
Task task = job.getTasks().values().iterator().next();
|
||||
TaskAttempt attempt = task.getAttempts().values().iterator().next();
|
||||
|
||||
HistoryClientService historyService = historyServer.getClientService();
|
||||
MRClientProtocol protocol = historyService.getClientHandler();
|
||||
|
||||
GetTaskAttemptReportRequest gtarRequest = recordFactory
|
||||
.newRecordInstance(GetTaskAttemptReportRequest.class);
|
||||
// test getTaskAttemptReport
|
||||
TaskAttemptId taId = attempt.getID();
|
||||
taId.setTaskId(task.getID());
|
||||
taId.getTaskId().setJobId(job.getID());
|
||||
gtarRequest.setTaskAttemptId(taId);
|
||||
GetTaskAttemptReportResponse response = protocol
|
||||
.getTaskAttemptReport(gtarRequest);
|
||||
assertEquals("container_0_0000_01_000000", response.getTaskAttemptReport()
|
||||
.getContainerId().toString());
|
||||
assertTrue(response.getTaskAttemptReport().getDiagnosticInfo().isEmpty());
|
||||
// counters
|
||||
assertNotNull(response.getTaskAttemptReport().getCounters()
|
||||
.getCounter(TaskCounter.PHYSICAL_MEMORY_BYTES));
|
||||
assertEquals(taId.toString(), response.getTaskAttemptReport()
|
||||
.getTaskAttemptId().toString());
|
||||
// test getTaskReport
|
||||
GetTaskReportRequest request = recordFactory
|
||||
.newRecordInstance(GetTaskReportRequest.class);
|
||||
TaskId taskId = task.getID();
|
||||
taskId.setJobId(job.getID());
|
||||
request.setTaskId(taskId);
|
||||
GetTaskReportResponse reportResponse = protocol.getTaskReport(request);
|
||||
assertEquals("", reportResponse.getTaskReport().getDiagnosticsList()
|
||||
.iterator().next());
|
||||
// progress
|
||||
assertEquals(1.0f, reportResponse.getTaskReport().getProgress(), 0.01);
|
||||
// report has corrected taskId
|
||||
assertEquals(taskId.toString(), reportResponse.getTaskReport().getTaskId()
|
||||
.toString());
|
||||
// Task state should be SUCCEEDED
|
||||
assertEquals(TaskState.SUCCEEDED, reportResponse.getTaskReport()
|
||||
.getTaskState());
|
||||
// test getTaskAttemptCompletionEvents
|
||||
GetTaskAttemptCompletionEventsRequest taskAttemptRequest = recordFactory
|
||||
.newRecordInstance(GetTaskAttemptCompletionEventsRequest.class);
|
||||
taskAttemptRequest.setJobId(job.getID());
|
||||
GetTaskAttemptCompletionEventsResponse taskAttemptCompletionEventsResponse = protocol
|
||||
.getTaskAttemptCompletionEvents(taskAttemptRequest);
|
||||
assertEquals(0, taskAttemptCompletionEventsResponse.getCompletionEventCount());
|
||||
|
||||
// test getDiagnostics
|
||||
GetDiagnosticsRequest diagnosticRequest = recordFactory
|
||||
.newRecordInstance(GetDiagnosticsRequest.class);
|
||||
diagnosticRequest.setTaskAttemptId(taId);
|
||||
GetDiagnosticsResponse diagnosticResponse = protocol
|
||||
.getDiagnostics(diagnosticRequest);
|
||||
// it is strange : why one empty string ?
|
||||
assertEquals(1, diagnosticResponse.getDiagnosticsCount());
|
||||
assertEquals("", diagnosticResponse.getDiagnostics(0));
|
||||
|
||||
}
|
||||
// test main method
|
||||
@Test (timeout =60000)
|
||||
public void testMainMethod() throws Exception {
|
||||
|
||||
ExitUtil.disableSystemExit();
|
||||
try {
|
||||
JobHistoryServer.main(new String[0]);
|
||||
|
||||
} catch (ExitUtil.ExitException e) {
|
||||
assertEquals(0,e.status);
|
||||
ExitUtil.resetFirstExitException();
|
||||
fail();
|
||||
}
|
||||
}
|
||||
|
||||
@After
|
||||
public void stop(){
|
||||
if(historyServer !=null && !STATE.STOPPED.equals(historyServer.getServiceState())){
|
||||
historyServer.stop();
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue