MAPREDUCE-5568. Fixed CompletedJob in JHS to show progress percentage correctly in case the number of mappers or reducers is zero. Contributed by MinJi Kim

This commit is contained in:
Jian He 2014-11-25 13:02:55 -08:00
parent 5805a81efb
commit 78f7cdbfd6
4 changed files with 70 additions and 7 deletions

View File

@ -253,6 +253,10 @@ Release 2.7.0 - UNRELEASED
MAPREDUCE-6049. AM JVM does not exit if MRClientService graceful shutdown
fails (Rohith via devaraj)
MAPREDUCE-5568. Fixed CompletedJob in JHS to show progress percentage
correctly in case the number of mappers or reducers is zero. (MinJi Kim via
jianhe)
Release 2.6.0 - 2014-11-18

View File

@ -141,8 +141,18 @@ public class CompletedJob implements org.apache.hadoop.mapreduce.v2.app.job.Job
report.setFinishTime(jobInfo.getFinishTime());
report.setJobName(jobInfo.getJobname());
report.setUser(jobInfo.getUsername());
report.setMapProgress((float) getCompletedMaps() / getTotalMaps());
report.setReduceProgress((float) getCompletedReduces() / getTotalReduces());
if ( getTotalMaps() == 0 ) {
report.setMapProgress(1.0f);
} else {
report.setMapProgress((float) getCompletedMaps() / getTotalMaps());
}
if ( getTotalReduces() == 0 ) {
report.setReduceProgress(1.0f);
} else {
report.setReduceProgress((float) getCompletedReduces() / getTotalReduces());
}
report.setJobFile(getConfFile().toString());
String historyUrl = "N/A";
try {

View File

@ -55,14 +55,19 @@ public class TestJobHistoryEntities {
private final String historyFileName =
"job_1329348432655_0001-1329348443227-user-Sleep+job-1329348468601-10-1-SUCCEEDED-default.jhist";
private final String historyFileNameZeroReduceTasks =
"job_1416424547277_0002-1416424775281-root-TeraGen-1416424785433-2-0-SUCCEEDED-default-1416424779349.jhist";
private final String confFileName = "job_1329348432655_0001_conf.xml";
private final Configuration conf = new Configuration();
private final JobACLsManager jobAclsManager = new JobACLsManager(conf);
private boolean loadTasks;
private JobId jobId = MRBuilderUtils.newJobId(1329348432655l, 1, 1);
Path fulleHistoryPath =
Path fullHistoryPath =
new Path(this.getClass().getClassLoader().getResource(historyFileName)
.getFile());
Path fullHistoryPathZeroReduces =
new Path(this.getClass().getClassLoader().getResource(historyFileNameZeroReduceTasks)
.getFile());
Path fullConfPath =
new Path(this.getClass().getClassLoader().getResource(confFileName)
.getFile());
@ -87,7 +92,7 @@ public class TestJobHistoryEntities {
when(info.getConfFile()).thenReturn(fullConfPath);
//Re-initialize to verify the delayed load.
completedJob =
new CompletedJob(conf, jobId, fulleHistoryPath, loadTasks, "user",
new CompletedJob(conf, jobId, fullHistoryPath, loadTasks, "user",
info, jobAclsManager);
//Verify tasks loaded based on loadTask parameter.
assertEquals(loadTasks, completedJob.tasksLoaded.get());
@ -106,12 +111,27 @@ public class TestJobHistoryEntities {
assertEquals(JobState.SUCCEEDED, jobReport.getJobState());
}
@Test (timeout=100000)
public void testCopmletedJobReportWithZeroTasks() throws Exception {
HistoryFileInfo info = mock(HistoryFileInfo.class);
when(info.getConfFile()).thenReturn(fullConfPath);
completedJob =
new CompletedJob(conf, jobId, fullHistoryPathZeroReduces, loadTasks, "user",
info, jobAclsManager);
JobReport jobReport = completedJob.getReport();
// Make sure that the number reduces (completed and total) are equal to zero.
assertEquals(0, completedJob.getTotalReduces());
assertEquals(0, completedJob.getCompletedReduces());
// Verify that the reduce progress is 1.0 (not NaN)
assertEquals(1.0, jobReport.getReduceProgress(), 0.001);
}
@Test (timeout=10000)
public void testCompletedTask() throws Exception {
HistoryFileInfo info = mock(HistoryFileInfo.class);
when(info.getConfFile()).thenReturn(fullConfPath);
completedJob =
new CompletedJob(conf, jobId, fulleHistoryPath, loadTasks, "user",
new CompletedJob(conf, jobId, fullHistoryPath, loadTasks, "user",
info, jobAclsManager);
TaskId mt1Id = MRBuilderUtils.newTaskId(jobId, 0, TaskType.MAP);
TaskId rt1Id = MRBuilderUtils.newTaskId(jobId, 0, TaskType.REDUCE);
@ -140,7 +160,7 @@ public class TestJobHistoryEntities {
HistoryFileInfo info = mock(HistoryFileInfo.class);
when(info.getConfFile()).thenReturn(fullConfPath);
completedJob =
new CompletedJob(conf, jobId, fulleHistoryPath, loadTasks, "user",
new CompletedJob(conf, jobId, fullHistoryPath, loadTasks, "user",
info, jobAclsManager);
TaskId mt1Id = MRBuilderUtils.newTaskId(jobId, 0, TaskType.MAP);
TaskId rt1Id = MRBuilderUtils.newTaskId(jobId, 0, TaskType.REDUCE);
@ -179,7 +199,7 @@ public class TestJobHistoryEntities {
HistoryFileInfo info = mock(HistoryFileInfo.class);
when(info.getConfFile()).thenReturn(fullConfPath);
completedJob =
new CompletedJob(conf, jobId, fulleHistoryPath, loadTasks, "user",
new CompletedJob(conf, jobId, fullHistoryPath, loadTasks, "user",
info, jobAclsManager);
TaskCompletionEvent[] events= completedJob.getMapAttemptCompletionEvents(0,1000);
assertEquals(10, completedJob.getMapAttemptCompletionEvents(0,10).length);