MAPREDUCE-7072. mapred job -history prints duplicate counter in human output (wilfreds via rkanter)
This commit is contained in:
parent
56788d759f
commit
1b9ecc264a
|
@ -148,7 +148,8 @@ class HumanReadableHistoryViewerPrinter implements HistoryViewerPrinter {
|
|||
"Total Value"));
|
||||
buff.append("\n------------------------------------------" +
|
||||
"---------------------------------------------");
|
||||
for (String groupName : totalCounters.getGroupNames()) {
|
||||
for (CounterGroup counterGroup : totalCounters) {
|
||||
String groupName = counterGroup.getName();
|
||||
CounterGroup totalGroup = totalCounters.getGroup(groupName);
|
||||
CounterGroup mapGroup = mapCounters.getGroup(groupName);
|
||||
CounterGroup reduceGroup = reduceCounters.getGroup(groupName);
|
||||
|
|
|
@ -104,7 +104,8 @@ class JSONHistoryViewerPrinter implements HistoryViewerPrinter {
|
|||
// Killed jobs might not have counters
|
||||
if (totalCounters != null) {
|
||||
JSONObject jGroups = new JSONObject();
|
||||
for (String groupName : totalCounters.getGroupNames()) {
|
||||
for (CounterGroup counterGroup : totalCounters) {
|
||||
String groupName = counterGroup.getName();
|
||||
CounterGroup totalGroup = totalCounters.getGroup(groupName);
|
||||
CounterGroup mapGroup = mapCounters.getGroup(groupName);
|
||||
CounterGroup reduceGroup = reduceCounters.getGroup(groupName);
|
||||
|
|
|
@ -859,6 +859,46 @@ public class TestHistoryViewerPrinter {
|
|||
"}\n", outStr, JSONCompareMode.NON_EXTENSIBLE);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testHumanDupePrinter() throws Exception {
|
||||
JobHistoryParser.JobInfo job = createJobInfo2();
|
||||
// Counters are only part of the overview so printAll can be false or true
|
||||
// this does not affect the test, task counters are not printed
|
||||
HumanReadableHistoryViewerPrinter printer =
|
||||
new HumanReadableHistoryViewerPrinter(job, false, "http://",
|
||||
TimeZone.getTimeZone("GMT"));
|
||||
String outStr = run(printer);
|
||||
// We are not interested in anything but the duplicate counter
|
||||
int count1 = outStr.indexOf(
|
||||
"|Map-Reduce Framework |Map input records |");
|
||||
Assert.assertNotEquals("First counter occurrence not found", -1, count1);
|
||||
int count2 = outStr.indexOf(
|
||||
"|Map-Reduce Framework |Map input records |",
|
||||
count1 + 1);
|
||||
Assert.assertEquals("Duplicate counter found at: " + count1 +
|
||||
" and " + count2, -1, count2);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testJSONDupePrinter() throws Exception {
|
||||
JobHistoryParser.JobInfo job = createJobInfo2();
|
||||
// Counters are part of the overview and task info
|
||||
// Tasks only have bogus counters in the test if that is changed printAll
|
||||
// must then be kept as false for this test to pass
|
||||
JSONHistoryViewerPrinter printer =
|
||||
new JSONHistoryViewerPrinter(job, false, "http://");
|
||||
String outStr = run(printer);
|
||||
// We are not interested in anything but the duplicate counter
|
||||
int count1 = outStr.indexOf(
|
||||
"\"counterName\":\"MAP_INPUT_RECORDS\"");
|
||||
Assert.assertNotEquals("First counter occurrence not found", -1, count1);
|
||||
int count2 = outStr.indexOf(
|
||||
"\"counterName\":\"MAP_INPUT_RECORDS\"",
|
||||
count1 + 1);
|
||||
Assert.assertEquals("Duplicate counter found at: " + count1 +
|
||||
" and " + count2, -1, count2);
|
||||
}
|
||||
|
||||
private String run(HistoryViewerPrinter printer) throws Exception {
|
||||
ByteArrayOutputStream boas = new ByteArrayOutputStream();
|
||||
PrintStream out = new PrintStream(boas, true);
|
||||
|
@ -901,6 +941,32 @@ public class TestHistoryViewerPrinter {
|
|||
addTaskInfo(job, TaskType.JOB_CLEANUP, 9, TaskStatus.State.SUCCEEDED);
|
||||
return job;
|
||||
}
|
||||
private static JobHistoryParser.JobInfo createJobInfo2() {
|
||||
JobHistoryParser.JobInfo job = new JobHistoryParser.JobInfo();
|
||||
job.submitTime = 1317928501754L;
|
||||
job.finishTime = job.submitTime + 15000;
|
||||
job.jobid = JobID.forName("job_1317928501754_0001");
|
||||
job.username = "test";
|
||||
job.jobname = "Dupe counter output";
|
||||
job.jobQueueName = "root.test";
|
||||
job.jobConfPath = "/tmp/job.xml";
|
||||
job.launchTime = job.submitTime + 1000;
|
||||
job.totalMaps = 1;
|
||||
job.totalReduces = 0;
|
||||
job.failedMaps = 0;
|
||||
job.failedReduces = 0;
|
||||
job.succeededMaps = 1;
|
||||
job.succeededReduces = 0;
|
||||
job.jobStatus = JobStatus.State.SUCCEEDED.name();
|
||||
job.totalCounters = createDeprecatedCounters();
|
||||
job.mapCounters = createDeprecatedCounters();
|
||||
job.reduceCounters = createDeprecatedCounters();
|
||||
job.tasksMap = new HashMap<>();
|
||||
addTaskInfo(job, TaskType.JOB_SETUP, 1, TaskStatus.State.SUCCEEDED);
|
||||
addTaskInfo(job, TaskType.MAP, 2, TaskStatus.State.SUCCEEDED);
|
||||
addTaskInfo(job, TaskType.JOB_CLEANUP, 3, TaskStatus.State.SUCCEEDED);
|
||||
return job;
|
||||
}
|
||||
|
||||
private static Counters createCounters() {
|
||||
Counters counters = new Counters();
|
||||
|
@ -910,6 +976,16 @@ public class TestHistoryViewerPrinter {
|
|||
return counters;
|
||||
}
|
||||
|
||||
private static Counters createDeprecatedCounters() {
|
||||
Counters counters = new Counters();
|
||||
// Deprecated counter: make sure it is only printed once
|
||||
counters.findCounter("org.apache.hadoop.mapred.Task$Counter",
|
||||
"MAP_INPUT_RECORDS").setValue(1);
|
||||
counters.findCounter("File System Counters",
|
||||
"FILE: Number of bytes read").setValue(1);
|
||||
return counters;
|
||||
}
|
||||
|
||||
private static void addTaskInfo(JobHistoryParser.JobInfo job,
|
||||
TaskType type, int id, TaskStatus.State status) {
|
||||
JobHistoryParser.TaskInfo task = new JobHistoryParser.TaskInfo();
|
||||
|
|
Loading…
Reference in New Issue