From 1b9ecc264a6abe9d9d5412318c67d3d2936bd9ac Mon Sep 17 00:00:00 2001 From: Robert Kanter Date: Tue, 24 Apr 2018 11:30:38 -0700 Subject: [PATCH] MAPREDUCE-7072. mapred job -history prints duplicate counter in human output (wilfreds via rkanter) --- .../HumanReadableHistoryViewerPrinter.java | 3 +- .../jobhistory/JSONHistoryViewerPrinter.java | 3 +- .../jobhistory/TestHistoryViewerPrinter.java | 76 +++++++++++++++++++ 3 files changed, 80 insertions(+), 2 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HumanReadableHistoryViewerPrinter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HumanReadableHistoryViewerPrinter.java index 685fa0592cb..060ba24c179 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HumanReadableHistoryViewerPrinter.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HumanReadableHistoryViewerPrinter.java @@ -148,7 +148,8 @@ class HumanReadableHistoryViewerPrinter implements HistoryViewerPrinter { "Total Value")); buff.append("\n------------------------------------------" + "---------------------------------------------"); - for (String groupName : totalCounters.getGroupNames()) { + for (CounterGroup counterGroup : totalCounters) { + String groupName = counterGroup.getName(); CounterGroup totalGroup = totalCounters.getGroup(groupName); CounterGroup mapGroup = mapCounters.getGroup(groupName); CounterGroup reduceGroup = reduceCounters.getGroup(groupName); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JSONHistoryViewerPrinter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JSONHistoryViewerPrinter.java index cfb66411fa2..5f8e9ad4b68 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JSONHistoryViewerPrinter.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JSONHistoryViewerPrinter.java @@ -104,7 +104,8 @@ class JSONHistoryViewerPrinter implements HistoryViewerPrinter { // Killed jobs might not have counters if (totalCounters != null) { JSONObject jGroups = new JSONObject(); - for (String groupName : totalCounters.getGroupNames()) { + for (CounterGroup counterGroup : totalCounters) { + String groupName = counterGroup.getName(); CounterGroup totalGroup = totalCounters.getGroup(groupName); CounterGroup mapGroup = mapCounters.getGroup(groupName); CounterGroup reduceGroup = reduceCounters.getGroup(groupName); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestHistoryViewerPrinter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestHistoryViewerPrinter.java index 588500c0b4b..3601ea762a3 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestHistoryViewerPrinter.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestHistoryViewerPrinter.java @@ -859,6 +859,46 @@ public class TestHistoryViewerPrinter { "}\n", outStr, JSONCompareMode.NON_EXTENSIBLE); } + @Test + public void testHumanDupePrinter() throws Exception { + JobHistoryParser.JobInfo job = createJobInfo2(); + // Counters are only part of the overview so printAll can be false or true + // this does not affect the test, task counters are not printed + HumanReadableHistoryViewerPrinter printer = + new HumanReadableHistoryViewerPrinter(job, false, "http://", + TimeZone.getTimeZone("GMT")); + String outStr = run(printer); + // We are not interested in anything but the duplicate counter + int count1 = outStr.indexOf( + "|Map-Reduce Framework |Map input records |"); + Assert.assertNotEquals("First counter occurrence not found", -1, count1); + int count2 = outStr.indexOf( + "|Map-Reduce Framework |Map input records |", + count1 + 1); + Assert.assertEquals("Duplicate counter found at: " + count1 + + " and " + count2, -1, count2); + } + + @Test + public void testJSONDupePrinter() throws Exception { + JobHistoryParser.JobInfo job = createJobInfo2(); + // Counters are part of the overview and task info + // Tasks only have bogus counters in the test if that is changed printAll + // must then be kept as false for this test to pass + JSONHistoryViewerPrinter printer = + new JSONHistoryViewerPrinter(job, false, "http://"); + String outStr = run(printer); + // We are not interested in anything but the duplicate counter + int count1 = outStr.indexOf( + "\"counterName\":\"MAP_INPUT_RECORDS\""); + Assert.assertNotEquals("First counter occurrence not found", -1, count1); + int count2 = outStr.indexOf( + "\"counterName\":\"MAP_INPUT_RECORDS\"", + count1 + 1); + Assert.assertEquals("Duplicate counter found at: " + count1 + + " and " + count2, -1, count2); + } + private String run(HistoryViewerPrinter printer) throws Exception { ByteArrayOutputStream boas = new ByteArrayOutputStream(); PrintStream out = new PrintStream(boas, true); @@ -901,6 +941,32 @@ public class TestHistoryViewerPrinter { addTaskInfo(job, TaskType.JOB_CLEANUP, 9, TaskStatus.State.SUCCEEDED); return job; } + private static JobHistoryParser.JobInfo createJobInfo2() { + JobHistoryParser.JobInfo job = new JobHistoryParser.JobInfo(); + job.submitTime = 1317928501754L; + job.finishTime = job.submitTime + 15000; + job.jobid = JobID.forName("job_1317928501754_0001"); + job.username = "test"; + job.jobname = "Dupe counter output"; + job.jobQueueName = "root.test"; + job.jobConfPath = "/tmp/job.xml"; + job.launchTime = job.submitTime + 1000; + job.totalMaps = 1; + job.totalReduces = 0; + job.failedMaps = 0; + job.failedReduces = 0; + job.succeededMaps = 1; + job.succeededReduces = 0; + job.jobStatus = JobStatus.State.SUCCEEDED.name(); + job.totalCounters = createDeprecatedCounters(); + job.mapCounters = createDeprecatedCounters(); + job.reduceCounters = createDeprecatedCounters(); + job.tasksMap = new HashMap<>(); + addTaskInfo(job, TaskType.JOB_SETUP, 1, TaskStatus.State.SUCCEEDED); + addTaskInfo(job, TaskType.MAP, 2, TaskStatus.State.SUCCEEDED); + addTaskInfo(job, TaskType.JOB_CLEANUP, 3, TaskStatus.State.SUCCEEDED); + return job; + } private static Counters createCounters() { Counters counters = new Counters(); @@ -910,6 +976,16 @@ public class TestHistoryViewerPrinter { return counters; } + private static Counters createDeprecatedCounters() { + Counters counters = new Counters(); + // Deprecated counter: make sure it is only printed once + counters.findCounter("org.apache.hadoop.mapred.Task$Counter", + "MAP_INPUT_RECORDS").setValue(1); + counters.findCounter("File System Counters", + "FILE: Number of bytes read").setValue(1); + return counters; + } + private static void addTaskInfo(JobHistoryParser.JobInfo job, TaskType type, int id, TaskStatus.State status) { JobHistoryParser.TaskInfo task = new JobHistoryParser.TaskInfo();