From 660fb6cbf834dee61135db2f786d04db583c5376 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Fri, 16 Dec 2011 01:52:12 +0000 Subject: [PATCH] MAPREDUCE-3487. Fixed JobHistory web-UI to display links to single task's counters' page. Contributed by Jason Lowe. svn merge -c 1215016 --ignore-ancestry ../../trunk/ git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.23@1215017 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 ++ .../v2/app/webapp/CountersBlock.java | 2 +- .../v2/app/webapp/SingleCounterBlock.java | 6 ++-- .../mapreduce/v2/app/webapp/TestAMWebApp.java | 29 ++++++++++++++++++- 4 files changed, 36 insertions(+), 4 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index c0c96d64513..9967c0803c9 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -263,6 +263,9 @@ Release 0.23.1 - Unreleased MAPREDUCE-3560. TestRMNodeTransitions is failing on trunk. (Siddharth Seth via mahadev) + MAPREDUCE-3487. Fixed JobHistory web-UI to display links to single task's + counters' page. (Jason Lowe via vinodkv) + Release 0.23.0 - 2011-11-01 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/CountersBlock.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/CountersBlock.java index cf6ab99a936..6accd8add73 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/CountersBlock.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/CountersBlock.java @@ -120,7 +120,7 @@ public class CountersBlock extends HtmlBlock { // Ditto TR>>>>>>> groupRow = group. tr(); - if (mg == null && rg == null) { + if (task == null && mg == null && rg == null) { groupRow.td().$title(counter.getName())._(counter.getDisplayName()). _(); } else { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/SingleCounterBlock.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/SingleCounterBlock.java index 1ec774e3fbe..bb728225428 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/SingleCounterBlock.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/SingleCounterBlock.java @@ -24,6 +24,7 @@ import java.util.TreeMap; import org.apache.hadoop.mapreduce.v2.api.records.Counter; import org.apache.hadoop.mapreduce.v2.api.records.CounterGroup; +import org.apache.hadoop.mapreduce.v2.api.records.Counters; import org.apache.hadoop.mapreduce.v2.api.records.JobId; import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId; import org.apache.hadoop.mapreduce.v2.api.records.TaskId; @@ -120,8 +121,9 @@ public class SingleCounterBlock extends HtmlBlock { for(Map.Entry entry : task.getAttempts().entrySet()) { long value = 0; - CounterGroup group = entry.getValue().getCounters() - .getCounterGroup($(COUNTER_GROUP)); + Counters counters = entry.getValue().getCounters(); + CounterGroup group = (counters != null) + ? counters.getCounterGroup($(COUNTER_GROUP)) : null; if(group != null) { Counter c = group.getCounter($(COUNTER_NAME)); if(c != null) { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebApp.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebApp.java index 745eedcb864..691ff657cdb 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebApp.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebApp.java @@ -31,6 +31,7 @@ import org.apache.hadoop.mapreduce.v2.app.AppContext; import org.apache.hadoop.mapreduce.v2.app.MockJobs; import org.apache.hadoop.mapreduce.v2.app.job.Job; import org.apache.hadoop.mapreduce.v2.app.job.Task; +import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt; import org.apache.hadoop.mapreduce.v2.util.MRApps; import org.apache.hadoop.yarn.Clock; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; @@ -153,7 +154,7 @@ public class TestAMWebApp { e.getValue().getType(); Map params = new HashMap(); params.put(AMParams.JOB_ID, MRApps.toString(jobId)); - params.put(AMParams.TASK_ID, e.getKey().toString()); + params.put(AMParams.TASK_ID, MRApps.toString(e.getKey())); params.put(AMParams.TASK_TYPE, MRApps.taskSymbol(e.getValue().getType())); return params; } @@ -179,6 +180,32 @@ public class TestAMWebApp { WebAppTests.testPage(SingleCounterPage.class, AppContext.class, appContext, params); } + + @Test public void testTaskCountersView() { + AppContext appContext = new TestAppContext(); + Map params = getTaskParams(appContext); + WebAppTests.testPage(CountersPage.class, AppContext.class, + appContext, params); + } + + @Test public void testSingleTaskCounterView() { + AppContext appContext = new TestAppContext(0, 1, 1, 2); + Map params = getTaskParams(appContext); + params.put(AMParams.COUNTER_GROUP, + "org.apache.hadoop.mapreduce.FileSystemCounter"); + params.put(AMParams.COUNTER_NAME, "HDFS_WRITE_OPS"); + + // remove counters from one task attempt + // to test handling of missing counters + TaskId taskID = MRApps.toTaskID(params.get(AMParams.TASK_ID)); + Job job = appContext.getJob(taskID.getJobId()); + Task task = job.getTask(taskID); + TaskAttempt attempt = task.getAttempts().values().iterator().next(); + attempt.getReport().setCounters(null); + + WebAppTests.testPage(SingleCounterPage.class, AppContext.class, + appContext, params); + } public static void main(String[] args) { WebApps.$for("yarn", AppContext.class, new TestAppContext(0, 8, 88, 4)).