diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index a013c818251..e6828b8a700 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -46,6 +46,10 @@ Release 2.4.0 - UNRELEASED MAPREDUCE-5773. Provide dedicated MRAppMaster syslog length limit (Gera Shegalov via jlowe) + MAPREDUCE-5553. Allow users to easily access + completed/pending/successful/failed tasks on MR AM web-ui. (Paul Han via + acmurthy) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AMParams.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AMParams.java index ad8a15d45a0..2ca7ff54717 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AMParams.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AMParams.java @@ -27,6 +27,7 @@ public interface AMParams { static final String JOB_ID = "job.id"; static final String TASK_ID = "task.id"; static final String TASK_TYPE = "task.type"; + static final String TASK_STATE = "task.state"; static final String ATTEMPT_STATE = "attempt.state"; static final String COUNTER_GROUP = "counter.group"; static final String COUNTER_NAME = "counter.name"; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AMWebApp.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AMWebApp.java index e3b62c71ec0..ee77c8a3917 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AMWebApp.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AMWebApp.java @@ -40,7 +40,7 @@ public class AMWebApp extends WebApp implements AMParams { route(pajoin("/jobcounters", JOB_ID), AppController.class, "jobCounters"); route(pajoin("/singlejobcounter",JOB_ID, COUNTER_GROUP, COUNTER_NAME), AppController.class, "singleJobCounter"); - route(pajoin("/tasks", JOB_ID, TASK_TYPE), AppController.class, "tasks"); + route(pajoin("/tasks", JOB_ID, TASK_TYPE, TASK_STATE), AppController.class, "tasks"); route(pajoin("/attempts", JOB_ID, TASK_TYPE, ATTEMPT_STATE), AppController.class, "attempts"); route(pajoin("/task", TASK_ID), AppController.class, "task"); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/JobBlock.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/JobBlock.java index a8a756d3059..0c6d7736fb1 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/JobBlock.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/JobBlock.java @@ -127,31 +127,28 @@ public class JobBlock extends HtmlBlock { th(_TH, "Running"). th(_TH, "Complete")._(). tr(_ODD). - th(). - a(url("tasks", jid, "m"), "Map")._(). + th("Map"). td(). div(_PROGRESSBAR). $title(join(jinfo.getMapProgressPercent(), '%')). // tooltip div(_PROGRESSBAR_VALUE). $style(join("width:", jinfo.getMapProgressPercent(), '%'))._()._()._(). - td(String.valueOf(jinfo.getMapsTotal())). - td(String.valueOf(jinfo.getMapsPending())). - td(String.valueOf(jinfo.getMapsRunning())). - td(String.valueOf(jinfo.getMapsCompleted()))._(). + td().a(url("tasks", jid, "m", "ALL"),String.valueOf(jinfo.getMapsTotal()))._(). + td().a(url("tasks", jid, "m", "PENDING"),String.valueOf(jinfo.getMapsPending()))._(). + td().a(url("tasks", jid, "m", "RUNNING"),String.valueOf(jinfo.getMapsRunning()))._(). + td().a(url("tasks", jid, "m", "COMPLETED"),String.valueOf(jinfo.getMapsCompleted()))._()._(). tr(_EVEN). - th(). - a(url("tasks", jid, "r"), "Reduce")._(). + th("Reduce"). td(). div(_PROGRESSBAR). $title(join(jinfo.getReduceProgressPercent(), '%')). // tooltip div(_PROGRESSBAR_VALUE). $style(join("width:", jinfo.getReduceProgressPercent(), '%'))._()._()._(). - td(String.valueOf(jinfo.getReducesTotal())). - td(String.valueOf(jinfo.getReducesPending())). - td(String.valueOf(jinfo.getReducesRunning())). - td(String.valueOf(jinfo.getReducesCompleted()))._() + td().a(url("tasks", jid, "r", "ALL"),String.valueOf(jinfo.getReducesTotal()))._(). + td().a(url("tasks", jid, "r", "PENDING"),String.valueOf(jinfo.getReducesPending()))._(). + td().a(url("tasks", jid, "r", "RUNNING"),String.valueOf(jinfo.getReducesRunning()))._(). + td().a(url("tasks", jid, "r", "COMPLETED"),String.valueOf(jinfo.getReducesCompleted()))._()._() ._(). - // Attempts table table("#job"). tr(). diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksBlock.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksBlock.java index e2c65e320b8..a2b19e4ae44 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksBlock.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksBlock.java @@ -18,6 +18,7 @@ package org.apache.hadoop.mapreduce.v2.app.webapp; +import static org.apache.hadoop.mapreduce.v2.app.webapp.AMParams.TASK_STATE; import static org.apache.hadoop.mapreduce.v2.app.webapp.AMParams.TASK_TYPE; import static org.apache.hadoop.yarn.util.StringHelper.join; import static org.apache.hadoop.yarn.util.StringHelper.percent; @@ -71,6 +72,25 @@ public class TasksBlock extends HtmlBlock { if (type != null && task.getType() != type) { continue; } + String taskStateStr = $(TASK_STATE); + if (taskStateStr == null || taskStateStr.trim().equals("")) { + taskStateStr = "ALL"; + } + + if (!taskStateStr.equalsIgnoreCase("ALL")) + { + try { + // get stateUI enum + MRApps.TaskStateUI stateUI = MRApps.taskState(taskStateStr); + if (!stateUI.correspondsTo(task.getState())) + { + continue; + } + } catch (IllegalArgumentException e) { + continue; // not supported state, ignore + } + } + TaskInfo info = new TaskInfo(task); String tid = info.getId(); String pct = percent(info.getProgress() / 100); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java index c12e84759c9..6b07fa16977 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java @@ -48,6 +48,7 @@ import org.apache.hadoop.mapreduce.v2.api.records.JobId; import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId; import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptState; import org.apache.hadoop.mapreduce.v2.api.records.TaskId; +import org.apache.hadoop.mapreduce.v2.api.records.TaskState; import org.apache.hadoop.mapreduce.v2.api.records.TaskType; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.ContainerLogAppender; @@ -125,6 +126,23 @@ public class MRApps extends Apps { } } + public static enum TaskStateUI { + RUNNING( + new TaskState[]{TaskState.RUNNING}), + PENDING(new TaskState[]{TaskState.SCHEDULED}), + COMPLETED(new TaskState[]{TaskState.SUCCEEDED, TaskState.FAILED, TaskState.KILLED}); + + private final List correspondingStates; + + private TaskStateUI(TaskState[] correspondingStates) { + this.correspondingStates = Arrays.asList(correspondingStates); + } + + public boolean correspondsTo(TaskState state) { + return this.correspondingStates.contains(state); + } + } + public static TaskType taskType(String symbol) { // JDK 7 supports switch on strings if (symbol.equals("m")) return TaskType.MAP; @@ -136,6 +154,10 @@ public class MRApps extends Apps { return TaskAttemptStateUI.valueOf(attemptStateStr); } + public static TaskStateUI taskState(String taskStateStr) { + return TaskStateUI.valueOf(taskStateStr); + } + // gets the base name of the MapReduce framework or null if no // framework was configured private static String getMRFrameworkName(Configuration conf) { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java index e9cecbee990..b8e4fa92fb7 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java @@ -37,6 +37,7 @@ import org.apache.hadoop.mapreduce.filecache.DistributedCache; import org.apache.hadoop.mapreduce.v2.api.records.JobId; import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId; import org.apache.hadoop.mapreduce.v2.api.records.TaskId; +import org.apache.hadoop.mapreduce.v2.api.records.TaskState; import org.apache.hadoop.mapreduce.v2.api.records.TaskType; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.ApplicationConstants; @@ -460,4 +461,13 @@ public class TestMRApps { public void initialize(URI name, Configuration conf) throws IOException {} } + + @Test + public void testTaskStateUI() { + assertTrue(MRApps.TaskStateUI.PENDING.correspondsTo(TaskState.SCHEDULED)); + assertTrue(MRApps.TaskStateUI.COMPLETED.correspondsTo(TaskState.SUCCEEDED)); + assertTrue(MRApps.TaskStateUI.COMPLETED.correspondsTo(TaskState.FAILED)); + assertTrue(MRApps.TaskStateUI.COMPLETED.correspondsTo(TaskState.KILLED)); + assertTrue(MRApps.TaskStateUI.RUNNING.correspondsTo(TaskState.RUNNING)); + } }