MAPREDUCE-4989. JSONify DataTables input data for Attempts page. Contributed by Ravi Prakash
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1445448 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
7e615c7f75
commit
c2d6407070
|
@ -696,6 +696,9 @@ Release 0.23.7 - UNRELEASED
|
||||||
MAPREDUCE-4905. test org.apache.hadoop.mapred.pipes
|
MAPREDUCE-4905. test org.apache.hadoop.mapred.pipes
|
||||||
(Aleksey Gorshkov via bobby)
|
(Aleksey Gorshkov via bobby)
|
||||||
|
|
||||||
|
MAPREDUCE-4989. JSONify DataTables input data for Attempts page (Ravi
|
||||||
|
Prakash via jlowe)
|
||||||
|
|
||||||
OPTIMIZATIONS
|
OPTIMIZATIONS
|
||||||
|
|
||||||
MAPREDUCE-4946. Fix a performance problem for large jobs by reducing the
|
MAPREDUCE-4946. Fix a performance problem for large jobs by reducing the
|
||||||
|
|
|
@ -27,18 +27,11 @@ import static org.apache.hadoop.yarn.webapp.view.JQueryUI.tableInit;
|
||||||
|
|
||||||
import java.util.Collection;
|
import java.util.Collection;
|
||||||
|
|
||||||
|
import org.apache.commons.lang.StringEscapeUtils;
|
||||||
import org.apache.hadoop.http.HttpConfig;
|
import org.apache.hadoop.http.HttpConfig;
|
||||||
import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt;
|
import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt;
|
||||||
import org.apache.hadoop.mapreduce.v2.app.webapp.dao.TaskAttemptInfo;
|
import org.apache.hadoop.mapreduce.v2.app.webapp.dao.TaskAttemptInfo;
|
||||||
import org.apache.hadoop.util.StringUtils;
|
|
||||||
import org.apache.hadoop.yarn.api.records.ContainerId;
|
|
||||||
import org.apache.hadoop.yarn.util.Times;
|
|
||||||
import org.apache.hadoop.yarn.webapp.SubView;
|
import org.apache.hadoop.yarn.webapp.SubView;
|
||||||
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
|
|
||||||
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
|
|
||||||
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY;
|
|
||||||
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TD;
|
|
||||||
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TR;
|
|
||||||
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
|
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
|
||||||
|
|
||||||
import com.google.inject.Inject;
|
import com.google.inject.Inject;
|
||||||
|
@ -60,7 +53,7 @@ public class TaskPage extends AppView {
|
||||||
h2($(TITLE));
|
h2($(TITLE));
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
TBODY<TABLE<Hamlet>> tbody = html.
|
html.
|
||||||
table("#attempts").
|
table("#attempts").
|
||||||
thead().
|
thead().
|
||||||
tr().
|
tr().
|
||||||
|
@ -72,49 +65,46 @@ public class TaskPage extends AppView {
|
||||||
th(".tsh", "Started").
|
th(".tsh", "Started").
|
||||||
th(".tsh", "Finished").
|
th(".tsh", "Finished").
|
||||||
th(".tsh", "Elapsed").
|
th(".tsh", "Elapsed").
|
||||||
th(".note", "Note")._()._().
|
th(".note", "Note")._()._();
|
||||||
tbody();
|
// Write all the data into a JavaScript array of arrays for JQuery
|
||||||
|
// DataTables to display
|
||||||
|
StringBuilder attemptsTableData = new StringBuilder("[\n");
|
||||||
|
|
||||||
for (TaskAttempt attempt : getTaskAttempts()) {
|
for (TaskAttempt attempt : getTaskAttempts()) {
|
||||||
TaskAttemptInfo ta = new TaskAttemptInfo(attempt, true);
|
TaskAttemptInfo ta = new TaskAttemptInfo(attempt, true);
|
||||||
String taid = ta.getId();
|
|
||||||
String progress = percent(ta.getProgress() / 100);
|
String progress = percent(ta.getProgress() / 100);
|
||||||
ContainerId containerId = ta.getAssignedContainerId();
|
|
||||||
|
|
||||||
String nodeHttpAddr = ta.getNode();
|
String nodeHttpAddr = ta.getNode();
|
||||||
long startTime = ta.getStartTime();
|
|
||||||
long finishTime = ta.getFinishTime();
|
|
||||||
long elapsed = ta.getElapsedTime();
|
|
||||||
String diag = ta.getNote() == null ? "" : ta.getNote();
|
String diag = ta.getNote() == null ? "" : ta.getNote();
|
||||||
TR<TBODY<TABLE<Hamlet>>> row = tbody.tr();
|
attemptsTableData.append("[\"")
|
||||||
TD<TR<TBODY<TABLE<Hamlet>>>> nodeTd = row.
|
.append(ta.getId()).append("\",\"")
|
||||||
td(".id", taid).
|
.append(progress).append("\",\"")
|
||||||
td(".progress", progress).
|
.append(ta.getState().toString()).append("\",\"")
|
||||||
td(".state", ta.getState()).td();
|
|
||||||
if (nodeHttpAddr == null) {
|
|
||||||
nodeTd._("N/A");
|
|
||||||
} else {
|
|
||||||
nodeTd.
|
|
||||||
a(".nodelink", url(HttpConfig.getSchemePrefix(),
|
|
||||||
nodeHttpAddr), nodeHttpAddr);
|
|
||||||
}
|
|
||||||
nodeTd._();
|
|
||||||
if (containerId != null) {
|
|
||||||
String containerIdStr = ta.getAssignedContainerIdStr();
|
|
||||||
row.td().
|
|
||||||
a(".logslink", url(HttpConfig.getSchemePrefix(),
|
|
||||||
nodeHttpAddr, "node", "containerlogs",
|
|
||||||
containerIdStr, app.getJob().getUserName()), "logs")._();
|
|
||||||
} else {
|
|
||||||
row.td()._("N/A")._();
|
|
||||||
}
|
|
||||||
|
|
||||||
row.
|
.append(nodeHttpAddr == null ? "N/A" :
|
||||||
td(".ts", Times.format(startTime)).
|
"<a class='nodelink' href='" + HttpConfig.getSchemePrefix() + nodeHttpAddr + "'>"
|
||||||
td(".ts", Times.format(finishTime)).
|
+ nodeHttpAddr + "</a>")
|
||||||
td(".dt", StringUtils.formatTime(elapsed)).
|
.append("\",\"")
|
||||||
td(".note", diag)._();
|
|
||||||
|
.append(ta.getAssignedContainerId() == null ? "N/A" :
|
||||||
|
"<a class='logslink' href='" + url(HttpConfig.getSchemePrefix(), nodeHttpAddr, "node"
|
||||||
|
, "containerlogs", ta.getAssignedContainerIdStr(), app.getJob()
|
||||||
|
.getUserName()) + "'>logs</a>")
|
||||||
|
.append("\",\"")
|
||||||
|
|
||||||
|
.append(ta.getStartTime()).append("\",\"")
|
||||||
|
.append(ta.getFinishTime()).append("\",\"")
|
||||||
|
.append(ta.getElapsedTime()).append("\",\"")
|
||||||
|
.append(StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml(
|
||||||
|
diag))).append("\"],\n");
|
||||||
}
|
}
|
||||||
tbody._()._();
|
//Remove the last comma and close off the array of arrays
|
||||||
|
if(attemptsTableData.charAt(attemptsTableData.length() - 2) == ',') {
|
||||||
|
attemptsTableData.delete(attemptsTableData.length()-2, attemptsTableData.length()-1);
|
||||||
|
}
|
||||||
|
attemptsTableData.append("]");
|
||||||
|
html.script().$type("text/javascript").
|
||||||
|
_("var attemptsTableData=" + attemptsTableData)._();
|
||||||
}
|
}
|
||||||
|
|
||||||
protected boolean isValidRequest() {
|
protected boolean isValidRequest() {
|
||||||
|
@ -140,9 +130,24 @@ public class TaskPage extends AppView {
|
||||||
}
|
}
|
||||||
|
|
||||||
private String attemptsTableInit() {
|
private String attemptsTableInit() {
|
||||||
return tableInit().
|
return tableInit()
|
||||||
|
.append(", 'aaData': attemptsTableData")
|
||||||
|
.append(", bDeferRender: true")
|
||||||
|
.append(", bProcessing: true")
|
||||||
|
.append("\n,aoColumnDefs:[\n")
|
||||||
|
|
||||||
|
//logs column should not filterable (it includes container ID which may pollute searches)
|
||||||
|
.append("\n{'aTargets': [ 4 ]")
|
||||||
|
.append(", 'bSearchable': false }")
|
||||||
|
|
||||||
|
.append("\n, {'sType':'numeric', 'aTargets': [ 5, 6")
|
||||||
|
.append(" ], 'mRender': renderHadoopDate }")
|
||||||
|
|
||||||
|
.append("\n, {'sType':'numeric', 'aTargets': [ 7")
|
||||||
|
.append(" ], 'mRender': renderHadoopElapsedTime }]")
|
||||||
|
|
||||||
// Sort by id upon page load
|
// Sort by id upon page load
|
||||||
append(", aaSorting: [[0, 'asc']]").
|
.append("\n, aaSorting: [[0, 'asc']]")
|
||||||
append("}").toString();
|
.append("}").toString();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -29,6 +29,7 @@ import static org.apache.hadoop.yarn.webapp.view.JQueryUI.tableInit;
|
||||||
|
|
||||||
import java.util.Collection;
|
import java.util.Collection;
|
||||||
|
|
||||||
|
import org.apache.commons.lang.StringEscapeUtils;
|
||||||
import org.apache.hadoop.http.HttpConfig;
|
import org.apache.hadoop.http.HttpConfig;
|
||||||
import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
|
import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
|
||||||
import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
|
import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
|
||||||
|
@ -110,6 +111,10 @@ public class HsTaskPage extends HsView {
|
||||||
th(".note", "Note");
|
th(".note", "Note");
|
||||||
|
|
||||||
TBODY<TABLE<Hamlet>> tbody = headRow._()._().tbody();
|
TBODY<TABLE<Hamlet>> tbody = headRow._()._().tbody();
|
||||||
|
// Write all the data into a JavaScript array of arrays for JQuery
|
||||||
|
// DataTables to display
|
||||||
|
StringBuilder attemptsTableData = new StringBuilder("[\n");
|
||||||
|
|
||||||
for (TaskAttempt ta : getTaskAttempts()) {
|
for (TaskAttempt ta : getTaskAttempts()) {
|
||||||
String taid = MRApps.toString(ta.getID());
|
String taid = MRApps.toString(ta.getID());
|
||||||
|
|
||||||
|
@ -139,56 +144,41 @@ public class HsTaskPage extends HsView {
|
||||||
Times.elapsed(attemptStartTime, attemptFinishTime, false);
|
Times.elapsed(attemptStartTime, attemptFinishTime, false);
|
||||||
int sortId = ta.getID().getId() + (ta.getID().getTaskId().getId() * 10000);
|
int sortId = ta.getID().getId() + (ta.getID().getTaskId().getId() * 10000);
|
||||||
|
|
||||||
TR<TBODY<TABLE<Hamlet>>> row = tbody.tr();
|
attemptsTableData.append("[\"")
|
||||||
TD<TR<TBODY<TABLE<Hamlet>>>> td = row.td();
|
.append(sortId + " ").append(taid).append("\",\"")
|
||||||
|
.append(ta.getState().toString()).append("\",\"")
|
||||||
|
|
||||||
td.br().$title(String.valueOf(sortId))._(). // sorting
|
.append("<a class='nodelink' href='" + HttpConfig.getSchemePrefix() + nodeHttpAddr + "'>")
|
||||||
_(taid)._().td(ta.getState().toString()).td().a(".nodelink",
|
.append(nodeRackName + "/" + nodeHttpAddr + "</a>\",\"")
|
||||||
HttpConfig.getSchemePrefix()+ nodeHttpAddr,
|
|
||||||
nodeRackName + "/" + nodeHttpAddr);
|
|
||||||
td._();
|
|
||||||
row.td().
|
|
||||||
a(".logslink",
|
|
||||||
url("logs", nodeIdString, containerIdString, taid, app.getJob()
|
|
||||||
.getUserName()), "logs")._();
|
|
||||||
|
|
||||||
row.td().
|
.append("<a class='logslink' href='").append(url("logs", nodeIdString
|
||||||
br().$title(String.valueOf(attemptStartTime))._().
|
, containerIdString, taid, app.getJob().getUserName()))
|
||||||
_(Times.format(attemptStartTime))._();
|
.append("'>logs</a>\",\"")
|
||||||
|
|
||||||
|
.append(attemptStartTime).append("\",\"");
|
||||||
|
|
||||||
if(type == TaskType.REDUCE) {
|
if(type == TaskType.REDUCE) {
|
||||||
row.td().
|
attemptsTableData.append(shuffleFinishTime).append("\",\"")
|
||||||
br().$title(String.valueOf(shuffleFinishTime))._().
|
.append(sortFinishTime).append("\",\"");
|
||||||
_(Times.format(shuffleFinishTime))._();
|
|
||||||
row.td().
|
|
||||||
br().$title(String.valueOf(sortFinishTime))._().
|
|
||||||
_(Times.format(sortFinishTime))._();
|
|
||||||
}
|
}
|
||||||
row.
|
attemptsTableData.append(attemptFinishTime).append("\",\"");
|
||||||
td().
|
|
||||||
br().$title(String.valueOf(attemptFinishTime))._().
|
|
||||||
_(Times.format(attemptFinishTime))._();
|
|
||||||
|
|
||||||
if(type == TaskType.REDUCE) {
|
if(type == TaskType.REDUCE) {
|
||||||
row.td().
|
attemptsTableData.append(elapsedShuffleTime).append("\",\"")
|
||||||
br().$title(String.valueOf(elapsedShuffleTime))._().
|
.append(elapsedSortTime).append("\",\"")
|
||||||
_(formatTime(elapsedShuffleTime))._();
|
.append(elapsedReduceTime).append("\",\"");
|
||||||
row.td().
|
|
||||||
br().$title(String.valueOf(elapsedSortTime))._().
|
|
||||||
_(formatTime(elapsedSortTime))._();
|
|
||||||
row.td().
|
|
||||||
br().$title(String.valueOf(elapsedReduceTime))._().
|
|
||||||
_(formatTime(elapsedReduceTime))._();
|
|
||||||
}
|
}
|
||||||
|
attemptsTableData.append(attemptElapsed).append("\",\"")
|
||||||
row.
|
.append(StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml(
|
||||||
td().
|
Joiner.on('\n').join(ta.getDiagnostics())))).append("\"],\n");
|
||||||
br().$title(String.valueOf(attemptElapsed))._().
|
|
||||||
_(formatTime(attemptElapsed))._().
|
|
||||||
td(".note", Joiner.on('\n').join(ta.getDiagnostics()));
|
|
||||||
row._();
|
|
||||||
}
|
}
|
||||||
|
//Remove the last comma and close off the array of arrays
|
||||||
|
if(attemptsTableData.charAt(attemptsTableData.length() - 2) == ',') {
|
||||||
|
attemptsTableData.delete(attemptsTableData.length()-2, attemptsTableData.length()-1);
|
||||||
|
}
|
||||||
|
attemptsTableData.append("]");
|
||||||
|
html.script().$type("text/javascript").
|
||||||
|
_("var attemptsTableData=" + attemptsTableData)._();
|
||||||
|
|
||||||
TR<TFOOT<TABLE<Hamlet>>> footRow = tbody._().tfoot().tr();
|
TR<TFOOT<TABLE<Hamlet>>> footRow = tbody._().tfoot().tr();
|
||||||
footRow.
|
footRow.
|
||||||
|
@ -237,10 +227,6 @@ public class HsTaskPage extends HsView {
|
||||||
footRow._()._()._();
|
footRow._()._()._();
|
||||||
}
|
}
|
||||||
|
|
||||||
private String formatTime(long elapsed) {
|
|
||||||
return elapsed < 0 ? "N/A" : StringUtils.formatTime(elapsed);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @return true if this is a valid request else false.
|
* @return true if this is a valid request else false.
|
||||||
*/
|
*/
|
||||||
|
@ -292,21 +278,31 @@ public class HsTaskPage extends HsView {
|
||||||
TaskId taskID = MRApps.toTaskID($(TASK_ID));
|
TaskId taskID = MRApps.toTaskID($(TASK_ID));
|
||||||
type = taskID.getTaskType();
|
type = taskID.getTaskType();
|
||||||
}
|
}
|
||||||
StringBuilder b = tableInit().
|
StringBuilder b = tableInit()
|
||||||
append(",aoColumnDefs:[");
|
.append(", 'aaData': attemptsTableData")
|
||||||
|
.append(", bDeferRender: true")
|
||||||
|
.append(", bProcessing: true")
|
||||||
|
.append("\n,aoColumnDefs:[\n")
|
||||||
|
|
||||||
b.append("{'sType':'title-numeric', 'aTargets': [ 0");
|
//logs column should not filterable (it includes container ID which may pollute searches)
|
||||||
if(type == TaskType.REDUCE) {
|
.append("\n{'aTargets': [ 3 ]")
|
||||||
b.append(", 7, 8, 9, 10");
|
.append(", 'bSearchable': false }")
|
||||||
} else { //MAP
|
|
||||||
b.append(", 5");
|
.append("\n, {'sType':'numeric', 'aTargets': [ 0 ]")
|
||||||
}
|
.append(", 'mRender': parseHadoopAttemptID }")
|
||||||
b.append(" ] }]");
|
|
||||||
|
.append("\n, {'sType':'numeric', 'aTargets': [ 4, 5")
|
||||||
|
//Column numbers are different for maps and reduces
|
||||||
|
.append(type == TaskType.REDUCE ? ", 6, 7" : "")
|
||||||
|
.append(" ], 'mRender': renderHadoopDate }")
|
||||||
|
|
||||||
|
.append("\n, {'sType':'numeric', 'aTargets': [")
|
||||||
|
.append(type == TaskType.REDUCE ? "8, 9, 10, 11" : "6")
|
||||||
|
.append(" ], 'mRender': renderHadoopElapsedTime }]")
|
||||||
|
|
||||||
// Sort by id upon page load
|
// Sort by id upon page load
|
||||||
b.append(", aaSorting: [[0, 'asc']]");
|
.append("\n, aaSorting: [[0, 'asc']]")
|
||||||
|
.append("}");
|
||||||
b.append("}");
|
|
||||||
return b.toString();
|
return b.toString();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -140,6 +140,7 @@ public class HsTasksBlock extends HtmlBlock {
|
||||||
attemptFinishTime = ta.getFinishTime();
|
attemptFinishTime = ta.getFinishTime();
|
||||||
attemptElapsed = ta.getElapsedTime();
|
attemptElapsed = ta.getElapsedTime();
|
||||||
}
|
}
|
||||||
|
|
||||||
tasksTableData.append("[\"")
|
tasksTableData.append("[\"")
|
||||||
.append("<a href='" + url("task", tid)).append("'>")
|
.append("<a href='" + url("task", tid)).append("'>")
|
||||||
.append(tid).append("</a>\",\"")
|
.append(tid).append("</a>\",\"")
|
||||||
|
@ -205,9 +206,4 @@ public class HsTasksBlock extends HtmlBlock {
|
||||||
|
|
||||||
footRow._()._()._();
|
footRow._()._()._();
|
||||||
}
|
}
|
||||||
|
|
||||||
private String formatTime(long elapsed) {
|
|
||||||
return elapsed < 0 ? "N/A" : StringUtils.formatTime(elapsed);
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -67,33 +67,25 @@ public class HsTasksPage extends HsView {
|
||||||
type = MRApps.taskType(symbol);
|
type = MRApps.taskType(symbol);
|
||||||
}
|
}
|
||||||
StringBuilder b = tableInit().
|
StringBuilder b = tableInit().
|
||||||
append(", 'aaData': tasksTableData");
|
append(", 'aaData': tasksTableData")
|
||||||
b.append(", bDeferRender: true");
|
.append(", bDeferRender: true")
|
||||||
b.append(", bProcessing: true");
|
.append(", bProcessing: true")
|
||||||
|
|
||||||
b.append("\n, aoColumnDefs: [\n");
|
.append("\n, aoColumnDefs: [\n")
|
||||||
b.append("{'sType':'numeric', 'aTargets': [ 0 ]");
|
.append("{'sType':'numeric', 'aTargets': [ 0 ]")
|
||||||
b.append(", 'mRender': parseHadoopID }");
|
.append(", 'mRender': parseHadoopID }")
|
||||||
|
|
||||||
b.append(", {'sType':'numeric', 'aTargets': [ 4");
|
.append(", {'sType':'numeric', 'aTargets': [ 4")
|
||||||
if(type == TaskType.REDUCE) {
|
.append(type == TaskType.REDUCE ? ", 9, 10, 11, 12" : ", 7")
|
||||||
b.append(", 9, 10, 11, 12");
|
.append(" ], 'mRender': renderHadoopElapsedTime }")
|
||||||
} else { //MAP
|
|
||||||
b.append(", 7");
|
|
||||||
}
|
|
||||||
b.append(" ], 'mRender': renderHadoopElapsedTime }");
|
|
||||||
|
|
||||||
b.append("\n, {'sType':'numeric', 'aTargets': [ 2, 3, 5");
|
.append("\n, {'sType':'numeric', 'aTargets': [ 2, 3, 5")
|
||||||
if(type == TaskType.REDUCE) {
|
.append(type == TaskType.REDUCE ? ", 6, 7, 8" : ", 6")
|
||||||
b.append(", 6, 7, 8");
|
.append(" ], 'mRender': renderHadoopDate }]")
|
||||||
} else { //MAP
|
|
||||||
b.append(", 6");
|
|
||||||
}
|
|
||||||
b.append(" ], 'mRender': renderHadoopDate }]");
|
|
||||||
|
|
||||||
// Sort by id upon page load
|
// Sort by id upon page load
|
||||||
b.append("\n, aaSorting: [[0, 'asc']]");
|
.append("\n, aaSorting: [[0, 'asc']]")
|
||||||
b.append("}");
|
.append("}");
|
||||||
return b.toString();
|
return b.toString();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue