MAPREDUCE-5052. Job History UI and web services confusing job start time and job submit time (Chen He via jeagles)
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1549903 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
507ce20d4f
commit
b43308af0b
|
@ -41,6 +41,9 @@ Release 2.4.0 - UNRELEASED
|
|||
|
||||
MAPREDUCE-5481. Enable uber jobs to have multiple reducers (Sandy Ryza)
|
||||
|
||||
MAPREDUCE-5052. Job History UI and web services confusing job start time and
|
||||
job submit time (Chen He via jeagles)
|
||||
|
||||
OPTIMIZATIONS
|
||||
|
||||
MAPREDUCE-5484. YarnChild unnecessarily loads job conf twice (Sandy Ryza)
|
||||
|
|
|
@ -520,6 +520,11 @@ public class JobHistoryEventHandler extends AbstractService
|
|||
mi.getJobIndexInfo().setSubmitTime(jobSubmittedEvent.getSubmitTime());
|
||||
mi.getJobIndexInfo().setQueueName(jobSubmittedEvent.getJobQueueName());
|
||||
}
|
||||
//initialize the launchTime in the JobIndexInfo of MetaInfo
|
||||
if(event.getHistoryEvent().getEventType() == EventType.JOB_INITED ){
|
||||
JobInitedEvent jie = (JobInitedEvent) event.getHistoryEvent();
|
||||
mi.getJobIndexInfo().setJobStartTime(jie.getLaunchTime());
|
||||
}
|
||||
|
||||
// If this is JobFinishedEvent, close the writer and setup the job-index
|
||||
if (event.getHistoryEvent().getEventType() == EventType.JOB_FINISHED) {
|
||||
|
|
|
@ -155,6 +155,7 @@ public class MockJobs extends MockApps {
|
|||
public static JobReport newJobReport(JobId id) {
|
||||
JobReport report = Records.newRecord(JobReport.class);
|
||||
report.setJobId(id);
|
||||
report.setSubmitTime(System.currentTimeMillis()-DT);
|
||||
report
|
||||
.setStartTime(System.currentTimeMillis() - (int) (Math.random() * DT));
|
||||
report.setFinishTime(System.currentTimeMillis()
|
||||
|
|
|
@ -51,6 +51,7 @@ public class FileNameIndexUtils {
|
|||
private static final int NUM_REDUCES_INDEX = 6;
|
||||
private static final int JOB_STATUS_INDEX = 7;
|
||||
private static final int QUEUE_NAME_INDEX = 8;
|
||||
private static final int JOB_START_TIME_INDEX = 9;
|
||||
|
||||
/**
|
||||
* Constructs the job history file name from the JobIndexInfo.
|
||||
|
@ -64,7 +65,7 @@ public class FileNameIndexUtils {
|
|||
sb.append(escapeDelimiters(TypeConverter.fromYarn(indexInfo.getJobId()).toString()));
|
||||
sb.append(DELIMITER);
|
||||
|
||||
//StartTime
|
||||
//SubmitTime
|
||||
sb.append(indexInfo.getSubmitTime());
|
||||
sb.append(DELIMITER);
|
||||
|
||||
|
@ -94,6 +95,10 @@ public class FileNameIndexUtils {
|
|||
|
||||
//QueueName
|
||||
sb.append(indexInfo.getQueueName());
|
||||
sb.append(DELIMITER);
|
||||
|
||||
//JobStartTime
|
||||
sb.append(indexInfo.getJobStartTime());
|
||||
|
||||
sb.append(JobHistoryUtils.JOB_HISTORY_FILE_EXTENSION);
|
||||
return encodeJobHistoryFileName(sb.toString());
|
||||
|
@ -161,6 +166,14 @@ public class FileNameIndexUtils {
|
|||
|
||||
indexInfo.setQueueName(
|
||||
decodeJobHistoryFileName(jobDetails[QUEUE_NAME_INDEX]));
|
||||
|
||||
try{
|
||||
indexInfo.setJobStartTime(
|
||||
Long.parseLong(decodeJobHistoryFileName(jobDetails[JOB_START_TIME_INDEX])));
|
||||
} catch (NumberFormatException e){
|
||||
LOG.warn("Unable to parse launch time from job history file "
|
||||
+ jhFileName + " : " + e);
|
||||
}
|
||||
} catch (IndexOutOfBoundsException e) {
|
||||
LOG.warn("Parsing job history file with partial data encoded into name: "
|
||||
+ jhFileName);
|
||||
|
|
|
@ -34,6 +34,7 @@ public class JobIndexInfo {
|
|||
private int numMaps;
|
||||
private int numReduces;
|
||||
private String jobStatus;
|
||||
private long jobStartTime;
|
||||
|
||||
public JobIndexInfo() {
|
||||
}
|
||||
|
@ -48,6 +49,7 @@ public class JobIndexInfo {
|
|||
this.numMaps = numMaps;
|
||||
this.numReduces = numReduces;
|
||||
this.jobStatus = jobStatus;
|
||||
this.jobStartTime = -1;
|
||||
}
|
||||
|
||||
public long getSubmitTime() {
|
||||
|
@ -104,6 +106,12 @@ public class JobIndexInfo {
|
|||
public void setJobStatus(String jobStatus) {
|
||||
this.jobStatus = jobStatus;
|
||||
}
|
||||
public long getJobStartTime() {
|
||||
return jobStartTime;
|
||||
}
|
||||
public void setJobStartTime(long lTime) {
|
||||
this.jobStartTime = lTime;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
|
|
|
@ -48,6 +48,7 @@ public class TestFileNameIndexUtils {
|
|||
+ FileNameIndexUtils.DELIMITER + "%s"
|
||||
+ FileNameIndexUtils.DELIMITER + "%s"
|
||||
+ FileNameIndexUtils.DELIMITER + "%s"
|
||||
+ FileNameIndexUtils.DELIMITER + "%s"
|
||||
+ JobHistoryUtils.JOB_HISTORY_FILE_EXTENSION;
|
||||
|
||||
private static final String JOB_ID = "job_1317928501754_0001";
|
||||
|
@ -67,6 +68,7 @@ public class TestFileNameIndexUtils {
|
|||
private static final String NUM_REDUCES = "1";
|
||||
private static final String JOB_STATUS = "SUCCEEDED";
|
||||
private static final String QUEUE_NAME = "default";
|
||||
private static final String JOB_START_TIME = "1317928742060";
|
||||
|
||||
@Test
|
||||
public void testEncodingDecodingEquivalence() throws IOException {
|
||||
|
@ -82,6 +84,7 @@ public class TestFileNameIndexUtils {
|
|||
info.setNumReduces(Integer.parseInt(NUM_REDUCES));
|
||||
info.setJobStatus(JOB_STATUS);
|
||||
info.setQueueName(QUEUE_NAME);
|
||||
info.setJobStartTime(Long.parseLong(JOB_START_TIME));
|
||||
|
||||
String jobHistoryFile = FileNameIndexUtils.getDoneFileName(info);
|
||||
JobIndexInfo parsedInfo = FileNameIndexUtils.getIndexInfo(jobHistoryFile);
|
||||
|
@ -104,6 +107,8 @@ public class TestFileNameIndexUtils {
|
|||
info.getJobStatus(), parsedInfo.getJobStatus());
|
||||
Assert.assertEquals("Queue name different after encoding and decoding",
|
||||
info.getQueueName(), parsedInfo.getQueueName());
|
||||
Assert.assertEquals("Job start time different after encoding and decoding",
|
||||
info.getJobStartTime(), parsedInfo.getJobStartTime());
|
||||
}
|
||||
|
||||
@Test
|
||||
|
@ -120,6 +125,7 @@ public class TestFileNameIndexUtils {
|
|||
info.setNumReduces(Integer.parseInt(NUM_REDUCES));
|
||||
info.setJobStatus(JOB_STATUS);
|
||||
info.setQueueName(QUEUE_NAME);
|
||||
info.setJobStartTime(Long.parseLong(JOB_START_TIME));
|
||||
|
||||
String jobHistoryFile = FileNameIndexUtils.getDoneFileName(info);
|
||||
Assert.assertTrue("User name not encoded correctly into job history file",
|
||||
|
@ -137,7 +143,8 @@ public class TestFileNameIndexUtils {
|
|||
NUM_MAPS,
|
||||
NUM_REDUCES,
|
||||
JOB_STATUS,
|
||||
QUEUE_NAME);
|
||||
QUEUE_NAME,
|
||||
JOB_START_TIME);
|
||||
|
||||
JobIndexInfo info = FileNameIndexUtils.getIndexInfo(jobHistoryFile);
|
||||
Assert.assertEquals("User name doesn't match",
|
||||
|
@ -158,6 +165,7 @@ public class TestFileNameIndexUtils {
|
|||
info.setNumReduces(Integer.parseInt(NUM_REDUCES));
|
||||
info.setJobStatus(JOB_STATUS);
|
||||
info.setQueueName(QUEUE_NAME);
|
||||
info.setJobStartTime(Long.parseLong(JOB_START_TIME));
|
||||
|
||||
String jobHistoryFile = FileNameIndexUtils.getDoneFileName(info);
|
||||
Assert.assertTrue("Job name not encoded correctly into job history file",
|
||||
|
@ -175,7 +183,8 @@ public class TestFileNameIndexUtils {
|
|||
NUM_MAPS,
|
||||
NUM_REDUCES,
|
||||
JOB_STATUS,
|
||||
QUEUE_NAME);
|
||||
QUEUE_NAME,
|
||||
JOB_START_TIME );
|
||||
|
||||
JobIndexInfo info = FileNameIndexUtils.getIndexInfo(jobHistoryFile);
|
||||
Assert.assertEquals("Job name doesn't match",
|
||||
|
|
|
@ -53,7 +53,8 @@ public class PartialJob implements org.apache.hadoop.mapreduce.v2.app.job.Job {
|
|||
this.jobIndexInfo = jobIndexInfo;
|
||||
this.jobId = jobId;
|
||||
jobReport = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(JobReport.class);
|
||||
jobReport.setStartTime(jobIndexInfo.getSubmitTime());
|
||||
jobReport.setSubmitTime(jobIndexInfo.getSubmitTime());
|
||||
jobReport.setStartTime(jobIndexInfo.getJobStartTime());
|
||||
jobReport.setFinishTime(jobIndexInfo.getFinishTime());
|
||||
jobReport.setJobState(getState());
|
||||
}
|
||||
|
|
|
@ -84,6 +84,7 @@ public class HsJobBlock extends HtmlBlock {
|
|||
_("Queue:", job.getQueueName()).
|
||||
_("State:", job.getState()).
|
||||
_("Uberized:", job.isUber()).
|
||||
_("Submitted:", new Date(job.getSubmitTime())).
|
||||
_("Started:", new Date(job.getStartTime())).
|
||||
_("Finished:", new Date(job.getFinishTime())).
|
||||
_("Elapsed:", StringUtils.formatTime(
|
||||
|
|
|
@ -55,6 +55,7 @@ public class HsJobsBlock extends HtmlBlock {
|
|||
table("#jobs").
|
||||
thead().
|
||||
tr().
|
||||
th("Submit Time").
|
||||
th("Start Time").
|
||||
th("Finish Time").
|
||||
th(".id", "Job ID").
|
||||
|
@ -74,6 +75,7 @@ public class HsJobsBlock extends HtmlBlock {
|
|||
for (Job j : appContext.getAllJobs().values()) {
|
||||
JobInfo job = new JobInfo(j);
|
||||
jobsTableData.append("[\"")
|
||||
.append(dateFormat.format(new Date(job.getSubmitTime()))).append("\",\"")
|
||||
.append(dateFormat.format(new Date(job.getStartTime()))).append("\",\"")
|
||||
.append(dateFormat.format(new Date(job.getFinishTime()))).append("\",\"")
|
||||
.append("<a href='").append(url("job", job.getId())).append("'>")
|
||||
|
@ -101,6 +103,7 @@ public class HsJobsBlock extends HtmlBlock {
|
|||
tbody._().
|
||||
tfoot().
|
||||
tr().
|
||||
th().input("search_init").$type(InputType.text).$name("submit_time").$value("Submit Time")._()._().
|
||||
th().input("search_init").$type(InputType.text).$name("start_time").$value("Start Time")._()._().
|
||||
th().input("search_init").$type(InputType.text).$name("finish_time").$value("Finish Time")._()._().
|
||||
th().input("search_init").$type(InputType.text).$name("start_time").$value("Job ID")._()._().
|
||||
|
|
|
@ -44,6 +44,7 @@ import org.apache.hadoop.security.authorize.AccessControlList;
|
|||
@XmlAccessorType(XmlAccessType.FIELD)
|
||||
public class JobInfo {
|
||||
|
||||
protected long submitTime;
|
||||
protected long startTime;
|
||||
protected long finishTime;
|
||||
protected String id;
|
||||
|
@ -85,6 +86,7 @@ public class JobInfo {
|
|||
this.mapsCompleted = job.getCompletedMaps();
|
||||
this.reducesTotal = job.getTotalReduces();
|
||||
this.reducesCompleted = job.getCompletedReduces();
|
||||
this.submitTime = report.getSubmitTime();
|
||||
this.startTime = report.getStartTime();
|
||||
this.finishTime = report.getFinishTime();
|
||||
this.name = job.getName().toString();
|
||||
|
@ -216,6 +218,10 @@ public class JobInfo {
|
|||
return this.id;
|
||||
}
|
||||
|
||||
public long getSubmitTime() {
|
||||
return this.submitTime;
|
||||
}
|
||||
|
||||
public long getStartTime() {
|
||||
return this.startTime;
|
||||
}
|
||||
|
|
|
@ -86,6 +86,7 @@ public class MockHistoryJobs extends MockJobs {
|
|||
report.getFinishTime(), mockJob.getUserName(), mockJob.getName(), id,
|
||||
mockJob.getCompletedMaps(), mockJob.getCompletedReduces(),
|
||||
String.valueOf(mockJob.getState()));
|
||||
info.setJobStartTime(report.getStartTime());
|
||||
info.setQueueName(mockJob.getQueueName());
|
||||
ret.partial.put(id, new PartialJob(info, id));
|
||||
|
||||
|
|
|
@ -33,7 +33,7 @@ import org.codehaus.jettison.json.JSONObject;
|
|||
public class VerifyJobsUtils {
|
||||
|
||||
public static void verifyHsJobPartial(JSONObject info, Job job) throws JSONException {
|
||||
assertEquals("incorrect number of elements", 11, info.length());
|
||||
assertEquals("incorrect number of elements", 12, info.length());
|
||||
|
||||
// everyone access fields
|
||||
verifyHsJobGeneric(job, info.getString("id"), info.getString("user"),
|
||||
|
@ -45,7 +45,7 @@ public class VerifyJobsUtils {
|
|||
}
|
||||
|
||||
public static void verifyHsJob(JSONObject info, Job job) throws JSONException {
|
||||
assertEquals("incorrect number of elements", 24, info.length());
|
||||
assertEquals("incorrect number of elements", 25, info.length());
|
||||
|
||||
// everyone access fields
|
||||
verifyHsJobGeneric(job, info.getString("id"), info.getString("user"),
|
||||
|
|
|
@ -201,6 +201,7 @@ History Server REST API's.
|
|||
"jobs" : {
|
||||
"job" : [
|
||||
{
|
||||
"submitTime" : 1326381344449,
|
||||
"state" : "SUCCEEDED",
|
||||
"user" : "user1",
|
||||
"reducesTotal" : 1,
|
||||
|
@ -214,6 +215,7 @@ History Server REST API's.
|
|||
"finishTime" : 1326381356010
|
||||
},
|
||||
{
|
||||
"submitTime" : 1326381446500
|
||||
"state" : "SUCCEEDED",
|
||||
"user" : "user1",
|
||||
"reducesTotal" : 1,
|
||||
|
@ -255,6 +257,7 @@ History Server REST API's.
|
|||
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<jobs>
|
||||
<job>
|
||||
<submitTime>1326381344449</submitTime>
|
||||
<startTime>1326381344489</startTime>
|
||||
<finishTime>1326381356010</finishTime>
|
||||
<id>job_1326381300833_1_1</id>
|
||||
|
@ -268,6 +271,7 @@ History Server REST API's.
|
|||
<reducesCompleted>1</reducesCompleted>
|
||||
</job>
|
||||
<job>
|
||||
<submitTime>1326381446500</submitTime>
|
||||
<startTime>1326381446529</startTime>
|
||||
<finishTime>1326381582106</finishTime>
|
||||
<id>job_1326381300833_2_2</id>
|
||||
|
@ -322,6 +326,8 @@ History Server REST API's.
|
|||
*---------------+--------------+-------------------------------+
|
||||
| diagnostics | string | A diagnostic message |
|
||||
*---------------+--------------+-------------------------------+
|
||||
| submitTime | long | The time the job submitted (in ms since epoch)|
|
||||
*---------------+--------------+-------------------------------+
|
||||
| startTime | long | The time the job started (in ms since epoch)|
|
||||
*---------------+--------------+-------------------------------+
|
||||
| finishTime | long | The time the job finished (in ms since epoch)|
|
||||
|
@ -393,6 +399,7 @@ History Server REST API's.
|
|||
+---+
|
||||
{
|
||||
"job" : {
|
||||
"submitTime": 1326381446500,
|
||||
"avgReduceTime" : 124961,
|
||||
"failedReduceAttempts" : 0,
|
||||
"state" : "SUCCEEDED",
|
||||
|
@ -453,6 +460,7 @@ History Server REST API's.
|
|||
+---+
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<job>
|
||||
<submitTime>1326381446500</submitTime>
|
||||
<startTime>1326381446529</startTime>
|
||||
<finishTime>1326381582106</finishTime>
|
||||
<id>job_1326381300833_2_2</id>
|
||||
|
@ -2663,5 +2671,4 @@ History Server REST API's.
|
|||
</counter>
|
||||
</taskAttemptCounterGroup>
|
||||
</jobTaskAttemptCounters>
|
||||
+---+
|
||||
|
||||
+---+
|
Loading…
Reference in New Issue