Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add Host name to history server web page 

Change-Id: I59a467a49d417aa76548fb8e00f1582b09b9e041
This commit is contained in:
gaurangi 2020-05-04 21:37:07 +00:00
parent 8f78aeb250
commit 884f2024fb
14 changed files with 93 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
View File

@ -1588,6 +1588,19 @@ public class JobHistoryEventHandler extends AbstractService
this.jobIndexInfo =
new JobIndexInfo(-1, -1, user, jobName, jobId, -1, -1, null,
queueName);
if (getConfig().getBoolean(
JHAdminConfig.MR_HISTORY_APPEND_RM_HOST_TO_HISTORY_FILE_NAME_ENABLED,
JHAdminConfig.DEFAULT_MR_HISTORY_APPEND_RM_HOST_TO_HISTORY_FILE_NAME_ENABLED)) {
String hostName = getConfig().get(YarnConfiguration.RM_HOSTNAME, "");
if (!hostName.isEmpty()) {
this.jobIndexInfo.setResourceManagerHost(hostName);
} else {
LOG.warn("Could not retreive a valid YARN host name");
}
}
this.jobSummary = new JobSummary();
this.flushTimer = new Timer("FlushTimer", true);
this.forcedJobStateOnShutDown = forcedJobStateOnShutDown;

1
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/Job.java
View File

@ -73,6 +73,7 @@ public interface Job {
boolean isUber();
String getUserName();
String getQueueName();
String getResourceManagerHost();
/**
* @return a path to where the config file for this job is located.

5
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
View File

@ -1163,6 +1163,11 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job,
return queueName;
}
@Override
public String getResourceManagerHost() {
throw new UnsupportedOperationException();
}
@Override
public void setQueueName(String queueName) {
this.queueName = queueName;

5
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java
View File

@ -605,6 +605,11 @@ public class MockJobs extends MockApps {
return "mockqueue";
}
@Override
public String getResourceManagerHost() {
return "mockresourcemanager";
}
@Override
public Path getConfFile() {
return configFile;

5
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java
View File

@ -490,6 +490,11 @@ public class TestRuntimeEstimators {
throw new UnsupportedOperationException("Not supported yet.");
}
@Override
public String getResourceManagerHost() {
throw new UnsupportedOperationException("Not supported yet.");
}
@Override
public int getTotalMaps() {
return mapTasks.size();

18
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java
View File

@ -52,6 +52,7 @@ public class FileNameIndexUtils {
private static final int JOB_STATUS_INDEX = 7;
private static final int QUEUE_NAME_INDEX = 8;
private static final int JOB_START_TIME_INDEX = 9;
private static final int RESOURCE_MANAGER_INDEX = 10;
/**
* Constructs the job history file name from the JobIndexInfo.
@ -116,6 +117,13 @@ public class FileNameIndexUtils {
sb.append(encodeJobHistoryFileName(
String.valueOf(indexInfo.getJobStartTime())));
String resourceManagerName = getResourceManagerHost(indexInfo);
if (resourceManagerName != null) {
sb.append(DELIMITER);
sb.append(escapeDelimiters(encodeJobHistoryFileName(
getResourceManagerHost(indexInfo))));
}
sb.append(encodeJobHistoryFileName(
JobHistoryUtils.JOB_HISTORY_FILE_EXTENSION));
return sb.toString();
@ -198,6 +206,12 @@ public class FileNameIndexUtils {
LOG.warn("Unable to parse start time from job history file "
+ jhFileName + " : " + e);
}
if (jobDetails.length > RESOURCE_MANAGER_INDEX) {
indexInfo.setResourceManagerHost(decodeJobHistoryFileName(jobDetails[RESOURCE_MANAGER_INDEX]));
} else {
LOG.warn("Could not parse cluster name from job history file " + jhFileName );
}
} catch (IndexOutOfBoundsException e) {
LOG.warn("Parsing job history file with partial data encoded into name: "
+ jhFileName);
@ -292,6 +306,10 @@ public class FileNameIndexUtils {
return getNonEmptyString(indexInfo.getQueueName());
}
private static String getResourceManagerHost(JobIndexInfo indexInfo) {
return getNonEmptyString(indexInfo.getResourceManagerHost());
}
//TODO Maybe handle default values for longs and integers here?
private static String getNonEmptyString(String in) {

5
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
View File

@ -103,6 +103,11 @@ public class JHAdminConfig {
public static final short
DEFAULT_MR_HISTORY_INTERMEDIATE_USER_DONE_DIR_PERMISSIONS = 0770;
public static final String MR_HISTORY_APPEND_RM_HOST_TO_HISTORY_FILE_NAME_ENABLED =
MR_HISTORY_PREFIX + "append-rm-host-to-history-file-name.enabled";
public static final boolean DEFAULT_MR_HISTORY_APPEND_RM_HOST_TO_HISTORY_FILE_NAME_ENABLED =
true;
/** Size of the job list cache.*/
public static final String MR_HISTORY_JOBLIST_CACHE_SIZE =
MR_HISTORY_PREFIX + "joblist.cache.size";

8
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobIndexInfo.java
View File

@ -36,6 +36,7 @@ public class JobIndexInfo {
private int numReduces;
private String jobStatus;
private long jobStartTime;
private String resourceManagerHost;
public JobIndexInfo() {
}
@ -59,6 +60,7 @@ public class JobIndexInfo {
this.jobStatus = jobStatus;
this.jobStartTime = -1;
this.queueName = queueName;
this.resourceManagerHost = null;
}
public long getSubmitTime() {
@ -121,6 +123,12 @@ public class JobIndexInfo {
public void setJobStartTime(long lTime) {
this.jobStartTime = lTime;
}
public String getResourceManagerHost() {
return resourceManagerHost;
}
public void setResourceManagerHost(String resourceManagerHost) {
this.resourceManagerHost = resourceManagerHost;
}
@Override
public String toString() {

5
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java
View File

@ -416,6 +416,11 @@ public class CompletedJob implements org.apache.hadoop.mapreduce.v2.app.job.Job
return jobInfo.getJobQueueName();
}
@Override
public String getResourceManagerHost() {
throw new UnsupportedOperationException();
}
@Override
public int getTotalMaps() {
return (int) jobInfo.getTotalMaps();

5
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/PartialJob.java
View File

@ -76,6 +76,11 @@ public class PartialJob implements org.apache.hadoop.mapreduce.v2.app.job.Job {
return jobIndexInfo.getQueueName();
}
@Override
public String getResourceManagerHost() {
return jobIndexInfo.getResourceManagerHost();
}
@Override
public JobState getState() {
JobState js = null;

5
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/UnparsedJob.java
View File

@ -159,6 +159,11 @@ public class UnparsedJob implements org.apache.hadoop.mapreduce.v2.app.job.Job {
return jobIndexInfo.getQueueName();
}
@Override
public String getResourceManagerHost() {
return jobIndexInfo.getResourceManagerHost();
}
@Override
public Path getConfFile() {
return jhfInfo.getConfFile();

10
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsJobsBlock.java
View File

@ -76,6 +76,7 @@ public class HsJobsBlock extends HtmlBlock {
th(".id", "Job ID").
th(".name", "Name").
th("User").
th(".resourceManagerHost", "Resource Manager Host").
th("Queue").
th(".state", "State").
th("Maps Total").
@ -96,6 +97,11 @@ public class HsJobsBlock extends HtmlBlock {
.checkAccess(ugi, JobACL.VIEW_JOB, job.getUserName(), null)) {
continue;
}
String resourceManagerHost = "";
if (j.getResourceManagerHost() != null) {
resourceManagerHost = j.getResourceManagerHost();
}
jobsTableData.append("[\"")
.append(dateFormat.format(new Date(job.getSubmitTime()))).append("\",\"")
.append(job.getFormattedStartTimeStr(dateFormat)).append("\",\"")
@ -106,6 +112,8 @@ public class HsJobsBlock extends HtmlBlock {
job.getName()))).append("\",\"")
.append(StringEscapeUtils.escapeEcmaScript(StringEscapeUtils.escapeHtml4(
job.getUserName()))).append("\",\"")
.append(StringEscapeUtils.escapeEcmaScript(StringEscapeUtils.escapeHtml4(
resourceManagerHost))).append("\",\"")
.append(StringEscapeUtils.escapeEcmaScript(StringEscapeUtils.escapeHtml4(
job.getQueueName()))).append("\",\"")
.append(job.getState()).append("\",\"")
@ -140,6 +148,8 @@ public class HsJobsBlock extends HtmlBlock {
.$name("name").$value("Name").__().__().
th().input("search_init").$type(InputType.text)
.$name("user").$value("User").__().__().
th().input("search_init").$type(InputType.text)
.$name("resourceManagerHost").$value("Resource Manager Host").__().__().
th().input("search_init").$type(InputType.text)
.$name("queue").$value("Queue").__().__().
th().input("search_init").$type(InputType.text)

1
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestBlocks.java
View File

@ -398,6 +398,7 @@ public class TestBlocks {
when(job.getReport()).thenReturn(report);
when(job.getName()).thenReturn("JobName");
when(job.getUserName()).thenReturn("UserName");
when(job.getResourceManagerHost()).thenReturn("ResourceManagerHost");
when(job.getQueueName()).thenReturn("QueueName");
when(job.getState()).thenReturn(JobState.SUCCEEDED);
when(job.getTotalMaps()).thenReturn(3);

5
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAcls.java
View File

@ -378,6 +378,11 @@ public class TestHsWebServicesAcls {
return mockJob.getQueueName();
}
@Override
public String getResourceManagerHost() {
return mockJob.getResourceManagerHost();
}
@Override
public Path getConfFile() {
return new Path("/some/path/to/conf");