Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

MAPREDUCE-3901. Modified JobHistory records in YARN to lazily load job and task reports so as to improve UI response times. Contributed by Siddarth Seth. 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1294417 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Vinod Kumar Vavilapalli 2012-02-28 00:32:19 +00:00
parent 7144c91c53
commit 7475e836dc
9 changed files with 825 additions and 158 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-mapreduce-project/CHANGES.txt
View File

@ -138,6 +138,9 @@ Release 0.23.2 - UNRELEASED
OPTIMIZATIONS OPTIMIZATIONS
MAPREDUCE-3901. Modified JobHistory records in YARN to lazily load job and
task reports so as to improve UI response times. (Siddarth Seth via vinodkv)
BUG FIXES BUG FIXES
MAPREDUCE-3918 proc_historyserver no longer in command line arguments for MAPREDUCE-3918 proc_historyserver no longer in command line arguments for
HistoryServer (Jon Eagles via bobby) HistoryServer (Jon Eagles via bobby)

6
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRBuilderUtils.java
View File

@ -30,6 +30,7 @@ import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.util.Records;
public class MRBuilderUtils { public class MRBuilderUtils {
@ -41,6 +42,11 @@ public class MRBuilderUtils {
return jobId; return jobId;
} }
public static JobId newJobId(long clusterTs, int appIdInt, int id) {
ApplicationId appId = BuilderUtils.newApplicationId(clusterTs, appIdInt);
return MRBuilderUtils.newJobId(appId, id);
}
public static TaskId newTaskId(JobId jobId, int id, TaskType taskType) { public static TaskId newTaskId(JobId jobId, int id, TaskType taskType) {
TaskId taskId = Records.newRecord(TaskId.class); TaskId taskId = Records.newRecord(TaskId.class);
taskId.setJobId(jobId); taskId.setJobId(jobId);

149
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java
View File

@ -19,13 +19,16 @@
package org.apache.hadoop.mapreduce.v2.hs; package org.apache.hadoop.mapreduce.v2.hs;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.net.UnknownHostException;
import java.util.Collections; import java.util.Collections;
import java.util.Comparator; import java.util.Comparator;
import java.util.HashMap; import java.util.HashMap;
import java.util.LinkedList; import java.util.LinkedList;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
@ -34,6 +37,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobACLsManager; import org.apache.hadoop.mapred.JobACLsManager;
import org.apache.hadoop.mapreduce.Counters; import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.JobACL; import org.apache.hadoop.mapreduce.JobACL;
import org.apache.hadoop.mapreduce.TaskID;
import org.apache.hadoop.mapreduce.TypeConverter; import org.apache.hadoop.mapreduce.TypeConverter;
import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser; import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser;
import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo; import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo;
@ -54,7 +58,7 @@ import org.apache.hadoop.mapreduce.v2.util.MRBuilderUtils;
import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.yarn.YarnException; import org.apache.hadoop.yarn.YarnException;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.util.Records;
/** /**
@ -64,19 +68,20 @@ import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
public class CompletedJob implements org.apache.hadoop.mapreduce.v2.app.job.Job { public class CompletedJob implements org.apache.hadoop.mapreduce.v2.app.job.Job {
static final Log LOG = LogFactory.getLog(CompletedJob.class); static final Log LOG = LogFactory.getLog(CompletedJob.class);
private final Counters counters;
private final Configuration conf; private final Configuration conf;
private final JobId jobId; private final JobId jobId; //Can be picked from JobInfo with a conversion.
private final List<String> diagnostics = new ArrayList<String>(); private final String user; //Can be picked up from JobInfo
private final JobReport report;
private final Map<TaskId, Task> tasks = new HashMap<TaskId, Task>();
private final Map<TaskId, Task> mapTasks = new HashMap<TaskId, Task>();
private final Map<TaskId, Task> reduceTasks = new HashMap<TaskId, Task>();
private final String user;
private final Path confFile; private final Path confFile;
private JobACLsManager aclsMgr;
private List<TaskAttemptCompletionEvent> completionEvents = null;
private JobInfo jobInfo; private JobInfo jobInfo;
private JobReport report;
AtomicBoolean tasksLoaded = new AtomicBoolean(false);
private Lock tasksLock = new ReentrantLock();
private Map<TaskId, Task> tasks = new HashMap<TaskId, Task>();
private Map<TaskId, Task> mapTasks = new HashMap<TaskId, Task>();
private Map<TaskId, Task> reduceTasks = new HashMap<TaskId, Task>();
private List<TaskAttemptCompletionEvent> completionEvents = null;
private JobACLsManager aclsMgr;
public CompletedJob(Configuration conf, JobId jobId, Path historyFile, public CompletedJob(Configuration conf, JobId jobId, Path historyFile,
boolean loadTasks, String userName, Path confFile, JobACLsManager aclsMgr) boolean loadTasks, String userName, Path confFile, JobACLsManager aclsMgr)
@ -84,30 +89,10 @@ public class CompletedJob implements org.apache.hadoop.mapreduce.v2.app.job.Job
LOG.info("Loading job: " + jobId + " from file: " + historyFile); LOG.info("Loading job: " + jobId + " from file: " + historyFile);
this.conf = conf; this.conf = conf;
this.jobId = jobId; this.jobId = jobId;
this.user = userName;
this.confFile = confFile; this.confFile = confFile;
this.aclsMgr = aclsMgr; this.aclsMgr = aclsMgr;
loadFullHistoryData(loadTasks, historyFile); loadFullHistoryData(loadTasks, historyFile);
user = userName;
counters = jobInfo.getTotalCounters();
diagnostics.add(jobInfo.getErrorInfo());
report =
RecordFactoryProvider.getRecordFactory(null).newRecordInstance(
JobReport.class);
report.setJobId(jobId);
report.setJobState(JobState.valueOf(jobInfo.getJobStatus()));
report.setSubmitTime(jobInfo.getSubmitTime());
report.setStartTime(jobInfo.getLaunchTime());
report.setFinishTime(jobInfo.getFinishTime());
report.setJobName(jobInfo.getJobname());
report.setUser(jobInfo.getUsername());
report.setMapProgress((float) getCompletedMaps() / getTotalMaps());
report.setReduceProgress((float) getCompletedReduces() / getTotalReduces());
report.setJobFile(confFile.toString());
report.setTrackingUrl(JobHistoryUtils.getHistoryUrl(conf, TypeConverter
.toYarn(TypeConverter.fromYarn(jobId)).getAppId()));
report.setAMInfos(getAMInfos());
report.setIsUber(isUber());
} }
@Override @Override
@ -122,7 +107,7 @@ public class CompletedJob implements org.apache.hadoop.mapreduce.v2.app.job.Job
@Override @Override
public Counters getAllCounters() { public Counters getAllCounters() {
return counters; return jobInfo.getTotalCounters();
} }
@Override @Override
@ -131,10 +116,36 @@ public class CompletedJob implements org.apache.hadoop.mapreduce.v2.app.job.Job
} }
@Override @Override
public JobReport getReport() { public synchronized JobReport getReport() {
if (report == null) {
constructJobReport();
}
return report; return report;
} }
private void constructJobReport() {
report = Records.newRecord(JobReport.class);
report.setJobId(jobId);
report.setJobState(JobState.valueOf(jobInfo.getJobStatus()));
report.setSubmitTime(jobInfo.getSubmitTime());
report.setStartTime(jobInfo.getLaunchTime());
report.setFinishTime(jobInfo.getFinishTime());
report.setJobName(jobInfo.getJobname());
report.setUser(jobInfo.getUsername());
report.setMapProgress((float) getCompletedMaps() / getTotalMaps());
report.setReduceProgress((float) getCompletedReduces() / getTotalReduces());
report.setJobFile(confFile.toString());
String historyUrl = "N/A";
try {
historyUrl = JobHistoryUtils.getHistoryUrl(conf, jobId.getAppId());
} catch (UnknownHostException e) {
//Ignore.
}
report.setTrackingUrl(historyUrl);
report.setAMInfos(getAMInfos());
report.setIsUber(isUber());
}
@Override @Override
public float getProgress() { public float getProgress() {
return 1.0f; return 1.0f;
@ -142,16 +153,23 @@ public class CompletedJob implements org.apache.hadoop.mapreduce.v2.app.job.Job
@Override @Override
public JobState getState() { public JobState getState() {
return report.getJobState(); return JobState.valueOf(jobInfo.getJobStatus());
} }
@Override @Override
public Task getTask(TaskId taskId) { public Task getTask(TaskId taskId) {
if (tasksLoaded.get()) {
return tasks.get(taskId); return tasks.get(taskId);
} else {
TaskID oldTaskId = TypeConverter.fromYarn(taskId);
CompletedTask completedTask =
new CompletedTask(taskId, jobInfo.getAllTasks().get(oldTaskId));
return completedTask;
}
} }
@Override @Override
public TaskAttemptCompletionEvent[] getTaskAttemptCompletionEvents( public synchronized TaskAttemptCompletionEvent[] getTaskAttemptCompletionEvents(
int fromEventId, int maxEvents) { int fromEventId, int maxEvents) {
if (completionEvents == null) { if (completionEvents == null) {
constructTaskAttemptCompletionEvents(); constructTaskAttemptCompletionEvents();
@ -167,6 +185,7 @@ public class CompletedJob implements org.apache.hadoop.mapreduce.v2.app.job.Job
} }
private void constructTaskAttemptCompletionEvents() { private void constructTaskAttemptCompletionEvents() {
loadAllTasks();
completionEvents = new LinkedList<TaskAttemptCompletionEvent>(); completionEvents = new LinkedList<TaskAttemptCompletionEvent>();
List<TaskAttempt> allTaskAttempts = new LinkedList<TaskAttempt>(); List<TaskAttempt> allTaskAttempts = new LinkedList<TaskAttempt>();
for (TaskId taskId : tasks.keySet()) { for (TaskId taskId : tasks.keySet()) {
@ -205,8 +224,8 @@ public class CompletedJob implements org.apache.hadoop.mapreduce.v2.app.job.Job
int eventId = 0; int eventId = 0;
for (TaskAttempt taskAttempt : allTaskAttempts) { for (TaskAttempt taskAttempt : allTaskAttempts) {
TaskAttemptCompletionEvent tace = RecordFactoryProvider.getRecordFactory( TaskAttemptCompletionEvent tace =
null).newRecordInstance(TaskAttemptCompletionEvent.class); Records.newRecord(TaskAttemptCompletionEvent.class);
int attemptRunTime = -1; int attemptRunTime = -1;
if (taskAttempt.getLaunchTime() != 0 && taskAttempt.getFinishTime() != 0) { if (taskAttempt.getLaunchTime() != 0 && taskAttempt.getFinishTime() != 0) {
@ -237,15 +256,42 @@ public class CompletedJob implements org.apache.hadoop.mapreduce.v2.app.job.Job
@Override @Override
public Map<TaskId, Task> getTasks() { public Map<TaskId, Task> getTasks() {
loadAllTasks();
return tasks; return tasks;
} }
private void loadAllTasks() {
if (tasksLoaded.get()) {
return;
}
tasksLock.lock();
try {
if (tasksLoaded.get()) {
return;
}
for (Map.Entry<TaskID, TaskInfo> entry : jobInfo.getAllTasks().entrySet()) {
TaskId yarnTaskID = TypeConverter.toYarn(entry.getKey());
TaskInfo taskInfo = entry.getValue();
Task task = new CompletedTask(yarnTaskID, taskInfo);
tasks.put(yarnTaskID, task);
if (task.getType() == TaskType.MAP) {
mapTasks.put(task.getID(), task);
} else if (task.getType() == TaskType.REDUCE) {
reduceTasks.put(task.getID(), task);
}
}
tasksLoaded.set(true);
} finally {
tasksLock.unlock();
}
}
//History data is leisurely loaded when task level data is requested //History data is leisurely loaded when task level data is requested
private synchronized void loadFullHistoryData(boolean loadTasks, private synchronized void loadFullHistoryData(boolean loadTasks,
Path historyFileAbsolute) throws IOException { Path historyFileAbsolute) throws IOException {
LOG.info("Loading history file: [" + historyFileAbsolute + "]"); LOG.info("Loading history file: [" + historyFileAbsolute + "]");
if (jobInfo != null) { if (this.jobInfo != null) {
return; //data already loaded return;
} }
if (historyFileAbsolute != null) { if (historyFileAbsolute != null) {
@ -254,7 +300,7 @@ public class CompletedJob implements org.apache.hadoop.mapreduce.v2.app.job.Job
parser = parser =
new JobHistoryParser(historyFileAbsolute.getFileSystem(conf), new JobHistoryParser(historyFileAbsolute.getFileSystem(conf),
historyFileAbsolute); historyFileAbsolute);
jobInfo = parser.parse(); this.jobInfo = parser.parse();
} catch (IOException e) { } catch (IOException e) {
throw new YarnException("Could not load history file " throw new YarnException("Could not load history file "
+ historyFileAbsolute, e); + historyFileAbsolute, e);
@ -268,27 +314,15 @@ public class CompletedJob implements org.apache.hadoop.mapreduce.v2.app.job.Job
} else { } else {
throw new IOException("History file not found"); throw new IOException("History file not found");
} }
if (loadTasks) { if (loadTasks) {
for (Map.Entry<org.apache.hadoop.mapreduce.TaskID, TaskInfo> entry : jobInfo loadAllTasks();
.getAllTasks().entrySet()) {
TaskId yarnTaskID = TypeConverter.toYarn(entry.getKey());
TaskInfo taskInfo = entry.getValue();
Task task = new CompletedTask(yarnTaskID, taskInfo);
tasks.put(yarnTaskID, task);
if (task.getType() == TaskType.MAP) {
mapTasks.put(task.getID(), task);
} else if (task.getType() == TaskType.REDUCE) {
reduceTasks.put(task.getID(), task);
}
}
}
LOG.info("TaskInfo loaded"); LOG.info("TaskInfo loaded");
} }
}
@Override @Override
public List<String> getDiagnostics() { public List<String> getDiagnostics() {
return diagnostics; return Collections.singletonList(jobInfo.getErrorInfo());
} }
@Override @Override
@ -318,6 +352,7 @@ public class CompletedJob implements org.apache.hadoop.mapreduce.v2.app.job.Job
@Override @Override
public Map<TaskId, Task> getTasks(TaskType taskType) { public Map<TaskId, Task> getTasks(TaskType taskType) {
loadAllTasks();
if (TaskType.MAP.equals(taskType)) { if (TaskType.MAP.equals(taskType)) {
return mapTasks; return mapTasks;
} else {//we have only two types of tasks } else {//we have only two types of tasks

120
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedTask.java
View File

@ -20,10 +20,13 @@ package org.apache.hadoop.mapreduce.v2.hs;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.mapreduce.Counters; import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.TypeConverter; import org.apache.hadoop.mapreduce.TypeConverter;
import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskAttemptInfo; import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskAttemptInfo;
@ -35,59 +38,24 @@ import org.apache.hadoop.mapreduce.v2.api.records.TaskState;
import org.apache.hadoop.mapreduce.v2.api.records.TaskType; import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
import org.apache.hadoop.mapreduce.v2.app.job.Task; import org.apache.hadoop.mapreduce.v2.app.job.Task;
import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt; import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.util.Records;
public class CompletedTask implements Task { public class CompletedTask implements Task {
private final TaskType type;
private Counters counters;
private final long startTime;
private final long finishTime;
private TaskState state;
private final TaskId taskId; private final TaskId taskId;
private final TaskReport report; private final TaskInfo taskInfo;
private TaskReport report;
private TaskAttemptId successfulAttempt;
private List<String> reportDiagnostics = new LinkedList<String>();
private Lock taskAttemptsLock = new ReentrantLock();
private AtomicBoolean taskAttemptsLoaded = new AtomicBoolean(false);
private final Map<TaskAttemptId, TaskAttempt> attempts = private final Map<TaskAttemptId, TaskAttempt> attempts =
new LinkedHashMap<TaskAttemptId, TaskAttempt>(); new LinkedHashMap<TaskAttemptId, TaskAttempt>();
private static final Log LOG = LogFactory.getLog(CompletedTask.class);
CompletedTask(TaskId taskId, TaskInfo taskInfo) { CompletedTask(TaskId taskId, TaskInfo taskInfo) {
//TODO JobHistoryParser.handleTaskFailedAttempt should use state from the event. //TODO JobHistoryParser.handleTaskFailedAttempt should use state from the event.
LOG.debug("HandlingTaskId: [" + taskId + "]"); this.taskInfo = taskInfo;
this.taskId = taskId; this.taskId = taskId;
this.startTime = taskInfo.getStartTime();
this.finishTime = taskInfo.getFinishTime();
this.type = TypeConverter.toYarn(taskInfo.getTaskType());
if (taskInfo.getCounters() != null)
this.counters = taskInfo.getCounters();
if (taskInfo.getTaskStatus() != null) {
this.state = TaskState.valueOf(taskInfo.getTaskStatus());
} else {
this.state = TaskState.KILLED;
}
report = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(TaskReport.class);
for (TaskAttemptInfo attemptHistory : taskInfo.getAllTaskAttempts()
.values()) {
CompletedTaskAttempt attempt = new CompletedTaskAttempt(taskId,
attemptHistory);
report.addAllDiagnostics(attempt.getDiagnostics()); //TODO TMI?
attempts.put(attempt.getID(), attempt);
if (attemptHistory.getTaskStatus() != null
&& attemptHistory.getTaskStatus().equals(
TaskState.SUCCEEDED.toString())
&& report.getSuccessfulAttempt() == null) {
report.setSuccessfulAttempt(TypeConverter.toYarn(attemptHistory
.getAttemptId()));
}
}
report.setTaskId(taskId);
report.setStartTime(startTime);
report.setFinishTime(finishTime);
report.setTaskState(state);
report.setProgress(getProgress());
report.setCounters(TypeConverter.toYarn(getCounters()));
report.addAllRunningAttempts(new ArrayList<TaskAttemptId>(attempts.keySet()));
} }
@Override @Override
@ -97,17 +65,19 @@ public class CompletedTask implements Task {
@Override @Override
public TaskAttempt getAttempt(TaskAttemptId attemptID) { public TaskAttempt getAttempt(TaskAttemptId attemptID) {
loadAllTaskAttempts();
return attempts.get(attemptID); return attempts.get(attemptID);
} }
@Override @Override
public Map<TaskAttemptId, TaskAttempt> getAttempts() { public Map<TaskAttemptId, TaskAttempt> getAttempts() {
loadAllTaskAttempts();
return attempts; return attempts;
} }
@Override @Override
public Counters getCounters() { public Counters getCounters() {
return counters; return taskInfo.getCounters();
} }
@Override @Override
@ -121,13 +91,18 @@ public class CompletedTask implements Task {
} }
@Override @Override
public TaskReport getReport() { public synchronized TaskReport getReport() {
if (report == null) {
constructTaskReport();
}
return report; return report;
} }
@Override @Override
public TaskType getType() { public TaskType getType() {
return type; return TypeConverter.toYarn(taskInfo.getTaskType());
} }
@Override @Override
@ -137,7 +112,54 @@ public class CompletedTask implements Task {
@Override @Override
public TaskState getState() { public TaskState getState() {
return state; return taskInfo.getTaskStatus() == null ? TaskState.KILLED : TaskState
.valueOf(taskInfo.getTaskStatus());
} }
private void constructTaskReport() {
loadAllTaskAttempts();
this.report = Records.newRecord(TaskReport.class);
report.setTaskId(taskId);
report.setStartTime(taskInfo.getStartTime());
report.setFinishTime(taskInfo.getFinishTime());
report.setTaskState(getState());
report.setProgress(getProgress());
report.setCounters(TypeConverter.toYarn(getCounters()));
if (successfulAttempt != null) {
report.setSuccessfulAttempt(successfulAttempt);
}
report.addAllDiagnostics(reportDiagnostics);
report
.addAllRunningAttempts(new ArrayList<TaskAttemptId>(attempts.keySet()));
}
private void loadAllTaskAttempts() {
if (taskAttemptsLoaded.get()) {
return;
}
taskAttemptsLock.lock();
try {
if (taskAttemptsLoaded.get()) {
return;
}
for (TaskAttemptInfo attemptHistory : taskInfo.getAllTaskAttempts()
.values()) {
CompletedTaskAttempt attempt =
new CompletedTaskAttempt(taskId, attemptHistory);
reportDiagnostics.addAll(attempt.getDiagnostics());
attempts.put(attempt.getID(), attempt);
if (successfulAttempt == null
&& attemptHistory.getTaskStatus() != null
&& attemptHistory.getTaskStatus().equals(
TaskState.SUCCEEDED.toString())) {
successfulAttempt =
TypeConverter.toYarn(attemptHistory.getAttemptId());
}
}
taskAttemptsLoaded.set(true);
} finally {
taskAttemptsLock.unlock();
}
}
} }

82
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedTaskAttempt.java
View File

@ -30,25 +30,21 @@ import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptState;
import org.apache.hadoop.mapreduce.v2.api.records.TaskId; import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt; import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt;
import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.util.Records;
public class CompletedTaskAttempt implements TaskAttempt { public class CompletedTaskAttempt implements TaskAttempt {
private final TaskAttemptInfo attemptInfo; private final TaskAttemptInfo attemptInfo;
private final TaskAttemptId attemptId; private final TaskAttemptId attemptId;
private Counters counters;
private final TaskAttemptState state; private final TaskAttemptState state;
private final TaskAttemptReport report;
private final List<String> diagnostics = new ArrayList<String>(); private final List<String> diagnostics = new ArrayList<String>();
private TaskAttemptReport report;
private String localDiagMessage; private String localDiagMessage;
CompletedTaskAttempt(TaskId taskId, TaskAttemptInfo attemptInfo) { CompletedTaskAttempt(TaskId taskId, TaskAttemptInfo attemptInfo) {
this.attemptInfo = attemptInfo; this.attemptInfo = attemptInfo;
this.attemptId = TypeConverter.toYarn(attemptInfo.getAttemptId()); this.attemptId = TypeConverter.toYarn(attemptInfo.getAttemptId());
if (attemptInfo.getCounters() != null) {
this.counters = attemptInfo.getCounters();
}
if (attemptInfo.getTaskStatus() != null) { if (attemptInfo.getTaskStatus() != null) {
this.state = TaskAttemptState.valueOf(attemptInfo.getTaskStatus()); this.state = TaskAttemptState.valueOf(attemptInfo.getTaskStatus());
} else { } else {
@ -56,37 +52,9 @@ public class CompletedTaskAttempt implements TaskAttempt {
localDiagMessage = "Attmpt state missing from History : marked as KILLED"; localDiagMessage = "Attmpt state missing from History : marked as KILLED";
diagnostics.add(localDiagMessage); diagnostics.add(localDiagMessage);
} }
if (attemptInfo.getError() != null) { if (attemptInfo.getError() != null) {
diagnostics.add(attemptInfo.getError()); diagnostics.add(attemptInfo.getError());
} }
report = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(TaskAttemptReport.class);
report.setTaskAttemptId(attemptId);
report.setTaskAttemptState(state);
report.setProgress(getProgress());
report.setStartTime(attemptInfo.getStartTime());
report.setFinishTime(attemptInfo.getFinishTime());
report.setShuffleFinishTime(attemptInfo.getShuffleFinishTime());
report.setSortFinishTime(attemptInfo.getSortFinishTime());
if (localDiagMessage != null) {
report.setDiagnosticInfo(attemptInfo.getError() + ", " + localDiagMessage);
} else {
report.setDiagnosticInfo(attemptInfo.getError());
}
// report.setPhase(attemptInfo.get); //TODO
report.setStateString(attemptInfo.getState());
report.setCounters(TypeConverter.toYarn(getCounters()));
report.setContainerId(attemptInfo.getContainerId());
if (attemptInfo.getHostname() == null) {
report.setNodeManagerHost("UNKNOWN");
} else {
report.setNodeManagerHost(attemptInfo.getHostname());
report.setNodeManagerPort(attemptInfo.getPort());
}
report.setNodeManagerHttpPort(attemptInfo.getHttpPort());
} }
@Override @Override
@ -111,7 +79,7 @@ public class CompletedTaskAttempt implements TaskAttempt {
@Override @Override
public Counters getCounters() { public Counters getCounters() {
return counters; return attemptInfo.getCounters();
} }
@Override @Override
@ -125,7 +93,10 @@ public class CompletedTaskAttempt implements TaskAttempt {
} }
@Override @Override
public TaskAttemptReport getReport() { public synchronized TaskAttemptReport getReport() {
if (report == null) {
constructTaskAttemptReport();
}
return report; return report;
} }
@ -146,26 +117,55 @@ public class CompletedTaskAttempt implements TaskAttempt {
@Override @Override
public long getLaunchTime() { public long getLaunchTime() {
return report.getStartTime(); return attemptInfo.getStartTime();
} }
@Override @Override
public long getFinishTime() { public long getFinishTime() {
return report.getFinishTime(); return attemptInfo.getFinishTime();
} }
@Override @Override
public long getShuffleFinishTime() { public long getShuffleFinishTime() {
return report.getShuffleFinishTime(); return attemptInfo.getShuffleFinishTime();
} }
@Override @Override
public long getSortFinishTime() { public long getSortFinishTime() {
return report.getSortFinishTime(); return attemptInfo.getSortFinishTime();
} }
@Override @Override
public int getShufflePort() { public int getShufflePort() {
throw new UnsupportedOperationException("Not supported yet."); return attemptInfo.getShufflePort();
}
private void constructTaskAttemptReport() {
report = Records.newRecord(TaskAttemptReport.class);
report.setTaskAttemptId(attemptId);
report.setTaskAttemptState(state);
report.setProgress(getProgress());
report.setStartTime(attemptInfo.getStartTime());
report.setFinishTime(attemptInfo.getFinishTime());
report.setShuffleFinishTime(attemptInfo.getShuffleFinishTime());
report.setSortFinishTime(attemptInfo.getSortFinishTime());
if (localDiagMessage != null) {
report
.setDiagnosticInfo(attemptInfo.getError() + ", " + localDiagMessage);
} else {
report.setDiagnosticInfo(attemptInfo.getError());
}
// report.setPhase(attemptInfo.get); //TODO
report.setStateString(attemptInfo.getState());
report.setCounters(TypeConverter.toYarn(getCounters()));
report.setContainerId(attemptInfo.getContainerId());
if (attemptInfo.getHostname() == null) {
report.setNodeManagerHost("UNKNOWN");
} else {
report.setNodeManagerHost(attemptInfo.getHostname());
report.setNodeManagerPort(attemptInfo.getPort());
}
report.setNodeManagerHttpPort(attemptInfo.getHttpPort());
} }
} }

20
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory.java
View File

@ -24,6 +24,7 @@ import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
@ -118,8 +119,7 @@ public class JobHistory extends AbstractService implements HistoryContext {
//Maintains a list of known done subdirectories. Not currently used. //Maintains a list of known done subdirectories. Not currently used.
private final Set<Path> existingDoneSubdirs = new HashSet<Path>(); private final Set<Path> existingDoneSubdirs = new HashSet<Path>();
private final SortedMap<JobId, Job> loadedJobCache = private Map<JobId, Job> loadedJobCache = null;
new ConcurrentSkipListMap<JobId, Job>();
/** /**
* Maintains a mapping between intermediate user directories and the last * Maintains a mapping between intermediate user directories and the last
@ -167,6 +167,7 @@ public class JobHistory extends AbstractService implements HistoryContext {
* .....${DONE_DIR}/VERSION_STRING/YYYY/MM/DD/HH/SERIAL_NUM/jh{index_entries}.jhist * .....${DONE_DIR}/VERSION_STRING/YYYY/MM/DD/HH/SERIAL_NUM/jh{index_entries}.jhist
*/ */
@SuppressWarnings("serial")
@Override @Override
public void init(Configuration conf) throws YarnException { public void init(Configuration conf) throws YarnException {
LOG.info("JobHistory Init"); LOG.info("JobHistory Init");
@ -224,6 +225,16 @@ public class JobHistory extends AbstractService implements HistoryContext {
DEFAULT_MOVE_THREAD_INTERVAL); DEFAULT_MOVE_THREAD_INTERVAL);
numMoveThreads = conf.getInt(JHAdminConfig.MR_HISTORY_MOVE_THREAD_COUNT, numMoveThreads = conf.getInt(JHAdminConfig.MR_HISTORY_MOVE_THREAD_COUNT,
DEFAULT_MOVE_THREAD_COUNT); DEFAULT_MOVE_THREAD_COUNT);
loadedJobCache =
Collections.synchronizedMap(new LinkedHashMap<JobId, Job>(
loadedJobCacheSize + 1, 0.75f, true) {
@Override
public boolean removeEldestEntry(final Map.Entry<JobId, Job> eldest) {
return super.size() > loadedJobCacheSize;
}
});
try { try {
initExisting(); initExisting();
} catch (IOException e) { } catch (IOException e) {
@ -465,9 +476,6 @@ public class JobHistory extends AbstractService implements HistoryContext {
LOG.debug("Adding "+job.getID()+" to loaded job cache"); LOG.debug("Adding "+job.getID()+" to loaded job cache");
} }
loadedJobCache.put(job.getID(), job); loadedJobCache.put(job.getID(), job);
if (loadedJobCache.size() > loadedJobCacheSize ) {
loadedJobCache.remove(loadedJobCache.firstKey());
}
} }
@ -655,7 +663,7 @@ public class JobHistory extends AbstractService implements HistoryContext {
synchronized(metaInfo) { synchronized(metaInfo) {
try { try {
Job job = new CompletedJob(conf, metaInfo.getJobIndexInfo().getJobId(), Job job = new CompletedJob(conf, metaInfo.getJobIndexInfo().getJobId(),
metaInfo.getHistoryFile(), true, metaInfo.getJobIndexInfo().getUser(), metaInfo.getHistoryFile(), false, metaInfo.getJobIndexInfo().getUser(),
metaInfo.getConfFile(), this.aclsMgr); metaInfo.getConfFile(), this.aclsMgr);
addToLoadedJobCache(job); addToLoadedJobCache(job);
return job; return job;

145
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEntities.java Normal file
View File

@ -0,0 +1,145 @@
package org.apache.hadoop.mapreduce.v2.hs;
import static junit.framework.Assert.assertEquals;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobACLsManager;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
import org.apache.hadoop.mapreduce.v2.api.records.JobReport;
import org.apache.hadoop.mapreduce.v2.api.records.JobState;
import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptReport;
import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptState;
import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskReport;
import org.apache.hadoop.mapreduce.v2.api.records.TaskState;
import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
import org.apache.hadoop.mapreduce.v2.app.job.Task;
import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt;
import org.apache.hadoop.mapreduce.v2.util.MRBuilderUtils;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
@RunWith(value = Parameterized.class)
public class TestJobHistoryEntities {
private final String historyFileName =
"job_1329348432655_0001-1329348443227-user-Sleep+job-1329348468601-10-1-SUCCEEDED-default.jhist";
private final String confFileName = "job_1329348432655_0001_conf.xml";
private final Configuration conf = new Configuration();
private final JobACLsManager jobAclsManager = new JobACLsManager(conf);
private boolean loadTasks;
private JobId jobId = MRBuilderUtils.newJobId(1329348432655l, 1, 1);
Path fulleHistoryPath =
new Path(this.getClass().getClassLoader().getResource(historyFileName)
.getFile());
Path fullConfPath =
new Path(this.getClass().getClassLoader().getResource(confFileName)
.getFile());
private CompletedJob completedJob;
public TestJobHistoryEntities(boolean loadTasks) throws Exception {
this.loadTasks = loadTasks;
}
@Parameters
public static Collection<Object[]> data() {
List<Object[]> list = new ArrayList<Object[]>(2);
list.add(new Object[] { true });
list.add(new Object[] { false });
return list;
}
/* Verify some expected values based on the history file */
@Test
public void testCompletedJob() throws Exception {
//Re-initialize to verify the delayed load.
completedJob =
new CompletedJob(conf, jobId, fulleHistoryPath, loadTasks, "user",
fullConfPath, jobAclsManager);
//Verify tasks loaded based on loadTask parameter.
assertEquals(loadTasks, completedJob.tasksLoaded.get());
assertEquals(1, completedJob.getAMInfos().size());
assertEquals(10, completedJob.getCompletedMaps());
assertEquals(1, completedJob.getCompletedReduces());
assertEquals(11, completedJob.getTasks().size());
//Verify tasks loaded at this point.
assertEquals(true, completedJob.tasksLoaded.get());
assertEquals(10, completedJob.getTasks(TaskType.MAP).size());
assertEquals(1, completedJob.getTasks(TaskType.REDUCE).size());
assertEquals("user", completedJob.getUserName());
assertEquals(JobState.SUCCEEDED, completedJob.getState());
JobReport jobReport = completedJob.getReport();
assertEquals("user", jobReport.getUser());
assertEquals(JobState.SUCCEEDED, jobReport.getJobState());
}
@Test
public void testCompletedTask() throws Exception {
completedJob =
new CompletedJob(conf, jobId, fulleHistoryPath, loadTasks, "user",
fullConfPath, jobAclsManager);
TaskId mt1Id = MRBuilderUtils.newTaskId(jobId, 0, TaskType.MAP);
TaskId rt1Id = MRBuilderUtils.newTaskId(jobId, 0, TaskType.REDUCE);
Map<TaskId, Task> mapTasks = completedJob.getTasks(TaskType.MAP);
Map<TaskId, Task> reduceTasks = completedJob.getTasks(TaskType.REDUCE);
assertEquals(10, mapTasks.size());
assertEquals(1, reduceTasks.size());
Task mt1 = mapTasks.get(mt1Id);
assertEquals(1, mt1.getAttempts().size());
assertEquals(TaskState.SUCCEEDED, mt1.getState());
TaskReport mt1Report = mt1.getReport();
assertEquals(TaskState.SUCCEEDED, mt1Report.getTaskState());
assertEquals(mt1Id, mt1Report.getTaskId());
Task rt1 = reduceTasks.get(rt1Id);
assertEquals(1, rt1.getAttempts().size());
assertEquals(TaskState.SUCCEEDED, rt1.getState());
TaskReport rt1Report = rt1.getReport();
assertEquals(TaskState.SUCCEEDED, rt1Report.getTaskState());
assertEquals(rt1Id, rt1Report.getTaskId());
}
@Test
public void testCompletedTaskAttempt() throws Exception {
completedJob =
new CompletedJob(conf, jobId, fulleHistoryPath, loadTasks, "user",
fullConfPath, jobAclsManager);
TaskId mt1Id = MRBuilderUtils.newTaskId(jobId, 0, TaskType.MAP);
TaskId rt1Id = MRBuilderUtils.newTaskId(jobId, 0, TaskType.REDUCE);
TaskAttemptId mta1Id = MRBuilderUtils.newTaskAttemptId(mt1Id, 0);
TaskAttemptId rta1Id = MRBuilderUtils.newTaskAttemptId(rt1Id, 0);
Task mt1 = completedJob.getTask(mt1Id);
Task rt1 = completedJob.getTask(rt1Id);
TaskAttempt mta1 = mt1.getAttempt(mta1Id);
assertEquals(TaskAttemptState.SUCCEEDED, mta1.getState());
assertEquals("localhost:45454", mta1.getAssignedContainerMgrAddress());
assertEquals("localhost:9999", mta1.getNodeHttpAddress());
TaskAttemptReport mta1Report = mta1.getReport();
assertEquals(TaskAttemptState.SUCCEEDED, mta1Report.getTaskAttemptState());
assertEquals("localhost", mta1Report.getNodeManagerHost());
assertEquals(45454, mta1Report.getNodeManagerPort());
assertEquals(9999, mta1Report.getNodeManagerHttpPort());
TaskAttempt rta1 = rt1.getAttempt(rta1Id);
assertEquals(TaskAttemptState.SUCCEEDED, rta1.getState());
assertEquals("localhost:45454", rta1.getAssignedContainerMgrAddress());
assertEquals("localhost:9999", rta1.getNodeHttpAddress());
TaskAttemptReport rta1Report = rta1.getReport();
assertEquals(TaskAttemptState.SUCCEEDED, rta1Report.getTaskAttemptState());
assertEquals("localhost", rta1Report.getNodeManagerHost());
assertEquals(45454, rta1Report.getNodeManagerPort());
assertEquals(9999, rta1Report.getNodeManagerHttpPort());
}
}

51
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1329348432655_0001-1329348443227-user-Sleep+job-1329348468601-10-1-SUCCEEDED-default.jhist Normal file
View File

File diff suppressed because one or more lines are too long

397
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1329348432655_0001_conf.xml Normal file
View File

@ -0,0 +1,397 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?><configuration>
<property><!--Loaded from job.xml--><name>mapreduce.job.ubertask.enable</name><value>false</value></property>
<property><!--Loaded from job.xml--><name>yarn.resourcemanager.max-completed-applications</name><value>10000</value></property>
<property><!--Loaded from job.xml--><name>yarn.resourcemanager.delayed.delegation-token.removal-interval-ms</name><value>30000</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.client.submit.file.replication</name><value>10</value></property>
<property><!--Loaded from job.xml--><name>yarn.nodemanager.container-manager.thread-count</name><value>20</value></property>
<property><!--Loaded from job.xml--><name>mapred.queue.default.acl-administer-jobs</name><value>*</value></property>
<property><!--Loaded from job.xml--><name>dfs.image.transfer.bandwidthPerSec</name><value>0</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.tasktracker.healthchecker.interval</name><value>60000</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.jobtracker.staging.root.dir</name><value>${hadoop.tmp.dir}/mapred/staging</value></property>
<property><!--Loaded from job.xml--><name>dfs.block.access.token.lifetime</name><value>600</value></property>
<property><!--Loaded from job.xml--><name>yarn.resourcemanager.am.max-retries</name><value>2</value></property>
<property><!--Loaded from job.xml--><name>fs.AbstractFileSystem.file.impl</name><value>org.apache.hadoop.fs.local.LocalFs</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.client.completion.pollinterval</name><value>5000</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.job.ubertask.maxreduces</name><value>1</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.reduce.shuffle.memory.limit.percent</name><value>0.25</value></property>
<property><!--Loaded from job.xml--><name>hadoop.http.authentication.kerberos.keytab</name><value>${user.home}/hadoop.keytab</value></property>
<property><!--Loaded from job.xml--><name>yarn.nodemanager.keytab</name><value>/etc/krb5.keytab</value></property>
<property><!--Loaded from job.xml--><name>io.seqfile.sorter.recordlimit</name><value>1000000</value></property>
<property><!--Loaded from job.xml--><name>s3.blocksize</name><value>67108864</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.task.io.sort.factor</name><value>10</value></property>
<property><!--Loaded from job.xml--><name>yarn.nodemanager.disk-health-checker.interval-ms</name><value>120000</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.job.working.dir</name><value>hdfs://localhost:8021/user/user</value></property>
<property><!--Loaded from job.xml--><name>yarn.admin.acl</name><value>*</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.job.speculative.speculativecap</name><value>0.1</value></property>
<property><!--Loaded from job.xml--><name>dfs.namenode.num.checkpoints.retained</name><value>2</value></property>
<property><!--Loaded from job.xml--><name>dfs.namenode.delegation.token.renew-interval</name><value>86400000</value></property>
<property><!--Loaded from job.xml--><name>yarn.nodemanager.resource.memory-mb</name><value>8192</value></property>
<property><!--Loaded from job.xml--><name>io.map.index.interval</name><value>128</value></property>
<property><!--Loaded from job.xml--><name>s3.client-write-packet-size</name><value>65536</value></property>
<property><!--Loaded from job.xml--><name>dfs.namenode.http-address</name><value>0.0.0.0:50070</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.task.files.preserve.failedtasks</name><value>false</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.job.reduce.class</name><value>org.apache.hadoop.mapreduce.SleepJob$SleepReducer</value></property>
<property><!--Loaded from job.xml--><name>hadoop.hdfs.configuration.version</name><value>1</value></property>
<property><!--Loaded from job.xml--><name>s3.replication</name><value>3</value></property>
<property><!--Loaded from job.xml--><name>dfs.datanode.balance.bandwidthPerSec</name><value>1048576</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.reduce.shuffle.connect.timeout</name><value>180000</value></property>
<property><!--Loaded from job.xml--><name>yarn.nodemanager.aux-services</name><value>mapreduce.shuffle</value></property>
<property><!--Loaded from job.xml--><name>dfs.datanode.block.volume.choice.policy</name><value>org.apache.hadoop.hdfs.server.datanode.RoundRobinVolumesPolicy</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.job.complete.cancel.delegation.tokens</name><value>true</value></property>
<property><!--Loaded from job.xml--><name>yarn.server.nodemanager.connect.rm</name><value>true</value></property>
<property><!--Loaded from job.xml--><name>dfs.namenode.checkpoint.dir</name><value>file://${hadoop.tmp.dir}/dfs/namesecondary</value></property>
<property><!--Loaded from job.xml--><name>fs.trash.interval</name><value>0</value></property>
<property><!--Loaded from job.xml--><name>yarn.resourcemanager.admin.address</name><value>0.0.0.0:8141</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.job.outputformat.class</name><value>org.apache.hadoop.mapreduce.lib.output.NullOutputFormat</value></property>
<property><!--Loaded from job.xml--><name>yarn.log.server.url</name><value>http://localhost:19888/jobhistory/nmlogs</value></property>
<property><!--Loaded from job.xml--><name>hadoop.http.authentication.kerberos.principal</name><value>HTTP/localhost@LOCALHOST</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.tasktracker.taskmemorymanager.monitoringinterval</name><value>5000</value></property>
<property><!--Loaded from job.xml--><name>s3native.blocksize</name><value>67108864</value></property>
<property><!--Loaded from job.xml--><name>dfs.namenode.edits.dir</name><value>${dfs.namenode.name.dir}</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.job.map.class</name><value>org.apache.hadoop.mapreduce.SleepJob$SleepMapper</value></property>
<property><!--Loaded from job.xml--><name>dfs.datanode.http.address</name><value>0.0.0.0:50075</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.jobtracker.jobhistory.task.numberprogresssplits</name><value>12</value></property>
<property><!--Loaded from job.xml--><name>yarn.acl.enable</name><value>true</value></property>
<property><!--Loaded from job.xml--><name>yarn.nodemanager.localizer.fetch.thread-count</name><value>4</value></property>
<property><!--Loaded from job.xml--><name>hadoop.proxyuser.user.hosts</name><value>127.0.0.1</value></property>
<property><!--Loaded from job.xml--><name>hadoop.security.authorization</name><value>false</value></property>
<property><!--Loaded from job.xml--><name>dfs.namenode.safemode.extension</name><value>30000</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.reduce.log.level</name><value>INFO</value></property>
<property><!--Loaded from job.xml--><name>yarn.log-aggregation-enable</name><value>false</value></property>
<property><!--Loaded from job.xml--><name>dfs.https.server.keystore.resource</name><value>ssl-server.xml</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.jobtracker.instrumentation</name><value>org.apache.hadoop.mapred.JobTrackerMetricsInst</value></property>
<property><!--Loaded from job.xml--><name>dfs.namenode.replication.min</name><value>1</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.map.java.opts</name><value>-Xmx500m</value></property>
<property><!--Loaded from job.xml--><name>s3native.bytes-per-checksum</name><value>512</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.tasktracker.tasks.sleeptimebeforesigkill</name><value>5000</value></property>
<property><!--Loaded from job.xml--><name>tfile.fs.output.buffer.size</name><value>262144</value></property>
<property><!--Loaded from job.xml--><name>yarn.nodemanager.local-dirs</name><value>/home/user/local-dir/</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.jobtracker.persist.jobstatus.active</name><value>true</value></property>
<property><!--Loaded from job.xml--><name>fs.AbstractFileSystem.hdfs.impl</name><value>org.apache.hadoop.fs.Hdfs</value></property>
<property><!--Loaded from job.xml--><name>dfs.namenode.safemode.min.datanodes</name><value>0</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.tasktracker.local.dir.minspacestart</name><value>0</value></property>
<property><!--Loaded from job.xml--><name>dfs.client.https.need-auth</name><value>false</value></property>
<property><!--Loaded from job.xml--><name>fs.har.impl.disable.cache</name><value>true</value></property>
<property><!--Loaded from job.xml--><name>dfs.client.https.keystore.resource</name><value>ssl-client.xml</value></property>
<property><!--Loaded from job.xml--><name>dfs.namenode.max.objects</name><value>0</value></property>
<property><!--Loaded from job.xml--><name>dfs.namenode.safemode.threshold-pct</name><value>0.999f</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.tasktracker.local.dir.minspacekill</name><value>0</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.jobtracker.retiredjobs.cache.size</name><value>1000</value></property>
<property><!--Loaded from job.xml--><name>dfs.blocksize</name><value>67108864</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.job.reduce.slowstart.completedmaps</name><value>0.05</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.job.end-notification.retry.attempts</name><value>5</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.job.inputformat.class</name><value>org.apache.hadoop.mapreduce.SleepJob$SleepInputFormat</value></property>
<property><!--Loaded from job.xml--><name>fs.s3n.impl</name><value>org.apache.hadoop.fs.s3native.NativeS3FileSystem</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.map.memory.mb</name><value>512</value></property>
<property><!--Loaded from Unknown--><name>mapreduce.job.user.name</name><value>user</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.tasktracker.outofband.heartbeat</name><value>false</value></property>
<property><!--Loaded from job.xml--><name>io.native.lib.available</name><value>true</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.jobtracker.persist.jobstatus.hours</name><value>1</value></property>
<property><!--Loaded from job.xml--><name>dfs.client-write-packet-size</name><value>65536</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.client.progressmonitor.pollinterval</name><value>1000</value></property>
<property><!--Loaded from job.xml--><name>dfs.namenode.name.dir</name><value>file:///home/user/hadoop-user/dfs/name</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.output.fileoutputformat.compression.codec</name><value>org.apache.hadoop.io.compress.DefaultCodec</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.reduce.input.buffer.percent</name><value>0.0</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.map.output.compress.codec</name><value>org.apache.hadoop.io.compress.DefaultCodec</value></property>
<property><!--Loaded from job.xml--><name>yarn.resourcemanager.delegation-token.keepalive-time-ms</name><value>300000</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.map.skip.proc.count.autoincr</name><value>true</value></property>
<property><!--Loaded from job.xml--><name>dfs.datanode.directoryscan.threads</name><value>1</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.jobtracker.address</name><value>local</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.cluster.local.dir</name><value>${hadoop.tmp.dir}/mapred/local</value></property>
<property><!--Loaded from Unknown--><name>mapreduce.job.application.attempt.id</name><value>1</value></property>
<property><!--Loaded from job.xml--><name>dfs.permissions.enabled</name><value>true</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.tasktracker.taskcontroller</name><value>org.apache.hadoop.mapred.DefaultTaskController</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.reduce.shuffle.parallelcopies</name><value>5</value></property>
<property><!--Loaded from job.xml--><name>yarn.nodemanager.env-whitelist</name><value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,YARN_HOME</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.jobtracker.heartbeats.in.second</name><value>100</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.job.maxtaskfailures.per.tracker</name><value>4</value></property>
<property><!--Loaded from job.xml--><name>ipc.client.connection.maxidletime</name><value>10000</value></property>
<property><!--Loaded from job.xml--><name>dfs.blockreport.intervalMsec</name><value>21600000</value></property>
<property><!--Loaded from job.xml--><name>fs.s3.sleepTimeSeconds</name><value>10</value></property>
<property><!--Loaded from job.xml--><name>dfs.namenode.replication.considerLoad</name><value>true</value></property>
<property><!--Loaded from job.xml--><name>dfs.client.block.write.retries</name><value>3</value></property>
<property><!--Loaded from job.xml--><name>hadoop.proxyuser.user.groups</name><value>users</value></property>
<property><!--Loaded from job.xml--><name>dfs.namenode.name.dir.restore</name><value>false</value></property>
<property><!--Loaded from job.xml--><name>io.seqfile.lazydecompress</name><value>true</value></property>
<property><!--Loaded from job.xml--><name>dfs.https.enable</name><value>false</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.reduce.merge.inmem.threshold</name><value>1000</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.input.fileinputformat.split.minsize</name><value>0</value></property>
<property><!--Loaded from job.xml--><name>dfs.replication</name><value>3</value></property>
<property><!--Loaded from job.xml--><name>ipc.client.tcpnodelay</name><value>false</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.map.output.value.class</name><value>org.apache.hadoop.io.NullWritable</value></property>
<property><!--Loaded from job.xml--><name>dfs.namenode.accesstime.precision</name><value>3600000</value></property>
<property><!--Loaded from job.xml--><name>s3.stream-buffer-size</name><value>4096</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.jobtracker.tasktracker.maxblacklists</name><value>4</value></property>
<property><!--Loaded from Unknown--><name>rpc.engine.com.google.protobuf.BlockingService</name><value>org.apache.hadoop.yarn.ipc.ProtoOverHadoopRpcEngine</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.job.jvm.numtasks</name><value>1</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.task.io.sort.mb</name><value>100</value></property>
<property><!--Loaded from job.xml--><name>io.compression.codecs</name><value>org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.BZip2Codec,org.apache.hadoop.io.compress.DeflateCodec,org.apache.hadoop.io.compress.SnappyCodec,org.apache.hadoop.io.compress.Lz4Codec</value></property>
<property><!--Loaded from job.xml--><name>io.file.buffer.size</name><value>4096</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.job.jar</name><value>/tmp/hadoop-yarn/staging/user/.staging/job_1329348432655_0001/job.jar</value></property>
<property><!--Loaded from job.xml--><name>dfs.namenode.checkpoint.txns</name><value>40000</value></property>
<property><!--Loaded from job.xml--><name>yarn.nodemanager.admin-env</name><value>MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.job.split.metainfo.maxsize</name><value>10000000</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.output.fileoutputformat.compression.type</name><value>RECORD</value></property>
<property><!--Loaded from job.xml--><name>kfs.replication</name><value>3</value></property>
<property><!--Loaded from job.xml--><name>yarn.app.mapreduce.am.scheduler.heartbeat.interval-ms</name><value>1000</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.reduce.maxattempts</name><value>4</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.sleepjob.map.sleep.time</name><value>1</value></property>
<property><!--Loaded from job.xml--><name>kfs.stream-buffer-size</name><value>4096</value></property>
<property><!--Loaded from job.xml--><name>fs.har.impl</name><value>org.apache.hadoop.fs.HarFileSystem</value></property>
<property><!--Loaded from job.xml--><name>hadoop.security.authentication</name><value>simple</value></property>
<property><!--Loaded from job.xml--><name>fs.s3.buffer.dir</name><value>${hadoop.tmp.dir}/s3</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.jobtracker.taskscheduler</name><value>org.apache.hadoop.mapred.JobQueueTaskScheduler</value></property>
<property><!--Loaded from job.xml--><name>yarn.app.mapreduce.am.job.task.listener.thread-count</name><value>30</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.job.reduces</name><value>1</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.map.sort.spill.percent</name><value>0.80</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.job.end-notification.retry.interval</name><value>1</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.job.maps</name><value>10</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.job.speculative.slownodethreshold</name><value>1.0</value></property>
<property><!--Loaded from job.xml--><name>dfs.block.access.token.enable</name><value>false</value></property>
<property><!--Loaded from job.xml--><name>tfile.fs.input.buffer.size</name><value>262144</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.map.speculative</name><value>false</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.job.acl-view-job</name><value> </value></property>
<property><!--Loaded from job.xml--><name>mapreduce.map.output.key.class</name><value>org.apache.hadoop.io.IntWritable</value></property>
<property><!--Loaded from job.xml--><name>yarn.ipc.serializer.type</name><value>protocolbuffers</value></property>
<property><!--Loaded from mapred-default.xml--><name>mapreduce.job.end-notification.max.retry.interval</name><value>5</value></property>
<property><!--Loaded from job.xml--><name>ftp.blocksize</name><value>67108864</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.tasktracker.http.threads</name><value>40</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.reduce.java.opts</name><value>-Xmx500m</value></property>
<property><!--Loaded from job.xml--><name>dfs.datanode.data.dir</name><value>file:///home/user/hadoop-user/dfs/data</value></property>
<property><!--Loaded from job.xml--><name>dfs.namenode.replication.interval</name><value>3</value></property>
<property><!--Loaded from job.xml--><name>fs.file.impl</name><value>org.apache.hadoop.fs.LocalFileSystem</value></property>
<property><!--Loaded from job.xml--><name>dfs.namenode.https-address</name><value>0.0.0.0:50470</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.task.skip.start.attempts</name><value>2</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.jobtracker.persist.jobstatus.dir</name><value>/jobtracker/jobsInfo</value></property>
<property><!--Loaded from job.xml--><name>ipc.client.kill.max</name><value>10</value></property>
<property><!--Loaded from mapred-default.xml--><name>mapreduce.job.end-notification.max.attempts</name><value>5</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.jobhistory.max-age-ms</name><value>10000000000</value></property>
<property><!--Loaded from job.xml--><name>yarn.resourcemanager.zookeeper-store.session.timeout-ms</name><value>60000</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.task.tmp.dir</name><value>./tmp</value></property>
<property><!--Loaded from job.xml--><name>dfs.default.chunk.view.size</name><value>32768</value></property>
<property><!--Loaded from job.xml--><name>kfs.bytes-per-checksum</name><value>512</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.reduce.memory.mb</name><value>512</value></property>
<property><!--Loaded from job.xml--><name>hadoop.http.filter.initializers</name><value>org.apache.hadoop.yarn.server.webproxy.amfilter.AmFilterInitializer</value></property>
<property><!--Loaded from job.xml--><name>dfs.datanode.failed.volumes.tolerated</name><value>0</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.sleepjob.reduce.sleep.count</name><value>1</value></property>
<property><!--Loaded from job.xml--><name>hadoop.http.authentication.type</name><value>simple</value></property>
<property><!--Loaded from job.xml--><name>dfs.datanode.data.dir.perm</name><value>700</value></property>
<property><!--Loaded from job.xml--><name>yarn.resourcemanager.client.thread-count</name><value>50</value></property>
<property><!--Loaded from job.xml--><name>ipc.server.listen.queue.size</name><value>128</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.reduce.skip.maxgroups</name><value>0</value></property>
<property><!--Loaded from job.xml--><name>file.stream-buffer-size</name><value>4096</value></property>
<property><!--Loaded from job.xml--><name>dfs.namenode.fs-limits.max-directory-items</name><value>0</value></property>
<property><!--Loaded from job.xml--><name>io.mapfile.bloom.size</name><value>1048576</value></property>
<property><!--Loaded from job.xml--><name>fs.hsftp.impl</name><value>org.apache.hadoop.hdfs.HsftpFileSystem</value></property>
<property><!--Loaded from job.xml--><name>yarn.nodemanager.container-executor.class</name><value>org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.map.maxattempts</name><value>4</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.jobtracker.jobhistory.block.size</name><value>3145728</value></property>
<property><!--Loaded from job.xml--><name>ftp.replication</name><value>3</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.jobtracker.http.address</name><value>0.0.0.0:50030</value></property>
<property><!--Loaded from job.xml--><name>yarn.nodemanager.health-checker.script.timeout-ms</name><value>1200000</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.jobhistory.address</name><value>0.0.0.0:10020</value></property>
<property><!--Loaded from job.xml--><name>dfs.datanode.dns.nameserver</name><value>default</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.jobtracker.taskcache.levels</name><value>2</value></property>
<property><!--Loaded from job.xml--><name>yarn.nodemanager.log.retain-seconds</name><value>12000</value></property>
<property><!--Loaded from job.xml--><name>mapred.child.java.opts</name><value>-Xmx200m</value></property>
<property><!--Loaded from job.xml--><name>dfs.replication.max</name><value>512</value></property>
<property><!--Loaded from job.xml--><name>map.sort.class</name><value>org.apache.hadoop.util.QuickSort</value></property>
<property><!--Loaded from job.xml--><name>dfs.stream-buffer-size</name><value>4096</value></property>
<property><!--Loaded from job.xml--><name>dfs.namenode.backup.address</name><value>0.0.0.0:50100</value></property>
<property><!--Loaded from job.xml--><name>hadoop.util.hash.type</name><value>murmur</value></property>
<property><!--Loaded from job.xml--><name>dfs.block.access.key.update.interval</name><value>600</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.jobhistory.move.interval-ms</name><value>30000</value></property>
<property><!--Loaded from job.xml--><name>dfs.datanode.dns.interface</name><value>default</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.reduce.skip.proc.count.autoincr</name><value>true</value></property>
<property><!--Loaded from job.xml--><name>dfs.namenode.backup.http-address</name><value>0.0.0.0:50105</value></property>
<property><!--Loaded from job.xml--><name>yarn.nodemanager.container-monitor.interval-ms</name><value>3000</value></property>
<property><!--Loaded from job.xml--><name>mapred.reducer.new-api</name><value>true</value></property>
<property><!--Loaded from job.xml--><name>yarn.nodemanager.disk-health-checker.min-healthy-disks</name><value>0.25</value></property>
<property><!--Loaded from job.xml--><name>kfs.client-write-packet-size</name><value>65536</value></property>
<property><!--Loaded from job.xml--><name>yarn.nodemanager.sleep-delay-before-sigkill.ms</name><value>250</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.job.dir</name><value>/tmp/hadoop-yarn/staging/user/.staging/job_1329348432655_0001</value></property>
<property><!--Loaded from job.xml--><name>io.map.index.skip</name><value>0</value></property>
<property><!--Loaded from job.xml--><name>net.topology.node.switch.mapping.impl</name><value>org.apache.hadoop.net.ScriptBasedMapping</value></property>
<property><!--Loaded from job.xml--><name>dfs.namenode.logging.level</name><value>info</value></property>
<property><!--Loaded from job.xml--><name>fs.s3.maxRetries</name><value>4</value></property>
<property><!--Loaded from job.xml--><name>s3native.client-write-packet-size</name><value>65536</value></property>
<property><!--Loaded from job.xml--><name>yarn.resourcemanager.amliveliness-monitor.interval-ms</name><value>1000</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.reduce.speculative</name><value>false</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.client.output.filter</name><value>FAILED</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.tasktracker.report.address</name><value>127.0.0.1:0</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.task.userlog.limit.kb</name><value>0</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.tasktracker.map.tasks.maximum</name><value>2</value></property>
<property><!--Loaded from job.xml--><name>hadoop.http.authentication.simple.anonymous.allowed</name><value>true</value></property>
<property><!--Loaded from job.xml--><name>hadoop.rpc.socket.factory.class.default</name><value>org.apache.hadoop.net.StandardSocketFactory</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.job.submithostname</name><value>localhost</value></property>
<property><!--Loaded from job.xml--><name>fs.hftp.impl</name><value>org.apache.hadoop.hdfs.HftpFileSystem</value></property>
<property><!--Loaded from job.xml--><name>dfs.namenode.handler.count</name><value>10</value></property>
<property><!--Loaded from job.xml--><name>fs.automatic.close</name><value>true</value></property>
<property><!--Loaded from job.xml--><name>fs.kfs.impl</name><value>org.apache.hadoop.fs.kfs.KosmosFileSystem</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.job.submithostaddress</name><value>127.0.0.1</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.tasktracker.healthchecker.script.timeout</name><value>600000</value></property>
<property><!--Loaded from job.xml--><name>dfs.datanode.directoryscan.interval</name><value>21600</value></property>
<property><!--Loaded from job.xml--><name>yarn.resourcemanager.address</name><value>0.0.0.0:8040</value></property>
<property><!--Loaded from job.xml--><name>yarn.nodemanager.log-aggregation-enable</name><value>false</value></property>
<property><!--Loaded from job.xml--><name>fs.hdfs.impl</name><value>org.apache.hadoop.hdfs.DistributedFileSystem</value></property>
<property><!--Loaded from job.xml--><name>yarn.nodemanager.health-checker.interval-ms</name><value>600000</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.reduce.markreset.buffer.percent</name><value>0.0</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.map.log.level</name><value>INFO</value></property>
<property><!--Loaded from job.xml--><name>yarn.nodemanager.localizer.address</name><value>0.0.0.0:4344</value></property>
<property><!--Loaded from job.xml--><name>dfs.bytes-per-checksum</name><value>512</value></property>
<property><!--Loaded from job.xml--><name>ftp.stream-buffer-size</name><value>4096</value></property>
<property><!--Loaded from job.xml--><name>yarn.resourcemanager.keytab</name><value>/etc/krb5.keytab</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.sleepjob.map.sleep.count</name><value>1</value></property>
<property><!--Loaded from job.xml--><name>dfs.blockreport.initialDelay</name><value>0</value></property>
<property><!--Loaded from job.xml--><name>yarn.nm.liveness-monitor.expiry-interval-ms</name><value>600000</value></property>
<property><!--Loaded from job.xml--><name>hadoop.http.authentication.token.validity</name><value>36000</value></property>
<property><!--Loaded from job.xml--><name>dfs.namenode.delegation.token.max-lifetime</name><value>604800000</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.job.hdfs-servers</name><value>${fs.default.name}</value></property>
<property><!--Loaded from job.xml--><name>fs.ftp.impl</name><value>org.apache.hadoop.fs.ftp.FTPFileSystem</value></property>
<property><!--Loaded from job.xml--><name>dfs.web.ugi</name><value>webuser,webgroup</value></property>
<property><!--Loaded from job.xml--><name>s3native.replication</name><value>3</value></property>
<property><!--Loaded from job.xml--><name>dfs.heartbeat.interval</name><value>3</value></property>
<property><!--Loaded from job.xml--><name>yarn.nodemanager.localizer.client.thread-count</name><value>5</value></property>
<property><!--Loaded from job.xml--><name>yarn.resourcemanager.container.liveness-monitor.interval-ms</name><value>600000</value></property>
<property><!--Loaded from job.xml--><name>yarn.am.liveness-monitor.expiry-interval-ms</name><value>600000</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.task.profile</name><value>false</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.tasktracker.instrumentation</name><value>org.apache.hadoop.mapred.TaskTrackerMetricsInst</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.tasktracker.http.address</name><value>0.0.0.0:50060</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.jobhistory.webapp.address</name><value>0.0.0.0:19888</value></property>
<property><!--Loaded from Unknown--><name>rpc.engine.org.apache.hadoop.yarn.proto.AMRMProtocol$AMRMProtocolService$BlockingInterface</name><value>org.apache.hadoop.yarn.ipc.ProtoOverHadoopRpcEngine</value></property>
<property><!--Loaded from job.xml--><name>yarn.ipc.rpc.class</name><value>org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.job.name</name><value>Sleep job</value></property>
<property><!--Loaded from job.xml--><name>kfs.blocksize</name><value>67108864</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.job.ubertask.maxmaps</name><value>9</value></property>
<property><!--Loaded from job.xml--><name>yarn.nodemanager.heartbeat.interval-ms</name><value>1000</value></property>
<property><!--Loaded from job.xml--><name>dfs.namenode.secondary.http-address</name><value>0.0.0.0:50090</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.job.userlog.retain.hours</name><value>24</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.task.timeout</name><value>600000</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.jobhistory.loadedjobs.cache.size</name><value>1</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.framework.name</name><value>yarn</value></property>
<property><!--Loaded from job.xml--><name>ipc.client.idlethreshold</name><value>4000</value></property>
<property><!--Loaded from job.xml--><name>ipc.server.tcpnodelay</name><value>false</value></property>
<property><!--Loaded from job.xml--><name>ftp.bytes-per-checksum</name><value>512</value></property>
<property><!--Loaded from job.xml--><name>s3.bytes-per-checksum</name><value>512</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.job.speculative.slowtaskthreshold</name><value>1.0</value></property>
<property><!--Loaded from job.xml--><name>yarn.nodemanager.localizer.cache.target-size-mb</name><value>1</value></property>
<property><!--Loaded from job.xml--><name>yarn.nodemanager.remote-app-log-dir</name><value>/tmp/logs</value></property>
<property><!--Loaded from job.xml--><name>fs.s3.block.size</name><value>67108864</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.job.queuename</name><value>default</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.sleepjob.reduce.sleep.time</name><value>1</value></property>
<property><!--Loaded from job.xml--><name>hadoop.rpc.protection</name><value>authentication</value></property>
<property><!--Loaded from job.xml--><name>yarn.app.mapreduce.client-am.ipc.max-retries</name><value>1</value></property>
<property><!--Loaded from job.xml--><name>ftp.client-write-packet-size</name><value>65536</value></property>
<property><!--Loaded from job.xml--><name>yarn.nodemanager.address</name><value>0.0.0.0:45454</value></property>
<property><!--Loaded from job.xml--><name>fs.defaultFS</name><value>hdfs://localhost:8021</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.task.merge.progress.records</name><value>10000</value></property>
<property><!--Loaded from job.xml--><name>yarn.resourcemanager.scheduler.client.thread-count</name><value>50</value></property>
<property><!--Loaded from job.xml--><name>file.client-write-packet-size</name><value>65536</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.job.partitioner.class</name><value>org.apache.hadoop.mapreduce.SleepJob$SleepJobPartitioner</value></property>
<property><!--Loaded from job.xml--><name>yarn.nodemanager.delete.thread-count</name><value>4</value></property>
<property><!--Loaded from job.xml--><name>yarn.resourcemanager.scheduler.address</name><value>0.0.0.0:8030</value></property>
<property><!--Loaded from job.xml--><name>fs.trash.checkpoint.interval</name><value>0</value></property>
<property><!--Loaded from job.xml--><name>s3native.stream-buffer-size</name><value>4096</value></property>
<property><!--Loaded from job.xml--><name>yarn.scheduler.fifo.minimum-allocation-mb</name><value>1024</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.reduce.shuffle.read.timeout</name><value>180000</value></property>
<property><!--Loaded from job.xml--><name>yarn.app.mapreduce.am.command-opts</name><value>-Xmx500m</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.admin.user.env</name><value>LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native</value></property>
<property><!--Loaded from job.xml--><name>dfs.namenode.checkpoint.edits.dir</name><value>${dfs.namenode.checkpoint.dir}</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.local.clientfactory.class.name</name><value>org.apache.hadoop.mapred.LocalClientFactory</value></property>
<property><!--Loaded from job.xml--><name>hadoop.common.configuration.version</name><value>0.23.0</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.tasktracker.dns.interface</name><value>default</value></property>
<property><!--Loaded from job.xml--><name>io.serializations</name><value>org.apache.hadoop.io.serializer.WritableSerialization,org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization,org.apache.hadoop.io.serializer.avro.AvroReflectSerialization</value></property>
<property><!--Loaded from job.xml--><name>yarn.nodemanager.aux-service.mapreduce.shuffle.class</name><value>org.apache.hadoop.mapred.ShuffleHandler</value></property>
<property><!--Loaded from job.xml--><name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name><value>org.apache.hadoop.mapred.ShuffleHandler</value></property>
<property><!--Loaded from job.xml--><name>fs.df.interval</name><value>60000</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.reduce.shuffle.input.buffer.percent</name><value>0.70</value></property>
<property><!--Loaded from job.xml--><name>io.seqfile.compress.blocksize</name><value>1000000</value></property>
<property><!--Loaded from job.xml--><name>ipc.client.connect.max.retries</name><value>10</value></property>
<property><!--Loaded from job.xml--><name>fs.viewfs.impl</name><value>org.apache.hadoop.fs.viewfs.ViewFileSystem</value></property>
<property><!--Loaded from job.xml--><name>hadoop.security.groups.cache.secs</name><value>300</value></property>
<property><!--Loaded from job.xml--><name>dfs.namenode.delegation.key.update-interval</name><value>86400000</value></property>
<property><!--Loaded from job.xml--><name>yarn.nodemanager.process-kill-wait.ms</name><value>2000</value></property>
<property><!--Loaded from job.xml--><name>yarn.application.classpath</name><value>
$HADOOP_CONF_DIR,
$HADOOP_COMMON_HOME/share/hadoop/common/*,
$HADOOP_COMMON_HOME/share/hadoop/common/lib/*,
$HADOOP_HDFS_HOME/share/hadoop/hdfs/*,
$HADOOP_HDFS_HOME/share/hadoop/hdfs/lib/*,
$YARN_HOME/share/hadoop/mapreduce/*,
$YARN_HOME/share/hadoop/mapreduce/lib/*
</value></property>
<property><!--Loaded from job.xml--><name>yarn.nodemanager.log-aggregation.compression-type</name><value>gz</value></property>
<property><!--Loaded from job.xml--><name>dfs.image.compress</name><value>false</value></property>
<property><!--Loaded from job.xml--><name>yarn.nodemanager.localizer.cache.cleanup.interval-ms</name><value>30000</value></property>
<property><!--Loaded from job.xml--><name>mapred.mapper.new-api</name><value>true</value></property>
<property><!--Loaded from job.xml--><name>yarn.nodemanager.log-dirs</name><value>/home/user/logs</value></property>
<property><!--Loaded from job.xml--><name>fs.s3n.block.size</name><value>67108864</value></property>
<property><!--Loaded from job.xml--><name>fs.ftp.host</name><value>0.0.0.0</value></property>
<property><!--Loaded from job.xml--><name>hadoop.security.group.mapping</name><value>org.apache.hadoop.security.ShellBasedUnixGroupsMapping</value></property>
<property><!--Loaded from job.xml--><name>dfs.datanode.address</name><value>0.0.0.0:50010</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.map.skip.maxrecords</name><value>0</value></property>
<property><!--Loaded from job.xml--><name>dfs.datanode.https.address</name><value>0.0.0.0:50475</value></property>
<property><!--Loaded from job.xml--><name>fs.s3.impl</name><value>org.apache.hadoop.fs.s3.S3FileSystem</value></property>
<property><!--Loaded from job.xml--><name>file.replication</name><value>1</value></property>
<property><!--Loaded from job.xml--><name>yarn.resourcemanager.resource-tracker.address</name><value>0.0.0.0:8025</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.jobtracker.restart.recover</name><value>false</value></property>
<property><!--Loaded from job.xml--><name>hadoop.work.around.non.threadsafe.getpwuid</name><value>false</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.client.genericoptionsparser.used</name><value>true</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.tasktracker.indexcache.mb</name><value>10</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.output.fileoutputformat.compress</name><value>false</value></property>
<property><!--Loaded from job.xml--><name>hadoop.tmp.dir</name><value>/tmp/hadoop-${user.name}</value></property>
<property><!--Loaded from job.xml--><name>dfs.client.block.write.replace-datanode-on-failure.policy</name><value>DEFAULT</value></property>
<property><!--Loaded from job.xml--><name>hadoop.kerberos.kinit.command</name><value>kinit</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.job.committer.setup.cleanup.needed</name><value>true</value></property>
<property><!--Loaded from job.xml--><name>dfs.datanode.du.reserved</name><value>0</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.task.profile.reduces</name><value>0-2</value></property>
<property><!--Loaded from job.xml--><name>file.bytes-per-checksum</name><value>512</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.input.fileinputformat.inputdir</name><value>hdfs://localhost:8021/user/user/ignored</value></property>
<property><!--Loaded from job.xml--><name>dfs.client.block.write.replace-datanode-on-failure.enable</name><value>ture</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.jobtracker.handler.count</name><value>10</value></property>
<property><!--Loaded from job.xml--><name>net.topology.script.number.args</name><value>100</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.task.profile.maps</name><value>0-2</value></property>
<property><!--Loaded from job.xml--><name>dfs.namenode.decommission.interval</name><value>30</value></property>
<property><!--Loaded from job.xml--><name>dfs.image.compression.codec</name><value>org.apache.hadoop.io.compress.DefaultCodec</value></property>
<property><!--Loaded from job.xml--><name>yarn.resourcemanager.webapp.address</name><value>0.0.0.0:8088</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.jobtracker.system.dir</name><value>${hadoop.tmp.dir}/mapred/system</value></property>
<property><!--Loaded from job.xml--><name>dfs.namenode.support.allow.format</name><value>true</value></property>
<property><!--Loaded from job.xml--><name>yarn.nodemanager.vmem-pmem-ratio</name><value>2.1</value></property>
<property><!--Loaded from job.xml--><name>io.mapfile.bloom.error.rate</name><value>0.005</value></property>
<property><!--Loaded from job.xml--><name>dfs.permissions.superusergroup</name><value>supergroup</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.jobtracker.expire.trackers.interval</name><value>600000</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.cluster.acls.enabled</name><value>false</value></property>
<property><!--Loaded from job.xml--><name>yarn.nodemanager.remote-app-log-dir-suffix</name><value>logs</value></property>
<property><!--Loaded from job.xml--><name>dfs.namenode.checkpoint.check.period</name><value>60</value></property>
<property><!--Loaded from job.xml--><name>io.seqfile.local.dir</name><value>${hadoop.tmp.dir}/io/local</value></property>
<property><!--Loaded from job.xml--><name>yarn.app.mapreduce.am.resource.mb</name><value>512</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.reduce.shuffle.merge.percent</name><value>0.66</value></property>
<property><!--Loaded from job.xml--><name>tfile.io.chunk.size</name><value>1048576</value></property>
<property><!--Loaded from job.xml--><name>file.blocksize</name><value>67108864</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.jobtracker.jobhistory.lru.cache.size</name><value>5</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.jobtracker.maxtasks.perjob</name><value>-1</value></property>
<property><!--Loaded from job.xml--><name>yarn.resourcemanager.nm.liveness-monitor.interval-ms</name><value>1000</value></property>
<property><!--Loaded from job.xml--><name>yarn.nodemanager.webapp.address</name><value>0.0.0.0:9999</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.job.acl-modify-job</name><value> </value></property>
<property><!--Loaded from job.xml--><name>mapreduce.tasktracker.reduce.tasks.maximum</name><value>2</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.cluster.temp.dir</name><value>${hadoop.tmp.dir}/mapred/temp</value></property>
<property><!--Loaded from job.xml--><name>io.skip.checksum.errors</name><value>false</value></property>
<property><!--Loaded from job.xml--><name>yarn.app.mapreduce.am.staging-dir</name><value>/tmp/hadoop-yarn/staging</value></property>
<property><!--Loaded from job.xml--><name>dfs.datanode.handler.count</name><value>3</value></property>
<property><!--Loaded from job.xml--><name>hadoop.http.authentication.signature.secret</name><value>hadoop</value></property>
<property><!--Loaded from job.xml--><name>dfs.namenode.decommission.nodes.per.interval</name><value>5</value></property>
<property><!--Loaded from job.xml--><name>fs.ftp.host.port</name><value>21</value></property>
<property><!--Loaded from job.xml--><name>dfs.namenode.checkpoint.period</name><value>3600</value></property>
<property><!--Loaded from job.xml--><name>dfs.namenode.fs-limits.max-component-length</name><value>0</value></property>
<property><!--Loaded from job.xml--><name>yarn.resourcemanager.admin.client.thread-count</name><value>1</value></property>
<property><!--Loaded from job.xml--><name>fs.AbstractFileSystem.viewfs.impl</name><value>org.apache.hadoop.fs.viewfs.ViewFs</value></property>
<property><!--Loaded from job.xml--><name>yarn.resourcemanager.resource-tracker.client.thread-count</name><value>50</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.tasktracker.dns.nameserver</name><value>default</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.clientfactory.class.name</name><value>org.apache.hadoop.mapred.YarnClientFactory</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.map.output.compress</name><value>false</value></property>
<property><!--Loaded from job.xml--><name>mapreduce.job.counters.limit</name><value>120</value></property>
<property><!--Loaded from job.xml--><name>dfs.datanode.ipc.address</name><value>0.0.0.0:50020</value></property>
<property><!--Loaded from job.xml--><name>fs.webhdfs.impl</name><value>org.apache.hadoop.hdfs.web.WebHdfsFileSystem</value></property>
<property><!--Loaded from job.xml--><name>yarn.nodemanager.delete.debug-delay-sec</name><value>0</value></property>
<property><!--Loaded from job.xml--><name>dfs.datanode.max.transfer.threads</name><value>4096</value></property>
</configuration>