From 2cc851a66e86b82ed6f9fc3b86c2df3001519c51 Mon Sep 17 00:00:00 2001 From: Sanford Ryza Date: Thu, 29 Aug 2013 23:45:16 +0000 Subject: [PATCH] MAPREDUCE-5484. YarnChild unnecessarily loads job conf twice (Sandy Ryza) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1518857 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 2 ++ .../org/apache/hadoop/mapred/YarnChild.java | 18 ++++++++---------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 8ba4b35d3af..1cc5e1b333c 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -162,6 +162,8 @@ Release 2.3.0 - UNRELEASED OPTIMIZATIONS + MAPREDUCE-5484. YarnChild unnecessarily loads job conf twice (Sandy Ryza) + BUG FIXES MAPREDUCE-5316. job -list-attempt-ids command does not handle illegal diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java index e4d0041d211..31007097129 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java @@ -75,9 +75,9 @@ public static void main(String[] args) throws Throwable { Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler()); LOG.debug("Child starting"); - final JobConf defaultConf = new JobConf(); - defaultConf.addResource(MRJobConfig.JOB_CONF_FILE); - UserGroupInformation.setConfiguration(defaultConf); + final JobConf job = new JobConf(); + job.addResource(MRJobConfig.JOB_CONF_FILE); + UserGroupInformation.setConfiguration(job); String host = args[0]; int port = Integer.parseInt(args[1]); @@ -111,7 +111,7 @@ public static void main(String[] args) throws Throwable { @Override public TaskUmbilicalProtocol run() throws Exception { return (TaskUmbilicalProtocol)RPC.getProxy(TaskUmbilicalProtocol.class, - TaskUmbilicalProtocol.versionID, address, defaultConf); + TaskUmbilicalProtocol.versionID, address, job); } }); @@ -140,7 +140,7 @@ public TaskUmbilicalProtocol run() throws Exception { YarnChild.taskid = task.getTaskID(); // Create the job-conf and set credentials - final JobConf job = configureTask(task, credentials, jt); + configureTask(job, task, credentials, jt); // log the system properties String systemPropsToLog = MRApps.getSystemPropertiesToLog(job); @@ -260,11 +260,10 @@ private static void configureLocalDirs(Task task, JobConf job) throws IOExceptio job.set(MRJobConfig.JOB_LOCAL_DIR,workDir.toString()); } - private static JobConf configureTask(Task task, Credentials credentials, - Token jt) throws IOException { - final JobConf job = new JobConf(MRJobConfig.JOB_CONF_FILE); + private static void configureTask(JobConf job, Task task, + Credentials credentials, Token jt) throws IOException { job.setCredentials(credentials); - + ApplicationAttemptId appAttemptId = ConverterUtils.toContainerId( System.getenv(Environment.CONTAINER_ID.name())) @@ -306,7 +305,6 @@ private static JobConf configureTask(Task task, Credentials credentials, writeLocalJobFile(localTaskFile, job); task.setJobFile(localTaskFile.toString()); task.setConf(job); - return job; } /**