diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 9ef4ace85af..b765963577d 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -625,6 +625,9 @@ Release 0.23.1 - Unreleased MAPREDUCE-3708. Metrics: Incorrect Apps Submitted Count (Bhallamudi via mahadev) + MAPREDUCE-3727. jobtoken location property in jobconf refers to wrong + jobtoken file (tucu) + Release 0.23.0 - 2011-11-01 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java index f63352b4dd1..4038f65cd46 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java @@ -369,6 +369,12 @@ class JobSubmitter { conf.set(toFullPropertyName(queue, QueueACL.ADMINISTER_JOBS.getAclName()), acl.getAclString()); + // removing jobtoken referrals before copying the jobconf to HDFS + // as the tasks don't need this setting, actually they may break + // because of it if present as the referral will point to a + // different job. + TokenCache.cleanUpTokenReferral(conf); + // Write job file to submit dir writeConf(conf, submitJobFile); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java index 312faf35925..63cbef73c82 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java @@ -79,7 +79,17 @@ public class TokenCache { } obtainTokensForNamenodesInternal(credentials, ps, conf); } - + + /** + * Remove jobtoken referrals which don't make sense in the context + * of the task execution. + * + * @param conf + */ + public static void cleanUpTokenReferral(Configuration conf) { + conf.unset(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY); + } + static void obtainTokensForNamenodesInternal(Credentials credentials, Path[] ps, Configuration conf) throws IOException { for(Path p: ps) { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java index 8c084fe0ba6..5efc33ddc4e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java @@ -19,6 +19,8 @@ package org.apache.hadoop.mapreduce.security; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; import static org.mockito.Matchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; @@ -33,6 +35,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapred.Master; +import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -158,4 +161,13 @@ public class TestTokenCache { return mockFs; } + + @Test + public void testCleanUpTokenReferral() throws Exception { + Configuration conf = new Configuration(); + conf.set(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY, "foo"); + TokenCache.cleanUpTokenReferral(conf); + assertNull(conf.get(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY)); + } + }