From 4705652915c32f9f8ef5d734e4b839e46691977f Mon Sep 17 00:00:00 2001 From: Robert Joseph Evans Date: Thu, 9 Aug 2012 17:39:37 +0000 Subject: [PATCH] svn merge -c 1371325 FIXES: MAPREDUCE-3782. teragen terasort jobs fail when using webhdfs:// (Jason Lowe via bobby) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1371332 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../apache/hadoop/examples/terasort/TeraOutputFormat.java | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 29efe373162..40106ea2a74 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -682,6 +682,9 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-4503. Should throw InvalidJobConfException if duplicates found in cacheArchives or cacheFiles (Robert Evans via jeagles) + MAPREDUCE-3782. teragen terasort jobs fail when using webhdfs:// (Jason + Lowe via bobby) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraOutputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraOutputFormat.java index 1900e117f5a..a4a67fbe695 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraOutputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraOutputFormat.java @@ -31,6 +31,7 @@ import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; +import org.apache.hadoop.mapreduce.security.TokenCache; /** * An output format that writes the key and value appended together. @@ -85,6 +86,10 @@ public void checkOutputSpecs(JobContext job if (outDir == null) { throw new InvalidJobConfException("Output directory not set in JobConf."); } + + // get delegation token for outDir's file system + TokenCache.obtainTokensForNamenodes(job.getCredentials(), + new Path[] { outDir }, job.getConfiguration()); } public RecordWriter getRecordWriter(TaskAttemptContext job