From d12c4631211f1ecee4af7d9700fc90781a962e37 Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Thu, 18 Apr 2013 18:51:45 +0000 Subject: [PATCH] MAPREDUCE-4898. FileOutputFormat.checkOutputSpecs and FileOutputFormat.setOutputPath incompatible with MR1. (rkanter via tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1469523 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 2 + .../lib/output/FileOutputFormat.java | 13 ++- .../lib/output/TestFileOutputFormat.java | 79 +++++++++++++++++++ 3 files changed, 90 insertions(+), 4 deletions(-) create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/TestFileOutputFormat.java diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 98656dfa6d0..5621586f185 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -310,6 +310,8 @@ Release 2.0.5-beta - UNRELEASED MAPREDUCE-5128. mapred-default.xml is missing a bunch of history server configs. (sandyr via tucu) + MAPREDUCE-4898. FileOutputFormat.checkOutputSpecs and FileOutputFormat.setOutputPath incompatible with MR1. (rkanter via tucu) + Release 2.0.4-alpha - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputFormat.java index a02f15e2f6f..60345a426e4 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputFormat.java @@ -27,7 +27,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.compress.CompressionCodec; -import org.apache.hadoop.fs.FileAlreadyExistsException; +import org.apache.hadoop.mapred.FileAlreadyExistsException; import org.apache.hadoop.mapred.InvalidJobConfException; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.JobContext; @@ -150,9 +150,14 @@ public void checkOutputSpecs(JobContext job * @param outputDir the {@link Path} of the output directory for * the map-reduce job. */ - public static void setOutputPath(Job job, Path outputDir) throws IOException { - outputDir = outputDir.getFileSystem(job.getConfiguration()).makeQualified( - outputDir); + public static void setOutputPath(Job job, Path outputDir) { + try { + outputDir = outputDir.getFileSystem(job.getConfiguration()).makeQualified( + outputDir); + } catch (IOException e) { + // Throw the IOException as a RuntimeException to be compatible with MR1 + throw new RuntimeException(e); + } job.getConfiguration().set(FileOutputFormat.OUTDIR, outputDir.toString()); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/TestFileOutputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/TestFileOutputFormat.java new file mode 100644 index 00000000000..a48fe3be2ea --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/TestFileOutputFormat.java @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mapreduce.lib.output; + +import java.io.IOException; +import junit.framework.TestCase; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.mapred.FileAlreadyExistsException; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.RecordWriter; +import org.apache.hadoop.mapreduce.TaskAttemptContext; + +public class TestFileOutputFormat extends TestCase { + + public void testSetOutputPathException() throws Exception { + Job job = Job.getInstance(); + try { + // Give it an invalid filesystem so it'll throw an exception + FileOutputFormat.setOutputPath(job, new Path("foo:///bar")); + fail("Should have thrown a RuntimeException with an IOException inside"); + } + catch (RuntimeException re) { + assertTrue(re.getCause() instanceof IOException); + } + } + + public void testCheckOutputSpecsException() throws Exception { + Job job = Job.getInstance(); + Path outDir = new Path(System.getProperty("test.build.data", "/tmp"), + "output"); + FileSystem fs = outDir.getFileSystem(new Configuration()); + // Create the output dir so it already exists and set it for the job + fs.mkdirs(outDir); + FileOutputFormat.setOutputPath(job, outDir); + // We don't need a "full" implementation of FileOutputFormat for this test + FileOutputFormat fof = new FileOutputFormat() { + @Override + public RecordWriter getRecordWriter(TaskAttemptContext job) + throws IOException, InterruptedException { + return null; + } + }; + try { + try { + // This should throw a FileAlreadyExistsException because the outputDir + // already exists + fof.checkOutputSpecs(job); + fail("Should have thrown a FileAlreadyExistsException"); + } + catch (FileAlreadyExistsException re) { + // correct behavior + } + } + finally { + // Cleanup + if (fs.exists(outDir)) { + fs.delete(outDir, true); + } + } + } +}