diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 507c6f44582..440c571ecd6 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -38,6 +38,9 @@ Release 2.8.0 - UNRELEASED BUG FIXES + MAPREDUCE-5448. MapFileOutputFormat#getReaders bug with hidden + files/folders. (Maysam Yabandeh via harsh) + MAPREDUCE-6286. A typo in HistoryViewer makes some code useless, which causes counter limits are not reset correctly. (Zhihai Xu via harsh) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/MapFileOutputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/MapFileOutputFormat.java index b8cb997c09c..da33770b4ba 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/MapFileOutputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/MapFileOutputFormat.java @@ -24,6 +24,7 @@ import java.util.Arrays; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.io.MapFile; import org.apache.hadoop.io.WritableComparable; @@ -88,7 +89,16 @@ public class MapFileOutputFormat public static MapFile.Reader[] getReaders(Path dir, Configuration conf) throws IOException { FileSystem fs = dir.getFileSystem(conf); - Path[] names = FileUtil.stat2Paths(fs.listStatus(dir)); + PathFilter filter = new PathFilter() { + @Override + public boolean accept(Path path) { + String name = path.getName(); + if (name.startsWith("_") || name.startsWith(".")) + return false; + return true; + } + }; + Path[] names = FileUtil.stat2Paths(fs.listStatus(dir, filter)); // sort names, so that hash partitioning works Arrays.sort(names); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/TestFileOutputCommitter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/TestFileOutputCommitter.java index 0d4ab98693d..5c4428b4864 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/TestFileOutputCommitter.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/TestFileOutputCommitter.java @@ -27,6 +27,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import junit.framework.Assert; import junit.framework.TestCase; import org.apache.commons.logging.Log; @@ -309,6 +310,15 @@ public class TestFileOutputCommitter extends TestCase { committer.commitTask(tContext); committer.commitJob(jContext); + // Ensure getReaders call works and also ignores + // hidden filenames (_ or . prefixes) + try { + MapFileOutputFormat.getReaders(outDir, conf); + } catch (Exception e) { + Assert.fail("Fail to read from MapFileOutputFormat: " + e); + e.printStackTrace(); + } + // validate output validateMapFileOutputContent(FileSystem.get(job.getConfiguration()), outDir); FileUtil.fullyDelete(new File(outDir.toString()));