MAPREDUCE-5448. MapFileOutputFormat#getReaders bug with invisible files/folders. Contributed by Maysam Yabandeh.

(cherry picked from commit b46c2bb51a)
This commit is contained in:
Harsh J 2015-03-22 09:45:48 +05:30
parent 7c72c7f529
commit a7f1c1b993
3 changed files with 24 additions and 1 deletions

View File

@ -38,6 +38,9 @@ Release 2.8.0 - UNRELEASED
BUG FIXES BUG FIXES
MAPREDUCE-5448. MapFileOutputFormat#getReaders bug with hidden
files/folders. (Maysam Yabandeh via harsh)
MAPREDUCE-6286. A typo in HistoryViewer makes some code useless, which MAPREDUCE-6286. A typo in HistoryViewer makes some code useless, which
causes counter limits are not reset correctly. causes counter limits are not reset correctly.
(Zhihai Xu via harsh) (Zhihai Xu via harsh)

View File

@ -24,6 +24,7 @@ import java.util.Arrays;
import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.io.MapFile; import org.apache.hadoop.io.MapFile;
import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.io.WritableComparable;
@ -88,7 +89,16 @@ public class MapFileOutputFormat
public static MapFile.Reader[] getReaders(Path dir, public static MapFile.Reader[] getReaders(Path dir,
Configuration conf) throws IOException { Configuration conf) throws IOException {
FileSystem fs = dir.getFileSystem(conf); FileSystem fs = dir.getFileSystem(conf);
Path[] names = FileUtil.stat2Paths(fs.listStatus(dir)); PathFilter filter = new PathFilter() {
@Override
public boolean accept(Path path) {
String name = path.getName();
if (name.startsWith("_") || name.startsWith("."))
return false;
return true;
}
};
Path[] names = FileUtil.stat2Paths(fs.listStatus(dir, filter));
// sort names, so that hash partitioning works // sort names, so that hash partitioning works
Arrays.sort(names); Arrays.sort(names);

View File

@ -27,6 +27,7 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors; import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import junit.framework.Assert;
import junit.framework.TestCase; import junit.framework.TestCase;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
@ -309,6 +310,15 @@ public class TestFileOutputCommitter extends TestCase {
committer.commitTask(tContext); committer.commitTask(tContext);
committer.commitJob(jContext); committer.commitJob(jContext);
// Ensure getReaders call works and also ignores
// hidden filenames (_ or . prefixes)
try {
MapFileOutputFormat.getReaders(outDir, conf);
} catch (Exception e) {
Assert.fail("Fail to read from MapFileOutputFormat: " + e);
e.printStackTrace();
}
// validate output // validate output
validateMapFileOutputContent(FileSystem.get(job.getConfiguration()), outDir); validateMapFileOutputContent(FileSystem.get(job.getConfiguration()), outDir);
FileUtil.fullyDelete(new File(outDir.toString())); FileUtil.fullyDelete(new File(outDir.toString()));