MAPREDUCE-5448. MapFileOutputFormat#getReaders bug with invisible files/folders. Contributed by Maysam Yabandeh.
(cherry picked from commit b46c2bb51a
)
This commit is contained in:
parent
7c72c7f529
commit
a7f1c1b993
|
@ -38,6 +38,9 @@ Release 2.8.0 - UNRELEASED
|
||||||
|
|
||||||
BUG FIXES
|
BUG FIXES
|
||||||
|
|
||||||
|
MAPREDUCE-5448. MapFileOutputFormat#getReaders bug with hidden
|
||||||
|
files/folders. (Maysam Yabandeh via harsh)
|
||||||
|
|
||||||
MAPREDUCE-6286. A typo in HistoryViewer makes some code useless, which
|
MAPREDUCE-6286. A typo in HistoryViewer makes some code useless, which
|
||||||
causes counter limits are not reset correctly.
|
causes counter limits are not reset correctly.
|
||||||
(Zhihai Xu via harsh)
|
(Zhihai Xu via harsh)
|
||||||
|
|
|
@ -24,6 +24,7 @@ import java.util.Arrays;
|
||||||
import org.apache.hadoop.fs.FileSystem;
|
import org.apache.hadoop.fs.FileSystem;
|
||||||
import org.apache.hadoop.fs.Path;
|
import org.apache.hadoop.fs.Path;
|
||||||
import org.apache.hadoop.fs.FileUtil;
|
import org.apache.hadoop.fs.FileUtil;
|
||||||
|
import org.apache.hadoop.fs.PathFilter;
|
||||||
|
|
||||||
import org.apache.hadoop.io.MapFile;
|
import org.apache.hadoop.io.MapFile;
|
||||||
import org.apache.hadoop.io.WritableComparable;
|
import org.apache.hadoop.io.WritableComparable;
|
||||||
|
@ -88,7 +89,16 @@ public class MapFileOutputFormat
|
||||||
public static MapFile.Reader[] getReaders(Path dir,
|
public static MapFile.Reader[] getReaders(Path dir,
|
||||||
Configuration conf) throws IOException {
|
Configuration conf) throws IOException {
|
||||||
FileSystem fs = dir.getFileSystem(conf);
|
FileSystem fs = dir.getFileSystem(conf);
|
||||||
Path[] names = FileUtil.stat2Paths(fs.listStatus(dir));
|
PathFilter filter = new PathFilter() {
|
||||||
|
@Override
|
||||||
|
public boolean accept(Path path) {
|
||||||
|
String name = path.getName();
|
||||||
|
if (name.startsWith("_") || name.startsWith("."))
|
||||||
|
return false;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
Path[] names = FileUtil.stat2Paths(fs.listStatus(dir, filter));
|
||||||
|
|
||||||
// sort names, so that hash partitioning works
|
// sort names, so that hash partitioning works
|
||||||
Arrays.sort(names);
|
Arrays.sort(names);
|
||||||
|
|
|
@ -27,6 +27,7 @@ import java.util.concurrent.ExecutorService;
|
||||||
import java.util.concurrent.Executors;
|
import java.util.concurrent.Executors;
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
|
import junit.framework.Assert;
|
||||||
import junit.framework.TestCase;
|
import junit.framework.TestCase;
|
||||||
|
|
||||||
import org.apache.commons.logging.Log;
|
import org.apache.commons.logging.Log;
|
||||||
|
@ -309,6 +310,15 @@ public class TestFileOutputCommitter extends TestCase {
|
||||||
committer.commitTask(tContext);
|
committer.commitTask(tContext);
|
||||||
committer.commitJob(jContext);
|
committer.commitJob(jContext);
|
||||||
|
|
||||||
|
// Ensure getReaders call works and also ignores
|
||||||
|
// hidden filenames (_ or . prefixes)
|
||||||
|
try {
|
||||||
|
MapFileOutputFormat.getReaders(outDir, conf);
|
||||||
|
} catch (Exception e) {
|
||||||
|
Assert.fail("Fail to read from MapFileOutputFormat: " + e);
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
|
||||||
// validate output
|
// validate output
|
||||||
validateMapFileOutputContent(FileSystem.get(job.getConfiguration()), outDir);
|
validateMapFileOutputContent(FileSystem.get(job.getConfiguration()), outDir);
|
||||||
FileUtil.fullyDelete(new File(outDir.toString()));
|
FileUtil.fullyDelete(new File(outDir.toString()));
|
||||||
|
|
Loading…
Reference in New Issue