Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-13163 Reuse pre-computed filestatus in Distcp-CopyMapper (Rajesh Balamohan via stevel)

This commit is contained in:
Steve Loughran 2016-05-17 12:59:59 +01:00
parent 19db965c27
commit dfc062a092
1 changed files with 2 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java
View File

@ -245,7 +245,7 @@ public class CopyMapper extends Mapper<Text, CopyListingFileStatus, Text, Text>
return; return;
} }
FileAction action = checkUpdate(sourceFS, sourceCurrStatus, target); FileAction action = checkUpdate(sourceFS, sourceCurrStatus, target, targetStatus);
if (action == FileAction.SKIP) { if (action == FileAction.SKIP) {
LOG.info("Skipping copy of " + sourceCurrStatus.getPath() LOG.info("Skipping copy of " + sourceCurrStatus.getPath()
+ " to " + target); + " to " + target);
@ -332,13 +332,7 @@ public class CopyMapper extends Mapper<Text, CopyListingFileStatus, Text, Text>
} }
private FileAction checkUpdate(FileSystem sourceFS, FileStatus source, private FileAction checkUpdate(FileSystem sourceFS, FileStatus source,
Path target) throws IOException { Path target, FileStatus targetFileStatus) throws IOException {
final FileStatus targetFileStatus;
try {
targetFileStatus = targetFS.getFileStatus(target);
} catch (FileNotFoundException e) {
return FileAction.OVERWRITE;
}
if (targetFileStatus != null && !overWrite) { if (targetFileStatus != null && !overWrite) {
if (canSkip(sourceFS, source, targetFileStatus)) { if (canSkip(sourceFS, source, targetFileStatus)) {
return FileAction.SKIP; return FileAction.SKIP;