Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-3054. distcp -skipcrccheck has no effect. Contributed by Colin Patrick McCabe. 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1381295 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Todd Lipcon 2012-09-05 18:54:32 +00:00
parent e7082c371b
commit 0c5de0e4f6
2 changed files with 15 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java
View File

@ -255,7 +255,7 @@ public class CopyMapper extends Mapper<Text, FileStatus, Text, Text> {
long bytesCopied; long bytesCopied;
try { try {
bytesCopied = (Long)new RetriableFileCopyCommand(description) bytesCopied = (Long)new RetriableFileCopyCommand(skipCrc, description)
.execute(sourceFileStatus, target, context, fileAttributes); .execute(sourceFileStatus, target, context, fileAttributes);
} catch (Exception e) { } catch (Exception e) {
context.setStatus("Copy Failure: " + sourceFileStatus.getPath()); context.setStatus("Copy Failure: " + sourceFileStatus.getPath());

14
hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java
View File

@ -41,6 +41,7 @@ public class RetriableFileCopyCommand extends RetriableCommand {
private static Log LOG = LogFactory.getLog(RetriableFileCopyCommand.class); private static Log LOG = LogFactory.getLog(RetriableFileCopyCommand.class);
private static int BUFFER_SIZE = 8 * 1024; private static int BUFFER_SIZE = 8 * 1024;
private boolean skipCrc = false;
/** /**
* Constructor, taking a description of the action. * Constructor, taking a description of the action.
@ -50,6 +51,17 @@ public class RetriableFileCopyCommand extends RetriableCommand {
super(description); super(description);
} }
/**
* Create a RetriableFileCopyCommand.
*
* @param skipCrc Whether to skip the crc check.
* @param description A verbose description of the copy operation.
*/
public RetriableFileCopyCommand(boolean skipCrc, String description) {
this(description);
this.skipCrc = skipCrc;
}
/** /**
* Implementation of RetriableCommand::doExecute(). * Implementation of RetriableCommand::doExecute().
* This is the actual copy-implementation. * This is the actual copy-implementation.
@ -92,7 +104,7 @@ public class RetriableFileCopyCommand extends RetriableCommand {
compareFileLengths(sourceFileStatus, tmpTargetPath, configuration, bytesRead); compareFileLengths(sourceFileStatus, tmpTargetPath, configuration, bytesRead);
//At this point, src&dest lengths are same. if length==0, we skip checksum //At this point, src&dest lengths are same. if length==0, we skip checksum
if (bytesRead != 0) { if ((bytesRead != 0) && (!skipCrc)) {
compareCheckSums(sourceFS, sourceFileStatus.getPath(), targetFS, tmpTargetPath); compareCheckSums(sourceFS, sourceFileStatus.getPath(), targetFS, tmpTargetPath);
} }
promoteTmpToTarget(tmpTargetPath, target, targetFS); promoteTmpToTarget(tmpTargetPath, target, targetFS);