diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java index 81c2be7e05d..238a37ba5fe 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java @@ -69,6 +69,7 @@ public class CopyCommitter extends FileOutputCommitter { private boolean overwrite = false; private boolean targetPathExists = true; private boolean ignoreFailures = false; + private int blocksPerChunk = 0; /** * Create a output committer @@ -79,6 +80,9 @@ public class CopyCommitter extends FileOutputCommitter { */ public CopyCommitter(Path outputPath, TaskAttemptContext context) throws IOException { super(outputPath, context); + blocksPerChunk = context.getConfiguration().getInt( + DistCpOptionSwitch.BLOCKS_PER_CHUNK.getConfigLabel(), 0); + LOG.debug("blocks per chunk {}", blocksPerChunk); this.taskAttemptContext = context; } @@ -93,7 +97,9 @@ public class CopyCommitter extends FileOutputCommitter { ignoreFailures = conf.getBoolean( DistCpOptionSwitch.IGNORE_FAILURES.getConfigLabel(), false); - concatFileChunks(conf); + if (blocksPerChunk > 0) { + concatFileChunks(conf); + } super.commitJob(jobContext); diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java index 6ee37ccd6a5..38964c1c2cc 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java @@ -82,10 +82,6 @@ public class TestCopyCommitter { @Before public void createMetaFolder() { config.set(DistCpConstants.CONF_LABEL_META_FOLDER, "/meta"); - // Unset listing file path since the config is shared by - // multiple tests, and some test doesn't set it, such as - // testNoCommitAction, but the distcp code will check it. - config.set(DistCpConstants.CONF_LABEL_LISTING_FILE_PATH, ""); Path meta = new Path("/meta"); try { cluster.getFileSystem().mkdirs(meta);