HADOOP-18582. Addendum: Skip unnecessary cleanup logic in DistCp. (#5409)
Followup to the original HADOOP-18582. Temporary path cleanup is re-enabled for -append jobs as these will create temporary files when creating or overwriting files. Contributed by Ayush Saxena
This commit is contained in:
parent
ee71318d72
commit
84e999b35c
|
@ -153,10 +153,7 @@ public class CopyCommitter extends FileOutputCommitter {
|
||||||
|
|
||||||
final boolean directWrite = conf.getBoolean(
|
final boolean directWrite = conf.getBoolean(
|
||||||
DistCpOptionSwitch.DIRECT_WRITE.getConfigLabel(), false);
|
DistCpOptionSwitch.DIRECT_WRITE.getConfigLabel(), false);
|
||||||
final boolean append = conf.getBoolean(
|
if (directWrite) {
|
||||||
DistCpOptionSwitch.APPEND.getConfigLabel(), false);
|
|
||||||
final boolean useTempTarget = !append && !directWrite;
|
|
||||||
if (!useTempTarget) {
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -586,13 +586,11 @@ public class TestCopyCommitter {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testCommitWithCleanupTempFiles() throws IOException {
|
public void testCommitWithCleanupTempFiles() throws IOException {
|
||||||
testCommitWithCleanup(true, false);
|
testCommitWithCleanup(true);
|
||||||
testCommitWithCleanup(false, true);
|
testCommitWithCleanup(false);
|
||||||
testCommitWithCleanup(true, true);
|
|
||||||
testCommitWithCleanup(false, false);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private void testCommitWithCleanup(boolean append, boolean directWrite)throws IOException {
|
private void testCommitWithCleanup(boolean directWrite) throws IOException {
|
||||||
TaskAttemptContext taskAttemptContext = getTaskAttemptContext(config);
|
TaskAttemptContext taskAttemptContext = getTaskAttemptContext(config);
|
||||||
JobID jobID = taskAttemptContext.getTaskAttemptID().getJobID();
|
JobID jobID = taskAttemptContext.getTaskAttemptID().getJobID();
|
||||||
JobContext jobContext = new JobContextImpl(
|
JobContext jobContext = new JobContextImpl(
|
||||||
|
@ -611,7 +609,7 @@ public class TestCopyCommitter {
|
||||||
DistCpOptions options = new DistCpOptions.Builder(
|
DistCpOptions options = new DistCpOptions.Builder(
|
||||||
Collections.singletonList(new Path(sourceBase)),
|
Collections.singletonList(new Path(sourceBase)),
|
||||||
new Path("/out"))
|
new Path("/out"))
|
||||||
.withAppend(append)
|
.withAppend(true)
|
||||||
.withSyncFolder(true)
|
.withSyncFolder(true)
|
||||||
.withDirectWrite(directWrite)
|
.withDirectWrite(directWrite)
|
||||||
.build();
|
.build();
|
||||||
|
@ -631,7 +629,7 @@ public class TestCopyCommitter {
|
||||||
null, taskAttemptContext);
|
null, taskAttemptContext);
|
||||||
committer.commitJob(jobContext);
|
committer.commitJob(jobContext);
|
||||||
|
|
||||||
if (append || directWrite) {
|
if (directWrite) {
|
||||||
ContractTestUtils.assertPathExists(fs, "Temp files should not be cleanup with append or direct option",
|
ContractTestUtils.assertPathExists(fs, "Temp files should not be cleanup with append or direct option",
|
||||||
tempFilePath);
|
tempFilePath);
|
||||||
} else {
|
} else {
|
||||||
|
|
Loading…
Reference in New Issue