From af5f90623c039573d8f9590a6bdf8d9043beae7d Mon Sep 17 00:00:00 2001 From: Ayush Saxena Date: Mon, 26 Oct 2020 22:07:37 +0530 Subject: [PATCH] HADOOP-17328. LazyPersist Overwrite fails in direct write mode. (#2413) (cherry picked from commit 872440610f066d4b12c9f93c05477848a260b21f) --- .../fs/shell/CommandWithDestination.java | 4 +++- .../org/apache/hadoop/fs/TestFsShellCopy.java | 24 +++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java index ca9961aeb65..90a709dffc0 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java @@ -54,6 +54,7 @@ import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_ import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY; import static org.apache.hadoop.fs.CreateFlag.CREATE; import static org.apache.hadoop.fs.CreateFlag.LAZY_PERSIST; +import static org.apache.hadoop.fs.CreateFlag.OVERWRITE; /** * Provides: argument processing to ensure the destination is valid @@ -515,7 +516,8 @@ abstract class CommandWithDestination extends FsCommand { defaultBlockSize = getDefaultBlockSize(item.path); } - EnumSet createFlags = EnumSet.of(CREATE, LAZY_PERSIST); + EnumSet createFlags = + EnumSet.of(CREATE, LAZY_PERSIST, OVERWRITE); return create(item.path, FsPermission.getFileDefault().applyUMask( FsPermission.getUMask(getConf())), diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellCopy.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellCopy.java index 117fad2924d..e3c4ee05c8c 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellCopy.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellCopy.java @@ -23,6 +23,7 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.not; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; @@ -697,4 +698,27 @@ public class TestFsShellCopy { lfs.setPermission(src, new FsPermission((short)0755)); } } + + @Test + public void testLazyPersistDirectOverwrite() throws Exception { + Path testRoot = new Path(testRootDir, "testLazyPersistDirectOverwrite"); + try { + lfs.delete(testRoot, true); + lfs.mkdirs(testRoot); + Path filePath = new Path(testRoot, new Path("srcFile")); + lfs.create(filePath).close(); + // Put with overwrite in direct mode. + String[] argv = + new String[] {"-put", "-f", "-l", "-d", filePath.toString(), + filePath.toString()}; + assertEquals(0, shell.run(argv)); + + // Put without overwrite in direct mode shouldn't be success. + argv = new String[] {"-put", "-l", "-d", filePath.toString(), + filePath.toString()}; + assertNotEquals(0, shell.run(argv)); + } finally { + lfs.delete(testRoot, true); + } + } }