From bbaa7dc28db75d9b3700c6ff95222d8e1de29c15 Mon Sep 17 00:00:00 2001 From: arp Date: Thu, 28 Aug 2014 15:53:12 -0700 Subject: [PATCH] HDFS-6928. 'hdfs put' command should accept lazyPersist flag for testing. (Arpit Agarwal) --- .../apache/hadoop/fs/FilterFileSystem.java | 13 ++++++- .../fs/shell/CommandWithDestination.java | 32 ++++++++++++++--- .../apache/hadoop/fs/shell/CopyCommands.java | 16 +++++---- .../java/org/apache/hadoop/fs/shell/Stat.java | 7 +++- .../src/test/resources/testConf.xml | 28 +++++++++++---- .../hadoop-hdfs/CHANGES-HDFS-6581.txt | 2 ++ .../src/test/resources/testHDFSConf.xml | 36 +++++++++++++++++++ 7 files changed, 115 insertions(+), 19 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java index 52706f4049a..e729e67a453 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java @@ -179,8 +179,19 @@ public FSDataOutputStream create(Path f, FsPermission permission, return fs.create(f, permission, overwrite, bufferSize, replication, blockSize, progress); } - + @Override + public FSDataOutputStream create(Path f, + FsPermission permission, + EnumSet flags, + int bufferSize, + short replication, + long blockSize, + Progressable progress, + ChecksumOpt checksumOpt) throws IOException { + return fs.create(f, permission, + flags, bufferSize, replication, blockSize, progress); + } @Override @Deprecated diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java index da67f1ccac5..5d0d9d6975d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java @@ -30,6 +30,7 @@ import java.util.Map.Entry; import java.util.NoSuchElementException; +import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FilterFileSystem; @@ -45,6 +46,9 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.io.IOUtils; +import static org.apache.hadoop.fs.CreateFlag.CREATE; +import static org.apache.hadoop.fs.CreateFlag.LAZY_PERSIST; + /** * Provides: argument processing to ensure the destination is valid * for the number of source arguments. A processPaths that accepts both @@ -56,6 +60,7 @@ abstract class CommandWithDestination extends FsCommand { private boolean overwrite = false; private boolean verifyChecksum = true; private boolean writeChecksum = true; + private boolean lazyPersist = false; /** * The name of the raw xattr namespace. It would be nice to use @@ -78,6 +83,10 @@ protected void setOverwrite(boolean flag) { overwrite = flag; } + protected void setLazyPersist(boolean flag) { + lazyPersist = flag; + } + protected void setVerifyChecksum(boolean flag) { verifyChecksum = flag; } @@ -379,7 +388,7 @@ protected void copyStreamToTarget(InputStream in, PathData target) try { PathData tempTarget = target.suffix("._COPYING_"); targetFs.setWriteChecksum(writeChecksum); - targetFs.writeStreamToFile(in, tempTarget); + targetFs.writeStreamToFile(in, tempTarget, lazyPersist); targetFs.rename(tempTarget, target); } finally { targetFs.close(); // last ditch effort to ensure temp file is removed @@ -449,10 +458,11 @@ private static class TargetFileSystem extends FilterFileSystem { super(fs); } - void writeStreamToFile(InputStream in, PathData target) throws IOException { + void writeStreamToFile(InputStream in, PathData target, + boolean lazyPersist) throws IOException { FSDataOutputStream out = null; try { - out = create(target); + out = create(target, lazyPersist); IOUtils.copyBytes(in, out, getConf(), true); } finally { IOUtils.closeStream(out); // just in case copyBytes didn't @@ -460,9 +470,21 @@ void writeStreamToFile(InputStream in, PathData target) throws IOException { } // tag created files as temp files - FSDataOutputStream create(PathData item) throws IOException { + FSDataOutputStream create(PathData item, boolean lazyPersist) + throws IOException { try { - return create(item.path, true); + EnumSet createFlags = EnumSet.of(CREATE); + if (lazyPersist) { + createFlags.add(LAZY_PERSIST); + } + return create(item.path, + null, + createFlags, + getConf().getInt("io.file.buffer.size", 4096), + lazyPersist ? 1 : getDefaultReplication(item.path), + getDefaultBlockSize(), + null, + null); } finally { // might have been created but stream was interrupted deleteOnExit(item.path); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java index 3fd870c1cf6..afd11150d98 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java @@ -215,21 +215,25 @@ protected void processOptions(LinkedList args) */ public static class Put extends CommandWithDestination { public static final String NAME = "put"; - public static final String USAGE = "[-f] [-p] ... "; + public static final String USAGE = "[-f] [-p] [-l] ... "; public static final String DESCRIPTION = "Copy files from the local file system " + "into fs. Copying fails if the file already " + - "exists, unless the -f flag is given. Passing " + - "-p preserves access and modification times, " + - "ownership and the mode. Passing -f overwrites " + - "the destination if it already exists.\n"; + "exists, unless the -f flag is given.\n" + + "Flags:\n" + + " -p : Preserves access and modification times, ownership and the mode.\n" + + " -f : Overwrites the destination if it already exists.\n" + + " -l : Allow DataNode to lazily persist the file to disk. Forces\n" + + " replication factor of 1. This flag will result in reduced\n" + + " durability. Use with care.\n"; @Override protected void processOptions(LinkedList args) throws IOException { - CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE, "f", "p"); + CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE, "f", "p", "l"); cf.parse(args); setOverwrite(cf.getOpt("f")); setPreserve(cf.getOpt("p")); + setLazyPersist(cf.getOpt("l")); getRemoteDestination(args); // should have a -r option setRecursive(true); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Stat.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Stat.java index 652c92890d3..8b32eb8e21f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Stat.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Stat.java @@ -39,6 +39,7 @@ * %u: User name of owner * %y: UTC date as "yyyy-MM-dd HH:mm:ss" * %Y: Milliseconds since January 1, 1970 UTC + * %l: Whether lazyPersist flag is set on the file. */ @InterfaceAudience.Private @InterfaceStability.Unstable @@ -53,7 +54,8 @@ public static void registerCommands(CommandFactory factory) { public static final String DESCRIPTION = "Print statistics about the file/directory at " + "in the specified format. Format accepts filesize in blocks (%b), group name of owner(%g), " + - "filename (%n), block size (%o), replication (%r), user name of owner(%u), modification date (%y, %Y)\n"; + "filename (%n), block size (%o), replication (%r), user name of owner(%u), modification date (%y, %Y), " + + "lazyPersist flag (%l)\n"; protected static final SimpleDateFormat timeFmt; static { @@ -115,6 +117,9 @@ protected void processPath(PathData item) throws IOException { case 'Y': buf.append(stat.getModificationTime()); break; + case 'l': + buf.append(stat.isLazyPersist()); + break; default: // this leaves % alone, which causes the potential for // future format options to break strings; should use %% to diff --git a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml index 804b504fdf7..29d4174b0ce 100644 --- a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml +++ b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml @@ -436,7 +436,7 @@ RegexpComparator - ^-put \[-f\] \[-p\] <localsrc> \.\.\. <dst> :\s* + ^-put \[-f\] \[-p\] \[-l\] <localsrc> \.\.\. <dst> :( )* RegexpComparator @@ -444,15 +444,31 @@ RegexpComparator - ^\s*exists, unless the -f flag is given.( )*Passing -p preserves access and( )* + ^\s*exists, unless the -f flag is given.( )* RegexpComparator - ^\s*modification times, ownership and the mode. Passing -f overwrites the( )* + ^\s*Flags:( )* RegexpComparator - ^( |\t)*destination if it already exists.( )* + ^\s*-p Preserves access and modification times, ownership and the mode.( )* + + + RegexpComparator + ^\s*-f Overwrites the destination if it already exists.( )* + + + RegexpComparator + ^\s*-l Allow DataNode to lazily persist the file to disk. Forces( )* + + + RegexpComparator + ^\s*replication factor of 1. This flag will result in reduced( )* + + + RegexpComparator + ^\s*durability. Use with care.( )* @@ -467,7 +483,7 @@ RegexpComparator - ^-copyFromLocal \[-f\] \[-p\] <localsrc> \.\.\. <dst> :\s* + ^-copyFromLocal \[-f\] \[-p\] \[-l\] <localsrc> \.\.\. <dst> :\s* RegexpComparator @@ -782,7 +798,7 @@ RegexpComparator - ^( |\t)*\(%y, %Y\)( )* + ^( |\t)*\(%y, %Y\), lazyPersist flag \(\%l\)( )* diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt index aac215cf138..b0fb070ada3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt @@ -21,4 +21,6 @@ HDFS-6929. NN periodically unlinks lazy persist files with missing replicas from namespace. (Arpit Agarwal) + HDFS-6928. 'hdfs put' command should accept lazyPersist flag for testing. + (Arpit Agarwal) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml index 087c3ab48aa..b8fba037e39 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml @@ -4554,6 +4554,42 @@ + + put: The LazyPersist flag is set when requested + + -fs NAMENODE -mkdir /dirLp + -fs NAMENODE -put -l CLITEST_DATA/data15bytes /dirLp/data15bytes + -fs NAMENODE -stat %l /dirLp/data15bytes + + + -fs NAMENODE -rm -r /dirLp/ + + + + RegexpComparator + ^true + + + + + + put: The LazyPersist flag is not set by default + + -fs NAMENODE -mkdir /dirLp + -fs NAMENODE -put CLITEST_DATA/data15bytes /dirLp/data15bytes + -fs NAMENODE -stat %l /dirLp/data15bytes + + + -fs NAMENODE -rm -r /dirLp/ + + + + RegexpComparator + ^false + + + + copyFromLocal: copying file into a file (absolute path)