HDFS-6928. 'hdfs put' command should accept lazyPersist flag for testing. (Arpit Agarwal)

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java

@@ -180,7 +180,18 @@ public FSDataOutputStream create(Path f, FsPermission permission,
         overwrite, bufferSize, replication, blockSize, progress);
   }
 
-
+  @Override
+  public FSDataOutputStream create(Path f,
+        FsPermission permission,
+        EnumSet<CreateFlag> flags,
+        int bufferSize,
+        short replication,
+        long blockSize,
+        Progressable progress,
+        ChecksumOpt checksumOpt) throws IOException {
+    return fs.create(f, permission,
+      flags, bufferSize, replication, blockSize, progress);
+  }
   
   @Override
   @Deprecated

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java

@@ -30,6 +30,7 @@
 import java.util.Map.Entry;
 import java.util.NoSuchElementException;
 
+import org.apache.hadoop.fs.CreateFlag;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FilterFileSystem;
@@ -45,6 +46,9 @@
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.io.IOUtils;
 
+import static org.apache.hadoop.fs.CreateFlag.CREATE;
+import static org.apache.hadoop.fs.CreateFlag.LAZY_PERSIST;
+
 /**
  * Provides: argument processing to ensure the destination is valid
  * for the number of source arguments.  A processPaths that accepts both
@@ -56,6 +60,7 @@ abstract class CommandWithDestination extends FsCommand {
   private boolean overwrite = false;
   private boolean verifyChecksum = true;
   private boolean writeChecksum = true;
+  private boolean lazyPersist = false;
   
   /**
    * The name of the raw xattr namespace. It would be nice to use
@@ -78,6 +83,10 @@ protected void setOverwrite(boolean flag) {
     overwrite = flag;
   }
   
+  protected void setLazyPersist(boolean flag) {
+    lazyPersist = flag;
+  }
+
   protected void setVerifyChecksum(boolean flag) {
     verifyChecksum = flag;
   }
@@ -379,7 +388,7 @@ protected void copyStreamToTarget(InputStream in, PathData target)
     try {
       PathData tempTarget = target.suffix("._COPYING_");
       targetFs.setWriteChecksum(writeChecksum);
-      targetFs.writeStreamToFile(in, tempTarget);
+      targetFs.writeStreamToFile(in, tempTarget, lazyPersist);
       targetFs.rename(tempTarget, target);
     } finally {
       targetFs.close(); // last ditch effort to ensure temp file is removed
@@ -449,10 +458,11 @@ private static class TargetFileSystem extends FilterFileSystem {
       super(fs);
     }
 
-    void writeStreamToFile(InputStream in, PathData target) throws IOException {
+    void writeStreamToFile(InputStream in, PathData target,
+                           boolean lazyPersist) throws IOException {
       FSDataOutputStream out = null;
       try {
-        out = create(target);
+        out = create(target, lazyPersist);
         IOUtils.copyBytes(in, out, getConf(), true);
       } finally {
         IOUtils.closeStream(out); // just in case copyBytes didn't
@@ -460,9 +470,21 @@ void writeStreamToFile(InputStream in, PathData target) throws IOException {
     }
     
     // tag created files as temp files
-    FSDataOutputStream create(PathData item) throws IOException {
+    FSDataOutputStream create(PathData item, boolean lazyPersist)
+        throws IOException {
       try {
-        return create(item.path, true);
+        EnumSet<CreateFlag> createFlags = EnumSet.of(CREATE);
+        if (lazyPersist) {
+          createFlags.add(LAZY_PERSIST);
+        }
+        return create(item.path,
+                      null,
+                      createFlags,
+                      getConf().getInt("io.file.buffer.size", 4096),
+                      lazyPersist ? 1 : getDefaultReplication(item.path),
+                      getDefaultBlockSize(),
+                      null,
+                      null);
       } finally { // might have been created but stream was interrupted
         deleteOnExit(item.path);
       }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java

@@ -215,21 +215,25 @@ protected void processOptions(LinkedList<String> args)
    */
   public static class Put extends CommandWithDestination {
     public static final String NAME = "put";
-    public static final String USAGE = "[-f] [-p] <localsrc> ... <dst>";
+    public static final String USAGE = "[-f] [-p] [-l] <localsrc> ... <dst>";
     public static final String DESCRIPTION =
       "Copy files from the local file system " +
       "into fs. Copying fails if the file already " +
-      "exists, unless the -f flag is given. Passing " +
+      "exists, unless the -f flag is given.\n" +
-      "-p preserves access and modification times, " +
+      "Flags:\n" +
-      "ownership and the mode. Passing -f overwrites " +
+      "  -p : Preserves access and modification times, ownership and the mode.\n" +
-      "the destination if it already exists.\n";
+      "  -f : Overwrites the destination if it already exists.\n" +
+      "  -l : Allow DataNode to lazily persist the file to disk. Forces\n" +
+      "       replication factor of 1. This flag will result in reduced\n" +
+      "       durability. Use with care.\n";
 
     @Override
     protected void processOptions(LinkedList<String> args) throws IOException {
-      CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE, "f", "p");
+      CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE, "f", "p", "l");
       cf.parse(args);
       setOverwrite(cf.getOpt("f"));
       setPreserve(cf.getOpt("p"));
+      setLazyPersist(cf.getOpt("l"));
       getRemoteDestination(args);
       // should have a -r option
       setRecursive(true);

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Stat.java

@@ -39,6 +39,7 @@
  *   %u: User name of owner
  *   %y: UTC date as "yyyy-MM-dd HH:mm:ss"
  *   %Y: Milliseconds since January 1, 1970 UTC
+ *   %l: Whether lazyPersist flag is set on the file.
  */
 @InterfaceAudience.Private
 @InterfaceStability.Unstable
@@ -53,7 +54,8 @@ public static void registerCommands(CommandFactory factory) {
   public static final String DESCRIPTION =
     "Print statistics about the file/directory at <path> " +
     "in the specified format. Format accepts filesize in blocks (%b), group name of owner(%g), " +
-    "filename (%n), block size (%o), replication (%r), user name of owner(%u), modification date (%y, %Y)\n";
+    "filename (%n), block size (%o), replication (%r), user name of owner(%u), modification date (%y, %Y), " +
+    "lazyPersist flag (%l)\n";
 
   protected static final SimpleDateFormat timeFmt;
   static {
@@ -115,6 +117,9 @@ protected void processPath(PathData item) throws IOException {
           case 'Y':
             buf.append(stat.getModificationTime());
             break;
+          case 'l':
+            buf.append(stat.isLazyPersist());
+            break;
           default:
             // this leaves %<unknown> alone, which causes the potential for
             // future format options to break strings; should use %% to

hadoop-common-project/hadoop-common/src/test/resources/testConf.xml

@@ -436,7 +436,7 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-put \[-f\] \[-p\] &lt;localsrc&gt; \.\.\. &lt;dst&gt; :\s*</expected-output>
+          <expected-output>^-put \[-f\] \[-p\] \[-l\] &lt;localsrc&gt; \.\.\. &lt;dst&gt; :( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
@@ -444,15 +444,31 @@
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^\s*exists, unless the -f flag is given.( )*Passing -p preserves access and( )*</expected-output>
+          <expected-output>^\s*exists, unless the -f flag is given.( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^\s*modification times, ownership and the mode. Passing -f overwrites the( )*</expected-output>
+          <expected-output>^\s*Flags:( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*destination if it already exists.( )*</expected-output>
+          <expected-output>^\s*-p  Preserves access and modification times, ownership and the mode.( )*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^\s*-f  Overwrites the destination if it already exists.( )*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^\s*-l  Allow DataNode to lazily persist the file to disk. Forces( )*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^\s*replication factor of 1. This flag will result in reduced( )*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^\s*durability. Use with care.( )*</expected-output>
         </comparator>
       </comparators>
     </test>
@@ -467,7 +483,7 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-copyFromLocal \[-f\] \[-p\] &lt;localsrc&gt; \.\.\. &lt;dst&gt; :\s*</expected-output>
+          <expected-output>^-copyFromLocal \[-f\] \[-p\] \[-l\] &lt;localsrc&gt; \.\.\. &lt;dst&gt; :\s*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
@@ -782,7 +798,7 @@
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*\(%y, %Y\)( )*</expected-output>
+          <expected-output>^( |\t)*\(%y, %Y\), lazyPersist flag \(\%l\)( )*</expected-output>
         </comparator>
       </comparators>
     </test>

hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt

@@ -21,4 +21,6 @@
     HDFS-6929. NN periodically unlinks lazy persist files with missing
     replicas from namespace. (Arpit Agarwal)
 
+    HDFS-6928. 'hdfs put' command should accept lazyPersist flag for testing.
+    (Arpit Agarwal)
 

hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml

@@ -4554,6 +4554,42 @@
       </comparators>
     </test>
 
+    <test> <!-- TESTED -->
+      <description>put: The LazyPersist flag is set when requested</description>
+      <test-commands>
+        <command>-fs NAMENODE -mkdir /dirLp</command>
+        <command>-fs NAMENODE -put -l CLITEST_DATA/data15bytes /dirLp/data15bytes</command>
+        <command>-fs NAMENODE -stat %l /dirLp/data15bytes</command>
+      </test-commands>
+      <cleanup-commands>
+        <command>-fs NAMENODE -rm -r /dirLp/</command>
+      </cleanup-commands>
+      <comparators>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^true</expected-output>
+        </comparator>
+      </comparators>
+    </test>
+
+    <test> <!-- TESTED -->
+      <description>put: The LazyPersist flag is not set by default</description>
+      <test-commands>
+        <command>-fs NAMENODE -mkdir /dirLp</command>
+        <command>-fs NAMENODE -put CLITEST_DATA/data15bytes /dirLp/data15bytes</command>
+        <command>-fs NAMENODE -stat %l /dirLp/data15bytes</command>
+      </test-commands>
+      <cleanup-commands>
+        <command>-fs NAMENODE -rm -r /dirLp/</command>
+      </cleanup-commands>
+      <comparators>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^false</expected-output>
+        </comparator>
+      </comparators>
+    </test>
+
     <!-- Tests for copyFromLocal -->
     <test> <!-- TESTED -->
       <description>copyFromLocal: copying file into a file (absolute path)</description>