HADOOP-9338. FsShell Copy Commands Should Optionally Preserve File Attributes. Contributed by Nick White.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1477717 13f79535-47bb-0310-9956-ffa450edef68

hadoop-common-project/hadoop-common/CHANGES.txt

@@ -13,6 +13,9 @@ Release 2.0.5-beta - UNRELEASED
     HADOOP-8415. Add getDouble() and setDouble() in 
     org.apache.hadoop.conf.Configuration (Jan van der Lugt via harsh)
 
+    HADOOP-9338. FsShell Copy Commands Should Optionally Preserve File
+    Attributes. (Nick White via atm)
+
   IMPROVEMENTS
 
     HADOOP-9253. Capture ulimit info in the logs at service start time.

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java

@@ -610,6 +610,28 @@ public class RawLocalFileSystem extends FileSystem {
     }
   }
 
+  /**
+   * Sets the {@link Path}'s last modified time <em>only</em> to the given
+   * valid time.
+   *
+   * @param mtime the modification time to set (only if greater than zero).
+   * @param atime currently ignored.
+   * @throws IOException if setting the last modified time fails.
+   */
+  @Override
+  public void setTimes(Path p, long mtime, long atime) throws IOException {
+    File f = pathToFile(p);
+    if(mtime >= 0) {
+      if(!f.setLastModified(mtime)) {
+        throw new IOException(
+          "couldn't set last-modified time to " +
+          mtime +
+          " for " +
+          f.getAbsolutePath());
+      }
+    }
+  }
+
   private static String execCommand(File f, String... cmd) throws IOException {
     String[] args = new String[cmd.length + 1];
     System.arraycopy(cmd, 0, args, 0, cmd.length);

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java

@@ -44,6 +44,7 @@ import org.apache.hadoop.io.IOUtils;
 abstract class CommandWithDestination extends FsCommand {  
   protected PathData dst;
   private boolean overwrite = false;
+  private boolean preserve = false;
   private boolean verifyChecksum = true;
   private boolean writeChecksum = true;
   
@@ -65,6 +66,16 @@ abstract class CommandWithDestination extends FsCommand {
     writeChecksum = flag;
   }
   
+  /**
+   * If true, the last modified time, last access time,
+   * owner, group and permission information of the source
+   * file will be preserved as far as target {@link FileSystem}
+   * implementation allows.
+   */
+  protected void setPreserve(boolean preserve) {
+    this.preserve = preserve;
+  }
+
   /**
    *  The last arg is expected to be a local path, if only one argument is
    *  given then the destination will be the current directory 
@@ -219,6 +230,19 @@ abstract class CommandWithDestination extends FsCommand {
   protected void copyFileToTarget(PathData src, PathData target) throws IOException {
     src.fs.setVerifyChecksum(verifyChecksum);
     copyStreamToTarget(src.fs.open(src.path), target);
+    if(preserve) {
+      target.fs.setTimes(
+        target.path,
+        src.stat.getModificationTime(),
+        src.stat.getAccessTime());
+      target.fs.setOwner(
+        target.path,
+        src.stat.getOwner(),
+        src.stat.getGroup());
+      target.fs.setPermission(
+        target.path,
+        src.stat.getPermission());
+  }
   }
   
   /**

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java

@@ -123,17 +123,19 @@ class CopyCommands {
 
   static class Cp extends CommandWithDestination {
     public static final String NAME = "cp";
-    public static final String USAGE = "<src> ... <dst>";
+    public static final String USAGE = "[-f] [-p] <src> ... <dst>";
     public static final String DESCRIPTION =
       "Copy files that match the file pattern <src> to a\n" +
       "destination.  When copying multiple files, the destination\n" +
-      "must be a directory.";
+      "must be a directory. Passing -p preserves access and\n" +
+      "modification times, ownership and the mode.\n";
     
     @Override
     protected void processOptions(LinkedList<String> args) throws IOException {
-      CommandFormat cf = new CommandFormat(2, Integer.MAX_VALUE, "f");
+      CommandFormat cf = new CommandFormat(2, Integer.MAX_VALUE, "f", "p");
       cf.parse(args);
       setOverwrite(cf.getOpt("f"));
+      setPreserve(cf.getOpt("p"));
       // should have a -r option
       setRecursive(true);
       getRemoteDestination(args);
@@ -146,20 +148,23 @@ class CopyCommands {
   public static class Get extends CommandWithDestination {
     public static final String NAME = "get";
     public static final String USAGE =
-      "[-ignoreCrc] [-crc] <src> ... <localdst>";
+      "[-p] [-ignoreCrc] [-crc] <src> ... <localdst>";
     public static final String DESCRIPTION =
       "Copy files that match the file pattern <src>\n" +
       "to the local name.  <src> is kept.  When copying multiple,\n" +
-      "files, the destination must be a directory.";
+      "files, the destination must be a directory. Passing\n" +
+      "-p preserves access and modification times,\n" +
+      "ownership and the mode.\n";
 
     @Override
     protected void processOptions(LinkedList<String> args)
     throws IOException {
       CommandFormat cf = new CommandFormat(
-          1, Integer.MAX_VALUE, "crc", "ignoreCrc");
+          1, Integer.MAX_VALUE, "crc", "ignoreCrc", "p");
       cf.parse(args);
       setWriteChecksum(cf.getOpt("crc"));
       setVerifyChecksum(!cf.getOpt("ignoreCrc"));
+      setPreserve(cf.getOpt("p"));
       setRecursive(true);
       getLocalDestination(args);
     }
@@ -170,16 +175,20 @@ class CopyCommands {
    */
   public static class Put extends CommandWithDestination {
     public static final String NAME = "put";
-    public static final String USAGE = "<localsrc> ... <dst>";
+    public static final String USAGE = "[-f] [-p] <localsrc> ... <dst>";
     public static final String DESCRIPTION =
       "Copy files from the local file system\n" +
-      "into fs.";
+      "into fs. Copying fails if the file already\n" +
+      "exists, unless the -f flag is given. Passing\n" +
+      "-p preserves access and modification times,\n" +
+      "ownership and the mode.\n";
 
     @Override
     protected void processOptions(LinkedList<String> args) throws IOException {
-      CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE, "f");
+      CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE, "f", "p");
       cf.parse(args);
       setOverwrite(cf.getOpt("f"));
+      setPreserve(cf.getOpt("p"));
       getRemoteDestination(args);
       // should have a -r option
       setRecursive(true);

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java

@@ -49,7 +49,8 @@ public class TestLocalFileSystem {
   
   @Before
   public void setup() throws IOException {
-    conf = new Configuration();
+    conf = new Configuration(false);
+    conf.set("fs.file.impl", LocalFileSystem.class.getName());
     fileSys = FileSystem.getLocal(conf);
     fileSys.delete(new Path(TEST_ROOT_DIR), true);
   }
@@ -64,7 +65,7 @@ public class TestLocalFileSystem {
   /**
    * Test the capability of setting the working directory.
    */
-  @Test
+  @Test(timeout = 1000)
   public void testWorkingDirectory() throws IOException {
     Path origDir = fileSys.getWorkingDirectory();
     Path subdir = new Path(TEST_ROOT_DIR, "new");
@@ -118,10 +119,9 @@ public class TestLocalFileSystem {
    * test Syncable interface on raw local file system
    * @throws IOException
    */
-  @Test
+  @Test(timeout = 1000)
   public void testSyncable() throws IOException {
-    Configuration conf = new Configuration();
+    FileSystem fs = fileSys.getRawFileSystem();
-    FileSystem fs = FileSystem.getLocal(conf).getRawFileSystem();
     Path file = new Path(TEST_ROOT_DIR, "syncable");
     FSDataOutputStream out = fs.create(file);;
     final int bytesWritten = 1;
@@ -152,76 +152,68 @@ public class TestLocalFileSystem {
     }
   }
   
-  @Test
+  @Test(timeout = 1000)
   public void testCopy() throws IOException {
-    Configuration conf = new Configuration();
-    LocalFileSystem fs = FileSystem.getLocal(conf);
     Path src = new Path(TEST_ROOT_DIR, "dingo");
     Path dst = new Path(TEST_ROOT_DIR, "yak");
-    writeFile(fs, src, 1);
+    writeFile(fileSys, src, 1);
-    assertTrue(FileUtil.copy(fs, src, fs, dst, true, false, conf));
+    assertTrue(FileUtil.copy(fileSys, src, fileSys, dst, true, false, conf));
-    assertTrue(!fs.exists(src) && fs.exists(dst));
+    assertTrue(!fileSys.exists(src) && fileSys.exists(dst));
-    assertTrue(FileUtil.copy(fs, dst, fs, src, false, false, conf));
+    assertTrue(FileUtil.copy(fileSys, dst, fileSys, src, false, false, conf));
-    assertTrue(fs.exists(src) && fs.exists(dst));
+    assertTrue(fileSys.exists(src) && fileSys.exists(dst));
-    assertTrue(FileUtil.copy(fs, src, fs, dst, true, true, conf));
+    assertTrue(FileUtil.copy(fileSys, src, fileSys, dst, true, true, conf));
-    assertTrue(!fs.exists(src) && fs.exists(dst));
+    assertTrue(!fileSys.exists(src) && fileSys.exists(dst));
-    fs.mkdirs(src);
+    fileSys.mkdirs(src);
-    assertTrue(FileUtil.copy(fs, dst, fs, src, false, false, conf));
+    assertTrue(FileUtil.copy(fileSys, dst, fileSys, src, false, false, conf));
     Path tmp = new Path(src, dst.getName());
-    assertTrue(fs.exists(tmp) && fs.exists(dst));
+    assertTrue(fileSys.exists(tmp) && fileSys.exists(dst));
-    assertTrue(FileUtil.copy(fs, dst, fs, src, false, true, conf));
+    assertTrue(FileUtil.copy(fileSys, dst, fileSys, src, false, true, conf));
-    assertTrue(fs.delete(tmp, true));
+    assertTrue(fileSys.delete(tmp, true));
-    fs.mkdirs(tmp);
+    fileSys.mkdirs(tmp);
     try {
-      FileUtil.copy(fs, dst, fs, src, true, true, conf);
+      FileUtil.copy(fileSys, dst, fileSys, src, true, true, conf);
       fail("Failed to detect existing dir");
     } catch (IOException e) {
       // Expected
     }
   }
 
-  @Test
+  @Test(timeout = 1000)
   public void testHomeDirectory() throws IOException {
-    Configuration conf = new Configuration();
-    FileSystem fileSys = FileSystem.getLocal(conf);
     Path home = new Path(System.getProperty("user.home"))
       .makeQualified(fileSys);
     Path fsHome = fileSys.getHomeDirectory();
     assertEquals(home, fsHome);
   }
 
-  @Test
+  @Test(timeout = 1000)
   public void testPathEscapes() throws IOException {
-    Configuration conf = new Configuration();
-    FileSystem fs = FileSystem.getLocal(conf);
     Path path = new Path(TEST_ROOT_DIR, "foo%bar");
-    writeFile(fs, path, 1);
+    writeFile(fileSys, path, 1);
-    FileStatus status = fs.getFileStatus(path);
+    FileStatus status = fileSys.getFileStatus(path);
-    assertEquals(path.makeQualified(fs), status.getPath());
+    assertEquals(path.makeQualified(fileSys), status.getPath());
-    cleanupFile(fs, path);
+    cleanupFile(fileSys, path);
   }
   
-  @Test
+  @Test(timeout = 1000)
   public void testMkdirs() throws IOException {
-    Configuration conf = new Configuration();
-    LocalFileSystem fs = FileSystem.getLocal(conf);
     Path test_dir = new Path(TEST_ROOT_DIR, "test_dir");
     Path test_file = new Path(TEST_ROOT_DIR, "file1");
-    assertTrue(fs.mkdirs(test_dir));
+    assertTrue(fileSys.mkdirs(test_dir));
    
-    writeFile(fs, test_file, 1);
+    writeFile(fileSys, test_file, 1);
     // creating dir over a file
     Path bad_dir = new Path(test_file, "another_dir");
     
     try {
-      fs.mkdirs(bad_dir);
+      fileSys.mkdirs(bad_dir);
       fail("Failed to detect existing file in path");
     } catch (FileAlreadyExistsException e) { 
       // Expected
     }
     
     try {
-      fs.mkdirs(null);
+        fileSys.mkdirs(null);
       fail("Failed to detect null in mkdir arg");
     } catch (IllegalArgumentException e) {
       // Expected
@@ -229,26 +221,23 @@ public class TestLocalFileSystem {
   }
 
   /** Test deleting a file, directory, and non-existent path */
-  @Test
+  @Test(timeout = 1000)
   public void testBasicDelete() throws IOException {
-    Configuration conf = new Configuration();
-    LocalFileSystem fs = FileSystem.getLocal(conf);
     Path dir1 = new Path(TEST_ROOT_DIR, "dir1");
     Path file1 = new Path(TEST_ROOT_DIR, "file1");
     Path file2 = new Path(TEST_ROOT_DIR+"/dir1", "file2");
     Path file3 = new Path(TEST_ROOT_DIR, "does-not-exist");
-    assertTrue(fs.mkdirs(dir1));
+    assertTrue(fileSys.mkdirs(dir1));
-    writeFile(fs, file1, 1);
+    writeFile(fileSys, file1, 1);
-    writeFile(fs, file2, 1);
+    writeFile(fileSys, file2, 1);
     assertFalse("Returned true deleting non-existant path", 
-        fs.delete(file3));
+            fileSys.delete(file3));
-    assertTrue("Did not delete file", fs.delete(file1));
+    assertTrue("Did not delete file", fileSys.delete(file1));
-    assertTrue("Did not delete non-empty dir", fs.delete(dir1));
+    assertTrue("Did not delete non-empty dir", fileSys.delete(dir1));
   }
   
-  @Test
+  @Test(timeout = 1000)
   public void testStatistics() throws Exception {
-    FileSystem.getLocal(new Configuration());
     int fileSchemeCount = 0;
     for (Statistics stats : FileSystem.getAllStatistics()) {
       if (stats.getScheme().equals("file")) {
@@ -258,30 +247,27 @@ public class TestLocalFileSystem {
     assertEquals(1, fileSchemeCount);
   }
 
-  @Test
+  @Test(timeout = 1000)
   public void testHasFileDescriptor() throws IOException {
-    Configuration conf = new Configuration();
-    LocalFileSystem fs = FileSystem.getLocal(conf);
     Path path = new Path(TEST_ROOT_DIR, "test-file");
-    writeFile(fs, path, 1);
+    writeFile(fileSys, path, 1);
     BufferedFSInputStream bis = new BufferedFSInputStream(
         new RawLocalFileSystem().new LocalFSFileInputStream(path), 1024);
     assertNotNull(bis.getFileDescriptor());
+    bis.close();
   }
 
-  @Test
+  @Test(timeout = 1000)
   public void testListStatusWithColons() throws IOException {
-    Configuration conf = new Configuration();
-    LocalFileSystem fs = FileSystem.getLocal(conf);
     File colonFile = new File(TEST_ROOT_DIR, "foo:bar");
     colonFile.mkdirs();
-    FileStatus[] stats = fs.listStatus(new Path(TEST_ROOT_DIR));
+    FileStatus[] stats = fileSys.listStatus(new Path(TEST_ROOT_DIR));
     assertEquals("Unexpected number of stats", 1, stats.length);
     assertEquals("Bad path from stat", colonFile.getAbsolutePath(),
         stats[0].getPath().toUri().getPath());
   }
   
-  @Test
+  @Test(timeout = 1000)
   public void testReportChecksumFailure() throws IOException {
     base.mkdirs();
     assertTrue(base.exists() && base.isDirectory());
@@ -354,4 +340,23 @@ public class TestLocalFileSystem {
     assertTrue(checksumFileFound);
   }
   
+  @Test(timeout = 1000)
+  public void testSetTimes() throws Exception {
+    Path path = new Path(TEST_ROOT_DIR, "set-times");
+    writeFile(fileSys, path, 1);
+
+    // test only to the nearest second, as the raw FS may not
+    // support millisecond timestamps
+    long newModTime = 12345000;
+
+    FileStatus status = fileSys.getFileStatus(path);
+    assertTrue("check we're actually changing something", newModTime != status.getModificationTime());
+    assertEquals(0, status.getAccessTime());
+
+    fileSys.setTimes(path, newModTime, -1);
+    status = fileSys.getFileStatus(path);
+    assertEquals(newModTime, status.getModificationTime());
+    assertEquals(0, status.getAccessTime());
+}
+
 }

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopyPreserveFlag.java

@@ -0,0 +1,135 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.shell;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.fs.shell.CopyCommands.Cp;
+import org.apache.hadoop.fs.shell.CopyCommands.Get;
+import org.apache.hadoop.fs.shell.CopyCommands.Put;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestCopyPreserveFlag {
+  private static final int MODIFICATION_TIME = 12345000;
+  private static final Path FROM = new Path("d1", "f1");
+  private static final Path TO = new Path("d2", "f2");
+  private static final FsPermission PERMISSIONS = new FsPermission(
+    FsAction.ALL,
+    FsAction.EXECUTE,
+    FsAction.READ_WRITE);
+
+  private FileSystem fs;
+  private Path testDir;
+  private Configuration conf;
+
+  @Before
+  public void initialize() throws Exception {
+    conf = new Configuration(false);
+    conf.set("fs.file.impl", LocalFileSystem.class.getName());
+    fs = FileSystem.getLocal(conf);
+    testDir = new Path(
+        System.getProperty("test.build.data", "build/test/data") + "/testStat"
+    );
+    // don't want scheme on the path, just an absolute path
+    testDir = new Path(fs.makeQualified(testDir).toUri().getPath());
+
+    FileSystem.setDefaultUri(conf, fs.getUri());
+    fs.setWorkingDirectory(testDir);
+    fs.mkdirs(new Path("d1"));
+    fs.mkdirs(new Path("d2"));
+    fs.createNewFile(FROM);
+
+    FSDataOutputStream output = fs.create(FROM, true);
+    for(int i = 0; i < 100; ++i) {
+        output.writeInt(i);
+        output.writeChar('\n');
+    }
+    output.close();
+    fs.setTimes(FROM, MODIFICATION_TIME, 0);
+    fs.setPermission(FROM, PERMISSIONS);
+  }
+
+  @After
+  public void cleanup() throws Exception {
+    fs.delete(testDir, true);
+    fs.close();
+  }
+
+  private void assertAttributesPreserved() throws IOException {
+    assertEquals(MODIFICATION_TIME, fs.getFileStatus(TO).getModificationTime());
+    assertEquals(PERMISSIONS, fs.getFileStatus(TO).getPermission());
+  }
+
+  private void assertAttributesChanged() throws IOException {
+      assertTrue(MODIFICATION_TIME != fs.getFileStatus(TO).getModificationTime());
+      assertTrue(!PERMISSIONS.equals(fs.getFileStatus(TO).getPermission()));
+  }
+
+  private void run(CommandWithDestination cmd, String... args) {
+    cmd.setConf(conf);
+    assertEquals(0, cmd.run(args));
+  }
+
+  @Test(timeout = 1000)
+  public void testPutWithP() throws Exception {
+    run(new Put(), "-p", FROM.toString(), TO.toString());
+    assertAttributesPreserved();
+  }
+
+  @Test(timeout = 1000)
+  public void testPutWithoutP() throws Exception {
+    run(new Put(), FROM.toString(), TO.toString());
+    assertAttributesChanged();
+  }
+
+  @Test(timeout = 1000)
+  public void testGetWithP() throws Exception {
+    run(new Get(), "-p", FROM.toString(), TO.toString());
+    assertAttributesPreserved();
+  }
+
+  @Test(timeout = 1000)
+  public void testGetWithoutP() throws Exception {
+    run(new Get(), FROM.toString(), TO.toString());
+    assertAttributesChanged();
+  }
+
+  @Test(timeout = 1000)
+  public void testCpWithP() throws Exception {
+      run(new Cp(), "-p", FROM.toString(), TO.toString());
+      assertAttributesPreserved();
+  }
+
+  @Test(timeout = 1000)
+  public void testCpWithoutP() throws Exception {
+      run(new Cp(), FROM.toString(), TO.toString());
+      assertAttributesChanged();
+  }
+}

hadoop-common-project/hadoop-common/src/test/resources/testConf.xml

@@ -133,7 +133,7 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-get( )*\[-ignoreCrc\]( )*\[-crc\]( )*&lt;src&gt; \.\.\. &lt;localdst&gt;:( |\t)*Copy files that match the file pattern &lt;src&gt;( )*</expected-output>
+          <expected-output>^-get( )*\[-p\]( )*\[-ignoreCrc\]( )*\[-crc\]( )*&lt;src&gt; \.\.\. &lt;localdst&gt;:( |\t)*Copy files that match the file pattern &lt;src&gt;( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
@@ -141,7 +141,15 @@
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*files, the destination must be a directory.( )*</expected-output>
+          <expected-output>^( |\t)*files, the destination must be a directory.( )*Passing( )*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^( |\t)*-p preserves access and modification times,( )*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^( |\t)*ownership and the mode.( )*</expected-output>
         </comparator>
       </comparators>
     </test>
@@ -276,7 +284,7 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-cp &lt;src&gt; \.\.\. &lt;dst&gt;:( |\t)*Copy files that match the file pattern &lt;src&gt; to a( )*</expected-output>
+          <expected-output>^-cp \[-f\] \[-p\] &lt;src&gt; \.\.\. &lt;dst&gt;:( |\t)*Copy files that match the file pattern &lt;src&gt; to a( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
@@ -284,7 +292,11 @@
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*must be a directory.( )*</expected-output>
+          <expected-output>^( |\t)*must be a directory.( )*Passing -p preserves access and( )*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^( |\t)*modification times, ownership and the mode.( )*</expected-output>
         </comparator>
       </comparators>
     </test>
@@ -372,11 +384,23 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-put &lt;localsrc&gt; \.\.\. &lt;dst&gt;:\s+Copy files from the local file system</expected-output>
+          <expected-output>^-put \[-f\] \[-p\] &lt;localsrc&gt; \.\.\. &lt;dst&gt;:\s+Copy files from the local file system</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*into fs.( )*</expected-output>
+          <expected-output>^( |\t)*into fs.( )*Copying fails if the file already( )*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^( |\t)*exists, unless the -f flag is given.( )*Passing( )*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^( |\t)*-p preserves access and modification times,( )*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^( |\t)*ownership and the mode.( )*</expected-output>
         </comparator>
       </comparators>
     </test>
@@ -391,7 +415,7 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-copyFromLocal &lt;localsrc&gt; \.\.\. &lt;dst&gt;:\s+Identical to the -put command\.</expected-output>
+          <expected-output>^-copyFromLocal \[-f\] \[-p\] &lt;localsrc&gt; \.\.\. &lt;dst&gt;:\s+Identical to the -put command\.</expected-output>
         </comparator>
       </comparators>
     </test>
@@ -426,7 +450,7 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-get( )*\[-ignoreCrc\]( )*\[-crc\]( )*&lt;src&gt; \.\.\. &lt;localdst&gt;:( |\t)*Copy files that match the file pattern &lt;src&gt;( )*</expected-output>
+          <expected-output>^-get( )*\[-p\]( )*\[-ignoreCrc\]( )*\[-crc\]( )*&lt;src&gt; \.\.\. &lt;localdst&gt;:( |\t)*Copy files that match the file pattern &lt;src&gt;( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
@@ -434,7 +458,15 @@
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*files, the destination must be a directory.( )*</expected-output>
+          <expected-output>^( |\t)*files, the destination must be a directory.( )*Passing( )*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^( |\t)*-p preserves access and modification times,( )*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^( |\t)*ownership and the mode.( )*</expected-output>
         </comparator>
       </comparators>
     </test>
@@ -512,7 +544,7 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-copyToLocal \[-ignoreCrc\] \[-crc\] &lt;src&gt; \.\.\. &lt;localdst&gt;:\s+Identical to the -get command.</expected-output>
+          <expected-output>^-copyToLocal \[-p\] \[-ignoreCrc\] \[-crc\] &lt;src&gt; \.\.\. &lt;localdst&gt;:\s+Identical to the -get command.</expected-output>
         </comparator>
       </comparators>
     </test>