HBASE-10567 Add overwrite manifest option to ExportSnapshot

git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1570502 13f79535-47bb-0310-9956-ffa450edef68

hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java

@@ -189,7 +189,7 @@ public final class ExportSnapshot extends Configured implements Tool {
           // FLAKY-TEST-WARN: lower is better, we can get some runs without the
           // retry, but at least we reduce the number of test failures due to
           // this test exception from the same map task.
-          if (random.nextFloat() < 0.05) {
+          if (random.nextFloat() < 0.03) {
             throw new IOException("TEST RETRY FAILURE: Unable to copy input=" + inputPath
                                   + " time=" + System.currentTimeMillis());
           }
@@ -587,6 +587,7 @@ public final class ExportSnapshot extends Configured implements Tool {
   public int run(String[] args) throws Exception {
     boolean verifyChecksum = true;
     String snapshotName = null;
+    boolean overwrite = false;
     String filesGroup = null;
     String filesUser = null;
     Path outputRoot = null;
@@ -611,6 +612,8 @@ public final class ExportSnapshot extends Configured implements Tool {
           filesGroup = args[++i];
         } else if (cmd.equals("-chmod")) {
           filesMode = Integer.parseInt(args[++i], 8);
+        } else if (cmd.equals("-overwrite")) {
+          overwrite = true;
         } else if (cmd.equals("-h") || cmd.equals("--help")) {
           printUsageAndExit();
         } else {
@@ -644,13 +647,20 @@ public final class ExportSnapshot extends Configured implements Tool {
 
     // Check if the snapshot already exists
     if (outputFs.exists(outputSnapshotDir)) {
+      if (overwrite) {
+        if (!outputFs.delete(outputSnapshotDir, true)) {
+          System.err.println("Unable to remove existing snapshot directory: " + outputSnapshotDir);
+          return 1;
+        }
+      } else {
         System.err.println("The snapshot '" + snapshotName +
           "' already exists in the destination: " + outputSnapshotDir);
         return 1;
       }
+    }
 
     // Check if the snapshot already in-progress
-    if (outputFs.exists(snapshotTmpDir)) {
+    if (!overwrite && outputFs.exists(snapshotTmpDir)) {
       System.err.println("A snapshot with the same name '" + snapshotName + "' may be in-progress");
       System.err.println("Please check " + snapshotTmpDir + ". If the snapshot has completed, ");
       System.err.println("consider removing " + snapshotTmpDir + " before retrying export");
@@ -664,7 +674,7 @@ public final class ExportSnapshot extends Configured implements Tool {
     // The snapshot references must be copied before the hfiles otherwise the cleaner
     // will remove them because they are unreferenced.
     try {
-      FileUtil.copy(inputFs, snapshotDir, outputFs, snapshotTmpDir, false, false, conf);
+      FileUtil.copy(inputFs, snapshotDir, outputFs, snapshotTmpDir, false, overwrite, conf);
     } catch (IOException e) {
       System.err.println("Failed to copy the snapshot directory: from=" + snapshotDir +
         " to=" + snapshotTmpDir);
@@ -710,6 +720,7 @@ public final class ExportSnapshot extends Configured implements Tool {
     System.err.println("  -snapshot NAME          Snapshot to restore.");
     System.err.println("  -copy-to NAME           Remote destination hdfs://");
     System.err.println("  -no-checksum-verify     Do not verify checksum.");
+    System.err.println("  -overwrite              Rewrite the snapshot manifest if already exists");
     System.err.println("  -chuser USERNAME        Change the owner of the files " +
         "to the specified one.");
     System.err.println("  -chgroup GROUP          Change the group of the files to " +
@@ -720,8 +731,8 @@ public final class ExportSnapshot extends Configured implements Tool {
         "copy (mapreduce.job.maps).");
     System.err.println();
     System.err.println("Examples:");
-    System.err.println("  hbase " + getClass() + " \\");
+    System.err.println("  hbase " + getClass().getName() + " \\");
-    System.err.println("    -snapshot MySnapshot -copy-to hdfs:///srv2:8082/hbase \\");
+    System.err.println("    -snapshot MySnapshot -copy-to hdfs://srv2:8082/hbase \\");
     System.err.println("    -chuser MyUser -chgroup MyGroup -chmod 700 -mappers 16");
     System.exit(1);
   }

hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java

@@ -82,6 +82,8 @@ public class TestExportSnapshot {
     TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
     TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
     TEST_UTIL.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", true);
+    TEST_UTIL.getConfiguration().setInt("mapreduce.map.max.attempts", 10);
+    TEST_UTIL.getConfiguration().setInt("mapred.map.max.attempts", 10);
     TEST_UTIL.startMiniCluster(3);
     TEST_UTIL.startMiniMapReduceCluster();
   }
@@ -176,6 +178,14 @@ public class TestExportSnapshot {
     testExportFileSystemState(tableName, emptySnapshotName, 1);
   }
 
+  @Test
+  public void testConsecutiveExports() throws Exception {
+    Path copyDir = TEST_UTIL.getDataTestDir("export-" + System.currentTimeMillis());
+    testExportFileSystemState(tableName, snapshotName, 2, copyDir, false);
+    testExportFileSystemState(tableName, snapshotName, 2, copyDir, true);
+    removeExportDir(copyDir);
+  }
+
   /**
    * Mock a snapshot with files in the archive dir,
    * two regions, and one reference file.
@@ -227,21 +237,32 @@ public class TestExportSnapshot {
     testExportFileSystemState(tableWithRefsName, Bytes.toBytes(snapshotName), 2);
   }
 
+  private void testExportFileSystemState(final TableName tableName, final byte[] snapshotName,
+      int filesExpected) throws Exception {
+    Path copyDir = TEST_UTIL.getDataTestDir("export-" + System.currentTimeMillis());
+    testExportFileSystemState(tableName, snapshotName, filesExpected, copyDir, false);
+    removeExportDir(copyDir);
+  }
+
   /**
    * Test ExportSnapshot
    */
   private void testExportFileSystemState(final TableName tableName, final byte[] snapshotName,
-      int filesExpected) throws Exception {
+      int filesExpected, Path copyDir, boolean overwrite) throws Exception {
-    Path copyDir = TEST_UTIL.getDataTestDir("export-" + System.currentTimeMillis());
     URI hdfsUri = FileSystem.get(TEST_UTIL.getConfiguration()).getUri();
     FileSystem fs = FileSystem.get(copyDir.toUri(), new Configuration());
     copyDir = copyDir.makeQualified(fs);
 
+    List<String> opts = new ArrayList<String>();
+    opts.add("-snapshot");
+    opts.add(Bytes.toString(snapshotName));
+    opts.add("-copy-to");
+    opts.add(copyDir.toString());
+    if (overwrite) opts.add("-overwrite");
+
     // Export Snapshot
-    int res = ExportSnapshot.innerMain(TEST_UTIL.getConfiguration(), new String[] {
+    int res = ExportSnapshot.innerMain(TEST_UTIL.getConfiguration(),
-      "-snapshot", Bytes.toString(snapshotName),
+        opts.toArray(new String[opts.size()]));
-      "-copy-to", copyDir.toString()
-    });
     assertEquals(0, res);
 
     // Verify File-System state
@@ -261,9 +282,6 @@ public class TestExportSnapshot {
         fs, new Path(copyDir, snapshotDir));
     verifyArchive(fs, copyDir, tableName, Bytes.toString(snapshotName));
     FSUtils.logFileSystemState(hdfs, snapshotDir, LOG);
-
-    // Remove the exported dir
-    fs.delete(copyDir, true);
   }
 
   /**
@@ -366,4 +384,10 @@ public class TestExportSnapshot {
     }
     return files;
   }
+
+  private void removeExportDir(final Path path) throws IOException {
+    FileSystem fs = FileSystem.get(path.toUri(), new Configuration());
+    FSUtils.logFileSystemState(fs, path, LOG);
+    fs.delete(path, true);
+  }
 }