HBASE-11119 Update ExportSnapShot to optionally not use a tmp file on external file system (Ted Malaska)

git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1593337 13f79535-47bb-0310-9956-ffa450edef68

hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java

@@ -93,6 +93,7 @@ public final class ExportSnapshot extends Configured implements Tool {
   private static final String CONF_BUFFER_SIZE = "snapshot.export.buffer.size";
   private static final String CONF_MAP_GROUP = "snapshot.export.default.map.group";
   private static final String CONF_BANDWIDTH_MB = "snapshot.export.map.bandwidth.mb";
+  protected static final String CONF_SKIP_TMP = "snapshot.export.skip.tmp";
 
   static final String CONF_TEST_FAILURE = "test.snapshot.export.failure";
   static final String CONF_TEST_RETRY = "test.snapshot.export.failure.retry";
@@ -751,9 +752,12 @@ public final class ExportSnapshot extends Configured implements Tool {
     FileSystem outputFs = FileSystem.get(outputRoot.toUri(), conf);
     LOG.debug("outputFs=" + outputFs.getUri().toString() + " outputRoot=" + outputRoot.toString());
 
+    boolean skipTmp = conf.getBoolean(CONF_SKIP_TMP, false);
+
     Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, inputRoot);
     Path snapshotTmpDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshotName, outputRoot);
     Path outputSnapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, outputRoot);
+    Path initialOutputSnapshotDir = skipTmp ? outputSnapshotDir : snapshotTmpDir;
 
     // Check if the snapshot already exists
     if (outputFs.exists(outputSnapshotDir)) {
@@ -769,20 +773,22 @@ public final class ExportSnapshot extends Configured implements Tool {
       }
     }
 
+    if (!skipTmp) {
       // Check if the snapshot already in-progress
       if (outputFs.exists(snapshotTmpDir)) {
         if (overwrite) {
           if (!outputFs.delete(snapshotTmpDir, true)) {
-          System.err.println("Unable to remove existing snapshot tmp directory: " + snapshotTmpDir);
+            System.err.println("Unable to remove existing snapshot tmp directory: "+snapshotTmpDir);
             return 1;
           }
         } else {
-        System.err.println("A snapshot with the same name '"+ snapshotName +"' may be in-progress");
+          System.err.println("A snapshot with the same name '"+snapshotName+"' may be in-progress");
-        System.err.println("Please check " + snapshotTmpDir + ". If the snapshot has completed, ");
+          System.err.println("Please check "+snapshotTmpDir+". If the snapshot has completed, ");
-        System.err.println("consider removing "+ snapshotTmpDir +" by using the -overwrite option");
+          System.err.println("consider removing "+snapshotTmpDir+" by using the -overwrite option");
           return 1;
         }
       }
+    }
 
     // Step 0 - Extract snapshot files to copy
     LOG.info("Loading Snapshot hfile list");
@@ -797,10 +803,10 @@ public final class ExportSnapshot extends Configured implements Tool {
     // will remove them because they are unreferenced.
     try {
       LOG.info("Copy Snapshot Manifest");
-      FileUtil.copy(inputFs, snapshotDir, outputFs, snapshotTmpDir, false, false, conf);
+      FileUtil.copy(inputFs, snapshotDir, outputFs, initialOutputSnapshotDir, false, false, conf);
     } catch (IOException e) {
       throw new ExportSnapshotException("Failed to copy the snapshot directory: from=" +
-        snapshotDir + " to=" + snapshotTmpDir, e);
+        snapshotDir + " to=" + initialOutputSnapshotDir, e);
     }
 
     // Step 2 - Start MR Job to copy files
@@ -814,12 +820,15 @@ public final class ExportSnapshot extends Configured implements Tool {
                    filesUser, filesGroup, filesMode, mappers, bandwidthMB);
       }
 
-      // Step 3 - Rename fs2:/.snapshot/.tmp/<snapshot> fs2:/.snapshot/<snapshot>
+
       LOG.info("Finalize the Snapshot Export");
+      if (!skipTmp) {
+        // Step 3 - Rename fs2:/.snapshot/.tmp/<snapshot> fs2:/.snapshot/<snapshot>
         if (!outputFs.rename(snapshotTmpDir, outputSnapshotDir)) {
           throw new ExportSnapshotException("Unable to rename snapshot directory from=" +
             snapshotTmpDir + " to=" + outputSnapshotDir);
         }
+      }
 
       // Step 4 - Verify snapshot validity
       LOG.info("Verify snapshot validity");
@@ -829,7 +838,9 @@ public final class ExportSnapshot extends Configured implements Tool {
       return 0;
     } catch (Exception e) {
       LOG.error("Snapshot export failed", e);
+      if (!skipTmp) {
         outputFs.delete(snapshotTmpDir, true);
+      }
       outputFs.delete(outputSnapshotDir, true);
       return 1;
     }

hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java

@@ -190,6 +190,12 @@ public class TestExportSnapshot {
     testExportFileSystemState(tableName, snapshotName, 2);
   }
 
+  @Test
+  public void testExportFileSystemStateWithSkipTmp() throws Exception {
+    TEST_UTIL.getConfiguration().setBoolean(ExportSnapshot.CONF_SKIP_TMP, true);
+    testExportFileSystemState(tableName, snapshotName, 2);
+  }
+
   @Test
   public void testEmptyExportFileSystemState() throws Exception {
     testExportFileSystemState(tableName, emptySnapshotName, 1);