HBASE-16464 archive folder grows bigger and bigger due to corrupt snapshot under tmp dir

hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java

@@ -39,6 +39,7 @@ import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.Stoppable;
+import org.apache.hadoop.hbase.snapshot.CorruptedSnapshotException;
 import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
 import org.apache.hadoop.hbase.util.FSUtils;
 
@@ -300,7 +301,21 @@ public class SnapshotFileCache implements Stoppable {
     FileStatus[] running = FSUtils.listStatus(fs, snapshotTmpDir);
     if (running != null) {
       for (FileStatus run : running) {
+        try {
           snapshotInProgress.addAll(fileInspector.filesUnderSnapshot(run.getPath()));
+        } catch (CorruptedSnapshotException e) {
+          // See HBASE-16464
+          if (e.getCause() instanceof FileNotFoundException) {
+            // If the snapshot is not in progress, we will delete it
+            if (!fs.exists(new Path(run.getPath(),
+              SnapshotDescriptionUtils.SNAPSHOT_IN_PROGRESS))) {
+              fs.delete(run.getPath(), true);
+              LOG.warn("delete the " + run.getPath() + " due to exception:", e.getCause());
+            }
+          } else {
+            throw e;
+          }
+        }
       }
     }
     return snapshotInProgress;

hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java

@@ -164,7 +164,7 @@ public abstract class TakeSnapshotHandler extends EventHandler implements Snapsh
     try {
       // If regions move after this meta scan, the region specific snapshot should fail, triggering
       // an external exception that gets captured here.
-
+      SnapshotDescriptionUtils.createInProgressTag(workingDir, fs);
       // write down the snapshot info in the working directory
       SnapshotDescriptionUtils.writeSnapshotInfo(snapshot, workingDir, fs);
       snapshotManifest.addTableDescriptor(this.htd);

hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java

@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.hbase.snapshot;
 
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.util.Collections;
 
@@ -104,6 +105,9 @@ public final class SnapshotDescriptionUtils {
 
   /** Temporary directory under the snapshot directory to store in-progress snapshots */
   public static final String SNAPSHOT_TMP_DIR_NAME = ".tmp";
+
+  /** This tag will be created in in-progess snapshots */
+  public static final String SNAPSHOT_IN_PROGRESS = ".inprogress";
   // snapshot operation values
   /** Default value if no start time is specified */
   public static final long NO_SNAPSHOT_START_TIME_SPECIFIED = 0;
@@ -292,6 +296,16 @@ public final class SnapshotDescriptionUtils {
     }
   }
 
+  /**
+   * Create in-progress tag under .tmp of in-progress snapshot
+   * */
+  public static void createInProgressTag(Path workingDir, FileSystem fs) throws IOException {
+    FsPermission perms = FSUtils.getFilePermissions(fs, fs.getConf(),
+      HConstants.DATA_FILE_UMASK_KEY);
+    Path snapshot_in_progress = new Path(workingDir, SnapshotDescriptionUtils.SNAPSHOT_IN_PROGRESS);
+    FSUtils.create(fs, snapshot_in_progress, perms, true);
+  }
+
   /**
    * Read in the {@link org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription} stored for the snapshot in the passed directory
    * @param fs filesystem where the snapshot was taken

hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV2.java

@@ -82,6 +82,9 @@ public final class SnapshotManifestV2 {
     }
 
     public void regionClose(final SnapshotRegionManifest.Builder region) throws IOException {
+      // we should ensure the snapshot dir exist, maybe it has been deleted by master
+      // see HBASE-16464
+      if (fs.exists(snapshotDir)) {
         SnapshotRegionManifest manifest = region.build();
         FSDataOutputStream stream = fs.create(getRegionManifestPath(snapshotDir, manifest));
         try {
@@ -89,6 +92,9 @@ public final class SnapshotManifestV2 {
         } finally {
           stream.close();
         }
+      } else {
+        LOG.warn("can't write manifest without parent dir, maybe it has been deleted by master?");
+      }
     }
 
     public SnapshotRegionManifest.FamilyFiles.Builder familyOpen(

hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotHFileCleaner.java

@@ -168,4 +168,24 @@ public class TestSnapshotHFileCleaner {
       fs.delete(SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir), true);
     }
   }
+
+
+  /**
+   * HBASE-16464
+   */
+  @Test
+  public void testMissedTmpSnapshot() throws IOException {
+    SnapshotTestingUtils.SnapshotMock
+      snapshotMock = new SnapshotTestingUtils.SnapshotMock(TEST_UTIL.getConfiguration(), fs, rootDir);
+    SnapshotTestingUtils.SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2(
+      SNAPSHOT_NAME_STR, TABLE_NAME_STR);
+    builder.addRegionV2();
+    builder.missOneRegionSnapshotFile();
+
+    long period = Long.MAX_VALUE;
+    SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
+      "test-snapshot-file-cache-refresh", new SnapshotFiles());
+    cache.getSnapshotsInProgress();
+    assertFalse(fs.exists(builder.getSnapshotsDir()));
+  }
 }

hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java

@@ -583,6 +583,18 @@ public final class SnapshotTestingUtils {
         corruptFile(p);
       }
 
+      public void missOneRegionSnapshotFile() throws IOException {
+        FileStatus[] manifestFiles = FSUtils.listStatus(fs, snapshotDir);
+        for (FileStatus fileStatus : manifestFiles) {
+          String fileName = fileStatus.getPath().getName();
+          if (fileName.endsWith(SnapshotDescriptionUtils.SNAPSHOTINFO_FILE)
+            || fileName.endsWith(".tabledesc")
+            || fileName.endsWith(SnapshotDescriptionUtils.SNAPSHOT_TMP_DIR_NAME)) {
+              fs.delete(fileStatus.getPath(), true);
+          }
+        }
+      }
+
       /**
        * Corrupt data-manifest file
        *