Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-16464 archive folder grows bigger and bigger due to corrupt snapshot under tmp dir

This commit is contained in:
chenheng 2016-08-22 16:45:20 +08:00
parent 97b164ac38
commit 3909b7c96f
6 changed files with 75 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
View File

@ -39,6 +39,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Stoppable;
import org.apache.hadoop.hbase.snapshot.CorruptedSnapshotException;
import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
import org.apache.hadoop.hbase.util.FSUtils;
@ -300,7 +301,21 @@ public class SnapshotFileCache implements Stoppable {
FileStatus[] running = FSUtils.listStatus(fs, snapshotTmpDir);
if (running != null) {
for (FileStatus run : running) {
snapshotInProgress.addAll(fileInspector.filesUnderSnapshot(run.getPath()));
try {
snapshotInProgress.addAll(fileInspector.filesUnderSnapshot(run.getPath()));
} catch (CorruptedSnapshotException e) {
// See HBASE-16464
if (e.getCause() instanceof FileNotFoundException) {
// If the snapshot is not in progress, we will delete it
if (!fs.exists(new Path(run.getPath(),
SnapshotDescriptionUtils.SNAPSHOT_IN_PROGRESS))) {
fs.delete(run.getPath(), true);
LOG.warn("delete the " + run.getPath() + " due to exception:", e.getCause());
}
} else {
throw e;
}
}
}
}
return snapshotInProgress;

2
hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
View File

@ -164,7 +164,7 @@ public abstract class TakeSnapshotHandler extends EventHandler implements Snapsh
try {
// If regions move after this meta scan, the region specific snapshot should fail, triggering
// an external exception that gets captured here.
SnapshotDescriptionUtils.createInProgressTag(workingDir, fs);
// write down the snapshot info in the working directory
SnapshotDescriptionUtils.writeSnapshotInfo(snapshot, workingDir, fs);
snapshotManifest.addTableDescriptor(this.htd);

14
hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java
View File

@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hbase.snapshot;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.Collections;
@ -104,6 +105,9 @@ public final class SnapshotDescriptionUtils {
/** Temporary directory under the snapshot directory to store in-progress snapshots */
public static final String SNAPSHOT_TMP_DIR_NAME = ".tmp";
/** This tag will be created in in-progess snapshots */
public static final String SNAPSHOT_IN_PROGRESS = ".inprogress";
// snapshot operation values
/** Default value if no start time is specified */
public static final long NO_SNAPSHOT_START_TIME_SPECIFIED = 0;
@ -292,6 +296,16 @@ public final class SnapshotDescriptionUtils {
}
}
/**
* Create in-progress tag under .tmp of in-progress snapshot
* */
public static void createInProgressTag(Path workingDir, FileSystem fs) throws IOException {
FsPermission perms = FSUtils.getFilePermissions(fs, fs.getConf(),
HConstants.DATA_FILE_UMASK_KEY);
Path snapshot_in_progress = new Path(workingDir, SnapshotDescriptionUtils.SNAPSHOT_IN_PROGRESS);
FSUtils.create(fs, snapshot_in_progress, perms, true);
}
/**
* Read in the {@link org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription} stored for the snapshot in the passed directory
* @param fs filesystem where the snapshot was taken

18
hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV2.java
View File

@ -82,12 +82,18 @@ public final class SnapshotManifestV2 {
}
public void regionClose(final SnapshotRegionManifest.Builder region) throws IOException {
SnapshotRegionManifest manifest = region.build();
FSDataOutputStream stream = fs.create(getRegionManifestPath(snapshotDir, manifest));
try {
manifest.writeTo(stream);
} finally {
stream.close();
// we should ensure the snapshot dir exist, maybe it has been deleted by master
// see HBASE-16464
if (fs.exists(snapshotDir)) {
SnapshotRegionManifest manifest = region.build();
FSDataOutputStream stream = fs.create(getRegionManifestPath(snapshotDir, manifest));
try {
manifest.writeTo(stream);
} finally {
stream.close();
}
} else {
LOG.warn("can't write manifest without parent dir, maybe it has been deleted by master?");
}
}

20
hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotHFileCleaner.java
View File

@ -168,4 +168,24 @@ public class TestSnapshotHFileCleaner {
fs.delete(SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir), true);
}
}
/**
* HBASE-16464
*/
@Test
public void testMissedTmpSnapshot() throws IOException {
SnapshotTestingUtils.SnapshotMock
snapshotMock = new SnapshotTestingUtils.SnapshotMock(TEST_UTIL.getConfiguration(), fs, rootDir);
SnapshotTestingUtils.SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2(
SNAPSHOT_NAME_STR, TABLE_NAME_STR);
builder.addRegionV2();
builder.missOneRegionSnapshotFile();
long period = Long.MAX_VALUE;
SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
"test-snapshot-file-cache-refresh", new SnapshotFiles());
cache.getSnapshotsInProgress();
assertFalse(fs.exists(builder.getSnapshotsDir()));
}
}

12
hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
View File

@ -583,6 +583,18 @@ public final class SnapshotTestingUtils {
corruptFile(p);
}
public void missOneRegionSnapshotFile() throws IOException {
FileStatus[] manifestFiles = FSUtils.listStatus(fs, snapshotDir);
for (FileStatus fileStatus : manifestFiles) {
String fileName = fileStatus.getPath().getName();
if (fileName.endsWith(SnapshotDescriptionUtils.SNAPSHOTINFO_FILE)
|| fileName.endsWith(".tabledesc")
|| fileName.endsWith(SnapshotDescriptionUtils.SNAPSHOT_TMP_DIR_NAME)) {
fs.delete(fileStatus.getPath(), true);
}
}
}
/**
* Corrupt data-manifest file
*