Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-11119 Update ExportSnapShot to optionally not use a tmp file on external file system (Ted Malaska) 

git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1593337 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
mbertozzi 2014-05-08 17:23:52 +00:00
parent 4325d15edf
commit 64675ef3c2
2 changed files with 34 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
View File

@ -93,6 +93,7 @@ public final class ExportSnapshot extends Configured implements Tool {
private static final String CONF_BUFFER_SIZE = "snapshot.export.buffer.size";
private static final String CONF_MAP_GROUP = "snapshot.export.default.map.group";
private static final String CONF_BANDWIDTH_MB = "snapshot.export.map.bandwidth.mb";
protected static final String CONF_SKIP_TMP = "snapshot.export.skip.tmp";
static final String CONF_TEST_FAILURE = "test.snapshot.export.failure";
static final String CONF_TEST_RETRY = "test.snapshot.export.failure.retry";
@ -751,9 +752,12 @@ public final class ExportSnapshot extends Configured implements Tool {
FileSystem outputFs = FileSystem.get(outputRoot.toUri(), conf);
LOG.debug("outputFs=" + outputFs.getUri().toString() + " outputRoot=" + outputRoot.toString());
boolean skipTmp = conf.getBoolean(CONF_SKIP_TMP, false);
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, inputRoot);
Path snapshotTmpDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshotName, outputRoot);
Path outputSnapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, outputRoot);
Path initialOutputSnapshotDir = skipTmp ? outputSnapshotDir : snapshotTmpDir;
// Check if the snapshot already exists
if (outputFs.exists(outputSnapshotDir)) {
@ -769,20 +773,22 @@ public final class ExportSnapshot extends Configured implements Tool {
}
}
if (!skipTmp) {
// Check if the snapshot already in-progress
if (outputFs.exists(snapshotTmpDir)) {
if (overwrite) {
if (!outputFs.delete(snapshotTmpDir, true)) {
System.err.println("Unable to remove existing snapshot tmp directory: " + snapshotTmpDir);
System.err.println("Unable to remove existing snapshot tmp directory: "+snapshotTmpDir);
return 1;
}
} else {
System.err.println("A snapshot with the same name '"+ snapshotName +"' may be in-progress");
System.err.println("Please check " + snapshotTmpDir + ". If the snapshot has completed, ");
System.err.println("consider removing "+ snapshotTmpDir +" by using the -overwrite option");
System.err.println("A snapshot with the same name '"+snapshotName+"' may be in-progress");
System.err.println("Please check "+snapshotTmpDir+". If the snapshot has completed, ");
System.err.println("consider removing "+snapshotTmpDir+" by using the -overwrite option");
return 1;
}
}
}
// Step 0 - Extract snapshot files to copy
LOG.info("Loading Snapshot hfile list");
@ -797,10 +803,10 @@ public final class ExportSnapshot extends Configured implements Tool {
// will remove them because they are unreferenced.
try {
LOG.info("Copy Snapshot Manifest");
FileUtil.copy(inputFs, snapshotDir, outputFs, snapshotTmpDir, false, false, conf);
FileUtil.copy(inputFs, snapshotDir, outputFs, initialOutputSnapshotDir, false, false, conf);
} catch (IOException e) {
throw new ExportSnapshotException("Failed to copy the snapshot directory: from=" +
snapshotDir + " to=" + snapshotTmpDir, e);
snapshotDir + " to=" + initialOutputSnapshotDir, e);
}
// Step 2 - Start MR Job to copy files
@ -814,12 +820,15 @@ public final class ExportSnapshot extends Configured implements Tool {
filesUser, filesGroup, filesMode, mappers, bandwidthMB);
}
// Step 3 - Rename fs2:/.snapshot/.tmp/<snapshot> fs2:/.snapshot/<snapshot>
LOG.info("Finalize the Snapshot Export");
if (!skipTmp) {
// Step 3 - Rename fs2:/.snapshot/.tmp/<snapshot> fs2:/.snapshot/<snapshot>
if (!outputFs.rename(snapshotTmpDir, outputSnapshotDir)) {
throw new ExportSnapshotException("Unable to rename snapshot directory from=" +
snapshotTmpDir + " to=" + outputSnapshotDir);
}
}
// Step 4 - Verify snapshot validity
LOG.info("Verify snapshot validity");
@ -829,7 +838,9 @@ public final class ExportSnapshot extends Configured implements Tool {
return 0;
} catch (Exception e) {
LOG.error("Snapshot export failed", e);
if (!skipTmp) {
outputFs.delete(snapshotTmpDir, true);
}
outputFs.delete(outputSnapshotDir, true);
return 1;
}

6
hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
View File

@ -190,6 +190,12 @@ public class TestExportSnapshot {
testExportFileSystemState(tableName, snapshotName, 2);
}
@Test
public void testExportFileSystemStateWithSkipTmp() throws Exception {
TEST_UTIL.getConfiguration().setBoolean(ExportSnapshot.CONF_SKIP_TMP, true);
testExportFileSystemState(tableName, snapshotName, 2);
}
@Test
public void testEmptyExportFileSystemState() throws Exception {
testExportFileSystemState(tableName, emptySnapshotName, 1);