Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-10567 Add overwrite manifest option to ExportSnapshot 

git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1570502 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
mbertozzi 2014-02-21 09:39:52 +00:00
parent e05863f8e5
commit 7378dd8521
2 changed files with 52 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
View File

@ -189,7 +189,7 @@ public final class ExportSnapshot extends Configured implements Tool {
// FLAKY-TEST-WARN: lower is better, we can get some runs without the
// retry, but at least we reduce the number of test failures due to
// this test exception from the same map task.
if (random.nextFloat() < 0.05) {
if (random.nextFloat() < 0.03) {
throw new IOException("TEST RETRY FAILURE: Unable to copy input=" + inputPath
+ " time=" + System.currentTimeMillis());
}
@ -587,6 +587,7 @@ public final class ExportSnapshot extends Configured implements Tool {
public int run(String[] args) throws Exception {
boolean verifyChecksum = true;
String snapshotName = null;
boolean overwrite = false;
String filesGroup = null;
String filesUser = null;
Path outputRoot = null;
@ -611,6 +612,8 @@ public final class ExportSnapshot extends Configured implements Tool {
filesGroup = args[++i];
} else if (cmd.equals("-chmod")) {
filesMode = Integer.parseInt(args[++i], 8);
} else if (cmd.equals("-overwrite")) {
overwrite = true;
} else if (cmd.equals("-h") || cmd.equals("--help")) {
printUsageAndExit();
} else {
@ -644,13 +647,20 @@ public final class ExportSnapshot extends Configured implements Tool {
// Check if the snapshot already exists
if (outputFs.exists(outputSnapshotDir)) {
System.err.println("The snapshot '" + snapshotName +
"' already exists in the destination: " + outputSnapshotDir);
return 1;
if (overwrite) {
if (!outputFs.delete(outputSnapshotDir, true)) {
System.err.println("Unable to remove existing snapshot directory: " + outputSnapshotDir);
return 1;
}
} else {
System.err.println("The snapshot '" + snapshotName +
"' already exists in the destination: " + outputSnapshotDir);
return 1;
}
}
// Check if the snapshot already in-progress
if (outputFs.exists(snapshotTmpDir)) {
if (!overwrite && outputFs.exists(snapshotTmpDir)) {
System.err.println("A snapshot with the same name '" + snapshotName + "' may be in-progress");
System.err.println("Please check " + snapshotTmpDir + ". If the snapshot has completed, ");
System.err.println("consider removing " + snapshotTmpDir + " before retrying export");
@ -664,7 +674,7 @@ public final class ExportSnapshot extends Configured implements Tool {
// The snapshot references must be copied before the hfiles otherwise the cleaner
// will remove them because they are unreferenced.
try {
FileUtil.copy(inputFs, snapshotDir, outputFs, snapshotTmpDir, false, false, conf);
FileUtil.copy(inputFs, snapshotDir, outputFs, snapshotTmpDir, false, overwrite, conf);
} catch (IOException e) {
System.err.println("Failed to copy the snapshot directory: from=" + snapshotDir +
" to=" + snapshotTmpDir);
@ -710,6 +720,7 @@ public final class ExportSnapshot extends Configured implements Tool {
System.err.println(" -snapshot NAME Snapshot to restore.");
System.err.println(" -copy-to NAME Remote destination hdfs://");
System.err.println(" -no-checksum-verify Do not verify checksum.");
System.err.println(" -overwrite Rewrite the snapshot manifest if already exists");
System.err.println(" -chuser USERNAME Change the owner of the files " +
"to the specified one.");
System.err.println(" -chgroup GROUP Change the group of the files to " +
@ -720,8 +731,8 @@ public final class ExportSnapshot extends Configured implements Tool {
"copy (mapreduce.job.maps).");
System.err.println();
System.err.println("Examples:");
System.err.println(" hbase " + getClass() + " \\");
System.err.println(" -snapshot MySnapshot -copy-to hdfs:///srv2:8082/hbase \\");
System.err.println(" hbase " + getClass().getName() + " \\");
System.err.println(" -snapshot MySnapshot -copy-to hdfs://srv2:8082/hbase \\");
System.err.println(" -chuser MyUser -chgroup MyGroup -chmod 700 -mappers 16");
System.exit(1);
}

42
hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
View File

@ -82,6 +82,8 @@ public class TestExportSnapshot {
TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
TEST_UTIL.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", true);
TEST_UTIL.getConfiguration().setInt("mapreduce.map.max.attempts", 10);
TEST_UTIL.getConfiguration().setInt("mapred.map.max.attempts", 10);
TEST_UTIL.startMiniCluster(3);
TEST_UTIL.startMiniMapReduceCluster();
}
@ -176,6 +178,14 @@ public class TestExportSnapshot {
testExportFileSystemState(tableName, emptySnapshotName, 1);
}
@Test
public void testConsecutiveExports() throws Exception {
Path copyDir = TEST_UTIL.getDataTestDir("export-" + System.currentTimeMillis());
testExportFileSystemState(tableName, snapshotName, 2, copyDir, false);
testExportFileSystemState(tableName, snapshotName, 2, copyDir, true);
removeExportDir(copyDir);
}
/**
* Mock a snapshot with files in the archive dir,
* two regions, and one reference file.
@ -227,21 +237,32 @@ public class TestExportSnapshot {
testExportFileSystemState(tableWithRefsName, Bytes.toBytes(snapshotName), 2);
}
private void testExportFileSystemState(final TableName tableName, final byte[] snapshotName,
int filesExpected) throws Exception {
Path copyDir = TEST_UTIL.getDataTestDir("export-" + System.currentTimeMillis());
testExportFileSystemState(tableName, snapshotName, filesExpected, copyDir, false);
removeExportDir(copyDir);
}
/**
* Test ExportSnapshot
*/
private void testExportFileSystemState(final TableName tableName, final byte[] snapshotName,
int filesExpected) throws Exception {
Path copyDir = TEST_UTIL.getDataTestDir("export-" + System.currentTimeMillis());
int filesExpected, Path copyDir, boolean overwrite) throws Exception {
URI hdfsUri = FileSystem.get(TEST_UTIL.getConfiguration()).getUri();
FileSystem fs = FileSystem.get(copyDir.toUri(), new Configuration());
copyDir = copyDir.makeQualified(fs);
List<String> opts = new ArrayList<String>();
opts.add("-snapshot");
opts.add(Bytes.toString(snapshotName));
opts.add("-copy-to");
opts.add(copyDir.toString());
if (overwrite) opts.add("-overwrite");
// Export Snapshot
int res = ExportSnapshot.innerMain(TEST_UTIL.getConfiguration(), new String[] {
"-snapshot", Bytes.toString(snapshotName),
"-copy-to", copyDir.toString()
});
int res = ExportSnapshot.innerMain(TEST_UTIL.getConfiguration(),
opts.toArray(new String[opts.size()]));
assertEquals(0, res);
// Verify File-System state
@ -261,9 +282,6 @@ public class TestExportSnapshot {
fs, new Path(copyDir, snapshotDir));
verifyArchive(fs, copyDir, tableName, Bytes.toString(snapshotName));
FSUtils.logFileSystemState(hdfs, snapshotDir, LOG);
// Remove the exported dir
fs.delete(copyDir, true);
}
/**
@ -366,4 +384,10 @@ public class TestExportSnapshot {
}
return files;
}
private void removeExportDir(final Path path) throws IOException {
FileSystem fs = FileSystem.get(path.toUri(), new Configuration());
FSUtils.logFileSystemState(fs, path, LOG);
fs.delete(path, true);
}
}