From 7378dd8521bc7630d4abf97243a07f3564f923a8 Mon Sep 17 00:00:00 2001 From: mbertozzi Date: Fri, 21 Feb 2014 09:39:52 +0000 Subject: [PATCH] HBASE-10567 Add overwrite manifest option to ExportSnapshot git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1570502 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop/hbase/snapshot/ExportSnapshot.java | 27 ++++++++---- .../hbase/snapshot/TestExportSnapshot.java | 42 +++++++++++++++---- 2 files changed, 52 insertions(+), 17 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java index 4f861746882..feae1c42a94 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java @@ -189,7 +189,7 @@ public final class ExportSnapshot extends Configured implements Tool { // FLAKY-TEST-WARN: lower is better, we can get some runs without the // retry, but at least we reduce the number of test failures due to // this test exception from the same map task. - if (random.nextFloat() < 0.05) { + if (random.nextFloat() < 0.03) { throw new IOException("TEST RETRY FAILURE: Unable to copy input=" + inputPath + " time=" + System.currentTimeMillis()); } @@ -587,6 +587,7 @@ public final class ExportSnapshot extends Configured implements Tool { public int run(String[] args) throws Exception { boolean verifyChecksum = true; String snapshotName = null; + boolean overwrite = false; String filesGroup = null; String filesUser = null; Path outputRoot = null; @@ -611,6 +612,8 @@ public final class ExportSnapshot extends Configured implements Tool { filesGroup = args[++i]; } else if (cmd.equals("-chmod")) { filesMode = Integer.parseInt(args[++i], 8); + } else if (cmd.equals("-overwrite")) { + overwrite = true; } else if (cmd.equals("-h") || cmd.equals("--help")) { printUsageAndExit(); } else { @@ -644,13 +647,20 @@ public final class ExportSnapshot extends Configured implements Tool { // Check if the snapshot already exists if (outputFs.exists(outputSnapshotDir)) { - System.err.println("The snapshot '" + snapshotName + - "' already exists in the destination: " + outputSnapshotDir); - return 1; + if (overwrite) { + if (!outputFs.delete(outputSnapshotDir, true)) { + System.err.println("Unable to remove existing snapshot directory: " + outputSnapshotDir); + return 1; + } + } else { + System.err.println("The snapshot '" + snapshotName + + "' already exists in the destination: " + outputSnapshotDir); + return 1; + } } // Check if the snapshot already in-progress - if (outputFs.exists(snapshotTmpDir)) { + if (!overwrite && outputFs.exists(snapshotTmpDir)) { System.err.println("A snapshot with the same name '" + snapshotName + "' may be in-progress"); System.err.println("Please check " + snapshotTmpDir + ". If the snapshot has completed, "); System.err.println("consider removing " + snapshotTmpDir + " before retrying export"); @@ -664,7 +674,7 @@ public final class ExportSnapshot extends Configured implements Tool { // The snapshot references must be copied before the hfiles otherwise the cleaner // will remove them because they are unreferenced. try { - FileUtil.copy(inputFs, snapshotDir, outputFs, snapshotTmpDir, false, false, conf); + FileUtil.copy(inputFs, snapshotDir, outputFs, snapshotTmpDir, false, overwrite, conf); } catch (IOException e) { System.err.println("Failed to copy the snapshot directory: from=" + snapshotDir + " to=" + snapshotTmpDir); @@ -710,6 +720,7 @@ public final class ExportSnapshot extends Configured implements Tool { System.err.println(" -snapshot NAME Snapshot to restore."); System.err.println(" -copy-to NAME Remote destination hdfs://"); System.err.println(" -no-checksum-verify Do not verify checksum."); + System.err.println(" -overwrite Rewrite the snapshot manifest if already exists"); System.err.println(" -chuser USERNAME Change the owner of the files " + "to the specified one."); System.err.println(" -chgroup GROUP Change the group of the files to " + @@ -720,8 +731,8 @@ public final class ExportSnapshot extends Configured implements Tool { "copy (mapreduce.job.maps)."); System.err.println(); System.err.println("Examples:"); - System.err.println(" hbase " + getClass() + " \\"); - System.err.println(" -snapshot MySnapshot -copy-to hdfs:///srv2:8082/hbase \\"); + System.err.println(" hbase " + getClass().getName() + " \\"); + System.err.println(" -snapshot MySnapshot -copy-to hdfs://srv2:8082/hbase \\"); System.err.println(" -chuser MyUser -chgroup MyGroup -chmod 700 -mappers 16"); System.exit(1); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java index 3aa5bacff07..86223d43fef 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java @@ -82,6 +82,8 @@ public class TestExportSnapshot { TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250); TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6); TEST_UTIL.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", true); + TEST_UTIL.getConfiguration().setInt("mapreduce.map.max.attempts", 10); + TEST_UTIL.getConfiguration().setInt("mapred.map.max.attempts", 10); TEST_UTIL.startMiniCluster(3); TEST_UTIL.startMiniMapReduceCluster(); } @@ -176,6 +178,14 @@ public class TestExportSnapshot { testExportFileSystemState(tableName, emptySnapshotName, 1); } + @Test + public void testConsecutiveExports() throws Exception { + Path copyDir = TEST_UTIL.getDataTestDir("export-" + System.currentTimeMillis()); + testExportFileSystemState(tableName, snapshotName, 2, copyDir, false); + testExportFileSystemState(tableName, snapshotName, 2, copyDir, true); + removeExportDir(copyDir); + } + /** * Mock a snapshot with files in the archive dir, * two regions, and one reference file. @@ -227,21 +237,32 @@ public class TestExportSnapshot { testExportFileSystemState(tableWithRefsName, Bytes.toBytes(snapshotName), 2); } + private void testExportFileSystemState(final TableName tableName, final byte[] snapshotName, + int filesExpected) throws Exception { + Path copyDir = TEST_UTIL.getDataTestDir("export-" + System.currentTimeMillis()); + testExportFileSystemState(tableName, snapshotName, filesExpected, copyDir, false); + removeExportDir(copyDir); + } + /** * Test ExportSnapshot */ private void testExportFileSystemState(final TableName tableName, final byte[] snapshotName, - int filesExpected) throws Exception { - Path copyDir = TEST_UTIL.getDataTestDir("export-" + System.currentTimeMillis()); + int filesExpected, Path copyDir, boolean overwrite) throws Exception { URI hdfsUri = FileSystem.get(TEST_UTIL.getConfiguration()).getUri(); FileSystem fs = FileSystem.get(copyDir.toUri(), new Configuration()); copyDir = copyDir.makeQualified(fs); + List opts = new ArrayList(); + opts.add("-snapshot"); + opts.add(Bytes.toString(snapshotName)); + opts.add("-copy-to"); + opts.add(copyDir.toString()); + if (overwrite) opts.add("-overwrite"); + // Export Snapshot - int res = ExportSnapshot.innerMain(TEST_UTIL.getConfiguration(), new String[] { - "-snapshot", Bytes.toString(snapshotName), - "-copy-to", copyDir.toString() - }); + int res = ExportSnapshot.innerMain(TEST_UTIL.getConfiguration(), + opts.toArray(new String[opts.size()])); assertEquals(0, res); // Verify File-System state @@ -261,9 +282,6 @@ public class TestExportSnapshot { fs, new Path(copyDir, snapshotDir)); verifyArchive(fs, copyDir, tableName, Bytes.toString(snapshotName)); FSUtils.logFileSystemState(hdfs, snapshotDir, LOG); - - // Remove the exported dir - fs.delete(copyDir, true); } /** @@ -366,4 +384,10 @@ public class TestExportSnapshot { } return files; } + + private void removeExportDir(final Path path) throws IOException { + FileSystem fs = FileSystem.get(path.toUri(), new Configuration()); + FSUtils.logFileSystemState(fs, path, LOG); + fs.delete(path, true); + } }