Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-11128 Add -target option to ExportSnapshot to export with a different name 

git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1593775 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
mbertozzi 2014-05-11 11:39:56 +00:00
parent a56a4c900e
commit f00161be20
2 changed files with 52 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
View File

@ -688,6 +688,7 @@ public final class ExportSnapshot extends Configured implements Tool {
boolean verifyTarget = true; boolean verifyTarget = true;
boolean verifyChecksum = true; boolean verifyChecksum = true;
String snapshotName = null; String snapshotName = null;
String targetName = null;
boolean overwrite = false; boolean overwrite = false;
String filesGroup = null; String filesGroup = null;
String filesUser = null; String filesUser = null;
@ -704,6 +705,8 @@ public final class ExportSnapshot extends Configured implements Tool {
try { try {
if (cmd.equals("-snapshot")) { if (cmd.equals("-snapshot")) {
snapshotName = args[++i]; snapshotName = args[++i];
} else if (cmd.equals("-target")) {
targetName = args[++i];
} else if (cmd.equals("-copy-to")) { } else if (cmd.equals("-copy-to")) {
outputRoot = new Path(args[++i]); outputRoot = new Path(args[++i]);
} else if (cmd.equals("-copy-from")) { } else if (cmd.equals("-copy-from")) {
@ -749,6 +752,10 @@ public final class ExportSnapshot extends Configured implements Tool {
printUsageAndExit(); printUsageAndExit();
} }
if (targetName == null) {
targetName = snapshotName;
}
Path inputRoot = FSUtils.getRootDir(conf); Path inputRoot = FSUtils.getRootDir(conf);
FileSystem inputFs = FileSystem.get(inputRoot.toUri(), conf); FileSystem inputFs = FileSystem.get(inputRoot.toUri(), conf);
LOG.debug("inputFs=" + inputFs.getUri().toString() + " inputRoot=" + inputRoot); LOG.debug("inputFs=" + inputFs.getUri().toString() + " inputRoot=" + inputRoot);
@ -758,8 +765,8 @@ public final class ExportSnapshot extends Configured implements Tool {
boolean skipTmp = conf.getBoolean(CONF_SKIP_TMP, false); boolean skipTmp = conf.getBoolean(CONF_SKIP_TMP, false);
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, inputRoot); Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, inputRoot);
Path snapshotTmpDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshotName, outputRoot); Path snapshotTmpDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(targetName, outputRoot);
Path outputSnapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, outputRoot); Path outputSnapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(targetName, outputRoot);
Path initialOutputSnapshotDir = skipTmp ? outputSnapshotDir : snapshotTmpDir; Path initialOutputSnapshotDir = skipTmp ? outputSnapshotDir : snapshotTmpDir;
// Check if the snapshot already exists // Check if the snapshot already exists
@ -770,7 +777,7 @@ public final class ExportSnapshot extends Configured implements Tool {
return 1; return 1;
} }
} else { } else {
System.err.println("The snapshot '" + snapshotName + System.err.println("The snapshot '" + targetName +
"' already exists in the destination: " + outputSnapshotDir); "' already exists in the destination: " + outputSnapshotDir);
return 1; return 1;
} }
@ -785,7 +792,7 @@ public final class ExportSnapshot extends Configured implements Tool {
return 1; return 1;
} }
} else { } else {
System.err.println("A snapshot with the same name '"+snapshotName+"' may be in-progress"); System.err.println("A snapshot with the same name '"+ targetName +"' may be in-progress");
System.err.println("Please check "+snapshotTmpDir+". If the snapshot has completed, "); System.err.println("Please check "+snapshotTmpDir+". If the snapshot has completed, ");
System.err.println("consider removing "+snapshotTmpDir+" by using the -overwrite option"); System.err.println("consider removing "+snapshotTmpDir+" by using the -overwrite option");
return 1; return 1;
@ -812,6 +819,16 @@ public final class ExportSnapshot extends Configured implements Tool {
snapshotDir + " to=" + initialOutputSnapshotDir, e); snapshotDir + " to=" + initialOutputSnapshotDir, e);
} }
// Write a new .snapshotinfo if the target name is different from the source name
if (!targetName.equals(snapshotName)) {
SnapshotDescription snapshotDesc =
SnapshotDescriptionUtils.readSnapshotInfo(inputFs, snapshotDir)
.toBuilder()
.setName(targetName)
.build();
SnapshotDescriptionUtils.writeSnapshotInfo(snapshotDesc, snapshotTmpDir, outputFs);
}
// Step 2 - Start MR Job to copy files // Step 2 - Start MR Job to copy files
// The snapshot references must be copied before the files otherwise the files gets removed // The snapshot references must be copied before the files otherwise the files gets removed
// by the HFileArchiver, since they have no references. // by the HFileArchiver, since they have no references.
@ -839,7 +856,7 @@ public final class ExportSnapshot extends Configured implements Tool {
verifySnapshot(conf, outputFs, outputRoot, outputSnapshotDir); verifySnapshot(conf, outputFs, outputRoot, outputSnapshotDir);
} }
LOG.info("Export Completed: " + snapshotName); LOG.info("Export Completed: " + targetName);
return 0; return 0;
} catch (Exception e) { } catch (Exception e) {
LOG.error("Snapshot export failed", e); LOG.error("Snapshot export failed", e);

42
hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
View File

@ -119,7 +119,7 @@ public class TestExportSnapshot {
// Add some rows // Add some rows
HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName); HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 1000, FAMILY); SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 500, FAMILY);
// take a snapshot // take a snapshot
admin.snapshot(snapshotName, tableName); admin.snapshot(snapshotName, tableName);
@ -187,28 +187,34 @@ public class TestExportSnapshot {
*/ */
@Test @Test
public void testExportFileSystemState() throws Exception { public void testExportFileSystemState() throws Exception {
testExportFileSystemState(tableName, snapshotName, 2); testExportFileSystemState(tableName, snapshotName, snapshotName, 2);
} }
@Test @Test
public void testExportFileSystemStateWithSkipTmp() throws Exception { public void testExportFileSystemStateWithSkipTmp() throws Exception {
TEST_UTIL.getConfiguration().setBoolean(ExportSnapshot.CONF_SKIP_TMP, true); TEST_UTIL.getConfiguration().setBoolean(ExportSnapshot.CONF_SKIP_TMP, true);
testExportFileSystemState(tableName, snapshotName, 2); testExportFileSystemState(tableName, snapshotName, snapshotName, 2);
} }
@Test @Test
public void testEmptyExportFileSystemState() throws Exception { public void testEmptyExportFileSystemState() throws Exception {
testExportFileSystemState(tableName, emptySnapshotName, 1); testExportFileSystemState(tableName, emptySnapshotName, emptySnapshotName, 1);
} }
@Test @Test
public void testConsecutiveExports() throws Exception { public void testConsecutiveExports() throws Exception {
Path copyDir = getLocalDestinationDir(); Path copyDir = getLocalDestinationDir();
testExportFileSystemState(tableName, snapshotName, 2, copyDir, false); testExportFileSystemState(tableName, snapshotName, snapshotName, 2, copyDir, false);
testExportFileSystemState(tableName, snapshotName, 2, copyDir, true); testExportFileSystemState(tableName, snapshotName, snapshotName, 2, copyDir, true);
removeExportDir(copyDir); removeExportDir(copyDir);
} }
@Test
public void testExportWithTargetName() throws Exception {
final byte[] targetName = Bytes.toBytes("testExportWithTargetName");
testExportFileSystemState(tableName, snapshotName, targetName, 2);
}
/** /**
* Mock a snapshot with files in the archive dir, * Mock a snapshot with files in the archive dir,
* two regions, and one reference file. * two regions, and one reference file.
@ -262,13 +268,14 @@ public class TestExportSnapshot {
FileUtil.copy(fs, tableDir, fs, snapshotDir, false, conf); FileUtil.copy(fs, tableDir, fs, snapshotDir, false, conf);
SnapshotDescriptionUtils.writeSnapshotInfo(sd, snapshotDir, fs); SnapshotDescriptionUtils.writeSnapshotInfo(sd, snapshotDir, fs);
testExportFileSystemState(tableWithRefsName, Bytes.toBytes(snapshotName), 2); byte[] name = Bytes.toBytes(snapshotName);
testExportFileSystemState(tableWithRefsName, name, name, 2);
} }
private void testExportFileSystemState(final TableName tableName, final byte[] snapshotName, private void testExportFileSystemState(final TableName tableName, final byte[] snapshotName,
int filesExpected) throws Exception { final byte[] targetName, int filesExpected) throws Exception {
Path copyDir = getHdfsDestinationDir(); Path copyDir = getHdfsDestinationDir();
testExportFileSystemState(tableName, snapshotName, filesExpected, copyDir, false); testExportFileSystemState(tableName, snapshotName, targetName, filesExpected, copyDir, false);
removeExportDir(copyDir); removeExportDir(copyDir);
} }
@ -276,7 +283,8 @@ public class TestExportSnapshot {
* Test ExportSnapshot * Test ExportSnapshot
*/ */
private void testExportFileSystemState(final TableName tableName, final byte[] snapshotName, private void testExportFileSystemState(final TableName tableName, final byte[] snapshotName,
int filesExpected, Path copyDir, boolean overwrite) throws Exception { final byte[] targetName, int filesExpected, Path copyDir, boolean overwrite)
throws Exception {
URI hdfsUri = FileSystem.get(TEST_UTIL.getConfiguration()).getUri(); URI hdfsUri = FileSystem.get(TEST_UTIL.getConfiguration()).getUri();
FileSystem fs = FileSystem.get(copyDir.toUri(), new Configuration()); FileSystem fs = FileSystem.get(copyDir.toUri(), new Configuration());
copyDir = copyDir.makeQualified(fs); copyDir = copyDir.makeQualified(fs);
@ -286,6 +294,10 @@ public class TestExportSnapshot {
opts.add(Bytes.toString(snapshotName)); opts.add(Bytes.toString(snapshotName));
opts.add("-copy-to"); opts.add("-copy-to");
opts.add(copyDir.toString()); opts.add(copyDir.toString());
if (targetName != snapshotName) {
opts.add("-target");
opts.add(Bytes.toString(targetName));
}
if (overwrite) opts.add("-overwrite"); if (overwrite) opts.add("-overwrite");
// Export Snapshot // Export Snapshot
@ -306,9 +318,10 @@ public class TestExportSnapshot {
// compare the snapshot metadata and verify the hfiles // compare the snapshot metadata and verify the hfiles
final FileSystem hdfs = FileSystem.get(hdfsUri, TEST_UTIL.getConfiguration()); final FileSystem hdfs = FileSystem.get(hdfsUri, TEST_UTIL.getConfiguration());
final Path snapshotDir = new Path(HConstants.SNAPSHOT_DIR_NAME, Bytes.toString(snapshotName)); final Path snapshotDir = new Path(HConstants.SNAPSHOT_DIR_NAME, Bytes.toString(snapshotName));
final Path targetDir = new Path(HConstants.SNAPSHOT_DIR_NAME, Bytes.toString(targetName));
verifySnapshot(hdfs, new Path(TEST_UTIL.getDefaultRootDirPath(), snapshotDir), verifySnapshot(hdfs, new Path(TEST_UTIL.getDefaultRootDirPath(), snapshotDir),
fs, new Path(copyDir, snapshotDir)); fs, new Path(copyDir, targetDir));
verifyArchive(fs, copyDir, tableName, Bytes.toString(snapshotName)); verifyArchive(fs, copyDir, tableName, Bytes.toString(targetName));
FSUtils.logFileSystemState(hdfs, snapshotDir, LOG); FSUtils.logFileSystemState(hdfs, snapshotDir, LOG);
} }
@ -396,6 +409,11 @@ public class TestExportSnapshot {
assertTrue(path + " should not be empty", fs.getFileStatus(path).getLen() > 0); assertTrue(path + " should not be empty", fs.getFileStatus(path).getLen() > 0);
} }
}); });
// Verify Snapshot description
SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, exportedSnapshot);
assertTrue(desc.getName().equals(snapshotName));
assertTrue(desc.getTable().equals(tableName.getNameAsString()));
} }
private Set<String> listFiles(final FileSystem fs, final Path root, final Path dir) private Set<String> listFiles(final FileSystem fs, final Path root, final Path dir)