Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-11128 Add -target option to ExportSnapshot to export with a different name 

git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1593775 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
mbertozzi 2014-05-11 11:39:56 +00:00
parent a56a4c900e
commit f00161be20
2 changed files with 52 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
View File

@ -688,6 +688,7 @@ public final class ExportSnapshot extends Configured implements Tool {
boolean verifyTarget = true;
boolean verifyChecksum = true;
String snapshotName = null;
String targetName = null;
boolean overwrite = false;
String filesGroup = null;
String filesUser = null;
@ -704,6 +705,8 @@ public final class ExportSnapshot extends Configured implements Tool {
try {
if (cmd.equals("-snapshot")) {
snapshotName = args[++i];
} else if (cmd.equals("-target")) {
targetName = args[++i];
} else if (cmd.equals("-copy-to")) {
outputRoot = new Path(args[++i]);
} else if (cmd.equals("-copy-from")) {
@ -749,6 +752,10 @@ public final class ExportSnapshot extends Configured implements Tool {
printUsageAndExit();
}
if (targetName == null) {
targetName = snapshotName;
}
Path inputRoot = FSUtils.getRootDir(conf);
FileSystem inputFs = FileSystem.get(inputRoot.toUri(), conf);
LOG.debug("inputFs=" + inputFs.getUri().toString() + " inputRoot=" + inputRoot);
@ -758,8 +765,8 @@ public final class ExportSnapshot extends Configured implements Tool {
boolean skipTmp = conf.getBoolean(CONF_SKIP_TMP, false);
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, inputRoot);
Path snapshotTmpDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshotName, outputRoot);
Path outputSnapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, outputRoot);
Path snapshotTmpDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(targetName, outputRoot);
Path outputSnapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(targetName, outputRoot);
Path initialOutputSnapshotDir = skipTmp ? outputSnapshotDir : snapshotTmpDir;
// Check if the snapshot already exists
@ -770,7 +777,7 @@ public final class ExportSnapshot extends Configured implements Tool {
return 1;
}
} else {
System.err.println("The snapshot '" + snapshotName +
System.err.println("The snapshot '" + targetName +
"' already exists in the destination: " + outputSnapshotDir);
return 1;
}
@ -785,7 +792,7 @@ public final class ExportSnapshot extends Configured implements Tool {
return 1;
}
} else {
System.err.println("A snapshot with the same name '"+snapshotName+"' may be in-progress");
System.err.println("A snapshot with the same name '"+ targetName +"' may be in-progress");
System.err.println("Please check "+snapshotTmpDir+". If the snapshot has completed, ");
System.err.println("consider removing "+snapshotTmpDir+" by using the -overwrite option");
return 1;
@ -812,6 +819,16 @@ public final class ExportSnapshot extends Configured implements Tool {
snapshotDir + " to=" + initialOutputSnapshotDir, e);
}
// Write a new .snapshotinfo if the target name is different from the source name
if (!targetName.equals(snapshotName)) {
SnapshotDescription snapshotDesc =
SnapshotDescriptionUtils.readSnapshotInfo(inputFs, snapshotDir)
.toBuilder()
.setName(targetName)
.build();
SnapshotDescriptionUtils.writeSnapshotInfo(snapshotDesc, snapshotTmpDir, outputFs);
}
// Step 2 - Start MR Job to copy files
// The snapshot references must be copied before the files otherwise the files gets removed
// by the HFileArchiver, since they have no references.
@ -839,7 +856,7 @@ public final class ExportSnapshot extends Configured implements Tool {
verifySnapshot(conf, outputFs, outputRoot, outputSnapshotDir);
}
LOG.info("Export Completed: " + snapshotName);
LOG.info("Export Completed: " + targetName);
return 0;
} catch (Exception e) {
LOG.error("Snapshot export failed", e);

42
hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
View File

@ -119,7 +119,7 @@ public class TestExportSnapshot {
// Add some rows
HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 1000, FAMILY);
SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 500, FAMILY);
// take a snapshot
admin.snapshot(snapshotName, tableName);
@ -187,28 +187,34 @@ public class TestExportSnapshot {
*/
@Test
public void testExportFileSystemState() throws Exception {
testExportFileSystemState(tableName, snapshotName, 2);
testExportFileSystemState(tableName, snapshotName, snapshotName, 2);
}
@Test
public void testExportFileSystemStateWithSkipTmp() throws Exception {
TEST_UTIL.getConfiguration().setBoolean(ExportSnapshot.CONF_SKIP_TMP, true);
testExportFileSystemState(tableName, snapshotName, 2);
testExportFileSystemState(tableName, snapshotName, snapshotName, 2);
}
@Test
public void testEmptyExportFileSystemState() throws Exception {
testExportFileSystemState(tableName, emptySnapshotName, 1);
testExportFileSystemState(tableName, emptySnapshotName, emptySnapshotName, 1);
}
@Test
public void testConsecutiveExports() throws Exception {
Path copyDir = getLocalDestinationDir();
testExportFileSystemState(tableName, snapshotName, 2, copyDir, false);
testExportFileSystemState(tableName, snapshotName, 2, copyDir, true);
testExportFileSystemState(tableName, snapshotName, snapshotName, 2, copyDir, false);
testExportFileSystemState(tableName, snapshotName, snapshotName, 2, copyDir, true);
removeExportDir(copyDir);
}
@Test
public void testExportWithTargetName() throws Exception {
final byte[] targetName = Bytes.toBytes("testExportWithTargetName");
testExportFileSystemState(tableName, snapshotName, targetName, 2);
}
/**
* Mock a snapshot with files in the archive dir,
* two regions, and one reference file.
@ -262,13 +268,14 @@ public class TestExportSnapshot {
FileUtil.copy(fs, tableDir, fs, snapshotDir, false, conf);
SnapshotDescriptionUtils.writeSnapshotInfo(sd, snapshotDir, fs);
testExportFileSystemState(tableWithRefsName, Bytes.toBytes(snapshotName), 2);
byte[] name = Bytes.toBytes(snapshotName);
testExportFileSystemState(tableWithRefsName, name, name, 2);
}
private void testExportFileSystemState(final TableName tableName, final byte[] snapshotName,
int filesExpected) throws Exception {
final byte[] targetName, int filesExpected) throws Exception {
Path copyDir = getHdfsDestinationDir();
testExportFileSystemState(tableName, snapshotName, filesExpected, copyDir, false);
testExportFileSystemState(tableName, snapshotName, targetName, filesExpected, copyDir, false);
removeExportDir(copyDir);
}
@ -276,7 +283,8 @@ public class TestExportSnapshot {
* Test ExportSnapshot
*/
private void testExportFileSystemState(final TableName tableName, final byte[] snapshotName,
int filesExpected, Path copyDir, boolean overwrite) throws Exception {
final byte[] targetName, int filesExpected, Path copyDir, boolean overwrite)
throws Exception {
URI hdfsUri = FileSystem.get(TEST_UTIL.getConfiguration()).getUri();
FileSystem fs = FileSystem.get(copyDir.toUri(), new Configuration());
copyDir = copyDir.makeQualified(fs);
@ -286,6 +294,10 @@ public class TestExportSnapshot {
opts.add(Bytes.toString(snapshotName));
opts.add("-copy-to");
opts.add(copyDir.toString());
if (targetName != snapshotName) {
opts.add("-target");
opts.add(Bytes.toString(targetName));
}
if (overwrite) opts.add("-overwrite");
// Export Snapshot
@ -306,9 +318,10 @@ public class TestExportSnapshot {
// compare the snapshot metadata and verify the hfiles
final FileSystem hdfs = FileSystem.get(hdfsUri, TEST_UTIL.getConfiguration());
final Path snapshotDir = new Path(HConstants.SNAPSHOT_DIR_NAME, Bytes.toString(snapshotName));
final Path targetDir = new Path(HConstants.SNAPSHOT_DIR_NAME, Bytes.toString(targetName));
verifySnapshot(hdfs, new Path(TEST_UTIL.getDefaultRootDirPath(), snapshotDir),
fs, new Path(copyDir, snapshotDir));
verifyArchive(fs, copyDir, tableName, Bytes.toString(snapshotName));
fs, new Path(copyDir, targetDir));
verifyArchive(fs, copyDir, tableName, Bytes.toString(targetName));
FSUtils.logFileSystemState(hdfs, snapshotDir, LOG);
}
@ -396,6 +409,11 @@ public class TestExportSnapshot {
assertTrue(path + " should not be empty", fs.getFileStatus(path).getLen() > 0);
}
});
// Verify Snapshot description
SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, exportedSnapshot);
assertTrue(desc.getName().equals(snapshotName));
assertTrue(desc.getTable().equals(tableName.getNameAsString()));
}
private Set<String> listFiles(final FileSystem fs, final Path root, final Path dir)