Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-26956 ExportSnapshot tool supports removing TTL (#4351) 

Signed-off-by: Duo Zhang <zhangduo@apache.org>
This commit is contained in:
XinSun 2022-06-15 15:04:17 +08:00 committed by GitHub
parent 7fc1674933
commit b365748485
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 73 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
View File

@ -154,6 +154,8 @@ public class ExportSnapshot extends AbstractHBaseTool implements Tool {
"Number of mappers to use during the copy (mapreduce.job.maps).");
static final Option BANDWIDTH =
new Option(null, "bandwidth", true, "Limit bandwidth to this value in MB/second.");
static final Option RESET_TTL =
new Option(null, "reset-ttl", false, "Do not copy TTL for the snapshot");
}
// Export Map-Reduce Counters, to keep track of the progress
@ -931,6 +933,7 @@ public class ExportSnapshot extends AbstractHBaseTool implements Tool {
private int bandwidthMB = Integer.MAX_VALUE;
private int filesMode = 0;
private int mappers = 0;
private boolean resetTtl = false;
@Override
protected void processOptions(CommandLine cmd) {
@ -952,6 +955,7 @@ public class ExportSnapshot extends AbstractHBaseTool implements Tool {
verifyChecksum = !cmd.hasOption(Options.NO_CHECKSUM_VERIFY.getLongOpt());
verifyTarget = !cmd.hasOption(Options.NO_TARGET_VERIFY.getLongOpt());
verifySource = !cmd.hasOption(Options.NO_SOURCE_VERIFY.getLongOpt());
resetTtl = cmd.hasOption(Options.RESET_TTL.getLongOpt());
}
/**
@ -1089,11 +1093,19 @@ public class ExportSnapshot extends AbstractHBaseTool implements Tool {
}
}
// Write a new .snapshotinfo if the target name is different from the source name
if (!targetName.equals(snapshotName)) {
SnapshotDescription snapshotDesc = SnapshotDescriptionUtils
.readSnapshotInfo(inputFs, snapshotDir).toBuilder().setName(targetName).build();
SnapshotDescriptionUtils.writeSnapshotInfo(snapshotDesc, initialOutputSnapshotDir, outputFs);
// Write a new .snapshotinfo if the target name is different from the source name or we want to
// reset TTL for target snapshot.
if (!targetName.equals(snapshotName) || resetTtl) {
SnapshotDescription.Builder snapshotDescBuilder =
SnapshotDescriptionUtils.readSnapshotInfo(inputFs, snapshotDir).toBuilder();
if (!targetName.equals(snapshotName)) {
snapshotDescBuilder.setName(targetName);
}
if (resetTtl) {
snapshotDescBuilder.setTtl(HConstants.DEFAULT_SNAPSHOT_TTL);
}
SnapshotDescriptionUtils.writeSnapshotInfo(snapshotDescBuilder.build(),
initialOutputSnapshotDir, outputFs);
if (filesUser != null || filesGroup != null) {
outputFs.setOwner(
new Path(initialOutputSnapshotDir, SnapshotDescriptionUtils.SNAPSHOTINFO_FILE), filesUser,
@ -1169,6 +1181,7 @@ public class ExportSnapshot extends AbstractHBaseTool implements Tool {
addOption(Options.CHMOD);
addOption(Options.MAPPERS);
addOption(Options.BANDWIDTH);
addOption(Options.RESET_TTL);
}
public static void main(String[] args) {

56
hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
View File

@ -24,9 +24,12 @@ import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import org.apache.hadoop.conf.Configuration;
@ -230,6 +233,37 @@ public class TestExportSnapshot {
testExportFileSystemState(tableName, snapshotName, targetName, tableNumFiles);
}
@Test
public void testExportWithResetTtl() throws Exception {
String name = "testExportWithResetTtl";
TableName tableName = TableName.valueOf(name);
String snapshotName = "snaptb-" + name;
Long ttl = 100000L;
try {
// create Table
createTable(tableName);
SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 50, FAMILY);
int tableNumFiles = admin.getRegions(tableName).size();
// take a snapshot with TTL
Map<String, Object> props = new HashMap<>();
props.put("TTL", ttl);
admin.snapshot(snapshotName, tableName, props);
Optional<Long> ttlOpt =
admin.listSnapshots().stream().filter(s -> s.getName().equals(snapshotName))
.map(org.apache.hadoop.hbase.client.SnapshotDescription::getTtl).findAny();
assertTrue(ttlOpt.isPresent());
assertEquals(ttl, ttlOpt.get());
testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles,
getHdfsDestinationDir(), false, true);
} catch (Exception e) {
throw e;
} finally {
TEST_UTIL.deleteTable(tableName);
}
}
private void testExportFileSystemState(final TableName tableName, final String snapshotName,
final String targetName, int filesExpected) throws Exception {
testExportFileSystemState(tableName, snapshotName, targetName, filesExpected,
@ -238,8 +272,15 @@ public class TestExportSnapshot {
protected void testExportFileSystemState(final TableName tableName, final String snapshotName,
final String targetName, int filesExpected, Path copyDir, boolean overwrite) throws Exception {
testExportFileSystemState(tableName, snapshotName, targetName, filesExpected, copyDir,
overwrite, false);
}
protected void testExportFileSystemState(final TableName tableName, final String snapshotName,
final String targetName, int filesExpected, Path copyDir, boolean overwrite, boolean resetTtl)
throws Exception {
testExportFileSystemState(TEST_UTIL.getConfiguration(), tableName, snapshotName, targetName,
filesExpected, TEST_UTIL.getDefaultRootDirPath(), copyDir, overwrite,
filesExpected, TEST_UTIL.getDefaultRootDirPath(), copyDir, overwrite, resetTtl,
getBypassRegionPredicate(), true);
}
@ -249,7 +290,8 @@ public class TestExportSnapshot {
protected static void testExportFileSystemState(final Configuration conf,
final TableName tableName, final String snapshotName, final String targetName,
final int filesExpected, final Path srcDir, Path rawTgtDir, final boolean overwrite,
final RegionPredicate bypassregionPredicate, boolean success) throws Exception {
final boolean resetTtl, final RegionPredicate bypassregionPredicate, boolean success)
throws Exception {
FileSystem tgtFs = rawTgtDir.getFileSystem(conf);
FileSystem srcFs = srcDir.getFileSystem(conf);
Path tgtDir = rawTgtDir.makeQualified(tgtFs.getUri(), tgtFs.getWorkingDirectory());
@ -267,6 +309,9 @@ public class TestExportSnapshot {
if (overwrite) {
opts.add("--overwrite");
}
if (resetTtl) {
opts.add("--reset-ttl");
}
// Export Snapshot
int res = run(conf, new ExportSnapshot(), opts.toArray(new String[opts.size()]));
@ -295,7 +340,7 @@ public class TestExportSnapshot {
final Path targetDir = new Path(HConstants.SNAPSHOT_DIR_NAME, targetName);
verifySnapshotDir(srcFs, new Path(srcDir, snapshotDir), tgtFs, new Path(tgtDir, targetDir));
Set<String> snapshotFiles =
verifySnapshot(conf, tgtFs, tgtDir, tableName, targetName, bypassregionPredicate);
verifySnapshot(conf, tgtFs, tgtDir, tableName, targetName, resetTtl, bypassregionPredicate);
assertEquals(filesExpected, snapshotFiles.size());
}
@ -312,7 +357,7 @@ public class TestExportSnapshot {
*/
protected static Set<String> verifySnapshot(final Configuration conf, final FileSystem fs,
final Path rootDir, final TableName tableName, final String snapshotName,
final RegionPredicate bypassregionPredicate) throws IOException {
final boolean resetTtl, final RegionPredicate bypassregionPredicate) throws IOException {
final Path exportedSnapshot =
new Path(rootDir, new Path(HConstants.SNAPSHOT_DIR_NAME, snapshotName));
final Set<String> snapshotFiles = new HashSet<>();
@ -354,6 +399,9 @@ public class TestExportSnapshot {
SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, exportedSnapshot);
assertTrue(desc.getName().equals(snapshotName));
assertTrue(desc.getTable().equals(tableName.getNameAsString()));
if (resetTtl) {
assertEquals(HConstants.DEFAULT_SNAPSHOT_TTL, desc.getTtl());
}
return snapshotFiles;
}

4
hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotAdjunct.java
View File

@ -151,7 +151,7 @@ public class TestExportSnapshotAdjunct {
conf.setInt(ExportSnapshot.Testing.CONF_TEST_FAILURE_COUNT, 2);
conf.setInt("mapreduce.map.maxattempts", 3);
TestExportSnapshot.testExportFileSystemState(conf, tableName, snapshotName, snapshotName,
tableNumFiles, TEST_UTIL.getDefaultRootDirPath(), copyDir, true, null, true);
tableNumFiles, TEST_UTIL.getDefaultRootDirPath(), copyDir, true, false, null, true);
}
/**
@ -167,6 +167,6 @@ public class TestExportSnapshotAdjunct {
conf.setInt(ExportSnapshot.Testing.CONF_TEST_FAILURE_COUNT, 4);
conf.setInt("mapreduce.map.maxattempts", 3);
TestExportSnapshot.testExportFileSystemState(conf, tableName, snapshotName, snapshotName,
tableNumFiles, TEST_UTIL.getDefaultRootDirPath(), copyDir, true, null, false);
tableNumFiles, TEST_UTIL.getDefaultRootDirPath(), copyDir, true, false, null, false);
}
}

2
hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotV1NoCluster.java
View File

@ -125,7 +125,7 @@ public class TestExportSnapshotV1NoCluster {
TableName tableName = builder.getTableDescriptor().getTableName();
TestExportSnapshot.testExportFileSystemState(testUtil.getConfiguration(), tableName,
snapshotName, snapshotName, snapshotFilesCount, testDir,
getDestinationDir(fs, testUtil, testDir), false, null, true);
getDestinationDir(fs, testUtil, testDir), false, false, null, true);
}
static Path getDestinationDir(FileSystem fs, HBaseCommonTestingUtil hctu, Path testDir)