Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-21070 Add Test for SnapshotFileCache for HBase backed by S3 (#209) 

SnapshotFileCache depends on getting the last modified time of the
snapshot directory, however, S3 FileSystem's do not update the
last modified time of the top 'folder' when objects are added/removed.
This commit adds a test for the previously fixed SnapshotFileCache.
This commit is contained in:
z-york 2019-05-06 11:52:08 -07:00 committed by GitHub
parent 45bcb4f307
commit 67c937fa92
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 41 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java
View File

@ -58,14 +58,18 @@ public class TestSnapshotFileCache {
private static final Logger LOG = LoggerFactory.getLogger(TestSnapshotFileCache.class);
private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
// don't refresh the cache unless we tell it to
private static final long PERIOD = Long.MAX_VALUE;
private static FileSystem fs;
private static Path rootDir;
private static Path snapshotDir;
@BeforeClass
public static void startCluster() throws Exception {
UTIL.startMiniDFSCluster(1);
fs = UTIL.getDFSCluster().getFileSystem();
rootDir = UTIL.getDefaultRootDirPath();
snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
}
@AfterClass
@ -76,48 +80,57 @@ public class TestSnapshotFileCache {
@After
public void cleanupFiles() throws Exception {
// cleanup the snapshot directory
Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
fs.delete(snapshotDir, true);
}
@Test
public void testLoadAndDelete() throws IOException {
// don't refresh the cache unless we tell it to
long period = Long.MAX_VALUE;
SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, PERIOD, 10000000,
"test-snapshot-file-cache-refresh", new SnapshotFiles());
createAndTestSnapshotV1(cache, "snapshot1a", false, true);
createAndTestSnapshotV1(cache, "snapshot1a", false, true, false);
createAndTestSnapshotV2(cache, "snapshot2a", false, true);
createAndTestSnapshotV2(cache, "snapshot2a", false, true, false);
}
@Test
public void testReloadModifiedDirectory() throws IOException {
// don't refresh the cache unless we tell it to
long period = Long.MAX_VALUE;
SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, PERIOD, 10000000,
"test-snapshot-file-cache-refresh", new SnapshotFiles());
createAndTestSnapshotV1(cache, "snapshot1", false, true);
createAndTestSnapshotV1(cache, "snapshot1", false, true, false);
// now delete the snapshot and add a file with a different name
createAndTestSnapshotV1(cache, "snapshot1", false, false);
createAndTestSnapshotV1(cache, "snapshot1", false, false, false);
createAndTestSnapshotV2(cache, "snapshot2", false, true);
createAndTestSnapshotV2(cache, "snapshot2", false, true, false);
// now delete the snapshot and add a file with a different name
createAndTestSnapshotV2(cache, "snapshot2", false, false);
createAndTestSnapshotV2(cache, "snapshot2", false, false, false);
}
@Test
public void testSnapshotTempDirReload() throws IOException {
long period = Long.MAX_VALUE;
// This doesn't refresh cache until we invoke it explicitly
SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
SnapshotFileCache cache =
new SnapshotFileCache(fs, rootDir, PERIOD, 10000000, "test-snapshot-file-cache-refresh", new SnapshotFiles());
// Add a new non-tmp snapshot
createAndTestSnapshotV1(cache, "snapshot0v1", false, false, false);
createAndTestSnapshotV1(cache, "snapshot0v2", false, false, false);
}
@Test
public void testCacheUpdatedWhenLastModifiedOfSnapDirNotUpdated() throws IOException {
SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, PERIOD, 10000000,
"test-snapshot-file-cache-refresh", new SnapshotFiles());
// Add a new non-tmp snapshot
createAndTestSnapshotV1(cache, "snapshot0v1", false, false);
createAndTestSnapshotV1(cache, "snapshot0v2", false, false);
createAndTestSnapshotV1(cache, "snapshot1v1", false, false, true);
createAndTestSnapshotV1(cache, "snapshot1v2", false, false, true);
// Add a new tmp snapshot
createAndTestSnapshotV2(cache, "snapshot2v1", true, false, true);
// Add another tmp snapshot
createAndTestSnapshotV2(cache, "snapshot2v2", true, false, true);
}
class SnapshotFiles implements SnapshotFileCache.SnapshotFileInspector {
@ -130,23 +143,24 @@ public class TestSnapshotFileCache {
}
private SnapshotMock.SnapshotBuilder createAndTestSnapshotV1(final SnapshotFileCache cache,
final String name, final boolean tmp, final boolean removeOnExit) throws IOException {
final String name, final boolean tmp, final boolean removeOnExit, boolean setFolderTime)
throws IOException {
SnapshotMock snapshotMock = new SnapshotMock(UTIL.getConfiguration(), fs, rootDir);
SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV1(name, name);
createAndTestSnapshot(cache, builder, tmp, removeOnExit);
createAndTestSnapshot(cache, builder, tmp, removeOnExit, setFolderTime);
return builder;
}
private void createAndTestSnapshotV2(final SnapshotFileCache cache, final String name,
final boolean tmp, final boolean removeOnExit) throws IOException {
final boolean tmp, final boolean removeOnExit, boolean setFolderTime) throws IOException {
SnapshotMock snapshotMock = new SnapshotMock(UTIL.getConfiguration(), fs, rootDir);
SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2(name, name);
createAndTestSnapshot(cache, builder, tmp, removeOnExit);
createAndTestSnapshot(cache, builder, tmp, removeOnExit, setFolderTime);
}
private void createAndTestSnapshot(final SnapshotFileCache cache,
final SnapshotMock.SnapshotBuilder builder,
final boolean tmp, final boolean removeOnExit) throws IOException {
final boolean tmp, final boolean removeOnExit, boolean setFolderTime) throws IOException {
List<Path> files = new ArrayList<>();
for (int i = 0; i < 3; ++i) {
for (Path filePath: builder.addRegion()) {
@ -157,6 +171,10 @@ public class TestSnapshotFileCache {
// Finalize the snapshot
builder.commit();
if (setFolderTime) {
fs.setTimes(snapshotDir, 0, -1);
}
// Make sure that all files are still present
for (Path path: files) {
assertFalse("Cache didn't find " + path, contains(getNonSnapshotFiles(cache, path), path));