Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-21070 Add Test for SnapshotFileCache for HBase backed by S3 (#209) 

SnapshotFileCache depends on getting the last modified time of the
snapshot directory, however, S3 FileSystem's do not update the
last modified time of the top 'folder' when objects are added/removed.
This commit adds a test for the previously fixed SnapshotFileCache.
This commit is contained in:
z-york 2019-05-06 11:52:08 -07:00 committed by Zach York
parent 26b9e76bbf
commit 13da268169
1 changed files with 41 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java
View File

@ -50,15 +50,18 @@ public class TestSnapshotFileCache {
private static final Log LOG = LogFactory.getLog(TestSnapshotFileCache.class);
private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
private static long sequenceId = 0;
// don't refresh the cache unless we tell it to
private static final long PERIOD = Long.MAX_VALUE;
private static FileSystem fs;
private static Path rootDir;
private static Path snapshotDir;
@BeforeClass
public static void startCluster() throws Exception {
UTIL.startMiniDFSCluster(1);
fs = UTIL.getDFSCluster().getFileSystem();
rootDir = UTIL.getDefaultRootDirPath();
snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
}
@AfterClass
@ -69,50 +72,59 @@ public class TestSnapshotFileCache {
@After
public void cleanupFiles() throws Exception {
// cleanup the snapshot directory
Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
fs.delete(snapshotDir, true);
}
@Test(timeout = 10000000)
@Ignore("See HBASE-19275")
public void testLoadAndDelete() throws IOException {
// don't refresh the cache unless we tell it to
long period = Long.MAX_VALUE;
SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, PERIOD, 10000000,
"test-snapshot-file-cache-refresh", new SnapshotFiles());
createAndTestSnapshotV1(cache, "snapshot1a", false, true);
createAndTestSnapshotV1(cache, "snapshot1a", false, true, false);
createAndTestSnapshotV2(cache, "snapshot2a", false, true);
createAndTestSnapshotV2(cache, "snapshot2a", false, true, false);
}
@Test
@Ignore("See HBASE-19275")
public void testReloadModifiedDirectory() throws IOException {
// don't refresh the cache unless we tell it to
long period = Long.MAX_VALUE;
SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, PERIOD, 10000000,
"test-snapshot-file-cache-refresh", new SnapshotFiles());
createAndTestSnapshotV1(cache, "snapshot1", false, true);
createAndTestSnapshotV1(cache, "snapshot1", false, true, false);
// now delete the snapshot and add a file with a different name
createAndTestSnapshotV1(cache, "snapshot1", false, false);
createAndTestSnapshotV1(cache, "snapshot1", false, false, false);
createAndTestSnapshotV2(cache, "snapshot2", false, true);
createAndTestSnapshotV2(cache, "snapshot2", false, true, false);
// now delete the snapshot and add a file with a different name
createAndTestSnapshotV2(cache, "snapshot2", false, false);
createAndTestSnapshotV2(cache, "snapshot2", false, false, false);
}
@Test
public void testSnapshotTempDirReload() throws IOException {
long period = Long.MAX_VALUE;
// This doesn't refresh cache until we invoke it explicitly
SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
SnapshotFileCache cache =
new SnapshotFileCache(fs, rootDir, PERIOD, 10000000, "test-snapshot-file-cache-refresh", new SnapshotFiles());
// Add a new non-tmp snapshot
createAndTestSnapshotV1(cache, "snapshot0v1", false, false, false);
createAndTestSnapshotV1(cache, "snapshot0v2", false, false, false);
}
@Test
public void testCacheUpdatedWhenLastModifiedOfSnapDirNotUpdated() throws IOException {
SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, PERIOD, 10000000,
"test-snapshot-file-cache-refresh", new SnapshotFiles());
// Add a new non-tmp snapshot
createAndTestSnapshotV1(cache, "snapshot0v1", false, false);
createAndTestSnapshotV1(cache, "snapshot0v2", false, false);
createAndTestSnapshotV1(cache, "snapshot1v1", false, false, true);
createAndTestSnapshotV1(cache, "snapshot1v2", false, false, true);
// Add a new tmp snapshot
createAndTestSnapshotV2(cache, "snapshot2v1", true, false, true);
// Add another tmp snapshot
createAndTestSnapshotV2(cache, "snapshot2v2", true, false, true);
}
class SnapshotFiles implements SnapshotFileCache.SnapshotFileInspector {
@ -124,23 +136,24 @@ public class TestSnapshotFileCache {
};
private SnapshotMock.SnapshotBuilder createAndTestSnapshotV1(final SnapshotFileCache cache,
final String name, final boolean tmp, final boolean removeOnExit) throws IOException {
final String name, final boolean tmp, final boolean removeOnExit, boolean setFolderTime)
throws IOException {
SnapshotMock snapshotMock = new SnapshotMock(UTIL.getConfiguration(), fs, rootDir);
SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV1(name, name);
createAndTestSnapshot(cache, builder, tmp, removeOnExit);
createAndTestSnapshot(cache, builder, tmp, removeOnExit, setFolderTime);
return builder;
}
private void createAndTestSnapshotV2(final SnapshotFileCache cache, final String name,
final boolean tmp, final boolean removeOnExit) throws IOException {
final boolean tmp, final boolean removeOnExit, boolean setFolderTime) throws IOException {
SnapshotMock snapshotMock = new SnapshotMock(UTIL.getConfiguration(), fs, rootDir);
SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2(name, name);
createAndTestSnapshot(cache, builder, tmp, removeOnExit);
createAndTestSnapshot(cache, builder, tmp, removeOnExit, setFolderTime);
}
private void createAndTestSnapshot(final SnapshotFileCache cache,
final SnapshotMock.SnapshotBuilder builder,
final boolean tmp, final boolean removeOnExit) throws IOException {
final boolean tmp, final boolean removeOnExit, boolean setFolderTime) throws IOException {
List<Path> files = new ArrayList<Path>();
for (int i = 0; i < 3; ++i) {
for (Path filePath: builder.addRegion()) {
@ -151,6 +164,10 @@ public class TestSnapshotFileCache {
// Finalize the snapshot
builder.commit();
if (setFolderTime) {
fs.setTimes(snapshotDir, 0, -1);
}
// Make sure that all files are still present
for (Path path: files) {
Iterable<FileStatus> nonSnapshotFiles = getNonSnapshotFiles(cache, path);