Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-21070 Add Test for SnapshotFileCache for HBase backed by S3 (#209) 

SnapshotFileCache depends on getting the last modified time of the
snapshot directory, however, S3 FileSystem's do not update the
last modified time of the top 'folder' when objects are added/removed.
This commit adds a test for the previously fixed SnapshotFileCache.
This commit is contained in:
z-york 2019-05-06 11:52:08 -07:00 committed by Zach York
parent 26b9e76bbf
commit 13da268169
1 changed files with 41 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java
View File

@ -50,15 +50,18 @@ public class TestSnapshotFileCache {
private static final Log LOG = LogFactory.getLog(TestSnapshotFileCache.class); private static final Log LOG = LogFactory.getLog(TestSnapshotFileCache.class);
private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
private static long sequenceId = 0; // don't refresh the cache unless we tell it to
private static final long PERIOD = Long.MAX_VALUE;
private static FileSystem fs; private static FileSystem fs;
private static Path rootDir; private static Path rootDir;
private static Path snapshotDir;
@BeforeClass @BeforeClass
public static void startCluster() throws Exception { public static void startCluster() throws Exception {
UTIL.startMiniDFSCluster(1); UTIL.startMiniDFSCluster(1);
fs = UTIL.getDFSCluster().getFileSystem(); fs = UTIL.getDFSCluster().getFileSystem();
rootDir = UTIL.getDefaultRootDirPath(); rootDir = UTIL.getDefaultRootDirPath();
snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
} }
@AfterClass @AfterClass
@ -69,50 +72,59 @@ public class TestSnapshotFileCache {
@After @After
public void cleanupFiles() throws Exception { public void cleanupFiles() throws Exception {
// cleanup the snapshot directory // cleanup the snapshot directory
Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
fs.delete(snapshotDir, true); fs.delete(snapshotDir, true);
} }
@Test(timeout = 10000000) @Test(timeout = 10000000)
@Ignore("See HBASE-19275") @Ignore("See HBASE-19275")
public void testLoadAndDelete() throws IOException { public void testLoadAndDelete() throws IOException {
// don't refresh the cache unless we tell it to SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, PERIOD, 10000000,
long period = Long.MAX_VALUE;
SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
"test-snapshot-file-cache-refresh", new SnapshotFiles()); "test-snapshot-file-cache-refresh", new SnapshotFiles());
createAndTestSnapshotV1(cache, "snapshot1a", false, true); createAndTestSnapshotV1(cache, "snapshot1a", false, true, false);
createAndTestSnapshotV2(cache, "snapshot2a", false, true); createAndTestSnapshotV2(cache, "snapshot2a", false, true, false);
} }
@Test @Test
@Ignore("See HBASE-19275") @Ignore("See HBASE-19275")
public void testReloadModifiedDirectory() throws IOException { public void testReloadModifiedDirectory() throws IOException {
// don't refresh the cache unless we tell it to SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, PERIOD, 10000000,
long period = Long.MAX_VALUE;
SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
"test-snapshot-file-cache-refresh", new SnapshotFiles()); "test-snapshot-file-cache-refresh", new SnapshotFiles());
createAndTestSnapshotV1(cache, "snapshot1", false, true); createAndTestSnapshotV1(cache, "snapshot1", false, true, false);
// now delete the snapshot and add a file with a different name // now delete the snapshot and add a file with a different name
createAndTestSnapshotV1(cache, "snapshot1", false, false); createAndTestSnapshotV1(cache, "snapshot1", false, false, false);
createAndTestSnapshotV2(cache, "snapshot2", false, true); createAndTestSnapshotV2(cache, "snapshot2", false, true, false);
// now delete the snapshot and add a file with a different name // now delete the snapshot and add a file with a different name
createAndTestSnapshotV2(cache, "snapshot2", false, false); createAndTestSnapshotV2(cache, "snapshot2", false, false, false);
} }
@Test @Test
public void testSnapshotTempDirReload() throws IOException { public void testSnapshotTempDirReload() throws IOException {
long period = Long.MAX_VALUE; SnapshotFileCache cache =
// This doesn't refresh cache until we invoke it explicitly new SnapshotFileCache(fs, rootDir, PERIOD, 10000000, "test-snapshot-file-cache-refresh", new SnapshotFiles());
SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
// Add a new non-tmp snapshot
createAndTestSnapshotV1(cache, "snapshot0v1", false, false, false);
createAndTestSnapshotV1(cache, "snapshot0v2", false, false, false);
}
@Test
public void testCacheUpdatedWhenLastModifiedOfSnapDirNotUpdated() throws IOException {
SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, PERIOD, 10000000,
"test-snapshot-file-cache-refresh", new SnapshotFiles()); "test-snapshot-file-cache-refresh", new SnapshotFiles());
// Add a new non-tmp snapshot // Add a new non-tmp snapshot
createAndTestSnapshotV1(cache, "snapshot0v1", false, false); createAndTestSnapshotV1(cache, "snapshot1v1", false, false, true);
createAndTestSnapshotV1(cache, "snapshot0v2", false, false); createAndTestSnapshotV1(cache, "snapshot1v2", false, false, true);
// Add a new tmp snapshot
createAndTestSnapshotV2(cache, "snapshot2v1", true, false, true);
// Add another tmp snapshot
createAndTestSnapshotV2(cache, "snapshot2v2", true, false, true);
} }
class SnapshotFiles implements SnapshotFileCache.SnapshotFileInspector { class SnapshotFiles implements SnapshotFileCache.SnapshotFileInspector {
@ -124,23 +136,24 @@ public class TestSnapshotFileCache {
}; };
private SnapshotMock.SnapshotBuilder createAndTestSnapshotV1(final SnapshotFileCache cache, private SnapshotMock.SnapshotBuilder createAndTestSnapshotV1(final SnapshotFileCache cache,
final String name, final boolean tmp, final boolean removeOnExit) throws IOException { final String name, final boolean tmp, final boolean removeOnExit, boolean setFolderTime)
throws IOException {
SnapshotMock snapshotMock = new SnapshotMock(UTIL.getConfiguration(), fs, rootDir); SnapshotMock snapshotMock = new SnapshotMock(UTIL.getConfiguration(), fs, rootDir);
SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV1(name, name); SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV1(name, name);
createAndTestSnapshot(cache, builder, tmp, removeOnExit); createAndTestSnapshot(cache, builder, tmp, removeOnExit, setFolderTime);
return builder; return builder;
} }
private void createAndTestSnapshotV2(final SnapshotFileCache cache, final String name, private void createAndTestSnapshotV2(final SnapshotFileCache cache, final String name,
final boolean tmp, final boolean removeOnExit) throws IOException { final boolean tmp, final boolean removeOnExit, boolean setFolderTime) throws IOException {
SnapshotMock snapshotMock = new SnapshotMock(UTIL.getConfiguration(), fs, rootDir); SnapshotMock snapshotMock = new SnapshotMock(UTIL.getConfiguration(), fs, rootDir);
SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2(name, name); SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2(name, name);
createAndTestSnapshot(cache, builder, tmp, removeOnExit); createAndTestSnapshot(cache, builder, tmp, removeOnExit, setFolderTime);
} }
private void createAndTestSnapshot(final SnapshotFileCache cache, private void createAndTestSnapshot(final SnapshotFileCache cache,
final SnapshotMock.SnapshotBuilder builder, final SnapshotMock.SnapshotBuilder builder,
final boolean tmp, final boolean removeOnExit) throws IOException { final boolean tmp, final boolean removeOnExit, boolean setFolderTime) throws IOException {
List<Path> files = new ArrayList<Path>(); List<Path> files = new ArrayList<Path>();
for (int i = 0; i < 3; ++i) { for (int i = 0; i < 3; ++i) {
for (Path filePath: builder.addRegion()) { for (Path filePath: builder.addRegion()) {
@ -151,6 +164,10 @@ public class TestSnapshotFileCache {
// Finalize the snapshot // Finalize the snapshot
builder.commit(); builder.commit();
if (setFolderTime) {
fs.setTimes(snapshotDir, 0, -1);
}
// Make sure that all files are still present // Make sure that all files are still present
for (Path path: files) { for (Path path: files) {
Iterable<FileStatus> nonSnapshotFiles = getNonSnapshotFiles(cache, path); Iterable<FileStatus> nonSnapshotFiles = getNonSnapshotFiles(cache, path);