pathsToDelete = null;
- if (oldMetas != null) {
- // put all metadata of old paths and verify
- ms.put(new DirListingMetadata(oldDir, oldMetas, false));
- assertEquals(0, ms.listChildren(newDir).withoutTombstones().numEntries());
- assertTrue(CollectionUtils.isEqualCollection(oldMetas,
- ms.listChildren(oldDir).getListing()));
-
- pathsToDelete = new ArrayList<>(oldMetas.size());
- for (PathMetadata meta : oldMetas) {
- pathsToDelete.add(meta.getFileStatus().getPath());
- }
- }
-
- // move the old paths to new paths and verify
- ms.move(pathsToDelete, newMetas);
- assertEquals(0, ms.listChildren(oldDir).withoutTombstones().numEntries());
- if (newMetas != null) {
- assertTrue(CollectionUtils.isEqualCollection(newMetas,
- ms.listChildren(newDir).getListing()));
- }
- }
-
- @Test
- public void testInitExistingTable() throws IOException {
- final DynamoDBMetadataStore ddbms = getDynamoMetadataStore();
- final String tableName = ddbms.getTable().getTableName();
- verifyTableInitialized(tableName);
- // create existing table
- ddbms.initTable();
- verifyTableInitialized(tableName);
- }
-
- /**
- * Test the low level version check code.
- */
- @Test
- public void testItemVersionCompatibility() throws Throwable {
- verifyVersionCompatibility("table",
- createVersionMarker(VERSION_MARKER, VERSION, 0));
- }
-
- /**
- * Test that a version marker entry without the version number field
- * is rejected as incompatible with a meaningful error message.
- */
- @Test
- public void testItemLacksVersion() throws Throwable {
- intercept(IOException.class, E_NOT_VERSION_MARKER,
- new VoidCallable() {
- @Override
- public void call() throws Exception {
- verifyVersionCompatibility("table",
- new Item().withPrimaryKey(
- createVersionMarkerPrimaryKey(VERSION_MARKER)));
- }
- });
- }
-
- /**
- * Delete the version marker and verify that table init fails.
- */
- @Test
- public void testTableVersionRequired() throws Exception {
- Configuration conf = getFileSystem().getConf();
- int maxRetries = conf.getInt(S3GUARD_DDB_MAX_RETRIES,
- S3GUARD_DDB_MAX_RETRIES_DEFAULT);
- conf.setInt(S3GUARD_DDB_MAX_RETRIES, 3);
-
- final DynamoDBMetadataStore ddbms = createContract(conf).getMetadataStore();
- String tableName = conf.get(S3GUARD_DDB_TABLE_NAME_KEY, BUCKET);
- Table table = verifyTableInitialized(tableName);
- table.deleteItem(VERSION_MARKER_PRIMARY_KEY);
-
- // create existing table
- intercept(IOException.class, E_NO_VERSION_MARKER,
- new VoidCallable() {
- @Override
- public void call() throws Exception {
- ddbms.initTable();
- }
- });
-
- conf.setInt(S3GUARD_DDB_MAX_RETRIES, maxRetries);
- }
-
- /**
- * Set the version value to a different number and verify that
- * table init fails.
- */
- @Test
- public void testTableVersionMismatch() throws Exception {
- final DynamoDBMetadataStore ddbms = createContract().getMetadataStore();
- String tableName = getFileSystem().getConf()
- .get(S3GUARD_DDB_TABLE_NAME_KEY, BUCKET);
- Table table = verifyTableInitialized(tableName);
- table.deleteItem(VERSION_MARKER_PRIMARY_KEY);
- Item v200 = createVersionMarker(VERSION_MARKER, 200, 0);
- table.putItem(v200);
-
- // create existing table
- intercept(IOException.class, E_INCOMPATIBLE_VERSION,
- new VoidCallable() {
- @Override
- public void call() throws Exception {
- ddbms.initTable();
- }
- });
- }
-
- /**
- * Test that initTable fails with IOException when table does not exist and
- * table auto-creation is disabled.
- */
- @Test
- public void testFailNonexistentTable() throws IOException {
- final String tableName = "testFailNonexistentTable";
- final S3AFileSystem s3afs = getFileSystem();
- final Configuration conf = s3afs.getConf();
- conf.set(S3GUARD_DDB_TABLE_NAME_KEY, tableName);
- conf.unset(S3GUARD_DDB_TABLE_CREATE_KEY);
- try (DynamoDBMetadataStore ddbms = new DynamoDBMetadataStore()) {
- ddbms.initialize(s3afs);
- fail("Should have failed as table does not exist and table auto-creation"
- + " is disabled");
- } catch (IOException ignored) {
- }
- }
-
- /**
- * Test cases about root directory as it is not in the DynamoDB table.
- */
- @Test
- public void testRootDirectory() throws IOException {
- final DynamoDBMetadataStore ddbms = getDynamoMetadataStore();
- Path rootPath = new Path(S3URI);
- verifyRootDirectory(ddbms.get(rootPath), true);
-
- ddbms.put(new PathMetadata(new S3AFileStatus(true,
- new Path(rootPath, "foo"),
- UserGroupInformation.getCurrentUser().getShortUserName())));
- verifyRootDirectory(ddbms.get(new Path(S3URI)), false);
- }
-
- private void verifyRootDirectory(PathMetadata rootMeta, boolean isEmpty) {
- assertNotNull(rootMeta);
- final FileStatus status = rootMeta.getFileStatus();
- assertNotNull(status);
- assertTrue(status.isDirectory());
- // UNKNOWN is always a valid option, but true / false should not contradict
- if (isEmpty) {
- assertNotSame("Should not be marked non-empty",
- Tristate.FALSE,
- rootMeta.isEmptyDirectory());
- } else {
- assertNotSame("Should not be marked empty",
- Tristate.TRUE,
- rootMeta.isEmptyDirectory());
- }
- }
-
- /**
- * Test that when moving nested paths, all its ancestors up to destination
- * root will also be created.
- * Here is the directory tree before move:
- *
- * testMovePopulateAncestors
- * ├── a
- * │ └── b
- * │ └── src
- * │ ├── dir1
- * │ │ └── dir2
- * │ └── file1.txt
- * └── c
- * └── d
- * └── dest
- *
- * As part of rename(a/b/src, d/c/dest), S3A will enumerate the subtree at
- * a/b/src. This test verifies that after the move, the new subtree at
- * 'dest' is reachable from the root (i.e. c/ and c/d exist in the table.
- * DynamoDBMetadataStore depends on this property to do recursive delete
- * without a full table scan.
- */
- @Test
- public void testMovePopulatesAncestors() throws IOException {
- final DynamoDBMetadataStore ddbms = getDynamoMetadataStore();
- final String testRoot = "/testMovePopulatesAncestors";
- final String srcRoot = testRoot + "/a/b/src";
- final String destRoot = testRoot + "/c/d/e/dest";
-
- final Path nestedPath1 = strToPath(srcRoot + "/file1.txt");
- ddbms.put(new PathMetadata(basicFileStatus(nestedPath1, 1024, false)));
- final Path nestedPath2 = strToPath(srcRoot + "/dir1/dir2");
- ddbms.put(new PathMetadata(basicFileStatus(nestedPath2, 0, true)));
-
- // We don't put the destRoot path here, since put() would create ancestor
- // entries, and we want to ensure that move() does it, instead.
-
- // Build enumeration of src / dest paths and do the move()
- final Collection fullSourcePaths = Lists.newArrayList(
- strToPath(srcRoot),
- strToPath(srcRoot + "/dir1"),
- strToPath(srcRoot + "/dir1/dir2"),
- strToPath(srcRoot + "/file1.txt")
- );
- final Collection pathsToCreate = Lists.newArrayList(
- new PathMetadata(basicFileStatus(strToPath(destRoot),
- 0, true)),
- new PathMetadata(basicFileStatus(strToPath(destRoot + "/dir1"),
- 0, true)),
- new PathMetadata(basicFileStatus(strToPath(destRoot + "/dir1/dir2"),
- 0, true)),
- new PathMetadata(basicFileStatus(strToPath(destRoot + "/file1.txt"),
- 1024, false))
- );
-
- ddbms.move(fullSourcePaths, pathsToCreate);
-
- // assert that all the ancestors should have been populated automatically
- assertCached(testRoot + "/c");
- assertCached(testRoot + "/c/d");
- assertCached(testRoot + "/c/d/e");
- assertCached(destRoot /* /c/d/e/dest */);
-
- // Also check moved files while we're at it
- assertCached(destRoot + "/dir1");
- assertCached(destRoot + "/dir1/dir2");
- assertCached(destRoot + "/file1.txt");
- }
-
- @Test
- public void testProvisionTable() throws IOException {
- final DynamoDBMetadataStore ddbms = getDynamoMetadataStore();
- final String tableName = ddbms.getTable().getTableName();
- final ProvisionedThroughputDescription oldProvision =
- dynamoDB.getTable(tableName).describe().getProvisionedThroughput();
- ddbms.provisionTable(oldProvision.getReadCapacityUnits() * 2,
- oldProvision.getWriteCapacityUnits() * 2);
- final ProvisionedThroughputDescription newProvision =
- dynamoDB.getTable(tableName).describe().getProvisionedThroughput();
- LOG.info("Old provision = {}, new provision = {}",
- oldProvision, newProvision);
- assertEquals(oldProvision.getReadCapacityUnits() * 2,
- newProvision.getReadCapacityUnits().longValue());
- assertEquals(oldProvision.getWriteCapacityUnits() * 2,
- newProvision.getWriteCapacityUnits().longValue());
- }
-
- @Test
- public void testDeleteTable() throws Exception {
- final String tableName = "testDeleteTable";
- final S3AFileSystem s3afs = getFileSystem();
- final Configuration conf = s3afs.getConf();
- conf.set(S3GUARD_DDB_TABLE_NAME_KEY, tableName);
- try (DynamoDBMetadataStore ddbms = new DynamoDBMetadataStore()) {
- ddbms.initialize(s3afs);
- // we can list the empty table
- ddbms.listChildren(new Path(S3URI));
-
- ddbms.destroy();
- verifyTableNotExist(tableName);
-
- // delete table once more; be ResourceNotFoundException swallowed silently
- ddbms.destroy();
- verifyTableNotExist(tableName);
- try {
- // we can no longer list the destroyed table
- ddbms.listChildren(new Path(S3URI));
- fail("Should have failed after the table is destroyed!");
- } catch (IOException ignored) {
- }
- }
- }
-
- /**
- * This validates the table is created and ACTIVE in DynamoDB.
- *
- * This should not rely on the {@link DynamoDBMetadataStore} implementation.
- * Return the table
- */
- private static Table verifyTableInitialized(String tableName) {
- final Table table = dynamoDB.getTable(tableName);
- final TableDescription td = table.describe();
- assertEquals(tableName, td.getTableName());
- assertEquals("ACTIVE", td.getTableStatus());
- return table;
- }
-
- /**
- * This validates the table is not found in DynamoDB.
- *
- * This should not rely on the {@link DynamoDBMetadataStore} implementation.
- */
- private static void verifyTableNotExist(String tableName) throws Exception{
- intercept(ResourceNotFoundException.class,
- () -> dynamoDB.getTable(tableName).describe());
- }
-
-}