HADOOP-16252. Add prefix to dynamo tables in tests.
Contributed by Ben Roling.
This commit is contained in:
parent
e95c055d91
commit
e1c5ddf2aa
|
@ -1074,6 +1074,12 @@ If the `s3guard` profile *is* set,
|
|||
property should be configured, and the name of that table should be different
|
||||
than what is used for fs.s3a.s3guard.ddb.table. The test table is destroyed
|
||||
and modified multiple times during the test.
|
||||
1. Several of the tests create and destroy DynamoDB tables. The table names
|
||||
are prefixed with the value defined by
|
||||
`fs.s3a.s3guard.test.dynamo.table.prefix` (default="s3guard.test."). The user
|
||||
executing the tests will need sufficient privilege to create and destroy such
|
||||
tables. If the tests abort uncleanly, these tables may be left behind,
|
||||
incurring AWS charges.
|
||||
|
||||
|
||||
### Scale Testing MetadataStore Directly
|
||||
|
|
|
@ -33,6 +33,7 @@ import java.io.IOException;
|
|||
|
||||
import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
|
||||
import static org.apache.hadoop.fs.contract.ContractTestUtils.writeDataset;
|
||||
import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestDynamoTablePrefix;
|
||||
|
||||
/**
|
||||
* An extension of the contract test base set up for S3A tests.
|
||||
|
@ -129,6 +130,10 @@ public abstract class AbstractS3ATestBase extends AbstractFSContractTestBase
|
|||
ContractTestUtils.verifyFileContents(getFileSystem(), path, data);
|
||||
}
|
||||
|
||||
protected String getTestTableName(String suffix) {
|
||||
return getTestDynamoTablePrefix(getConfiguration()) + suffix;
|
||||
}
|
||||
|
||||
/**
|
||||
* Assert that an exception failed with a specific status code.
|
||||
* @param e exception
|
||||
|
|
|
@ -158,6 +158,10 @@ public interface S3ATestConstants {
|
|||
String TEST_S3GUARD_IMPLEMENTATION_DYNAMO = "dynamo";
|
||||
String TEST_S3GUARD_IMPLEMENTATION_NONE = "none";
|
||||
|
||||
String TEST_S3GUARD_DYNAMO_TABLE_PREFIX =
|
||||
"fs.s3a.s3guard.test.dynamo.table.prefix";
|
||||
String TEST_S3GUARD_DYNAMO_TABLE_PREFIX_DEFAULT = "s3guard.test.";
|
||||
|
||||
/**
|
||||
* Timeout in Milliseconds for standard tests: {@value}.
|
||||
*/
|
||||
|
|
|
@ -708,6 +708,16 @@ public final class S3ATestUtils {
|
|||
return URI.create(bucket).getHost();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the prefix for DynamoDB table names used in tests.
|
||||
* @param conf configuration to scan.
|
||||
* @return the table name prefix
|
||||
*/
|
||||
public static String getTestDynamoTablePrefix(final Configuration conf) {
|
||||
return getTestProperty(conf, TEST_S3GUARD_DYNAMO_TABLE_PREFIX,
|
||||
TEST_S3GUARD_DYNAMO_TABLE_PREFIX_DEFAULT);
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove any values from a bucket.
|
||||
* @param bucket bucket whose overrides are to be removed. Can be null/empty
|
||||
|
|
|
@ -59,6 +59,7 @@ import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_TABLE_CREATE_KEY;
|
|||
import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_TABLE_NAME_KEY;
|
||||
import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_METASTORE_NULL;
|
||||
import static org.apache.hadoop.fs.s3a.Constants.S3_METADATA_STORE_IMPL;
|
||||
import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestDynamoTablePrefix;
|
||||
import static org.apache.hadoop.fs.s3a.S3AUtils.clearBucketOption;
|
||||
import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.E_BAD_STATE;
|
||||
import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.SUCCESS;
|
||||
|
@ -71,7 +72,7 @@ import static org.apache.hadoop.test.LambdaTestUtils.intercept;
|
|||
public abstract class AbstractS3GuardToolTestBase extends AbstractS3ATestBase {
|
||||
|
||||
protected static final String OWNER = "hdfs";
|
||||
protected static final String DYNAMODB_TABLE = "dynamodb://ireland-team";
|
||||
protected static final String DYNAMODB_TABLE = "ireland-team";
|
||||
protected static final String S3A_THIS_BUCKET_DOES_NOT_EXIST
|
||||
= "s3a://this-bucket-does-not-exist-00000000000";
|
||||
|
||||
|
@ -520,7 +521,8 @@ public abstract class AbstractS3GuardToolTestBase extends AbstractS3ATestBase {
|
|||
ByteArrayOutputStream buf = new ByteArrayOutputStream();
|
||||
S3GuardTool.Diff cmd = new S3GuardTool.Diff(fs.getConf());
|
||||
cmd.setStore(ms);
|
||||
exec(0, "", cmd, buf, "diff", "-meta", DYNAMODB_TABLE, testPath.toString());
|
||||
String table = "dynamo://" + getTestTableName(DYNAMODB_TABLE);
|
||||
exec(0, "", cmd, buf, "diff", "-meta", table, testPath.toString());
|
||||
|
||||
Set<Path> actualOnS3 = new HashSet<>();
|
||||
Set<Path> actualOnMS = new HashSet<>();
|
||||
|
|
|
@ -269,7 +269,8 @@ public class ITestDynamoDBMetadataStore extends MetadataStoreTestBase {
|
|||
@Test
|
||||
public void testInitialize() throws IOException {
|
||||
final S3AFileSystem s3afs = this.fileSystem;
|
||||
final String tableName = "testInitialize";
|
||||
final String tableName =
|
||||
getTestTableName("testInitialize");
|
||||
final Configuration conf = s3afs.getConf();
|
||||
conf.set(S3GUARD_DDB_TABLE_NAME_KEY, tableName);
|
||||
try (DynamoDBMetadataStore ddbms = new DynamoDBMetadataStore()) {
|
||||
|
@ -293,7 +294,8 @@ public class ITestDynamoDBMetadataStore extends MetadataStoreTestBase {
|
|||
*/
|
||||
@Test
|
||||
public void testInitializeWithConfiguration() throws IOException {
|
||||
final String tableName = "testInitializeWithConfiguration";
|
||||
final String tableName =
|
||||
getTestTableName("testInitializeWithConfiguration");
|
||||
final Configuration conf = getFileSystem().getConf();
|
||||
conf.unset(S3GUARD_DDB_TABLE_NAME_KEY);
|
||||
String savedRegion = conf.get(S3GUARD_DDB_REGION_KEY,
|
||||
|
@ -430,7 +432,7 @@ public class ITestDynamoDBMetadataStore extends MetadataStoreTestBase {
|
|||
*/
|
||||
@Test
|
||||
public void testTableVersionRequired() throws Exception {
|
||||
String tableName = "testTableVersionRequired";
|
||||
String tableName = getTestTableName("testTableVersionRequired");
|
||||
Configuration conf = getFileSystem().getConf();
|
||||
int maxRetries = conf.getInt(S3GUARD_DDB_MAX_RETRIES,
|
||||
S3GUARD_DDB_MAX_RETRIES_DEFAULT);
|
||||
|
@ -457,7 +459,7 @@ public class ITestDynamoDBMetadataStore extends MetadataStoreTestBase {
|
|||
*/
|
||||
@Test
|
||||
public void testTableVersionMismatch() throws Exception {
|
||||
String tableName = "testTableVersionMismatch";
|
||||
String tableName = getTestTableName("testTableVersionMismatch");
|
||||
Configuration conf = getFileSystem().getConf();
|
||||
conf.set(S3GUARD_DDB_TABLE_NAME_KEY, tableName);
|
||||
|
||||
|
@ -484,7 +486,8 @@ public class ITestDynamoDBMetadataStore extends MetadataStoreTestBase {
|
|||
*/
|
||||
@Test
|
||||
public void testFailNonexistentTable() throws IOException {
|
||||
final String tableName = "testFailNonexistentTable";
|
||||
final String tableName =
|
||||
getTestTableName("testFailNonexistentTable");
|
||||
final S3AFileSystem s3afs = getFileSystem();
|
||||
final Configuration conf = s3afs.getConf();
|
||||
conf.set(S3GUARD_DDB_TABLE_NAME_KEY, tableName);
|
||||
|
@ -600,7 +603,8 @@ public class ITestDynamoDBMetadataStore extends MetadataStoreTestBase {
|
|||
|
||||
@Test
|
||||
public void testProvisionTable() throws Exception {
|
||||
final String tableName = "testProvisionTable-" + UUID.randomUUID();
|
||||
final String tableName
|
||||
= getTestTableName("testProvisionTable-" + UUID.randomUUID());
|
||||
Configuration conf = getFileSystem().getConf();
|
||||
conf.set(S3GUARD_DDB_TABLE_NAME_KEY, tableName);
|
||||
|
||||
|
@ -631,7 +635,7 @@ public class ITestDynamoDBMetadataStore extends MetadataStoreTestBase {
|
|||
|
||||
@Test
|
||||
public void testDeleteTable() throws Exception {
|
||||
final String tableName = "testDeleteTable";
|
||||
final String tableName = getTestTableName("testDeleteTable");
|
||||
Path testPath = new Path(new Path(fsUri), "/" + tableName);
|
||||
final S3AFileSystem s3afs = getFileSystem();
|
||||
final Configuration conf = s3afs.getConf();
|
||||
|
@ -666,7 +670,8 @@ public class ITestDynamoDBMetadataStore extends MetadataStoreTestBase {
|
|||
propKey -> conf.unset(S3GUARD_DDB_TABLE_TAG + propKey)
|
||||
);
|
||||
|
||||
String tableName = "testTableTagging-" + UUID.randomUUID();
|
||||
String tableName =
|
||||
getTestTableName("testTableTagging-" + UUID.randomUUID());
|
||||
conf.set(S3GUARD_DDB_TABLE_NAME_KEY, tableName);
|
||||
conf.set(S3GUARD_DDB_TABLE_CREATE_KEY, "true");
|
||||
|
||||
|
@ -765,4 +770,7 @@ public class ITestDynamoDBMetadataStore extends MetadataStoreTestBase {
|
|||
() -> dynamoDB.getTable(tableName).describe());
|
||||
}
|
||||
|
||||
private String getTestTableName(String suffix) {
|
||||
return getTestDynamoTablePrefix(s3AContract.getConf()) + suffix;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -106,7 +106,9 @@ public class ITestS3GuardConcurrentOps extends AbstractS3ATestBase {
|
|||
try {
|
||||
DynamoDB db = ms.getDynamoDB();
|
||||
|
||||
String tableName = "testConcurrentTableCreations" + new Random().nextInt();
|
||||
String tableName =
|
||||
getTestTableName("testConcurrentTableCreations" +
|
||||
new Random().nextInt());
|
||||
conf.setBoolean(Constants.S3GUARD_DDB_TABLE_CREATE_KEY, true);
|
||||
conf.set(Constants.S3GUARD_DDB_TABLE_NAME_KEY, tableName);
|
||||
|
||||
|
|
|
@ -44,6 +44,7 @@ import org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.Init;
|
|||
import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_REGION_KEY;
|
||||
import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_TABLE_NAME_KEY;
|
||||
import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_TABLE_TAG;
|
||||
import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestDynamoTablePrefix;
|
||||
import static org.apache.hadoop.fs.s3a.S3AUtils.setBucketOption;
|
||||
import static org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStore.*;
|
||||
import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.*;
|
||||
|
@ -80,7 +81,8 @@ public class ITestS3GuardToolDynamoDB extends AbstractS3GuardToolTestBase {
|
|||
|
||||
@Test
|
||||
public void testInvalidRegion() throws Exception {
|
||||
final String testTableName = "testInvalidRegion" + new Random().nextInt();
|
||||
final String testTableName =
|
||||
getTestTableName("testInvalidRegion" + new Random().nextInt());
|
||||
final String testRegion = "invalidRegion";
|
||||
// Initialize MetadataStore
|
||||
final Init initCmd = new Init(getFileSystem().getConf());
|
||||
|
@ -117,7 +119,7 @@ public class ITestS3GuardToolDynamoDB extends AbstractS3GuardToolTestBase {
|
|||
);
|
||||
|
||||
conf.set(S3GUARD_DDB_TABLE_NAME_KEY,
|
||||
"testDynamoTableTagging-" + UUID.randomUUID());
|
||||
getTestTableName("testDynamoTableTagging-" + UUID.randomUUID()));
|
||||
S3GuardTool.Init cmdR = new S3GuardTool.Init(conf);
|
||||
Map<String, String> tagMap = new HashMap<>();
|
||||
tagMap.put("hello", "dynamo");
|
||||
|
@ -165,7 +167,8 @@ public class ITestS3GuardToolDynamoDB extends AbstractS3GuardToolTestBase {
|
|||
|
||||
@Test
|
||||
public void testDynamoDBInitDestroyCycle() throws Throwable {
|
||||
String testTableName = "testDynamoDBInitDestroy" + new Random().nextInt();
|
||||
String testTableName =
|
||||
getTestTableName("testDynamoDBInitDestroy" + new Random().nextInt());
|
||||
String testS3Url = path(testTableName).toString();
|
||||
S3AFileSystem fs = getFileSystem();
|
||||
DynamoDB db = null;
|
||||
|
@ -284,7 +287,7 @@ public class ITestS3GuardToolDynamoDB extends AbstractS3GuardToolTestBase {
|
|||
public void testDestroyUnknownTable() throws Throwable {
|
||||
run(S3GuardTool.Destroy.NAME,
|
||||
"-region", "us-west-2",
|
||||
"-meta", DYNAMODB_TABLE);
|
||||
"-meta", "dynamodb://" + getTestTableName(DYNAMODB_TABLE));
|
||||
}
|
||||
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue