Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-15428. s3guard bucket-info will create s3guard table if FS is set to do this automatically. (Contributed by Gabor Bota)

This commit is contained in:
Sean Mackrory 2018-12-07 09:56:16 -07:00
parent 80e59e7876
commit 3ff8580f22
3 changed files with 29 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
View File

@ -1061,7 +1061,7 @@ public abstract class S3GuardTool extends Configured implements Tool {
+ "\t" + PURPOSE + "\n\n"
+ "Common options:\n"
+ " -" + GUARDED_FLAG + " - Require S3Guard\n"
+ " -" + UNGUARDED_FLAG + " - Require S3Guard to be disabled\n"
+ " -" + UNGUARDED_FLAG + " - Force S3Guard to be disabled\n"
+ " -" + AUTH_FLAG + " - Require the S3Guard mode to be \"authoritative\"\n"
+ " -" + NONAUTH_FLAG + " - Require the S3Guard mode to be \"non-authoritative\"\n"
+ " -" + MAGIC_FLAG + " - Require the S3 filesystem to be support the \"magic\" committer\n"
@ -1092,6 +1092,15 @@ public abstract class S3GuardTool extends Configured implements Tool {
throw invalidArgs("No bucket specified");
}
String s3Path = paths.get(0);
CommandFormat commands = getCommandFormat();
// check if UNGUARDED_FLAG is passed and use NullMetadataStore in
// config to avoid side effects like creating the table if not exists
if (commands.getOpt(UNGUARDED_FLAG)) {
LOG.debug("Unguarded flag is passed to command :" + this.getName());
getConf().set(S3_METADATA_STORE_IMPL, S3GUARD_METASTORE_NULL);
}
S3AFileSystem fs = (S3AFileSystem) FileSystem.newInstance(
toUri(s3Path), getConf());
setFilesystem(fs);
@ -1128,7 +1137,6 @@ public abstract class S3GuardTool extends Configured implements Tool {
"none");
printOption(out, "\tInput seek policy", INPUT_FADVISE, INPUT_FADV_NORMAL);
CommandFormat commands = getCommandFormat();
if (usingS3Guard) {
if (commands.getOpt(UNGUARDED_FLAG)) {
throw badState("S3Guard is enabled for %s", fsUri);

2
hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3guard.md
View File

@ -505,7 +505,7 @@ Options
| argument | meaning |
|-----------|-------------|
| `-guarded` | Require S3Guard to be enabled |
| `-unguarded` | Require S3Guard to be disabled |
| `-unguarded` | Force S3Guard to be disabled |
| `-auth` | Require the S3Guard mode to be "authoritative" |
| `-nonauth` | Require the S3Guard mode to be "non-authoritative" |
| `-magic` | Require the S3 filesystem to be support the "magic" committer |

18
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
View File

@ -316,6 +316,24 @@ public abstract class AbstractS3GuardToolTestBase extends AbstractS3ATestBase {
S3GuardTool.SetCapacity.WRITE_CAP_INVALID, () -> cmdW.run(argsW));
}
@Test
public void testBucketInfoUnguarded() throws Exception {
final Configuration conf = getConfiguration();
conf.set(S3GUARD_DDB_TABLE_CREATE_KEY, Boolean.FALSE.toString());
conf.set(S3GUARD_DDB_TABLE_NAME_KEY,
"testBucketInfoUnguarded-" + UUID.randomUUID());
// run a bucket info command and look for
// confirmation that it got the output from DDB diags
S3GuardTool.BucketInfo infocmd = new S3GuardTool.BucketInfo(conf);
String info = exec(infocmd, S3GuardTool.BucketInfo.NAME,
"-" + S3GuardTool.BucketInfo.UNGUARDED_FLAG,
getFileSystem().getUri().toString());
assertTrue("Output should contain information about S3A client " + info,
info.contains("S3A Client"));
}
@Test
public void testSetCapacityFailFastIfNotGuarded() throws Exception{
Configuration conf = getConfiguration();