Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-16393. S3Guard init command uses global settings, not those of target bucket. 

Contributed by Steve Loughran.

Change-Id: I226a91ab8d7758340f8d221aa80a7abf9a0d3e8f
This commit is contained in:
Steve Loughran 2019-07-10 20:57:02 +01:00
parent 6872efcabf
commit c7b5f858a0
No known key found for this signature in database
GPG Key ID: D22CF846DBB162A0
2 changed files with 15 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
View File

@ -65,6 +65,7 @@ import org.apache.hadoop.util.ToolRunner;
import static org.apache.hadoop.fs.s3a.Constants.*; import static org.apache.hadoop.fs.s3a.Constants.*;
import static org.apache.hadoop.fs.s3a.Invoker.LOG_EVENT; import static org.apache.hadoop.fs.s3a.Invoker.LOG_EVENT;
import static org.apache.hadoop.fs.s3a.S3AUtils.clearBucketOption; import static org.apache.hadoop.fs.s3a.S3AUtils.clearBucketOption;
import static org.apache.hadoop.fs.s3a.S3AUtils.propagateBucketOptions;
import static org.apache.hadoop.service.launcher.LauncherExitCodes.*; import static org.apache.hadoop.service.launcher.LauncherExitCodes.*;
/** /**
@ -480,6 +481,13 @@ public abstract class S3GuardTool extends Configured implements Tool {
int writeCapacity = Integer.parseInt(writeCap); int writeCapacity = Integer.parseInt(writeCap);
getConf().setInt(S3GUARD_DDB_TABLE_CAPACITY_WRITE_KEY, writeCapacity); getConf().setInt(S3GUARD_DDB_TABLE_CAPACITY_WRITE_KEY, writeCapacity);
} }
if (!paths.isEmpty()) {
String s3path = paths.get(0);
URI fsURI = new URI(s3path);
Configuration bucketConf = propagateBucketOptions(getConf(),
fsURI.getHost());
setConf(bucketConf);
}
String tags = getCommandFormat().getOptValue(TAG_FLAG); String tags = getCommandFormat().getOptValue(TAG_FLAG);
if (tags != null && !tags.isEmpty()) { if (tags != null && !tags.isEmpty()) {

8
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java
View File

@ -45,6 +45,7 @@ import org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.Init;
import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_REGION_KEY; import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_REGION_KEY;
import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_TABLE_NAME_KEY; import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_TABLE_NAME_KEY;
import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_TABLE_TAG; import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_TABLE_TAG;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBucketOverrides;
import static org.apache.hadoop.fs.s3a.S3AUtils.setBucketOption; import static org.apache.hadoop.fs.s3a.S3AUtils.setBucketOption;
import static org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStore.*; import static org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStore.*;
import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.*; import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.*;
@ -120,6 +121,11 @@ public class ITestS3GuardToolDynamoDB extends AbstractS3GuardToolTestBase {
conf.set(S3GUARD_DDB_TABLE_NAME_KEY, conf.set(S3GUARD_DDB_TABLE_NAME_KEY,
getTestTableName("testDynamoTableTagging-" + UUID.randomUUID())); getTestTableName("testDynamoTableTagging-" + UUID.randomUUID()));
String bucket = getFileSystem().getBucket();
removeBucketOverrides(bucket, conf,
S3GUARD_DDB_TABLE_NAME_KEY,
S3GUARD_DDB_REGION_KEY);
S3GuardTool.Init cmdR = new S3GuardTool.Init(conf); S3GuardTool.Init cmdR = new S3GuardTool.Init(conf);
Map<String, String> tagMap = new HashMap<>(); Map<String, String> tagMap = new HashMap<>();
tagMap.put("hello", "dynamo"); tagMap.put("hello", "dynamo");
@ -128,7 +134,7 @@ public class ITestS3GuardToolDynamoDB extends AbstractS3GuardToolTestBase {
String[] argsR = new String[]{ String[] argsR = new String[]{
cmdR.getName(), cmdR.getName(),
"-tag", tagMapToStringParams(tagMap), "-tag", tagMapToStringParams(tagMap),
getFileSystem().getBucket() "s3a://" + bucket + "/"
}; };
// run // run