Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-14220 Enhance S3GuardTool with bucket-info and set-capacity commands, tests. 

Contributed by Steve Loughran
This commit is contained in:
Steve Loughran 2017-10-03 19:59:09 +01:00
parent 42e873d4ea
commit 28e05dc1da
No known key found for this signature in database
GPG Key ID: 950CC3E032B79CA2
17 changed files with 1418 additions and 188 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
View File

@ -292,6 +292,10 @@ public class S3AFileSystem extends FileSystem {
metadataStore = S3Guard.getMetadataStore(this);
allowAuthoritative = conf.getBoolean(METADATASTORE_AUTHORITATIVE,
DEFAULT_METADATASTORE_AUTHORITATIVE);
if (hasMetadataStore()) {
LOG.debug("Using metadata store {}, authoritative={}",
getMetadataStore(), allowAuthoritative);
}
} catch (AmazonClientException e) {
throw translateException("initializing ", new Path(name), e);
}
@ -975,7 +979,7 @@ public class S3AFileSystem extends FileSystem {
* @return the metadata store of this FS instance
*/
@VisibleForTesting
MetadataStore getMetadataStore() {
public MetadataStore getMetadataStore() {
return metadataStore;
}
@ -2430,9 +2434,11 @@ public class S3AFileSystem extends FileSystem {
sb.append(", statistics {")
.append(statistics)
.append("}");
sb.append(", metrics {")
.append(instrumentation.dump("{", "=", "} ", true))
.append("}");
if (instrumentation != null) {
sb.append(", metrics {")
.append(instrumentation.dump("{", "=", "} ", true))
.append("}");
}
sb.append('}');
return sb.toString();
}

60
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
View File

@ -743,9 +743,11 @@ public final class S3AUtils {
// propagate the value, building a new origin field.
// to track overwrites, the generic key is overwritten even if
// already matches the new one.
String origin = "[" + StringUtils.join(
source.getPropertySources(key), ", ") +"]";
final String generic = FS_S3A_PREFIX + stripped;
LOG.debug("Updating {}", generic);
dest.set(generic, value, key);
LOG.debug("Updating {} from {}", generic, origin);
dest.set(generic, value, key + " via " + origin);
}
}
return dest;
@ -888,4 +890,58 @@ public final class S3AUtils {
}
}
/**
* Set a bucket-specific property to a particular value.
* If the generic key passed in has an {@code fs.s3a. prefix},
* that's stripped off, so that when the the bucket properties are propagated
* down to the generic values, that value gets copied down.
* @param conf configuration to set
* @param bucket bucket name
* @param genericKey key; can start with "fs.s3a."
* @param value value to set
*/
public static void setBucketOption(Configuration conf, String bucket,
String genericKey, String value) {
final String baseKey = genericKey.startsWith(FS_S3A_PREFIX) ?
genericKey.substring(FS_S3A_PREFIX.length())
: genericKey;
conf.set(FS_S3A_BUCKET_PREFIX + bucket + '.' + baseKey, value, "S3AUtils");
}
/**
* Clear a bucket-specific property.
* If the generic key passed in has an {@code fs.s3a. prefix},
* that's stripped off, so that when the the bucket properties are propagated
* down to the generic values, that value gets copied down.
* @param conf configuration to set
* @param bucket bucket name
* @param genericKey key; can start with "fs.s3a."
*/
public static void clearBucketOption(Configuration conf, String bucket,
String genericKey) {
final String baseKey = genericKey.startsWith(FS_S3A_PREFIX) ?
genericKey.substring(FS_S3A_PREFIX.length())
: genericKey;
String k = FS_S3A_BUCKET_PREFIX + bucket + '.' + baseKey;
LOG.debug("Unset {}", k);
conf.unset(k);
}
/**
* Get a bucket-specific property.
* If the generic key passed in has an {@code fs.s3a. prefix},
* that's stripped off.
* @param conf configuration to set
* @param bucket bucket name
* @param genericKey key; can start with "fs.s3a."
* @return the bucket option, null if there is none
*/
public static String getBucketOption(Configuration conf, String bucket,
String genericKey) {
final String baseKey = genericKey.startsWith(FS_S3A_PREFIX) ?
genericKey.substring(FS_S3A_PREFIX.length())
: genericKey;
return conf.get(FS_S3A_BUCKET_PREFIX + bucket + '.' + baseKey);
}
}

104
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
View File

@ -29,6 +29,7 @@ import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import java.util.concurrent.TimeUnit;
import com.amazonaws.AmazonClientException;
@ -51,6 +52,7 @@ import com.amazonaws.services.dynamodbv2.model.ProvisionedThroughput;
import com.amazonaws.services.dynamodbv2.model.ProvisionedThroughputDescription;
import com.amazonaws.services.dynamodbv2.model.ResourceInUseException;
import com.amazonaws.services.dynamodbv2.model.ResourceNotFoundException;
import com.amazonaws.services.dynamodbv2.model.TableDescription;
import com.amazonaws.services.dynamodbv2.model.WriteRequest;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
@ -184,6 +186,18 @@ public class DynamoDBMetadataStore implements MetadataStore {
* DynamoDB. Value is {@value} msec. */
public static final long MIN_RETRY_SLEEP_MSEC = 100;
@VisibleForTesting
static final String DESCRIPTION
= "S3Guard metadata store in DynamoDB";
@VisibleForTesting
static final String READ_CAPACITY = "read-capacity";
@VisibleForTesting
static final String WRITE_CAPACITY = "write-capacity";
@VisibleForTesting
static final String STATUS = "status";
@VisibleForTesting
static final String TABLE = "table";
private static ValueMap deleteTrackingValueMap =
new ValueMap().withBoolean(":false", false);
@ -788,7 +802,9 @@ public class DynamoDBMetadataStore implements MetadataStore {
try {
try {
LOG.debug("Binding to table {}", tableName);
final String status = table.describe().getTableStatus();
TableDescription description = table.describe();
LOG.debug("Table state: {}", description);
final String status = description.getTableStatus();
switch (status) {
case "CREATING":
case "UPDATING":
@ -824,9 +840,10 @@ public class DynamoDBMetadataStore implements MetadataStore {
createTable(capacity);
} else {
throw new FileNotFoundException("DynamoDB table "
+ "'" + tableName + "' does not "
+ "exist in region " + region + "; auto-creation is turned off");
throw (FileNotFoundException)new FileNotFoundException(
"DynamoDB table '" + tableName + "' does not "
+ "exist in region " + region + "; auto-creation is turned off")
.initCause(rnfe);
}
}
@ -1007,4 +1024,83 @@ public class DynamoDBMetadataStore implements MetadataStore {
Preconditions.checkNotNull(meta.getFileStatus().getPath());
}
@Override
public Map<String, String> getDiagnostics() throws IOException {
Map<String, String> map = new TreeMap<>();
if (table != null) {
TableDescription desc = getTableDescription(true);
map.put("name", desc.getTableName());
map.put(STATUS, desc.getTableStatus());
map.put("ARN", desc.getTableArn());
map.put("size", desc.getTableSizeBytes().toString());
map.put(TABLE, desc.toString());
ProvisionedThroughputDescription throughput
= desc.getProvisionedThroughput();
map.put(READ_CAPACITY, throughput.getReadCapacityUnits().toString());
map.put(WRITE_CAPACITY, throughput.getWriteCapacityUnits().toString());
map.put(TABLE, desc.toString());
} else {
map.put("name", "DynamoDB Metadata Store");
map.put(TABLE, "none");
map.put(STATUS, "undefined");
}
map.put("description", DESCRIPTION);
map.put("region", region);
if (dataAccessRetryPolicy != null) {
map.put("retryPolicy", dataAccessRetryPolicy.toString());
}
return map;
}
private TableDescription getTableDescription(boolean forceUpdate) {
TableDescription desc = table.getDescription();
if (desc == null || forceUpdate) {
desc = table.describe();
}
return desc;
}
@Override
public void updateParameters(Map<String, String> parameters)
throws IOException {
Preconditions.checkNotNull(table, "Not initialized");
TableDescription desc = getTableDescription(true);
ProvisionedThroughputDescription current
= desc.getProvisionedThroughput();
long currentRead = current.getReadCapacityUnits();
long newRead = getLongParam(parameters,
S3GUARD_DDB_TABLE_CAPACITY_READ_KEY,
currentRead);
long currentWrite = current.getWriteCapacityUnits();
long newWrite = getLongParam(parameters,
S3GUARD_DDB_TABLE_CAPACITY_WRITE_KEY,
currentWrite);
ProvisionedThroughput throughput = new ProvisionedThroughput()
.withReadCapacityUnits(newRead)
.withWriteCapacityUnits(newWrite);
if (newRead != currentRead || newWrite != currentWrite) {
LOG.info("Current table capacity is read: {}, write: {}",
currentRead, currentWrite);
LOG.info("Changing capacity of table to read: {}, write: {}",
newRead, newWrite);
table.updateTable(throughput);
} else {
LOG.info("Table capacity unchanged at read: {}, write: {}",
newRead, newWrite);
}
}
private long getLongParam(Map<String, String> parameters,
String key,
long defVal) {
String k = parameters.get(key);
if (k != null) {
return Long.parseLong(k);
} else {
return defVal;
}
}
}

25
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataStore.java
View File

@ -20,6 +20,8 @@ package org.apache.hadoop.fs.s3a.s3guard;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@ -31,6 +33,7 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.net.URI;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.Map;
@ -100,7 +103,7 @@ public class LocalMetadataStore implements MetadataStore {
public String toString() {
final StringBuilder sb = new StringBuilder(
"LocalMetadataStore{");
sb.append(", uriHost='").append(uriHost).append('\'');
sb.append("uriHost='").append(uriHost).append('\'');
sb.append('}');
return sb.toString();
}
@ -153,7 +156,9 @@ public class LocalMetadataStore implements MetadataStore {
m.setIsEmptyDirectory(isEmptyDirectory(p));
}
LOG.debug("get({}) -> {}", path, m == null ? "null" : m.prettyPrint());
if (LOG.isDebugEnabled()) {
LOG.debug("get({}) -> {}", path, m == null ? "null" : m.prettyPrint());
}
return m;
}
}
@ -424,12 +429,22 @@ public class LocalMetadataStore implements MetadataStore {
Preconditions.checkArgument(p.isAbsolute(), "Path must be absolute");
URI uri = p.toUri();
if (uriHost != null) {
Preconditions.checkArgument(!isEmpty(uri.getHost()));
Preconditions.checkArgument(StringUtils.isNotEmpty(uri.getHost()));
}
return p;
}
private static boolean isEmpty(String s) {
return (s == null || s.isEmpty());
@Override
public Map<String, String> getDiagnostics() throws IOException {
Map<String, String> map = new HashMap<>();
map.put("name", "local://metadata");
map.put("uriHost", uriHost);
map.put("description", "Local in-VM metadata store for testing");
return map;
}
@Override
public void updateParameters(Map<String, String> parameters)
throws IOException {
}
}

17
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStore.java
View File

@ -21,6 +21,7 @@ package org.apache.hadoop.fs.s3a.s3guard;
import java.io.Closeable;
import java.io.IOException;
import java.util.Collection;
import java.util.Map;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.classification.InterfaceAudience;
@ -218,4 +219,20 @@ public interface MetadataStore extends Closeable {
* @throws UnsupportedOperationException if not implemented
*/
void prune(long modTime) throws IOException, UnsupportedOperationException;
/**
* Get any diagnostics information from a store, as a list of (key, value)
* tuples for display. Arbitrary values; no guarantee of stability.
* These are for debugging only.
* @return a map of strings.
* @throws IOException if there is an error
*/
Map<String, String> getDiagnostics() throws IOException;
/**
* Tune/update parameters for an existing table.
* @param parameters map of params to change.
* @throws IOException if there is an error
*/
void updateParameters(Map<String, String> parameters) throws IOException;
}

15
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/NullMetadataStore.java
View File

@ -24,6 +24,8 @@ import org.apache.hadoop.fs.Path;
import java.io.IOException;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
/**
* A no-op implementation of MetadataStore. Clients that use this
@ -101,4 +103,17 @@ public class NullMetadataStore implements MetadataStore {
public String toString() {
return "NullMetadataStore";
}
@Override
public Map<String, String> getDiagnostics() throws IOException {
Map<String, String> map = new HashMap<>();
map.put("name", "Null Metadata Store");
map.put("description", "This is not a real metadata store");
return map;
}
@Override
public void updateParameters(Map<String, String> parameters)
throws IOException {
}
}

17
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3Guard.java
View File

@ -18,6 +18,7 @@
package org.apache.hadoop.fs.s3a.s3guard;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
@ -95,6 +96,10 @@ public final class S3Guard {
msClass.getSimpleName(), fs.getScheme());
msInstance.initialize(fs);
return msInstance;
} catch (FileNotFoundException e) {
// Don't log this exception as it means the table doesn't exist yet;
// rely on callers to catch and treat specially
throw e;
} catch (RuntimeException | IOException e) {
String message = "Failed to instantiate metadata store " +
conf.get(S3_METADATA_STORE_IMPL)
@ -109,14 +114,20 @@ public final class S3Guard {
}
}
private static Class<? extends MetadataStore> getMetadataStoreClass(
static Class<? extends MetadataStore> getMetadataStoreClass(
Configuration conf) {
if (conf == null) {
return NullMetadataStore.class;
}
if (conf.get(S3_METADATA_STORE_IMPL) != null && LOG.isDebugEnabled()) {
LOG.debug("Metastore option source {}",
conf.getPropertySources(S3_METADATA_STORE_IMPL));
}
return conf.getClass(S3_METADATA_STORE_IMPL, NullMetadataStore.class,
MetadataStore.class);
Class<? extends MetadataStore> aClass = conf.getClass(
S3_METADATA_STORE_IMPL, NullMetadataStore.class,
MetadataStore.class);
return aClass;
}

541
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
View File

@ -36,6 +36,7 @@ import com.google.common.base.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileStatus;
@ -43,15 +44,17 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.s3a.Constants;
import org.apache.hadoop.fs.s3a.S3AFileStatus;
import org.apache.hadoop.fs.s3a.S3AFileSystem;
import org.apache.hadoop.fs.s3a.S3AUtils;
import org.apache.hadoop.fs.shell.CommandFormat;
import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.util.GenericOptionsParser;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import static org.apache.hadoop.fs.s3a.Constants.*;
import static org.apache.hadoop.service.launcher.LauncherExitCodes.*;
/**
* CLI to manage S3Guard Metadata Store.
@ -74,40 +77,46 @@ public abstract class S3GuardTool extends Configured implements Tool {
"\t" + Init.NAME + " - " + Init.PURPOSE + "\n" +
"\t" + Destroy.NAME + " - " + Destroy.PURPOSE + "\n" +
"\t" + Import.NAME + " - " + Import.PURPOSE + "\n" +
"\t" + BucketInfo.NAME + " - " + BucketInfo.PURPOSE + "\n" +
"\t" + Diff.NAME + " - " + Diff.PURPOSE + "\n" +
"\t" + Prune.NAME + " - " + Prune.PURPOSE + "\n";
"\t" + Prune.NAME + " - " + Prune.PURPOSE + "\n" +
"\t" + SetCapacity.NAME + " - " +SetCapacity.PURPOSE + "\n";
private static final String DATA_IN_S3_IS_PRESERVED
= "(all data in S3 is preserved";
= "(all data in S3 is preserved)";
abstract public String getUsage();
// Exit codes
static final int SUCCESS = 0;
static final int INVALID_ARGUMENT = 1;
static final int ERROR = 99;
static final int SUCCESS = EXIT_SUCCESS;
static final int INVALID_ARGUMENT = EXIT_COMMAND_ARGUMENT_ERROR;
static final int E_USAGE = EXIT_USAGE;
static final int ERROR = EXIT_FAIL;
static final int E_BAD_STATE = EXIT_NOT_ACCEPTABLE;
static final int E_NOT_FOUND = EXIT_NOT_FOUND;
private S3AFileSystem filesystem;
private MetadataStore store;
private final CommandFormat commandFormat;
private static final String META_FLAG = "meta";
private static final String DAYS_FLAG = "days";
private static final String HOURS_FLAG = "hours";
private static final String MINUTES_FLAG = "minutes";
private static final String SECONDS_FLAG = "seconds";
public static final String META_FLAG = "meta";
public static final String DAYS_FLAG = "days";
public static final String HOURS_FLAG = "hours";
public static final String MINUTES_FLAG = "minutes";
public static final String SECONDS_FLAG = "seconds";
private static final String REGION_FLAG = "region";
private static final String READ_FLAG = "read";
private static final String WRITE_FLAG = "write";
public static final String REGION_FLAG = "region";
public static final String READ_FLAG = "read";
public static final String WRITE_FLAG = "write";
/**
* Constructor a S3Guard tool with HDFS configuration.
* @param conf Configuration.
* @param opts any boolean options to support
*/
protected S3GuardTool(Configuration conf) {
protected S3GuardTool(Configuration conf, String...opts) {
super(conf);
commandFormat = new CommandFormat(0, Integer.MAX_VALUE);
commandFormat = new CommandFormat(0, Integer.MAX_VALUE, opts);
// For metadata store URI
commandFormat.addOptionWithValue(META_FLAG);
// DDB region.
@ -126,10 +135,10 @@ public abstract class S3GuardTool extends Configured implements Tool {
* {@link Destroy}.
*
* @param paths remaining parameters from CLI.
* @return false for invalid parameters.
* @throws IOException on I/O errors.
* @throws ExitUtil.ExitException on validation errors
*/
boolean parseDynamoDBRegion(List<String> paths) throws IOException {
void parseDynamoDBRegion(List<String> paths) throws IOException {
Configuration conf = getConf();
String fromCli = getCommandFormat().getOptValue(REGION_FLAG);
String fromConf = conf.get(S3GUARD_DDB_REGION_KEY);
@ -137,37 +146,34 @@ public abstract class S3GuardTool extends Configured implements Tool {
if (fromCli != null) {
if (fromCli.isEmpty()) {
System.err.println("No region provided with -" + REGION_FLAG + " flag");
return false;
throw invalidArgs("No region provided with -" + REGION_FLAG + " flag");
}
if (hasS3Path) {
System.err.println("Providing both an S3 path and the -" + REGION_FLAG
throw invalidArgs("Providing both an S3 path and the"
+ " -" + REGION_FLAG
+ " flag is not supported. If you need to specify a different "
+ "region than the S3 bucket, configure " + S3GUARD_DDB_REGION_KEY);
return false;
}
conf.set(S3GUARD_DDB_REGION_KEY, fromCli);
return true;
return;
}
if (fromConf != null) {
if (fromConf.isEmpty()) {
System.err.printf("No region provided with config %s, %n",
throw invalidArgs("No region provided with config %s",
S3GUARD_DDB_REGION_KEY);
return false;
}
return true;
return;
}
if (hasS3Path) {
String s3Path = paths.get(0);
initS3AFileSystem(s3Path);
return true;
return;
}
System.err.println("No region found from -" + REGION_FLAG + " flag, " +
throw invalidArgs("No region found from -" + REGION_FLAG + " flag, " +
"config, or S3 bucket");
return false;
}
/**
@ -189,7 +195,7 @@ public abstract class S3GuardTool extends Configured implements Tool {
String metaURI = getCommandFormat().getOptValue(META_FLAG);
if (metaURI != null && !metaURI.isEmpty()) {
URI uri = URI.create(metaURI);
LOG.info("create metadata store: {}", uri + " scheme: "
LOG.info("Create metadata store: {}", uri + " scheme: "
+ uri.getScheme());
switch (uri.getScheme().toLowerCase(Locale.ENGLISH)) {
case "local":
@ -225,35 +231,48 @@ public abstract class S3GuardTool extends Configured implements Tool {
}
/**
* Initialize S3A FileSystem instance.
* Create and initialize a new S3A FileSystem instance.
* This instance is always created without S3Guard, so allowing
* a previously created metastore to be patched in.
*
* Note: this is a bit convoluted as it needs to also handle the situation
* of a per-bucket option in core-site.xml, which isn't easily overridden.
* The new config and the setting of the values before any
* {@code Configuration.get()} calls are critical.
*
* @param path s3a URI
* @throws IOException
* @throws IOException failure to init filesystem
* @throws ExitUtil.ExitException if the FS is not an S3A FS
*/
void initS3AFileSystem(String path) throws IOException {
URI uri;
try {
uri = new URI(path);
} catch (URISyntaxException e) {
throw new IOException(e);
}
URI uri = toUri(path);
// Make sure that S3AFileSystem does not hold an actual MetadataStore
// implementation.
Configuration conf = getConf();
conf.setClass(S3_METADATA_STORE_IMPL, NullMetadataStore.class,
MetadataStore.class);
FileSystem fs = FileSystem.get(uri, getConf());
Configuration conf = new Configuration(getConf());
String nullStore = NullMetadataStore.class.getName();
conf.set(S3_METADATA_STORE_IMPL, nullStore);
String bucket = uri.getHost();
S3AUtils.setBucketOption(conf,
bucket,
S3_METADATA_STORE_IMPL, S3GUARD_METASTORE_NULL);
String updatedBucketOption = S3AUtils.getBucketOption(conf, bucket,
S3_METADATA_STORE_IMPL);
LOG.debug("updated bucket store option {}", updatedBucketOption);
Preconditions.checkState(S3GUARD_METASTORE_NULL.equals(updatedBucketOption),
"Expected bucket option to be %s but was %s",
S3GUARD_METASTORE_NULL, updatedBucketOption);
FileSystem fs = FileSystem.newInstance(uri, conf);
if (!(fs instanceof S3AFileSystem)) {
throw new IOException(
String.format("URI %s is not a S3A file system: %s", uri,
fs.getClass().getName()));
throw invalidArgs("URI %s is not a S3A file system: %s",
uri, fs.getClass().getName());
}
filesystem = (S3AFileSystem) fs;
}
/**
* Parse CLI arguments and returns the position arguments.
* The options are stored in {@link #commandFormat}
* The options are stored in {@link #commandFormat}.
*
* @param args command line arguments.
* @return the position arguments from CLI.
@ -285,11 +304,32 @@ public abstract class S3GuardTool extends Configured implements Tool {
return commandFormat;
}
@Override
public final int run(String[] args) throws Exception {
return run(args, System.out);
}
/**
* Run the tool, capturing the output (if the tool supports that).
*
* As well as returning an exit code, the implementations can choose to
* throw an instance of {@link ExitUtil.ExitException} with their exit
* code set to the desired exit value. The exit code of auch an exception
* is used for the tool's exit code, and the stack trace only logged at
* debug.
* @param args argument list
* @param out output stream
* @return the exit code to return.
* @throws Exception on any failure
* @throws ExitUtil.ExitException for an alternative clean exit
*/
public abstract int run(String[] args, PrintStream out) throws Exception;
/**
* Create the metadata store.
*/
static class Init extends S3GuardTool {
private static final String NAME = "init";
public static final String NAME = "init";
public static final String PURPOSE = "initialize metadata repository";
private static final String USAGE = NAME + " [OPTIONS] [s3a://BUCKET]\n" +
"\t" + PURPOSE + "\n\n" +
@ -325,7 +365,7 @@ public abstract class S3GuardTool extends Configured implements Tool {
}
@Override
public int run(String[] args) throws IOException {
public int run(String[] args, PrintStream out) throws Exception {
List<String> paths = parseArgs(args);
String readCap = getCommandFormat().getOptValue(READ_FLAG);
@ -340,20 +380,92 @@ public abstract class S3GuardTool extends Configured implements Tool {
}
// Validate parameters.
if (!parseDynamoDBRegion(paths)) {
System.err.println(USAGE);
return INVALID_ARGUMENT;
try {
parseDynamoDBRegion(paths);
} catch (ExitUtil.ExitException e) {
errorln(USAGE);
throw e;
}
initMetadataStore(true);
MetadataStore store = initMetadataStore(true);
printStoreDiagnostics(out, store);
return SUCCESS;
}
}
/**
* Change the capacity of the metadata store.
*/
static class SetCapacity extends S3GuardTool {
public static final String NAME = "set-capacity";
public static final String PURPOSE = "Alter metadata store IO capacity";
private static final String USAGE = NAME + " [OPTIONS] [s3a://BUCKET]\n" +
"\t" + PURPOSE + "\n\n" +
"Common options:\n" +
" -" + META_FLAG + " URL - Metadata repository details " +
"(implementation-specific)\n" +
"\n" +
"Amazon DynamoDB-specific options:\n" +
" -" + READ_FLAG + " UNIT - Provisioned read throughput units\n" +
" -" + WRITE_FLAG + " UNIT - Provisioned write through put units\n" +
"\n" +
" URLs for Amazon DynamoDB are of the form dynamodb://TABLE_NAME.\n" +
" Specifying both the -" + REGION_FLAG + " option and an S3A path\n" +
" is not supported.";
SetCapacity(Configuration conf) {
super(conf);
// read capacity.
getCommandFormat().addOptionWithValue(READ_FLAG);
// write capacity.
getCommandFormat().addOptionWithValue(WRITE_FLAG);
}
@Override
String getName() {
return NAME;
}
@Override
public String getUsage() {
return USAGE;
}
@Override
public int run(String[] args, PrintStream out) throws Exception {
List<String> paths = parseArgs(args);
Map<String, String> options = new HashMap<>();
String readCap = getCommandFormat().getOptValue(READ_FLAG);
if (StringUtils.isNotEmpty(readCap)) {
S3GuardTool.println(out, "Read capacity set to %s", readCap);
options.put(S3GUARD_DDB_TABLE_CAPACITY_READ_KEY, readCap);
}
String writeCap = getCommandFormat().getOptValue(WRITE_FLAG);
if (StringUtils.isNotEmpty(writeCap)) {
S3GuardTool.println(out, "Write capacity set to %s", writeCap);
options.put(S3GUARD_DDB_TABLE_CAPACITY_WRITE_KEY, writeCap);
}
// Validate parameters.
try {
parseDynamoDBRegion(paths);
} catch (ExitUtil.ExitException e) {
errorln(USAGE);
throw e;
}
MetadataStore store = initMetadataStore(false);
store.updateParameters(options);
printStoreDiagnostics(out, store);
return SUCCESS;
}
}
/**
* Destroy a metadata store.
*/
static class Destroy extends S3GuardTool {
private static final String NAME = "destroy";
public static final String NAME = "destroy";
public static final String PURPOSE = "destroy Metadata Store data "
+ DATA_IN_S3_IS_PRESERVED;
private static final String USAGE = NAME + " [OPTIONS] [s3a://BUCKET]\n" +
@ -383,19 +495,21 @@ public abstract class S3GuardTool extends Configured implements Tool {
return USAGE;
}
public int run(String[] args) throws IOException {
public int run(String[] args, PrintStream out) throws Exception {
List<String> paths = parseArgs(args);
if (!parseDynamoDBRegion(paths)) {
System.err.println(USAGE);
return INVALID_ARGUMENT;
try {
parseDynamoDBRegion(paths);
} catch (ExitUtil.ExitException e) {
errorln(USAGE);
throw e;
}
try {
initMetadataStore(false);
} catch (FileNotFoundException e) {
// indication that the table was not found
println(out, "Metadata Store does not exist.");
LOG.debug("Failed to bind to store to be destroyed", e);
LOG.info("Metadata Store does not exist.");
return SUCCESS;
}
@ -403,7 +517,7 @@ public abstract class S3GuardTool extends Configured implements Tool {
"Metadata Store is not initialized");
getStore().destroy();
LOG.info("Metadata store is deleted.");
println(out, "Metadata store is deleted.");
return SUCCESS;
}
}
@ -412,7 +526,7 @@ public abstract class S3GuardTool extends Configured implements Tool {
* Import s3 metadata to the metadata store.
*/
static class Import extends S3GuardTool {
private static final String NAME = "import";
public static final String NAME = "import";
public static final String PURPOSE = "import metadata from existing S3 " +
"data";
private static final String USAGE = NAME + " [OPTIONS] [s3a://BUCKET]\n" +
@ -498,21 +612,16 @@ public abstract class S3GuardTool extends Configured implements Tool {
}
@Override
public int run(String[] args) throws IOException {
public int run(String[] args, PrintStream out) throws Exception {
List<String> paths = parseArgs(args);
if (paths.isEmpty()) {
System.err.println(getUsage());
return INVALID_ARGUMENT;
errorln(getUsage());
throw invalidArgs("no arguments");
}
String s3Path = paths.get(0);
initS3AFileSystem(s3Path);
URI uri;
try {
uri = new URI(s3Path);
} catch (URISyntaxException e) {
throw new IOException(e);
}
URI uri = toUri(s3Path);
String filePath = uri.getPath();
if (filePath.isEmpty()) {
// If they specify a naked S3 URI (e.g. s3a://bucket), we'll consider
@ -522,7 +631,11 @@ public abstract class S3GuardTool extends Configured implements Tool {
Path path = new Path(filePath);
FileStatus status = getFilesystem().getFileStatus(path);
initMetadataStore(false);
try {
initMetadataStore(false);
} catch (FileNotFoundException e) {
throw storeNotFound(e);
}
long items = 1;
if (status.isFile()) {
@ -532,17 +645,18 @@ public abstract class S3GuardTool extends Configured implements Tool {
items = importDir(status);
}
System.out.printf("Inserted %d items into Metadata Store%n", items);
println(out, "Inserted %d items into Metadata Store", items);
return SUCCESS;
}
}
/**
* Show diffs between the s3 and metadata store.
*/
static class Diff extends S3GuardTool {
private static final String NAME = "diff";
public static final String NAME = "diff";
public static final String PURPOSE = "report on delta between S3 and " +
"repository";
private static final String USAGE = NAME + " [OPTIONS] s3a://BUCKET\n" +
@ -625,10 +739,10 @@ public abstract class S3GuardTool extends Configured implements Tool {
if (differ(msStatus, s3Status)) {
if (s3Status != null) {
out.printf("%s%s%s%n", S3_PREFIX, SEP, formatFileStatus(s3Status));
println(out, "%s%s%s", S3_PREFIX, SEP, formatFileStatus(s3Status));
}
if (msStatus != null) {
out.printf("%s%s%s%n", MS_PREFIX, SEP, formatFileStatus(msStatus));
println(out, "%s%s%s", MS_PREFIX, SEP, formatFileStatus(msStatus));
}
}
}
@ -705,6 +819,7 @@ public abstract class S3GuardTool extends Configured implements Tool {
try {
s3Status = getFilesystem().getFileStatus(qualified);
} catch (FileNotFoundException e) {
/* ignored */
}
PathMetadata meta = getStore().get(qualified);
FileStatus msStatus = (meta != null && !meta.isDeleted()) ?
@ -717,18 +832,13 @@ public abstract class S3GuardTool extends Configured implements Tool {
List<String> paths = parseArgs(args);
if (paths.isEmpty()) {
out.println(USAGE);
return INVALID_ARGUMENT;
throw invalidArgs("no arguments");
}
String s3Path = paths.get(0);
initS3AFileSystem(s3Path);
initMetadataStore(true);
initMetadataStore(false);
URI uri;
try {
uri = new URI(s3Path);
} catch (URISyntaxException e) {
throw new IOException(e);
}
URI uri = toUri(s3Path);
Path root;
if (uri.getPath().isEmpty()) {
root = new Path("/");
@ -741,17 +851,13 @@ public abstract class S3GuardTool extends Configured implements Tool {
return SUCCESS;
}
@Override
public int run(String[] args) throws IOException {
return run(args, System.out);
}
}
/**
* Prune metadata that has not been modified recently.
*/
static class Prune extends S3GuardTool {
private static final String NAME = "prune";
public static final String NAME = "prune";
public static final String PURPOSE = "truncate older metadata from " +
"repository "
+ DATA_IN_S3_IS_PRESERVED;;
@ -803,18 +909,19 @@ public abstract class S3GuardTool extends Configured implements Tool {
return unit.toMillis(parsed);
}
@VisibleForTesting
public int run(String[] args, PrintStream out) throws
InterruptedException, IOException {
List<String> paths = parseArgs(args);
if (!parseDynamoDBRegion(paths)) {
System.err.println(USAGE);
return INVALID_ARGUMENT;
try {
parseDynamoDBRegion(paths);
} catch (ExitUtil.ExitException e) {
errorln(USAGE);
throw e;
}
initMetadataStore(false);
Configuration conf = getConf();
long confDelta = conf.getLong(Constants.S3GUARD_CLI_PRUNE_AGE, 0);
long confDelta = conf.getLong(S3GUARD_CLI_PRUNE_AGE, 0);
long cliDelta = 0;
cliDelta += getDeltaComponent(TimeUnit.DAYS, "days");
@ -823,8 +930,7 @@ public abstract class S3GuardTool extends Configured implements Tool {
cliDelta += getDeltaComponent(TimeUnit.SECONDS, "seconds");
if (confDelta <= 0 && cliDelta <= 0) {
System.err.println(
"You must specify a positive age for metadata to prune.");
errorln("You must specify a positive age for metadata to prune.");
}
// A delta provided on the CLI overrides if one is configured
@ -842,35 +948,235 @@ public abstract class S3GuardTool extends Configured implements Tool {
return SUCCESS;
}
@Override
public int run(String[] args) throws InterruptedException, IOException {
return run(args, System.out);
}
/**
* Get info about a bucket and its S3Guard integration status.
*/
static class BucketInfo extends S3GuardTool {
public static final String NAME = "bucket-info";
public static final String GUARDED_FLAG = "guarded";
public static final String UNGUARDED_FLAG = "unguarded";
public static final String AUTH_FLAG = "auth";
public static final String NONAUTH_FLAG = "nonauth";
public static final String ENCRYPTION_FLAG = "encryption";
public static final String PURPOSE = "provide/check S3Guard information"
+ " about a specific bucket";
private static final String USAGE = NAME + " [OPTIONS] s3a://BUCKET\n"
+ "\t" + PURPOSE + "\n\n"
+ "Common options:\n"
+ " -" + GUARDED_FLAG + " - Require S3Guard\n"
+ " -" + ENCRYPTION_FLAG
+ " -require {none, sse-s3, sse-kms} - Require encryption policy";
BucketInfo(Configuration conf) {
super(conf, GUARDED_FLAG, UNGUARDED_FLAG, AUTH_FLAG, NONAUTH_FLAG);
CommandFormat format = getCommandFormat();
format.addOptionWithValue(ENCRYPTION_FLAG);
}
@Override
String getName() {
return NAME;
}
@Override
public String getUsage() {
return USAGE;
}
public int run(String[] args, PrintStream out)
throws InterruptedException, IOException {
List<String> paths = parseArgs(args);
if (paths.isEmpty()) {
errorln(getUsage());
throw invalidArgs("No bucket specified");
}
String s3Path = paths.get(0);
S3AFileSystem fs = (S3AFileSystem) FileSystem.newInstance(
toUri(s3Path), getConf());
setFilesystem(fs);
Configuration conf = fs.getConf();
URI fsUri = fs.getUri();
MetadataStore store = fs.getMetadataStore();
println(out, "Filesystem %s", fsUri);
println(out, "Location: %s", fs.getBucketLocation());
boolean usingS3Guard = !(store instanceof NullMetadataStore);
boolean authMode = false;
if (usingS3Guard) {
out.printf("Filesystem %s is using S3Guard with store %s%n",
fsUri, store.toString());
printOption(out, "Authoritative S3Guard",
METADATASTORE_AUTHORITATIVE, "false");
authMode = conf.getBoolean(METADATASTORE_AUTHORITATIVE, false);
printStoreDiagnostics(out, store);
} else {
println(out, "Filesystem %s is not using S3Guard", fsUri);
}
println(out, "%nS3A Client");
String endpoint = conf.getTrimmed(ENDPOINT, "");
println(out, "\tEndpoint: %s=%s",
ENDPOINT,
StringUtils.isNotEmpty(endpoint) ? endpoint : "(unset)");
String encryption =
printOption(out, "\tEncryption", SERVER_SIDE_ENCRYPTION_ALGORITHM,
"none");
printOption(out, "\tInput seek policy", INPUT_FADVISE, INPUT_FADV_NORMAL);
CommandFormat commands = getCommandFormat();
if (usingS3Guard) {
if (commands.getOpt(UNGUARDED_FLAG)) {
throw badState("S3Guard is enabled for %s", fsUri);
}
if (commands.getOpt(AUTH_FLAG) && !authMode) {
throw badState("S3Guard is enabled for %s,"
+ " but not in authoritative mode", fsUri);
}
if (commands.getOpt(NONAUTH_FLAG) && authMode) {
throw badState("S3Guard is enabled in authoritative mode for %s",
fsUri);
}
} else {
if (commands.getOpt(GUARDED_FLAG)) {
throw badState("S3Guard is not enabled for %s", fsUri);
}
}
String desiredEncryption = getCommandFormat()
.getOptValue(ENCRYPTION_FLAG);
if (StringUtils.isNotEmpty(desiredEncryption)
&& !desiredEncryption.equalsIgnoreCase(encryption)) {
throw badState("Bucket %s: required encryption is %s"
+ " but actual encryption is %s",
fsUri, desiredEncryption, encryption);
}
out.flush();
return SUCCESS;
}
private String printOption(PrintStream out,
String description, String key, String defVal) {
String t = getFilesystem().getConf().getTrimmed(key, defVal);
println(out, "%s: %s=%s", description, key, t);
return t;
}
}
private static S3GuardTool command;
/**
* Convert a path to a URI, catching any {@code URISyntaxException}
* and converting to an invalid args exception.
* @param s3Path path to convert to a URI
* @return a URI of the path
* @throws ExitUtil.ExitException INVALID_ARGUMENT if the URI is invalid
*/
protected static URI toUri(String s3Path) {
URI uri;
try {
uri = new URI(s3Path);
} catch (URISyntaxException e) {
throw invalidArgs("Not a valid fileystem path: %s", s3Path);
}
return uri;
}
private static void printHelp() {
if (command == null) {
System.err.println("Usage: hadoop " + USAGE);
System.err.println("\tperform S3Guard metadata store " +
errorln("Usage: hadoop " + USAGE);
errorln("\tperform S3Guard metadata store " +
"administrative commands.");
} else {
System.err.println("Usage: hadoop " + command.getUsage());
errorln("Usage: hadoop " + command.getUsage());
}
errorln();
errorln(COMMON_USAGE);
}
private static void errorln() {
System.err.println();
System.err.println(COMMON_USAGE);
}
private static void errorln(String x) {
System.err.println(x);
}
/**
* Print a formatted string followed by a newline to the output stream.
* @param out destination
* @param format format string
* @param args optional arguments
*/
private static void println(PrintStream out, String format, Object... args) {
out.println(String.format(format, args));
}
/**
* Retrieve and Print store diagnostics.
* @param out output stream
* @param store store
* @throws IOException Failure to retrieve the data.
*/
protected static void printStoreDiagnostics(PrintStream out,
MetadataStore store)
throws IOException {
Map<String, String> diagnostics = store.getDiagnostics();
out.println("Metadata Store Diagnostics:");
for (Map.Entry<String, String> entry : diagnostics.entrySet()) {
println(out, "\t%s=%s", entry.getKey(), entry.getValue());
}
}
/**
* Handle store not found by converting to an exit exception
* with specific error code.
* @param e exception
* @return a new exception to throw
*/
protected static ExitUtil.ExitException storeNotFound(
FileNotFoundException e) {
return new ExitUtil.ExitException(
E_NOT_FOUND, e.toString(), e);
}
/**
* Build the exception to raise on invalid arguments.
* @param format string format
* @param args optional arguments for the string
* @return a new exception to throw
*/
protected static ExitUtil.ExitException invalidArgs(
String format, Object...args) {
return new ExitUtil.ExitException(INVALID_ARGUMENT,
String.format(format, args));
}
/**
* Build the exception to raise on a bad store/bucket state.
* @param format string format
* @param args optional arguments for the string
* @return a new exception to throw
*/
protected static ExitUtil.ExitException badState(
String format, Object...args) {
return new ExitUtil.ExitException(E_BAD_STATE,
String.format(format, args));
}
/**
* Execute the command with the given arguments.
*
* @param args command specific arguments.
* @param conf Hadoop configuration.
* @param args command specific arguments.
* @return exit code.
* @throws Exception on I/O errors.
*/
public static int run(String[] args, Configuration conf) throws
public static int run(Configuration conf, String...args) throws
Exception {
/* ToolRunner.run does this too, but we must do it before looking at
subCommand or instantiating the cmd object below */
@ -878,9 +1184,10 @@ public abstract class S3GuardTool extends Configured implements Tool {
.getRemainingArgs();
if (otherArgs.length == 0) {
printHelp();
return INVALID_ARGUMENT;
throw new ExitUtil.ExitException(E_USAGE, "No arguments provided");
}
final String subCommand = otherArgs[0];
LOG.debug("Executing command {}", subCommand);
switch (subCommand) {
case Init.NAME:
command = new Init(conf);
@ -891,15 +1198,22 @@ public abstract class S3GuardTool extends Configured implements Tool {
case Import.NAME:
command = new Import(conf);
break;
case BucketInfo.NAME:
command = new BucketInfo(conf);
break;
case Diff.NAME:
command = new Diff(conf);
break;
case Prune.NAME:
command = new Prune(conf);
break;
case SetCapacity.NAME:
command = new SetCapacity(conf);
break;
default:
printHelp();
return INVALID_ARGUMENT;
throw new ExitUtil.ExitException(E_USAGE,
"Unknown command " + subCommand);
}
return ToolRunner.run(conf, command, otherArgs);
}
@ -910,15 +1224,22 @@ public abstract class S3GuardTool extends Configured implements Tool {
*/
public static void main(String[] args) {
try {
int ret = run(args, new Configuration());
System.exit(ret);
int ret = run(new Configuration(), args);
exit(ret, "");
} catch (CommandFormat.UnknownOptionException e) {
System.err.println(e.getMessage());
errorln(e.getMessage());
printHelp();
System.exit(INVALID_ARGUMENT);
exit(E_USAGE, e.getMessage());
} catch (ExitUtil.ExitException e) {
// explicitly raised exit code
exit(e.getExitCode(), e.toString());
} catch (Throwable e) {
e.printStackTrace(System.err);
System.exit(ERROR);
exit(ERROR, e.toString());
}
}
protected static void exit(int status, String text) {
ExitUtil.terminate(status, text);
}
}

165
hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3guard.md
View File

@ -388,7 +388,7 @@ Example 2
hadoop s3guard init -meta dynamodb://ireland-team -region eu-west-1
```
Creates a table "ireland-team" in the same region "s3-eu-west-1.amazonaws.com"
Creates a table "ireland-team" in the region "eu-west-1.amazonaws.com"
### Import a bucket: `s3guard import`
@ -422,6 +422,98 @@ Example
hadoop s3guard diff s3a://ireland-1
```
### Display information about a bucket, `s3guard bucket-info`
Prints and optionally checks the s3guard and encryption status of a bucket.
```bash
hadoop s3guard bucket-info [ -guarded ] [-unguarded] [-auth] [-nonauth] [-encryption ENCRYPTION] s3a://BUCKET
```
Options
| argument | meaning |
|-----------|-------------|
| `-guarded` | Require S3Guard to be enabled |
| `-unguarded` | Require S3Guard to be disabled |
| `-auth` | Require the S3Guard mode to be "authoritative" |
| `-nonauth` | Require the S3Guard mode to be "non-authoritative" |
| `-encryption <type>` | Require a specific server-side encryption algorithm |
The server side encryption options are not directly related to S3Guard, but
it is often convenient to check them at the same time.
Example
```bash
hadoop s3guard bucket-info -guarded s3a://ireland-1
```
List the details of bucket `s3a://ireland-1`, mandating that it must have S3Guard enabled
```
Filesystem s3a://ireland-1
Location: eu-west-1
Filesystem s3a://ireland-1 is using S3Guard with store DynamoDBMetadataStore{region=eu-west-1, tableName=ireland-1}
Authoritative S3Guard: fs.s3a.metadatastore.authoritative=false
Metadata Store Diagnostics:
ARN=arn:aws:dynamodb:eu-west-1:00000000:table/ireland-1
description=S3Guard metadata store in DynamoDB
name=ireland-1
read-capacity=20
region=eu-west-1
retryPolicy=ExponentialBackoffRetry(maxRetries=9, sleepTime=100 MILLISECONDS)
size=12812
status=ACTIVE
table={AttributeDefinitions: [{AttributeName: child,AttributeType: S},
{AttributeName: parent,AttributeType: S}],TableName: ireland-1,
KeySchema: [{AttributeName: parent,KeyType: HASH}, {AttributeName: child,KeyType: RANGE}],
TableStatus: ACTIVE,
CreationDateTime: Fri Aug 25 19:07:25 BST 2017,
ProvisionedThroughput: {LastIncreaseDateTime: Tue Aug 29 11:45:18 BST 2017,
LastDecreaseDateTime: Wed Aug 30 15:37:51 BST 2017,
NumberOfDecreasesToday: 1,
ReadCapacityUnits: 20,WriteCapacityUnits: 20},
TableSizeBytes: 12812,ItemCount: 91,
TableArn: arn:aws:dynamodb:eu-west-1:00000000:table/ireland-1,}
write-capacity=20
S3A Client
Endpoint: fs.s3a.endpoint=s3-eu-west-1.amazonaws.com
Encryption: fs.s3a.server-side-encryption-algorithm=none
Input seek policy: fs.s3a.experimental.input.fadvise=normal
```
This listing includes all the information about the table supplied from
```bash
hadoop s3guard bucket-info -unguarded -encryption none s3a://landsat-pds
```
List the S3Guard status of clients of the public `landsat-pds` bucket,
and verifies that the data is neither tracked with S3Guard nor encrypted.
```
Filesystem s3a://landsat-pdsLocation: us-west-2
Filesystem s3a://landsat-pds is not using S3Guard
Endpoint: fs.s3a.endpoints3.amazonaws.com
Encryption: fs.s3a.server-side-encryption-algorithm=none
Input seek policy: fs.s3a.experimental.input.fadvise=normal
```
Note that other clients may have a S3Guard table set up to store metadata
on this bucket; the checks are all done from the perspective of the configuration
setttings of the current client.
```bash
hadoop s3guard bucket-info -guarded -auth s3a://landsat-pds
```
Require the bucket to be using S3Guard in authoritative mode. This will normally
fail against this specific bucket.
### Delete a table: `s3guard destroy`
@ -450,7 +542,6 @@ hadoop s3guard destroy -meta dynamodb://ireland-team -region eu-west-1
```
### Clean up a table, `s3guard prune`
Delete all file entries in the MetadataStore table whose object "modification
@ -461,7 +552,7 @@ hadoop s3guard prune [-days DAYS] [-hours HOURS] [-minutes MINUTES]
[-seconds SECONDS] [-m URI] ( -region REGION | s3a://BUCKET )
```
A time value must be supplied.
A time value of hours, minutes and/or seconds must be supplied.
1. This does not delete the entries in the bucket itself.
1. The modification time is effectively the creation time of the objects
@ -486,6 +577,63 @@ Delete all entries more than 90 minutes old from the table "ireland-team" in
the region "eu-west-1".
### Tune the IO capacity of the DynamoDB Table, `s3guard set-capacity`
Alter the read and/or write capacity of a s3guard table.
```bash
hadoop s3guard set-capacity [--read UNIT] [--write UNIT] ( -region REGION | s3a://BUCKET )
```
The `--read` and `--write` units are those of `s3guard init`.
Example
```
hadoop s3guard set-capacity -read 20 -write 20 s3a://ireland-1
```
Set the capacity of the table used by bucket `s3a://ireland-1` to 20 read
and 20 write. (This is a low number, incidentally)
```
2017-08-30 16:21:26,343 [main] INFO s3guard.S3GuardTool (S3GuardTool.java:initMetadataStore(229)) - Metadata store DynamoDBMetadataStore{region=eu-west-1, tableName=ireland-1} is initialized.
2017-08-30 16:21:26,344 [main] INFO s3guard.DynamoDBMetadataStore (DynamoDBMetadataStore.java:updateParameters(1084)) - Current table capacity is read: 25, write: 25
2017-08-30 16:21:26,344 [main] INFO s3guard.DynamoDBMetadataStore (DynamoDBMetadataStore.java:updateParameters(1086)) - Changing capacity of table to read: 20, write: 20
Metadata Store Diagnostics:
ARN=arn:aws:dynamodb:eu-west-1:00000000000:table/ireland-1
description=S3Guard metadata store in DynamoDB
name=ireland-1
read-capacity=25
region=eu-west-1
retryPolicy=ExponentialBackoffRetry(maxRetries=9, sleepTime=100 MILLISECONDS)
size=12812
status=UPDATING
table={ ... }
write-capacity=25
```
After the update, the table status changes to `UPDATING`; this is a sign that
the capacity has been changed
Repeating the same command will not change the capacity, as both read and
write values match that already in use
```
2017-08-30 16:24:35,337 [main] INFO s3guard.DynamoDBMetadataStore (DynamoDBMetadataStore.java:updateParameters(1090)) - Table capacity unchanged at read: 20, write: 20
Metadata Store Diagnostics:
ARN=arn:aws:dynamodb:eu-west-1:00000000000:table/ireland-1
description=S3Guard metadata store in DynamoDB
name=ireland-1
read-capacity=20
region=eu-west-1
retryPolicy=ExponentialBackoffRetry(maxRetries=9, sleepTime=100 MILLISECONDS)
size=12812
status=ACTIVE
table={ ... }
write-capacity=20
```
## Debugging and Error Handling
@ -608,3 +756,14 @@ or the configuration is preventing S3Guard from finding the table.
1. If the region is not set, verify that the table exists in the same
region as the bucket being used.
1. Create the table if necessary.
### Error `"The level of configured provisioned throughput for the table was exceeded"`
The IO load of clients of the (shared) DynamoDB table was exceeded.
Currently S3Guard doesn't do any throttling and retries here; the way to address
this is to increase capacity via the AWS console or the `set-capacity` command.
## Other Topis
For details on how to test S3Guard, see [Testing S3Guard](./testing.html#s3guard)

10
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
View File

@ -591,6 +591,16 @@ public class ITestS3AConfiguration {
assertOptionEquals(updated, USER_AGENT_PREFIX, "UA-c");
}
@Test
public void testClearBucketOption() throws Throwable {
Configuration config = new Configuration();
config.set(USER_AGENT_PREFIX, "base");
setBucketOption(config, "bucket", USER_AGENT_PREFIX, "overridden");
clearBucketOption(config, "bucket", USER_AGENT_PREFIX);
Configuration updated = propagateBucketOptions(config, "c");
assertOptionEquals(updated, USER_AGENT_PREFIX, "base");
}
@Test
public void testBucketConfigurationSkipsUnmodifiable() throws Throwable {
Configuration config = new Configuration(false);

8
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3GuardListConsistency.java
View File

@ -435,16 +435,16 @@ public class ITestS3GuardListConsistency extends AbstractS3ATestBase {
// files to list are delaying visibility
if (!recursive) {
// in this case only the top level files are listed
verifyFileIsListed(listedFiles, baseTestDir, fileNames);
assertEquals("Unexpected number of files returned by listFiles() call",
normalFileNum + delayedFileNum, listedFiles.size());
verifyFileIsListed(listedFiles, baseTestDir, fileNames);
} else {
assertEquals("Unexpected number of files returned by listFiles() call",
filesAndEmptyDirectories,
listedFiles.size());
for (Path dir : testDirs) {
verifyFileIsListed(listedFiles, dir, fileNames);
}
assertEquals("Unexpected number of files returned by listFiles() call",
filesAndEmptyDirectories,
listedFiles.size());
}
}

37
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3GuardWriteBack.java
View File

@ -22,15 +22,17 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.contract.ContractTestUtils;
import org.apache.hadoop.fs.s3a.s3guard.DirListingMetadata;
import org.junit.Assume;
import org.junit.Test;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
import java.util.Arrays;
import static org.apache.hadoop.fs.s3a.Constants.*;
/**
* Test cases that validate S3Guard's behavior for writing things like
* directory listings back to the MetadataStore.
@ -66,7 +68,7 @@ public class ITestS3GuardWriteBack extends AbstractS3ATestBase {
noS3Guard.mkdirs(new Path(directory, "OnS3"));
// Create a directory on both S3 and metadata store
Path p = new Path(directory, "OnS3AndMS");
assertPathDoesntExist(noWriteBack, p);
ContractTestUtils.assertPathDoesNotExist(noWriteBack, "path", p);
noWriteBack.mkdirs(p);
FileStatus[] fsResults;
@ -87,7 +89,7 @@ public class ITestS3GuardWriteBack extends AbstractS3ATestBase {
// FS should return both (and will write it back)
fsResults = yesWriteBack.listStatus(directory);
assertEquals("Filesystem enabled S3Guard with write back should have "
assertEquals("Filesystem enabled S3Guard with write back should have"
+ " both /OnS3 and /OnS3AndMS: " + Arrays.toString(fsResults),
2, fsResults.length);
@ -104,7 +106,12 @@ public class ITestS3GuardWriteBack extends AbstractS3ATestBase {
new Path(directory, "OnS3"));
}
/** Create a separate S3AFileSystem instance for testing. */
/**
* Create a separate S3AFileSystem instance for testing.
* There's a bit of complexity as it forces pushes up s3guard options from
* the base values to the per-bucket options. This stops explicit bucket
* settings in test XML configs from unintentionally breaking tests.
*/
private S3AFileSystem createTestFS(URI fsURI, boolean disableS3Guard,
boolean authoritativeMeta) throws IOException {
Configuration conf;
@ -112,12 +119,22 @@ public class ITestS3GuardWriteBack extends AbstractS3ATestBase {
// Create a FileSystem that is S3-backed only
conf = createConfiguration();
S3ATestUtils.disableFilesystemCaching(conf);
String host = fsURI.getHost();
if (disableS3Guard) {
conf.set(Constants.S3_METADATA_STORE_IMPL,
Constants.S3GUARD_METASTORE_NULL);
S3AUtils.setBucketOption(conf, host,
S3_METADATA_STORE_IMPL,
S3GUARD_METASTORE_NULL);
} else {
S3ATestUtils.maybeEnableS3Guard(conf);
conf.setBoolean(Constants.METADATASTORE_AUTHORITATIVE, authoritativeMeta);
conf.setBoolean(METADATASTORE_AUTHORITATIVE, authoritativeMeta);
S3AUtils.setBucketOption(conf, host,
METADATASTORE_AUTHORITATIVE,
Boolean.toString(authoritativeMeta));
S3AUtils.setBucketOption(conf, host,
S3_METADATA_STORE_IMPL,
conf.get(S3_METADATA_STORE_IMPL));
}
FileSystem fs = FileSystem.get(fsURI, conf);
return asS3AFS(fs);
@ -128,14 +145,4 @@ public class ITestS3GuardWriteBack extends AbstractS3ATestBase {
return (S3AFileSystem)fs;
}
private static void assertPathDoesntExist(FileSystem fs, Path p)
throws IOException {
try {
FileStatus s = fs.getFileStatus(p);
} catch (FileNotFoundException e) {
return;
}
fail("Path should not exist: " + p);
}
}

18
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
View File

@ -696,24 +696,6 @@ public final class S3ATestUtils {
assertEquals("Permission: " + details, permission, status.getPermission());
}
/**
* Set a bucket specific property to a particular value.
* If the generic key passed in has an {@code fs.s3a. prefix},
* that's stripped off, so that when the the bucket properties are propagated
* down to the generic values, that value gets copied down.
* @param conf configuration to set
* @param bucket bucket name
* @param genericKey key; can start with "fs.s3a."
* @param value value to set
*/
public static void setBucketOption(Configuration conf, String bucket,
String genericKey, String value) {
final String baseKey = genericKey.startsWith(FS_S3A_PREFIX) ?
genericKey.substring(FS_S3A_PREFIX.length())
: genericKey;
conf.set(FS_S3A_BUCKET_PREFIX + bucket + '.' + baseKey, value);
}
/**
* Assert that a configuration option matches the expected value.
* @param conf configuration

138
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
View File

@ -18,9 +18,15 @@
package org.apache.hadoop.fs.s3a.s3guard;
import java.io.ByteArrayOutputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.PrintStream;
import java.util.Collection;
import java.util.concurrent.Callable;
import java.util.concurrent.TimeUnit;
import org.junit.Assume;
import org.junit.Test;
import org.apache.hadoop.conf.Configuration;
@ -31,8 +37,11 @@ import org.apache.hadoop.fs.s3a.Constants;
import org.apache.hadoop.fs.s3a.S3AFileStatus;
import org.apache.hadoop.fs.s3a.S3ATestUtils;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.util.StringUtils;
import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.SUCCESS;
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
/**
* Common functionality for S3GuardTool test cases.
@ -40,6 +49,9 @@ import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.SUCCESS;
public abstract class AbstractS3GuardToolTestBase extends AbstractS3ATestBase {
protected static final String OWNER = "hdfs";
protected static final String DYNAMODB_TABLE = "dynamodb://ireland-team";
protected static final String S3A_THIS_BUCKET_DOES_NOT_EXIST
= "s3a://this-bucket-does-not-exist-00000000000";
private MetadataStore ms;
@ -57,6 +69,51 @@ public abstract class AbstractS3GuardToolTestBase extends AbstractS3ATestBase {
assertEquals(message, SUCCESS, tool.run(args));
}
/**
* Run a S3GuardTool command from a varags list.
* @param conf configuration
* @param args argument list
* @return the return code
* @throws Exception any exception
*/
protected int run(Configuration conf, String... args)
throws Exception {
return S3GuardTool.run(conf, args);
}
/**
* Run a S3GuardTool command from a varags list and the
* configuration returned by {@code getConfiguration()}.
* @param args argument list
* @return the return code
* @throws Exception any exception
*/
protected int run(String... args) throws Exception {
return S3GuardTool.run(getConfiguration(), args);
}
/**
* Run a S3GuardTool command from a varags list, catch any raised
* ExitException and verify the status code matches that expected.
* @param status expected status code of the exception
* @param args argument list
* @throws Exception any exception
*/
protected void runToFailure(int status, final String... args)
throws Exception {
ExitUtil.ExitException ex =
intercept(ExitUtil.ExitException.class,
new Callable<Integer>() {
@Override
public Integer call() throws Exception {
return run(args);
}
});
if (ex.status != status) {
throw ex;
}
}
protected MetadataStore getMetadataStore() {
return ms;
}
@ -134,16 +191,23 @@ public abstract class AbstractS3GuardToolTestBase extends AbstractS3ATestBase {
Thread.sleep(TimeUnit.SECONDS.toMillis(2));
createFile(new Path(parent, "fresh"), true, true);
assertEquals(2, ms.listChildren(parent).getListing().size());
expectSuccess("Prune command did not exit successfully - see output", cmd,
args);
assertEquals(1, ms.listChildren(parent).getListing().size());
assertMetastoreListingCount(parent, "Children count before pruning", 2);
exec(cmd, args);
assertMetastoreListingCount(parent, "Pruned children count", 1);
} finally {
getFileSystem().delete(parent, true);
ms.prune(Long.MAX_VALUE);
}
}
private void assertMetastoreListingCount(Path parent,
String message,
int expected) throws IOException {
Collection<PathMetadata> listing = ms.listChildren(parent).getListing();
assertEquals(message +" [" + StringUtils.join(", ", listing) + "]",
expected, listing.size());
}
@Test
public void testPruneCommandCLI() throws Exception {
String testPath = path("testPruneCommandCLI").toString();
@ -158,4 +222,70 @@ public abstract class AbstractS3GuardToolTestBase extends AbstractS3ATestBase {
String testPath = path("testPruneCommandConf").toString();
testPruneCommand(getConfiguration(), "prune", testPath);
}
@Test
public void testDestroyNoBucket() throws Throwable {
intercept(FileNotFoundException.class,
new Callable<Integer>() {
@Override
public Integer call() throws Exception {
return run(S3GuardTool.Destroy.NAME,
S3A_THIS_BUCKET_DOES_NOT_EXIST);
}
});
}
/**
* Get the test CSV file; assume() that it is not modified (i.e. we haven't
* switched to a new storage infrastructure where the bucket is no longer
* read only).
* @return test file.
*/
protected String getLandsatCSVFile() {
String csvFile = getConfiguration()
.getTrimmed(KEY_CSVTEST_FILE, DEFAULT_CSVTEST_FILE);
Assume.assumeTrue("CSV test file is not the default",
DEFAULT_CSVTEST_FILE.equals(csvFile));
return csvFile;
}
/**
* Execute a command, returning the buffer if the command actually completes.
* If an exception is raised the output is logged instead.
* @param cmd command
* @param buf buffer to use for tool output (not SLF4J output)
* @param args argument list
* @throws Exception on any failure
*/
public String exec(S3GuardTool cmd, String...args) throws Exception {
ByteArrayOutputStream buf = new ByteArrayOutputStream();
try {
exec(cmd, buf, args);
return buf.toString();
} catch (AssertionError e) {
throw e;
} catch (Exception e) {
LOG.error("Command {} failed: \n{}", cmd, buf);
throw e;
}
}
/**
* Execute a command, saving the output into the buffer.
* @param cmd command
* @param buf buffer to use for tool output (not SLF4J output)
* @param args argument list
* @throws Exception on any failure
*/
protected void exec(S3GuardTool cmd, ByteArrayOutputStream buf, String...args)
throws Exception {
LOG.info("exec {}", (Object) args);
int r = 0;
try(PrintStream out =new PrintStream(buf)) {
r = cmd.run(args, out);
out.flush();
}
assertEquals("Command " + cmd + " failed\n"+ buf, 0, r);
}
}

167
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java
View File

@ -19,19 +19,28 @@
package org.apache.hadoop.fs.s3a.s3guard;
import java.io.IOException;
import java.util.Map;
import java.util.Objects;
import java.util.Random;
import java.util.concurrent.Callable;
import java.util.concurrent.atomic.AtomicInteger;
import com.amazonaws.services.dynamodbv2.document.DynamoDB;
import com.amazonaws.services.dynamodbv2.document.Table;
import com.amazonaws.services.dynamodbv2.model.ResourceNotFoundException;
import org.junit.Test;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.s3a.Constants;
import org.apache.hadoop.fs.s3a.S3AFileSystem;
import org.apache.hadoop.fs.s3a.S3AUtils;
import org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.Destroy;
import org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.Init;
import org.apache.hadoop.test.LambdaTestUtils;
import static org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStore.*;
import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.*;
/**
* Test S3Guard related CLI commands against DynamoDB.
*/
@ -78,8 +87,76 @@ public class ITestS3GuardToolDynamoDB extends AbstractS3GuardToolTestBase {
});
}
private static class Capacities {
private final long read, write;
Capacities(long read, long write) {
this.read = read;
this.write = write;
}
public long getRead() {
return read;
}
public long getWrite() {
return write;
}
String getReadStr() {
return Long.toString(read);
}
String getWriteStr() {
return Long.toString(write);
}
void checkEquals(String text, Capacities that) throws Exception {
if (!this.equals(that)) {
throw new Exception(text + " expected = " + this +"; actual = "+ that);
}
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
Capacities that = (Capacities) o;
return read == that.read && write == that.write;
}
@Override
public int hashCode() {
return Objects.hash(read, write);
}
@Override
public String toString() {
final StringBuilder sb = new StringBuilder("Capacities{");
sb.append("read=").append(read);
sb.append(", write=").append(write);
sb.append('}');
return sb.toString();
}
}
private Capacities getCapacities() throws IOException {
Map<String, String> diagnostics = getMetadataStore().getDiagnostics();
return getCapacities(diagnostics);
}
private Capacities getCapacities(Map<String, String> diagnostics) {
return new Capacities(
Long.parseLong(diagnostics.get(DynamoDBMetadataStore.READ_CAPACITY)),
Long.parseLong(diagnostics.get(DynamoDBMetadataStore.WRITE_CAPACITY)));
}
@Test
public void testDynamoDBInitDestroyCycle() throws Exception {
public void testDynamoDBInitDestroyCycle() throws Throwable {
String testTableName = "testDynamoDBInitDestroy" + new Random().nextInt();
String testS3Url = path(testTableName).toString();
S3AFileSystem fs = getFileSystem();
@ -99,11 +176,80 @@ public class ITestS3GuardToolDynamoDB extends AbstractS3GuardToolTestBase {
assertTrue(String.format("%s does not exist", testTableName),
exist(db, testTableName));
Configuration conf = fs.getConf();
String bucket = fs.getBucket();
// force in a new bucket
S3AUtils.setBucketOption(conf, bucket, Constants.S3_METADATA_STORE_IMPL,
Constants.S3GUARD_METASTORE_DYNAMO);
initCmd = new Init(conf);
String initOutput = exec(initCmd,
"init", "-meta", "dynamodb://" + testTableName, testS3Url);
assertTrue("No Dynamo diagnostics in output " + initOutput,
initOutput.contains(DESCRIPTION));
// run a bucket info command and look for
// confirmation that it got the output from DDB diags
S3GuardTool.BucketInfo infocmd = new S3GuardTool.BucketInfo(conf);
String info = exec(infocmd, S3GuardTool.BucketInfo.NAME,
"-" + S3GuardTool.BucketInfo.GUARDED_FLAG,
testS3Url);
assertTrue("No Dynamo diagnostics in output " + info,
info.contains(DESCRIPTION));
// get the current values to set again
// play with the set-capacity option
Capacities original = getCapacities();
String fsURI = getFileSystem().getUri().toString();
String capacityOut = exec(newSetCapacity(),
S3GuardTool.SetCapacity.NAME,
fsURI);
LOG.info("Set Capacity output=\n{}", capacityOut);
capacityOut = exec(newSetCapacity(),
S3GuardTool.SetCapacity.NAME,
"-" + READ_FLAG, original.getReadStr(),
"-" + WRITE_FLAG, original.getWriteStr(),
fsURI);
LOG.info("Set Capacity output=\n{}", capacityOut);
// that call does not change the values
original.checkEquals("unchanged", getCapacities());
// now update the value
long readCap = original.getRead();
long writeCap = original.getWrite();
long rc2 = readCap + 1;
long wc2 = writeCap + 1;
final Capacities desired = new Capacities(rc2, wc2);
capacityOut = exec(newSetCapacity(),
S3GuardTool.SetCapacity.NAME,
"-" + READ_FLAG, Long.toString(rc2),
"-" + WRITE_FLAG, Long.toString(wc2),
fsURI);
LOG.info("Set Capacity output=\n{}", capacityOut);
// to avoid race conditions, spin for the state change
final AtomicInteger c = new AtomicInteger(0);
LambdaTestUtils.eventually(60000,
new LambdaTestUtils.VoidCallable() {
@Override
public void call() throws Exception {
c.incrementAndGet();
Map<String, String> diags = getMetadataStore().getDiagnostics();
Capacities updated = getCapacities(diags);
String tableInfo = String.format("[%02d] table state: %s",
c.intValue(), diags.get(STATUS));
LOG.info("{}; capacities {}",
tableInfo, updated);
desired.checkEquals(tableInfo, updated);
}
},
new LambdaTestUtils.ProportionalRetryInterval(500, 5000));
// Destroy MetadataStore
Destroy destroyCmd = new Destroy(fs.getConf());
expectSuccess("Destroy command did not exit successfully - see output",
destroyCmd,
String destroyed = exec(destroyCmd,
"destroy", "-meta", "dynamodb://" + testTableName, testS3Url);
// Verify it does not exist
assertFalse(String.format("%s still exists", testTableName),
@ -131,4 +277,19 @@ public class ITestS3GuardToolDynamoDB extends AbstractS3GuardToolTestBase {
}
}
}
private S3GuardTool newSetCapacity() {
S3GuardTool setCapacity = new S3GuardTool.SetCapacity(
getFileSystem().getConf());
setCapacity.setStore(getMetadataStore());
return setCapacity;
}
@Test
public void testDestroyUnknownTable() throws Throwable {
run(S3GuardTool.Destroy.NAME,
"-region", "us-west-2",
"-meta", DYNAMODB_TABLE);
}
}

142
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolLocal.java
View File

@ -21,11 +21,12 @@ package org.apache.hadoop.fs.s3a.s3guard;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.PrintStream;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.Callable;
import org.junit.Test;
@ -34,13 +35,16 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.s3a.S3AFileSystem;
import org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.Diff;
import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.SUCCESS;
import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.*;
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
/**
* Test S3Guard related CLI commands against a LocalMetadataStore.
*/
public class ITestS3GuardToolLocal extends AbstractS3GuardToolTestBase {
private static final String LOCAL_METADATA = "local://metadata";
@Override
protected MetadataStore newMetadataStore() {
return new LocalMetadataStore();
@ -65,10 +69,7 @@ public class ITestS3GuardToolLocal extends AbstractS3GuardToolTestBase {
S3GuardTool.Import cmd = new S3GuardTool.Import(fs.getConf());
cmd.setStore(ms);
expectSuccess("Import command did not exit successfully - see output",
cmd,
"import", parent.toString());
exec(cmd, "import", parent.toString());
DirListingMetadata children =
ms.listChildren(dir);
@ -80,7 +81,7 @@ public class ITestS3GuardToolLocal extends AbstractS3GuardToolTestBase {
}
@Test
public void testDiffCommand() throws IOException {
public void testDiffCommand() throws Exception {
S3AFileSystem fs = getFileSystem();
MetadataStore ms = getMetadataStore();
Set<Path> filesOnS3 = new HashSet<>(); // files on S3.
@ -108,13 +109,10 @@ public class ITestS3GuardToolLocal extends AbstractS3GuardToolTestBase {
}
ByteArrayOutputStream buf = new ByteArrayOutputStream();
PrintStream out = new PrintStream(buf);
Diff cmd = new Diff(fs.getConf());
cmd.setStore(ms);
assertEquals("Diff command did not exit successfully - see output", SUCCESS,
cmd.run(new String[]{"diff", "-meta", "local://metadata",
testPath.toString()}, out));
out.close();
exec(cmd, buf, "diff", "-meta", LOCAL_METADATA,
testPath.toString());
Set<Path> actualOnS3 = new HashSet<>();
Set<Path> actualOnMS = new HashSet<>();
@ -140,10 +138,128 @@ public class ITestS3GuardToolLocal extends AbstractS3GuardToolTestBase {
}
}
}
String actualOut = out.toString();
String actualOut = buf.toString();
assertEquals("Mismatched metadata store outputs: " + actualOut,
filesOnMS, actualOnMS);
assertEquals("Mismatched s3 outputs: " + actualOut, filesOnS3, actualOnS3);
assertFalse("Diff contained duplicates", duplicates);
}
@Test
public void testDestroyBucketExistsButNoTable() throws Throwable {
run(Destroy.NAME,
"-meta", LOCAL_METADATA,
getLandsatCSVFile());
}
@Test
public void testImportNoFilesystem() throws Throwable {
final Import importer =
new S3GuardTool.Import(getConfiguration());
importer.setStore(getMetadataStore());
intercept(IOException.class,
new Callable<Integer>() {
@Override
public Integer call() throws Exception {
return importer.run(
new String[]{
"import",
"-meta", LOCAL_METADATA,
S3A_THIS_BUCKET_DOES_NOT_EXIST
});
}
});
}
@Test
public void testInfoBucketAndRegionNoFS() throws Throwable {
intercept(FileNotFoundException.class,
new Callable<Integer>() {
@Override
public Integer call() throws Exception {
return run(BucketInfo.NAME, "-meta",
LOCAL_METADATA, "-region",
"any-region", S3A_THIS_BUCKET_DOES_NOT_EXIST);
}
});
}
@Test
public void testInitNegativeRead() throws Throwable {
runToFailure(INVALID_ARGUMENT,
Init.NAME, "-meta", LOCAL_METADATA, "-region",
"eu-west-1",
READ_FLAG, "-10");
}
@Test
public void testInit() throws Throwable {
run(Init.NAME,
"-meta", LOCAL_METADATA,
"-region", "us-west-1");
}
@Test
public void testInitTwice() throws Throwable {
run(Init.NAME,
"-meta", LOCAL_METADATA,
"-region", "us-west-1");
run(Init.NAME,
"-meta", LOCAL_METADATA,
"-region", "us-west-1");
}
@Test
public void testLandsatBucketUnguarded() throws Throwable {
run(BucketInfo.NAME,
"-" + BucketInfo.UNGUARDED_FLAG,
getLandsatCSVFile());
}
@Test
public void testLandsatBucketRequireGuarded() throws Throwable {
runToFailure(E_BAD_STATE,
BucketInfo.NAME,
"-" + BucketInfo.GUARDED_FLAG,
ITestS3GuardToolLocal.this.getLandsatCSVFile());
}
@Test
public void testLandsatBucketRequireUnencrypted() throws Throwable {
run(BucketInfo.NAME,
"-" + BucketInfo.ENCRYPTION_FLAG, "none",
getLandsatCSVFile());
}
@Test
public void testLandsatBucketRequireEncrypted() throws Throwable {
runToFailure(E_BAD_STATE,
BucketInfo.NAME,
"-" + BucketInfo.ENCRYPTION_FLAG,
"AES256", ITestS3GuardToolLocal.this.getLandsatCSVFile());
}
@Test
public void testStoreInfo() throws Throwable {
S3GuardTool.BucketInfo cmd = new S3GuardTool.BucketInfo(
getFileSystem().getConf());
cmd.setStore(getMetadataStore());
String output = exec(cmd, cmd.getName(),
"-" + S3GuardTool.BucketInfo.GUARDED_FLAG,
getFileSystem().getUri().toString());
LOG.info("Exec output=\n{}", output);
}
@Test
public void testSetCapacity() throws Throwable {
S3GuardTool cmd = new S3GuardTool.SetCapacity(getFileSystem().getConf());
cmd.setStore(getMetadataStore());
String output = exec(cmd, cmd.getName(),
"-" + READ_FLAG, "100",
"-" + WRITE_FLAG, "100",
getFileSystem().getUri().toString());
LOG.info("Exec output=\n{}", output);
}
}

128
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestS3GuardCLI.java Normal file
View File

@ -0,0 +1,128 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.s3a.s3guard;
import java.util.concurrent.Callable;
import org.junit.Assert;
import org.junit.Test;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.s3a.S3ATestConstants;
import org.apache.hadoop.test.LambdaTestUtils;
import org.apache.hadoop.util.ExitUtil;
import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.*;
/**
* Test the S3Guard CLI entry point.
*/
public class TestS3GuardCLI extends Assert {
/**
* Run a S3GuardTool command from a varags list.
* @param args argument list
* @return the return code
* @throws Exception any exception
*/
protected int run(String... args)
throws Exception {
Configuration conf = new Configuration(false);
return S3GuardTool.run(conf, args);
}
/**
* Run a S3GuardTool command from a varags list, catch any raised
* ExitException and verify the status code matches that expected.
* @param status expected status code of an exception
* @param args argument list
* @throws Exception any exception
*/
protected void runToFailure(int status, final String... args)
throws Exception {
ExitUtil.ExitException ex =
LambdaTestUtils.intercept(ExitUtil.ExitException.class,
new Callable<Integer>() {
@Override
public Integer call() throws Exception {
return run(args);
}
});
if (ex.status != status) {
throw ex;
}
}
@Test
public void testInfoNoArgs() throws Throwable {
runToFailure(INVALID_ARGUMENT, BucketInfo.NAME);
}
@Test
public void testInfoWrongFilesystem() throws Throwable {
runToFailure(INVALID_ARGUMENT,
BucketInfo.NAME, "file://");
}
@Test
public void testNoCommand() throws Throwable {
runToFailure(E_USAGE);
}
@Test
public void testUnknownCommand() throws Throwable {
runToFailure(E_USAGE, "unknown");
}
@Test
public void testPruneNoArgs() throws Throwable {
runToFailure(INVALID_ARGUMENT, Prune.NAME);
}
@Test
public void testDiffNoArgs() throws Throwable {
runToFailure(INVALID_ARGUMENT, Diff.NAME);
}
@Test
public void testImportNoArgs() throws Throwable {
runToFailure(INVALID_ARGUMENT, Import.NAME);
}
@Test
public void testDestroyNoArgs() throws Throwable {
runToFailure(INVALID_ARGUMENT, Destroy.NAME);
}
@Test
public void testDestroyUnknownTableNoRegion() throws Throwable {
runToFailure(INVALID_ARGUMENT, Destroy.NAME,
"-meta", "dynamodb://ireland-team");
}
@Test
public void testInitBucketAndRegion() throws Throwable {
runToFailure(INVALID_ARGUMENT, Init.NAME,
"-meta", "dynamodb://ireland-team",
"-region", "eu-west-1",
S3ATestConstants.DEFAULT_CSVTEST_FILE
);
}
}