HADOOP-13336 S3A to support per-bucket configuration. Contributed by Steve Loughran

This commit is contained in:
Steve Loughran 2017-01-11 17:13:05 +00:00
parent 357eab9566
commit e648b6e138
11 changed files with 442 additions and 71 deletions

View File

@ -954,6 +954,15 @@
</description>
</property>
<property>
<name>fs.s3a.security.credential.provider.path</name>
<value />
<description>
Optional comma separated list of credential providers, a list
which is prepended to that set in hadoop.security.credential.provider.path
</description>
</property>
<property>
<name>fs.s3a.connection.maximum</name>
<value>15</value>

View File

@ -48,6 +48,14 @@ public final class Constants {
public static final String AWS_CREDENTIALS_PROVIDER =
"fs.s3a.aws.credentials.provider";
/**
* Extra set of security credentials which will be prepended to that
* set in {@code "hadoop.security.credential.provider.path"}.
* This extra option allows for per-bucket overrides.
*/
public static final String S3A_SECURITY_CREDENTIAL_PROVIDER_PATH =
"fs.s3a.security.credential.provider.path";
// session token for when using TemporaryAWSCredentialsProvider
public static final String SESSION_TOKEN = "fs.s3a.session.token";
@ -226,6 +234,12 @@ public final class Constants {
public static final String FS_S3A_BLOCK_SIZE = "fs.s3a.block.size";
public static final String FS_S3A = "s3a";
/** Prefix for all S3A properties: {@value}. */
public static final String FS_S3A_PREFIX = "fs.s3a.";
/** Prefix for S3A bucket-specific properties: {@value}. */
public static final String FS_S3A_BUCKET_PREFIX = "fs.s3a.bucket.";
public static final int S3A_DEFAULT_PORT = -1;
public static final String USER_AGENT_PREFIX = "fs.s3a.user.agent.prefix";

View File

@ -29,7 +29,6 @@ import java.util.Date;
import java.util.EnumSet;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.Objects;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
@ -153,21 +152,28 @@ public class S3AFileSystem extends FileSystem {
/** Called after a new FileSystem instance is constructed.
* @param name a uri whose authority section names the host, port, etc.
* for this FileSystem
* @param conf the configuration
* @param originalConf the configuration to use for the FS. The
* bucket-specific options are patched over the base ones before any use is
* made of the config.
*/
public void initialize(URI name, Configuration conf) throws IOException {
public void initialize(URI name, Configuration originalConf)
throws IOException {
uri = S3xLoginHelper.buildFSURI(name);
// get the host; this is guaranteed to be non-null, non-empty
bucket = name.getHost();
// clone the configuration into one with propagated bucket options
Configuration conf = propagateBucketOptions(originalConf, bucket);
patchSecurityCredentialProviders(conf);
super.initialize(name, conf);
setConf(conf);
try {
instrumentation = new S3AInstrumentation(name);
uri = S3xLoginHelper.buildFSURI(name);
// Username is the current user at the time the FS was instantiated.
username = UserGroupInformation.getCurrentUser().getShortUserName();
workingDir = new Path("/user", username)
.makeQualified(this.uri, this.getWorkingDirectory());
bucket = name.getHost();
Class<? extends S3ClientFactory> s3ClientFactoryClass = conf.getClass(
S3_CLIENT_FACTORY_IMPL, DEFAULT_S3_CLIENT_FACTORY_IMPL,

View File

@ -35,6 +35,8 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.s3native.S3xLoginHelper;
import org.apache.hadoop.security.ProviderUtils;
import com.google.common.collect.Lists;
import org.slf4j.Logger;
import java.io.EOFException;
@ -46,15 +48,13 @@ import java.lang.reflect.Method;
import java.lang.reflect.Modifier;
import java.net.URI;
import java.nio.file.AccessDeniedException;
import java.util.Collection;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import static org.apache.hadoop.fs.s3a.Constants.ACCESS_KEY;
import static org.apache.hadoop.fs.s3a.Constants.AWS_CREDENTIALS_PROVIDER;
import static org.apache.hadoop.fs.s3a.Constants.ENDPOINT;
import static org.apache.hadoop.fs.s3a.Constants.MULTIPART_MIN_SIZE;
import static org.apache.hadoop.fs.s3a.Constants.SECRET_KEY;
import static org.apache.hadoop.fs.s3a.Constants.*;
/**
* Utility methods for S3A code.
@ -74,6 +74,13 @@ public final class S3AUtils {
"is abstract and therefore cannot be created";
static final String ENDPOINT_KEY = "Endpoint";
/**
* Core property for provider path. Duplicated here for consistent
* code across Hadoop version: {@value}.
*/
static final String CREDENTIAL_PROVIDER_PATH =
"hadoop.security.credential.provider.path";
private S3AUtils() {
}
@ -636,4 +643,84 @@ public final class S3AUtils {
return null;
}
}
/**
* Propagates bucket-specific settings into generic S3A configuration keys.
* This is done by propagating the values of the form
* {@code fs.s3a.bucket.${bucket}.key} to
* {@code fs.s3a.key}, for all values of "key" other than a small set
* of unmodifiable values.
*
* The source of the updated property is set to the key name of the bucket
* property, to aid in diagnostics of where things came from.
*
* Returns a new configuration. Why the clone?
* You can use the same conf for different filesystems, and the original
* values are not updated.
*
* The {@code fs.s3a.impl} property cannot be set, nor can
* any with the prefix {@code fs.s3a.bucket}.
*
* This method does not propagate security provider path information from
* the S3A property into the Hadoop common provider: callers must call
* {@link #patchSecurityCredentialProviders(Configuration)} explicitly.
* @param source Source Configuration object.
* @param bucket bucket name. Must not be empty.
* @return a (potentially) patched clone of the original.
*/
public static Configuration propagateBucketOptions(Configuration source,
String bucket) {
Preconditions.checkArgument(StringUtils.isNotEmpty(bucket), "bucket");
final String bucketPrefix = FS_S3A_BUCKET_PREFIX + bucket +'.';
LOG.debug("Propagating entries under {}", bucketPrefix);
final Configuration dest = new Configuration(source);
for (Map.Entry<String, String> entry : source) {
final String key = entry.getKey();
// get the (unexpanded) value.
final String value = entry.getValue();
if (!key.startsWith(bucketPrefix) || bucketPrefix.equals(key)) {
continue;
}
// there's a bucket prefix, so strip it
final String stripped = key.substring(bucketPrefix.length());
if (stripped.startsWith("bucket.") || "impl".equals(stripped)) {
//tell user off
LOG.debug("Ignoring bucket option {}", key);
} else {
// propagate the value, building a new origin field.
// to track overwrites, the generic key is overwritten even if
// already matches the new one.
final String generic = FS_S3A_PREFIX + stripped;
LOG.debug("Updating {}", generic);
dest.set(generic, value, key);
}
}
return dest;
}
/**
* Patch the security credential provider information in
* {@link #CREDENTIAL_PROVIDER_PATH}
* with the providers listed in
* {@link Constants#S3A_SECURITY_CREDENTIAL_PROVIDER_PATH}.
*
* This allows different buckets to use different credential files.
* @param conf configuration to patch
*/
static void patchSecurityCredentialProviders(Configuration conf) {
Collection<String> customCredentials = conf.getStringCollection(
S3A_SECURITY_CREDENTIAL_PROVIDER_PATH);
Collection<String> hadoopCredentials = conf.getStringCollection(
CREDENTIAL_PROVIDER_PATH);
if (!customCredentials.isEmpty()) {
List<String> all = Lists.newArrayList(customCredentials);
all.addAll(hadoopCredentials);
String joined = StringUtils.join(all, ',');
LOG.debug("Setting {} to {}", CREDENTIAL_PROVIDER_PATH,
joined);
conf.set(CREDENTIAL_PROVIDER_PATH, joined,
"patch of " + S3A_SECURITY_CREDENTIAL_PROVIDER_PATH);
}
}
}

View File

@ -255,6 +255,7 @@ properties, the Hadoop key management store and IAM roles.
* Test suites includes distcp and suites in downstream projects.
* Available since Hadoop 2.6; considered production ready in Hadoop 2.7.
* Actively maintained.
* Supports per-bucket configuration.
S3A is now the recommended client for working with S3 objects. It is also the
one where patches for functionality and performance are very welcome.
@ -609,6 +610,29 @@ in XML configuration files.
Because this property only supplies the path to the secrets file, the configuration
option itself is no longer a sensitive item.
The property `hadoop.security.credential.provider.path` is global to all
filesystems and secrets.
There is another property, `fs.s3a.security.credential.provider.path`
which only lists credential providers for S3A filesystems.
The two properties are combined into one, with the list of providers in the
`fs.s3a.` property taking precedence
over that of the `hadoop.security` list (i.e. they are prepended to the common list).
```xml
<property>
<name>fs.s3a.security.credential.provider.path</name>
<value />
<description>
Optional comma separated list of credential providers, a list
which is prepended to that set in hadoop.security.credential.provider.path
</description>
</property>
```
Supporting a separate list in an `fs.s3a.` prefix permits per-bucket configuration
of credential files.
###### Using the credentials
Once the provider is set in the Hadoop configuration, hadoop commands
@ -631,7 +655,7 @@ hadoop distcp \
hdfs://nn1.example.com:9001/user/backup/007020615 s3a://glacier1/
hadoop fs \
-D hadoop.security.credential.provider.path=jceks://hdfs@nn1.example.com:9001/user/backup/s3.jceks \
-D fs.s3a.security.credential.provider.path=jceks://hdfs@nn1.example.com:9001/user/backup/s3.jceks \
-ls s3a://glacier1/
```
@ -869,6 +893,78 @@ from placing its declaration on the command line.
any call to setReadahead() is made to an open stream.</description>
</property>
### Configurations different S3 buckets
Different S3 buckets can be accessed with different S3A client configurations.
This allows for different endpoints, data read and write strategies, as well
as login details.
1. All `fs.s3a` options other than a small set of unmodifiable values
(currently `fs.s3a.impl`) can be set on a per bucket basis.
1. The bucket specific option is set by replacing the `fs.s3a.` prefix on an option
with `fs.s3a.bucket.BUCKETNAME.`, where `BUCKETNAME` is the name of the bucket.
1. When connecting to a bucket, all options explicitly set will override
the base `fs.s3a.` values.
As an example, a configuration could have a base configuration to use the IAM
role information available when deployed in Amazon EC2.
```xml
<property>
<name>fs.s3a.aws.credentials.provider</name>
<value>org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider</value>
</property>
```
This will be the default authentication mechanism for S3A buckets.
A bucket `s3a://nightly/` used for nightly data uses a session key:
```xml
<property>
<name>fs.s3a.bucket.nightly.access.key</name>
<value>AKAACCESSKEY-2</value>
</property>
<property>
<name>fs.s3a.bucket.nightly.secret.key</name>
<value>SESSIONSECRETKEY</value>
</property>
<property>
<name>fs.s3a.bucket.nightly.session.token</name>
<value>Short-lived-session-token</value>
</property>
<property>
<name>fs.s3a.bucket.nightly.aws.credentials.provider</name>
<value>org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider</value>
</property>
```
Finally, the public `s3a://landsat-pds/` bucket is accessed anonymously:
```xml
<property>
<name>fs.s3a.bucket.landsat-pds.aws.credentials.provider</name>
<value>org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider</value>
</property>
```
**Customizing S3A secrets held in credential files**
Although most properties are automatically propagated from their
`fs.s3a.bucket.`-prefixed custom entry to that of the base `fs.s3a.` option
supporting secrets kept in Hadoop credential files is slightly more complex.
This is because the property values are kept in these files, and cannot be
dynamically patched.
Instead, callers need to create different configuration files for each
bucket, setting the base secrets (`fs.s3a.bucket.nightly.access.key`, etc),
then declare the path to the appropriate credential file in
a bucket-specific version of the property `fs.s3a.security.credential.provider.path`.
### Working with buckets in different regions
S3 Buckets are hosted in different regions, the default being US-East.
@ -924,6 +1020,16 @@ If the wrong endpoint is used, the request may fail. This may be reported as a 3
or as a 400 Bad Request.
If you are trying to mix endpoints for different buckets, use a per-bucket endpoint
declaration. For example:
```xml
<property>
<name>fs.s3a.bucket.landsat-pds.endpoint</name>
<value>s3.amazonaws.com</value>
<description>The endpoint for s3a://landsat-pds URLs</description>
</property>
```
### <a name="s3a_fast_upload"></a>Stabilizing: S3A Fast Upload
@ -1603,15 +1709,15 @@ org.apache.hadoop.fs.s3a.AWSS3IOException: Received permanent redirect response
1. If not using "V4" authentication (see above), the original S3 endpoint
can be used:
```
<property>
<name>fs.s3a.endpoint</name>
<value>s3.amazonaws.com</value>
</property>
```xml
<property>
<name>fs.s3a.endpoint</name>
<value>s3.amazonaws.com</value>
</property>
```
Using the explicit endpoint for the region is recommended for speed and the
ability to use the V4 signing API.
Using the explicit endpoint for the region is recommended for speed and
to use the V4 signing API.
### "Timeout waiting for connection from pool" when writing to S3A
@ -2163,7 +2269,22 @@ is hosted in Amazon's US-east datacenter.
1. If the property is set to a different path, then that data must be readable
and "sufficiently" large.
To test on different S3 endpoints, or alternate infrastructures supporting
(the reason the space or newline is needed is to add "an empty entry"; an empty
`<value/>` would be considered undefined and pick up the default)
Of using a test file in an S3 region requiring a different endpoint value
set in `fs.s3a.endpoint`, a bucket-specific endpoint must be defined.
For the default test dataset, hosted in the `landsat-pds` bucket, this is:
```xml
<property>
<name>fs.s3a.bucket.landsat-pds.endpoint</name>
<value>s3.amazonaws.com</value>
<description>The endpoint for s3a://landsat-pds URLs</description>
</property>
```
To test on alternate infrastructures supporting
the same APIs, the option `fs.s3a.scale.test.csvfile` must either be
set to " ", or an object of at least 10MB is uploaded to the object store, and
the `fs.s3a.scale.test.csvfile` option set to its path.
@ -2175,20 +2296,6 @@ the `fs.s3a.scale.test.csvfile` option set to its path.
</property>
```
(the reason the space or newline is needed is to add "an empty entry"; an empty
`<value/>` would be considered undefined and pick up the default)
*Note:* if using a test file in an S3 region requiring a different endpoint value
set in `fs.s3a.endpoint`, define it in `fs.s3a.scale.test.csvfile.endpoint`.
If the default CSV file is used, the tests will automatically use the us-east
endpoint:
```xml
<property>
<name>fs.s3a.scale.test.csvfile.endpoint</name>
<value>s3.amazonaws.com</value>
</property>
```
### Viewing Integration Test Reports

View File

@ -129,7 +129,6 @@ public class ITestS3AAWSCredentialsProvider {
AnonymousAWSCredentialsProvider.class.getName());
Path testFile = new Path(
conf.getTrimmed(KEY_CSVTEST_FILE, DEFAULT_CSVTEST_FILE));
S3ATestUtils.useCSVDataEndpoint(conf);
FileSystem fs = FileSystem.newInstance(testFile.toUri(), conf);
assertNotNull(fs);
assertTrue(fs instanceof S3AFileSystem);

View File

@ -35,6 +35,7 @@ import org.junit.rules.Timeout;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.apache.hadoop.fs.s3a.S3ATestConstants.TEST_FS_S3A_NAME;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
@ -45,6 +46,7 @@ import static org.junit.Assert.fail;
import java.io.File;
import java.net.URI;
import java.security.PrivilegedExceptionAction;
import java.util.Collection;
import org.apache.hadoop.security.ProviderUtils;
import org.apache.hadoop.security.UserGroupInformation;
@ -54,6 +56,10 @@ import org.apache.hadoop.util.VersionInfo;
import org.apache.http.HttpStatus;
import org.junit.rules.TemporaryFolder;
import static org.apache.hadoop.fs.s3a.Constants.*;
import static org.apache.hadoop.fs.s3a.S3AUtils.*;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.*;
/**
* S3A tests for configuration.
*/
@ -505,4 +511,128 @@ public class ITestS3AConfiguration {
fieldType.isAssignableFrom(obj.getClass()));
return fieldType.cast(obj);
}
@Test
public void testBucketConfigurationPropagation() throws Throwable {
Configuration config = new Configuration(false);
setBucketOption(config, "b", "base", "1024");
String basekey = "fs.s3a.base";
assertOptionEquals(config, basekey, null);
String bucketKey = "fs.s3a.bucket.b.base";
assertOptionEquals(config, bucketKey, "1024");
Configuration updated = propagateBucketOptions(config, "b");
assertOptionEquals(updated, basekey, "1024");
// original conf is not updated
assertOptionEquals(config, basekey, null);
String[] sources = updated.getPropertySources(basekey);
assertEquals(1, sources.length);
String sourceInfo = sources[0];
assertTrue("Wrong source " + sourceInfo, sourceInfo.contains(bucketKey));
}
@Test
public void testBucketConfigurationPropagationResolution() throws Throwable {
Configuration config = new Configuration(false);
String basekey = "fs.s3a.base";
String baseref = "fs.s3a.baseref";
String baseref2 = "fs.s3a.baseref2";
config.set(basekey, "orig");
config.set(baseref2, "${fs.s3a.base}");
setBucketOption(config, "b", basekey, "1024");
setBucketOption(config, "b", baseref, "${fs.s3a.base}");
Configuration updated = propagateBucketOptions(config, "b");
assertOptionEquals(updated, basekey, "1024");
assertOptionEquals(updated, baseref, "1024");
assertOptionEquals(updated, baseref2, "1024");
}
@Test
public void testMultipleBucketConfigurations() throws Throwable {
Configuration config = new Configuration(false);
setBucketOption(config, "b", USER_AGENT_PREFIX, "UA-b");
setBucketOption(config, "c", USER_AGENT_PREFIX, "UA-c");
config.set(USER_AGENT_PREFIX, "UA-orig");
Configuration updated = propagateBucketOptions(config, "c");
assertOptionEquals(updated, USER_AGENT_PREFIX, "UA-c");
}
@Test
public void testBucketConfigurationSkipsUnmodifiable() throws Throwable {
Configuration config = new Configuration(false);
String impl = "fs.s3a.impl";
config.set(impl, "orig");
setBucketOption(config, "b", impl, "b");
String metastoreImpl = "fs.s3a.metadatastore.impl";
String ddb = "org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStore";
setBucketOption(config, "b", metastoreImpl, ddb);
setBucketOption(config, "b", "impl2", "b2");
setBucketOption(config, "b", "bucket.b.loop", "b3");
assertOptionEquals(config, "fs.s3a.bucket.b.impl", "b");
Configuration updated = propagateBucketOptions(config, "b");
assertOptionEquals(updated, impl, "orig");
assertOptionEquals(updated, "fs.s3a.impl2", "b2");
assertOptionEquals(updated, metastoreImpl, ddb);
assertOptionEquals(updated, "fs.s3a.bucket.b.loop", null);
}
@Test
public void testConfOptionPropagationToFS() throws Exception {
Configuration config = new Configuration();
String testFSName = config.getTrimmed(TEST_FS_S3A_NAME, "");
String bucket = new URI(testFSName).getHost();
setBucketOption(config, bucket, "propagation", "propagated");
fs = S3ATestUtils.createTestFileSystem(config);
Configuration updated = fs.getConf();
assertOptionEquals(updated, "fs.s3a.propagation", "propagated");
}
@Test
public void testSecurityCredentialPropagationNoOverride() throws Exception {
Configuration config = new Configuration();
config.set(CREDENTIAL_PROVIDER_PATH, "base");
patchSecurityCredentialProviders(config);
assertOptionEquals(config, CREDENTIAL_PROVIDER_PATH,
"base");
}
@Test
public void testSecurityCredentialPropagationOverrideNoBase()
throws Exception {
Configuration config = new Configuration();
config.unset(CREDENTIAL_PROVIDER_PATH);
config.set(S3A_SECURITY_CREDENTIAL_PROVIDER_PATH, "override");
patchSecurityCredentialProviders(config);
assertOptionEquals(config, CREDENTIAL_PROVIDER_PATH,
"override");
}
@Test
public void testSecurityCredentialPropagationOverride() throws Exception {
Configuration config = new Configuration();
config.set(CREDENTIAL_PROVIDER_PATH, "base");
config.set(S3A_SECURITY_CREDENTIAL_PROVIDER_PATH, "override");
patchSecurityCredentialProviders(config);
assertOptionEquals(config, CREDENTIAL_PROVIDER_PATH,
"override,base");
Collection<String> all = config.getStringCollection(
CREDENTIAL_PROVIDER_PATH);
assertTrue(all.contains("override"));
assertTrue(all.contains("base"));
}
@Test
public void testSecurityCredentialPropagationEndToEnd() throws Exception {
Configuration config = new Configuration();
config.set(CREDENTIAL_PROVIDER_PATH, "base");
setBucketOption(config, "b", S3A_SECURITY_CREDENTIAL_PROVIDER_PATH,
"override");
Configuration updated = propagateBucketOptions(config, "b");
patchSecurityCredentialProviders(updated);
assertOptionEquals(updated, CREDENTIAL_PROVIDER_PATH,
"override,base");
}
}

View File

@ -85,18 +85,6 @@ public interface S3ATestConstants {
*/
String DEFAULT_CSVTEST_FILE = "s3a://landsat-pds/scene_list.gz";
/**
* Endpoint for the S3 CSV/scale tests. This defaults to
* being us-east.
*/
String KEY_CSVTEST_ENDPOINT = S3A_SCALE_TEST + "csvfile.endpoint";
/**
* Endpoint for the S3 CSV/scale tests. This defaults to
* being us-east.
*/
String DEFAULT_CSVTEST_ENDPOINT = "s3.amazonaws.com";
/**
* Name of the property to define the timeout for scale tests: {@value}.
* Measured in seconds.

View File

@ -22,10 +22,11 @@ import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.s3a.scale.S3AScaleTestBase;
import org.junit.Assert;
import org.junit.Assume;
import org.junit.internal.AssumptionViolatedException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.net.URI;
@ -34,11 +35,14 @@ import java.util.List;
import static org.apache.hadoop.fs.contract.ContractTestUtils.skip;
import static org.apache.hadoop.fs.s3a.S3ATestConstants.*;
import static org.apache.hadoop.fs.s3a.Constants.*;
import static org.junit.Assert.*;
/**
* Utilities for the S3A tests.
*/
public final class S3ATestUtils {
private static final Logger LOG = LoggerFactory.getLogger(
S3ATestUtils.class);
/**
* Value to set a system property to (in maven) to declare that
@ -136,20 +140,6 @@ public final class S3ATestUtils {
return fc;
}
/**
* patch the endpoint option so that irrespective of where other tests
* are working, the IO performance tests can work with the landsat
* images.
* @param conf configuration to patch
*/
public static void useCSVDataEndpoint(Configuration conf) {
String endpoint = conf.getTrimmed(S3AScaleTestBase.KEY_CSVTEST_ENDPOINT,
S3AScaleTestBase.DEFAULT_CSVTEST_ENDPOINT);
if (!endpoint.isEmpty()) {
conf.set(ENDPOINT, endpoint);
}
}
/**
* Get a long test property.
* <ol>
@ -511,4 +501,47 @@ public final class S3ATestUtils {
*/
private S3ATestUtils() {
}
/**
* Set a bucket specific property to a particular value.
* If the generic key passed in has an {@code fs.s3a. prefix},
* that's stripped off, so that when the the bucket properties are propagated
* down to the generic values, that value gets copied down.
* @param conf configuration to set
* @param bucket bucket name
* @param genericKey key; can start with "fs.s3a."
* @param value value to set
*/
public static void setBucketOption(Configuration conf, String bucket,
String genericKey, String value) {
final String baseKey = genericKey.startsWith(FS_S3A_PREFIX) ?
genericKey.substring(FS_S3A_PREFIX.length())
: genericKey;
conf.set(FS_S3A_BUCKET_PREFIX + bucket + '.' + baseKey, value);
}
/**
* Assert that a configuration option matches the expected value.
* @param conf configuration
* @param key option key
* @param expected expected value
*/
public static void assertOptionEquals(Configuration conf,
String key,
String expected) {
assertEquals("Value of " + key, expected, conf.get(key));
}
/**
* Assume that a condition is met. If not: log at WARN and
* then throw an {@link AssumptionViolatedException}.
* @param message
* @param condition
*/
public static void assume(String message, boolean condition) {
if (!condition) {
LOG.warn(message);
}
Assume.assumeTrue(message, condition);
}
}

View File

@ -28,7 +28,6 @@ import org.apache.hadoop.fs.s3a.S3AFileSystem;
import org.apache.hadoop.fs.s3a.S3AInputPolicy;
import org.apache.hadoop.fs.s3a.S3AInputStream;
import org.apache.hadoop.fs.s3a.S3AInstrumentation;
import org.apache.hadoop.fs.s3a.S3ATestUtils;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.compress.CompressionCodec;
@ -36,7 +35,6 @@ import org.apache.hadoop.io.compress.CompressionCodecFactory;
import org.apache.hadoop.util.LineReader;
import org.junit.After;
import org.junit.Assert;
import org.junit.Assume;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
@ -47,6 +45,7 @@ import java.io.IOException;
import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
import static org.apache.hadoop.fs.s3a.Constants.*;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.assume;
/**
* Look at the performance of S3a operations.
@ -79,10 +78,11 @@ public class ITestS3AInputStreamPerformance extends S3AScaleTestBase {
String testFile = conf.getTrimmed(KEY_CSVTEST_FILE, DEFAULT_CSVTEST_FILE);
if (testFile.isEmpty()) {
assumptionMessage = "Empty test property: " + KEY_CSVTEST_FILE;
LOG.warn(assumptionMessage);
testDataAvailable = false;
} else {
S3ATestUtils.useCSVDataEndpoint(conf);
testData = new Path(testFile);
LOG.info("Using {} as input stream source", testData);
Path path = this.testData;
bindS3aFS(path);
try {
@ -113,7 +113,7 @@ public class ITestS3AInputStreamPerformance extends S3AScaleTestBase {
* Declare that the test requires the CSV test dataset.
*/
private void requireCSVTestData() {
Assume.assumeTrue(assumptionMessage, testDataAvailable);
assume(assumptionMessage, testDataAvailable);
}
/**
@ -146,7 +146,7 @@ public class ITestS3AInputStreamPerformance extends S3AScaleTestBase {
/**
* Open a test file with the read buffer specified in the setting
* {@link #KEY_READ_BUFFER_SIZE}.
* {@link org.apache.hadoop.fs.s3a.S3ATestConstants#KEY_READ_BUFFER_SIZE}.
*
* @param path path to open
* @param inputPolicy input policy to use

View File

@ -28,7 +28,6 @@ import org.apache.hadoop.fs.s3a.S3ATestConstants;
import org.apache.hadoop.fs.s3a.Statistic;
import org.apache.hadoop.metrics2.lib.MutableGaugeLong;
import org.junit.Assume;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -91,14 +90,13 @@ public class S3AScaleTestBase extends AbstractS3ATestBase {
super.setup();
testPath = path("/tests3ascale");
LOG.debug("Scale test operation count = {}", getOperationCount());
// multipart purges are disabled on the scale tests
// check for the test being enabled
enabled = getTestPropertyBool(
getConf(),
KEY_SCALE_TESTS_ENABLED,
DEFAULT_SCALE_TESTS_ENABLED);
Assume.assumeTrue("Scale test disabled: to enable set property " +
KEY_SCALE_TESTS_ENABLED, isEnabled());
assume("Scale test disabled: to enable set property " +
KEY_SCALE_TESTS_ENABLED,
isEnabled());
}
/**