HADOOP-13237: s3a initialization against public bucket fails if caller lacks any credentials. Contributed by Chris Nauroth
This commit is contained in:
parent
3b2a25bc4b
commit
7e09601a90
|
@ -746,7 +746,18 @@
|
|||
|
||||
<property>
|
||||
<name>fs.s3a.aws.credentials.provider</name>
|
||||
<description>Class name of a credentials provider that implements com.amazonaws.auth.AWSCredentialsProvider. Omit if using access/secret keys or another authentication mechanism.</description>
|
||||
<description>
|
||||
Class name of a credentials provider that implements
|
||||
com.amazonaws.auth.AWSCredentialsProvider. Omit if using access/secret keys
|
||||
or another authentication mechanism. The specified class must provide an
|
||||
accessible constructor accepting java.net.URI and
|
||||
org.apache.hadoop.conf.Configuration, or an accessible default constructor.
|
||||
Specifying org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider allows
|
||||
anonymous access to a publicly accessible S3 bucket without any credentials.
|
||||
Please note that allowing anonymous access to an S3 bucket compromises
|
||||
security and therefore is unsuitable for most use cases. It can be useful
|
||||
for accessing public data sets without requiring AWS credentials.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
|
|
|
@ -24,6 +24,17 @@ import com.amazonaws.auth.AWSCredentials;
|
|||
import org.apache.hadoop.classification.InterfaceAudience;
|
||||
import org.apache.hadoop.classification.InterfaceStability;
|
||||
|
||||
/**
|
||||
* AnonymousAWSCredentialsProvider supports anonymous access to AWS services
|
||||
* through the AWS SDK. AWS requests will not be signed. This is not suitable
|
||||
* for most cases, because allowing anonymous access to an S3 bucket compromises
|
||||
* security. This can be useful for accessing public data sets without
|
||||
* requiring AWS credentials.
|
||||
*
|
||||
* Please note that users may reference this class name from configuration
|
||||
* property fs.s3a.aws.credentials.provider. Therefore, changing the class name
|
||||
* would be a backward-incompatible change.
|
||||
*/
|
||||
@InterfaceAudience.Private
|
||||
@InterfaceStability.Stable
|
||||
public class AnonymousAWSCredentialsProvider implements AWSCredentialsProvider {
|
||||
|
|
|
@ -26,6 +26,14 @@ import org.apache.commons.lang.StringUtils;
|
|||
import org.apache.hadoop.classification.InterfaceAudience;
|
||||
import org.apache.hadoop.classification.InterfaceStability;
|
||||
|
||||
/**
|
||||
* BasicAWSCredentialsProvider supports static configuration of access key ID
|
||||
* and secret access key for use with the AWS SDK.
|
||||
*
|
||||
* Please note that users may reference this class name from configuration
|
||||
* property fs.s3a.aws.credentials.provider. Therefore, changing the class name
|
||||
* would be a backward-incompatible change.
|
||||
*/
|
||||
@InterfaceAudience.Private
|
||||
@InterfaceStability.Stable
|
||||
public class BasicAWSCredentialsProvider implements AWSCredentialsProvider {
|
||||
|
|
|
@ -527,20 +527,28 @@ public class S3AFileSystem extends FileSystem {
|
|||
new BasicAWSCredentialsProvider(
|
||||
creds.getAccessKey(), creds.getAccessSecret()),
|
||||
new InstanceProfileCredentialsProvider(),
|
||||
new EnvironmentVariableCredentialsProvider(),
|
||||
new AnonymousAWSCredentialsProvider()
|
||||
);
|
||||
new EnvironmentVariableCredentialsProvider());
|
||||
|
||||
} else {
|
||||
try {
|
||||
LOG.debug("Credential provider class is {}", className);
|
||||
credentials = (AWSCredentialsProvider) Class.forName(className)
|
||||
.getDeclaredConstructor(URI.class, Configuration.class)
|
||||
.newInstance(this.uri, conf);
|
||||
Class<?> credClass = Class.forName(className);
|
||||
try {
|
||||
credentials =
|
||||
(AWSCredentialsProvider)credClass.getDeclaredConstructor(
|
||||
URI.class, Configuration.class).newInstance(this.uri, conf);
|
||||
} catch (NoSuchMethodException | SecurityException e) {
|
||||
credentials =
|
||||
(AWSCredentialsProvider)credClass.getDeclaredConstructor()
|
||||
.newInstance();
|
||||
}
|
||||
} catch (ClassNotFoundException e) {
|
||||
throw new IOException(className + " not found.", e);
|
||||
} catch (NoSuchMethodException | SecurityException e) {
|
||||
throw new IOException(className + " constructor exception.", e);
|
||||
throw new IOException(String.format("%s constructor exception. A "
|
||||
+ "class specified in %s must provide an accessible constructor "
|
||||
+ "accepting URI and Configuration, or an accessible default "
|
||||
+ "constructor.", className, AWS_CREDENTIALS_PROVIDER), e);
|
||||
} catch (ReflectiveOperationException | IllegalArgumentException e) {
|
||||
throw new IOException(className + " instantiation exception.", e);
|
||||
}
|
||||
|
|
|
@ -184,8 +184,18 @@ If you do any of these: change your credentials immediately!
|
|||
|
||||
<property>
|
||||
<name>fs.s3a.aws.credentials.provider</name>
|
||||
<description>Class name of a credentials provider that implements com.amazonaws.auth.AWSCredentialsProvider.
|
||||
Omit if using access/secret keys or another authentication mechanism.</description>
|
||||
<description>
|
||||
Class name of a credentials provider that implements
|
||||
com.amazonaws.auth.AWSCredentialsProvider. Omit if using access/secret keys
|
||||
or another authentication mechanism. The specified class must provide an
|
||||
accessible constructor accepting java.net.URI and
|
||||
org.apache.hadoop.conf.Configuration, or an accessible default constructor.
|
||||
Specifying org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider allows
|
||||
anonymous access to a publicly accessible S3 bucket without any credentials.
|
||||
Please note that allowing anonymous access to an S3 bucket compromises
|
||||
security and therefore is unsuitable for most use cases. It can be useful
|
||||
for accessing public data sets without requiring AWS credentials.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
#### Protecting the AWS Credentials in S3A
|
||||
|
|
|
@ -19,6 +19,7 @@
|
|||
package org.apache.hadoop.fs.s3a;
|
||||
|
||||
import static org.apache.hadoop.fs.s3a.Constants.*;
|
||||
import static org.apache.hadoop.fs.s3a.S3ATestConstants.*;
|
||||
import static org.junit.Assert.*;
|
||||
|
||||
import java.io.IOException;
|
||||
|
@ -26,8 +27,13 @@ import java.net.URI;
|
|||
import java.nio.file.AccessDeniedException;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.FileStatus;
|
||||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
import org.junit.rules.ExpectedException;
|
||||
import org.junit.rules.Timeout;
|
||||
|
||||
import com.amazonaws.auth.AWSCredentials;
|
||||
import com.amazonaws.auth.AWSCredentialsProvider;
|
||||
|
@ -45,6 +51,12 @@ public class TestS3AAWSCredentialsProvider {
|
|||
private static final Logger LOG =
|
||||
LoggerFactory.getLogger(TestS3AAWSCredentialsProvider.class);
|
||||
|
||||
@Rule
|
||||
public Timeout testTimeout = new Timeout(1 * 60 * 1000);
|
||||
|
||||
@Rule
|
||||
public ExpectedException exception = ExpectedException.none();
|
||||
|
||||
@Test
|
||||
public void testBadConfiguration() throws IOException {
|
||||
Configuration conf = new Configuration();
|
||||
|
@ -113,4 +125,47 @@ public class TestS3AAWSCredentialsProvider {
|
|||
conf.set(AWS_CREDENTIALS_PROVIDER, GoodCredentialsProvider.class.getName());
|
||||
S3ATestUtils.createTestFileSystem(conf);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAnonymousProvider() throws Exception {
|
||||
Configuration conf = new Configuration();
|
||||
conf.set(AWS_CREDENTIALS_PROVIDER,
|
||||
AnonymousAWSCredentialsProvider.class.getName());
|
||||
Path testFile = new Path(
|
||||
conf.getTrimmed(KEY_CSVTEST_FILE, DEFAULT_CSVTEST_FILE));
|
||||
FileSystem fs = FileSystem.newInstance(testFile.toUri(), conf);
|
||||
assertNotNull(fs);
|
||||
assertTrue(fs instanceof S3AFileSystem);
|
||||
FileStatus stat = fs.getFileStatus(testFile);
|
||||
assertNotNull(stat);
|
||||
assertEquals(testFile, stat.getPath());
|
||||
}
|
||||
|
||||
static class ConstructorErrorProvider implements AWSCredentialsProvider {
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
public ConstructorErrorProvider(String str) {
|
||||
}
|
||||
|
||||
@Override
|
||||
public AWSCredentials getCredentials() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void refresh() {
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testProviderConstructorError() throws Exception {
|
||||
Configuration conf = new Configuration();
|
||||
conf.set(AWS_CREDENTIALS_PROVIDER,
|
||||
ConstructorErrorProvider.class.getName());
|
||||
Path testFile = new Path(
|
||||
conf.getTrimmed(KEY_CSVTEST_FILE, DEFAULT_CSVTEST_FILE));
|
||||
exception.expect(IOException.class);
|
||||
exception.expectMessage("constructor exception");
|
||||
FileSystem fs = FileSystem.newInstance(testFile.toUri(), conf);
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue