HADOOP-13237: s3a initialization against public bucket fails if caller lacks any credentials. Contributed by Chris Nauroth

hadoop-common-project/hadoop-common/src/main/resources/core-default.xml

@@ -791,7 +791,18 @@
 
 <property>
   <name>fs.s3a.aws.credentials.provider</name>
-  <description>Class name of a credentials provider that implements com.amazonaws.auth.AWSCredentialsProvider. Omit if using access/secret keys or another authentication mechanism.</description>
+  <description>
+    Class name of a credentials provider that implements
+    com.amazonaws.auth.AWSCredentialsProvider.  Omit if using access/secret keys
+    or another authentication mechanism.  The specified class must provide an
+    accessible constructor accepting java.net.URI and
+    org.apache.hadoop.conf.Configuration, or an accessible default constructor.
+    Specifying org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider allows
+    anonymous access to a publicly accessible S3 bucket without any credentials.
+    Please note that allowing anonymous access to an S3 bucket compromises
+    security and therefore is unsuitable for most use cases.  It can be useful
+    for accessing public data sets without requiring AWS credentials.
+  </description>
 </property>
 
 <property>

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AnonymousAWSCredentialsProvider.java

@@ -24,6 +24,17 @@ import com.amazonaws.auth.AWSCredentials;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 
+/**
+ * AnonymousAWSCredentialsProvider supports anonymous access to AWS services
+ * through the AWS SDK.  AWS requests will not be signed.  This is not suitable
+ * for most cases, because allowing anonymous access to an S3 bucket compromises
+ * security.  This can be useful for accessing public data sets without
+ * requiring AWS credentials.
+ *
+ * Please note that users may reference this class name from configuration
+ * property fs.s3a.aws.credentials.provider.  Therefore, changing the class name
+ * would be a backward-incompatible change.
+ */
 @InterfaceAudience.Private
 @InterfaceStability.Stable
 public class AnonymousAWSCredentialsProvider implements AWSCredentialsProvider {

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/BasicAWSCredentialsProvider.java

@@ -26,6 +26,14 @@ import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 
+/**
+ * BasicAWSCredentialsProvider supports static configuration of access key ID
+ * and secret access key for use with the AWS SDK.
+ *
+ * Please note that users may reference this class name from configuration
+ * property fs.s3a.aws.credentials.provider.  Therefore, changing the class name
+ * would be a backward-incompatible change.
+ */
 @InterfaceAudience.Private
 @InterfaceStability.Stable
 public class BasicAWSCredentialsProvider implements AWSCredentialsProvider {

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java

@@ -465,20 +465,28 @@ public class S3AFileSystem extends FileSystem {
           new BasicAWSCredentialsProvider(
               creds.getAccessKey(), creds.getAccessSecret()),
           new InstanceProfileCredentialsProvider(),
-          new EnvironmentVariableCredentialsProvider(),
+          new EnvironmentVariableCredentialsProvider());
-          new AnonymousAWSCredentialsProvider()
-      );
 
     } else {
       try {
         LOG.debug("Credential provider class is {}", className);
-        credentials = (AWSCredentialsProvider) Class.forName(className)
+        Class<?> credClass = Class.forName(className);
-            .getDeclaredConstructor(URI.class, Configuration.class)
+        try {
-            .newInstance(this.uri, conf);
+          credentials =
+              (AWSCredentialsProvider)credClass.getDeclaredConstructor(
+                  URI.class, Configuration.class).newInstance(this.uri, conf);
+        } catch (NoSuchMethodException | SecurityException e) {
+          credentials =
+              (AWSCredentialsProvider)credClass.getDeclaredConstructor()
+                  .newInstance();
+        }
       } catch (ClassNotFoundException e) {
         throw new IOException(className + " not found.", e);
       } catch (NoSuchMethodException | SecurityException e) {
-        throw new IOException(className + " constructor exception.", e);
+        throw new IOException(String.format("%s constructor exception.  A "
+            + "class specified in %s must provide an accessible constructor "
+            + "accepting URI and Configuration, or an accessible default "
+            + "constructor.", className, AWS_CREDENTIALS_PROVIDER), e);
       } catch (ReflectiveOperationException | IllegalArgumentException e) {
         throw new IOException(className + " instantiation exception.", e);
       }

hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md

@@ -187,8 +187,18 @@ If you do any of these: change your credentials immediately!
 
     <property>
       <name>fs.s3a.aws.credentials.provider</name>
-      <description>Class name of a credentials provider that implements com.amazonaws.auth.AWSCredentialsProvider.
+      <description>
-      Omit if using access/secret keys or another authentication mechanism.</description>
+        Class name of a credentials provider that implements
+        com.amazonaws.auth.AWSCredentialsProvider.  Omit if using access/secret keys
+        or another authentication mechanism.  The specified class must provide an
+        accessible constructor accepting java.net.URI and
+        org.apache.hadoop.conf.Configuration, or an accessible default constructor.
+        Specifying org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider allows
+        anonymous access to a publicly accessible S3 bucket without any credentials.
+        Please note that allowing anonymous access to an S3 bucket compromises
+        security and therefore is unsuitable for most use cases.  It can be useful
+        for accessing public data sets without requiring AWS credentials.
+      </description>
     </property>
 
 #### Protecting the AWS Credentials in S3A

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java

@@ -19,6 +19,7 @@
 package org.apache.hadoop.fs.s3a;
 
 import static org.apache.hadoop.fs.s3a.Constants.*;
+import static org.apache.hadoop.fs.s3a.S3ATestConstants.*;
 import static org.junit.Assert.*;
 
 import java.io.IOException;
@@ -26,8 +27,13 @@ import java.net.URI;
 import java.nio.file.AccessDeniedException;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.rules.Timeout;
 
 import com.amazonaws.auth.AWSCredentials;
 import com.amazonaws.auth.AWSCredentialsProvider;
@@ -45,6 +51,12 @@ public class TestS3AAWSCredentialsProvider {
   private static final Logger LOG =
       LoggerFactory.getLogger(TestS3AAWSCredentialsProvider.class);
 
+  @Rule
+  public Timeout testTimeout = new Timeout(1 * 60 * 1000);
+
+  @Rule
+  public ExpectedException exception = ExpectedException.none();
+
   @Test
   public void testBadConfiguration() throws IOException {
     Configuration conf = new Configuration();
@@ -113,4 +125,47 @@ public class TestS3AAWSCredentialsProvider {
     conf.set(AWS_CREDENTIALS_PROVIDER, GoodCredentialsProvider.class.getName());
     S3ATestUtils.createTestFileSystem(conf);
   }
+
+  @Test
+  public void testAnonymousProvider() throws Exception {
+    Configuration conf = new Configuration();
+    conf.set(AWS_CREDENTIALS_PROVIDER,
+        AnonymousAWSCredentialsProvider.class.getName());
+    Path testFile = new Path(
+        conf.getTrimmed(KEY_CSVTEST_FILE, DEFAULT_CSVTEST_FILE));
+    FileSystem fs = FileSystem.newInstance(testFile.toUri(), conf);
+    assertNotNull(fs);
+    assertTrue(fs instanceof S3AFileSystem);
+    FileStatus stat = fs.getFileStatus(testFile);
+    assertNotNull(stat);
+    assertEquals(testFile, stat.getPath());
+  }
+
+  static class ConstructorErrorProvider implements AWSCredentialsProvider {
+
+    @SuppressWarnings("unused")
+    public ConstructorErrorProvider(String str) {
+    }
+
+    @Override
+    public AWSCredentials getCredentials() {
+      return null;
+    }
+
+    @Override
+    public void refresh() {
+    }
+  }
+
+  @Test
+  public void testProviderConstructorError() throws Exception {
+    Configuration conf = new Configuration();
+    conf.set(AWS_CREDENTIALS_PROVIDER,
+        ConstructorErrorProvider.class.getName());
+    Path testFile = new Path(
+        conf.getTrimmed(KEY_CSVTEST_FILE, DEFAULT_CSVTEST_FILE));
+    exception.expect(IOException.class);
+    exception.expectMessage("constructor exception");
+    FileSystem fs = FileSystem.newInstance(testFile.toUri(), conf);
+  }
 }