HADOOP-14301. Deprecate SharedInstanceProfileCredentialsProvider in branch-2. Contributed by Mingliang Liu.

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java

@@ -339,14 +339,15 @@ public final class S3AUtils {
       credentials.add(new BasicAWSCredentialsProvider(
               creds.getUser(), creds.getPassword()));
       credentials.add(new EnvironmentVariableCredentialsProvider());
-      credentials.add(
+      credentials.add(InstanceProfileCredentialsProvider.getInstance());
-          SharedInstanceProfileCredentialsProvider.getInstance());
     } else {
       for (Class<?> aClass : awsClasses) {
-        if (aClass == InstanceProfileCredentialsProvider.class) {
+        if (aClass == SharedInstanceProfileCredentialsProvider.class) {
-          LOG.debug("Found {}, but will use {} instead.", aClass.getName(),
+          LOG.warn("{} is deprecated and will be removed in future. " +
-              SharedInstanceProfileCredentialsProvider.class.getName());
+                  "Fall back to {} automatically.",
-          aClass = SharedInstanceProfileCredentialsProvider.class;
+              aClass.getName(),
+              InstanceProfileCredentialsProvider.class.getName());
+          aClass = InstanceProfileCredentialsProvider.class;
         }
         credentials.add(createAWSCredentialProvider(conf, aClass));
       }

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceProfileCredentialsProvider.java

@@ -37,12 +37,15 @@ import org.apache.hadoop.classification.InterfaceStability;
  * {@link S3AFileSystem} connecting to different buckets, so sharing a singleton
  * instance is safe.
  *
- * As of AWS SDK 1.11.39, the SDK code internally enforces a singleton.  After
+ * As of AWS SDK 1.11.39, the SDK code internally enforces a singleton.  Hadoop
- * Hadoop upgrades to that version or higher, it's likely that we can remove
+ * has upgraded its dependency to 1.11.39+ so this class is deprecated. In
- * this class.
+ * next major version, this will be removed.
+ *
+ * @deprecated Please use {@link InstanceProfileCredentialsProvider} instead.
  */
 @InterfaceAudience.Private
 @InterfaceStability.Stable
+@Deprecated
 public final class SharedInstanceProfileCredentialsProvider
     extends InstanceProfileCredentialsProvider {
 

hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md

@@ -357,13 +357,8 @@ of `com.amazonaws.auth.AWSCredentialsProvider` may also be used.
             configuration of AWS access key ID and secret access key in
             environment variables named AWS_ACCESS_KEY_ID and
             AWS_SECRET_ACCESS_KEY, as documented in the AWS SDK.
-        3. org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider:
+        3. com.amazonaws.auth.InstanceProfileCredentialsProvider: supports use
-            a shared instance of
+            of instance profile credentials if running in an EC2 VM.
-            com.amazonaws.auth.InstanceProfileCredentialsProvider from the AWS
-            SDK, which supports use of instance profile credentials if running
-            in an EC2 VM.  Using this shared instance potentially reduces load
-            on the EC2 instance metadata service for multi-threaded
-            applications.
       </description>
     </property>
 
@@ -459,19 +454,23 @@ set up in the authentication chain:
 Applications running in EC2 may associate an IAM role with the VM and query the
 [EC2 Instance Metadata Service](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html)
 for credentials to access S3.  Within the AWS SDK, this functionality is
-provided by `InstanceProfileCredentialsProvider`.  Heavily multi-threaded
+provided by `InstanceProfileCredentialsProvider`.  In previous AWS SDK versions
-applications may trigger a high volume of calls to the instance metadata service
+earlier than 1.11.39, heavily multi-threaded applications may trigger a high
-and trigger throttling: either an HTTP 429 response or a forcible close of the
+volume of calls to the instance metadata service and trigger throttling: either
-connection.
+an HTTP 429 response or a forcible close of the connection.
 
-To mitigate against this problem, `hadoop-aws` ships with a variant of
+To mitigate against that problem, `hadoop-aws` ships with a variant of
 `InstanceProfileCredentialsProvider` called
 `SharedInstanceProfileCredentialsProvider`.  Using this ensures that all
 instances of S3A reuse the same instance profile credentials instead of issuing
-a large volume of redundant metadata service calls.  If
+a large volume of redundant metadata service calls.
-`fs.s3a.aws.credentials.provider` refers to
+
-`com.amazonaws.auth.InstanceProfileCredentialsProvider`, S3A automatically uses
+As of AWS SDK 1.11.39, the SDK code internally enforces a singleton.  Hadoop has
-`org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider` instead.
+upgraded its dependency so that this class is deprecated. In next major
+version, this will be removed. If `fs.s3a.aws.credentials.provider` refers to
+`org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider`, S3A
+automatically uses `com.amazonaws.auth.InstanceProfileCredentialsProvider`
+instead, along with a warning message.
 
 *Session Credentials with `TemporaryAWSCredentialsProvider`*
 
@@ -571,7 +570,7 @@ This means that the default S3A authentication chain can be defined as
       <value>
       org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider,
       com.amazonaws.auth.EnvironmentVariableCredentialsProvider,
-      org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider
+      com.amazonaws.auth.InstanceProfileCredentialsProvider
       </value>
     </property>
 
@@ -958,7 +957,7 @@ role information available when deployed in Amazon EC2.
 ```xml
 <property>
   <name>fs.s3a.aws.credentials.provider</name>
-  <value>org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider</value>
+  <value>com.amazonaws.auth.InstanceProfileCredentialsProvider</value>
 </property>
 ```
 

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java

@@ -114,7 +114,7 @@ public class TestS3AAWSCredentialsProvider {
         Arrays.asList(
             BasicAWSCredentialsProvider.class,
             EnvironmentVariableCredentialsProvider.class,
-            SharedInstanceProfileCredentialsProvider.class);
+            InstanceProfileCredentialsProvider.class);
     assertCredentialProviders(expectedClasses, list1);
     assertCredentialProviders(expectedClasses, list2);
     assertSameInstanceProfileCredentialsProvider(list1.getProviders().get(2),
@@ -125,12 +125,20 @@ public class TestS3AAWSCredentialsProvider {
   public void testConfiguredChain() throws Exception {
     URI uri1 = new URI("s3a://bucket1"), uri2 = new URI("s3a://bucket2");
     Configuration conf = new Configuration();
-    List<Class<? extends AWSCredentialsProvider>> expectedClasses =
+    List<Class<? extends AWSCredentialsProvider>> originalClasses =
         Arrays.asList(
             EnvironmentVariableCredentialsProvider.class,
             SharedInstanceProfileCredentialsProvider.class,
             AnonymousAWSCredentialsProvider.class);
-    conf.set(AWS_CREDENTIALS_PROVIDER, buildClassListString(expectedClasses));
+    conf.set(AWS_CREDENTIALS_PROVIDER, buildClassListString(originalClasses));
+
+    // SharedInstanceProfileCredentialsProvider is deprecated and should have
+    // been replaced with InstanceProfileCredentialsProvider automatically
+    List<Class<? extends AWSCredentialsProvider>> expectedClasses =
+        Arrays.asList(
+            EnvironmentVariableCredentialsProvider.class,
+            InstanceProfileCredentialsProvider.class,
+            AnonymousAWSCredentialsProvider.class);
     AWSCredentialProviderList list1 = S3AUtils.createAWSCredentialProviderSet(
         uri1, conf);
     AWSCredentialProviderList list2 = S3AUtils.createAWSCredentialProviderSet(