diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
index 6a11699ba5f..b652b3b0b38 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
@@ -339,14 +339,15 @@ public static AWSCredentialProviderList createAWSCredentialProviderSet(
credentials.add(new BasicAWSCredentialsProvider(
creds.getUser(), creds.getPassword()));
credentials.add(new EnvironmentVariableCredentialsProvider());
- credentials.add(
- SharedInstanceProfileCredentialsProvider.getInstance());
+ credentials.add(InstanceProfileCredentialsProvider.getInstance());
} else {
for (Class> aClass : awsClasses) {
- if (aClass == InstanceProfileCredentialsProvider.class) {
- LOG.debug("Found {}, but will use {} instead.", aClass.getName(),
- SharedInstanceProfileCredentialsProvider.class.getName());
- aClass = SharedInstanceProfileCredentialsProvider.class;
+ if (aClass == SharedInstanceProfileCredentialsProvider.class) {
+ LOG.warn("{} is deprecated and will be removed in future. " +
+ "Fall back to {} automatically.",
+ aClass.getName(),
+ InstanceProfileCredentialsProvider.class.getName());
+ aClass = InstanceProfileCredentialsProvider.class;
}
credentials.add(createAWSCredentialProvider(conf, aClass));
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceProfileCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceProfileCredentialsProvider.java
index cbc07873f08..d30444bd75e 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceProfileCredentialsProvider.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceProfileCredentialsProvider.java
@@ -37,12 +37,15 @@
* {@link S3AFileSystem} connecting to different buckets, so sharing a singleton
* instance is safe.
*
- * As of AWS SDK 1.11.39, the SDK code internally enforces a singleton. After
- * Hadoop upgrades to that version or higher, it's likely that we can remove
- * this class.
+ * As of AWS SDK 1.11.39, the SDK code internally enforces a singleton. Hadoop
+ * has upgraded its dependency to 1.11.39+ so this class is deprecated. In
+ * next major version, this will be removed.
+ *
+ * @deprecated Please use {@link InstanceProfileCredentialsProvider} instead.
*/
@InterfaceAudience.Private
@InterfaceStability.Stable
+@Deprecated
public final class SharedInstanceProfileCredentialsProvider
extends InstanceProfileCredentialsProvider {
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
index 07cc9030941..4174141fbda 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
@@ -357,13 +357,8 @@ of `com.amazonaws.auth.AWSCredentialsProvider` may also be used.
configuration of AWS access key ID and secret access key in
environment variables named AWS_ACCESS_KEY_ID and
AWS_SECRET_ACCESS_KEY, as documented in the AWS SDK.
- 3. org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider:
- a shared instance of
- com.amazonaws.auth.InstanceProfileCredentialsProvider from the AWS
- SDK, which supports use of instance profile credentials if running
- in an EC2 VM. Using this shared instance potentially reduces load
- on the EC2 instance metadata service for multi-threaded
- applications.
+ 3. com.amazonaws.auth.InstanceProfileCredentialsProvider: supports use
+ of instance profile credentials if running in an EC2 VM.
@@ -459,19 +454,23 @@ set up in the authentication chain:
Applications running in EC2 may associate an IAM role with the VM and query the
[EC2 Instance Metadata Service](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html)
for credentials to access S3. Within the AWS SDK, this functionality is
-provided by `InstanceProfileCredentialsProvider`. Heavily multi-threaded
-applications may trigger a high volume of calls to the instance metadata service
-and trigger throttling: either an HTTP 429 response or a forcible close of the
-connection.
+provided by `InstanceProfileCredentialsProvider`. In previous AWS SDK versions
+earlier than 1.11.39, heavily multi-threaded applications may trigger a high
+volume of calls to the instance metadata service and trigger throttling: either
+an HTTP 429 response or a forcible close of the connection.
-To mitigate against this problem, `hadoop-aws` ships with a variant of
+To mitigate against that problem, `hadoop-aws` ships with a variant of
`InstanceProfileCredentialsProvider` called
`SharedInstanceProfileCredentialsProvider`. Using this ensures that all
instances of S3A reuse the same instance profile credentials instead of issuing
-a large volume of redundant metadata service calls. If
-`fs.s3a.aws.credentials.provider` refers to
-`com.amazonaws.auth.InstanceProfileCredentialsProvider`, S3A automatically uses
-`org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider` instead.
+a large volume of redundant metadata service calls.
+
+As of AWS SDK 1.11.39, the SDK code internally enforces a singleton. Hadoop has
+upgraded its dependency so that this class is deprecated. In next major
+version, this will be removed. If `fs.s3a.aws.credentials.provider` refers to
+`org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider`, S3A
+automatically uses `com.amazonaws.auth.InstanceProfileCredentialsProvider`
+instead, along with a warning message.
*Session Credentials with `TemporaryAWSCredentialsProvider`*
@@ -571,7 +570,7 @@ This means that the default S3A authentication chain can be defined as
org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider,
com.amazonaws.auth.EnvironmentVariableCredentialsProvider,
- org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider
+ com.amazonaws.auth.InstanceProfileCredentialsProvider
@@ -958,7 +957,7 @@ role information available when deployed in Amazon EC2.
```xml
fs.s3a.aws.credentials.provider
- org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider
+ com.amazonaws.auth.InstanceProfileCredentialsProvider
```
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java
index 33740c8ab93..cfa6dd4759a 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java
@@ -114,7 +114,7 @@ public void testDefaultChain() throws Exception {
Arrays.asList(
BasicAWSCredentialsProvider.class,
EnvironmentVariableCredentialsProvider.class,
- SharedInstanceProfileCredentialsProvider.class);
+ InstanceProfileCredentialsProvider.class);
assertCredentialProviders(expectedClasses, list1);
assertCredentialProviders(expectedClasses, list2);
assertSameInstanceProfileCredentialsProvider(list1.getProviders().get(2),
@@ -125,12 +125,20 @@ public void testDefaultChain() throws Exception {
public void testConfiguredChain() throws Exception {
URI uri1 = new URI("s3a://bucket1"), uri2 = new URI("s3a://bucket2");
Configuration conf = new Configuration();
- List> expectedClasses =
+ List> originalClasses =
Arrays.asList(
EnvironmentVariableCredentialsProvider.class,
SharedInstanceProfileCredentialsProvider.class,
AnonymousAWSCredentialsProvider.class);
- conf.set(AWS_CREDENTIALS_PROVIDER, buildClassListString(expectedClasses));
+ conf.set(AWS_CREDENTIALS_PROVIDER, buildClassListString(originalClasses));
+
+ // SharedInstanceProfileCredentialsProvider is deprecated and should have
+ // been replaced with InstanceProfileCredentialsProvider automatically
+ List> expectedClasses =
+ Arrays.asList(
+ EnvironmentVariableCredentialsProvider.class,
+ InstanceProfileCredentialsProvider.class,
+ AnonymousAWSCredentialsProvider.class);
AWSCredentialProviderList list1 = S3AUtils.createAWSCredentialProviderSet(
uri1, conf);
AWSCredentialProviderList list2 = S3AUtils.createAWSCredentialProviderSet(