From 1bfb382296ab6f980341c926801b0eec57a68b87 Mon Sep 17 00:00:00 2001 From: Chris Nauroth Date: Wed, 12 Apr 2017 10:07:00 -0700 Subject: [PATCH] HADOOP-14301. Deprecate SharedInstanceProfileCredentialsProvider in branch-2. Contributed by Mingliang Liu. --- .../org/apache/hadoop/fs/s3a/S3AUtils.java | 13 +++---- ...redInstanceProfileCredentialsProvider.java | 9 +++-- .../site/markdown/tools/hadoop-aws/index.md | 35 +++++++++---------- .../fs/s3a/TestS3AAWSCredentialsProvider.java | 14 ++++++-- 4 files changed, 41 insertions(+), 30 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java index 6a11699ba5f..b652b3b0b38 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java @@ -339,14 +339,15 @@ public final class S3AUtils { credentials.add(new BasicAWSCredentialsProvider( creds.getUser(), creds.getPassword())); credentials.add(new EnvironmentVariableCredentialsProvider()); - credentials.add( - SharedInstanceProfileCredentialsProvider.getInstance()); + credentials.add(InstanceProfileCredentialsProvider.getInstance()); } else { for (Class aClass : awsClasses) { - if (aClass == InstanceProfileCredentialsProvider.class) { - LOG.debug("Found {}, but will use {} instead.", aClass.getName(), - SharedInstanceProfileCredentialsProvider.class.getName()); - aClass = SharedInstanceProfileCredentialsProvider.class; + if (aClass == SharedInstanceProfileCredentialsProvider.class) { + LOG.warn("{} is deprecated and will be removed in future. " + + "Fall back to {} automatically.", + aClass.getName(), + InstanceProfileCredentialsProvider.class.getName()); + aClass = InstanceProfileCredentialsProvider.class; } credentials.add(createAWSCredentialProvider(conf, aClass)); } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceProfileCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceProfileCredentialsProvider.java index cbc07873f08..d30444bd75e 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceProfileCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceProfileCredentialsProvider.java @@ -37,12 +37,15 @@ import org.apache.hadoop.classification.InterfaceStability; * {@link S3AFileSystem} connecting to different buckets, so sharing a singleton * instance is safe. * - * As of AWS SDK 1.11.39, the SDK code internally enforces a singleton. After - * Hadoop upgrades to that version or higher, it's likely that we can remove - * this class. + * As of AWS SDK 1.11.39, the SDK code internally enforces a singleton. Hadoop + * has upgraded its dependency to 1.11.39+ so this class is deprecated. In + * next major version, this will be removed. + * + * @deprecated Please use {@link InstanceProfileCredentialsProvider} instead. */ @InterfaceAudience.Private @InterfaceStability.Stable +@Deprecated public final class SharedInstanceProfileCredentialsProvider extends InstanceProfileCredentialsProvider { diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md index 07cc9030941..4174141fbda 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md @@ -357,13 +357,8 @@ of `com.amazonaws.auth.AWSCredentialsProvider` may also be used. configuration of AWS access key ID and secret access key in environment variables named AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY, as documented in the AWS SDK. - 3. org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider: - a shared instance of - com.amazonaws.auth.InstanceProfileCredentialsProvider from the AWS - SDK, which supports use of instance profile credentials if running - in an EC2 VM. Using this shared instance potentially reduces load - on the EC2 instance metadata service for multi-threaded - applications. + 3. com.amazonaws.auth.InstanceProfileCredentialsProvider: supports use + of instance profile credentials if running in an EC2 VM. @@ -459,19 +454,23 @@ set up in the authentication chain: Applications running in EC2 may associate an IAM role with the VM and query the [EC2 Instance Metadata Service](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) for credentials to access S3. Within the AWS SDK, this functionality is -provided by `InstanceProfileCredentialsProvider`. Heavily multi-threaded -applications may trigger a high volume of calls to the instance metadata service -and trigger throttling: either an HTTP 429 response or a forcible close of the -connection. +provided by `InstanceProfileCredentialsProvider`. In previous AWS SDK versions +earlier than 1.11.39, heavily multi-threaded applications may trigger a high +volume of calls to the instance metadata service and trigger throttling: either +an HTTP 429 response or a forcible close of the connection. -To mitigate against this problem, `hadoop-aws` ships with a variant of +To mitigate against that problem, `hadoop-aws` ships with a variant of `InstanceProfileCredentialsProvider` called `SharedInstanceProfileCredentialsProvider`. Using this ensures that all instances of S3A reuse the same instance profile credentials instead of issuing -a large volume of redundant metadata service calls. If -`fs.s3a.aws.credentials.provider` refers to -`com.amazonaws.auth.InstanceProfileCredentialsProvider`, S3A automatically uses -`org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider` instead. +a large volume of redundant metadata service calls. + +As of AWS SDK 1.11.39, the SDK code internally enforces a singleton. Hadoop has +upgraded its dependency so that this class is deprecated. In next major +version, this will be removed. If `fs.s3a.aws.credentials.provider` refers to +`org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider`, S3A +automatically uses `com.amazonaws.auth.InstanceProfileCredentialsProvider` +instead, along with a warning message. *Session Credentials with `TemporaryAWSCredentialsProvider`* @@ -571,7 +570,7 @@ This means that the default S3A authentication chain can be defined as org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider, com.amazonaws.auth.EnvironmentVariableCredentialsProvider, - org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider + com.amazonaws.auth.InstanceProfileCredentialsProvider @@ -958,7 +957,7 @@ role information available when deployed in Amazon EC2. ```xml fs.s3a.aws.credentials.provider - org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider + com.amazonaws.auth.InstanceProfileCredentialsProvider ``` diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java index 33740c8ab93..cfa6dd4759a 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java @@ -114,7 +114,7 @@ public class TestS3AAWSCredentialsProvider { Arrays.asList( BasicAWSCredentialsProvider.class, EnvironmentVariableCredentialsProvider.class, - SharedInstanceProfileCredentialsProvider.class); + InstanceProfileCredentialsProvider.class); assertCredentialProviders(expectedClasses, list1); assertCredentialProviders(expectedClasses, list2); assertSameInstanceProfileCredentialsProvider(list1.getProviders().get(2), @@ -125,12 +125,20 @@ public class TestS3AAWSCredentialsProvider { public void testConfiguredChain() throws Exception { URI uri1 = new URI("s3a://bucket1"), uri2 = new URI("s3a://bucket2"); Configuration conf = new Configuration(); - List> expectedClasses = + List> originalClasses = Arrays.asList( EnvironmentVariableCredentialsProvider.class, SharedInstanceProfileCredentialsProvider.class, AnonymousAWSCredentialsProvider.class); - conf.set(AWS_CREDENTIALS_PROVIDER, buildClassListString(expectedClasses)); + conf.set(AWS_CREDENTIALS_PROVIDER, buildClassListString(originalClasses)); + + // SharedInstanceProfileCredentialsProvider is deprecated and should have + // been replaced with InstanceProfileCredentialsProvider automatically + List> expectedClasses = + Arrays.asList( + EnvironmentVariableCredentialsProvider.class, + InstanceProfileCredentialsProvider.class, + AnonymousAWSCredentialsProvider.class); AWSCredentialProviderList list1 = S3AUtils.createAWSCredentialProviderSet( uri1, conf); AWSCredentialProviderList list2 = S3AUtils.createAWSCredentialProviderSet(