HADOOP-14248. Retire SharedInstanceProfileCredentialsProvider in trunk. Contributed by Mingliang Liu.

hadoop-common-project/hadoop-common/src/main/resources/core-default.xml

@@ -955,13 +955,8 @@
         configuration of AWS access key ID and secret access key in
         environment variables named AWS_ACCESS_KEY_ID and
         AWS_SECRET_ACCESS_KEY, as documented in the AWS SDK.
-    3. org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider:
+    3. com.amazonaws.auth.InstanceProfileCredentialsProvider: supports use
-        a shared instance of
+        of instance profile credentials if running in an EC2 VM.
-        com.amazonaws.auth.InstanceProfileCredentialsProvider from the AWS
-        SDK, which supports use of instance profile credentials if running
-        in an EC2 VM.  Using this shared instance potentially reduces load
-        on the EC2 instance metadata service for multi-threaded
-        applications.
   </description>
 </property>
 

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java

@@ -339,15 +339,9 @@ public final class S3AUtils {
       credentials.add(new BasicAWSCredentialsProvider(
               creds.getUser(), creds.getPassword()));
       credentials.add(new EnvironmentVariableCredentialsProvider());
-      credentials.add(
+      credentials.add(InstanceProfileCredentialsProvider.getInstance());
-          SharedInstanceProfileCredentialsProvider.getInstance());
     } else {
       for (Class<?> aClass : awsClasses) {
-        if (aClass == InstanceProfileCredentialsProvider.class) {
-          LOG.debug("Found {}, but will use {} instead.", aClass.getName(),
-              SharedInstanceProfileCredentialsProvider.class.getName());
-          aClass = SharedInstanceProfileCredentialsProvider.class;
-        }
         credentials.add(createAWSCredentialProvider(conf, aClass));
       }
     }

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceProfileCredentialsProvider.java

@@ -1,67 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import com.amazonaws.auth.InstanceProfileCredentialsProvider;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * A subclass of {@link InstanceProfileCredentialsProvider} that enforces
- * instantiation of only a single instance.
- * This credential provider calls the EC2 instance metadata service to obtain
- * credentials.  For highly multi-threaded applications, it's possible that
- * multiple instances call the service simultaneously and overwhelm it with
- * load.  The service handles this by throttling the client with an HTTP 429
- * response or forcibly terminating the connection.  Forcing use of a single
- * instance reduces load on the metadata service by allowing all threads to
- * share the credentials.  The base class is thread-safe, and there is nothing
- * that varies in the credentials across different instances of
- * {@link S3AFileSystem} connecting to different buckets, so sharing a singleton
- * instance is safe.
- *
- * As of AWS SDK 1.11.39, the SDK code internally enforces a singleton.  After
- * Hadoop upgrades to that version or higher, it's likely that we can remove
- * this class.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Stable
-public final class SharedInstanceProfileCredentialsProvider
-    extends InstanceProfileCredentialsProvider {
-
-  private static final SharedInstanceProfileCredentialsProvider INSTANCE =
-      new SharedInstanceProfileCredentialsProvider();
-
-  /**
-   * Returns the singleton instance.
-   *
-   * @return singleton instance
-   */
-  public static SharedInstanceProfileCredentialsProvider getInstance() {
-    return INSTANCE;
-  }
-
-  /**
-   * Default constructor, defined explicitly as private to enforce singleton.
-   */
-  private SharedInstanceProfileCredentialsProvider() {
-    super();
-  }
-}

hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md

@@ -328,13 +328,8 @@ of `com.amazonaws.auth.AWSCredentialsProvider` may also be used.
             configuration of AWS access key ID and secret access key in
             environment variables named AWS_ACCESS_KEY_ID and
             AWS_SECRET_ACCESS_KEY, as documented in the AWS SDK.
-        3. org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider:
+        3. com.amazonaws.auth.InstanceProfileCredentialsProvider: supports use
-            a shared instance of
+            of instance profile credentials if running in an EC2 VM.
-            com.amazonaws.auth.InstanceProfileCredentialsProvider from the AWS
-            SDK, which supports use of instance profile credentials if running
-            in an EC2 VM.  Using this shared instance potentially reduces load
-            on the EC2 instance metadata service for multi-threaded
-            applications.
       </description>
     </property>
 
@@ -407,13 +402,12 @@ AWS Credential Providers are classes which can be used by the Amazon AWS SDK to
 obtain an AWS login from a different source in the system, including environment
 variables, JVM properties and configuration files.
 
-There are four AWS Credential Providers inside the `hadoop-aws` JAR:
+There are three AWS Credential Providers inside the `hadoop-aws` JAR:
 
 | classname | description |
 |-----------|-------------|
 | `org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider`| Session Credentials |
 | `org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider`| Simple name/secret credentials |
-| `org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider`| Shared instance of EC2 Metadata Credentials, which can reduce load on the EC2 instance metadata service.  (See below.) |
 | `org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider`| Anonymous Login |
 
 There are also many in the Amazon SDKs, in particular two which are automatically
@@ -425,24 +419,13 @@ set up in the authentication chain:
 | `com.amazonaws.auth.EnvironmentVariableCredentialsProvider`| AWS Environment Variables |
 
 
-*EC2 Metadata Credentials with `SharedInstanceProfileCredentialsProvider`*
+*EC2 Metadata Credentials with `InstanceProfileCredentialsProvider`*
 
 Applications running in EC2 may associate an IAM role with the VM and query the
 [EC2 Instance Metadata Service](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html)
 for credentials to access S3.  Within the AWS SDK, this functionality is
-provided by `InstanceProfileCredentialsProvider`.  Heavily multi-threaded
+provided by `InstanceProfileCredentialsProvider`, which internally enforces a
-applications may trigger a high volume of calls to the instance metadata service
+singleton instance in order to prevent throttling problem.
-and trigger throttling: either an HTTP 429 response or a forcible close of the
-connection.
-
-To mitigate against this problem, `hadoop-aws` ships with a variant of
-`InstanceProfileCredentialsProvider` called
-`SharedInstanceProfileCredentialsProvider`.  Using this ensures that all
-instances of S3A reuse the same instance profile credentials instead of issuing
-a large volume of redundant metadata service calls.  If
-`fs.s3a.aws.credentials.provider` refers to
-`com.amazonaws.auth.InstanceProfileCredentialsProvider`, S3A automatically uses
-`org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider` instead.
 
 *Session Credentials with `TemporaryAWSCredentialsProvider`*
 
@@ -542,7 +525,7 @@ This means that the default S3A authentication chain can be defined as
       <value>
       org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider,
       com.amazonaws.auth.EnvironmentVariableCredentialsProvider,
-      org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider
+      com.amazonaws.auth.InstanceProfileCredentialsProvider
       </value>
     </property>
 
@@ -929,7 +912,7 @@ role information available when deployed in Amazon EC2.
 ```xml
 <property>
   <name>fs.s3a.aws.credentials.provider</name>
-  <value>org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider</value>
+  <value>com.amazonaws.auth.InstanceProfileCredentialsProvider</value>
 </property>
 ```
 

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java

@@ -114,7 +114,7 @@ public class TestS3AAWSCredentialsProvider {
         Arrays.asList(
             BasicAWSCredentialsProvider.class,
             EnvironmentVariableCredentialsProvider.class,
-            SharedInstanceProfileCredentialsProvider.class);
+            InstanceProfileCredentialsProvider.class);
     assertCredentialProviders(expectedClasses, list1);
     assertCredentialProviders(expectedClasses, list2);
     assertSameInstanceProfileCredentialsProvider(list1.getProviders().get(2),
@@ -128,7 +128,7 @@ public class TestS3AAWSCredentialsProvider {
     List<Class<? extends AWSCredentialsProvider>> expectedClasses =
         Arrays.asList(
             EnvironmentVariableCredentialsProvider.class,
-            SharedInstanceProfileCredentialsProvider.class,
+            InstanceProfileCredentialsProvider.class,
             AnonymousAWSCredentialsProvider.class);
     conf.set(AWS_CREDENTIALS_PROVIDER, buildClassListString(expectedClasses));
     AWSCredentialProviderList list1 = S3AUtils.createAWSCredentialProviderSet(