HADOOP-14248. Retire SharedInstanceProfileCredentialsProvider in trunk. Contributed by Mingliang Liu.
This commit is contained in:
parent
a16ab2be91
commit
b8305e6d06
|
@ -955,13 +955,8 @@
|
|||
configuration of AWS access key ID and secret access key in
|
||||
environment variables named AWS_ACCESS_KEY_ID and
|
||||
AWS_SECRET_ACCESS_KEY, as documented in the AWS SDK.
|
||||
3. org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider:
|
||||
a shared instance of
|
||||
com.amazonaws.auth.InstanceProfileCredentialsProvider from the AWS
|
||||
SDK, which supports use of instance profile credentials if running
|
||||
in an EC2 VM. Using this shared instance potentially reduces load
|
||||
on the EC2 instance metadata service for multi-threaded
|
||||
applications.
|
||||
3. com.amazonaws.auth.InstanceProfileCredentialsProvider: supports use
|
||||
of instance profile credentials if running in an EC2 VM.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
|
|
|
@ -339,15 +339,9 @@ public final class S3AUtils {
|
|||
credentials.add(new BasicAWSCredentialsProvider(
|
||||
creds.getUser(), creds.getPassword()));
|
||||
credentials.add(new EnvironmentVariableCredentialsProvider());
|
||||
credentials.add(
|
||||
SharedInstanceProfileCredentialsProvider.getInstance());
|
||||
credentials.add(InstanceProfileCredentialsProvider.getInstance());
|
||||
} else {
|
||||
for (Class<?> aClass : awsClasses) {
|
||||
if (aClass == InstanceProfileCredentialsProvider.class) {
|
||||
LOG.debug("Found {}, but will use {} instead.", aClass.getName(),
|
||||
SharedInstanceProfileCredentialsProvider.class.getName());
|
||||
aClass = SharedInstanceProfileCredentialsProvider.class;
|
||||
}
|
||||
credentials.add(createAWSCredentialProvider(conf, aClass));
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,67 +0,0 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.fs.s3a;
|
||||
|
||||
import com.amazonaws.auth.InstanceProfileCredentialsProvider;
|
||||
|
||||
import org.apache.hadoop.classification.InterfaceAudience;
|
||||
import org.apache.hadoop.classification.InterfaceStability;
|
||||
|
||||
/**
|
||||
* A subclass of {@link InstanceProfileCredentialsProvider} that enforces
|
||||
* instantiation of only a single instance.
|
||||
* This credential provider calls the EC2 instance metadata service to obtain
|
||||
* credentials. For highly multi-threaded applications, it's possible that
|
||||
* multiple instances call the service simultaneously and overwhelm it with
|
||||
* load. The service handles this by throttling the client with an HTTP 429
|
||||
* response or forcibly terminating the connection. Forcing use of a single
|
||||
* instance reduces load on the metadata service by allowing all threads to
|
||||
* share the credentials. The base class is thread-safe, and there is nothing
|
||||
* that varies in the credentials across different instances of
|
||||
* {@link S3AFileSystem} connecting to different buckets, so sharing a singleton
|
||||
* instance is safe.
|
||||
*
|
||||
* As of AWS SDK 1.11.39, the SDK code internally enforces a singleton. After
|
||||
* Hadoop upgrades to that version or higher, it's likely that we can remove
|
||||
* this class.
|
||||
*/
|
||||
@InterfaceAudience.Private
|
||||
@InterfaceStability.Stable
|
||||
public final class SharedInstanceProfileCredentialsProvider
|
||||
extends InstanceProfileCredentialsProvider {
|
||||
|
||||
private static final SharedInstanceProfileCredentialsProvider INSTANCE =
|
||||
new SharedInstanceProfileCredentialsProvider();
|
||||
|
||||
/**
|
||||
* Returns the singleton instance.
|
||||
*
|
||||
* @return singleton instance
|
||||
*/
|
||||
public static SharedInstanceProfileCredentialsProvider getInstance() {
|
||||
return INSTANCE;
|
||||
}
|
||||
|
||||
/**
|
||||
* Default constructor, defined explicitly as private to enforce singleton.
|
||||
*/
|
||||
private SharedInstanceProfileCredentialsProvider() {
|
||||
super();
|
||||
}
|
||||
}
|
|
@ -328,13 +328,8 @@ of `com.amazonaws.auth.AWSCredentialsProvider` may also be used.
|
|||
configuration of AWS access key ID and secret access key in
|
||||
environment variables named AWS_ACCESS_KEY_ID and
|
||||
AWS_SECRET_ACCESS_KEY, as documented in the AWS SDK.
|
||||
3. org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider:
|
||||
a shared instance of
|
||||
com.amazonaws.auth.InstanceProfileCredentialsProvider from the AWS
|
||||
SDK, which supports use of instance profile credentials if running
|
||||
in an EC2 VM. Using this shared instance potentially reduces load
|
||||
on the EC2 instance metadata service for multi-threaded
|
||||
applications.
|
||||
3. com.amazonaws.auth.InstanceProfileCredentialsProvider: supports use
|
||||
of instance profile credentials if running in an EC2 VM.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
|
@ -407,13 +402,12 @@ AWS Credential Providers are classes which can be used by the Amazon AWS SDK to
|
|||
obtain an AWS login from a different source in the system, including environment
|
||||
variables, JVM properties and configuration files.
|
||||
|
||||
There are four AWS Credential Providers inside the `hadoop-aws` JAR:
|
||||
There are three AWS Credential Providers inside the `hadoop-aws` JAR:
|
||||
|
||||
| classname | description |
|
||||
|-----------|-------------|
|
||||
| `org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider`| Session Credentials |
|
||||
| `org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider`| Simple name/secret credentials |
|
||||
| `org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider`| Shared instance of EC2 Metadata Credentials, which can reduce load on the EC2 instance metadata service. (See below.) |
|
||||
| `org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider`| Anonymous Login |
|
||||
|
||||
There are also many in the Amazon SDKs, in particular two which are automatically
|
||||
|
@ -425,24 +419,13 @@ set up in the authentication chain:
|
|||
| `com.amazonaws.auth.EnvironmentVariableCredentialsProvider`| AWS Environment Variables |
|
||||
|
||||
|
||||
*EC2 Metadata Credentials with `SharedInstanceProfileCredentialsProvider`*
|
||||
*EC2 Metadata Credentials with `InstanceProfileCredentialsProvider`*
|
||||
|
||||
Applications running in EC2 may associate an IAM role with the VM and query the
|
||||
[EC2 Instance Metadata Service](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html)
|
||||
for credentials to access S3. Within the AWS SDK, this functionality is
|
||||
provided by `InstanceProfileCredentialsProvider`. Heavily multi-threaded
|
||||
applications may trigger a high volume of calls to the instance metadata service
|
||||
and trigger throttling: either an HTTP 429 response or a forcible close of the
|
||||
connection.
|
||||
|
||||
To mitigate against this problem, `hadoop-aws` ships with a variant of
|
||||
`InstanceProfileCredentialsProvider` called
|
||||
`SharedInstanceProfileCredentialsProvider`. Using this ensures that all
|
||||
instances of S3A reuse the same instance profile credentials instead of issuing
|
||||
a large volume of redundant metadata service calls. If
|
||||
`fs.s3a.aws.credentials.provider` refers to
|
||||
`com.amazonaws.auth.InstanceProfileCredentialsProvider`, S3A automatically uses
|
||||
`org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider` instead.
|
||||
provided by `InstanceProfileCredentialsProvider`, which internally enforces a
|
||||
singleton instance in order to prevent throttling problem.
|
||||
|
||||
*Session Credentials with `TemporaryAWSCredentialsProvider`*
|
||||
|
||||
|
@ -542,7 +525,7 @@ This means that the default S3A authentication chain can be defined as
|
|||
<value>
|
||||
org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider,
|
||||
com.amazonaws.auth.EnvironmentVariableCredentialsProvider,
|
||||
org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider
|
||||
com.amazonaws.auth.InstanceProfileCredentialsProvider
|
||||
</value>
|
||||
</property>
|
||||
|
||||
|
@ -929,7 +912,7 @@ role information available when deployed in Amazon EC2.
|
|||
```xml
|
||||
<property>
|
||||
<name>fs.s3a.aws.credentials.provider</name>
|
||||
<value>org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider</value>
|
||||
<value>com.amazonaws.auth.InstanceProfileCredentialsProvider</value>
|
||||
</property>
|
||||
```
|
||||
|
||||
|
|
|
@ -114,7 +114,7 @@ public class TestS3AAWSCredentialsProvider {
|
|||
Arrays.asList(
|
||||
BasicAWSCredentialsProvider.class,
|
||||
EnvironmentVariableCredentialsProvider.class,
|
||||
SharedInstanceProfileCredentialsProvider.class);
|
||||
InstanceProfileCredentialsProvider.class);
|
||||
assertCredentialProviders(expectedClasses, list1);
|
||||
assertCredentialProviders(expectedClasses, list2);
|
||||
assertSameInstanceProfileCredentialsProvider(list1.getProviders().get(2),
|
||||
|
@ -128,7 +128,7 @@ public class TestS3AAWSCredentialsProvider {
|
|||
List<Class<? extends AWSCredentialsProvider>> expectedClasses =
|
||||
Arrays.asList(
|
||||
EnvironmentVariableCredentialsProvider.class,
|
||||
SharedInstanceProfileCredentialsProvider.class,
|
||||
InstanceProfileCredentialsProvider.class,
|
||||
AnonymousAWSCredentialsProvider.class);
|
||||
conf.set(AWS_CREDENTIALS_PROVIDER, buildClassListString(expectedClasses));
|
||||
AWSCredentialProviderList list1 = S3AUtils.createAWSCredentialProviderSet(
|
||||
|
|
Loading…
Reference in New Issue