diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index d392d8efe34..d4776799b3f 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -34,6 +34,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import com.amazonaws.auth.EnvironmentVariableCredentialsProvider; import com.amazonaws.AmazonClientException; import com.amazonaws.AmazonServiceException; import com.amazonaws.ClientConfiguration; @@ -526,6 +527,7 @@ private AWSCredentialsProvider getAWSCredentialsProvider(URI binding, new BasicAWSCredentialsProvider( creds.getAccessKey(), creds.getAccessSecret()), new InstanceProfileCredentialsProvider(), + new EnvironmentVariableCredentialsProvider(), new AnonymousAWSCredentialsProvider() ); diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md index 8cd2155d1f8..8ca10a7a6b8 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md @@ -199,6 +199,25 @@ credentials in S3AFileSystem. For additional reading on the credential provider API see: [Credential Provider API](../../../hadoop-project-dist/hadoop-common/CredentialProviderAPI.html). +#### Authenticating via environment variables + +S3A supports configuration via [the standard AWS environment variables](http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html#cli-environment). + +The core environment variables are for the access key and associated secret: + +``` +export AWS_ACCESS_KEY_ID=my.aws.key +export AWS_SECRET_ACCESS_KEY=my.secret.key +``` + +These environment variables can be used to set the authentication credentials +instead of properties in the Hadoop configuration. *Important:* these +environment variables are not propagated from client to server when +YARN applications are launched. That is: having the AWS environment variables +set when an application is launched will not permit the launched application +to access S3 resources. The environment variables must (somehow) be set +on the hosts/processes where the work is executed. + ##### End to End Steps for Distcp and S3 with Credential Providers ###### provision