From 858aff396c0c593a37d17a2e8f09fad4a1a5b42e Mon Sep 17 00:00:00 2001 From: Glenn Nethercutt Date: Fri, 3 Oct 2014 23:06:42 -0400 Subject: [PATCH 1/3] move to latest AWS sdk (1.8.11) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1ad6ccb9338..e915d5a1006 100644 --- a/pom.xml +++ b/pom.xml @@ -130,7 +130,7 @@ com.amazonaws aws-java-sdk - 1.6.0.1 + 1.8.11 javax.mail From 39a7af28d6ffbbfbd8cb2dca85970c93da50c66f Mon Sep 17 00:00:00 2001 From: Glenn Nethercutt Date: Fri, 3 Oct 2014 23:09:38 -0400 Subject: [PATCH 2/3] Custom AWSCredentialsProviderChain for the S3 storage module: supports existing druid config, file, environment variable, system property, profile and (most importantly) instance profile based credential options --- .../storage/s3/S3StorageDruidModule.java | 79 ++++++++++++++++--- 1 file changed, 66 insertions(+), 13 deletions(-) diff --git a/s3-extensions/src/main/java/io/druid/storage/s3/S3StorageDruidModule.java b/s3-extensions/src/main/java/io/druid/storage/s3/S3StorageDruidModule.java index 3d2434365ae..5ee68e8fb44 100644 --- a/s3-extensions/src/main/java/io/druid/storage/s3/S3StorageDruidModule.java +++ b/s3-extensions/src/main/java/io/druid/storage/s3/S3StorageDruidModule.java @@ -19,7 +19,13 @@ package io.druid.storage.s3; +import com.amazonaws.AmazonClientException; import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.auth.AWSCredentialsProviderChain; +import com.amazonaws.auth.EnvironmentVariableCredentialsProvider; +import com.amazonaws.auth.InstanceProfileCredentialsProvider; +import com.amazonaws.auth.SystemPropertiesCredentialsProvider; +import com.amazonaws.auth.profile.ProfileCredentialsProvider; import com.fasterxml.jackson.databind.Module; import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; @@ -62,14 +68,18 @@ public class S3StorageDruidModule implements DruidModule binder.bind(S3TaskLogs.class).in(LazySingleton.class); } - @Provides - @LazySingleton - public AWSCredentialsProvider getAWSCredentialsProvider(final AWSCredentialsConfig config) + private class ConfigDrivenAwsCredentialsConfigProvider implements AWSCredentialsProvider { - if (!Strings.isNullOrEmpty(config.getAccessKey()) && !Strings.isNullOrEmpty(config.getSecretKey())) { - return new AWSCredentialsProvider() { - @Override - public com.amazonaws.auth.AWSCredentials getCredentials() { + private AWSCredentialsConfig config; + + public ConfigDrivenAwsCredentialsConfigProvider(AWSCredentialsConfig config) { + this.config = config; + } + + @Override + public com.amazonaws.auth.AWSCredentials getCredentials() + { + if (!Strings.isNullOrEmpty(config.getAccessKey()) && !Strings.isNullOrEmpty(config.getSecretKey())) { return new com.amazonaws.auth.AWSCredentials() { @Override public String getAWSAccessKeyId() { @@ -82,13 +92,56 @@ public class S3StorageDruidModule implements DruidModule } }; } - - @Override - public void refresh() {} - }; - } else { - return new FileSessionCredentialsProvider(config.getFileSessionCredentials()); + throw new AmazonClientException("Unable to load AWS credentials from druid AWSCredentialsConfig"); } + + @Override + public void refresh() {} + } + + private class LazyFileSessionCredentialsProvider implements AWSCredentialsProvider + { + private AWSCredentialsConfig config; + private FileSessionCredentialsProvider provider; + + public LazyFileSessionCredentialsProvider(AWSCredentialsConfig config) { + this.config = config; + } + + private FileSessionCredentialsProvider getUnderlyingProvider() { + if (provider == null) { + synchronized (config) { + if (provider == null) { + provider = new FileSessionCredentialsProvider(config.getFileSessionCredentials()); + } + } + } + return provider; + } + + @Override + public com.amazonaws.auth.AWSCredentials getCredentials() + { + return getUnderlyingProvider().getCredentials(); + } + + @Override + public void refresh() { + getUnderlyingProvider().refresh(); + } + } + + @Provides + @LazySingleton + public AWSCredentialsProvider getAWSCredentialsProvider(final AWSCredentialsConfig config) + { + return new AWSCredentialsProviderChain( + new ConfigDrivenAwsCredentialsConfigProvider(config), + new LazyFileSessionCredentialsProvider(config), + new EnvironmentVariableCredentialsProvider(), + new SystemPropertiesCredentialsProvider(), + new ProfileCredentialsProvider(), + new InstanceProfileCredentialsProvider()); } @Provides From e30ac109962a82dbfe5757287052f38a769081fb Mon Sep 17 00:00:00 2001 From: Glenn Nethercutt Date: Tue, 11 Nov 2014 08:22:00 -0500 Subject: [PATCH 3/3] make the inner provider classes static --- .../main/java/io/druid/storage/s3/S3StorageDruidModule.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/s3-extensions/src/main/java/io/druid/storage/s3/S3StorageDruidModule.java b/s3-extensions/src/main/java/io/druid/storage/s3/S3StorageDruidModule.java index 5ee68e8fb44..792136ae8b9 100644 --- a/s3-extensions/src/main/java/io/druid/storage/s3/S3StorageDruidModule.java +++ b/s3-extensions/src/main/java/io/druid/storage/s3/S3StorageDruidModule.java @@ -68,7 +68,7 @@ public class S3StorageDruidModule implements DruidModule binder.bind(S3TaskLogs.class).in(LazySingleton.class); } - private class ConfigDrivenAwsCredentialsConfigProvider implements AWSCredentialsProvider + private static class ConfigDrivenAwsCredentialsConfigProvider implements AWSCredentialsProvider { private AWSCredentialsConfig config; @@ -99,7 +99,7 @@ public class S3StorageDruidModule implements DruidModule public void refresh() {} } - private class LazyFileSessionCredentialsProvider implements AWSCredentialsProvider + private static class LazyFileSessionCredentialsProvider implements AWSCredentialsProvider { private AWSCredentialsConfig config; private FileSessionCredentialsProvider provider;