diff --git a/aws-common/pom.xml b/aws-common/pom.xml new file mode 100644 index 00000000000..02822ee29fc --- /dev/null +++ b/aws-common/pom.xml @@ -0,0 +1,80 @@ + + + + + 4.0.0 + + druid-aws-common + druid-aws-common + druid-aws-common + + + io.druid + druid + 0.7.0-SNAPSHOT + + + + + io.druid + druid-common + ${project.parent.version} + + + com.amazonaws + aws-java-sdk + + + + + junit + junit + test + + + + + + + org.apache.maven.plugins + maven-source-plugin + + + attach-sources + + jar + + + + + + maven-jar-plugin + + + + true + true + + + + + + + diff --git a/extensions/s3-extensions/src/main/java/io/druid/storage/s3/AWSCredentialsConfig.java b/aws-common/src/main/java/io/druid/common/aws/AWSCredentialsConfig.java similarity index 97% rename from extensions/s3-extensions/src/main/java/io/druid/storage/s3/AWSCredentialsConfig.java rename to aws-common/src/main/java/io/druid/common/aws/AWSCredentialsConfig.java index dc2615ae264..330b31306bf 100644 --- a/extensions/s3-extensions/src/main/java/io/druid/storage/s3/AWSCredentialsConfig.java +++ b/aws-common/src/main/java/io/druid/common/aws/AWSCredentialsConfig.java @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package io.druid.storage.s3; +package io.druid.common.aws; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/aws-common/src/main/java/io/druid/common/aws/AWSCredentialsUtils.java b/aws-common/src/main/java/io/druid/common/aws/AWSCredentialsUtils.java new file mode 100644 index 00000000000..4df2def1b83 --- /dev/null +++ b/aws-common/src/main/java/io/druid/common/aws/AWSCredentialsUtils.java @@ -0,0 +1,39 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2015 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.common.aws; + +import com.amazonaws.auth.AWSCredentialsProviderChain; +import com.amazonaws.auth.EnvironmentVariableCredentialsProvider; +import com.amazonaws.auth.InstanceProfileCredentialsProvider; +import com.amazonaws.auth.SystemPropertiesCredentialsProvider; +import com.amazonaws.auth.profile.ProfileCredentialsProvider; + +public class AWSCredentialsUtils +{ + public static AWSCredentialsProviderChain defaultAWSCredentialsProviderChain(final AWSCredentialsConfig config) { + return new AWSCredentialsProviderChain( + new ConfigDrivenAwsCredentialsConfigProvider(config), + new LazyFileSessionCredentialsProvider(config), + new EnvironmentVariableCredentialsProvider(), + new SystemPropertiesCredentialsProvider(), + new ProfileCredentialsProvider(), + new InstanceProfileCredentialsProvider()); + } +} diff --git a/aws-common/src/main/java/io/druid/common/aws/ConfigDrivenAwsCredentialsConfigProvider.java b/aws-common/src/main/java/io/druid/common/aws/ConfigDrivenAwsCredentialsConfigProvider.java new file mode 100644 index 00000000000..e94b479ba4a --- /dev/null +++ b/aws-common/src/main/java/io/druid/common/aws/ConfigDrivenAwsCredentialsConfigProvider.java @@ -0,0 +1,55 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2015 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.common.aws; + +import com.amazonaws.AmazonClientException; +import com.amazonaws.auth.AWSCredentialsProvider; +import com.google.common.base.Strings; + +public class ConfigDrivenAwsCredentialsConfigProvider implements AWSCredentialsProvider +{ + private AWSCredentialsConfig config; + + public ConfigDrivenAwsCredentialsConfigProvider(AWSCredentialsConfig config) { + this.config = config; + } + + @Override + public com.amazonaws.auth.AWSCredentials getCredentials() + { + if (!Strings.isNullOrEmpty(config.getAccessKey()) && !Strings.isNullOrEmpty(config.getSecretKey())) { + return new com.amazonaws.auth.AWSCredentials() { + @Override + public String getAWSAccessKeyId() { + return config.getAccessKey(); + } + + @Override + public String getAWSSecretKey() { + return config.getSecretKey(); + } + }; + } + throw new AmazonClientException("Unable to load AWS credentials from druid AWSCredentialsConfig"); + } + + @Override + public void refresh() {} +} diff --git a/extensions/s3-extensions/src/main/java/io/druid/storage/s3/FileSessionCredentialsProvider.java b/aws-common/src/main/java/io/druid/common/aws/FileSessionCredentialsProvider.java similarity index 99% rename from extensions/s3-extensions/src/main/java/io/druid/storage/s3/FileSessionCredentialsProvider.java rename to aws-common/src/main/java/io/druid/common/aws/FileSessionCredentialsProvider.java index 9c4fe4e2fdd..01dd5dc04d9 100644 --- a/extensions/s3-extensions/src/main/java/io/druid/storage/s3/FileSessionCredentialsProvider.java +++ b/aws-common/src/main/java/io/druid/common/aws/FileSessionCredentialsProvider.java @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package io.druid.storage.s3; +package io.druid.common.aws; import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.AWSCredentialsProvider; diff --git a/aws-common/src/main/java/io/druid/common/aws/LazyFileSessionCredentialsProvider.java b/aws-common/src/main/java/io/druid/common/aws/LazyFileSessionCredentialsProvider.java new file mode 100644 index 00000000000..dfdf9032d78 --- /dev/null +++ b/aws-common/src/main/java/io/druid/common/aws/LazyFileSessionCredentialsProvider.java @@ -0,0 +1,54 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2015 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.common.aws; + +import com.amazonaws.auth.AWSCredentialsProvider; + +public class LazyFileSessionCredentialsProvider implements AWSCredentialsProvider +{ + private AWSCredentialsConfig config; + private FileSessionCredentialsProvider provider; + + public LazyFileSessionCredentialsProvider(AWSCredentialsConfig config) { + this.config = config; + } + + private FileSessionCredentialsProvider getUnderlyingProvider() { + if (provider == null) { + synchronized (config) { + if (provider == null) { + provider = new FileSessionCredentialsProvider(config.getFileSessionCredentials()); + } + } + } + return provider; + } + + @Override + public com.amazonaws.auth.AWSCredentials getCredentials() + { + return getUnderlyingProvider().getCredentials(); + } + + @Override + public void refresh() { + getUnderlyingProvider().refresh(); + } +} diff --git a/extensions/s3-extensions/pom.xml b/extensions/s3-extensions/pom.xml index 9594c84650a..3fa0b632e58 100644 --- a/extensions/s3-extensions/pom.xml +++ b/extensions/s3-extensions/pom.xml @@ -37,6 +37,12 @@ io.druid druid-api + + io.druid + druid-aws-common + ${project.parent.version} + + net.java.dev.jets3t @@ -47,10 +53,6 @@ org.apache.logging.log4j log4j-1.2-api - - com.amazonaws - aws-java-sdk - org.apache.httpcomponents diff --git a/extensions/s3-extensions/src/main/java/io/druid/storage/s3/S3StorageDruidModule.java b/extensions/s3-extensions/src/main/java/io/druid/storage/s3/S3StorageDruidModule.java index 792136ae8b9..b85b7a22d3a 100644 --- a/extensions/s3-extensions/src/main/java/io/druid/storage/s3/S3StorageDruidModule.java +++ b/extensions/s3-extensions/src/main/java/io/druid/storage/s3/S3StorageDruidModule.java @@ -19,7 +19,6 @@ package io.druid.storage.s3; -import com.amazonaws.AmazonClientException; import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.auth.AWSCredentialsProviderChain; import com.amazonaws.auth.EnvironmentVariableCredentialsProvider; @@ -27,10 +26,13 @@ import com.amazonaws.auth.InstanceProfileCredentialsProvider; import com.amazonaws.auth.SystemPropertiesCredentialsProvider; import com.amazonaws.auth.profile.ProfileCredentialsProvider; import com.fasterxml.jackson.databind.Module; -import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.inject.Binder; import com.google.inject.Provides; +import io.druid.common.aws.AWSCredentialsConfig; +import io.druid.common.aws.AWSCredentialsUtils; +import io.druid.common.aws.ConfigDrivenAwsCredentialsConfigProvider; +import io.druid.common.aws.LazyFileSessionCredentialsProvider; import io.druid.guice.Binders; import io.druid.guice.JsonConfigProvider; import io.druid.guice.LazySingleton; @@ -68,80 +70,11 @@ public class S3StorageDruidModule implements DruidModule binder.bind(S3TaskLogs.class).in(LazySingleton.class); } - private static class ConfigDrivenAwsCredentialsConfigProvider implements AWSCredentialsProvider - { - private AWSCredentialsConfig config; - - public ConfigDrivenAwsCredentialsConfigProvider(AWSCredentialsConfig config) { - this.config = config; - } - - @Override - public com.amazonaws.auth.AWSCredentials getCredentials() - { - if (!Strings.isNullOrEmpty(config.getAccessKey()) && !Strings.isNullOrEmpty(config.getSecretKey())) { - return new com.amazonaws.auth.AWSCredentials() { - @Override - public String getAWSAccessKeyId() { - return config.getAccessKey(); - } - - @Override - public String getAWSSecretKey() { - return config.getSecretKey(); - } - }; - } - throw new AmazonClientException("Unable to load AWS credentials from druid AWSCredentialsConfig"); - } - - @Override - public void refresh() {} - } - - private static class LazyFileSessionCredentialsProvider implements AWSCredentialsProvider - { - private AWSCredentialsConfig config; - private FileSessionCredentialsProvider provider; - - public LazyFileSessionCredentialsProvider(AWSCredentialsConfig config) { - this.config = config; - } - - private FileSessionCredentialsProvider getUnderlyingProvider() { - if (provider == null) { - synchronized (config) { - if (provider == null) { - provider = new FileSessionCredentialsProvider(config.getFileSessionCredentials()); - } - } - } - return provider; - } - - @Override - public com.amazonaws.auth.AWSCredentials getCredentials() - { - return getUnderlyingProvider().getCredentials(); - } - - @Override - public void refresh() { - getUnderlyingProvider().refresh(); - } - } - @Provides @LazySingleton public AWSCredentialsProvider getAWSCredentialsProvider(final AWSCredentialsConfig config) { - return new AWSCredentialsProviderChain( - new ConfigDrivenAwsCredentialsConfigProvider(config), - new LazyFileSessionCredentialsProvider(config), - new EnvironmentVariableCredentialsProvider(), - new SystemPropertiesCredentialsProvider(), - new ProfileCredentialsProvider(), - new InstanceProfileCredentialsProvider()); + return AWSCredentialsUtils.defaultAWSCredentialsProviderChain(config); } @Provides diff --git a/extensions/s3-extensions/src/test/java/io/druid/storage/s3/TestAWSCredentialsProvider.java b/extensions/s3-extensions/src/test/java/io/druid/storage/s3/TestAWSCredentialsProvider.java index ad3d33995d4..e132dbb9685 100644 --- a/extensions/s3-extensions/src/test/java/io/druid/storage/s3/TestAWSCredentialsProvider.java +++ b/extensions/s3-extensions/src/test/java/io/druid/storage/s3/TestAWSCredentialsProvider.java @@ -22,6 +22,7 @@ package io.druid.storage.s3; import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.auth.AWSSessionCredentials; +import io.druid.common.aws.AWSCredentialsConfig; import org.easymock.EasyMock; import org.junit.Rule; import org.junit.Test; diff --git a/extensions/s3-extensions/src/test/java/io/druid/storage/s3/TestFileSessionCredentialsProvider.java b/extensions/s3-extensions/src/test/java/io/druid/storage/s3/TestFileSessionCredentialsProvider.java index 8d9743e80c8..c9dd7769e69 100644 --- a/extensions/s3-extensions/src/test/java/io/druid/storage/s3/TestFileSessionCredentialsProvider.java +++ b/extensions/s3-extensions/src/test/java/io/druid/storage/s3/TestFileSessionCredentialsProvider.java @@ -20,6 +20,7 @@ package io.druid.storage.s3; import com.amazonaws.auth.AWSSessionCredentials; +import io.druid.common.aws.FileSessionCredentialsProvider; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; diff --git a/pom.xml b/pom.xml index 8d20e709797..f2a293ee6ad 100644 --- a/pom.xml +++ b/pom.xml @@ -56,6 +56,7 @@ server services integration-tests + aws-common extensions/cassandra-storage extensions/hdfs-storage diff --git a/server/pom.xml b/server/pom.xml index 439c968a536..2bc86b39ce6 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -37,6 +37,11 @@ druid-processing ${project.parent.version} + + io.druid + druid-aws-common + ${project.parent.version} + com.metamx http-client @@ -61,10 +66,6 @@ org.glassfish javax.el - - com.amazonaws - aws-java-sdk - org.apache.curator curator-framework diff --git a/server/src/main/java/io/druid/guice/AWSModule.java b/server/src/main/java/io/druid/guice/AWSModule.java index 76ea8bbe142..6c8f8e2a829 100644 --- a/server/src/main/java/io/druid/guice/AWSModule.java +++ b/server/src/main/java/io/druid/guice/AWSModule.java @@ -20,13 +20,14 @@ package io.druid.guice; import com.amazonaws.auth.AWSCredentials; -import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.services.ec2.AmazonEC2; import com.amazonaws.services.ec2.AmazonEC2Client; -import com.fasterxml.jackson.annotation.JsonProperty; import com.google.inject.Binder; import com.google.inject.Module; import com.google.inject.Provides; +import io.druid.common.aws.AWSCredentialsConfig; +import io.druid.common.aws.AWSCredentialsUtils; /** */ @@ -40,35 +41,15 @@ public class AWSModule implements Module @Provides @LazySingleton - public AWSCredentials getAWSCredentials(AWSCredentialsConfig config) + public AWSCredentialsProvider getAWSCredentialsProvider(final AWSCredentialsConfig config) { - return new BasicAWSCredentials(config.getAccessKey(), config.getSecretKey()); + return AWSCredentialsUtils.defaultAWSCredentialsProviderChain(config); } @Provides @LazySingleton - public AmazonEC2 getEc2Client(AWSCredentials credentials) + public AmazonEC2 getEc2Client(AWSCredentialsProvider credentials) { return new AmazonEC2Client(credentials); } - - public static class AWSCredentialsConfig - { - @JsonProperty - private String accessKey = ""; - - @JsonProperty - private String secretKey = ""; - - public String getAccessKey() - { - return accessKey; - } - - public String getSecretKey() - { - return secretKey; - } - } - }