Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1072 from metamx/common-aws-config 

Common AWS module + consistent credential chain for both S3 and auto-scaling
This commit is contained in:
Fangjin Yang 2015-01-30 14:41:24 -08:00
parent 14837d7808 f00872c41b
commit b94b24c46e
13 changed files with 255 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

80
aws-common/pom.xml Normal file
View File

@ -0,0 +1,80 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Druid - a distributed column store.
~ Copyright (C) 2015 Metamarkets Group Inc.
~
~ This program is free software; you can redistribute it and/or
~ modify it under the terms of the GNU General Public License
~ as published by the Free Software Foundation; either version 2
~ of the License, or (at your option) any later version.
~
~ This program is distributed in the hope that it will be useful,
~ but WITHOUT ANY WARRANTY; without even the implied warranty of
~ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
~ GNU General Public License for more details.
~
~ You should have received a copy of the GNU General Public License
~ along with this program; if not, write to the Free Software
~ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<artifactId>druid-aws-common</artifactId>
<name>druid-aws-common</name>
<description>druid-aws-common</description>
<parent>
<groupId>io.druid</groupId>
<artifactId>druid</artifactId>
<version>0.7.0-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
<groupId>io.druid</groupId>
<artifactId>druid-common</artifactId>
<version>${project.parent.version}</version>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk</artifactId>
</dependency>
<!-- Tests -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
<executions>
<execution>
<id>attach-sources</id>
<goals>
<goal>jar</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-jar-plugin</artifactId>
<configuration>
<archive>
<manifest>
<addDefaultImplementationEntries>true</addDefaultImplementationEntries>
<addDefaultSpecificationEntries>true</addDefaultSpecificationEntries>
</manifest>
</archive>
</configuration>
</plugin>
</plugins>
</build>
</project>

2
extensions/s3-extensions/src/main/java/io/druid/storage/s3/AWSCredentialsConfig.java → aws-common/src/main/java/io/druid/common/aws/AWSCredentialsConfig.java
View File

@ -17,7 +17,7 @@
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package io.druid.storage.s3;
package io.druid.common.aws;
import com.fasterxml.jackson.annotation.JsonProperty;

39
aws-common/src/main/java/io/druid/common/aws/AWSCredentialsUtils.java Normal file
View File

@ -0,0 +1,39 @@
/*
* Druid - a distributed column store.
* Copyright (C) 2015 Metamarkets Group Inc.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package io.druid.common.aws;
import com.amazonaws.auth.AWSCredentialsProviderChain;
import com.amazonaws.auth.EnvironmentVariableCredentialsProvider;
import com.amazonaws.auth.InstanceProfileCredentialsProvider;
import com.amazonaws.auth.SystemPropertiesCredentialsProvider;
import com.amazonaws.auth.profile.ProfileCredentialsProvider;
public class AWSCredentialsUtils
{
public static AWSCredentialsProviderChain defaultAWSCredentialsProviderChain(final AWSCredentialsConfig config) {
return new AWSCredentialsProviderChain(
new ConfigDrivenAwsCredentialsConfigProvider(config),
new LazyFileSessionCredentialsProvider(config),
new EnvironmentVariableCredentialsProvider(),
new SystemPropertiesCredentialsProvider(),
new ProfileCredentialsProvider(),
new InstanceProfileCredentialsProvider());
}
}

55
aws-common/src/main/java/io/druid/common/aws/ConfigDrivenAwsCredentialsConfigProvider.java Normal file
View File

@ -0,0 +1,55 @@
/*
* Druid - a distributed column store.
* Copyright (C) 2015 Metamarkets Group Inc.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package io.druid.common.aws;
import com.amazonaws.AmazonClientException;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.google.common.base.Strings;
public class ConfigDrivenAwsCredentialsConfigProvider implements AWSCredentialsProvider
{
private AWSCredentialsConfig config;
public ConfigDrivenAwsCredentialsConfigProvider(AWSCredentialsConfig config) {
this.config = config;
}
@Override
public com.amazonaws.auth.AWSCredentials getCredentials()
{
if (!Strings.isNullOrEmpty(config.getAccessKey()) && !Strings.isNullOrEmpty(config.getSecretKey())) {
return new com.amazonaws.auth.AWSCredentials() {
@Override
public String getAWSAccessKeyId() {
return config.getAccessKey();
}
@Override
public String getAWSSecretKey() {
return config.getSecretKey();
}
};
}
throw new AmazonClientException("Unable to load AWS credentials from druid AWSCredentialsConfig");
}
@Override
public void refresh() {}
}

2
extensions/s3-extensions/src/main/java/io/druid/storage/s3/FileSessionCredentialsProvider.java → aws-common/src/main/java/io/druid/common/aws/FileSessionCredentialsProvider.java
View File

@ -17,7 +17,7 @@
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package io.druid.storage.s3;
package io.druid.common.aws;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSCredentialsProvider;

54
aws-common/src/main/java/io/druid/common/aws/LazyFileSessionCredentialsProvider.java Normal file
View File

@ -0,0 +1,54 @@
/*
* Druid - a distributed column store.
* Copyright (C) 2015 Metamarkets Group Inc.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package io.druid.common.aws;
import com.amazonaws.auth.AWSCredentialsProvider;
public class LazyFileSessionCredentialsProvider implements AWSCredentialsProvider
{
private AWSCredentialsConfig config;
private FileSessionCredentialsProvider provider;
public LazyFileSessionCredentialsProvider(AWSCredentialsConfig config) {
this.config = config;
}
private FileSessionCredentialsProvider getUnderlyingProvider() {
if (provider == null) {
synchronized (config) {
if (provider == null) {
provider = new FileSessionCredentialsProvider(config.getFileSessionCredentials());
}
}
}
return provider;
}
@Override
public com.amazonaws.auth.AWSCredentials getCredentials()
{
return getUnderlyingProvider().getCredentials();
}
@Override
public void refresh() {
getUnderlyingProvider().refresh();
}
}

10
extensions/s3-extensions/pom.xml
View File

@ -37,6 +37,12 @@
<groupId>io.druid</groupId>
<artifactId>druid-api</artifactId>
</dependency>
<dependency>
<groupId>io.druid</groupId>
<artifactId>druid-aws-common</artifactId>
<version>${project.parent.version}</version>
</dependency>
<!-- override jets3t from hadoop-core -->
<dependency>
<groupId>net.java.dev.jets3t</groupId>
@ -47,10 +53,6 @@
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-1.2-api</artifactId>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk</artifactId>
</dependency>
<!-- override httpclient / httpcore version from jets3t -->
<dependency>
<groupId>org.apache.httpcomponents</groupId>

77
extensions/s3-extensions/src/main/java/io/druid/storage/s3/S3StorageDruidModule.java
View File

@ -19,7 +19,6 @@
package io.druid.storage.s3;
import com.amazonaws.AmazonClientException;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.AWSCredentialsProviderChain;
import com.amazonaws.auth.EnvironmentVariableCredentialsProvider;
@ -27,10 +26,13 @@ import com.amazonaws.auth.InstanceProfileCredentialsProvider;
import com.amazonaws.auth.SystemPropertiesCredentialsProvider;
import com.amazonaws.auth.profile.ProfileCredentialsProvider;
import com.fasterxml.jackson.databind.Module;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
import com.google.inject.Binder;
import com.google.inject.Provides;
import io.druid.common.aws.AWSCredentialsConfig;
import io.druid.common.aws.AWSCredentialsUtils;
import io.druid.common.aws.ConfigDrivenAwsCredentialsConfigProvider;
import io.druid.common.aws.LazyFileSessionCredentialsProvider;
import io.druid.guice.Binders;
import io.druid.guice.JsonConfigProvider;
import io.druid.guice.LazySingleton;
@ -68,80 +70,11 @@ public class S3StorageDruidModule implements DruidModule
binder.bind(S3TaskLogs.class).in(LazySingleton.class);
}
private static class ConfigDrivenAwsCredentialsConfigProvider implements AWSCredentialsProvider
{
private AWSCredentialsConfig config;
public ConfigDrivenAwsCredentialsConfigProvider(AWSCredentialsConfig config) {
this.config = config;
}
@Override
public com.amazonaws.auth.AWSCredentials getCredentials()
{
if (!Strings.isNullOrEmpty(config.getAccessKey()) && !Strings.isNullOrEmpty(config.getSecretKey())) {
return new com.amazonaws.auth.AWSCredentials() {
@Override
public String getAWSAccessKeyId() {
return config.getAccessKey();
}
@Override
public String getAWSSecretKey() {
return config.getSecretKey();
}
};
}
throw new AmazonClientException("Unable to load AWS credentials from druid AWSCredentialsConfig");
}
@Override
public void refresh() {}
}
private static class LazyFileSessionCredentialsProvider implements AWSCredentialsProvider
{
private AWSCredentialsConfig config;
private FileSessionCredentialsProvider provider;
public LazyFileSessionCredentialsProvider(AWSCredentialsConfig config) {
this.config = config;
}
private FileSessionCredentialsProvider getUnderlyingProvider() {
if (provider == null) {
synchronized (config) {
if (provider == null) {
provider = new FileSessionCredentialsProvider(config.getFileSessionCredentials());
}
}
}
return provider;
}
@Override
public com.amazonaws.auth.AWSCredentials getCredentials()
{
return getUnderlyingProvider().getCredentials();
}
@Override
public void refresh() {
getUnderlyingProvider().refresh();
}
}
@Provides
@LazySingleton
public AWSCredentialsProvider getAWSCredentialsProvider(final AWSCredentialsConfig config)
{
return new AWSCredentialsProviderChain(
new ConfigDrivenAwsCredentialsConfigProvider(config),
new LazyFileSessionCredentialsProvider(config),
new EnvironmentVariableCredentialsProvider(),
new SystemPropertiesCredentialsProvider(),
new ProfileCredentialsProvider(),
new InstanceProfileCredentialsProvider());
return AWSCredentialsUtils.defaultAWSCredentialsProviderChain(config);
}
@Provides

1
extensions/s3-extensions/src/test/java/io/druid/storage/s3/TestAWSCredentialsProvider.java
View File

@ -22,6 +22,7 @@ package io.druid.storage.s3;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.AWSSessionCredentials;
import io.druid.common.aws.AWSCredentialsConfig;
import org.easymock.EasyMock;
import org.junit.Rule;
import org.junit.Test;

1
extensions/s3-extensions/src/test/java/io/druid/storage/s3/TestFileSessionCredentialsProvider.java
View File

@ -20,6 +20,7 @@
package io.druid.storage.s3;
import com.amazonaws.auth.AWSSessionCredentials;
import io.druid.common.aws.FileSessionCredentialsProvider;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;

1
pom.xml
View File

@ -56,6 +56,7 @@
<module>server</module>
<module>services</module>
<module>integration-tests</module>
<module>aws-common</module>
<!-- Non-default modules -->
<module>extensions/cassandra-storage</module>
<module>extensions/hdfs-storage</module>

9
server/pom.xml
View File

@ -37,6 +37,11 @@
<artifactId>druid-processing</artifactId>
<version>${project.parent.version}</version>
</dependency>
<dependency>
<groupId>io.druid</groupId>
<artifactId>druid-aws-common</artifactId>
<version>${project.parent.version}</version>
</dependency>
<dependency>
<groupId>com.metamx</groupId>
<artifactId>http-client</artifactId>
@ -61,10 +66,6 @@
<groupId>org.glassfish</groupId>
<artifactId>javax.el</artifactId>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk</artifactId>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-framework</artifactId>

31
server/src/main/java/io/druid/guice/AWSModule.java
View File

@ -20,13 +20,14 @@
package io.druid.guice;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.services.ec2.AmazonEC2;
import com.amazonaws.services.ec2.AmazonEC2Client;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.inject.Binder;
import com.google.inject.Module;
import com.google.inject.Provides;
import io.druid.common.aws.AWSCredentialsConfig;
import io.druid.common.aws.AWSCredentialsUtils;
/**
*/
@ -40,35 +41,15 @@ public class AWSModule implements Module
@Provides
@LazySingleton
public AWSCredentials getAWSCredentials(AWSCredentialsConfig config)
public AWSCredentialsProvider getAWSCredentialsProvider(final AWSCredentialsConfig config)
{
return new BasicAWSCredentials(config.getAccessKey(), config.getSecretKey());
return AWSCredentialsUtils.defaultAWSCredentialsProviderChain(config);
}
@Provides
@LazySingleton
public AmazonEC2 getEc2Client(AWSCredentials credentials)
public AmazonEC2 getEc2Client(AWSCredentialsProvider credentials)
{
return new AmazonEC2Client(credentials);
}
public static class AWSCredentialsConfig
{
@JsonProperty
private String accessKey = "";
@JsonProperty
private String secretKey = "";
public String getAccessKey()
{
return accessKey;
}
public String getSecretKey()
{
return secretKey;
}
}
}