Merge pull request #1072 from metamx/common-aws-config

Common AWS module + consistent credential chain for both S3 and auto-scaling

aws-common/pom.xml

@@ -0,0 +1,80 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Druid - a distributed column store.
+  ~ Copyright (C) 2015  Metamarkets Group Inc.
+  ~
+  ~ This program is free software; you can redistribute it and/or
+  ~ modify it under the terms of the GNU General Public License
+  ~ as published by the Free Software Foundation; either version 2
+  ~ of the License, or (at your option) any later version.
+  ~
+  ~ This program is distributed in the hope that it will be useful,
+  ~ but WITHOUT ANY WARRANTY; without even the implied warranty of
+  ~ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  ~ GNU General Public License for more details.
+  ~
+  ~ You should have received a copy of the GNU General Public License
+  ~ along with this program; if not, write to the Free Software
+  ~ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <artifactId>druid-aws-common</artifactId>
+  <name>druid-aws-common</name>
+  <description>druid-aws-common</description>
+
+  <parent>
+    <groupId>io.druid</groupId>
+    <artifactId>druid</artifactId>
+    <version>0.7.0-SNAPSHOT</version>
+  </parent>
+
+  <dependencies>
+    <dependency>
+      <groupId>io.druid</groupId>
+      <artifactId>druid-common</artifactId>
+      <version>${project.parent.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>com.amazonaws</groupId>
+      <artifactId>aws-java-sdk</artifactId>
+    </dependency>
+
+    <!-- Tests -->
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-source-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>attach-sources</id>
+            <goals>
+              <goal>jar</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <artifactId>maven-jar-plugin</artifactId>
+        <configuration>
+          <archive>
+            <manifest>
+              <addDefaultImplementationEntries>true</addDefaultImplementationEntries>
+              <addDefaultSpecificationEntries>true</addDefaultSpecificationEntries>
+            </manifest>
+          </archive>
+        </configuration>
+      </plugin>
+    </plugins>
+  </build>
+</project>

aws-common/src/main/java/io/druid/common/aws/AWSCredentialsConfig.java

@@ -17,7 +17,7 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */
 
-package io.druid.storage.s3;
+package io.druid.common.aws;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 

aws-common/src/main/java/io/druid/common/aws/AWSCredentialsUtils.java

@@ -0,0 +1,39 @@
+/*
+ * Druid - a distributed column store.
+ * Copyright (C) 2015  Metamarkets Group Inc.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+package io.druid.common.aws;
+
+import com.amazonaws.auth.AWSCredentialsProviderChain;
+import com.amazonaws.auth.EnvironmentVariableCredentialsProvider;
+import com.amazonaws.auth.InstanceProfileCredentialsProvider;
+import com.amazonaws.auth.SystemPropertiesCredentialsProvider;
+import com.amazonaws.auth.profile.ProfileCredentialsProvider;
+
+public class AWSCredentialsUtils
+{
+  public static AWSCredentialsProviderChain defaultAWSCredentialsProviderChain(final AWSCredentialsConfig config) {
+    return new AWSCredentialsProviderChain(
+        new ConfigDrivenAwsCredentialsConfigProvider(config),
+        new LazyFileSessionCredentialsProvider(config),
+        new EnvironmentVariableCredentialsProvider(),
+        new SystemPropertiesCredentialsProvider(),
+        new ProfileCredentialsProvider(),
+        new InstanceProfileCredentialsProvider());
+  }
+}

aws-common/src/main/java/io/druid/common/aws/ConfigDrivenAwsCredentialsConfigProvider.java

@@ -0,0 +1,55 @@
+/*
+ * Druid - a distributed column store.
+ * Copyright (C) 2015  Metamarkets Group Inc.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+package io.druid.common.aws;
+
+import com.amazonaws.AmazonClientException;
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.google.common.base.Strings;
+
+public class ConfigDrivenAwsCredentialsConfigProvider implements AWSCredentialsProvider
+{
+  private AWSCredentialsConfig config;
+
+  public ConfigDrivenAwsCredentialsConfigProvider(AWSCredentialsConfig config) {
+    this.config = config;
+  }
+
+  @Override
+  public com.amazonaws.auth.AWSCredentials getCredentials()
+  {
+      if (!Strings.isNullOrEmpty(config.getAccessKey()) && !Strings.isNullOrEmpty(config.getSecretKey())) {
+        return new com.amazonaws.auth.AWSCredentials() {
+          @Override
+          public String getAWSAccessKeyId() {
+            return config.getAccessKey();
+          }
+
+          @Override
+          public String getAWSSecretKey() {
+            return config.getSecretKey();
+          }
+        };
+      }
+      throw new AmazonClientException("Unable to load AWS credentials from druid AWSCredentialsConfig");
+  }
+
+  @Override
+  public void refresh() {}
+}

aws-common/src/main/java/io/druid/common/aws/FileSessionCredentialsProvider.java

@@ -17,7 +17,7 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */
 
-package io.druid.storage.s3;
+package io.druid.common.aws;
 
 import com.amazonaws.auth.AWSCredentials;
 import com.amazonaws.auth.AWSCredentialsProvider;

aws-common/src/main/java/io/druid/common/aws/LazyFileSessionCredentialsProvider.java

@@ -0,0 +1,54 @@
+/*
+ * Druid - a distributed column store.
+ * Copyright (C) 2015  Metamarkets Group Inc.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+package io.druid.common.aws;
+
+import com.amazonaws.auth.AWSCredentialsProvider;
+
+public class LazyFileSessionCredentialsProvider implements AWSCredentialsProvider
+{
+  private AWSCredentialsConfig config;
+  private FileSessionCredentialsProvider provider;
+
+  public LazyFileSessionCredentialsProvider(AWSCredentialsConfig config) {
+    this.config = config;
+  }
+
+  private FileSessionCredentialsProvider getUnderlyingProvider() {
+    if (provider == null) {
+      synchronized (config) {
+        if (provider == null) {
+          provider = new FileSessionCredentialsProvider(config.getFileSessionCredentials());
+        }
+      }
+    }
+    return provider;
+  }
+
+  @Override
+  public com.amazonaws.auth.AWSCredentials getCredentials()
+  {
+    return getUnderlyingProvider().getCredentials();
+  }
+
+  @Override
+  public void refresh() {
+    getUnderlyingProvider().refresh();
+  }
+}

extensions/s3-extensions/pom.xml

@@ -37,6 +37,12 @@
             <groupId>io.druid</groupId>
             <artifactId>druid-api</artifactId>
         </dependency>
+        <dependency>
+          <groupId>io.druid</groupId>
+          <artifactId>druid-aws-common</artifactId>
+          <version>${project.parent.version}</version>
+        </dependency>
+
         <!-- override jets3t from hadoop-core -->
         <dependency>
             <groupId>net.java.dev.jets3t</groupId>
@@ -47,10 +53,6 @@
             <groupId>org.apache.logging.log4j</groupId>
             <artifactId>log4j-1.2-api</artifactId>
         </dependency>
-        <dependency>
-            <groupId>com.amazonaws</groupId>
-            <artifactId>aws-java-sdk</artifactId>
-        </dependency>
         <!-- override httpclient / httpcore version from jets3t -->
         <dependency>
             <groupId>org.apache.httpcomponents</groupId>

extensions/s3-extensions/src/main/java/io/druid/storage/s3/S3StorageDruidModule.java

@@ -19,7 +19,6 @@
 
 package io.druid.storage.s3;
 
-import com.amazonaws.AmazonClientException;
 import com.amazonaws.auth.AWSCredentialsProvider;
 import com.amazonaws.auth.AWSCredentialsProviderChain;
 import com.amazonaws.auth.EnvironmentVariableCredentialsProvider;
@@ -27,10 +26,13 @@ import com.amazonaws.auth.InstanceProfileCredentialsProvider;
 import com.amazonaws.auth.SystemPropertiesCredentialsProvider;
 import com.amazonaws.auth.profile.ProfileCredentialsProvider;
 import com.fasterxml.jackson.databind.Module;
-import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableList;
 import com.google.inject.Binder;
 import com.google.inject.Provides;
+import io.druid.common.aws.AWSCredentialsConfig;
+import io.druid.common.aws.AWSCredentialsUtils;
+import io.druid.common.aws.ConfigDrivenAwsCredentialsConfigProvider;
+import io.druid.common.aws.LazyFileSessionCredentialsProvider;
 import io.druid.guice.Binders;
 import io.druid.guice.JsonConfigProvider;
 import io.druid.guice.LazySingleton;
@@ -68,80 +70,11 @@ public class S3StorageDruidModule implements DruidModule
     binder.bind(S3TaskLogs.class).in(LazySingleton.class);
   }
 
-  private static class ConfigDrivenAwsCredentialsConfigProvider implements AWSCredentialsProvider 
-  {
-    private AWSCredentialsConfig config;
-
-    public ConfigDrivenAwsCredentialsConfigProvider(AWSCredentialsConfig config) {
-      this.config = config;
-    }
-    
-    @Override
-    public com.amazonaws.auth.AWSCredentials getCredentials() 
-    {
-        if (!Strings.isNullOrEmpty(config.getAccessKey()) && !Strings.isNullOrEmpty(config.getSecretKey())) {
-          return new com.amazonaws.auth.AWSCredentials() {
-            @Override
-            public String getAWSAccessKeyId() {
-              return config.getAccessKey();
-            }
-
-            @Override
-            public String getAWSSecretKey() {
-              return config.getSecretKey();
-            }
-          };
-        }
-        throw new AmazonClientException("Unable to load AWS credentials from druid AWSCredentialsConfig");
-    }
-    
-    @Override
-    public void refresh() {}
-  }
-  
-  private static class LazyFileSessionCredentialsProvider implements AWSCredentialsProvider 
-  {
-    private AWSCredentialsConfig config;
-    private FileSessionCredentialsProvider provider;
-    
-    public LazyFileSessionCredentialsProvider(AWSCredentialsConfig config) {
-      this.config = config;
-    }
-    
-    private FileSessionCredentialsProvider getUnderlyingProvider() {
-      if (provider == null) {
-        synchronized (config) {
-          if (provider == null) {
-            provider = new FileSessionCredentialsProvider(config.getFileSessionCredentials());
-          }
-        }
-      }
-      return provider;
-    }
-    
-    @Override
-    public com.amazonaws.auth.AWSCredentials getCredentials() 
-    {
-      return getUnderlyingProvider().getCredentials();
-    }
-
-    @Override
-    public void refresh() {
-      getUnderlyingProvider().refresh();
-    }
-  }
-  
   @Provides
   @LazySingleton
   public AWSCredentialsProvider getAWSCredentialsProvider(final AWSCredentialsConfig config)
   {
-    return new AWSCredentialsProviderChain(
+    return AWSCredentialsUtils.defaultAWSCredentialsProviderChain(config);
-           new ConfigDrivenAwsCredentialsConfigProvider(config),
-           new LazyFileSessionCredentialsProvider(config),
-           new EnvironmentVariableCredentialsProvider(),
-           new SystemPropertiesCredentialsProvider(),
-           new ProfileCredentialsProvider(),
-           new InstanceProfileCredentialsProvider());
   }
 
   @Provides

extensions/s3-extensions/src/test/java/io/druid/storage/s3/TestAWSCredentialsProvider.java

@@ -22,6 +22,7 @@ package io.druid.storage.s3;
 import com.amazonaws.auth.AWSCredentials;
 import com.amazonaws.auth.AWSCredentialsProvider;
 import com.amazonaws.auth.AWSSessionCredentials;
+import io.druid.common.aws.AWSCredentialsConfig;
 import org.easymock.EasyMock;
 import org.junit.Rule;
 import org.junit.Test;

extensions/s3-extensions/src/test/java/io/druid/storage/s3/TestFileSessionCredentialsProvider.java

@@ -20,6 +20,7 @@
 package io.druid.storage.s3;
 
 import com.amazonaws.auth.AWSSessionCredentials;
+import io.druid.common.aws.FileSessionCredentialsProvider;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;

pom.xml

@@ -56,6 +56,7 @@
         <module>server</module>
         <module>services</module>
         <module>integration-tests</module>
+        <module>aws-common</module>
         <!-- Non-default modules -->
         <module>extensions/cassandra-storage</module>
         <module>extensions/hdfs-storage</module>

server/pom.xml

@@ -37,6 +37,11 @@
             <artifactId>druid-processing</artifactId>
             <version>${project.parent.version}</version>
         </dependency>
+        <dependency>
+            <groupId>io.druid</groupId>
+            <artifactId>druid-aws-common</artifactId>
+            <version>${project.parent.version}</version>
+        </dependency>
         <dependency>
             <groupId>com.metamx</groupId>
             <artifactId>http-client</artifactId>
@@ -61,10 +66,6 @@
             <groupId>org.glassfish</groupId>
             <artifactId>javax.el</artifactId>
         </dependency>
-        <dependency>
-            <groupId>com.amazonaws</groupId>
-            <artifactId>aws-java-sdk</artifactId>
-        </dependency>
         <dependency>
             <groupId>org.apache.curator</groupId>
             <artifactId>curator-framework</artifactId>

server/src/main/java/io/druid/guice/AWSModule.java

@@ -20,13 +20,14 @@
 package io.druid.guice;
 
 import com.amazonaws.auth.AWSCredentials;
-import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.auth.AWSCredentialsProvider;
 import com.amazonaws.services.ec2.AmazonEC2;
 import com.amazonaws.services.ec2.AmazonEC2Client;
-import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.inject.Binder;
 import com.google.inject.Module;
 import com.google.inject.Provides;
+import io.druid.common.aws.AWSCredentialsConfig;
+import io.druid.common.aws.AWSCredentialsUtils;
 
 /**
  */
@@ -40,35 +41,15 @@ public class AWSModule implements Module
 
   @Provides
   @LazySingleton
-  public AWSCredentials getAWSCredentials(AWSCredentialsConfig config)
+  public AWSCredentialsProvider getAWSCredentialsProvider(final AWSCredentialsConfig config)
   {
-    return new BasicAWSCredentials(config.getAccessKey(), config.getSecretKey());
+    return AWSCredentialsUtils.defaultAWSCredentialsProviderChain(config);
   }
 
   @Provides
   @LazySingleton
-  public AmazonEC2 getEc2Client(AWSCredentials credentials)
+  public AmazonEC2 getEc2Client(AWSCredentialsProvider credentials)
   {
     return new AmazonEC2Client(credentials);
   }
-
-  public static class AWSCredentialsConfig
-  {
-    @JsonProperty
-    private String accessKey = "";
-
-    @JsonProperty
-    private String secretKey = "";
-
-    public String getAccessKey()
-    {
-      return accessKey;
-    }
-
-    public String getSecretKey()
-    {
-      return secretKey;
-    }
-  }
-
 }