Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Make s3 work better (#3898)

This commit is contained in:
Charles Allen 2017-02-02 10:04:30 -08:00 committed by Gian Merlino
parent e6b95e80aa
commit a73f1c9c70
2 changed files with 20 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3StorageDruidModule.java
View File

@ -27,7 +27,6 @@ import com.google.inject.Binder;
import com.google.inject.Provides;
import com.google.inject.multibindings.MapBinder;
import io.druid.common.aws.AWSCredentialsConfig;
import io.druid.common.aws.AWSCredentialsUtils;
import io.druid.data.SearchableVersionedDataFinder;
import io.druid.guice.Binders;
import io.druid.guice.JsonConfigProvider;
@ -43,6 +42,7 @@ import java.util.List;
public class S3StorageDruidModule implements DruidModule
{
public static final String SCHEME = "s3_zip";
@Override
public List<? extends Module> getJacksonModules()
{
@ -85,7 +85,10 @@ public class S3StorageDruidModule implements DruidModule
Binders.dataSegmentPullerBinder(binder).addBinding(SCHEME).to(S3DataSegmentPuller.class).in(LazySingleton.class);
Binders.dataSegmentKillerBinder(binder).addBinding(SCHEME).to(S3DataSegmentKiller.class).in(LazySingleton.class);
Binders.dataSegmentMoverBinder(binder).addBinding(SCHEME).to(S3DataSegmentMover.class).in(LazySingleton.class);
Binders.dataSegmentArchiverBinder(binder).addBinding(SCHEME).to(S3DataSegmentArchiver.class).in(LazySingleton.class);
Binders.dataSegmentArchiverBinder(binder)
.addBinding(SCHEME)
.to(S3DataSegmentArchiver.class)
.in(LazySingleton.class);
Binders.dataSegmentPusherBinder(binder).addBinding("s3").to(S3DataSegmentPusher.class).in(LazySingleton.class);
Binders.dataSegmentFinderBinder(binder).addBinding("s3").to(S3DataSegmentFinder.class).in(LazySingleton.class);
JsonConfigProvider.bind(binder, "druid.storage", S3DataSegmentPusherConfig.class);
@ -96,18 +99,11 @@ public class S3StorageDruidModule implements DruidModule
binder.bind(S3TaskLogs.class).in(LazySingleton.class);
}
@Provides
@LazySingleton
public AWSCredentialsProvider getAWSCredentialsProvider(final AWSCredentialsConfig config)
{
return AWSCredentialsUtils.defaultAWSCredentialsProviderChain(config);
}
@Provides
@LazySingleton
public RestS3Service getRestS3Service(AWSCredentialsProvider provider)
{
if(provider.getCredentials() instanceof com.amazonaws.auth.AWSSessionCredentials) {
if (provider.getCredentials() instanceof com.amazonaws.auth.AWSSessionCredentials) {
return new RestS3Service(new AWSSessionCredentialsAdapter(provider));
} else {
return new RestS3Service(new AWSCredentials(

25
extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/TestAWSCredentialsProvider.java
View File

@ -23,6 +23,7 @@ import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.AWSSessionCredentials;
import io.druid.common.aws.AWSCredentialsConfig;
import io.druid.guice.AWSModule;
import org.easymock.EasyMock;
import org.junit.Rule;
import org.junit.Test;
@ -35,32 +36,34 @@ import java.io.PrintWriter;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
public class TestAWSCredentialsProvider {
@Test
public void testWithFixedAWSKeys() {
S3StorageDruidModule module = new S3StorageDruidModule();
public class TestAWSCredentialsProvider
{
private final AWSModule awsModule = new AWSModule();
private final S3StorageDruidModule s3Module = new S3StorageDruidModule();
@Test
public void testWithFixedAWSKeys()
{
AWSCredentialsConfig config = EasyMock.createMock(AWSCredentialsConfig.class);
EasyMock.expect(config.getAccessKey()).andReturn("accessKeySample").atLeastOnce();
EasyMock.expect(config.getSecretKey()).andReturn("secretKeySample").atLeastOnce();
EasyMock.replay(config);
AWSCredentialsProvider provider = module.getAWSCredentialsProvider(config);
AWSCredentialsProvider provider = awsModule.getAWSCredentialsProvider(config);
AWSCredentials credentials = provider.getCredentials();
assertEquals(credentials.getAWSAccessKeyId(), "accessKeySample");
assertEquals(credentials.getAWSSecretKey(), "secretKeySample");
// try to create
module.getRestS3Service(provider);
s3Module.getRestS3Service(provider);
}
@Rule
public TemporaryFolder folder = new TemporaryFolder();
@Test
public void testWithFileSessionCredentials() throws IOException {
S3StorageDruidModule module = new S3StorageDruidModule();
public void testWithFileSessionCredentials() throws IOException
{
AWSCredentialsConfig config = EasyMock.createMock(AWSCredentialsConfig.class);
EasyMock.expect(config.getAccessKey()).andReturn("");
EasyMock.expect(config.getSecretKey()).andReturn("");
@ -71,7 +74,7 @@ public class TestAWSCredentialsProvider {
EasyMock.expect(config.getFileSessionCredentials()).andReturn(file.getAbsolutePath()).atLeastOnce();
EasyMock.replay(config);
AWSCredentialsProvider provider = module.getAWSCredentialsProvider(config);
AWSCredentialsProvider provider = awsModule.getAWSCredentialsProvider(config);
AWSCredentials credentials = provider.getCredentials();
assertTrue(credentials instanceof AWSSessionCredentials);
AWSSessionCredentials sessionCredentials = (AWSSessionCredentials) credentials;
@ -80,6 +83,6 @@ public class TestAWSCredentialsProvider {
assertEquals(sessionCredentials.getSessionToken(), "sessionTokenSample");
// try to create
module.getRestS3Service(provider);
s3Module.getRestS3Service(provider);
}
}