diff --git a/aws-common/pom.xml b/cloud/aws-common/pom.xml similarity index 97% rename from aws-common/pom.xml rename to cloud/aws-common/pom.xml index 210e5717c8a..7e031733750 100644 --- a/aws-common/pom.xml +++ b/cloud/aws-common/pom.xml @@ -29,6 +29,7 @@ org.apache.druid druid 0.15.0-incubating-SNAPSHOT + ../../pom.xml diff --git a/aws-common/src/main/java/org/apache/druid/common/aws/AWSClientConfig.java b/cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSClientConfig.java similarity index 100% rename from aws-common/src/main/java/org/apache/druid/common/aws/AWSClientConfig.java rename to cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSClientConfig.java diff --git a/aws-common/src/main/java/org/apache/druid/common/aws/AWSCredentialsConfig.java b/cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSCredentialsConfig.java similarity index 100% rename from aws-common/src/main/java/org/apache/druid/common/aws/AWSCredentialsConfig.java rename to cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSCredentialsConfig.java diff --git a/aws-common/src/main/java/org/apache/druid/common/aws/AWSCredentialsUtils.java b/cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSCredentialsUtils.java similarity index 100% rename from aws-common/src/main/java/org/apache/druid/common/aws/AWSCredentialsUtils.java rename to cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSCredentialsUtils.java diff --git a/aws-common/src/main/java/org/apache/druid/common/aws/AWSEndpointConfig.java b/cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSEndpointConfig.java similarity index 100% rename from aws-common/src/main/java/org/apache/druid/common/aws/AWSEndpointConfig.java rename to cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSEndpointConfig.java diff --git a/server/src/main/java/org/apache/druid/guice/AWSModule.java b/cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSModule.java similarity index 79% rename from server/src/main/java/org/apache/druid/guice/AWSModule.java rename to cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSModule.java index 9d0c568e8b4..79b60c956c3 100644 --- a/server/src/main/java/org/apache/druid/guice/AWSModule.java +++ b/cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSModule.java @@ -17,23 +17,22 @@ * under the License. */ -package org.apache.druid.guice; +package org.apache.druid.common.aws; import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.services.ec2.AmazonEC2; import com.amazonaws.services.ec2.AmazonEC2Client; +import com.fasterxml.jackson.databind.Module; import com.google.inject.Binder; -import com.google.inject.Module; import com.google.inject.Provides; -import org.apache.druid.common.aws.AWSClientConfig; -import org.apache.druid.common.aws.AWSCredentialsConfig; -import org.apache.druid.common.aws.AWSCredentialsUtils; -import org.apache.druid.common.aws.AWSEndpointConfig; -import org.apache.druid.common.aws.AWSProxyConfig; +import org.apache.druid.guice.JsonConfigProvider; +import org.apache.druid.guice.LazySingleton; +import org.apache.druid.initialization.DruidModule; -/** - */ -public class AWSModule implements Module +import java.util.Collections; +import java.util.List; + +public class AWSModule implements DruidModule { @Override public void configure(Binder binder) @@ -57,4 +56,10 @@ public class AWSModule implements Module { return new AmazonEC2Client(credentials); } + + @Override + public List getJacksonModules() + { + return Collections.emptyList(); + } } diff --git a/aws-common/src/main/java/org/apache/druid/common/aws/AWSProxyConfig.java b/cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSProxyConfig.java similarity index 100% rename from aws-common/src/main/java/org/apache/druid/common/aws/AWSProxyConfig.java rename to cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSProxyConfig.java diff --git a/aws-common/src/main/java/org/apache/druid/common/aws/ConfigDrivenAwsCredentialsConfigProvider.java b/cloud/aws-common/src/main/java/org/apache/druid/common/aws/ConfigDrivenAwsCredentialsConfigProvider.java similarity index 100% rename from aws-common/src/main/java/org/apache/druid/common/aws/ConfigDrivenAwsCredentialsConfigProvider.java rename to cloud/aws-common/src/main/java/org/apache/druid/common/aws/ConfigDrivenAwsCredentialsConfigProvider.java diff --git a/aws-common/src/main/java/org/apache/druid/common/aws/FileSessionCredentialsProvider.java b/cloud/aws-common/src/main/java/org/apache/druid/common/aws/FileSessionCredentialsProvider.java similarity index 100% rename from aws-common/src/main/java/org/apache/druid/common/aws/FileSessionCredentialsProvider.java rename to cloud/aws-common/src/main/java/org/apache/druid/common/aws/FileSessionCredentialsProvider.java diff --git a/aws-common/src/main/java/org/apache/druid/common/aws/LazyFileSessionCredentialsProvider.java b/cloud/aws-common/src/main/java/org/apache/druid/common/aws/LazyFileSessionCredentialsProvider.java similarity index 100% rename from aws-common/src/main/java/org/apache/druid/common/aws/LazyFileSessionCredentialsProvider.java rename to cloud/aws-common/src/main/java/org/apache/druid/common/aws/LazyFileSessionCredentialsProvider.java diff --git a/cloud/aws-common/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule b/cloud/aws-common/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule new file mode 100644 index 00000000000..2d43a9c333d --- /dev/null +++ b/cloud/aws-common/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +org.apache.druid.common.aws.AWSModule diff --git a/aws-common/src/test/java/org/apache/druid/common/aws/AWSCredentialsConfigTest.java b/cloud/aws-common/src/test/java/org/apache/druid/common/aws/AWSCredentialsConfigTest.java similarity index 100% rename from aws-common/src/test/java/org/apache/druid/common/aws/AWSCredentialsConfigTest.java rename to cloud/aws-common/src/test/java/org/apache/druid/common/aws/AWSCredentialsConfigTest.java diff --git a/cloud/gcp-common/pom.xml b/cloud/gcp-common/pom.xml new file mode 100644 index 00000000000..7b914aa5188 --- /dev/null +++ b/cloud/gcp-common/pom.xml @@ -0,0 +1,63 @@ + + + + + 4.0.0 + + druid-gcp-common + druid-gcp-common + Druid GCP Base Module + + + org.apache.druid + druid + 0.15.0-incubating-SNAPSHOT + ../../pom.xml + + + + + org.apache.druid + druid-core + ${project.parent.version} + provided + + + com.google.api-client + google-api-client + + + com.google.http-client + google-http-client-jackson2 + + + com.fasterxml.jackson.module + jackson-module-guice + + + + + junit + junit + test + + + + diff --git a/cloud/gcp-common/src/main/java/org/apache/druid/common/gcp/GcpModule.java b/cloud/gcp-common/src/main/java/org/apache/druid/common/gcp/GcpModule.java new file mode 100644 index 00000000000..56458ee75bc --- /dev/null +++ b/cloud/gcp-common/src/main/java/org/apache/druid/common/gcp/GcpModule.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.common.gcp; + +import com.fasterxml.jackson.databind.Module; +import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; +import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; +import com.google.api.client.http.HttpRequestInitializer; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.json.JsonFactory; +import com.google.api.client.json.jackson2.JacksonFactory; +import com.google.common.collect.ImmutableList; +import com.google.inject.Binder; +import com.google.inject.Provides; +import org.apache.druid.guice.LazySingleton; +import org.apache.druid.initialization.DruidModule; + +import java.io.IOException; +import java.security.GeneralSecurityException; +import java.util.Collections; +import java.util.List; + +public class GcpModule implements DruidModule +{ + @Override + public List getJacksonModules() + { + return ImmutableList.of(); + } + + @Override + public void configure(Binder binder) + { + // Nothing to proactively bind + } + + + @Provides + @LazySingleton + public HttpRequestInitializer getHttpRequestInitializer(HttpTransport transport, JsonFactory factory) + throws IOException + { + GoogleCredential credential = GoogleCredential.getApplicationDefault(transport, factory); + if (credential.createScopedRequired()) { + credential = credential.createScoped(Collections.singleton("https://www.googleapis.com/auth/cloud-platform")); + } + return credential; + } + + @Provides + @LazySingleton + public HttpTransport getHttpTransport() throws GeneralSecurityException, IOException + { + return GoogleNetHttpTransport.newTrustedTransport(); + } + + @Provides + @LazySingleton + public JsonFactory getJsonFactory() + { + return JacksonFactory.getDefaultInstance(); + } +} diff --git a/cloud/gcp-common/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule b/cloud/gcp-common/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule new file mode 100644 index 00000000000..c94639ec271 --- /dev/null +++ b/cloud/gcp-common/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +org.apache.druid.common.gcp.GcpModule diff --git a/cloud/gcp-common/src/test/java/org/apache/druid/common/gcp/GcpMockModule.java b/cloud/gcp-common/src/test/java/org/apache/druid/common/gcp/GcpMockModule.java new file mode 100644 index 00000000000..c1f57541615 --- /dev/null +++ b/cloud/gcp-common/src/test/java/org/apache/druid/common/gcp/GcpMockModule.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.common.gcp; + +import com.fasterxml.jackson.databind.Module; +import com.google.api.client.googleapis.testing.auth.oauth2.MockGoogleCredential; +import com.google.api.client.http.HttpRequestInitializer; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.json.JsonFactory; +import com.google.api.client.json.jackson2.JacksonFactory; +import com.google.api.client.testing.http.MockHttpTransport; +import com.google.common.collect.ImmutableList; +import com.google.inject.Binder; +import com.google.inject.Provides; +import org.apache.druid.guice.LazySingleton; +import org.apache.druid.initialization.DruidModule; + +import java.util.List; + +public class GcpMockModule implements DruidModule +{ + @Override + public List getJacksonModules() + { + return ImmutableList.of(); + } + + @Override + public void configure(Binder binder) + { + + } + + + @Provides + @LazySingleton + public HttpRequestInitializer mockRequestInitializer( + HttpTransport transport, + JsonFactory factory + ) + { + return new MockGoogleCredential.Builder().setTransport(transport).setJsonFactory(factory).build(); + } + + @Provides + @LazySingleton + public HttpTransport buildMockTransport() + { + return new MockHttpTransport.Builder().build(); + } + + @Provides + @LazySingleton + public JsonFactory getJsonFactory() + { + return JacksonFactory.getDefaultInstance(); + } +} diff --git a/cloud/gcp-common/src/test/java/org/apache/druid/common/gcp/GcpModuleTest.java b/cloud/gcp-common/src/test/java/org/apache/druid/common/gcp/GcpModuleTest.java new file mode 100644 index 00000000000..4790a914b72 --- /dev/null +++ b/cloud/gcp-common/src/test/java/org/apache/druid/common/gcp/GcpModuleTest.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.common.gcp; + +import com.google.api.client.googleapis.testing.auth.oauth2.MockGoogleCredential; +import com.google.api.client.http.HttpRequestInitializer; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.testing.http.MockHttpTransport; +import com.google.inject.Binder; +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.Scopes; +import com.google.inject.util.Modules; +import org.apache.druid.guice.LazySingleton; +import org.junit.Assert; +import org.junit.Test; + +public class GcpModuleTest +{ + @Test + public void testSimpleModuleLoads() + { + final Injector injector = Guice.createInjector(Modules.override(new GcpModule()).with(new GcpMockModule() + { + @Override + public void configure(Binder binder) + { + binder.bindScope(LazySingleton.class, Scopes.SINGLETON); + } + })); + Assert.assertTrue(injector.getInstance(HttpRequestInitializer.class) instanceof MockGoogleCredential); + Assert.assertTrue(injector.getInstance(HttpTransport.class) instanceof MockHttpTransport); + } +} diff --git a/distribution/pom.xml b/distribution/pom.xml index 82ea3e5593f..7579b46b6d8 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -178,6 +178,10 @@ -c org.apache.druid.extensions:druid-s3-extensions -c + org.apache.druid.extensions:druid-ec2-extensions + -c + org.apache.druid.extensions:druid-google-extensions + -c org.apache.druid.extensions:druid-stats -c org.apache.druid.extensions:druid-examples @@ -300,8 +304,6 @@ -c org.apache.druid.extensions.contrib:druid-rocketmq -c - org.apache.druid.extensions.contrib:druid-google-extensions - -c org.apache.druid.extensions.contrib:graphite-emitter -c org.apache.druid.extensions.contrib:druid-influx-extensions diff --git a/docs/content/development/extensions.md b/docs/content/development/extensions.md index f24bfc687e1..1775edbd725 100644 --- a/docs/content/development/extensions.md +++ b/docs/content/development/extensions.md @@ -58,6 +58,7 @@ Core extensions are maintained by Druid committers. |druid-parquet-extensions|Support for data in Apache Parquet data format. Requires druid-avro-extensions to be loaded.|[link](../development/extensions-core/parquet.html)| |druid-protobuf-extensions| Support for data in Protobuf data format.|[link](../development/extensions-core/protobuf.html)| |druid-s3-extensions|Interfacing with data in AWS S3, and using S3 as deep storage.|[link](../development/extensions-core/s3.html)| +|druid-ec2-extensions|Interfacing with AWS EC2 for autoscaling middle managers|UNDOCUMENTED| |druid-stats|Statistics related module including variance and standard deviation.|[link](../development/extensions-core/stats.html)| |mysql-metadata-storage|MySQL metadata store.|[link](../development/extensions-core/mysql.html)| |postgresql-metadata-storage|PostgreSQL metadata store.|[link](../development/extensions-core/postgresql.html)| diff --git a/extensions-core/ec2-extensions/pom.xml b/extensions-core/ec2-extensions/pom.xml new file mode 100644 index 00000000000..fb399489712 --- /dev/null +++ b/extensions-core/ec2-extensions/pom.xml @@ -0,0 +1,64 @@ + + + + + 4.0.0 + + org.apache.druid.extensions + druid-ec2-extensions + druid-ec2-extensions + druid-ec2-extensions + + + org.apache.druid + druid + 0.15.0-incubating-SNAPSHOT + ../../pom.xml + + + + + org.apache.druid + druid-indexing-service + ${project.parent.version} + provided + + + org.apache.druid + druid-aws-common + ${project.parent.version} + provided + + + + + junit + junit + test + + + org.easymock + easymock + test + + + + diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2AutoScaler.java b/extensions-core/ec2-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2AutoScaler.java similarity index 99% rename from indexing-service/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2AutoScaler.java rename to extensions-core/ec2-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2AutoScaler.java index 30402377006..56ffc6b74ef 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2AutoScaler.java +++ b/extensions-core/ec2-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2AutoScaler.java @@ -33,6 +33,7 @@ import com.amazonaws.services.ec2.model.TerminateInstancesRequest; import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeName; import com.google.common.base.Function; import com.google.common.collect.FluentIterable; import com.google.common.collect.Lists; @@ -46,6 +47,7 @@ import java.util.List; /** */ +@JsonTypeName("ec2") public class EC2AutoScaler implements AutoScaler { private static final EmittingLogger log = new EmittingLogger(EC2AutoScaler.class); diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2EnvironmentConfig.java b/extensions-core/ec2-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2EnvironmentConfig.java similarity index 100% rename from indexing-service/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2EnvironmentConfig.java rename to extensions-core/ec2-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2EnvironmentConfig.java diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2IamProfileData.java b/extensions-core/ec2-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2IamProfileData.java similarity index 100% rename from indexing-service/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2IamProfileData.java rename to extensions-core/ec2-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2IamProfileData.java diff --git a/extensions-core/ec2-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2Module.java b/extensions-core/ec2-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2Module.java new file mode 100644 index 00000000000..aa533d94cc4 --- /dev/null +++ b/extensions-core/ec2-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2Module.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.indexing.overlord.autoscaling.ec2; + +import com.fasterxml.jackson.databind.Module; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.google.inject.Binder; +import org.apache.druid.initialization.DruidModule; + +import java.util.Collections; +import java.util.List; + +public class EC2Module implements DruidModule +{ + @Override + public List getJacksonModules() + { + return Collections.singletonList(new SimpleModule("DruidEc2Module").registerSubtypes(EC2AutoScaler.class)); + } + + @Override + public void configure(Binder binder) + { + + } +} diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2NodeData.java b/extensions-core/ec2-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2NodeData.java similarity index 100% rename from indexing-service/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2NodeData.java rename to extensions-core/ec2-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2NodeData.java diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2UserData.java b/extensions-core/ec2-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2UserData.java similarity index 100% rename from indexing-service/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2UserData.java rename to extensions-core/ec2-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2UserData.java diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/GalaxyEC2UserData.java b/extensions-core/ec2-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/GalaxyEC2UserData.java similarity index 100% rename from indexing-service/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/GalaxyEC2UserData.java rename to extensions-core/ec2-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/GalaxyEC2UserData.java diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/StringEC2UserData.java b/extensions-core/ec2-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/StringEC2UserData.java similarity index 100% rename from indexing-service/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/StringEC2UserData.java rename to extensions-core/ec2-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/StringEC2UserData.java diff --git a/extensions-core/ec2-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule b/extensions-core/ec2-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule new file mode 100644 index 00000000000..082c02d1265 --- /dev/null +++ b/extensions-core/ec2-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.druid.indexing.overlord.autoscaling.ec2.EC2Module diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/autoscaling/EC2AutoScalerSerdeTest.java b/extensions-core/ec2-extensions/src/test/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2AutoScalerSerdeTest.java similarity index 94% rename from indexing-service/src/test/java/org/apache/druid/indexing/overlord/autoscaling/EC2AutoScalerSerdeTest.java rename to extensions-core/ec2-extensions/src/test/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2AutoScalerSerdeTest.java index f49d6dcf20b..29ba9a0c5a3 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/autoscaling/EC2AutoScalerSerdeTest.java +++ b/extensions-core/ec2-extensions/src/test/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2AutoScalerSerdeTest.java @@ -17,13 +17,14 @@ * under the License. */ -package org.apache.druid.indexing.overlord.autoscaling; +package org.apache.druid.indexing.overlord.autoscaling.ec2; import com.fasterxml.jackson.databind.BeanProperty; import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.InjectableValues; +import com.fasterxml.jackson.databind.Module; import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.druid.indexing.overlord.autoscaling.ec2.EC2AutoScaler; +import org.apache.druid.indexing.overlord.autoscaling.AutoScaler; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.StringUtils; import org.junit.Assert; @@ -57,38 +58,6 @@ public class EC2AutoScalerSerdeTest + " \"type\" : \"ec2\"\n" + "}"; - @Test - public void testSerde() throws Exception - { - final ObjectMapper objectMapper = new DefaultObjectMapper(); - objectMapper.setInjectableValues( - new InjectableValues() - { - @Override - public Object findInjectableValue( - Object o, - DeserializationContext deserializationContext, - BeanProperty beanProperty, - Object o1 - ) - { - return null; - } - } - ); - - final EC2AutoScaler autoScaler = (EC2AutoScaler) objectMapper.readValue(json, AutoScaler.class); - verifyAutoScaler(autoScaler); - - final EC2AutoScaler roundTripAutoScaler = (EC2AutoScaler) objectMapper.readValue( - objectMapper.writeValueAsBytes(autoScaler), - AutoScaler.class - ); - verifyAutoScaler(roundTripAutoScaler); - - Assert.assertEquals("Round trip equals", autoScaler, roundTripAutoScaler); - } - private static void verifyAutoScaler(final EC2AutoScaler autoScaler) { Assert.assertEquals(3, autoScaler.getMaxNumWorkers()); @@ -132,4 +101,37 @@ public class EC2AutoScalerSerdeTest ) ); } + + @Test + public void testSerde() throws Exception + { + final ObjectMapper objectMapper = new DefaultObjectMapper() + .registerModules((Iterable) new EC2Module().getJacksonModules()); + objectMapper.setInjectableValues( + new InjectableValues() + { + @Override + public Object findInjectableValue( + Object o, + DeserializationContext deserializationContext, + BeanProperty beanProperty, + Object o1 + ) + { + return null; + } + } + ); + + final EC2AutoScaler autoScaler = (EC2AutoScaler) objectMapper.readValue(json, AutoScaler.class); + verifyAutoScaler(autoScaler); + + final EC2AutoScaler roundTripAutoScaler = (EC2AutoScaler) objectMapper.readValue( + objectMapper.writeValueAsBytes(autoScaler), + AutoScaler.class + ); + verifyAutoScaler(roundTripAutoScaler); + + Assert.assertEquals("Round trip equals", autoScaler, roundTripAutoScaler); + } } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/autoscaling/EC2AutoScalerTest.java b/extensions-core/ec2-extensions/src/test/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2AutoScalerTest.java similarity index 95% rename from indexing-service/src/test/java/org/apache/druid/indexing/overlord/autoscaling/EC2AutoScalerTest.java rename to extensions-core/ec2-extensions/src/test/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2AutoScalerTest.java index efdc794ae04..bdeb9e81221 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/autoscaling/EC2AutoScalerTest.java +++ b/extensions-core/ec2-extensions/src/test/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2AutoScalerTest.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.druid.indexing.overlord.autoscaling; +package org.apache.druid.indexing.overlord.autoscaling.ec2; import com.amazonaws.services.ec2.AmazonEC2Client; import com.amazonaws.services.ec2.model.DescribeInstancesRequest; @@ -33,10 +33,8 @@ import com.google.common.collect.ContiguousSet; import com.google.common.collect.DiscreteDomain; import com.google.common.collect.Lists; import com.google.common.collect.Range; -import org.apache.druid.indexing.overlord.autoscaling.ec2.EC2AutoScaler; -import org.apache.druid.indexing.overlord.autoscaling.ec2.EC2EnvironmentConfig; -import org.apache.druid.indexing.overlord.autoscaling.ec2.EC2NodeData; -import org.apache.druid.indexing.overlord.autoscaling.ec2.GalaxyEC2UserData; +import org.apache.druid.indexing.overlord.autoscaling.AutoScalingData; +import org.apache.druid.indexing.overlord.autoscaling.SimpleWorkerProvisioningConfig; import org.apache.druid.jackson.DefaultObjectMapper; import org.easymock.EasyMock; import org.junit.After; diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2NodeDataTest.java b/extensions-core/ec2-extensions/src/test/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2NodeDataTest.java similarity index 100% rename from indexing-service/src/test/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2NodeDataTest.java rename to extensions-core/ec2-extensions/src/test/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2NodeDataTest.java diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/WorkerBehaviorConfigTest.java b/extensions-core/ec2-extensions/src/test/java/org/apache/druid/indexing/overlord/setup/WorkerBehaviorConfigTest.java similarity index 98% rename from indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/WorkerBehaviorConfigTest.java rename to extensions-core/ec2-extensions/src/test/java/org/apache/druid/indexing/overlord/setup/WorkerBehaviorConfigTest.java index d15264191af..a47a79fa9bc 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/WorkerBehaviorConfigTest.java +++ b/extensions-core/ec2-extensions/src/test/java/org/apache/druid/indexing/overlord/setup/WorkerBehaviorConfigTest.java @@ -75,6 +75,7 @@ public class WorkerBehaviorConfigTest ); final ObjectMapper mapper = new DefaultObjectMapper(); + mapper.registerSubtypes(EC2AutoScaler.class); mapper.setInjectableValues( new InjectableValues() { diff --git a/extensions-contrib/google-extensions/pom.xml b/extensions-core/google-extensions/pom.xml similarity index 75% rename from extensions-contrib/google-extensions/pom.xml rename to extensions-core/google-extensions/pom.xml index bbd20b2837e..70ba19d5818 100644 --- a/extensions-contrib/google-extensions/pom.xml +++ b/extensions-core/google-extensions/pom.xml @@ -33,6 +33,10 @@ ../../pom.xml + + v1-rev79-${com.google.apis.client.version} + + org.apache.druid @@ -40,35 +44,21 @@ ${project.parent.version} provided + + org.apache.druid + druid-gcp-common + ${project.parent.version} + provided + com.google.apis google-api-services-storage - v1-rev79-1.22.0 - - - com.google.http-client - google-http-client-jackson2 - 1.22.0 + ${com.google.apis.storage.version} - com.fasterxml.jackson.core - jackson-core - - - - - com.fasterxml.jackson.module - jackson-module-guice - ${jackson.version} - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-databind + com.google.api-client + google-api-client diff --git a/extensions-contrib/google-extensions/src/main/java/org/apache/druid/firehose/google/GoogleBlob.java b/extensions-core/google-extensions/src/main/java/org/apache/druid/firehose/google/GoogleBlob.java similarity index 100% rename from extensions-contrib/google-extensions/src/main/java/org/apache/druid/firehose/google/GoogleBlob.java rename to extensions-core/google-extensions/src/main/java/org/apache/druid/firehose/google/GoogleBlob.java diff --git a/extensions-contrib/google-extensions/src/main/java/org/apache/druid/firehose/google/StaticGoogleBlobStoreFirehoseFactory.java b/extensions-core/google-extensions/src/main/java/org/apache/druid/firehose/google/StaticGoogleBlobStoreFirehoseFactory.java similarity index 100% rename from extensions-contrib/google-extensions/src/main/java/org/apache/druid/firehose/google/StaticGoogleBlobStoreFirehoseFactory.java rename to extensions-core/google-extensions/src/main/java/org/apache/druid/firehose/google/StaticGoogleBlobStoreFirehoseFactory.java diff --git a/extensions-contrib/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleAccountConfig.java b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleAccountConfig.java similarity index 100% rename from extensions-contrib/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleAccountConfig.java rename to extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleAccountConfig.java diff --git a/extensions-contrib/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleByteSource.java b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleByteSource.java similarity index 100% rename from extensions-contrib/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleByteSource.java rename to extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleByteSource.java diff --git a/extensions-contrib/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleDataSegmentKiller.java b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleDataSegmentKiller.java similarity index 100% rename from extensions-contrib/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleDataSegmentKiller.java rename to extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleDataSegmentKiller.java diff --git a/extensions-contrib/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleDataSegmentPuller.java b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleDataSegmentPuller.java similarity index 100% rename from extensions-contrib/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleDataSegmentPuller.java rename to extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleDataSegmentPuller.java diff --git a/extensions-contrib/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleDataSegmentPusher.java b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleDataSegmentPusher.java similarity index 100% rename from extensions-contrib/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleDataSegmentPusher.java rename to extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleDataSegmentPusher.java diff --git a/extensions-contrib/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleLoadSpec.java b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleLoadSpec.java similarity index 100% rename from extensions-contrib/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleLoadSpec.java rename to extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleLoadSpec.java diff --git a/extensions-contrib/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorage.java b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorage.java similarity index 100% rename from extensions-contrib/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorage.java rename to extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorage.java diff --git a/extensions-contrib/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorageDruidModule.java b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorageDruidModule.java similarity index 78% rename from extensions-contrib/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorageDruidModule.java rename to extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorageDruidModule.java index 038f3d2550a..74a58c41919 100644 --- a/extensions-contrib/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorageDruidModule.java +++ b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorageDruidModule.java @@ -23,13 +23,10 @@ import com.fasterxml.jackson.core.Version; import com.fasterxml.jackson.databind.Module; import com.fasterxml.jackson.databind.jsontype.NamedType; import com.fasterxml.jackson.databind.module.SimpleModule; -import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; -import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; +import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpTransport; import com.google.api.client.json.JsonFactory; -import com.google.api.client.json.jackson2.JacksonFactory; import com.google.api.services.storage.Storage; -import com.google.api.services.storage.StorageScopes; import com.google.common.collect.ImmutableList; import com.google.inject.Binder; import com.google.inject.Provides; @@ -40,15 +37,12 @@ import org.apache.druid.guice.LazySingleton; import org.apache.druid.initialization.DruidModule; import org.apache.druid.java.util.common.logger.Logger; -import java.io.IOException; -import java.security.GeneralSecurityException; import java.util.List; public class GoogleStorageDruidModule implements DruidModule { - private static final Logger LOG = new Logger(GoogleStorageDruidModule.class); - public static final String SCHEME = "google"; + private static final Logger LOG = new Logger(GoogleStorageDruidModule.class); private static final String APPLICATION_NAME = "druid-google-extensions"; @Override @@ -101,19 +95,18 @@ public class GoogleStorageDruidModule implements DruidModule @Provides @LazySingleton - public GoogleStorage getGoogleStorage(final GoogleAccountConfig config) - throws IOException, GeneralSecurityException + public GoogleStorage getGoogleStorage( + HttpTransport httpTransport, + JsonFactory jsonFactory, + HttpRequestInitializer requestInitializer + ) { LOG.info("Building Cloud Storage Client..."); - HttpTransport httpTransport = GoogleNetHttpTransport.newTrustedTransport(); - JsonFactory jsonFactory = JacksonFactory.getDefaultInstance(); - - GoogleCredential credential = GoogleCredential.getApplicationDefault(httpTransport, jsonFactory); - if (credential.createScopedRequired()) { - credential = credential.createScoped(StorageScopes.all()); - } - Storage storage = new Storage.Builder(httpTransport, jsonFactory, credential).setApplicationName(APPLICATION_NAME).build(); + Storage storage = new Storage + .Builder(httpTransport, jsonFactory, requestInitializer) + .setApplicationName(APPLICATION_NAME) + .build(); return new GoogleStorage(storage); } diff --git a/extensions-contrib/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleTaskLogs.java b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleTaskLogs.java similarity index 100% rename from extensions-contrib/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleTaskLogs.java rename to extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleTaskLogs.java diff --git a/extensions-contrib/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleTaskLogsConfig.java b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleTaskLogsConfig.java similarity index 100% rename from extensions-contrib/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleTaskLogsConfig.java rename to extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleTaskLogsConfig.java diff --git a/extensions-contrib/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleUtils.java b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleUtils.java similarity index 100% rename from extensions-contrib/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleUtils.java rename to extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleUtils.java diff --git a/extensions-contrib/google-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule b/extensions-core/google-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule similarity index 51% rename from extensions-contrib/google-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule rename to extensions-core/google-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule index 4c2440ca571..3d89b732031 100644 --- a/extensions-contrib/google-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule +++ b/extensions-core/google-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule @@ -13,23 +13,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - org.apache.druid.storage.google.GoogleStorageDruidModule diff --git a/extensions-contrib/google-extensions/src/test/java/org/apache/druid/firehose/google/StaticGoogleBlobStoreFirehoseFactoryTest.java b/extensions-core/google-extensions/src/test/java/org/apache/druid/firehose/google/StaticGoogleBlobStoreFirehoseFactoryTest.java similarity index 100% rename from extensions-contrib/google-extensions/src/test/java/org/apache/druid/firehose/google/StaticGoogleBlobStoreFirehoseFactoryTest.java rename to extensions-core/google-extensions/src/test/java/org/apache/druid/firehose/google/StaticGoogleBlobStoreFirehoseFactoryTest.java diff --git a/extensions-contrib/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleByteSourceTest.java b/extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleByteSourceTest.java similarity index 100% rename from extensions-contrib/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleByteSourceTest.java rename to extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleByteSourceTest.java diff --git a/extensions-contrib/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleDataSegmentKillerTest.java b/extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleDataSegmentKillerTest.java similarity index 100% rename from extensions-contrib/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleDataSegmentKillerTest.java rename to extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleDataSegmentKillerTest.java diff --git a/extensions-contrib/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleDataSegmentPullerTest.java b/extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleDataSegmentPullerTest.java similarity index 100% rename from extensions-contrib/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleDataSegmentPullerTest.java rename to extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleDataSegmentPullerTest.java diff --git a/extensions-contrib/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleDataSegmentPusherTest.java b/extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleDataSegmentPusherTest.java similarity index 100% rename from extensions-contrib/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleDataSegmentPusherTest.java rename to extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleDataSegmentPusherTest.java diff --git a/extensions-contrib/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleTaskLogsTest.java b/extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleTaskLogsTest.java similarity index 100% rename from extensions-contrib/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleTaskLogsTest.java rename to extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleTaskLogsTest.java diff --git a/extensions-contrib/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleUtilsTest.java b/extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleUtilsTest.java similarity index 100% rename from extensions-contrib/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleUtilsTest.java rename to extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleUtilsTest.java diff --git a/extensions-core/kinesis-indexing-service/pom.xml b/extensions-core/kinesis-indexing-service/pom.xml index 9c5b708f70e..a11a3322028 100644 --- a/extensions-core/kinesis-indexing-service/pom.xml +++ b/extensions-core/kinesis-indexing-service/pom.xml @@ -66,10 +66,23 @@ + + + org.apache.druid + druid-aws-common + ${project.parent.version} + provided + com.amazonaws aws-java-sdk-sts ${aws.sdk.version} + + + aws-java-sdk-core + com.amazonaws + + diff --git a/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3StorageDruidModule.java b/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3StorageDruidModule.java index 61d145d591e..2daaadbe7e2 100644 --- a/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3StorageDruidModule.java +++ b/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3StorageDruidModule.java @@ -34,7 +34,6 @@ import com.google.inject.Provides; import com.google.inject.multibindings.MapBinder; import org.apache.commons.lang.StringUtils; import org.apache.druid.common.aws.AWSClientConfig; -import org.apache.druid.common.aws.AWSCredentialsConfig; import org.apache.druid.common.aws.AWSEndpointConfig; import org.apache.druid.common.aws.AWSProxyConfig; import org.apache.druid.data.SearchableVersionedDataFinder; @@ -51,6 +50,7 @@ import java.net.URI; import java.util.List; /** + * */ public class S3StorageDruidModule implements DruidModule { @@ -58,6 +58,56 @@ public class S3StorageDruidModule implements DruidModule private static final Logger log = new Logger(S3StorageDruidModule.class); + private static ClientConfiguration setProxyConfig(ClientConfiguration conf, AWSProxyConfig proxyConfig) + { + if (StringUtils.isNotEmpty(proxyConfig.getHost())) { + conf.setProxyHost(proxyConfig.getHost()); + } + if (proxyConfig.getPort() != -1) { + conf.setProxyPort(proxyConfig.getPort()); + } + if (StringUtils.isNotEmpty(proxyConfig.getUsername())) { + conf.setProxyUsername(proxyConfig.getUsername()); + } + if (StringUtils.isNotEmpty(proxyConfig.getPassword())) { + conf.setProxyPassword(proxyConfig.getPassword()); + } + return conf; + } + + @Nullable + private static Protocol parseProtocol(@Nullable String protocol) + { + if (protocol == null) { + return null; + } + + if (protocol.equalsIgnoreCase("http")) { + return Protocol.HTTP; + } else if (protocol.equalsIgnoreCase("https")) { + return Protocol.HTTPS; + } else { + throw new IAE("Unknown protocol[%s]", protocol); + } + } + + private static Protocol determineProtocol(AWSClientConfig clientConfig, AWSEndpointConfig endpointConfig) + { + final Protocol protocolFromClientConfig = parseProtocol(clientConfig.getProtocol()); + final String endpointUrl = endpointConfig.getUrl(); + if (StringUtils.isNotEmpty(endpointUrl)) { + //noinspection ConstantConditions + final URI uri = URIs.parse(endpointUrl, protocolFromClientConfig.toString()); + final Protocol protocol = parseProtocol(uri.getScheme()); + if (protocol != null && (protocol != protocolFromClientConfig)) { + log.warn("[%s] protocol will be used for endpoint [%s]", protocol, endpointUrl); + } + return protocol; + } else { + return protocolFromClientConfig; + } + } + @Override public List getJacksonModules() { @@ -88,10 +138,6 @@ public class S3StorageDruidModule implements DruidModule @Override public void configure(Binder binder) { - JsonConfigProvider.bind(binder, "druid.s3", AWSCredentialsConfig.class); - JsonConfigProvider.bind(binder, "druid.s3", AWSClientConfig.class); - JsonConfigProvider.bind(binder, "druid.s3.proxy", AWSProxyConfig.class); - JsonConfigProvider.bind(binder, "druid.s3.endpoint", AWSEndpointConfig.class); MapBinder.newMapBinder(binder, String.class, SearchableVersionedDataFinder.class) .addBinding("s3") .to(S3TimestampVersionedDataFinder.class) @@ -146,54 +192,4 @@ public class S3StorageDruidModule implements DruidModule return new ServerSideEncryptingAmazonS3(builder.build(), storageConfig.getServerSideEncryption()); } - - private static ClientConfiguration setProxyConfig(ClientConfiguration conf, AWSProxyConfig proxyConfig) - { - if (StringUtils.isNotEmpty(proxyConfig.getHost())) { - conf.setProxyHost(proxyConfig.getHost()); - } - if (proxyConfig.getPort() != -1) { - conf.setProxyPort(proxyConfig.getPort()); - } - if (StringUtils.isNotEmpty(proxyConfig.getUsername())) { - conf.setProxyUsername(proxyConfig.getUsername()); - } - if (StringUtils.isNotEmpty(proxyConfig.getPassword())) { - conf.setProxyPassword(proxyConfig.getPassword()); - } - return conf; - } - - @Nullable - private static Protocol parseProtocol(@Nullable String protocol) - { - if (protocol == null) { - return null; - } - - if (protocol.equalsIgnoreCase("http")) { - return Protocol.HTTP; - } else if (protocol.equalsIgnoreCase("https")) { - return Protocol.HTTPS; - } else { - throw new IAE("Unknown protocol[%s]", protocol); - } - } - - private static Protocol determineProtocol(AWSClientConfig clientConfig, AWSEndpointConfig endpointConfig) - { - final Protocol protocolFromClientConfig = parseProtocol(clientConfig.getProtocol()); - final String endpointUrl = endpointConfig.getUrl(); - if (StringUtils.isNotEmpty(endpointUrl)) { - //noinspection ConstantConditions - final URI uri = URIs.parse(endpointUrl, protocolFromClientConfig.toString()); - final Protocol protocol = parseProtocol(uri.getScheme()); - if (protocol != null && (protocol != protocolFromClientConfig)) { - log.warn("[%s] protocol will be used for endpoint [%s]", protocol, endpointUrl); - } - return protocol; - } else { - return protocolFromClientConfig; - } - } } diff --git a/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/TestAWSCredentialsProvider.java b/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/TestAWSCredentialsProvider.java index 27534a498c6..d2ec870fc6f 100644 --- a/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/TestAWSCredentialsProvider.java +++ b/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/TestAWSCredentialsProvider.java @@ -26,8 +26,8 @@ import com.google.common.io.Files; import org.apache.druid.common.aws.AWSClientConfig; import org.apache.druid.common.aws.AWSCredentialsConfig; import org.apache.druid.common.aws.AWSEndpointConfig; +import org.apache.druid.common.aws.AWSModule; import org.apache.druid.common.aws.AWSProxyConfig; -import org.apache.druid.guice.AWSModule; import org.apache.druid.metadata.DefaultPasswordProvider; import org.easymock.EasyMock; import org.junit.Rule; diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/autoscaling/AutoScaler.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/autoscaling/AutoScaler.java index 1bbaf8b955a..76b20cf4228 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/autoscaling/AutoScaler.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/autoscaling/AutoScaler.java @@ -19,10 +19,8 @@ package org.apache.druid.indexing.overlord.autoscaling; -import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import org.apache.druid.guice.annotations.ExtensionPoint; -import org.apache.druid.indexing.overlord.autoscaling.ec2.EC2AutoScaler; import javax.annotation.Nullable; import java.util.List; @@ -31,9 +29,6 @@ import java.util.List; * The AutoScaler has the actual methods to provision and terminate worker nodes. */ @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = NoopAutoScaler.class) -@JsonSubTypes(value = { - @JsonSubTypes.Type(name = "ec2", value = EC2AutoScaler.class) -}) @ExtensionPoint public interface AutoScaler { @@ -71,6 +66,7 @@ public interface AutoScaler * This method is unused, but AutoScaler is an {@link ExtensionPoint}, so we cannot remove it. * * @param nodeIds - nodes ids + * * @return IPs associated with the node */ @SuppressWarnings("unused") diff --git a/pom.xml b/pom.xml index ddb91076a3b..1239bee82de 100644 --- a/pom.xml +++ b/pom.xml @@ -101,6 +101,8 @@ 3.4.11 2.5.7 + 1.22.0 + apache.snapshots Apache Snapshot Repository https://repository.apache.org/snapshots @@ -120,9 +122,11 @@ services integration-tests benchmarks - aws-common extendedset hll + + cloud/aws-common + cloud/gcp-common web-console extensions-core/avro-extensions @@ -142,9 +146,11 @@ extensions-core/protobuf-extensions extensions-core/lookups-cached-global extensions-core/lookups-cached-single + extensions-core/ec2-extensions extensions-core/s3-extensions extensions-core/simple-client-sslcontext extensions-core/druid-basic-security + extensions-core/google-extensions extensions-contrib/influx-extensions extensions-contrib/azure-extensions @@ -158,7 +164,6 @@ extensions-contrib/statsd-emitter extensions-contrib/orc-extensions extensions-contrib/time-min-max - extensions-contrib/google-extensions extensions-contrib/virtual-columns extensions-contrib/thrift-extensions extensions-contrib/ambari-metrics-emitter @@ -865,6 +870,66 @@ + + + + com.google.api-client + google-api-client + ${com.google.apis.client.version} + + + com.google.code.findbugs + jsr305 + + + com.fasterxml.jackson.core + jackson-core + + + com.google.guava + guava-jdk5 + + + org.apache.httpcomponents + httpcore + + + org.apache.httpcomponents + httpclient + + + + + com.google.http-client + google-http-client-jackson2 + ${com.google.apis.client.version} + + + com.fasterxml.jackson.core + jackson-core + + + org.apache.httpcomponents + httpclient + + + + + com.fasterxml.jackson.module + jackson-module-guice + ${jackson.version} + + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-databind + + + + org.testng testng diff --git a/server/pom.xml b/server/pom.xml index 67d62e6e8e7..c6c9b61f912 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -42,6 +42,11 @@ druid-aws-common ${project.parent.version} + + org.apache.druid + druid-gcp-common + ${project.parent.version} + org.apache.druid druid-console diff --git a/server/src/main/java/org/apache/druid/initialization/Initialization.java b/server/src/main/java/org/apache/druid/initialization/Initialization.java index 5082e43b2d4..1bbaee84d16 100644 --- a/server/src/main/java/org/apache/druid/initialization/Initialization.java +++ b/server/src/main/java/org/apache/druid/initialization/Initialization.java @@ -29,7 +29,6 @@ import com.google.inject.util.Modules; import org.apache.commons.io.FileUtils; import org.apache.druid.curator.CuratorModule; import org.apache.druid.curator.discovery.DiscoveryModule; -import org.apache.druid.guice.AWSModule; import org.apache.druid.guice.AnnouncerModule; import org.apache.druid.guice.CoordinatorDiscoveryModule; import org.apache.druid.guice.DruidProcessingConfigModule; @@ -378,7 +377,6 @@ public class Initialization new HttpClientModule("druid.broker.http", EscalatedClient.class), new CuratorModule(), new AnnouncerModule(), - new AWSModule(), new MetricsModule(), new SegmentWriteOutMediumModule(), new ServerModule(),