Move GCP to a core extension (#6953)

* Move GCP to a core extension

* Don't provide druid-core >.<

* Keep AWS and GCP modules separate

* Move AWSModule to its own module

* Add aws ec2 extension and more modules in more places

* Fix bad imports

* Fix test jackson module

* Include AWS and GCP core in server

* Add simple empty method comment

* Update version to 15

* One more 0.13.0-->0.15.0 change

* Fix multi-binding problem

* Grep for s3-extensions and update docs

* Update extensions.md

cloud/aws-common/pom.xml

@@ -29,6 +29,7 @@
         <groupId>org.apache.druid</groupId>
         <artifactId>druid</artifactId>
         <version>0.15.0-incubating-SNAPSHOT</version>
+        <relativePath>../../pom.xml</relativePath>
     </parent>
 
     <dependencies>

cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSModule.java

@@ -17,23 +17,22 @@
  * under the License.
  */
 
-package org.apache.druid.guice;
+package org.apache.druid.common.aws;
 
 import com.amazonaws.auth.AWSCredentialsProvider;
 import com.amazonaws.services.ec2.AmazonEC2;
 import com.amazonaws.services.ec2.AmazonEC2Client;
+import com.fasterxml.jackson.databind.Module;
 import com.google.inject.Binder;
-import com.google.inject.Module;
 import com.google.inject.Provides;
-import org.apache.druid.common.aws.AWSClientConfig;
-import org.apache.druid.common.aws.AWSCredentialsConfig;
-import org.apache.druid.common.aws.AWSCredentialsUtils;
-import org.apache.druid.common.aws.AWSEndpointConfig;
-import org.apache.druid.common.aws.AWSProxyConfig;
+import org.apache.druid.guice.JsonConfigProvider;
+import org.apache.druid.guice.LazySingleton;
+import org.apache.druid.initialization.DruidModule;
 
-/**
- */
-public class AWSModule implements Module
+import java.util.Collections;
+import java.util.List;
+
+public class AWSModule implements DruidModule
 {
   @Override
   public void configure(Binder binder)
@@ -57,4 +56,10 @@ public class AWSModule implements Module
   {
     return new AmazonEC2Client(credentials);
   }
+
+  @Override
+  public List<? extends Module> getJacksonModules()
+  {
+    return Collections.emptyList();
+  }
 }

cloud/aws-common/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule

@@ -0,0 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+org.apache.druid.common.aws.AWSModule

cloud/gcp-common/pom.xml

@@ -0,0 +1,63 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~   http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing,
+  ~ software distributed under the License is distributed on an
+  ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  ~ KIND, either express or implied.  See the License for the
+  ~ specific language governing permissions and limitations
+  ~ under the License.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>druid-gcp-common</artifactId>
+    <name>druid-gcp-common</name>
+    <description>Druid GCP Base Module</description>
+
+    <parent>
+        <groupId>org.apache.druid</groupId>
+        <artifactId>druid</artifactId>
+        <version>0.15.0-incubating-SNAPSHOT</version>
+        <relativePath>../../pom.xml</relativePath>
+    </parent>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.druid</groupId>
+            <artifactId>druid-core</artifactId>
+            <version>${project.parent.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.google.api-client</groupId>
+            <artifactId>google-api-client</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.google.http-client</groupId>
+            <artifactId>google-http-client-jackson2</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.module</groupId>
+            <artifactId>jackson-module-guice</artifactId>
+        </dependency>
+
+        <!-- Tests -->
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+</project>

cloud/gcp-common/src/main/java/org/apache/druid/common/gcp/GcpModule.java

@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.common.gcp;
+
+import com.fasterxml.jackson.databind.Module;
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpRequestInitializer;
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.common.collect.ImmutableList;
+import com.google.inject.Binder;
+import com.google.inject.Provides;
+import org.apache.druid.guice.LazySingleton;
+import org.apache.druid.initialization.DruidModule;
+
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.Collections;
+import java.util.List;
+
+public class GcpModule implements DruidModule
+{
+  @Override
+  public List<? extends Module> getJacksonModules()
+  {
+    return ImmutableList.of();
+  }
+
+  @Override
+  public void configure(Binder binder)
+  {
+    // Nothing to proactively bind
+  }
+
+
+  @Provides
+  @LazySingleton
+  public HttpRequestInitializer getHttpRequestInitializer(HttpTransport transport, JsonFactory factory)
+      throws IOException
+  {
+    GoogleCredential credential = GoogleCredential.getApplicationDefault(transport, factory);
+    if (credential.createScopedRequired()) {
+      credential = credential.createScoped(Collections.singleton("https://www.googleapis.com/auth/cloud-platform"));
+    }
+    return credential;
+  }
+
+  @Provides
+  @LazySingleton
+  public HttpTransport getHttpTransport() throws GeneralSecurityException, IOException
+  {
+    return GoogleNetHttpTransport.newTrustedTransport();
+  }
+
+  @Provides
+  @LazySingleton
+  public JsonFactory getJsonFactory()
+  {
+    return JacksonFactory.getDefaultInstance();
+  }
+}

cloud/gcp-common/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule

@@ -0,0 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+org.apache.druid.common.gcp.GcpModule

cloud/gcp-common/src/test/java/org/apache/druid/common/gcp/GcpMockModule.java

@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.common.gcp;
+
+import com.fasterxml.jackson.databind.Module;
+import com.google.api.client.googleapis.testing.auth.oauth2.MockGoogleCredential;
+import com.google.api.client.http.HttpRequestInitializer;
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.client.testing.http.MockHttpTransport;
+import com.google.common.collect.ImmutableList;
+import com.google.inject.Binder;
+import com.google.inject.Provides;
+import org.apache.druid.guice.LazySingleton;
+import org.apache.druid.initialization.DruidModule;
+
+import java.util.List;
+
+public class GcpMockModule implements DruidModule
+{
+  @Override
+  public List<? extends Module> getJacksonModules()
+  {
+    return ImmutableList.of();
+  }
+
+  @Override
+  public void configure(Binder binder)
+  {
+
+  }
+
+
+  @Provides
+  @LazySingleton
+  public HttpRequestInitializer mockRequestInitializer(
+      HttpTransport transport,
+      JsonFactory factory
+  )
+  {
+    return new MockGoogleCredential.Builder().setTransport(transport).setJsonFactory(factory).build();
+  }
+
+  @Provides
+  @LazySingleton
+  public HttpTransport buildMockTransport()
+  {
+    return new MockHttpTransport.Builder().build();
+  }
+
+  @Provides
+  @LazySingleton
+  public JsonFactory getJsonFactory()
+  {
+    return JacksonFactory.getDefaultInstance();
+  }
+}

cloud/gcp-common/src/test/java/org/apache/druid/common/gcp/GcpModuleTest.java

@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.common.gcp;
+
+import com.google.api.client.googleapis.testing.auth.oauth2.MockGoogleCredential;
+import com.google.api.client.http.HttpRequestInitializer;
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.testing.http.MockHttpTransport;
+import com.google.inject.Binder;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.google.inject.Scopes;
+import com.google.inject.util.Modules;
+import org.apache.druid.guice.LazySingleton;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class GcpModuleTest
+{
+  @Test
+  public void testSimpleModuleLoads()
+  {
+    final Injector injector = Guice.createInjector(Modules.override(new GcpModule()).with(new GcpMockModule()
+    {
+      @Override
+      public void configure(Binder binder)
+      {
+        binder.bindScope(LazySingleton.class, Scopes.SINGLETON);
+      }
+    }));
+    Assert.assertTrue(injector.getInstance(HttpRequestInitializer.class) instanceof MockGoogleCredential);
+    Assert.assertTrue(injector.getInstance(HttpTransport.class) instanceof MockHttpTransport);
+  }
+}

distribution/pom.xml

@@ -178,6 +178,10 @@
                                         <argument>-c</argument>
                                         <argument>org.apache.druid.extensions:druid-s3-extensions</argument>
                                         <argument>-c</argument>
+                                        <argument>org.apache.druid.extensions:druid-ec2-extensions</argument>
+                                        <argument>-c</argument>
+                                        <argument>org.apache.druid.extensions:druid-google-extensions</argument>
+                                        <argument>-c</argument>
                                         <argument>org.apache.druid.extensions:druid-stats</argument>
                                         <argument>-c</argument>
                                         <argument>org.apache.druid.extensions:druid-examples</argument>
@@ -300,8 +304,6 @@
                                         <argument>-c</argument>
                                         <argument>org.apache.druid.extensions.contrib:druid-rocketmq</argument>
                                         <argument>-c</argument>
-                                        <argument>org.apache.druid.extensions.contrib:druid-google-extensions</argument>
-                                        <argument>-c</argument>
                                         <argument>org.apache.druid.extensions.contrib:graphite-emitter</argument>
                                         <argument>-c</argument>
                                         <argument>org.apache.druid.extensions.contrib:druid-influx-extensions</argument>

docs/content/development/extensions.md

@@ -58,6 +58,7 @@ Core extensions are maintained by Druid committers.
 |druid-parquet-extensions|Support for data in Apache Parquet data format. Requires druid-avro-extensions to be loaded.|[link](../development/extensions-core/parquet.html)|
 |druid-protobuf-extensions| Support for data in Protobuf data format.|[link](../development/extensions-core/protobuf.html)|
 |druid-s3-extensions|Interfacing with data in AWS S3, and using S3 as deep storage.|[link](../development/extensions-core/s3.html)|
+|druid-ec2-extensions|Interfacing with AWS EC2 for autoscaling middle managers|UNDOCUMENTED|
 |druid-stats|Statistics related module including variance and standard deviation.|[link](../development/extensions-core/stats.html)|
 |mysql-metadata-storage|MySQL metadata store.|[link](../development/extensions-core/mysql.html)|
 |postgresql-metadata-storage|PostgreSQL metadata store.|[link](../development/extensions-core/postgresql.html)|

extensions-core/ec2-extensions/pom.xml

@@ -0,0 +1,64 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~   http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing,
+  ~ software distributed under the License is distributed on an
+  ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  ~ KIND, either express or implied.  See the License for the
+  ~ specific language governing permissions and limitations
+  ~ under the License.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <groupId>org.apache.druid.extensions</groupId>
+  <artifactId>druid-ec2-extensions</artifactId>
+  <name>druid-ec2-extensions</name>
+  <description>druid-ec2-extensions</description>
+
+  <parent>
+    <groupId>org.apache.druid</groupId>
+    <artifactId>druid</artifactId>
+    <version>0.15.0-incubating-SNAPSHOT</version>
+    <relativePath>../../pom.xml</relativePath>
+  </parent>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.druid</groupId>
+      <artifactId>druid-indexing-service</artifactId>
+      <version>${project.parent.version}</version>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.druid</groupId>
+      <artifactId>druid-aws-common</artifactId>
+      <version>${project.parent.version}</version>
+      <scope>provided</scope>
+    </dependency>
+
+    <!-- Tests -->
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.easymock</groupId>
+      <artifactId>easymock</artifactId>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+
+</project>

extensions-core/ec2-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2AutoScaler.java

@@ -33,6 +33,7 @@ import com.amazonaws.services.ec2.model.TerminateInstancesRequest;
 import com.fasterxml.jackson.annotation.JacksonInject;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
 import com.google.common.base.Function;
 import com.google.common.collect.FluentIterable;
 import com.google.common.collect.Lists;
@@ -46,6 +47,7 @@ import java.util.List;
 
 /**
  */
+@JsonTypeName("ec2")
 public class EC2AutoScaler implements AutoScaler<EC2EnvironmentConfig>
 {
   private static final EmittingLogger log = new EmittingLogger(EC2AutoScaler.class);

extensions-core/ec2-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2Module.java

@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.indexing.overlord.autoscaling.ec2;
+
+import com.fasterxml.jackson.databind.Module;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import com.google.inject.Binder;
+import org.apache.druid.initialization.DruidModule;
+
+import java.util.Collections;
+import java.util.List;
+
+public class EC2Module implements DruidModule
+{
+  @Override
+  public List<? extends Module> getJacksonModules()
+  {
+    return Collections.singletonList(new SimpleModule("DruidEc2Module").registerSubtypes(EC2AutoScaler.class));
+  }
+
+  @Override
+  public void configure(Binder binder)
+  {
+
+  }
+}

extensions-core/ec2-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule

@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+org.apache.druid.indexing.overlord.autoscaling.ec2.EC2Module

extensions-core/ec2-extensions/src/test/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2AutoScalerSerdeTest.java

@@ -17,13 +17,14 @@
  * under the License.
  */
 
-package org.apache.druid.indexing.overlord.autoscaling;
+package org.apache.druid.indexing.overlord.autoscaling.ec2;
 
 import com.fasterxml.jackson.databind.BeanProperty;
 import com.fasterxml.jackson.databind.DeserializationContext;
 import com.fasterxml.jackson.databind.InjectableValues;
+import com.fasterxml.jackson.databind.Module;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.druid.indexing.overlord.autoscaling.ec2.EC2AutoScaler;
+import org.apache.druid.indexing.overlord.autoscaling.AutoScaler;
 import org.apache.druid.jackson.DefaultObjectMapper;
 import org.apache.druid.java.util.common.StringUtils;
 import org.junit.Assert;
@@ -57,38 +58,6 @@ public class EC2AutoScalerSerdeTest
                       + "   \"type\" : \"ec2\"\n"
                       + "}";
 
-  @Test
-  public void testSerde() throws Exception
-  {
-    final ObjectMapper objectMapper = new DefaultObjectMapper();
-    objectMapper.setInjectableValues(
-        new InjectableValues()
-        {
-          @Override
-          public Object findInjectableValue(
-              Object o,
-              DeserializationContext deserializationContext,
-              BeanProperty beanProperty,
-              Object o1
-          )
-          {
-            return null;
-          }
-        }
-    );
-
-    final EC2AutoScaler autoScaler = (EC2AutoScaler) objectMapper.readValue(json, AutoScaler.class);
-    verifyAutoScaler(autoScaler);
-
-    final EC2AutoScaler roundTripAutoScaler = (EC2AutoScaler) objectMapper.readValue(
-        objectMapper.writeValueAsBytes(autoScaler),
-        AutoScaler.class
-    );
-    verifyAutoScaler(roundTripAutoScaler);
-
-    Assert.assertEquals("Round trip equals", autoScaler, roundTripAutoScaler);
-  }
-
   private static void verifyAutoScaler(final EC2AutoScaler autoScaler)
   {
     Assert.assertEquals(3, autoScaler.getMaxNumWorkers());
@@ -132,4 +101,37 @@ public class EC2AutoScalerSerdeTest
         )
     );
   }
+
+  @Test
+  public void testSerde() throws Exception
+  {
+    final ObjectMapper objectMapper = new DefaultObjectMapper()
+        .registerModules((Iterable<Module>) new EC2Module().getJacksonModules());
+    objectMapper.setInjectableValues(
+        new InjectableValues()
+        {
+          @Override
+          public Object findInjectableValue(
+              Object o,
+              DeserializationContext deserializationContext,
+              BeanProperty beanProperty,
+              Object o1
+          )
+          {
+            return null;
+          }
+        }
+    );
+
+    final EC2AutoScaler autoScaler = (EC2AutoScaler) objectMapper.readValue(json, AutoScaler.class);
+    verifyAutoScaler(autoScaler);
+
+    final EC2AutoScaler roundTripAutoScaler = (EC2AutoScaler) objectMapper.readValue(
+        objectMapper.writeValueAsBytes(autoScaler),
+        AutoScaler.class
+    );
+    verifyAutoScaler(roundTripAutoScaler);
+
+    Assert.assertEquals("Round trip equals", autoScaler, roundTripAutoScaler);
+  }
 }

extensions-core/ec2-extensions/src/test/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2AutoScalerTest.java

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.apache.druid.indexing.overlord.autoscaling;
+package org.apache.druid.indexing.overlord.autoscaling.ec2;
 
 import com.amazonaws.services.ec2.AmazonEC2Client;
 import com.amazonaws.services.ec2.model.DescribeInstancesRequest;
@@ -33,10 +33,8 @@ import com.google.common.collect.ContiguousSet;
 import com.google.common.collect.DiscreteDomain;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Range;
-import org.apache.druid.indexing.overlord.autoscaling.ec2.EC2AutoScaler;
-import org.apache.druid.indexing.overlord.autoscaling.ec2.EC2EnvironmentConfig;
-import org.apache.druid.indexing.overlord.autoscaling.ec2.EC2NodeData;
-import org.apache.druid.indexing.overlord.autoscaling.ec2.GalaxyEC2UserData;
+import org.apache.druid.indexing.overlord.autoscaling.AutoScalingData;
+import org.apache.druid.indexing.overlord.autoscaling.SimpleWorkerProvisioningConfig;
 import org.apache.druid.jackson.DefaultObjectMapper;
 import org.easymock.EasyMock;
 import org.junit.After;

extensions-core/ec2-extensions/src/test/java/org/apache/druid/indexing/overlord/setup/WorkerBehaviorConfigTest.java

@@ -75,6 +75,7 @@ public class WorkerBehaviorConfigTest
     );
 
     final ObjectMapper mapper = new DefaultObjectMapper();
+    mapper.registerSubtypes(EC2AutoScaler.class);
     mapper.setInjectableValues(
         new InjectableValues()
         {

extensions-core/google-extensions/pom.xml

@@ -33,6 +33,10 @@
         <relativePath>../../pom.xml</relativePath>
     </parent>
 
+    <properties>
+        <com.google.apis.storage.version>v1-rev79-${com.google.apis.client.version}</com.google.apis.storage.version>
+    </properties>
+
     <dependencies>
         <dependency>
             <groupId>org.apache.druid</groupId>
@@ -40,35 +44,21 @@
             <version>${project.parent.version}</version>
             <scope>provided</scope>
         </dependency>
+        <dependency>
+            <groupId>org.apache.druid</groupId>
+            <artifactId>druid-gcp-common</artifactId>
+            <version>${project.parent.version}</version>
+            <scope>provided</scope>
+        </dependency>
 
         <dependency>
             <groupId>com.google.apis</groupId>
             <artifactId>google-api-services-storage</artifactId>
-            <version>v1-rev79-1.22.0</version>
-        </dependency>
-        <dependency>
-            <groupId>com.google.http-client</groupId>
-            <artifactId>google-http-client-jackson2</artifactId>
-            <version>1.22.0</version>
+            <version>${com.google.apis.storage.version}</version>
             <exclusions>
                 <exclusion>
-                    <groupId>com.fasterxml.jackson.core</groupId>
-                    <artifactId>jackson-core</artifactId>
-                </exclusion>
-            </exclusions>
-        </dependency>
-        <dependency>
-            <groupId>com.fasterxml.jackson.module</groupId>
-            <artifactId>jackson-module-guice</artifactId>
-            <version>${jackson.version}</version>
-            <exclusions>
-                <exclusion>
-                    <groupId>com.fasterxml.jackson.core</groupId>
-                    <artifactId>jackson-core</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>com.fasterxml.jackson.core</groupId>
-                    <artifactId>jackson-databind</artifactId>
+                    <groupId>com.google.api-client</groupId>
+                    <artifactId>google-api-client</artifactId>
                 </exclusion>
             </exclusions>
         </dependency>

extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorageDruidModule.java

@@ -23,13 +23,10 @@ import com.fasterxml.jackson.core.Version;
 import com.fasterxml.jackson.databind.Module;
 import com.fasterxml.jackson.databind.jsontype.NamedType;
 import com.fasterxml.jackson.databind.module.SimpleModule;
-import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
-import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpRequestInitializer;
 import com.google.api.client.http.HttpTransport;
 import com.google.api.client.json.JsonFactory;
-import com.google.api.client.json.jackson2.JacksonFactory;
 import com.google.api.services.storage.Storage;
-import com.google.api.services.storage.StorageScopes;
 import com.google.common.collect.ImmutableList;
 import com.google.inject.Binder;
 import com.google.inject.Provides;
@@ -40,15 +37,12 @@ import org.apache.druid.guice.LazySingleton;
 import org.apache.druid.initialization.DruidModule;
 import org.apache.druid.java.util.common.logger.Logger;
 
-import java.io.IOException;
-import java.security.GeneralSecurityException;
 import java.util.List;
 
 public class GoogleStorageDruidModule implements DruidModule
 {
-  private static final Logger LOG = new Logger(GoogleStorageDruidModule.class);
-
   public static final String SCHEME = "google";
+  private static final Logger LOG = new Logger(GoogleStorageDruidModule.class);
   private static final String APPLICATION_NAME = "druid-google-extensions";
 
   @Override
@@ -101,19 +95,18 @@ public class GoogleStorageDruidModule implements DruidModule
 
   @Provides
   @LazySingleton
-  public GoogleStorage getGoogleStorage(final GoogleAccountConfig config)
-      throws IOException, GeneralSecurityException
+  public GoogleStorage getGoogleStorage(
+      HttpTransport httpTransport,
+      JsonFactory jsonFactory,
+      HttpRequestInitializer requestInitializer
+  )
   {
     LOG.info("Building Cloud Storage Client...");
 
-    HttpTransport httpTransport = GoogleNetHttpTransport.newTrustedTransport();
-    JsonFactory jsonFactory = JacksonFactory.getDefaultInstance();
-
-    GoogleCredential credential = GoogleCredential.getApplicationDefault(httpTransport, jsonFactory);
-    if (credential.createScopedRequired()) {
-      credential = credential.createScoped(StorageScopes.all());
-    }
-    Storage storage = new Storage.Builder(httpTransport, jsonFactory, credential).setApplicationName(APPLICATION_NAME).build();
+    Storage storage = new Storage
+        .Builder(httpTransport, jsonFactory, requestInitializer)
+        .setApplicationName(APPLICATION_NAME)
+        .build();
 
     return new GoogleStorage(storage);
   }

extensions-core/google-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule

@@ -13,23 +13,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
 org.apache.druid.storage.google.GoogleStorageDruidModule

extensions-core/kinesis-indexing-service/pom.xml

@@ -66,10 +66,23 @@
             </exclusions>
         </dependency>
 
+
+        <dependency>
+            <groupId>org.apache.druid</groupId>
+            <artifactId>druid-aws-common</artifactId>
+            <version>${project.parent.version}</version>
+            <scope>provided</scope>
+        </dependency>
         <dependency>
             <groupId>com.amazonaws</groupId>
             <artifactId>aws-java-sdk-sts</artifactId>
             <version>${aws.sdk.version}</version>
+            <exclusions>
+                <exclusion>
+                    <artifactId>aws-java-sdk-core</artifactId>
+                    <groupId>com.amazonaws</groupId>
+                </exclusion>
+            </exclusions>
         </dependency>
 
         <dependency>

extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3StorageDruidModule.java

@@ -34,7 +34,6 @@ import com.google.inject.Provides;
 import com.google.inject.multibindings.MapBinder;
 import org.apache.commons.lang.StringUtils;
 import org.apache.druid.common.aws.AWSClientConfig;
-import org.apache.druid.common.aws.AWSCredentialsConfig;
 import org.apache.druid.common.aws.AWSEndpointConfig;
 import org.apache.druid.common.aws.AWSProxyConfig;
 import org.apache.druid.data.SearchableVersionedDataFinder;
@@ -51,6 +50,7 @@ import java.net.URI;
 import java.util.List;
 
 /**
+ *
  */
 public class S3StorageDruidModule implements DruidModule
 {
@@ -58,6 +58,56 @@ public class S3StorageDruidModule implements DruidModule
 
   private static final Logger log = new Logger(S3StorageDruidModule.class);
 
+  private static ClientConfiguration setProxyConfig(ClientConfiguration conf, AWSProxyConfig proxyConfig)
+  {
+    if (StringUtils.isNotEmpty(proxyConfig.getHost())) {
+      conf.setProxyHost(proxyConfig.getHost());
+    }
+    if (proxyConfig.getPort() != -1) {
+      conf.setProxyPort(proxyConfig.getPort());
+    }
+    if (StringUtils.isNotEmpty(proxyConfig.getUsername())) {
+      conf.setProxyUsername(proxyConfig.getUsername());
+    }
+    if (StringUtils.isNotEmpty(proxyConfig.getPassword())) {
+      conf.setProxyPassword(proxyConfig.getPassword());
+    }
+    return conf;
+  }
+
+  @Nullable
+  private static Protocol parseProtocol(@Nullable String protocol)
+  {
+    if (protocol == null) {
+      return null;
+    }
+
+    if (protocol.equalsIgnoreCase("http")) {
+      return Protocol.HTTP;
+    } else if (protocol.equalsIgnoreCase("https")) {
+      return Protocol.HTTPS;
+    } else {
+      throw new IAE("Unknown protocol[%s]", protocol);
+    }
+  }
+
+  private static Protocol determineProtocol(AWSClientConfig clientConfig, AWSEndpointConfig endpointConfig)
+  {
+    final Protocol protocolFromClientConfig = parseProtocol(clientConfig.getProtocol());
+    final String endpointUrl = endpointConfig.getUrl();
+    if (StringUtils.isNotEmpty(endpointUrl)) {
+      //noinspection ConstantConditions
+      final URI uri = URIs.parse(endpointUrl, protocolFromClientConfig.toString());
+      final Protocol protocol = parseProtocol(uri.getScheme());
+      if (protocol != null && (protocol != protocolFromClientConfig)) {
+        log.warn("[%s] protocol will be used for endpoint [%s]", protocol, endpointUrl);
+      }
+      return protocol;
+    } else {
+      return protocolFromClientConfig;
+    }
+  }
+
   @Override
   public List<? extends Module> getJacksonModules()
   {
@@ -88,10 +138,6 @@ public class S3StorageDruidModule implements DruidModule
   @Override
   public void configure(Binder binder)
   {
-    JsonConfigProvider.bind(binder, "druid.s3", AWSCredentialsConfig.class);
-    JsonConfigProvider.bind(binder, "druid.s3", AWSClientConfig.class);
-    JsonConfigProvider.bind(binder, "druid.s3.proxy", AWSProxyConfig.class);
-    JsonConfigProvider.bind(binder, "druid.s3.endpoint", AWSEndpointConfig.class);
     MapBinder.newMapBinder(binder, String.class, SearchableVersionedDataFinder.class)
              .addBinding("s3")
              .to(S3TimestampVersionedDataFinder.class)
@@ -146,54 +192,4 @@ public class S3StorageDruidModule implements DruidModule
 
     return new ServerSideEncryptingAmazonS3(builder.build(), storageConfig.getServerSideEncryption());
   }
-
-  private static ClientConfiguration setProxyConfig(ClientConfiguration conf, AWSProxyConfig proxyConfig)
-  {
-    if (StringUtils.isNotEmpty(proxyConfig.getHost())) {
-      conf.setProxyHost(proxyConfig.getHost());
-    }
-    if (proxyConfig.getPort() != -1) {
-      conf.setProxyPort(proxyConfig.getPort());
-    }
-    if (StringUtils.isNotEmpty(proxyConfig.getUsername())) {
-      conf.setProxyUsername(proxyConfig.getUsername());
-    }
-    if (StringUtils.isNotEmpty(proxyConfig.getPassword())) {
-      conf.setProxyPassword(proxyConfig.getPassword());
-    }
-    return conf;
-  }
-
-  @Nullable
-  private static Protocol parseProtocol(@Nullable String protocol)
-  {
-    if (protocol == null) {
-      return null;
-    }
-
-    if (protocol.equalsIgnoreCase("http")) {
-      return Protocol.HTTP;
-    } else if (protocol.equalsIgnoreCase("https")) {
-      return Protocol.HTTPS;
-    } else {
-      throw new IAE("Unknown protocol[%s]", protocol);
-    }
-  }
-
-  private static Protocol determineProtocol(AWSClientConfig clientConfig, AWSEndpointConfig endpointConfig)
-  {
-    final Protocol protocolFromClientConfig = parseProtocol(clientConfig.getProtocol());
-    final String endpointUrl = endpointConfig.getUrl();
-    if (StringUtils.isNotEmpty(endpointUrl)) {
-      //noinspection ConstantConditions
-      final URI uri = URIs.parse(endpointUrl, protocolFromClientConfig.toString());
-      final Protocol protocol = parseProtocol(uri.getScheme());
-      if (protocol != null && (protocol != protocolFromClientConfig)) {
-        log.warn("[%s] protocol will be used for endpoint [%s]", protocol, endpointUrl);
-      }
-      return protocol;
-    } else {
-      return protocolFromClientConfig;
-    }
-  }
 }

extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/TestAWSCredentialsProvider.java

@@ -26,8 +26,8 @@ import com.google.common.io.Files;
 import org.apache.druid.common.aws.AWSClientConfig;
 import org.apache.druid.common.aws.AWSCredentialsConfig;
 import org.apache.druid.common.aws.AWSEndpointConfig;
+import org.apache.druid.common.aws.AWSModule;
 import org.apache.druid.common.aws.AWSProxyConfig;
-import org.apache.druid.guice.AWSModule;
 import org.apache.druid.metadata.DefaultPasswordProvider;
 import org.easymock.EasyMock;
 import org.junit.Rule;

indexing-service/src/main/java/org/apache/druid/indexing/overlord/autoscaling/AutoScaler.java

@@ -19,10 +19,8 @@
 
 package org.apache.druid.indexing.overlord.autoscaling;
 
-import com.fasterxml.jackson.annotation.JsonSubTypes;
 import com.fasterxml.jackson.annotation.JsonTypeInfo;
 import org.apache.druid.guice.annotations.ExtensionPoint;
-import org.apache.druid.indexing.overlord.autoscaling.ec2.EC2AutoScaler;
 
 import javax.annotation.Nullable;
 import java.util.List;
@@ -31,9 +29,6 @@ import java.util.List;
  * The AutoScaler has the actual methods to provision and terminate worker nodes.
  */
 @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = NoopAutoScaler.class)
-@JsonSubTypes(value = {
-    @JsonSubTypes.Type(name = "ec2", value = EC2AutoScaler.class)
-})
 @ExtensionPoint
 public interface AutoScaler<T>
 {
@@ -71,6 +66,7 @@ public interface AutoScaler<T>
    * This method is unused, but AutoScaler is an {@link ExtensionPoint}, so we cannot remove it.
    *
    * @param nodeIds - nodes ids
+   *
    * @return IPs associated with the node
    */
   @SuppressWarnings("unused")

pom.xml

@@ -101,6 +101,8 @@
         <!-- When upgrading ZK, edit docs and integration tests as well (integration-tests/docker-base/setup.sh) -->
         <zookeeper.version>3.4.11</zookeeper.version>
         <checkerframework.version>2.5.7</checkerframework.version>
+        <com.google.apis.client.version>1.22.0</com.google.apis.client.version>
+
         <repoOrgId>apache.snapshots</repoOrgId>
         <repoOrgName>Apache Snapshot Repository</repoOrgName>
         <repoOrgUrl>https://repository.apache.org/snapshots</repoOrgUrl>
@@ -120,9 +122,11 @@
         <module>services</module>
         <module>integration-tests</module>
         <module>benchmarks</module>
-        <module>aws-common</module>
         <module>extendedset</module>
         <module>hll</module>
+        <!-- Core cloud functionality -->
+        <module>cloud/aws-common</module>
+        <module>cloud/gcp-common</module>
         <module>web-console</module>
         <!-- Core extensions -->
         <module>extensions-core/avro-extensions</module>
@@ -142,9 +146,11 @@
         <module>extensions-core/protobuf-extensions</module>
         <module>extensions-core/lookups-cached-global</module>
         <module>extensions-core/lookups-cached-single</module>
+        <module>extensions-core/ec2-extensions</module>
         <module>extensions-core/s3-extensions</module>
         <module>extensions-core/simple-client-sslcontext</module>
         <module>extensions-core/druid-basic-security</module>
+        <module>extensions-core/google-extensions</module>
         <!-- Community extensions -->
         <module>extensions-contrib/influx-extensions</module>
         <module>extensions-contrib/azure-extensions</module>
@@ -158,7 +164,6 @@
         <module>extensions-contrib/statsd-emitter</module>
         <module>extensions-contrib/orc-extensions</module>
         <module>extensions-contrib/time-min-max</module>
-        <module>extensions-contrib/google-extensions</module>
         <module>extensions-contrib/virtual-columns</module>
         <module>extensions-contrib/thrift-extensions</module>
         <module>extensions-contrib/ambari-metrics-emitter</module>
@@ -865,6 +870,66 @@
                     </exclusion>
                 </exclusions>
             </dependency>
+
+            <!-- GCP -->
+            <dependency>
+                <groupId>com.google.api-client</groupId>
+                <artifactId>google-api-client</artifactId>
+                <version>${com.google.apis.client.version}</version>
+                <exclusions>
+                    <exclusion>
+                        <groupId>com.google.code.findbugs</groupId>
+                        <artifactId>jsr305</artifactId>
+                    </exclusion>
+                    <exclusion>
+                        <groupId>com.fasterxml.jackson.core</groupId>
+                        <artifactId>jackson-core</artifactId>
+                    </exclusion>
+                    <exclusion>
+                        <groupId>com.google.guava</groupId>
+                        <artifactId>guava-jdk5</artifactId>
+                    </exclusion>
+                    <exclusion>
+                        <groupId>org.apache.httpcomponents</groupId>
+                        <artifactId>httpcore</artifactId>
+                    </exclusion>
+                    <exclusion>
+                        <groupId>org.apache.httpcomponents</groupId>
+                        <artifactId>httpclient</artifactId>
+                    </exclusion>
+                </exclusions>
+            </dependency>
+            <dependency>
+                <groupId>com.google.http-client</groupId>
+                <artifactId>google-http-client-jackson2</artifactId>
+                <version>${com.google.apis.client.version}</version>
+                <exclusions>
+                    <exclusion>
+                        <groupId>com.fasterxml.jackson.core</groupId>
+                        <artifactId>jackson-core</artifactId>
+                    </exclusion>
+                    <exclusion>
+                        <groupId>org.apache.httpcomponents</groupId>
+                        <artifactId>httpclient</artifactId>
+                    </exclusion>
+                </exclusions>
+            </dependency>
+            <dependency>
+                <groupId>com.fasterxml.jackson.module</groupId>
+                <artifactId>jackson-module-guice</artifactId>
+                <version>${jackson.version}</version>
+                <exclusions>
+                    <exclusion>
+                        <groupId>com.fasterxml.jackson.core</groupId>
+                        <artifactId>jackson-core</artifactId>
+                    </exclusion>
+                    <exclusion>
+                        <groupId>com.fasterxml.jackson.core</groupId>
+                        <artifactId>jackson-databind</artifactId>
+                    </exclusion>
+                </exclusions>
+            </dependency>
+
             <dependency>
                 <groupId>org.testng</groupId>
                 <artifactId>testng</artifactId>

server/pom.xml

@@ -42,6 +42,11 @@
             <artifactId>druid-aws-common</artifactId>
             <version>${project.parent.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.apache.druid</groupId>
+            <artifactId>druid-gcp-common</artifactId>
+            <version>${project.parent.version}</version>
+        </dependency>
         <dependency>
             <groupId>org.apache.druid</groupId>
             <artifactId>druid-console</artifactId>

server/src/main/java/org/apache/druid/initialization/Initialization.java

@@ -29,7 +29,6 @@ import com.google.inject.util.Modules;
 import org.apache.commons.io.FileUtils;
 import org.apache.druid.curator.CuratorModule;
 import org.apache.druid.curator.discovery.DiscoveryModule;
-import org.apache.druid.guice.AWSModule;
 import org.apache.druid.guice.AnnouncerModule;
 import org.apache.druid.guice.CoordinatorDiscoveryModule;
 import org.apache.druid.guice.DruidProcessingConfigModule;
@@ -378,7 +377,6 @@ public class Initialization
         new HttpClientModule("druid.broker.http", EscalatedClient.class),
         new CuratorModule(),
         new AnnouncerModule(),
-        new AWSModule(),
         new MetricsModule(),
         new SegmentWriteOutMediumModule(),
         new ServerModule(),