diff --git a/distribution/pom.xml b/distribution/pom.xml index cdde9dbe537..eb6d4e20f56 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -71,10 +71,6 @@ -c io.druid.extensions:druid-examples -c - io.druid.extensions:druid-azure-extensions - -c - io.druid.extensions:druid-cassandra-storage - -c io.druid.extensions:druid-datasketches -c io.druid.extensions:druid-hdfs-storage @@ -83,8 +79,6 @@ -c io.druid.extensions:druid-kafka-eight -c - io.druid.extensions:druid-kafka-eight-simple-consumer - -c io.druid.extensions:druid-kafka-extraction-namespace -c io.druid.extensions:mysql-metadata-storage @@ -93,11 +87,7 @@ -c io.druid.extensions:postgresql-metadata-storage -c - io.druid.extensions:druid-rabbitmq - -c io.druid.extensions:druid-s3-extensions - -c - io.druid.extensions:druid-cloudfiles-extensions diff --git a/docs/content/dependencies/deep-storage.md b/docs/content/dependencies/deep-storage.md index 95e51b3dee1..880242e74c2 100644 --- a/docs/content/dependencies/deep-storage.md +++ b/docs/content/dependencies/deep-storage.md @@ -4,9 +4,7 @@ layout: doc_page # Deep Storage Deep storage is where segments are stored. It is a storage mechanism that Druid does not provide. This deep storage infrastructure defines the level of durability of your data, as long as Druid nodes can see this storage infrastructure and get at the segments stored on it, you will not lose data no matter how many Druid nodes you lose. If segments disappear from this storage layer, then you will lose whatever data those segments represented. -## Production Tested Deep Stores - -### Local Mount +## Local Mount A local mount can be used for storage of segments as well. This allows you to use just your local file system or anything else that can be mount locally like NFS, Ceph, etc. This is the default deep storage implementation. @@ -21,14 +19,12 @@ Note that you should generally set `druid.storage.storageDirectory` to something If you are using the Hadoop indexer in local mode, then just give it a local file as your output directory and it will work. - -### S3-compatible +## S3-compatible S3-compatible deep storage is basically either S3 or something like Google Storage which exposes the same API as S3. S3 configuration parameters are - |Property|Possible Values|Description|Default| |--------|---------------|-----------|-------| |`druid.s3.accessKey`||S3 access key.|Must be set.| @@ -36,7 +32,7 @@ S3 configuration parameters are |`druid.storage.bucket`||Bucket to store in.|Must be set.| |`druid.storage.baseKey`||Base key prefix to use, i.e. what directory.|Must be set.| -### HDFS +## HDFS In order to use hdfs for deep storage, you need to set the following configuration in your common configs. @@ -46,44 +42,3 @@ In order to use hdfs for deep storage, you need to set the following configurati |`druid.storage.storageDirectory`||Directory for storing segments.|Must be set.| If you are using the Hadoop indexer, set your output directory to be a location on Hadoop and it will work - -## Community Contributed Deep Stores - -### Cassandra - -[Apache Cassandra](http://www.datastax.com/what-we-offer/products-services/datastax-enterprise/apache-cassandra) can also be leveraged for deep storage. This requires some additional druid configuration as well as setting up the necessary schema within a Cassandra keystore. - -Please note that this is a community contributed module and does not support Cassandra 2.x or hadoop-based batch indexing. For more information on using Cassandra as deep storage, see [Cassandra Deep Storage](../dependencies/cassandra-deep-storage.html). - -## Azure - -[Microsoft Azure Storage](http://azure.microsoft.com/en-us/services/storage/) is another option for deep storage. This requires some additional druid configuration. - -|Property|Possible Values|Description|Default| -|--------|---------------|-----------|-------| -|`druid.storage.type`|azure||Must be set.| -|`druid.azure.account`||Azure Storage account name.|Must be set.| -|`druid.azure.key`||Azure Storage account key.|Must be set.| -|`druid.azure.container`||Azure Storage container name.|Must be set.| -|`druid.azure.protocol`|http or https||https| -|`druid.azure.maxTries`||Number of tries before cancel an Azure operation.|3| - -Please note that this is a community contributed module. See [Azure Services](http://azure.microsoft.com/en-us/pricing/free-trial/) for more information. - -### Rackspace - -[Rackspace Cloud Files](http://www.rackspace.com/cloud/files/) is another option for deep storage. This requires some additional druid configuration. - -|Property|Possible Values|Description|Default| -|--------|---------------|-----------|-------| -|`druid.storage.type`|cloudfiles||Must be set.| -|`druid.storage.region`||Rackspace Cloud Files region.|Must be set.| -|`druid.storage.container`||Rackspace Cloud Files container name.|Must be set.| -|`druid.storage.basePath`||Rackspace Cloud Files base path to use in the container.|Must be set.| -|`druid.storage.operationMaxRetries`||Number of tries before cancel a Rackspace operation.|10| -|`druid.cloudfiles.userName`||Rackspace Cloud username|Must be set.| -|`druid.cloudfiles.apiKey`||Rackspace Cloud api key.|Must be set.| -|`druid.cloudfiles.provider`|rackspace-cloudfiles-us,rackspace-cloudfiles-uk|Name of the provider depending on the region.|Must be set.| -|`druid.cloudfiles.useServiceNet`|true,false|Whether to use the internal service net.|true| - -Please note that this is a community contributed module. diff --git a/docs/content/development/community-extensions/azure.md b/docs/content/development/community-extensions/azure.md new file mode 100644 index 00000000000..a23d9d769ae --- /dev/null +++ b/docs/content/development/community-extensions/azure.md @@ -0,0 +1,62 @@ +--- +layout: doc_page +--- + +# Microsoft Azure + +## Deep Storage + +[Microsoft Azure Storage](http://azure.microsoft.com/en-us/services/storage/) is another option for deep storage. This requires some additional druid configuration. + +|Property|Possible Values|Description|Default| +|--------|---------------|-----------|-------| +|`druid.storage.type`|azure||Must be set.| +|`druid.azure.account`||Azure Storage account name.|Must be set.| +|`druid.azure.key`||Azure Storage account key.|Must be set.| +|`druid.azure.container`||Azure Storage container name.|Must be set.| +|`druid.azure.protocol`|http or https||https| +|`druid.azure.maxTries`||Number of tries before cancel an Azure operation.|3| + +See [Azure Services](http://azure.microsoft.com/en-us/pricing/free-trial/) for more information. + +## Firehose + +#### StaticAzureBlobStoreFirehose + +This firehose ingests events, similar to the StaticS3Firehose, but from an Azure Blob Store. + +Data is newline delimited, with one JSON object per line and parsed as per the `InputRowParser` configuration. + +The storage account is shared with the one used for Azure deep storage functionality, but blobs can be in a different container. + +As with the S3 blobstore, it is assumed to be gzipped if the extension ends in .gz + +Sample spec: + +```json +"firehose" : { + "type" : "static-azure-blobstore", + "blobs": [ + { + "container": "container", + "path": "/path/to/your/file.json" + }, + { + "container": "anothercontainer", + "path": "/another/path.json" + } + ] +} +``` + +|property|description|default|required?| +|--------|-----------|-------|---------| +|type|This should be "static-azure-blobstore".|N/A|yes| +|blobs|JSON array of [Azure blobs](https://msdn.microsoft.com/en-us/library/azure/ee691964.aspx).|N/A|yes| + +Azure Blobs: + +|property|description|default|required?| +|--------|-----------|-------|---------| +|container|Name of the azure [container](https://azure.microsoft.com/en-us/documentation/articles/storage-dotnet-how-to-use-blobs/#create-a-container)|N/A|yes| +|path|The path where data is located.|N/A|yes| diff --git a/docs/content/development/community-extensions/cassandra.md b/docs/content/development/community-extensions/cassandra.md new file mode 100644 index 00000000000..c0c02872202 --- /dev/null +++ b/docs/content/development/community-extensions/cassandra.md @@ -0,0 +1,9 @@ +--- +layout: doc_page +--- + +# Apache Cassandra + +[Apache Cassandra](http://www.datastax.com/what-we-offer/products-services/datastax-enterprise/apache-cassandra) can also +be leveraged for deep storage. This requires some additional druid configuration as well as setting up the necessary +schema within a Cassandra keystore. diff --git a/docs/content/development/community-extensions/cloudfiles.md b/docs/content/development/community-extensions/cloudfiles.md new file mode 100644 index 00000000000..de56c6cc79c --- /dev/null +++ b/docs/content/development/community-extensions/cloudfiles.md @@ -0,0 +1,65 @@ +--- +layout: doc_page +--- + +# Rackspace Cloud Files + +## Deep Storage + +[Rackspace Cloud Files](http://www.rackspace.com/cloud/files/) is another option for deep storage. This requires some additional druid configuration. + +|Property|Possible Values|Description|Default| +|--------|---------------|-----------|-------| +|`druid.storage.type`|cloudfiles||Must be set.| +|`druid.storage.region`||Rackspace Cloud Files region.|Must be set.| +|`druid.storage.container`||Rackspace Cloud Files container name.|Must be set.| +|`druid.storage.basePath`||Rackspace Cloud Files base path to use in the container.|Must be set.| +|`druid.storage.operationMaxRetries`||Number of tries before cancel a Rackspace operation.|10| +|`druid.cloudfiles.userName`||Rackspace Cloud username|Must be set.| +|`druid.cloudfiles.apiKey`||Rackspace Cloud api key.|Must be set.| +|`druid.cloudfiles.provider`|rackspace-cloudfiles-us,rackspace-cloudfiles-uk|Name of the provider depending on the region.|Must be set.| +|`druid.cloudfiles.useServiceNet`|true,false|Whether to use the internal service net.|true| + +## Firehose + +#### StaticCloudFilesFirehose + +This firehose ingests events, similar to the StaticAzureBlobStoreFirehose, but from Rackspace's Cloud Files. + +Data is newline delimited, with one JSON object per line and parsed as per the `InputRowParser` configuration. + +The storage account is shared with the one used for Racksapce's Cloud Files deep storage functionality, but blobs can be in a different region and container. + +As with the Azure blobstore, it is assumed to be gzipped if the extension ends in .gz + +Sample spec: + +```json +"firehose" : { + "type" : "static-cloudfiles", + "blobs": [ + { + "region": "DFW" + "container": "container", + "path": "/path/to/your/file.json" + }, + { + "region": "ORD" + "container": "anothercontainer", + "path": "/another/path.json" + } + ] +} +``` + +|property|description|default|required?| +|--------|-----------|-------|---------| +|type|This should be "static-cloudfiles".|N/A|yes| +|blobs|JSON array of Cloud Files blobs.|N/A|yes| + +Cloud Files Blobs: + +|property|description|default|required?| +|--------|-----------|-------|---------| +|container|Name of the Cloud Files container|N/A|yes| +|path|The path where data is located.|N/A|yes| diff --git a/extensions/graphite-emitter/README.md b/docs/content/development/community-extensions/graphite.md similarity index 97% rename from extensions/graphite-emitter/README.md rename to docs/content/development/community-extensions/graphite.md index 9ae350c2b03..9eba568944b 100644 --- a/extensions/graphite-emitter/README.md +++ b/docs/content/development/community-extensions/graphite.md @@ -1,9 +1,15 @@ -## introduction +--- +layout: doc_page +--- + +# Graphite Emitter + +## Introduction This extension emits druid metrics to a graphite carbon server. Events are sent after been [pickled](http://graphite.readthedocs.org/en/latest/feeding-carbon.html#the-pickle-protocol); the size of the batch is configurable. -## configuration +## Configuration All the configuration parameters for graphite emitter are under `druid.emitter.graphite`. @@ -69,4 +75,4 @@ druid.emitter.graphite.eventConverter={"type":"whiteList", "namespacePrefix": "d ``` -**Druid emits a huge number of metrics we highly recommend to use the `whiteList` converter** \ No newline at end of file +**Druid emits a huge number of metrics we highly recommend to use the `whiteList` converter** diff --git a/docs/content/development/kafka-simple-consumer-firehose.md b/docs/content/development/community-extensions/kafka-simple.md similarity index 74% rename from docs/content/development/kafka-simple-consumer-firehose.md rename to docs/content/development/community-extensions/kafka-simple.md index 39971bc7abd..69a66dfb97b 100644 --- a/docs/content/development/kafka-simple-consumer-firehose.md +++ b/docs/content/development/community-extensions/kafka-simple.md @@ -1,19 +1,23 @@ --- layout: doc_page --- -# KafkaSimpleConsumerFirehose + +# Kafka Simple Consumer + +## Firehose + This is an experimental firehose to ingest data from kafka using kafka simple consumer api. Currently, this firehose would only work inside standalone realtime nodes. -The configuration for KafkaSimpleConsumerFirehose is similar to the KafkaFirehose [Kafka firehose example](../ingestion/stream-pull.html#realtime-specfile), except `firehose` should be replaced with `firehoseV2` like this: +The configuration for KafkaSimpleConsumerFirehose is similar to the Kafka Eight Firehose , except `firehose` should be replaced with `firehoseV2` like this: ```json "firehoseV2": { -"type" : "kafka-0.8-v2", -"brokerList" : ["localhost:4443"], -"queueBufferLength":10001, -"resetOffsetToEarliest":"true", -"partitionIdList" : ["0"], -"clientId" : "localclient", -"feed": "wikipedia" + "type" : "kafka-0.8-v2", + "brokerList" : ["localhost:4443"], + "queueBufferLength":10001, + "resetOffsetToEarliest":"true", + "partitionIdList" : ["0"], + "clientId" : "localclient", + "feed": "wikipedia" } ``` diff --git a/docs/content/development/community-extensions/rabbitmq.md b/docs/content/development/community-extensions/rabbitmq.md new file mode 100644 index 00000000000..d047022e839 --- /dev/null +++ b/docs/content/development/community-extensions/rabbitmq.md @@ -0,0 +1,59 @@ +--- +layout: doc_page +--- + +# RabbitMQ + +## Firehose + +#### RabbitMQFirehose + +This firehose ingests events from a define rabbit-mq queue. + +**Note:** Add **amqp-client-3.2.1.jar** to lib directory of druid to use this firehose. + +A sample spec for rabbitmq firehose: + +```json +"firehose" : { + "type" : "rabbitmq", + "connection" : { + "host": "localhost", + "port": "5672", + "username": "test-dude", + "password": "test-word", + "virtualHost": "test-vhost", + "uri": "amqp://mqserver:1234/vhost" + }, + "config" : { + "exchange": "test-exchange", + "queue" : "druidtest", + "routingKey": "#", + "durable": "true", + "exclusive": "false", + "autoDelete": "false", + "maxRetries": "10", + "retryIntervalSeconds": "1", + "maxDurationSeconds": "300" + } +} +``` + +|property|description|default|required?| +|--------|-----------|-------|---------| +|type|This should be "rabbitmq"|N/A|yes| +|host|The hostname of the RabbitMQ broker to connect to|localhost|no| +|port|The port number to connect to on the RabbitMQ broker|5672|no| +|username|The username to use to connect to RabbitMQ|guest|no| +|password|The password to use to connect to RabbitMQ|guest|no| +|virtualHost|The virtual host to connect to|/|no| +|uri|The URI string to use to connect to RabbitMQ| |no| +|exchange|The exchange to connect to| |yes| +|queue|The queue to connect to or create| |yes| +|routingKey|The routing key to use to bind the queue to the exchange| |yes| +|durable|Whether the queue should be durable|false|no| +|exclusive|Whether the queue should be exclusive|false|no| +|autoDelete|Whether the queue should auto-delete on disconnect|false|no| +|maxRetries|The max number of reconnection retry attempts| |yes| +|retryIntervalSeconds|The reconnection interval| |yes| +|maxDurationSeconds|The max duration of trying to reconnect| |yes| diff --git a/docs/content/development/community-extensions/rocketmq.md b/docs/content/development/community-extensions/rocketmq.md new file mode 100644 index 00000000000..f5195ac4ddf --- /dev/null +++ b/docs/content/development/community-extensions/rocketmq.md @@ -0,0 +1,7 @@ +--- +layout: doc_page +--- + +# RocketMQ + +Original author: [https://github.com/lizhanhui](https://github.com/lizhanhui). diff --git a/docs/content/development/extensions.md b/docs/content/development/extensions.md new file mode 100644 index 00000000000..2cf0d835a73 --- /dev/null +++ b/docs/content/development/extensions.md @@ -0,0 +1,54 @@ +--- +layout: doc_page +--- + +# Druid extensions + +Druid implements an extension system that allows for adding functionality at runtime. Extensions +are commonly used to add support for deep storages (like HDFS and S3), metadata stores (like MySQL +and PostgreSQL), new aggregators, new input formats, and so on. + +Production clusters will generally use at least two extensions; one for deep storage and one for a +metadata store. Many clusters will also use additional extensions. + +## Including extensions + +Please see [here](../operations/including-extensions.html). + +## Core extensions + +Core extensions are maintained by Druid committers. + +|Name|Description|Docs| +|----|-----------|----| +|druid-avro-extensions|Support for data in Apache Avro data format.|[link](../ingestion/index.html)| +|druid-datasketches|Support for approximate counts and set operations with [DataSketches](http://datasketches.github.io/).|[link](../development/datasketches-aggregators.html)| +|druid-hdfs-storage|Support for data in Apache Avro data format.|[link](../ingestion/index.html)| +|druid-histogram|HDFS deep storage.|[link](../dependencies/deep-storage.html#hdfs)| +|druid-kafka-eight|Kafka ingest firehose (high level consumer).|[link](../ingestion/firehose.html#kafkaeightfirehose)| +|druid-kafka-extraction-namespace|Kafka namespaced lookup.|[link](../querying/lookups.html#kafka-namespaced-lookup)| +|druid-namespace-lookup|Namespaced lookups.|[link](../querying/lookups.html)| +|druid-s3-extensions|S3 deep storage.|[link](../dependencies/deep-storage.html#s3-compatible/)| +|mysql-metadata-storage|MySQL metadata store.|[link](../dependencies/metadata-storage.html#setting-up-mysql)| +|postgresql-metadata-storage|PostgreSQL metadata store.|[link](../dependencies/metadata-storage.html#setting-up-postgresql)| + +# Community Extensions + +A number of community members have contributed their own extensions to Druid that are not packaged with the default Druid tarball. +Community extensions are not maintained by Druid committers, although we accept patches from community members using these extensions. +If you'd like to take on maintenance for a community extension, please post on [druid-development group](https://groups.google.com/forum/#!forum/druid-development) to let us know! + +|Name|Description|Docs| +|----|-----------|----| +|druid-azure-extensions|Microsoft Azure deep storage.|[link](../development/community-extensions/azure.html)| +|druid-cassandra-storage|Apache Cassandra deep storage.|[link](../development/community-extensions/cassandra.html)| +|druid-cloudfiles-extensions|Rackspace Cloudfiles deep storage and firehose.|[link](../development/community-extensions/cloudfiles.html)| +|druid-kafka-eight-simpleConsumer|Kafka ingest firehose (low level consumer).|[link](../development/community-extensions/kafka-simple.html)| +|druid-rabbitmq|RabbitMQ firehose.|[link](../development/community-extensions/rabbitmq.html)| +|druid-rocketmq|RocketMQ firehose.|[link](../development/community-extensions/rocketmq.html)| +|graphite-emitter|Graphite metrics emitter|[link](../development/community-extensions/graphite.html)| + +## Promoting Community Extension to Core Extension + +Please [let us know](https://groups.google.com/forum/#!forum/druid-development) if you'd like an extension to be promoted to core. +If we see a community extension actively supported by the community, we can promote it to core based on community feedback. diff --git a/docs/content/ingestion/firehose.md b/docs/content/ingestion/firehose.md index 7c0a47d8ae1..12b33d59c28 100644 --- a/docs/content/ingestion/firehose.md +++ b/docs/content/ingestion/firehose.md @@ -3,6 +3,7 @@ layout: doc_page --- # Druid Firehoses + Firehoses describe the data stream source. They are pluggable and thus the configuration schema can and will vary based on the `type` of the firehose. | Field | Type | Description | Required | @@ -14,8 +15,7 @@ We describe the configuration of the [Kafka firehose example](../ingestion/strea - `consumerProps` is a map of properties for the Kafka consumer. The JSON object is converted into a Properties object and passed along to the Kafka consumer. - `feed` is the feed that the Kafka consumer should read from. -Available Firehoses -------------------- +## Available Firehoses There are several firehoses readily available in Druid, some are meant for examples, others can be used directly in a production environment. @@ -66,160 +66,6 @@ Sample spec: |type|This should be "static-s3"|N/A|yes| |uris|JSON array of URIs where s3 files to be ingested are located.|N/A|yes| -#### StaticAzureBlobStoreFirehose - -This firehose ingests events, similar to the StaticS3Firehose, but from an Azure Blob Store. - -Data is newline delimited, with one JSON object per line and parsed as per the `InputRowParser` configuration. - -The storage account is shared with the one used for Azure deep storage functionality, but blobs can be in a different container. - -As with the S3 blobstore, it is assumed to be gzipped if the extension ends in .gz - -Sample spec: - -```json -"firehose" : { - "type" : "static-azure-blobstore", - "blobs": [ - { - "container": "container", - "path": "/path/to/your/file.json" - }, - { - "container": "anothercontainer", - "path": "/another/path.json" - } - ] -} -``` - -|property|description|default|required?| -|--------|-----------|-------|---------| -|type|This should be "static-azure-blobstore".|N/A|yes| -|blobs|JSON array of [Azure blobs](https://msdn.microsoft.com/en-us/library/azure/ee691964.aspx).|N/A|yes| - -Azure Blobs: - -|property|description|default|required?| -|--------|-----------|-------|---------| -|container|Name of the azure [container](https://azure.microsoft.com/en-us/documentation/articles/storage-dotnet-how-to-use-blobs/#create-a-container)|N/A|yes| -|path|The path where data is located.|N/A|yes| - -#### StaticCloudFilesFirehose - -This firehose ingests events, similar to the StaticAzureBlobStoreFirehose, but from Rackspace's Cloud Files. - -Data is newline delimited, with one JSON object per line and parsed as per the `InputRowParser` configuration. - -The storage account is shared with the one used for Racksapce's Cloud Files deep storage functionality, but blobs can be in a different region and container. - -As with the Azure blobstore, it is assumed to be gzipped if the extension ends in .gz - -Sample spec: - -```json -"firehose" : { - "type" : "static-cloudfiles", - "blobs": [ - { - "region": "DFW" - "container": "container", - "path": "/path/to/your/file.json" - }, - { - "region": "ORD" - "container": "anothercontainer", - "path": "/another/path.json" - } - ] -} -``` - -|property|description|default|required?| -|--------|-----------|-------|---------| -|type|This should be "static-cloudfiles".|N/A|yes| -|blobs|JSON array of Cloud Files blobs.|N/A|yes| - -Cloud Files Blobs: - -|property|description|default|required?| -|--------|-----------|-------|---------| -|container|Name of the Cloud Files container|N/A|yes| -|path|The path where data is located.|N/A|yes| - -#### TwitterSpritzerFirehose - -This firehose connects directly to the twitter spritzer data stream. - -Sample spec: - -```json -"firehose" : { - "type" : "twitzer", - "maxEventCount": -1, - "maxRunMinutes": 0 -} -``` - -|property|description|default|required?| -|--------|-----------|-------|---------| -|type|This should be "twitzer"|N/A|yes| -|maxEventCount|max events to receive, -1 is infinite, 0 means nothing is delivered; use this to prevent infinite space consumption or to prevent getting throttled at an inconvenient time.|N/A|yes| -|maxRunMinutes|maximum number of minutes to fetch Twitter events. Use this to prevent getting throttled at an inconvenient time. If zero or less, no time limit for run.|N/A|yes| - -#### RabbitMQFirehose - -This firehose ingests events from a define rabbit-mq queue. - -**Note:** Add **amqp-client-3.2.1.jar** to lib directory of druid to use this firehose. - -A sample spec for rabbitmq firehose: - -```json -"firehose" : { - "type" : "rabbitmq", - "connection" : { - "host": "localhost", - "port": "5672", - "username": "test-dude", - "password": "test-word", - "virtualHost": "test-vhost", - "uri": "amqp://mqserver:1234/vhost" - }, - "config" : { - "exchange": "test-exchange", - "queue" : "druidtest", - "routingKey": "#", - "durable": "true", - "exclusive": "false", - "autoDelete": "false", - "maxRetries": "10", - "retryIntervalSeconds": "1", - "maxDurationSeconds": "300" - } -} -``` - -|property|description|default|required?| -|--------|-----------|-------|---------| -|type|This should be "rabbitmq"|N/A|yes| -|host|The hostname of the RabbitMQ broker to connect to|localhost|no| -|port|The port number to connect to on the RabbitMQ broker|5672|no| -|username|The username to use to connect to RabbitMQ|guest|no| -|password|The password to use to connect to RabbitMQ|guest|no| -|virtualHost|The virtual host to connect to|/|no| -|uri|The URI string to use to connect to RabbitMQ| |no| -|exchange|The exchange to connect to| |yes| -|queue|The queue to connect to or create| |yes| -|routingKey|The routing key to use to bind the queue to the exchange| |yes| -|durable|Whether the queue should be durable|false|no| -|exclusive|Whether the queue should be exclusive|false|no| -|autoDelete|Whether the queue should auto-delete on disconnect|false|no| -|maxRetries|The max number of reconnection retry attempts| |yes| -|retryIntervalSeconds|The reconnection interval| |yes| -|maxDurationSeconds|The max duration of trying to reconnect| |yes| - #### LocalFirehose This Firehose can be used to read the data from files on local disk. @@ -280,7 +126,6 @@ This can be used to merge data from more than one firehose. |type|This should be "combining"|yes| |delegates|list of firehoses to combine data from|yes| - #### EventReceiverFirehose EventReceiverFirehoseFactory can be used to ingest events using an http endpoint. @@ -324,4 +169,23 @@ An example is shown below: |type|This should be "timed"|yes| |shutoffTime|time at which the firehose should shut down, in ISO8601 format|yes| |delegate|firehose to use|yes| -======= + +#### TwitterSpritzerFirehose + +This firehose connects directly to the twitter spritzer data stream. + +Sample spec: + +```json +"firehose" : { + "type" : "twitzer", + "maxEventCount": -1, + "maxRunMinutes": 0 +} +``` + +|property|description|default|required?| +|--------|-----------|-------|---------| +|type|This should be "twitzer"|N/A|yes| +|maxEventCount|max events to receive, -1 is infinite, 0 means nothing is delivered; use this to prevent infinite space consumption or to prevent getting throttled at an inconvenient time.|N/A|yes| +|maxRunMinutes|maximum number of minutes to fetch Twitter events. Use this to prevent getting throttled at an inconvenient time. If zero or less, no time limit for run.|N/A|yes| diff --git a/docs/content/operations/including-extensions.md b/docs/content/operations/including-extensions.md index b8f059b88c3..68e538eeb9c 100644 --- a/docs/content/operations/including-extensions.md +++ b/docs/content/operations/including-extensions.md @@ -3,7 +3,8 @@ layout: doc_page --- # Including Extensions -Druid uses a module system that allows for the addition of extensions at runtime. To instruct Druid to load extensions, follow the steps below. +Druid uses a module system that allows for the addition of extensions at runtime. Core extensions are bundled with the Druid tarball. +Community extensions be download locally via the [pull-deps](../operations/pull-deps.html) tool. ## Download extensions diff --git a/docs/content/operations/pull-deps.md b/docs/content/operations/pull-deps.md index ad5c794f24b..d8fdab30e7f 100644 --- a/docs/content/operations/pull-deps.md +++ b/docs/content/operations/pull-deps.md @@ -44,9 +44,11 @@ To run `pull-deps`, you should Example: -Suppose you want to download ```druid-examples```, ```mysql-metadata-storage``` and ```hadoop-client```(both 2.3.0 and 2.4.0) with a specific version, you can run `pull-deps` command with `-c io.druid.extensions:druid-examples:0.9.0`, `-c io.druid.extensions:mysql-metadata-storage:0.9.0`, `-h org.apache.hadoop:hadoop-client:2.3.0` and `-h org.apache.hadoop:hadoop-client:2.4.0`, an example command would be: +Suppose you want to download ```druid-rabbitmq```, ```mysql-metadata-storage``` and ```hadoop-client```(both 2.3.0 and 2.4.0) with a specific version, you can run `pull-deps` command with `-c io.druid.extensions:druid-examples:0.9.0`, `-c io.druid.extensions:mysql-metadata-storage:0.9.0`, `-h org.apache.hadoop:hadoop-client:2.3.0` and `-h org.apache.hadoop:hadoop-client:2.4.0`, an example command would be: -```java -classpath "/my/druid/library/*" io.druid.cli.Main tools pull-deps --clean -c io.druid.extensions:mysql-metadata-storage:0.9.0 -c io.druid.extensions:druid-examples:0.9.0 -h org.apache.hadoop:hadoop-client:2.3.0 -h org.apache.hadoop:hadoop-client:2.4.0``` +``` +java -classpath "/my/druid/library/*" io.druid.cli.Main tools pull-deps --clean -c io.druid.extensions:mysql-metadata-storage:0.9.0 -c io.druid.extensions.contrib:druid-rabbitmq:0.9.0 -h org.apache.hadoop:hadoop-client:2.3.0 -h org.apache.hadoop:hadoop-client:2.4.0 +``` Because `--clean` is supplied, this command will first remove the directories specified at `druid.extensions.directory` and `druid.extensions.hadoopDependenciesDir`, then recreate them and start downloading the extensions there. After finishing downloading, if you go to the extension directories you specified, you will see @@ -90,8 +92,14 @@ hadoop-dependencies/ ..... lots of jars ``` -Note that if you specify `--defaultVersion`, you don't have to put version information in the coordinate. For example, if you want both `druid-examples` and `mysql-metadata-storage` to use version `0.9.0`, you can change the command above to +Note that if you specify `--defaultVersion`, you don't have to put version information in the coordinate. For example, if you want both `druid-rabbitmq` and `mysql-metadata-storage` to use version `0.9.0`, you can change the command above to ``` -java -classpath "/my/druid/library/*" io.druid.cli.Main tools pull-deps --defaultVersion 0.9.0 --clean -c io.druid.extensions:mysql-metadata-storage -c io.druid.extensions:druid-examples -h org.apache.hadoop:hadoop-client:2.3.0 -h org.apache.hadoop:hadoop-client:2.4.0 +java -classpath "/my/druid/library/*" io.druid.cli.Main tools pull-deps --defaultVersion 0.9.0 --clean -c io.druid.extensions:mysql-metadata-storage -c io.druid.extensions.contrib:druid-rabbitmq -h org.apache.hadoop:hadoop-client:2.3.0 -h org.apache.hadoop:hadoop-client:2.4.0 ``` + +
+Please note to use the pull-deps tool you must know the Maven groupId, artifactId, and version of your extension. + +For Druid community extensions listed here, the groupId is "io.druid.extensions.contrib" and the artifactId is the name of the extension. +
diff --git a/docs/content/toc.md b/docs/content/toc.md index 42b7d0b994a..81b7061ceca 100644 --- a/docs/content/toc.md +++ b/docs/content/toc.md @@ -85,8 +85,9 @@ ## Development * [Overview](../development/overview.html) - * [Libraries](../development/libraries.html) + * [Libraries](../development/libraries.html) * [Extending Druid](../development/modules.html) + * [Available Modules](../development/extensions.html) * [Build From Source](../development/build.html) * [Versioning](../development/versioning.html) * [Integration](../development/integrating-druid-with-other-technologies.html) @@ -95,8 +96,7 @@ * [Geographic Queries](../development/geo.html) * [Approximate Histograms and Quantiles](../development/approximate-histograms.html) * [Datasketches](../development/datasketches-aggregators.html) - * [Router](../development/router.html) - * [Kafka Simple Consumer Firehose](../development/kafka-simple-consumer-firehose.html) + * [Router](../development/router.html) ## Misc * [Papers & Talks](../misc/papers-and-talks.html) diff --git a/extensions-contrib/README.md b/extensions-contrib/README.md new file mode 100644 index 00000000000..943b30a6759 --- /dev/null +++ b/extensions-contrib/README.md @@ -0,0 +1,6 @@ +# Community Extensions + +Please contribute all community extensions in this directory and include a doc of how your extension can be used under /docs/content/development/community-extensions/. + +Please note that community extensions are maintained by their original contributors and are not packaged with the core Druid distribution. +If you'd like to take on maintenance for a community extension, please post on [druid-development group](https://groups.google.com/forum/#!forum/druid-development) to let us know! diff --git a/extensions/azure-extensions/pom.xml b/extensions-contrib/azure-extensions/pom.xml similarity index 64% rename from extensions/azure-extensions/pom.xml rename to extensions-contrib/azure-extensions/pom.xml index df00f7ee7d3..823454fb6c4 100644 --- a/extensions/azure-extensions/pom.xml +++ b/extensions-contrib/azure-extensions/pom.xml @@ -1,25 +1,28 @@ 4.0.0 - io.druid.extensions + + io.druid.extensions.contrib druid-azure-extensions druid-azure-extensions druid-azure-extensions diff --git a/extensions/azure-extensions/src/main/java/io/druid/firehose/azure/AzureBlob.java b/extensions-contrib/azure-extensions/src/main/java/io/druid/firehose/azure/AzureBlob.java similarity index 100% rename from extensions/azure-extensions/src/main/java/io/druid/firehose/azure/AzureBlob.java rename to extensions-contrib/azure-extensions/src/main/java/io/druid/firehose/azure/AzureBlob.java diff --git a/extensions/azure-extensions/src/main/java/io/druid/firehose/azure/StaticAzureBlobStoreFirehoseFactory.java b/extensions-contrib/azure-extensions/src/main/java/io/druid/firehose/azure/StaticAzureBlobStoreFirehoseFactory.java similarity index 100% rename from extensions/azure-extensions/src/main/java/io/druid/firehose/azure/StaticAzureBlobStoreFirehoseFactory.java rename to extensions-contrib/azure-extensions/src/main/java/io/druid/firehose/azure/StaticAzureBlobStoreFirehoseFactory.java diff --git a/extensions/azure-extensions/src/main/java/io/druid/storage/azure/AzureAccountConfig.java b/extensions-contrib/azure-extensions/src/main/java/io/druid/storage/azure/AzureAccountConfig.java similarity index 100% rename from extensions/azure-extensions/src/main/java/io/druid/storage/azure/AzureAccountConfig.java rename to extensions-contrib/azure-extensions/src/main/java/io/druid/storage/azure/AzureAccountConfig.java diff --git a/extensions/azure-extensions/src/main/java/io/druid/storage/azure/AzureByteSource.java b/extensions-contrib/azure-extensions/src/main/java/io/druid/storage/azure/AzureByteSource.java similarity index 100% rename from extensions/azure-extensions/src/main/java/io/druid/storage/azure/AzureByteSource.java rename to extensions-contrib/azure-extensions/src/main/java/io/druid/storage/azure/AzureByteSource.java diff --git a/extensions/azure-extensions/src/main/java/io/druid/storage/azure/AzureDataSegmentKiller.java b/extensions-contrib/azure-extensions/src/main/java/io/druid/storage/azure/AzureDataSegmentKiller.java similarity index 100% rename from extensions/azure-extensions/src/main/java/io/druid/storage/azure/AzureDataSegmentKiller.java rename to extensions-contrib/azure-extensions/src/main/java/io/druid/storage/azure/AzureDataSegmentKiller.java diff --git a/extensions/azure-extensions/src/main/java/io/druid/storage/azure/AzureDataSegmentPuller.java b/extensions-contrib/azure-extensions/src/main/java/io/druid/storage/azure/AzureDataSegmentPuller.java similarity index 100% rename from extensions/azure-extensions/src/main/java/io/druid/storage/azure/AzureDataSegmentPuller.java rename to extensions-contrib/azure-extensions/src/main/java/io/druid/storage/azure/AzureDataSegmentPuller.java diff --git a/extensions/azure-extensions/src/main/java/io/druid/storage/azure/AzureDataSegmentPusher.java b/extensions-contrib/azure-extensions/src/main/java/io/druid/storage/azure/AzureDataSegmentPusher.java similarity index 100% rename from extensions/azure-extensions/src/main/java/io/druid/storage/azure/AzureDataSegmentPusher.java rename to extensions-contrib/azure-extensions/src/main/java/io/druid/storage/azure/AzureDataSegmentPusher.java diff --git a/extensions/azure-extensions/src/main/java/io/druid/storage/azure/AzureLoadSpec.java b/extensions-contrib/azure-extensions/src/main/java/io/druid/storage/azure/AzureLoadSpec.java similarity index 100% rename from extensions/azure-extensions/src/main/java/io/druid/storage/azure/AzureLoadSpec.java rename to extensions-contrib/azure-extensions/src/main/java/io/druid/storage/azure/AzureLoadSpec.java diff --git a/extensions/azure-extensions/src/main/java/io/druid/storage/azure/AzureStorage.java b/extensions-contrib/azure-extensions/src/main/java/io/druid/storage/azure/AzureStorage.java similarity index 100% rename from extensions/azure-extensions/src/main/java/io/druid/storage/azure/AzureStorage.java rename to extensions-contrib/azure-extensions/src/main/java/io/druid/storage/azure/AzureStorage.java diff --git a/extensions/azure-extensions/src/main/java/io/druid/storage/azure/AzureStorageDruidModule.java b/extensions-contrib/azure-extensions/src/main/java/io/druid/storage/azure/AzureStorageDruidModule.java similarity index 100% rename from extensions/azure-extensions/src/main/java/io/druid/storage/azure/AzureStorageDruidModule.java rename to extensions-contrib/azure-extensions/src/main/java/io/druid/storage/azure/AzureStorageDruidModule.java diff --git a/extensions/azure-extensions/src/main/java/io/druid/storage/azure/AzureTaskLogs.java b/extensions-contrib/azure-extensions/src/main/java/io/druid/storage/azure/AzureTaskLogs.java similarity index 100% rename from extensions/azure-extensions/src/main/java/io/druid/storage/azure/AzureTaskLogs.java rename to extensions-contrib/azure-extensions/src/main/java/io/druid/storage/azure/AzureTaskLogs.java diff --git a/extensions/azure-extensions/src/main/java/io/druid/storage/azure/AzureTaskLogsConfig.java b/extensions-contrib/azure-extensions/src/main/java/io/druid/storage/azure/AzureTaskLogsConfig.java similarity index 100% rename from extensions/azure-extensions/src/main/java/io/druid/storage/azure/AzureTaskLogsConfig.java rename to extensions-contrib/azure-extensions/src/main/java/io/druid/storage/azure/AzureTaskLogsConfig.java diff --git a/extensions/azure-extensions/src/main/java/io/druid/storage/azure/AzureUtils.java b/extensions-contrib/azure-extensions/src/main/java/io/druid/storage/azure/AzureUtils.java similarity index 100% rename from extensions/azure-extensions/src/main/java/io/druid/storage/azure/AzureUtils.java rename to extensions-contrib/azure-extensions/src/main/java/io/druid/storage/azure/AzureUtils.java diff --git a/extensions/azure-extensions/src/main/resources/META-INF/services/io.druid.initialization.DruidModule b/extensions-contrib/azure-extensions/src/main/resources/META-INF/services/io.druid.initialization.DruidModule similarity index 100% rename from extensions/azure-extensions/src/main/resources/META-INF/services/io.druid.initialization.DruidModule rename to extensions-contrib/azure-extensions/src/main/resources/META-INF/services/io.druid.initialization.DruidModule diff --git a/extensions/azure-extensions/src/test/java/io/druid/storage/azure/AzureByteSourceTest.java b/extensions-contrib/azure-extensions/src/test/java/io/druid/storage/azure/AzureByteSourceTest.java similarity index 100% rename from extensions/azure-extensions/src/test/java/io/druid/storage/azure/AzureByteSourceTest.java rename to extensions-contrib/azure-extensions/src/test/java/io/druid/storage/azure/AzureByteSourceTest.java diff --git a/extensions/azure-extensions/src/test/java/io/druid/storage/azure/AzureDataSegmentKillerTest.java b/extensions-contrib/azure-extensions/src/test/java/io/druid/storage/azure/AzureDataSegmentKillerTest.java similarity index 100% rename from extensions/azure-extensions/src/test/java/io/druid/storage/azure/AzureDataSegmentKillerTest.java rename to extensions-contrib/azure-extensions/src/test/java/io/druid/storage/azure/AzureDataSegmentKillerTest.java diff --git a/extensions/azure-extensions/src/test/java/io/druid/storage/azure/AzureDataSegmentPullerTest.java b/extensions-contrib/azure-extensions/src/test/java/io/druid/storage/azure/AzureDataSegmentPullerTest.java similarity index 100% rename from extensions/azure-extensions/src/test/java/io/druid/storage/azure/AzureDataSegmentPullerTest.java rename to extensions-contrib/azure-extensions/src/test/java/io/druid/storage/azure/AzureDataSegmentPullerTest.java diff --git a/extensions/azure-extensions/src/test/java/io/druid/storage/azure/AzureDataSegmentPusherTest.java b/extensions-contrib/azure-extensions/src/test/java/io/druid/storage/azure/AzureDataSegmentPusherTest.java similarity index 100% rename from extensions/azure-extensions/src/test/java/io/druid/storage/azure/AzureDataSegmentPusherTest.java rename to extensions-contrib/azure-extensions/src/test/java/io/druid/storage/azure/AzureDataSegmentPusherTest.java diff --git a/extensions/azure-extensions/src/test/java/io/druid/storage/azure/AzureTaskLogsTest.java b/extensions-contrib/azure-extensions/src/test/java/io/druid/storage/azure/AzureTaskLogsTest.java similarity index 100% rename from extensions/azure-extensions/src/test/java/io/druid/storage/azure/AzureTaskLogsTest.java rename to extensions-contrib/azure-extensions/src/test/java/io/druid/storage/azure/AzureTaskLogsTest.java diff --git a/extensions/azure-extensions/src/test/java/io/druid/storage/azure/AzureTestUtils.java b/extensions-contrib/azure-extensions/src/test/java/io/druid/storage/azure/AzureTestUtils.java similarity index 100% rename from extensions/azure-extensions/src/test/java/io/druid/storage/azure/AzureTestUtils.java rename to extensions-contrib/azure-extensions/src/test/java/io/druid/storage/azure/AzureTestUtils.java diff --git a/extensions/cassandra-storage/pom.xml b/extensions-contrib/cassandra-storage/pom.xml similarity index 60% rename from extensions/cassandra-storage/pom.xml rename to extensions-contrib/cassandra-storage/pom.xml index a855e114540..812f47895d4 100644 --- a/extensions/cassandra-storage/pom.xml +++ b/extensions-contrib/cassandra-storage/pom.xml @@ -1,24 +1,27 @@ 4.0.0 - io.druid.extensions + + io.druid.extensions.contrib druid-cassandra-storage druid-cassandra-storage druid-cassandra-storage diff --git a/extensions/cassandra-storage/src/main/java/io/druid/storage/cassandra/CassandraDataSegmentConfig.java b/extensions-contrib/cassandra-storage/src/main/java/io/druid/storage/cassandra/CassandraDataSegmentConfig.java similarity index 100% rename from extensions/cassandra-storage/src/main/java/io/druid/storage/cassandra/CassandraDataSegmentConfig.java rename to extensions-contrib/cassandra-storage/src/main/java/io/druid/storage/cassandra/CassandraDataSegmentConfig.java diff --git a/extensions/cassandra-storage/src/main/java/io/druid/storage/cassandra/CassandraDataSegmentPuller.java b/extensions-contrib/cassandra-storage/src/main/java/io/druid/storage/cassandra/CassandraDataSegmentPuller.java similarity index 100% rename from extensions/cassandra-storage/src/main/java/io/druid/storage/cassandra/CassandraDataSegmentPuller.java rename to extensions-contrib/cassandra-storage/src/main/java/io/druid/storage/cassandra/CassandraDataSegmentPuller.java diff --git a/extensions/cassandra-storage/src/main/java/io/druid/storage/cassandra/CassandraDataSegmentPusher.java b/extensions-contrib/cassandra-storage/src/main/java/io/druid/storage/cassandra/CassandraDataSegmentPusher.java similarity index 100% rename from extensions/cassandra-storage/src/main/java/io/druid/storage/cassandra/CassandraDataSegmentPusher.java rename to extensions-contrib/cassandra-storage/src/main/java/io/druid/storage/cassandra/CassandraDataSegmentPusher.java diff --git a/extensions/cassandra-storage/src/main/java/io/druid/storage/cassandra/CassandraDruidModule.java b/extensions-contrib/cassandra-storage/src/main/java/io/druid/storage/cassandra/CassandraDruidModule.java similarity index 100% rename from extensions/cassandra-storage/src/main/java/io/druid/storage/cassandra/CassandraDruidModule.java rename to extensions-contrib/cassandra-storage/src/main/java/io/druid/storage/cassandra/CassandraDruidModule.java diff --git a/extensions/cassandra-storage/src/main/java/io/druid/storage/cassandra/CassandraLoadSpec.java b/extensions-contrib/cassandra-storage/src/main/java/io/druid/storage/cassandra/CassandraLoadSpec.java similarity index 100% rename from extensions/cassandra-storage/src/main/java/io/druid/storage/cassandra/CassandraLoadSpec.java rename to extensions-contrib/cassandra-storage/src/main/java/io/druid/storage/cassandra/CassandraLoadSpec.java diff --git a/extensions/cassandra-storage/src/main/java/io/druid/storage/cassandra/CassandraStorage.java b/extensions-contrib/cassandra-storage/src/main/java/io/druid/storage/cassandra/CassandraStorage.java similarity index 100% rename from extensions/cassandra-storage/src/main/java/io/druid/storage/cassandra/CassandraStorage.java rename to extensions-contrib/cassandra-storage/src/main/java/io/druid/storage/cassandra/CassandraStorage.java diff --git a/extensions/cassandra-storage/src/main/resources/META-INF/services/io.druid.initialization.DruidModule b/extensions-contrib/cassandra-storage/src/main/resources/META-INF/services/io.druid.initialization.DruidModule similarity index 100% rename from extensions/cassandra-storage/src/main/resources/META-INF/services/io.druid.initialization.DruidModule rename to extensions-contrib/cassandra-storage/src/main/resources/META-INF/services/io.druid.initialization.DruidModule diff --git a/extensions/cloudfiles-extensions/pom.xml b/extensions-contrib/cloudfiles-extensions/pom.xml similarity index 74% rename from extensions/cloudfiles-extensions/pom.xml rename to extensions-contrib/cloudfiles-extensions/pom.xml index 0766b5765d9..f554d8f7555 100644 --- a/extensions/cloudfiles-extensions/pom.xml +++ b/extensions-contrib/cloudfiles-extensions/pom.xml @@ -1,29 +1,28 @@ - - + 4.0.0 - io.druid.extensions + + io.druid.extensions.contrib druid-cloudfiles-extensions druid-cloudfiles-extensions druid-cloudfiles-extensions diff --git a/extensions/cloudfiles-extensions/src/main/java/io/druid/firehose/cloudfiles/CloudFilesBlob.java b/extensions-contrib/cloudfiles-extensions/src/main/java/io/druid/firehose/cloudfiles/CloudFilesBlob.java similarity index 64% rename from extensions/cloudfiles-extensions/src/main/java/io/druid/firehose/cloudfiles/CloudFilesBlob.java rename to extensions-contrib/cloudfiles-extensions/src/main/java/io/druid/firehose/cloudfiles/CloudFilesBlob.java index 1f16ce1c49b..fc25ba9b676 100644 --- a/extensions/cloudfiles-extensions/src/main/java/io/druid/firehose/cloudfiles/CloudFilesBlob.java +++ b/extensions-contrib/cloudfiles-extensions/src/main/java/io/druid/firehose/cloudfiles/CloudFilesBlob.java @@ -25,41 +25,41 @@ import javax.validation.constraints.NotNull; public class CloudFilesBlob { - @JsonProperty - @NotNull - private String container = null; + @JsonProperty + @NotNull + private String container = null; - @JsonProperty - @NotNull - private String path = null; + @JsonProperty + @NotNull + private String path = null; - @JsonProperty - @NotNull - private String region = null; + @JsonProperty + @NotNull + private String region = null; - public CloudFilesBlob() - { - } + public CloudFilesBlob() + { + } - public CloudFilesBlob(String container, String path, String region) - { - this.container = container; - this.path = path; - this.region = region; - } + public CloudFilesBlob(String container, String path, String region) + { + this.container = container; + this.path = path; + this.region = region; + } - public String getContainer() - { - return container; - } + public String getContainer() + { + return container; + } - public String getPath() - { - return path; - } + public String getPath() + { + return path; + } - public String getRegion() - { - return region; - } + public String getRegion() + { + return region; + } } diff --git a/extensions/cloudfiles-extensions/src/main/java/io/druid/firehose/cloudfiles/CloudFilesFirehoseDruidModule.java b/extensions-contrib/cloudfiles-extensions/src/main/java/io/druid/firehose/cloudfiles/CloudFilesFirehoseDruidModule.java similarity index 80% rename from extensions/cloudfiles-extensions/src/main/java/io/druid/firehose/cloudfiles/CloudFilesFirehoseDruidModule.java rename to extensions-contrib/cloudfiles-extensions/src/main/java/io/druid/firehose/cloudfiles/CloudFilesFirehoseDruidModule.java index 99300da94a0..01c564a4f47 100644 --- a/extensions/cloudfiles-extensions/src/main/java/io/druid/firehose/cloudfiles/CloudFilesFirehoseDruidModule.java +++ b/extensions-contrib/cloudfiles-extensions/src/main/java/io/druid/firehose/cloudfiles/CloudFilesFirehoseDruidModule.java @@ -19,31 +19,30 @@ package io.druid.firehose.cloudfiles; -import java.util.List; - import com.fasterxml.jackson.databind.Module; import com.fasterxml.jackson.databind.jsontype.NamedType; import com.fasterxml.jackson.databind.module.SimpleModule; import com.google.common.collect.ImmutableList; import com.google.inject.Binder; - import io.druid.initialization.DruidModule; +import java.util.List; + public class CloudFilesFirehoseDruidModule implements DruidModule { - @Override - public List getJacksonModules() - { - return ImmutableList.of( - new SimpleModule().registerSubtypes( - new NamedType(StaticCloudFilesFirehoseFactory.class, "static-cloudfiles"))); - } + @Override + public List getJacksonModules() + { + return ImmutableList.of( + new SimpleModule().registerSubtypes( + new NamedType(StaticCloudFilesFirehoseFactory.class, "staticcloudfiles"))); + } - @Override - public void configure(Binder arg0) - { + @Override + public void configure(Binder arg0) + { - } + } } diff --git a/extensions-contrib/cloudfiles-extensions/src/main/java/io/druid/firehose/cloudfiles/StaticCloudFilesFirehoseFactory.java b/extensions-contrib/cloudfiles-extensions/src/main/java/io/druid/firehose/cloudfiles/StaticCloudFilesFirehoseFactory.java new file mode 100644 index 00000000000..e6dfb20ac21 --- /dev/null +++ b/extensions-contrib/cloudfiles-extensions/src/main/java/io/druid/firehose/cloudfiles/StaticCloudFilesFirehoseFactory.java @@ -0,0 +1,138 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.firehose.cloudfiles; + +import com.fasterxml.jackson.annotation.JacksonInject; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Charsets; +import com.google.common.base.Preconditions; +import com.google.common.base.Throwables; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import com.metamx.common.CompressionUtils; +import com.metamx.common.logger.Logger; +import com.metamx.common.parsers.ParseException; +import io.druid.data.input.Firehose; +import io.druid.data.input.FirehoseFactory; +import io.druid.data.input.impl.FileIteratingFirehose; +import io.druid.data.input.impl.StringInputRowParser; +import io.druid.storage.cloudfiles.CloudFilesByteSource; +import io.druid.storage.cloudfiles.CloudFilesObjectApiProxy; +import org.apache.commons.io.IOUtils; +import org.apache.commons.io.LineIterator; +import org.jclouds.rackspace.cloudfiles.v1.CloudFilesApi; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; + +public class StaticCloudFilesFirehoseFactory implements FirehoseFactory +{ + private static final Logger log = new Logger(StaticCloudFilesFirehoseFactory.class); + + private final CloudFilesApi cloudFilesApi; + private final List blobs; + + @JsonCreator + public StaticCloudFilesFirehoseFactory( + @JacksonInject("objectApi") CloudFilesApi cloudFilesApi, + @JsonProperty("blobs") CloudFilesBlob[] blobs + ) + { + this.cloudFilesApi = cloudFilesApi; + this.blobs = ImmutableList.copyOf(blobs); + } + + @JsonProperty + public List getBlobs() + { + return blobs; + } + + @Override + public Firehose connect(StringInputRowParser stringInputRowParser) throws IOException, ParseException + { + Preconditions.checkNotNull(cloudFilesApi, "null cloudFilesApi"); + + final LinkedList objectQueue = Lists.newLinkedList(blobs); + + return new FileIteratingFirehose( + new Iterator() + { + + @Override + public boolean hasNext() + { + return !objectQueue.isEmpty(); + } + + @Override + public LineIterator next() + { + final CloudFilesBlob nextURI = objectQueue.poll(); + + final String region = nextURI.getRegion(); + final String container = nextURI.getContainer(); + final String path = nextURI.getPath(); + + log.info("Retrieving file from region[%s], container[%s] and path [%s]", + region, container, path + ); + CloudFilesObjectApiProxy objectApi = new CloudFilesObjectApiProxy( + cloudFilesApi, region, container); + final CloudFilesByteSource byteSource = new CloudFilesByteSource(objectApi, path); + + try { + final InputStream innerInputStream = byteSource.openStream(); + final InputStream outerInputStream = path.endsWith(".gz") + ? CompressionUtils.gzipInputStream(innerInputStream) + : innerInputStream; + + return IOUtils.lineIterator( + new BufferedReader( + new InputStreamReader(outerInputStream, Charsets.UTF_8))); + } + catch (IOException e) { + log.error(e, + "Exception opening container[%s] blob[%s] from region[%s]", + container, path, region + ); + + throw Throwables.propagate(e); + } + } + + @Override + public void remove() + { + throw new UnsupportedOperationException(); + } + + }, + stringInputRowParser + ); + } + +} diff --git a/extensions/cloudfiles-extensions/src/main/java/io/druid/storage/cloudfiles/CloudFilesAccountConfig.java b/extensions-contrib/cloudfiles-extensions/src/main/java/io/druid/storage/cloudfiles/CloudFilesAccountConfig.java similarity index 100% rename from extensions/cloudfiles-extensions/src/main/java/io/druid/storage/cloudfiles/CloudFilesAccountConfig.java rename to extensions-contrib/cloudfiles-extensions/src/main/java/io/druid/storage/cloudfiles/CloudFilesAccountConfig.java diff --git a/extensions/cloudfiles-extensions/src/main/java/io/druid/storage/cloudfiles/CloudFilesByteSource.java b/extensions-contrib/cloudfiles-extensions/src/main/java/io/druid/storage/cloudfiles/CloudFilesByteSource.java similarity index 100% rename from extensions/cloudfiles-extensions/src/main/java/io/druid/storage/cloudfiles/CloudFilesByteSource.java rename to extensions-contrib/cloudfiles-extensions/src/main/java/io/druid/storage/cloudfiles/CloudFilesByteSource.java diff --git a/extensions/cloudfiles-extensions/src/main/java/io/druid/storage/cloudfiles/CloudFilesDataSegmentPuller.java b/extensions-contrib/cloudfiles-extensions/src/main/java/io/druid/storage/cloudfiles/CloudFilesDataSegmentPuller.java similarity index 100% rename from extensions/cloudfiles-extensions/src/main/java/io/druid/storage/cloudfiles/CloudFilesDataSegmentPuller.java rename to extensions-contrib/cloudfiles-extensions/src/main/java/io/druid/storage/cloudfiles/CloudFilesDataSegmentPuller.java diff --git a/extensions/cloudfiles-extensions/src/main/java/io/druid/storage/cloudfiles/CloudFilesDataSegmentPusher.java b/extensions-contrib/cloudfiles-extensions/src/main/java/io/druid/storage/cloudfiles/CloudFilesDataSegmentPusher.java similarity index 100% rename from extensions/cloudfiles-extensions/src/main/java/io/druid/storage/cloudfiles/CloudFilesDataSegmentPusher.java rename to extensions-contrib/cloudfiles-extensions/src/main/java/io/druid/storage/cloudfiles/CloudFilesDataSegmentPusher.java diff --git a/extensions/cloudfiles-extensions/src/main/java/io/druid/storage/cloudfiles/CloudFilesDataSegmentPusherConfig.java b/extensions-contrib/cloudfiles-extensions/src/main/java/io/druid/storage/cloudfiles/CloudFilesDataSegmentPusherConfig.java similarity index 100% rename from extensions/cloudfiles-extensions/src/main/java/io/druid/storage/cloudfiles/CloudFilesDataSegmentPusherConfig.java rename to extensions-contrib/cloudfiles-extensions/src/main/java/io/druid/storage/cloudfiles/CloudFilesDataSegmentPusherConfig.java diff --git a/extensions/cloudfiles-extensions/src/main/java/io/druid/storage/cloudfiles/CloudFilesLoadSpec.java b/extensions-contrib/cloudfiles-extensions/src/main/java/io/druid/storage/cloudfiles/CloudFilesLoadSpec.java similarity index 100% rename from extensions/cloudfiles-extensions/src/main/java/io/druid/storage/cloudfiles/CloudFilesLoadSpec.java rename to extensions-contrib/cloudfiles-extensions/src/main/java/io/druid/storage/cloudfiles/CloudFilesLoadSpec.java diff --git a/extensions/cloudfiles-extensions/src/main/java/io/druid/storage/cloudfiles/CloudFilesObject.java b/extensions-contrib/cloudfiles-extensions/src/main/java/io/druid/storage/cloudfiles/CloudFilesObject.java similarity index 100% rename from extensions/cloudfiles-extensions/src/main/java/io/druid/storage/cloudfiles/CloudFilesObject.java rename to extensions-contrib/cloudfiles-extensions/src/main/java/io/druid/storage/cloudfiles/CloudFilesObject.java diff --git a/extensions/cloudfiles-extensions/src/main/java/io/druid/storage/cloudfiles/CloudFilesObjectApiProxy.java b/extensions-contrib/cloudfiles-extensions/src/main/java/io/druid/storage/cloudfiles/CloudFilesObjectApiProxy.java similarity index 100% rename from extensions/cloudfiles-extensions/src/main/java/io/druid/storage/cloudfiles/CloudFilesObjectApiProxy.java rename to extensions-contrib/cloudfiles-extensions/src/main/java/io/druid/storage/cloudfiles/CloudFilesObjectApiProxy.java diff --git a/extensions/cloudfiles-extensions/src/main/java/io/druid/storage/cloudfiles/CloudFilesStorageDruidModule.java b/extensions-contrib/cloudfiles-extensions/src/main/java/io/druid/storage/cloudfiles/CloudFilesStorageDruidModule.java similarity index 100% rename from extensions/cloudfiles-extensions/src/main/java/io/druid/storage/cloudfiles/CloudFilesStorageDruidModule.java rename to extensions-contrib/cloudfiles-extensions/src/main/java/io/druid/storage/cloudfiles/CloudFilesStorageDruidModule.java diff --git a/extensions/cloudfiles-extensions/src/main/java/io/druid/storage/cloudfiles/CloudFilesUtils.java b/extensions-contrib/cloudfiles-extensions/src/main/java/io/druid/storage/cloudfiles/CloudFilesUtils.java similarity index 100% rename from extensions/cloudfiles-extensions/src/main/java/io/druid/storage/cloudfiles/CloudFilesUtils.java rename to extensions-contrib/cloudfiles-extensions/src/main/java/io/druid/storage/cloudfiles/CloudFilesUtils.java diff --git a/extensions/cloudfiles-extensions/src/main/resources/META-INF/services/io.druid.initialization.DruidModule b/extensions-contrib/cloudfiles-extensions/src/main/resources/META-INF/services/io.druid.initialization.DruidModule similarity index 100% rename from extensions/cloudfiles-extensions/src/main/resources/META-INF/services/io.druid.initialization.DruidModule rename to extensions-contrib/cloudfiles-extensions/src/main/resources/META-INF/services/io.druid.initialization.DruidModule diff --git a/extensions/cloudfiles-extensions/src/test/java/io/druid/storage/cloudfiles/CloudFilesByteSourceTest.java b/extensions-contrib/cloudfiles-extensions/src/test/java/io/druid/storage/cloudfiles/CloudFilesByteSourceTest.java similarity index 100% rename from extensions/cloudfiles-extensions/src/test/java/io/druid/storage/cloudfiles/CloudFilesByteSourceTest.java rename to extensions-contrib/cloudfiles-extensions/src/test/java/io/druid/storage/cloudfiles/CloudFilesByteSourceTest.java diff --git a/extensions/cloudfiles-extensions/src/test/java/io/druid/storage/cloudfiles/CloudFilesDataSegmentPusherTest.java b/extensions-contrib/cloudfiles-extensions/src/test/java/io/druid/storage/cloudfiles/CloudFilesDataSegmentPusherTest.java similarity index 100% rename from extensions/cloudfiles-extensions/src/test/java/io/druid/storage/cloudfiles/CloudFilesDataSegmentPusherTest.java rename to extensions-contrib/cloudfiles-extensions/src/test/java/io/druid/storage/cloudfiles/CloudFilesDataSegmentPusherTest.java diff --git a/extensions/cloudfiles-extensions/src/test/java/io/druid/storage/cloudfiles/CloudFilesObjectApiProxyTest.java b/extensions-contrib/cloudfiles-extensions/src/test/java/io/druid/storage/cloudfiles/CloudFilesObjectApiProxyTest.java similarity index 100% rename from extensions/cloudfiles-extensions/src/test/java/io/druid/storage/cloudfiles/CloudFilesObjectApiProxyTest.java rename to extensions-contrib/cloudfiles-extensions/src/test/java/io/druid/storage/cloudfiles/CloudFilesObjectApiProxyTest.java diff --git a/extensions/druid-rocketmq/pom.xml b/extensions-contrib/druid-rocketmq/pom.xml similarity index 97% rename from extensions/druid-rocketmq/pom.xml rename to extensions-contrib/druid-rocketmq/pom.xml index adcc523744c..191919d3ac6 100644 --- a/extensions/druid-rocketmq/pom.xml +++ b/extensions-contrib/druid-rocketmq/pom.xml @@ -27,6 +27,8 @@ 0.9.0-SNAPSHOT ../../pom.xml + + io.druid.extensions.contrib druid-rocketmq diff --git a/extensions/druid-rocketmq/src/main/java/io/druid/firehose/rocketmq/RocketMQDruidModule.java b/extensions-contrib/druid-rocketmq/src/main/java/io/druid/firehose/rocketmq/RocketMQDruidModule.java similarity index 100% rename from extensions/druid-rocketmq/src/main/java/io/druid/firehose/rocketmq/RocketMQDruidModule.java rename to extensions-contrib/druid-rocketmq/src/main/java/io/druid/firehose/rocketmq/RocketMQDruidModule.java diff --git a/extensions/druid-rocketmq/src/main/java/io/druid/firehose/rocketmq/RocketMQFirehoseFactory.java b/extensions-contrib/druid-rocketmq/src/main/java/io/druid/firehose/rocketmq/RocketMQFirehoseFactory.java similarity index 100% rename from extensions/druid-rocketmq/src/main/java/io/druid/firehose/rocketmq/RocketMQFirehoseFactory.java rename to extensions-contrib/druid-rocketmq/src/main/java/io/druid/firehose/rocketmq/RocketMQFirehoseFactory.java diff --git a/extensions/graphite-emitter/pom.xml b/extensions-contrib/graphite-emitter/pom.xml similarity index 72% rename from extensions/graphite-emitter/pom.xml rename to extensions-contrib/graphite-emitter/pom.xml index ebc1a469309..fd41131377a 100644 --- a/extensions/graphite-emitter/pom.xml +++ b/extensions-contrib/graphite-emitter/pom.xml @@ -1,22 +1,22 @@ + ~ Licensed to Metamarkets Group Inc. (Metamarkets) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. Metamarkets licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, + ~ software distributed under the License is distributed on an + ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + ~ KIND, either express or implied. See the License for the + ~ specific language governing permissions and limitations + ~ under the License. + --> @@ -29,7 +29,7 @@ ../../pom.xml - io.druid.extensions + io.druid.extensions.contrib graphite-emitter graphite-emitter Druid emitter extension to support graphite diff --git a/extensions/graphite-emitter/src/main/java/io/druid/emitter/graphite/DruidToGraphiteEventConverter.java b/extensions-contrib/graphite-emitter/src/main/java/io/druid/emitter/graphite/DruidToGraphiteEventConverter.java similarity index 100% rename from extensions/graphite-emitter/src/main/java/io/druid/emitter/graphite/DruidToGraphiteEventConverter.java rename to extensions-contrib/graphite-emitter/src/main/java/io/druid/emitter/graphite/DruidToGraphiteEventConverter.java diff --git a/extensions/graphite-emitter/src/main/java/io/druid/emitter/graphite/GraphiteEmitter.java b/extensions-contrib/graphite-emitter/src/main/java/io/druid/emitter/graphite/GraphiteEmitter.java similarity index 100% rename from extensions/graphite-emitter/src/main/java/io/druid/emitter/graphite/GraphiteEmitter.java rename to extensions-contrib/graphite-emitter/src/main/java/io/druid/emitter/graphite/GraphiteEmitter.java diff --git a/extensions/graphite-emitter/src/main/java/io/druid/emitter/graphite/GraphiteEmitterConfig.java b/extensions-contrib/graphite-emitter/src/main/java/io/druid/emitter/graphite/GraphiteEmitterConfig.java similarity index 100% rename from extensions/graphite-emitter/src/main/java/io/druid/emitter/graphite/GraphiteEmitterConfig.java rename to extensions-contrib/graphite-emitter/src/main/java/io/druid/emitter/graphite/GraphiteEmitterConfig.java diff --git a/extensions/graphite-emitter/src/main/java/io/druid/emitter/graphite/GraphiteEmitterModule.java b/extensions-contrib/graphite-emitter/src/main/java/io/druid/emitter/graphite/GraphiteEmitterModule.java similarity index 100% rename from extensions/graphite-emitter/src/main/java/io/druid/emitter/graphite/GraphiteEmitterModule.java rename to extensions-contrib/graphite-emitter/src/main/java/io/druid/emitter/graphite/GraphiteEmitterModule.java diff --git a/extensions/graphite-emitter/src/main/java/io/druid/emitter/graphite/GraphiteEvent.java b/extensions-contrib/graphite-emitter/src/main/java/io/druid/emitter/graphite/GraphiteEvent.java similarity index 100% rename from extensions/graphite-emitter/src/main/java/io/druid/emitter/graphite/GraphiteEvent.java rename to extensions-contrib/graphite-emitter/src/main/java/io/druid/emitter/graphite/GraphiteEvent.java diff --git a/extensions/graphite-emitter/src/main/java/io/druid/emitter/graphite/SendAllGraphiteEventConverter.java b/extensions-contrib/graphite-emitter/src/main/java/io/druid/emitter/graphite/SendAllGraphiteEventConverter.java similarity index 100% rename from extensions/graphite-emitter/src/main/java/io/druid/emitter/graphite/SendAllGraphiteEventConverter.java rename to extensions-contrib/graphite-emitter/src/main/java/io/druid/emitter/graphite/SendAllGraphiteEventConverter.java diff --git a/extensions/graphite-emitter/src/main/java/io/druid/emitter/graphite/WhiteListBasedConverter.java b/extensions-contrib/graphite-emitter/src/main/java/io/druid/emitter/graphite/WhiteListBasedConverter.java similarity index 100% rename from extensions/graphite-emitter/src/main/java/io/druid/emitter/graphite/WhiteListBasedConverter.java rename to extensions-contrib/graphite-emitter/src/main/java/io/druid/emitter/graphite/WhiteListBasedConverter.java diff --git a/extensions/graphite-emitter/src/main/resources/META-INF/services/io.druid.initialization.DruidModule b/extensions-contrib/graphite-emitter/src/main/resources/META-INF/services/io.druid.initialization.DruidModule similarity index 100% rename from extensions/graphite-emitter/src/main/resources/META-INF/services/io.druid.initialization.DruidModule rename to extensions-contrib/graphite-emitter/src/main/resources/META-INF/services/io.druid.initialization.DruidModule diff --git a/extensions/graphite-emitter/src/main/resources/defaultWhiteListMap.json b/extensions-contrib/graphite-emitter/src/main/resources/defaultWhiteListMap.json similarity index 100% rename from extensions/graphite-emitter/src/main/resources/defaultWhiteListMap.json rename to extensions-contrib/graphite-emitter/src/main/resources/defaultWhiteListMap.json diff --git a/extensions/graphite-emitter/src/test/java/io/druid/emitter/graphite/DruidToWhiteListBasedConverterTest.java b/extensions-contrib/graphite-emitter/src/test/java/io/druid/emitter/graphite/DruidToWhiteListBasedConverterTest.java similarity index 100% rename from extensions/graphite-emitter/src/test/java/io/druid/emitter/graphite/DruidToWhiteListBasedConverterTest.java rename to extensions-contrib/graphite-emitter/src/test/java/io/druid/emitter/graphite/DruidToWhiteListBasedConverterTest.java diff --git a/extensions/graphite-emitter/src/test/java/io/druid/emitter/graphite/GraphiteEmitterConfigTest.java b/extensions-contrib/graphite-emitter/src/test/java/io/druid/emitter/graphite/GraphiteEmitterConfigTest.java similarity index 100% rename from extensions/graphite-emitter/src/test/java/io/druid/emitter/graphite/GraphiteEmitterConfigTest.java rename to extensions-contrib/graphite-emitter/src/test/java/io/druid/emitter/graphite/GraphiteEmitterConfigTest.java diff --git a/extensions/graphite-emitter/src/test/java/io/druid/emitter/graphite/WhiteListBasedConverterTest.java b/extensions-contrib/graphite-emitter/src/test/java/io/druid/emitter/graphite/WhiteListBasedConverterTest.java similarity index 100% rename from extensions/graphite-emitter/src/test/java/io/druid/emitter/graphite/WhiteListBasedConverterTest.java rename to extensions-contrib/graphite-emitter/src/test/java/io/druid/emitter/graphite/WhiteListBasedConverterTest.java diff --git a/extensions/kafka-eight-simpleConsumer/pom.xml b/extensions-contrib/kafka-eight-simpleConsumer/pom.xml similarity index 98% rename from extensions/kafka-eight-simpleConsumer/pom.xml rename to extensions-contrib/kafka-eight-simpleConsumer/pom.xml index 9f441d8fb4a..b4782c8c359 100644 --- a/extensions/kafka-eight-simpleConsumer/pom.xml +++ b/extensions-contrib/kafka-eight-simpleConsumer/pom.xml @@ -19,7 +19,7 @@ 4.0.0 - io.druid.extensions + io.druid.extensions.contrib druid-kafka-eight-simple-consumer druid-kafka-eight-simple-consumer druid-kafka-eight-simple-consumer diff --git a/extensions/kafka-eight-simpleConsumer/src/main/java/io/druid/firehose/kafka/KafkaEightSimpleConsumerDruidModule.java b/extensions-contrib/kafka-eight-simpleConsumer/src/main/java/io/druid/firehose/kafka/KafkaEightSimpleConsumerDruidModule.java similarity index 100% rename from extensions/kafka-eight-simpleConsumer/src/main/java/io/druid/firehose/kafka/KafkaEightSimpleConsumerDruidModule.java rename to extensions-contrib/kafka-eight-simpleConsumer/src/main/java/io/druid/firehose/kafka/KafkaEightSimpleConsumerDruidModule.java diff --git a/extensions/kafka-eight-simpleConsumer/src/main/java/io/druid/firehose/kafka/KafkaEightSimpleConsumerFirehoseFactory.java b/extensions-contrib/kafka-eight-simpleConsumer/src/main/java/io/druid/firehose/kafka/KafkaEightSimpleConsumerFirehoseFactory.java similarity index 100% rename from extensions/kafka-eight-simpleConsumer/src/main/java/io/druid/firehose/kafka/KafkaEightSimpleConsumerFirehoseFactory.java rename to extensions-contrib/kafka-eight-simpleConsumer/src/main/java/io/druid/firehose/kafka/KafkaEightSimpleConsumerFirehoseFactory.java diff --git a/extensions/kafka-eight-simpleConsumer/src/main/java/io/druid/firehose/kafka/KafkaSimpleConsumer.java b/extensions-contrib/kafka-eight-simpleConsumer/src/main/java/io/druid/firehose/kafka/KafkaSimpleConsumer.java similarity index 100% rename from extensions/kafka-eight-simpleConsumer/src/main/java/io/druid/firehose/kafka/KafkaSimpleConsumer.java rename to extensions-contrib/kafka-eight-simpleConsumer/src/main/java/io/druid/firehose/kafka/KafkaSimpleConsumer.java diff --git a/extensions/kafka-eight-simpleConsumer/src/main/resources/META-INF/services/io.druid.initialization.DruidModule b/extensions-contrib/kafka-eight-simpleConsumer/src/main/resources/META-INF/services/io.druid.initialization.DruidModule similarity index 100% rename from extensions/kafka-eight-simpleConsumer/src/main/resources/META-INF/services/io.druid.initialization.DruidModule rename to extensions-contrib/kafka-eight-simpleConsumer/src/main/resources/META-INF/services/io.druid.initialization.DruidModule diff --git a/extensions/rabbitmq/pom.xml b/extensions-contrib/rabbitmq/pom.xml similarity index 66% rename from extensions/rabbitmq/pom.xml rename to extensions-contrib/rabbitmq/pom.xml index ba6f74ffe18..ed3f70ea228 100644 --- a/extensions/rabbitmq/pom.xml +++ b/extensions-contrib/rabbitmq/pom.xml @@ -1,25 +1,27 @@ 4.0.0 - io.druid.extensions + io.druid.extensions.contrib druid-rabbitmq druid-rabbitmq druid-rabbitmq diff --git a/extensions/rabbitmq/src/main/java/io/druid/firehose/rabbitmq/JacksonifiedConnectionFactory.java b/extensions-contrib/rabbitmq/src/main/java/io/druid/firehose/rabbitmq/JacksonifiedConnectionFactory.java similarity index 100% rename from extensions/rabbitmq/src/main/java/io/druid/firehose/rabbitmq/JacksonifiedConnectionFactory.java rename to extensions-contrib/rabbitmq/src/main/java/io/druid/firehose/rabbitmq/JacksonifiedConnectionFactory.java diff --git a/extensions/rabbitmq/src/main/java/io/druid/firehose/rabbitmq/RabbitMQDruidModule.java b/extensions-contrib/rabbitmq/src/main/java/io/druid/firehose/rabbitmq/RabbitMQDruidModule.java similarity index 100% rename from extensions/rabbitmq/src/main/java/io/druid/firehose/rabbitmq/RabbitMQDruidModule.java rename to extensions-contrib/rabbitmq/src/main/java/io/druid/firehose/rabbitmq/RabbitMQDruidModule.java diff --git a/extensions/rabbitmq/src/main/java/io/druid/firehose/rabbitmq/RabbitMQFirehoseConfig.java b/extensions-contrib/rabbitmq/src/main/java/io/druid/firehose/rabbitmq/RabbitMQFirehoseConfig.java similarity index 100% rename from extensions/rabbitmq/src/main/java/io/druid/firehose/rabbitmq/RabbitMQFirehoseConfig.java rename to extensions-contrib/rabbitmq/src/main/java/io/druid/firehose/rabbitmq/RabbitMQFirehoseConfig.java diff --git a/extensions/rabbitmq/src/main/java/io/druid/firehose/rabbitmq/RabbitMQFirehoseFactory.java b/extensions-contrib/rabbitmq/src/main/java/io/druid/firehose/rabbitmq/RabbitMQFirehoseFactory.java similarity index 100% rename from extensions/rabbitmq/src/main/java/io/druid/firehose/rabbitmq/RabbitMQFirehoseFactory.java rename to extensions-contrib/rabbitmq/src/main/java/io/druid/firehose/rabbitmq/RabbitMQFirehoseFactory.java diff --git a/extensions/rabbitmq/src/main/resources/META-INF/services/io.druid.initialization.DruidModule b/extensions-contrib/rabbitmq/src/main/resources/META-INF/services/io.druid.initialization.DruidModule similarity index 100% rename from extensions/rabbitmq/src/main/resources/META-INF/services/io.druid.initialization.DruidModule rename to extensions-contrib/rabbitmq/src/main/resources/META-INF/services/io.druid.initialization.DruidModule diff --git a/extensions/rabbitmq/src/test/java/io/druid/examples/rabbitmq/RabbitMQFirehoseFactoryTest.java b/extensions-contrib/rabbitmq/src/test/java/io/druid/examples/rabbitmq/RabbitMQFirehoseFactoryTest.java similarity index 100% rename from extensions/rabbitmq/src/test/java/io/druid/examples/rabbitmq/RabbitMQFirehoseFactoryTest.java rename to extensions-contrib/rabbitmq/src/test/java/io/druid/examples/rabbitmq/RabbitMQFirehoseFactoryTest.java diff --git a/extensions/rabbitmq/src/test/java/io/druid/examples/rabbitmq/RabbitMQProducerMain.java b/extensions-contrib/rabbitmq/src/test/java/io/druid/examples/rabbitmq/RabbitMQProducerMain.java similarity index 100% rename from extensions/rabbitmq/src/test/java/io/druid/examples/rabbitmq/RabbitMQProducerMain.java rename to extensions-contrib/rabbitmq/src/test/java/io/druid/examples/rabbitmq/RabbitMQProducerMain.java diff --git a/extensions/avro-extensions/pom.xml b/extensions-core/avro-extensions/pom.xml similarity index 77% rename from extensions/avro-extensions/pom.xml rename to extensions-core/avro-extensions/pom.xml index 029d3005934..49aaef8eed2 100644 --- a/extensions/avro-extensions/pom.xml +++ b/extensions-core/avro-extensions/pom.xml @@ -1,24 +1,27 @@ 4.0.0 + io.druid.extensions druid-avro-extensions druid-avro-extensions diff --git a/extensions/avro-extensions/src/main/java/io/druid/data/input/AvroHadoopInputRowParser.java b/extensions-core/avro-extensions/src/main/java/io/druid/data/input/AvroHadoopInputRowParser.java similarity index 100% rename from extensions/avro-extensions/src/main/java/io/druid/data/input/AvroHadoopInputRowParser.java rename to extensions-core/avro-extensions/src/main/java/io/druid/data/input/AvroHadoopInputRowParser.java diff --git a/extensions/avro-extensions/src/main/java/io/druid/data/input/AvroStreamInputRowParser.java b/extensions-core/avro-extensions/src/main/java/io/druid/data/input/AvroStreamInputRowParser.java similarity index 100% rename from extensions/avro-extensions/src/main/java/io/druid/data/input/AvroStreamInputRowParser.java rename to extensions-core/avro-extensions/src/main/java/io/druid/data/input/AvroStreamInputRowParser.java diff --git a/extensions/avro-extensions/src/main/java/io/druid/data/input/avro/AvroBytesDecoder.java b/extensions-core/avro-extensions/src/main/java/io/druid/data/input/avro/AvroBytesDecoder.java similarity index 100% rename from extensions/avro-extensions/src/main/java/io/druid/data/input/avro/AvroBytesDecoder.java rename to extensions-core/avro-extensions/src/main/java/io/druid/data/input/avro/AvroBytesDecoder.java diff --git a/extensions/avro-extensions/src/main/java/io/druid/data/input/avro/AvroExtensionsModule.java b/extensions-core/avro-extensions/src/main/java/io/druid/data/input/avro/AvroExtensionsModule.java similarity index 100% rename from extensions/avro-extensions/src/main/java/io/druid/data/input/avro/AvroExtensionsModule.java rename to extensions-core/avro-extensions/src/main/java/io/druid/data/input/avro/AvroExtensionsModule.java diff --git a/extensions/avro-extensions/src/main/java/io/druid/data/input/avro/AvroValueInputFormat.java b/extensions-core/avro-extensions/src/main/java/io/druid/data/input/avro/AvroValueInputFormat.java similarity index 100% rename from extensions/avro-extensions/src/main/java/io/druid/data/input/avro/AvroValueInputFormat.java rename to extensions-core/avro-extensions/src/main/java/io/druid/data/input/avro/AvroValueInputFormat.java diff --git a/extensions/avro-extensions/src/main/java/io/druid/data/input/avro/AvroValueRecordReader.java b/extensions-core/avro-extensions/src/main/java/io/druid/data/input/avro/AvroValueRecordReader.java similarity index 100% rename from extensions/avro-extensions/src/main/java/io/druid/data/input/avro/AvroValueRecordReader.java rename to extensions-core/avro-extensions/src/main/java/io/druid/data/input/avro/AvroValueRecordReader.java diff --git a/extensions/avro-extensions/src/main/java/io/druid/data/input/avro/GenericRecordAsMap.java b/extensions-core/avro-extensions/src/main/java/io/druid/data/input/avro/GenericRecordAsMap.java similarity index 100% rename from extensions/avro-extensions/src/main/java/io/druid/data/input/avro/GenericRecordAsMap.java rename to extensions-core/avro-extensions/src/main/java/io/druid/data/input/avro/GenericRecordAsMap.java diff --git a/extensions/avro-extensions/src/main/java/io/druid/data/input/avro/SchemaRepoBasedAvroBytesDecoder.java b/extensions-core/avro-extensions/src/main/java/io/druid/data/input/avro/SchemaRepoBasedAvroBytesDecoder.java similarity index 100% rename from extensions/avro-extensions/src/main/java/io/druid/data/input/avro/SchemaRepoBasedAvroBytesDecoder.java rename to extensions-core/avro-extensions/src/main/java/io/druid/data/input/avro/SchemaRepoBasedAvroBytesDecoder.java diff --git a/extensions/avro-extensions/src/main/java/io/druid/data/input/schemarepo/Avro1124RESTRepositoryClientWrapper.java b/extensions-core/avro-extensions/src/main/java/io/druid/data/input/schemarepo/Avro1124RESTRepositoryClientWrapper.java similarity index 100% rename from extensions/avro-extensions/src/main/java/io/druid/data/input/schemarepo/Avro1124RESTRepositoryClientWrapper.java rename to extensions-core/avro-extensions/src/main/java/io/druid/data/input/schemarepo/Avro1124RESTRepositoryClientWrapper.java diff --git a/extensions/avro-extensions/src/main/java/io/druid/data/input/schemarepo/Avro1124SubjectAndIdConverter.java b/extensions-core/avro-extensions/src/main/java/io/druid/data/input/schemarepo/Avro1124SubjectAndIdConverter.java similarity index 100% rename from extensions/avro-extensions/src/main/java/io/druid/data/input/schemarepo/Avro1124SubjectAndIdConverter.java rename to extensions-core/avro-extensions/src/main/java/io/druid/data/input/schemarepo/Avro1124SubjectAndIdConverter.java diff --git a/extensions/avro-extensions/src/main/java/io/druid/data/input/schemarepo/SubjectAndIdConverter.java b/extensions-core/avro-extensions/src/main/java/io/druid/data/input/schemarepo/SubjectAndIdConverter.java similarity index 100% rename from extensions/avro-extensions/src/main/java/io/druid/data/input/schemarepo/SubjectAndIdConverter.java rename to extensions-core/avro-extensions/src/main/java/io/druid/data/input/schemarepo/SubjectAndIdConverter.java diff --git a/extensions/avro-extensions/src/main/resources/META-INF/services/io.druid.initialization.DruidModule b/extensions-core/avro-extensions/src/main/resources/META-INF/services/io.druid.initialization.DruidModule similarity index 100% rename from extensions/avro-extensions/src/main/resources/META-INF/services/io.druid.initialization.DruidModule rename to extensions-core/avro-extensions/src/main/resources/META-INF/services/io.druid.initialization.DruidModule diff --git a/extensions/avro-extensions/src/test/avro/some-datum.avsc b/extensions-core/avro-extensions/src/test/avro/some-datum.avsc similarity index 100% rename from extensions/avro-extensions/src/test/avro/some-datum.avsc rename to extensions-core/avro-extensions/src/test/avro/some-datum.avsc diff --git a/extensions/avro-extensions/src/test/java/io/druid/data/input/AvroHadoopInputRowParserTest.java b/extensions-core/avro-extensions/src/test/java/io/druid/data/input/AvroHadoopInputRowParserTest.java similarity index 100% rename from extensions/avro-extensions/src/test/java/io/druid/data/input/AvroHadoopInputRowParserTest.java rename to extensions-core/avro-extensions/src/test/java/io/druid/data/input/AvroHadoopInputRowParserTest.java diff --git a/extensions/avro-extensions/src/test/java/io/druid/data/input/AvroStreamInputRowParserTest.java b/extensions-core/avro-extensions/src/test/java/io/druid/data/input/AvroStreamInputRowParserTest.java similarity index 100% rename from extensions/avro-extensions/src/test/java/io/druid/data/input/AvroStreamInputRowParserTest.java rename to extensions-core/avro-extensions/src/test/java/io/druid/data/input/AvroStreamInputRowParserTest.java diff --git a/extensions/datasketches/README.md b/extensions-core/datasketches/README.md similarity index 100% rename from extensions/datasketches/README.md rename to extensions-core/datasketches/README.md diff --git a/extensions/datasketches/pom.xml b/extensions-core/datasketches/pom.xml similarity index 100% rename from extensions/datasketches/pom.xml rename to extensions-core/datasketches/pom.xml diff --git a/extensions/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/EmptySketchAggregator.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/EmptySketchAggregator.java similarity index 100% rename from extensions/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/EmptySketchAggregator.java rename to extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/EmptySketchAggregator.java diff --git a/extensions/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/EmptySketchBufferAggregator.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/EmptySketchBufferAggregator.java similarity index 100% rename from extensions/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/EmptySketchBufferAggregator.java rename to extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/EmptySketchBufferAggregator.java diff --git a/extensions/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/MemoryJsonSerializer.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/MemoryJsonSerializer.java similarity index 100% rename from extensions/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/MemoryJsonSerializer.java rename to extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/MemoryJsonSerializer.java diff --git a/extensions/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchAggregator.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchAggregator.java similarity index 100% rename from extensions/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchAggregator.java rename to extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchAggregator.java diff --git a/extensions/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchAggregatorFactory.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchAggregatorFactory.java similarity index 100% rename from extensions/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchAggregatorFactory.java rename to extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchAggregatorFactory.java diff --git a/extensions/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchBufferAggregator.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchBufferAggregator.java similarity index 100% rename from extensions/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchBufferAggregator.java rename to extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchBufferAggregator.java diff --git a/extensions/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchBuildComplexMetricSerde.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchBuildComplexMetricSerde.java similarity index 100% rename from extensions/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchBuildComplexMetricSerde.java rename to extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchBuildComplexMetricSerde.java diff --git a/extensions/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchEstimatePostAggregator.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchEstimatePostAggregator.java similarity index 100% rename from extensions/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchEstimatePostAggregator.java rename to extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchEstimatePostAggregator.java diff --git a/extensions/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchEstimateWithErrorBounds.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchEstimateWithErrorBounds.java similarity index 100% rename from extensions/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchEstimateWithErrorBounds.java rename to extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchEstimateWithErrorBounds.java diff --git a/extensions/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchJsonSerializer.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchJsonSerializer.java similarity index 100% rename from extensions/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchJsonSerializer.java rename to extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchJsonSerializer.java diff --git a/extensions/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchMergeAggregatorFactory.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchMergeAggregatorFactory.java similarity index 100% rename from extensions/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchMergeAggregatorFactory.java rename to extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchMergeAggregatorFactory.java diff --git a/extensions/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchMergeComplexMetricSerde.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchMergeComplexMetricSerde.java similarity index 100% rename from extensions/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchMergeComplexMetricSerde.java rename to extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchMergeComplexMetricSerde.java diff --git a/extensions/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchModule.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchModule.java similarity index 100% rename from extensions/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchModule.java rename to extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchModule.java diff --git a/extensions/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchObjectStrategy.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchObjectStrategy.java similarity index 100% rename from extensions/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchObjectStrategy.java rename to extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchObjectStrategy.java diff --git a/extensions/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchOperations.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchOperations.java similarity index 100% rename from extensions/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchOperations.java rename to extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchOperations.java diff --git a/extensions/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchSetPostAggregator.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchSetPostAggregator.java similarity index 100% rename from extensions/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchSetPostAggregator.java rename to extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SketchSetPostAggregator.java diff --git a/extensions/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SynchronizedUnion.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SynchronizedUnion.java similarity index 100% rename from extensions/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SynchronizedUnion.java rename to extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/SynchronizedUnion.java diff --git a/extensions/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/oldapi/OldApiSketchModule.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/oldapi/OldApiSketchModule.java similarity index 100% rename from extensions/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/oldapi/OldApiSketchModule.java rename to extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/oldapi/OldApiSketchModule.java diff --git a/extensions/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/oldapi/OldSketchBuildAggregatorFactory.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/oldapi/OldSketchBuildAggregatorFactory.java similarity index 100% rename from extensions/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/oldapi/OldSketchBuildAggregatorFactory.java rename to extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/oldapi/OldSketchBuildAggregatorFactory.java diff --git a/extensions/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/oldapi/OldSketchEstimatePostAggregator.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/oldapi/OldSketchEstimatePostAggregator.java similarity index 100% rename from extensions/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/oldapi/OldSketchEstimatePostAggregator.java rename to extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/oldapi/OldSketchEstimatePostAggregator.java diff --git a/extensions/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/oldapi/OldSketchMergeAggregatorFactory.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/oldapi/OldSketchMergeAggregatorFactory.java similarity index 100% rename from extensions/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/oldapi/OldSketchMergeAggregatorFactory.java rename to extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/oldapi/OldSketchMergeAggregatorFactory.java diff --git a/extensions/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/oldapi/OldSketchSetPostAggregator.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/oldapi/OldSketchSetPostAggregator.java similarity index 100% rename from extensions/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/oldapi/OldSketchSetPostAggregator.java rename to extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/theta/oldapi/OldSketchSetPostAggregator.java diff --git a/extensions/datasketches/src/main/resources/META-INF/services/io.druid.initialization.DruidModule b/extensions-core/datasketches/src/main/resources/META-INF/services/io.druid.initialization.DruidModule similarity index 100% rename from extensions/datasketches/src/main/resources/META-INF/services/io.druid.initialization.DruidModule rename to extensions-core/datasketches/src/main/resources/META-INF/services/io.druid.initialization.DruidModule diff --git a/extensions/datasketches/src/test/java/io/druid/query/aggregation/datasketches/theta/SketchAggregationTest.java b/extensions-core/datasketches/src/test/java/io/druid/query/aggregation/datasketches/theta/SketchAggregationTest.java similarity index 100% rename from extensions/datasketches/src/test/java/io/druid/query/aggregation/datasketches/theta/SketchAggregationTest.java rename to extensions-core/datasketches/src/test/java/io/druid/query/aggregation/datasketches/theta/SketchAggregationTest.java diff --git a/extensions/datasketches/src/test/java/io/druid/query/aggregation/datasketches/theta/SketchEstimateWithErrorBoundsTest.java b/extensions-core/datasketches/src/test/java/io/druid/query/aggregation/datasketches/theta/SketchEstimateWithErrorBoundsTest.java similarity index 100% rename from extensions/datasketches/src/test/java/io/druid/query/aggregation/datasketches/theta/SketchEstimateWithErrorBoundsTest.java rename to extensions-core/datasketches/src/test/java/io/druid/query/aggregation/datasketches/theta/SketchEstimateWithErrorBoundsTest.java diff --git a/extensions/datasketches/src/test/java/io/druid/query/aggregation/datasketches/theta/oldapi/OldApiSketchAggregationTest.java b/extensions-core/datasketches/src/test/java/io/druid/query/aggregation/datasketches/theta/oldapi/OldApiSketchAggregationTest.java similarity index 100% rename from extensions/datasketches/src/test/java/io/druid/query/aggregation/datasketches/theta/oldapi/OldApiSketchAggregationTest.java rename to extensions-core/datasketches/src/test/java/io/druid/query/aggregation/datasketches/theta/oldapi/OldApiSketchAggregationTest.java diff --git a/extensions/datasketches/src/test/resources/oldapi/old_simple_test_data_aggregators.json b/extensions-core/datasketches/src/test/resources/oldapi/old_simple_test_data_aggregators.json similarity index 100% rename from extensions/datasketches/src/test/resources/oldapi/old_simple_test_data_aggregators.json rename to extensions-core/datasketches/src/test/resources/oldapi/old_simple_test_data_aggregators.json diff --git a/extensions/datasketches/src/test/resources/oldapi/old_simple_test_data_group_by_query.json b/extensions-core/datasketches/src/test/resources/oldapi/old_simple_test_data_group_by_query.json similarity index 100% rename from extensions/datasketches/src/test/resources/oldapi/old_simple_test_data_group_by_query.json rename to extensions-core/datasketches/src/test/resources/oldapi/old_simple_test_data_group_by_query.json diff --git a/extensions/datasketches/src/test/resources/oldapi/old_sketch_test_data_aggregators.json b/extensions-core/datasketches/src/test/resources/oldapi/old_sketch_test_data_aggregators.json similarity index 100% rename from extensions/datasketches/src/test/resources/oldapi/old_sketch_test_data_aggregators.json rename to extensions-core/datasketches/src/test/resources/oldapi/old_sketch_test_data_aggregators.json diff --git a/extensions/datasketches/src/test/resources/oldapi/old_sketch_test_data_group_by_query.json b/extensions-core/datasketches/src/test/resources/oldapi/old_sketch_test_data_group_by_query.json similarity index 100% rename from extensions/datasketches/src/test/resources/oldapi/old_sketch_test_data_group_by_query.json rename to extensions-core/datasketches/src/test/resources/oldapi/old_sketch_test_data_group_by_query.json diff --git a/extensions/datasketches/src/test/resources/select_query.json b/extensions-core/datasketches/src/test/resources/select_query.json similarity index 100% rename from extensions/datasketches/src/test/resources/select_query.json rename to extensions-core/datasketches/src/test/resources/select_query.json diff --git a/extensions/datasketches/src/test/resources/simple_test_data.tsv b/extensions-core/datasketches/src/test/resources/simple_test_data.tsv similarity index 100% rename from extensions/datasketches/src/test/resources/simple_test_data.tsv rename to extensions-core/datasketches/src/test/resources/simple_test_data.tsv diff --git a/extensions/datasketches/src/test/resources/simple_test_data_aggregators.json b/extensions-core/datasketches/src/test/resources/simple_test_data_aggregators.json similarity index 100% rename from extensions/datasketches/src/test/resources/simple_test_data_aggregators.json rename to extensions-core/datasketches/src/test/resources/simple_test_data_aggregators.json diff --git a/extensions/datasketches/src/test/resources/simple_test_data_group_by_query.json b/extensions-core/datasketches/src/test/resources/simple_test_data_group_by_query.json similarity index 100% rename from extensions/datasketches/src/test/resources/simple_test_data_group_by_query.json rename to extensions-core/datasketches/src/test/resources/simple_test_data_group_by_query.json diff --git a/extensions/datasketches/src/test/resources/simple_test_data_record_parser.json b/extensions-core/datasketches/src/test/resources/simple_test_data_record_parser.json similarity index 100% rename from extensions/datasketches/src/test/resources/simple_test_data_record_parser.json rename to extensions-core/datasketches/src/test/resources/simple_test_data_record_parser.json diff --git a/extensions/datasketches/src/test/resources/simple_test_data_record_parser2.json b/extensions-core/datasketches/src/test/resources/simple_test_data_record_parser2.json similarity index 100% rename from extensions/datasketches/src/test/resources/simple_test_data_record_parser2.json rename to extensions-core/datasketches/src/test/resources/simple_test_data_record_parser2.json diff --git a/extensions/datasketches/src/test/resources/sketch_test_data.tsv b/extensions-core/datasketches/src/test/resources/sketch_test_data.tsv similarity index 100% rename from extensions/datasketches/src/test/resources/sketch_test_data.tsv rename to extensions-core/datasketches/src/test/resources/sketch_test_data.tsv diff --git a/extensions/datasketches/src/test/resources/sketch_test_data_aggregators.json b/extensions-core/datasketches/src/test/resources/sketch_test_data_aggregators.json similarity index 100% rename from extensions/datasketches/src/test/resources/sketch_test_data_aggregators.json rename to extensions-core/datasketches/src/test/resources/sketch_test_data_aggregators.json diff --git a/extensions/datasketches/src/test/resources/sketch_test_data_group_by_query.json b/extensions-core/datasketches/src/test/resources/sketch_test_data_group_by_query.json similarity index 100% rename from extensions/datasketches/src/test/resources/sketch_test_data_group_by_query.json rename to extensions-core/datasketches/src/test/resources/sketch_test_data_group_by_query.json diff --git a/extensions/datasketches/src/test/resources/sketch_test_data_record_parser.json b/extensions-core/datasketches/src/test/resources/sketch_test_data_record_parser.json similarity index 100% rename from extensions/datasketches/src/test/resources/sketch_test_data_record_parser.json rename to extensions-core/datasketches/src/test/resources/sketch_test_data_record_parser.json diff --git a/extensions/hdfs-storage/pom.xml b/extensions-core/hdfs-storage/pom.xml similarity index 100% rename from extensions/hdfs-storage/pom.xml rename to extensions-core/hdfs-storage/pom.xml diff --git a/extensions/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsDataSegmentFinder.java b/extensions-core/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsDataSegmentFinder.java similarity index 100% rename from extensions/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsDataSegmentFinder.java rename to extensions-core/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsDataSegmentFinder.java diff --git a/extensions/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsDataSegmentKiller.java b/extensions-core/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsDataSegmentKiller.java similarity index 100% rename from extensions/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsDataSegmentKiller.java rename to extensions-core/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsDataSegmentKiller.java diff --git a/extensions/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsDataSegmentPuller.java b/extensions-core/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsDataSegmentPuller.java similarity index 100% rename from extensions/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsDataSegmentPuller.java rename to extensions-core/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsDataSegmentPuller.java diff --git a/extensions/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsDataSegmentPusher.java b/extensions-core/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsDataSegmentPusher.java similarity index 100% rename from extensions/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsDataSegmentPusher.java rename to extensions-core/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsDataSegmentPusher.java diff --git a/extensions/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsDataSegmentPusherConfig.java b/extensions-core/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsDataSegmentPusherConfig.java similarity index 100% rename from extensions/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsDataSegmentPusherConfig.java rename to extensions-core/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsDataSegmentPusherConfig.java diff --git a/extensions/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsFileTimestampVersionFinder.java b/extensions-core/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsFileTimestampVersionFinder.java similarity index 100% rename from extensions/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsFileTimestampVersionFinder.java rename to extensions-core/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsFileTimestampVersionFinder.java diff --git a/extensions/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsLoadSpec.java b/extensions-core/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsLoadSpec.java similarity index 100% rename from extensions/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsLoadSpec.java rename to extensions-core/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsLoadSpec.java diff --git a/extensions/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsStorageDruidModule.java b/extensions-core/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsStorageDruidModule.java similarity index 100% rename from extensions/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsStorageDruidModule.java rename to extensions-core/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsStorageDruidModule.java diff --git a/extensions/hdfs-storage/src/main/java/io/druid/storage/hdfs/tasklog/HdfsTaskLogs.java b/extensions-core/hdfs-storage/src/main/java/io/druid/storage/hdfs/tasklog/HdfsTaskLogs.java similarity index 100% rename from extensions/hdfs-storage/src/main/java/io/druid/storage/hdfs/tasklog/HdfsTaskLogs.java rename to extensions-core/hdfs-storage/src/main/java/io/druid/storage/hdfs/tasklog/HdfsTaskLogs.java diff --git a/extensions/hdfs-storage/src/main/java/io/druid/storage/hdfs/tasklog/HdfsTaskLogsConfig.java b/extensions-core/hdfs-storage/src/main/java/io/druid/storage/hdfs/tasklog/HdfsTaskLogsConfig.java similarity index 100% rename from extensions/hdfs-storage/src/main/java/io/druid/storage/hdfs/tasklog/HdfsTaskLogsConfig.java rename to extensions-core/hdfs-storage/src/main/java/io/druid/storage/hdfs/tasklog/HdfsTaskLogsConfig.java diff --git a/extensions/hdfs-storage/src/main/resources/META-INF/services/io.druid.initialization.DruidModule b/extensions-core/hdfs-storage/src/main/resources/META-INF/services/io.druid.initialization.DruidModule similarity index 100% rename from extensions/hdfs-storage/src/main/resources/META-INF/services/io.druid.initialization.DruidModule rename to extensions-core/hdfs-storage/src/main/resources/META-INF/services/io.druid.initialization.DruidModule diff --git a/extensions/hdfs-storage/src/test/java/io/druid/indexing/common/tasklogs/HdfsTaskLogsTest.java b/extensions-core/hdfs-storage/src/test/java/io/druid/indexing/common/tasklogs/HdfsTaskLogsTest.java similarity index 100% rename from extensions/hdfs-storage/src/test/java/io/druid/indexing/common/tasklogs/HdfsTaskLogsTest.java rename to extensions-core/hdfs-storage/src/test/java/io/druid/indexing/common/tasklogs/HdfsTaskLogsTest.java diff --git a/extensions/hdfs-storage/src/test/java/io/druid/segment/loading/HdfsDataSegmentFinderTest.java b/extensions-core/hdfs-storage/src/test/java/io/druid/segment/loading/HdfsDataSegmentFinderTest.java similarity index 100% rename from extensions/hdfs-storage/src/test/java/io/druid/segment/loading/HdfsDataSegmentFinderTest.java rename to extensions-core/hdfs-storage/src/test/java/io/druid/segment/loading/HdfsDataSegmentFinderTest.java diff --git a/extensions/hdfs-storage/src/test/java/io/druid/segment/loading/HdfsDataSegmentPullerTest.java b/extensions-core/hdfs-storage/src/test/java/io/druid/segment/loading/HdfsDataSegmentPullerTest.java similarity index 100% rename from extensions/hdfs-storage/src/test/java/io/druid/segment/loading/HdfsDataSegmentPullerTest.java rename to extensions-core/hdfs-storage/src/test/java/io/druid/segment/loading/HdfsDataSegmentPullerTest.java diff --git a/extensions/hdfs-storage/src/test/java/io/druid/segment/loading/HdfsFileTimestampVersionFinderTest.java b/extensions-core/hdfs-storage/src/test/java/io/druid/segment/loading/HdfsFileTimestampVersionFinderTest.java similarity index 100% rename from extensions/hdfs-storage/src/test/java/io/druid/segment/loading/HdfsFileTimestampVersionFinderTest.java rename to extensions-core/hdfs-storage/src/test/java/io/druid/segment/loading/HdfsFileTimestampVersionFinderTest.java diff --git a/extensions/hdfs-storage/src/test/java/io/druid/storage/hdfs/HdfsDataSegmentKillerTest.java b/extensions-core/hdfs-storage/src/test/java/io/druid/storage/hdfs/HdfsDataSegmentKillerTest.java similarity index 100% rename from extensions/hdfs-storage/src/test/java/io/druid/storage/hdfs/HdfsDataSegmentKillerTest.java rename to extensions-core/hdfs-storage/src/test/java/io/druid/storage/hdfs/HdfsDataSegmentKillerTest.java diff --git a/extensions/hdfs-storage/src/test/java/io/druid/storage/hdfs/HdfsDataSegmentPusherTest.java b/extensions-core/hdfs-storage/src/test/java/io/druid/storage/hdfs/HdfsDataSegmentPusherTest.java similarity index 100% rename from extensions/hdfs-storage/src/test/java/io/druid/storage/hdfs/HdfsDataSegmentPusherTest.java rename to extensions-core/hdfs-storage/src/test/java/io/druid/storage/hdfs/HdfsDataSegmentPusherTest.java diff --git a/extensions/histogram/pom.xml b/extensions-core/histogram/pom.xml similarity index 100% rename from extensions/histogram/pom.xml rename to extensions-core/histogram/pom.xml diff --git a/extensions/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogram.java b/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogram.java similarity index 100% rename from extensions/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogram.java rename to extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogram.java diff --git a/extensions/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramAggregator.java b/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramAggregator.java similarity index 100% rename from extensions/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramAggregator.java rename to extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramAggregator.java diff --git a/extensions/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramAggregatorFactory.java b/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramAggregatorFactory.java similarity index 100% rename from extensions/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramAggregatorFactory.java rename to extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramAggregatorFactory.java diff --git a/extensions/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramBufferAggregator.java b/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramBufferAggregator.java similarity index 100% rename from extensions/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramBufferAggregator.java rename to extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramBufferAggregator.java diff --git a/extensions/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramDruidModule.java b/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramDruidModule.java similarity index 100% rename from extensions/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramDruidModule.java rename to extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramDruidModule.java diff --git a/extensions/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramFoldingAggregator.java b/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramFoldingAggregator.java similarity index 100% rename from extensions/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramFoldingAggregator.java rename to extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramFoldingAggregator.java diff --git a/extensions/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramFoldingAggregatorFactory.java b/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramFoldingAggregatorFactory.java similarity index 100% rename from extensions/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramFoldingAggregatorFactory.java rename to extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramFoldingAggregatorFactory.java diff --git a/extensions/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramFoldingBufferAggregator.java b/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramFoldingBufferAggregator.java similarity index 100% rename from extensions/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramFoldingBufferAggregator.java rename to extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramFoldingBufferAggregator.java diff --git a/extensions/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramFoldingSerde.java b/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramFoldingSerde.java similarity index 100% rename from extensions/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramFoldingSerde.java rename to extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramFoldingSerde.java diff --git a/extensions/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramPostAggregator.java b/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramPostAggregator.java similarity index 100% rename from extensions/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramPostAggregator.java rename to extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramPostAggregator.java diff --git a/extensions/histogram/src/main/java/io/druid/query/aggregation/histogram/ArrayUtils.java b/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ArrayUtils.java similarity index 100% rename from extensions/histogram/src/main/java/io/druid/query/aggregation/histogram/ArrayUtils.java rename to extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ArrayUtils.java diff --git a/extensions/histogram/src/main/java/io/druid/query/aggregation/histogram/BucketsPostAggregator.java b/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/BucketsPostAggregator.java similarity index 100% rename from extensions/histogram/src/main/java/io/druid/query/aggregation/histogram/BucketsPostAggregator.java rename to extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/BucketsPostAggregator.java diff --git a/extensions/histogram/src/main/java/io/druid/query/aggregation/histogram/BufferUtils.java b/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/BufferUtils.java similarity index 100% rename from extensions/histogram/src/main/java/io/druid/query/aggregation/histogram/BufferUtils.java rename to extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/BufferUtils.java diff --git a/extensions/histogram/src/main/java/io/druid/query/aggregation/histogram/CustomBucketsPostAggregator.java b/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/CustomBucketsPostAggregator.java similarity index 100% rename from extensions/histogram/src/main/java/io/druid/query/aggregation/histogram/CustomBucketsPostAggregator.java rename to extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/CustomBucketsPostAggregator.java diff --git a/extensions/histogram/src/main/java/io/druid/query/aggregation/histogram/EqualBucketsPostAggregator.java b/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/EqualBucketsPostAggregator.java similarity index 100% rename from extensions/histogram/src/main/java/io/druid/query/aggregation/histogram/EqualBucketsPostAggregator.java rename to extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/EqualBucketsPostAggregator.java diff --git a/extensions/histogram/src/main/java/io/druid/query/aggregation/histogram/Histogram.java b/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/Histogram.java similarity index 100% rename from extensions/histogram/src/main/java/io/druid/query/aggregation/histogram/Histogram.java rename to extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/Histogram.java diff --git a/extensions/histogram/src/main/java/io/druid/query/aggregation/histogram/MaxPostAggregator.java b/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/MaxPostAggregator.java similarity index 100% rename from extensions/histogram/src/main/java/io/druid/query/aggregation/histogram/MaxPostAggregator.java rename to extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/MaxPostAggregator.java diff --git a/extensions/histogram/src/main/java/io/druid/query/aggregation/histogram/MinPostAggregator.java b/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/MinPostAggregator.java similarity index 100% rename from extensions/histogram/src/main/java/io/druid/query/aggregation/histogram/MinPostAggregator.java rename to extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/MinPostAggregator.java diff --git a/extensions/histogram/src/main/java/io/druid/query/aggregation/histogram/QuantilePostAggregator.java b/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/QuantilePostAggregator.java similarity index 100% rename from extensions/histogram/src/main/java/io/druid/query/aggregation/histogram/QuantilePostAggregator.java rename to extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/QuantilePostAggregator.java diff --git a/extensions/histogram/src/main/java/io/druid/query/aggregation/histogram/Quantiles.java b/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/Quantiles.java similarity index 100% rename from extensions/histogram/src/main/java/io/druid/query/aggregation/histogram/Quantiles.java rename to extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/Quantiles.java diff --git a/extensions/histogram/src/main/java/io/druid/query/aggregation/histogram/QuantilesPostAggregator.java b/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/QuantilesPostAggregator.java similarity index 100% rename from extensions/histogram/src/main/java/io/druid/query/aggregation/histogram/QuantilesPostAggregator.java rename to extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/QuantilesPostAggregator.java diff --git a/extensions/histogram/src/main/resources/META-INF/services/io.druid.initialization.DruidModule b/extensions-core/histogram/src/main/resources/META-INF/services/io.druid.initialization.DruidModule similarity index 100% rename from extensions/histogram/src/main/resources/META-INF/services/io.druid.initialization.DruidModule rename to extensions-core/histogram/src/main/resources/META-INF/services/io.druid.initialization.DruidModule diff --git a/extensions/histogram/src/test/java/io/druid/query/aggregation/histogram/ApproximateHistogramAggregationTest.java b/extensions-core/histogram/src/test/java/io/druid/query/aggregation/histogram/ApproximateHistogramAggregationTest.java similarity index 100% rename from extensions/histogram/src/test/java/io/druid/query/aggregation/histogram/ApproximateHistogramAggregationTest.java rename to extensions-core/histogram/src/test/java/io/druid/query/aggregation/histogram/ApproximateHistogramAggregationTest.java diff --git a/extensions/histogram/src/test/java/io/druid/query/aggregation/histogram/ApproximateHistogramAggregatorTest.java b/extensions-core/histogram/src/test/java/io/druid/query/aggregation/histogram/ApproximateHistogramAggregatorTest.java similarity index 100% rename from extensions/histogram/src/test/java/io/druid/query/aggregation/histogram/ApproximateHistogramAggregatorTest.java rename to extensions-core/histogram/src/test/java/io/druid/query/aggregation/histogram/ApproximateHistogramAggregatorTest.java diff --git a/extensions/histogram/src/test/java/io/druid/query/aggregation/histogram/ApproximateHistogramErrorBenchmark.java b/extensions-core/histogram/src/test/java/io/druid/query/aggregation/histogram/ApproximateHistogramErrorBenchmark.java similarity index 100% rename from extensions/histogram/src/test/java/io/druid/query/aggregation/histogram/ApproximateHistogramErrorBenchmark.java rename to extensions-core/histogram/src/test/java/io/druid/query/aggregation/histogram/ApproximateHistogramErrorBenchmark.java diff --git a/extensions/histogram/src/test/java/io/druid/query/aggregation/histogram/ApproximateHistogramGroupByQueryTest.java b/extensions-core/histogram/src/test/java/io/druid/query/aggregation/histogram/ApproximateHistogramGroupByQueryTest.java similarity index 100% rename from extensions/histogram/src/test/java/io/druid/query/aggregation/histogram/ApproximateHistogramGroupByQueryTest.java rename to extensions-core/histogram/src/test/java/io/druid/query/aggregation/histogram/ApproximateHistogramGroupByQueryTest.java diff --git a/extensions/histogram/src/test/java/io/druid/query/aggregation/histogram/ApproximateHistogramPostAggregatorTest.java b/extensions-core/histogram/src/test/java/io/druid/query/aggregation/histogram/ApproximateHistogramPostAggregatorTest.java similarity index 100% rename from extensions/histogram/src/test/java/io/druid/query/aggregation/histogram/ApproximateHistogramPostAggregatorTest.java rename to extensions-core/histogram/src/test/java/io/druid/query/aggregation/histogram/ApproximateHistogramPostAggregatorTest.java diff --git a/extensions/histogram/src/test/java/io/druid/query/aggregation/histogram/ApproximateHistogramTest.java b/extensions-core/histogram/src/test/java/io/druid/query/aggregation/histogram/ApproximateHistogramTest.java similarity index 100% rename from extensions/histogram/src/test/java/io/druid/query/aggregation/histogram/ApproximateHistogramTest.java rename to extensions-core/histogram/src/test/java/io/druid/query/aggregation/histogram/ApproximateHistogramTest.java diff --git a/extensions/histogram/src/test/java/io/druid/query/aggregation/histogram/ApproximateHistogramTopNQueryTest.java b/extensions-core/histogram/src/test/java/io/druid/query/aggregation/histogram/ApproximateHistogramTopNQueryTest.java similarity index 100% rename from extensions/histogram/src/test/java/io/druid/query/aggregation/histogram/ApproximateHistogramTopNQueryTest.java rename to extensions-core/histogram/src/test/java/io/druid/query/aggregation/histogram/ApproximateHistogramTopNQueryTest.java diff --git a/extensions/histogram/src/test/java/io/druid/query/aggregation/histogram/QuantilesTest.java b/extensions-core/histogram/src/test/java/io/druid/query/aggregation/histogram/QuantilesTest.java similarity index 100% rename from extensions/histogram/src/test/java/io/druid/query/aggregation/histogram/QuantilesTest.java rename to extensions-core/histogram/src/test/java/io/druid/query/aggregation/histogram/QuantilesTest.java diff --git a/extensions/histogram/src/test/resources/sample.data.tsv b/extensions-core/histogram/src/test/resources/sample.data.tsv similarity index 100% rename from extensions/histogram/src/test/resources/sample.data.tsv rename to extensions-core/histogram/src/test/resources/sample.data.tsv diff --git a/extensions/kafka-eight/pom.xml b/extensions-core/kafka-eight/pom.xml similarity index 100% rename from extensions/kafka-eight/pom.xml rename to extensions-core/kafka-eight/pom.xml diff --git a/extensions/kafka-eight/src/main/java/io/druid/firehose/kafka/KafkaEightDruidModule.java b/extensions-core/kafka-eight/src/main/java/io/druid/firehose/kafka/KafkaEightDruidModule.java similarity index 100% rename from extensions/kafka-eight/src/main/java/io/druid/firehose/kafka/KafkaEightDruidModule.java rename to extensions-core/kafka-eight/src/main/java/io/druid/firehose/kafka/KafkaEightDruidModule.java diff --git a/extensions/kafka-eight/src/main/java/io/druid/firehose/kafka/KafkaEightFirehoseFactory.java b/extensions-core/kafka-eight/src/main/java/io/druid/firehose/kafka/KafkaEightFirehoseFactory.java similarity index 100% rename from extensions/kafka-eight/src/main/java/io/druid/firehose/kafka/KafkaEightFirehoseFactory.java rename to extensions-core/kafka-eight/src/main/java/io/druid/firehose/kafka/KafkaEightFirehoseFactory.java diff --git a/extensions/kafka-eight/src/main/resources/META-INF/services/io.druid.initialization.DruidModule b/extensions-core/kafka-eight/src/main/resources/META-INF/services/io.druid.initialization.DruidModule similarity index 100% rename from extensions/kafka-eight/src/main/resources/META-INF/services/io.druid.initialization.DruidModule rename to extensions-core/kafka-eight/src/main/resources/META-INF/services/io.druid.initialization.DruidModule diff --git a/extensions/kafka-extraction-namespace/pom.xml b/extensions-core/kafka-extraction-namespace/pom.xml similarity index 100% rename from extensions/kafka-extraction-namespace/pom.xml rename to extensions-core/kafka-extraction-namespace/pom.xml diff --git a/extensions/kafka-extraction-namespace/src/main/java/io/druid/query/extraction/namespace/KafkaExtractionNamespace.java b/extensions-core/kafka-extraction-namespace/src/main/java/io/druid/query/extraction/namespace/KafkaExtractionNamespace.java similarity index 100% rename from extensions/kafka-extraction-namespace/src/main/java/io/druid/query/extraction/namespace/KafkaExtractionNamespace.java rename to extensions-core/kafka-extraction-namespace/src/main/java/io/druid/query/extraction/namespace/KafkaExtractionNamespace.java diff --git a/extensions/kafka-extraction-namespace/src/main/java/io/druid/server/namespace/KafkaExtractionManager.java b/extensions-core/kafka-extraction-namespace/src/main/java/io/druid/server/namespace/KafkaExtractionManager.java similarity index 100% rename from extensions/kafka-extraction-namespace/src/main/java/io/druid/server/namespace/KafkaExtractionManager.java rename to extensions-core/kafka-extraction-namespace/src/main/java/io/druid/server/namespace/KafkaExtractionManager.java diff --git a/extensions/kafka-extraction-namespace/src/main/java/io/druid/server/namespace/KafkaExtractionNamespaceFactory.java b/extensions-core/kafka-extraction-namespace/src/main/java/io/druid/server/namespace/KafkaExtractionNamespaceFactory.java similarity index 100% rename from extensions/kafka-extraction-namespace/src/main/java/io/druid/server/namespace/KafkaExtractionNamespaceFactory.java rename to extensions-core/kafka-extraction-namespace/src/main/java/io/druid/server/namespace/KafkaExtractionNamespaceFactory.java diff --git a/extensions/kafka-extraction-namespace/src/main/java/io/druid/server/namespace/KafkaExtractionNamespaceModule.java b/extensions-core/kafka-extraction-namespace/src/main/java/io/druid/server/namespace/KafkaExtractionNamespaceModule.java similarity index 100% rename from extensions/kafka-extraction-namespace/src/main/java/io/druid/server/namespace/KafkaExtractionNamespaceModule.java rename to extensions-core/kafka-extraction-namespace/src/main/java/io/druid/server/namespace/KafkaExtractionNamespaceModule.java diff --git a/extensions/kafka-extraction-namespace/src/main/resources/META-INF/services/io.druid.initialization.DruidModule b/extensions-core/kafka-extraction-namespace/src/main/resources/META-INF/services/io.druid.initialization.DruidModule similarity index 100% rename from extensions/kafka-extraction-namespace/src/main/resources/META-INF/services/io.druid.initialization.DruidModule rename to extensions-core/kafka-extraction-namespace/src/main/resources/META-INF/services/io.druid.initialization.DruidModule diff --git a/extensions/kafka-extraction-namespace/src/test/java/io/druid/query/extraction/namespace/KafkaExtractionNamespaceTest.java b/extensions-core/kafka-extraction-namespace/src/test/java/io/druid/query/extraction/namespace/KafkaExtractionNamespaceTest.java similarity index 100% rename from extensions/kafka-extraction-namespace/src/test/java/io/druid/query/extraction/namespace/KafkaExtractionNamespaceTest.java rename to extensions-core/kafka-extraction-namespace/src/test/java/io/druid/query/extraction/namespace/KafkaExtractionNamespaceTest.java diff --git a/extensions/kafka-extraction-namespace/src/test/java/io/druid/query/extraction/namespace/TestKafkaExtractionCluster.java b/extensions-core/kafka-extraction-namespace/src/test/java/io/druid/query/extraction/namespace/TestKafkaExtractionCluster.java similarity index 100% rename from extensions/kafka-extraction-namespace/src/test/java/io/druid/query/extraction/namespace/TestKafkaExtractionCluster.java rename to extensions-core/kafka-extraction-namespace/src/test/java/io/druid/query/extraction/namespace/TestKafkaExtractionCluster.java diff --git a/extensions/kafka-extraction-namespace/src/test/resources/log4j2.xml b/extensions-core/kafka-extraction-namespace/src/test/resources/log4j2.xml similarity index 100% rename from extensions/kafka-extraction-namespace/src/test/resources/log4j2.xml rename to extensions-core/kafka-extraction-namespace/src/test/resources/log4j2.xml diff --git a/extensions/mysql-metadata-storage/pom.xml b/extensions-core/mysql-metadata-storage/pom.xml similarity index 100% rename from extensions/mysql-metadata-storage/pom.xml rename to extensions-core/mysql-metadata-storage/pom.xml diff --git a/extensions/mysql-metadata-storage/src/main/java/io/druid/metadata/storage/mysql/MySQLConnector.java b/extensions-core/mysql-metadata-storage/src/main/java/io/druid/metadata/storage/mysql/MySQLConnector.java similarity index 100% rename from extensions/mysql-metadata-storage/src/main/java/io/druid/metadata/storage/mysql/MySQLConnector.java rename to extensions-core/mysql-metadata-storage/src/main/java/io/druid/metadata/storage/mysql/MySQLConnector.java diff --git a/extensions/mysql-metadata-storage/src/main/java/io/druid/metadata/storage/mysql/MySQLMetadataStorageModule.java b/extensions-core/mysql-metadata-storage/src/main/java/io/druid/metadata/storage/mysql/MySQLMetadataStorageModule.java similarity index 100% rename from extensions/mysql-metadata-storage/src/main/java/io/druid/metadata/storage/mysql/MySQLMetadataStorageModule.java rename to extensions-core/mysql-metadata-storage/src/main/java/io/druid/metadata/storage/mysql/MySQLMetadataStorageModule.java diff --git a/extensions/mysql-metadata-storage/src/main/resources/META-INF/services/io.druid.initialization.DruidModule b/extensions-core/mysql-metadata-storage/src/main/resources/META-INF/services/io.druid.initialization.DruidModule similarity index 100% rename from extensions/mysql-metadata-storage/src/main/resources/META-INF/services/io.druid.initialization.DruidModule rename to extensions-core/mysql-metadata-storage/src/main/resources/META-INF/services/io.druid.initialization.DruidModule diff --git a/extensions/namespace-lookup/pom.xml b/extensions-core/namespace-lookup/pom.xml similarity index 100% rename from extensions/namespace-lookup/pom.xml rename to extensions-core/namespace-lookup/pom.xml diff --git a/extensions/namespace-lookup/src/main/java/io/druid/data/input/MapPopulator.java b/extensions-core/namespace-lookup/src/main/java/io/druid/data/input/MapPopulator.java similarity index 100% rename from extensions/namespace-lookup/src/main/java/io/druid/data/input/MapPopulator.java rename to extensions-core/namespace-lookup/src/main/java/io/druid/data/input/MapPopulator.java diff --git a/extensions/namespace-lookup/src/main/java/io/druid/query/extraction/NamespacedExtractor.java b/extensions-core/namespace-lookup/src/main/java/io/druid/query/extraction/NamespacedExtractor.java similarity index 100% rename from extensions/namespace-lookup/src/main/java/io/druid/query/extraction/NamespacedExtractor.java rename to extensions-core/namespace-lookup/src/main/java/io/druid/query/extraction/NamespacedExtractor.java diff --git a/extensions/namespace-lookup/src/main/java/io/druid/query/extraction/namespace/ExtractionNamespace.java b/extensions-core/namespace-lookup/src/main/java/io/druid/query/extraction/namespace/ExtractionNamespace.java similarity index 100% rename from extensions/namespace-lookup/src/main/java/io/druid/query/extraction/namespace/ExtractionNamespace.java rename to extensions-core/namespace-lookup/src/main/java/io/druid/query/extraction/namespace/ExtractionNamespace.java diff --git a/extensions/namespace-lookup/src/main/java/io/druid/query/extraction/namespace/ExtractionNamespaceFunctionFactory.java b/extensions-core/namespace-lookup/src/main/java/io/druid/query/extraction/namespace/ExtractionNamespaceFunctionFactory.java similarity index 100% rename from extensions/namespace-lookup/src/main/java/io/druid/query/extraction/namespace/ExtractionNamespaceFunctionFactory.java rename to extensions-core/namespace-lookup/src/main/java/io/druid/query/extraction/namespace/ExtractionNamespaceFunctionFactory.java diff --git a/extensions/namespace-lookup/src/main/java/io/druid/query/extraction/namespace/JDBCExtractionNamespace.java b/extensions-core/namespace-lookup/src/main/java/io/druid/query/extraction/namespace/JDBCExtractionNamespace.java similarity index 100% rename from extensions/namespace-lookup/src/main/java/io/druid/query/extraction/namespace/JDBCExtractionNamespace.java rename to extensions-core/namespace-lookup/src/main/java/io/druid/query/extraction/namespace/JDBCExtractionNamespace.java diff --git a/extensions/namespace-lookup/src/main/java/io/druid/query/extraction/namespace/URIExtractionNamespace.java b/extensions-core/namespace-lookup/src/main/java/io/druid/query/extraction/namespace/URIExtractionNamespace.java similarity index 100% rename from extensions/namespace-lookup/src/main/java/io/druid/query/extraction/namespace/URIExtractionNamespace.java rename to extensions-core/namespace-lookup/src/main/java/io/druid/query/extraction/namespace/URIExtractionNamespace.java diff --git a/extensions/namespace-lookup/src/main/java/io/druid/server/initialization/NamespaceLookupStaticConfig.java b/extensions-core/namespace-lookup/src/main/java/io/druid/server/initialization/NamespaceLookupStaticConfig.java similarity index 100% rename from extensions/namespace-lookup/src/main/java/io/druid/server/initialization/NamespaceLookupStaticConfig.java rename to extensions-core/namespace-lookup/src/main/java/io/druid/server/initialization/NamespaceLookupStaticConfig.java diff --git a/extensions/namespace-lookup/src/main/java/io/druid/server/namespace/JDBCExtractionNamespaceFunctionFactory.java b/extensions-core/namespace-lookup/src/main/java/io/druid/server/namespace/JDBCExtractionNamespaceFunctionFactory.java similarity index 100% rename from extensions/namespace-lookup/src/main/java/io/druid/server/namespace/JDBCExtractionNamespaceFunctionFactory.java rename to extensions-core/namespace-lookup/src/main/java/io/druid/server/namespace/JDBCExtractionNamespaceFunctionFactory.java diff --git a/extensions/namespace-lookup/src/main/java/io/druid/server/namespace/NamespacedExtractionModule.java b/extensions-core/namespace-lookup/src/main/java/io/druid/server/namespace/NamespacedExtractionModule.java similarity index 100% rename from extensions/namespace-lookup/src/main/java/io/druid/server/namespace/NamespacedExtractionModule.java rename to extensions-core/namespace-lookup/src/main/java/io/druid/server/namespace/NamespacedExtractionModule.java diff --git a/extensions/namespace-lookup/src/main/java/io/druid/server/namespace/URIExtractionNamespaceFunctionFactory.java b/extensions-core/namespace-lookup/src/main/java/io/druid/server/namespace/URIExtractionNamespaceFunctionFactory.java similarity index 100% rename from extensions/namespace-lookup/src/main/java/io/druid/server/namespace/URIExtractionNamespaceFunctionFactory.java rename to extensions-core/namespace-lookup/src/main/java/io/druid/server/namespace/URIExtractionNamespaceFunctionFactory.java diff --git a/extensions/namespace-lookup/src/main/java/io/druid/server/namespace/cache/NamespaceExtractionCacheManager.java b/extensions-core/namespace-lookup/src/main/java/io/druid/server/namespace/cache/NamespaceExtractionCacheManager.java similarity index 100% rename from extensions/namespace-lookup/src/main/java/io/druid/server/namespace/cache/NamespaceExtractionCacheManager.java rename to extensions-core/namespace-lookup/src/main/java/io/druid/server/namespace/cache/NamespaceExtractionCacheManager.java diff --git a/extensions/namespace-lookup/src/main/java/io/druid/server/namespace/cache/OffHeapNamespaceExtractionCacheManager.java b/extensions-core/namespace-lookup/src/main/java/io/druid/server/namespace/cache/OffHeapNamespaceExtractionCacheManager.java similarity index 100% rename from extensions/namespace-lookup/src/main/java/io/druid/server/namespace/cache/OffHeapNamespaceExtractionCacheManager.java rename to extensions-core/namespace-lookup/src/main/java/io/druid/server/namespace/cache/OffHeapNamespaceExtractionCacheManager.java diff --git a/extensions/namespace-lookup/src/main/java/io/druid/server/namespace/cache/OnHeapNamespaceExtractionCacheManager.java b/extensions-core/namespace-lookup/src/main/java/io/druid/server/namespace/cache/OnHeapNamespaceExtractionCacheManager.java similarity index 100% rename from extensions/namespace-lookup/src/main/java/io/druid/server/namespace/cache/OnHeapNamespaceExtractionCacheManager.java rename to extensions-core/namespace-lookup/src/main/java/io/druid/server/namespace/cache/OnHeapNamespaceExtractionCacheManager.java diff --git a/extensions/namespace-lookup/src/main/java/io/druid/server/namespace/http/NamespacesCacheResource.java b/extensions-core/namespace-lookup/src/main/java/io/druid/server/namespace/http/NamespacesCacheResource.java similarity index 100% rename from extensions/namespace-lookup/src/main/java/io/druid/server/namespace/http/NamespacesCacheResource.java rename to extensions-core/namespace-lookup/src/main/java/io/druid/server/namespace/http/NamespacesCacheResource.java diff --git a/extensions/namespace-lookup/src/main/resources/META-INF/services/io.druid.initialization.DruidModule b/extensions-core/namespace-lookup/src/main/resources/META-INF/services/io.druid.initialization.DruidModule similarity index 100% rename from extensions/namespace-lookup/src/main/resources/META-INF/services/io.druid.initialization.DruidModule rename to extensions-core/namespace-lookup/src/main/resources/META-INF/services/io.druid.initialization.DruidModule diff --git a/extensions/namespace-lookup/src/test/java/io/druid/query/extraction/namespace/NamespacedExtractorTest.java b/extensions-core/namespace-lookup/src/test/java/io/druid/query/extraction/namespace/NamespacedExtractorTest.java similarity index 100% rename from extensions/namespace-lookup/src/test/java/io/druid/query/extraction/namespace/NamespacedExtractorTest.java rename to extensions-core/namespace-lookup/src/test/java/io/druid/query/extraction/namespace/NamespacedExtractorTest.java diff --git a/extensions/namespace-lookup/src/test/java/io/druid/query/extraction/namespace/URIExtractionNamespaceTest.java b/extensions-core/namespace-lookup/src/test/java/io/druid/query/extraction/namespace/URIExtractionNamespaceTest.java similarity index 100% rename from extensions/namespace-lookup/src/test/java/io/druid/query/extraction/namespace/URIExtractionNamespaceTest.java rename to extensions-core/namespace-lookup/src/test/java/io/druid/query/extraction/namespace/URIExtractionNamespaceTest.java diff --git a/extensions/namespace-lookup/src/test/java/io/druid/server/namespace/NamespacedExtractorModuleTest.java b/extensions-core/namespace-lookup/src/test/java/io/druid/server/namespace/NamespacedExtractorModuleTest.java similarity index 100% rename from extensions/namespace-lookup/src/test/java/io/druid/server/namespace/NamespacedExtractorModuleTest.java rename to extensions-core/namespace-lookup/src/test/java/io/druid/server/namespace/NamespacedExtractorModuleTest.java diff --git a/extensions/namespace-lookup/src/test/java/io/druid/server/namespace/URIExtractionNamespaceFunctionFactoryTest.java b/extensions-core/namespace-lookup/src/test/java/io/druid/server/namespace/URIExtractionNamespaceFunctionFactoryTest.java similarity index 100% rename from extensions/namespace-lookup/src/test/java/io/druid/server/namespace/URIExtractionNamespaceFunctionFactoryTest.java rename to extensions-core/namespace-lookup/src/test/java/io/druid/server/namespace/URIExtractionNamespaceFunctionFactoryTest.java diff --git a/extensions/namespace-lookup/src/test/java/io/druid/server/namespace/cache/JDBCExtractionNamespaceTest.java b/extensions-core/namespace-lookup/src/test/java/io/druid/server/namespace/cache/JDBCExtractionNamespaceTest.java similarity index 100% rename from extensions/namespace-lookup/src/test/java/io/druid/server/namespace/cache/JDBCExtractionNamespaceTest.java rename to extensions-core/namespace-lookup/src/test/java/io/druid/server/namespace/cache/JDBCExtractionNamespaceTest.java diff --git a/extensions/namespace-lookup/src/test/java/io/druid/server/namespace/cache/NamespaceExtractionCacheManagerExecutorsTest.java b/extensions-core/namespace-lookup/src/test/java/io/druid/server/namespace/cache/NamespaceExtractionCacheManagerExecutorsTest.java similarity index 100% rename from extensions/namespace-lookup/src/test/java/io/druid/server/namespace/cache/NamespaceExtractionCacheManagerExecutorsTest.java rename to extensions-core/namespace-lookup/src/test/java/io/druid/server/namespace/cache/NamespaceExtractionCacheManagerExecutorsTest.java diff --git a/extensions/namespace-lookup/src/test/java/io/druid/server/namespace/cache/NamespaceExtractionCacheManagersTest.java b/extensions-core/namespace-lookup/src/test/java/io/druid/server/namespace/cache/NamespaceExtractionCacheManagersTest.java similarity index 100% rename from extensions/namespace-lookup/src/test/java/io/druid/server/namespace/cache/NamespaceExtractionCacheManagersTest.java rename to extensions-core/namespace-lookup/src/test/java/io/druid/server/namespace/cache/NamespaceExtractionCacheManagersTest.java diff --git a/extensions/postgresql-metadata-storage/pom.xml b/extensions-core/postgresql-metadata-storage/pom.xml similarity index 100% rename from extensions/postgresql-metadata-storage/pom.xml rename to extensions-core/postgresql-metadata-storage/pom.xml diff --git a/extensions/postgresql-metadata-storage/src/main/java/io/druid/metadata/storage/postgresql/PostgreSQLConnector.java b/extensions-core/postgresql-metadata-storage/src/main/java/io/druid/metadata/storage/postgresql/PostgreSQLConnector.java similarity index 100% rename from extensions/postgresql-metadata-storage/src/main/java/io/druid/metadata/storage/postgresql/PostgreSQLConnector.java rename to extensions-core/postgresql-metadata-storage/src/main/java/io/druid/metadata/storage/postgresql/PostgreSQLConnector.java diff --git a/extensions/postgresql-metadata-storage/src/main/java/io/druid/metadata/storage/postgresql/PostgreSQLMetadataStorageModule.java b/extensions-core/postgresql-metadata-storage/src/main/java/io/druid/metadata/storage/postgresql/PostgreSQLMetadataStorageModule.java similarity index 100% rename from extensions/postgresql-metadata-storage/src/main/java/io/druid/metadata/storage/postgresql/PostgreSQLMetadataStorageModule.java rename to extensions-core/postgresql-metadata-storage/src/main/java/io/druid/metadata/storage/postgresql/PostgreSQLMetadataStorageModule.java diff --git a/extensions/postgresql-metadata-storage/src/main/resources/META-INF/services/io.druid.initialization.DruidModule b/extensions-core/postgresql-metadata-storage/src/main/resources/META-INF/services/io.druid.initialization.DruidModule similarity index 100% rename from extensions/postgresql-metadata-storage/src/main/resources/META-INF/services/io.druid.initialization.DruidModule rename to extensions-core/postgresql-metadata-storage/src/main/resources/META-INF/services/io.druid.initialization.DruidModule diff --git a/extensions/postgresql-metadata-storage/src/test/java/io/druid/metadata/storage/postgresql/PostgreSQLConnectorTest.java b/extensions-core/postgresql-metadata-storage/src/test/java/io/druid/metadata/storage/postgresql/PostgreSQLConnectorTest.java similarity index 100% rename from extensions/postgresql-metadata-storage/src/test/java/io/druid/metadata/storage/postgresql/PostgreSQLConnectorTest.java rename to extensions-core/postgresql-metadata-storage/src/test/java/io/druid/metadata/storage/postgresql/PostgreSQLConnectorTest.java diff --git a/extensions/s3-extensions/pom.xml b/extensions-core/s3-extensions/pom.xml similarity index 100% rename from extensions/s3-extensions/pom.xml rename to extensions-core/s3-extensions/pom.xml diff --git a/extensions/s3-extensions/src/main/java/io/druid/firehose/s3/S3FirehoseDruidModule.java b/extensions-core/s3-extensions/src/main/java/io/druid/firehose/s3/S3FirehoseDruidModule.java similarity index 100% rename from extensions/s3-extensions/src/main/java/io/druid/firehose/s3/S3FirehoseDruidModule.java rename to extensions-core/s3-extensions/src/main/java/io/druid/firehose/s3/S3FirehoseDruidModule.java diff --git a/extensions/s3-extensions/src/main/java/io/druid/firehose/s3/StaticS3FirehoseFactory.java b/extensions-core/s3-extensions/src/main/java/io/druid/firehose/s3/StaticS3FirehoseFactory.java similarity index 100% rename from extensions/s3-extensions/src/main/java/io/druid/firehose/s3/StaticS3FirehoseFactory.java rename to extensions-core/s3-extensions/src/main/java/io/druid/firehose/s3/StaticS3FirehoseFactory.java diff --git a/extensions/s3-extensions/src/main/java/io/druid/storage/s3/AWSSessionCredentialsAdapter.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/AWSSessionCredentialsAdapter.java similarity index 100% rename from extensions/s3-extensions/src/main/java/io/druid/storage/s3/AWSSessionCredentialsAdapter.java rename to extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/AWSSessionCredentialsAdapter.java diff --git a/extensions/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentArchiver.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentArchiver.java similarity index 100% rename from extensions/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentArchiver.java rename to extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentArchiver.java diff --git a/extensions/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentArchiverConfig.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentArchiverConfig.java similarity index 100% rename from extensions/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentArchiverConfig.java rename to extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentArchiverConfig.java diff --git a/extensions/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentKiller.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentKiller.java similarity index 100% rename from extensions/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentKiller.java rename to extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentKiller.java diff --git a/extensions/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentMover.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentMover.java similarity index 100% rename from extensions/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentMover.java rename to extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentMover.java diff --git a/extensions/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPuller.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPuller.java similarity index 100% rename from extensions/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPuller.java rename to extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPuller.java diff --git a/extensions/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPusher.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPusher.java similarity index 100% rename from extensions/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPusher.java rename to extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPusher.java diff --git a/extensions/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPusherConfig.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPusherConfig.java similarity index 100% rename from extensions/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPusherConfig.java rename to extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPusherConfig.java diff --git a/extensions/s3-extensions/src/main/java/io/druid/storage/s3/S3LoadSpec.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3LoadSpec.java similarity index 100% rename from extensions/s3-extensions/src/main/java/io/druid/storage/s3/S3LoadSpec.java rename to extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3LoadSpec.java diff --git a/extensions/s3-extensions/src/main/java/io/druid/storage/s3/S3StorageDruidModule.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3StorageDruidModule.java similarity index 100% rename from extensions/s3-extensions/src/main/java/io/druid/storage/s3/S3StorageDruidModule.java rename to extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3StorageDruidModule.java diff --git a/extensions/s3-extensions/src/main/java/io/druid/storage/s3/S3TaskLogs.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3TaskLogs.java similarity index 100% rename from extensions/s3-extensions/src/main/java/io/druid/storage/s3/S3TaskLogs.java rename to extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3TaskLogs.java diff --git a/extensions/s3-extensions/src/main/java/io/druid/storage/s3/S3TaskLogsConfig.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3TaskLogsConfig.java similarity index 100% rename from extensions/s3-extensions/src/main/java/io/druid/storage/s3/S3TaskLogsConfig.java rename to extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3TaskLogsConfig.java diff --git a/extensions/s3-extensions/src/main/java/io/druid/storage/s3/S3TimestampVersionedDataFinder.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3TimestampVersionedDataFinder.java similarity index 100% rename from extensions/s3-extensions/src/main/java/io/druid/storage/s3/S3TimestampVersionedDataFinder.java rename to extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3TimestampVersionedDataFinder.java diff --git a/extensions/s3-extensions/src/main/java/io/druid/storage/s3/S3Utils.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3Utils.java similarity index 100% rename from extensions/s3-extensions/src/main/java/io/druid/storage/s3/S3Utils.java rename to extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3Utils.java diff --git a/extensions/s3-extensions/src/main/resources/META-INF/services/io.druid.initialization.DruidModule b/extensions-core/s3-extensions/src/main/resources/META-INF/services/io.druid.initialization.DruidModule similarity index 100% rename from extensions/s3-extensions/src/main/resources/META-INF/services/io.druid.initialization.DruidModule rename to extensions-core/s3-extensions/src/main/resources/META-INF/services/io.druid.initialization.DruidModule diff --git a/extensions/s3-extensions/src/test/java/io/druid/firehose/s3/StaticS3FirehoseFactoryTest.java b/extensions-core/s3-extensions/src/test/java/io/druid/firehose/s3/StaticS3FirehoseFactoryTest.java similarity index 100% rename from extensions/s3-extensions/src/test/java/io/druid/firehose/s3/StaticS3FirehoseFactoryTest.java rename to extensions-core/s3-extensions/src/test/java/io/druid/firehose/s3/StaticS3FirehoseFactoryTest.java diff --git a/extensions/s3-extensions/src/test/java/io/druid/storage/s3/S3DataSegmentMoverTest.java b/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/S3DataSegmentMoverTest.java similarity index 100% rename from extensions/s3-extensions/src/test/java/io/druid/storage/s3/S3DataSegmentMoverTest.java rename to extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/S3DataSegmentMoverTest.java diff --git a/extensions/s3-extensions/src/test/java/io/druid/storage/s3/S3DataSegmentPullerTest.java b/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/S3DataSegmentPullerTest.java similarity index 100% rename from extensions/s3-extensions/src/test/java/io/druid/storage/s3/S3DataSegmentPullerTest.java rename to extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/S3DataSegmentPullerTest.java diff --git a/extensions/s3-extensions/src/test/java/io/druid/storage/s3/S3DataSegmentPusherTest.java b/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/S3DataSegmentPusherTest.java similarity index 100% rename from extensions/s3-extensions/src/test/java/io/druid/storage/s3/S3DataSegmentPusherTest.java rename to extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/S3DataSegmentPusherTest.java diff --git a/extensions/s3-extensions/src/test/java/io/druid/storage/s3/S3TimestampVersionedDataFinderTest.java b/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/S3TimestampVersionedDataFinderTest.java similarity index 100% rename from extensions/s3-extensions/src/test/java/io/druid/storage/s3/S3TimestampVersionedDataFinderTest.java rename to extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/S3TimestampVersionedDataFinderTest.java diff --git a/extensions/s3-extensions/src/test/java/io/druid/storage/s3/TestAWSCredentialsProvider.java b/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/TestAWSCredentialsProvider.java similarity index 100% rename from extensions/s3-extensions/src/test/java/io/druid/storage/s3/TestAWSCredentialsProvider.java rename to extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/TestAWSCredentialsProvider.java diff --git a/extensions/s3-extensions/src/test/java/io/druid/storage/s3/TestFileSessionCredentialsProvider.java b/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/TestFileSessionCredentialsProvider.java similarity index 100% rename from extensions/s3-extensions/src/test/java/io/druid/storage/s3/TestFileSessionCredentialsProvider.java rename to extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/TestFileSessionCredentialsProvider.java diff --git a/extensions/cloudfiles-extensions/src/main/java/io/druid/firehose/cloudfiles/StaticCloudFilesFirehoseFactory.java b/extensions/cloudfiles-extensions/src/main/java/io/druid/firehose/cloudfiles/StaticCloudFilesFirehoseFactory.java deleted file mode 100644 index c3d1c0c7ad1..00000000000 --- a/extensions/cloudfiles-extensions/src/main/java/io/druid/firehose/cloudfiles/StaticCloudFilesFirehoseFactory.java +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Licensed to Metamarkets Group Inc. (Metamarkets) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Metamarkets licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package io.druid.firehose.cloudfiles; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; - -import com.fasterxml.jackson.annotation.JacksonInject; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.base.Charsets; -import com.google.common.base.Preconditions; -import com.google.common.base.Throwables; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; -import com.metamx.common.CompressionUtils; -import com.metamx.common.logger.Logger; -import com.metamx.common.parsers.ParseException; - -import io.druid.data.input.impl.FileIteratingFirehose; -import io.druid.data.input.Firehose; -import io.druid.data.input.FirehoseFactory; - -import org.apache.commons.io.IOUtils; -import org.apache.commons.io.LineIterator; -import org.jclouds.rackspace.cloudfiles.v1.CloudFilesApi; - -import io.druid.data.input.impl.StringInputRowParser; -import io.druid.storage.cloudfiles.CloudFilesByteSource; -import io.druid.storage.cloudfiles.CloudFilesObjectApiProxy; - -public class StaticCloudFilesFirehoseFactory implements FirehoseFactory -{ - private static final Logger log = new Logger(StaticCloudFilesFirehoseFactory.class); - - private final CloudFilesApi cloudFilesApi; - private final List blobs; - - @JsonCreator - public StaticCloudFilesFirehoseFactory( - @JacksonInject("objectApi") CloudFilesApi cloudFilesApi, - @JsonProperty("blobs") CloudFilesBlob[] blobs) - { - this.cloudFilesApi = cloudFilesApi; - this.blobs = ImmutableList.copyOf(blobs); - } - - @JsonProperty - public List getBlobs() - { - return blobs; - } - - @Override - public Firehose connect(StringInputRowParser stringInputRowParser) throws IOException, ParseException - { - Preconditions.checkNotNull(cloudFilesApi, "null cloudFilesApi"); - - final LinkedList objectQueue = Lists.newLinkedList(blobs); - - return new FileIteratingFirehose( - new Iterator() - { - - @Override - public boolean hasNext() - { - return !objectQueue.isEmpty(); - } - - @Override - public LineIterator next() - { - final CloudFilesBlob nextURI = objectQueue.poll(); - - final String region = nextURI.getRegion(); - final String container = nextURI.getContainer(); - final String path = nextURI.getPath(); - - log.info("Retrieving file from region[%s], container[%s] and path [%s]", - region, container, path); - CloudFilesObjectApiProxy objectApi = new CloudFilesObjectApiProxy( - cloudFilesApi, region, container); - final CloudFilesByteSource byteSource = new CloudFilesByteSource(objectApi, path); - - try - { - final InputStream innerInputStream = byteSource.openStream(); - final InputStream outerInputStream = path.endsWith(".gz") - ? CompressionUtils.gzipInputStream(innerInputStream) - : innerInputStream; - - return IOUtils.lineIterator( - new BufferedReader( - new InputStreamReader(outerInputStream, Charsets.UTF_8))); - } catch (IOException e) - { - log.error(e, - "Exception opening container[%s] blob[%s] from region[%s]", - container, path, region); - - throw Throwables.propagate(e); - } - } - - @Override - public void remove() - { - throw new UnsupportedOperationException(); - } - - }, - stringInputRowParser); - } - -} diff --git a/pom.xml b/pom.xml index 4d5472812fe..e17b3b58bd2 100644 --- a/pom.xml +++ b/pom.xml @@ -16,7 +16,8 @@ ~ limitations under the License. --> - + 4.0.0 @@ -80,30 +81,28 @@ integration-tests benchmarks aws-common - - extensions/cassandra-storage - extensions/hdfs-storage - extensions/s3-extensions - extensions/graphite-emitter - - - extensions/kafka-eight - extensions/kafka-eight-simpleConsumer - extensions/druid-rocketmq - extensions/rabbitmq - extensions/histogram - extensions/mysql-metadata-storage - extensions/postgresql-metadata-storage - extensions/azure-extensions - extensions/namespace-lookup - extensions/kafka-extraction-namespace - extensions/cloudfiles-extensions - extensions/datasketches - extensions/avro-extensions + + extensions-core/avro-extensions + extensions-core/datasketches + extensions-core/hdfs-storage + extensions-core/histogram + extensions-core/kafka-eight + extensions-core/kafka-extraction-namespace + extensions-core/mysql-metadata-storage + extensions-core/postgresql-metadata-storage + extensions-core/namespace-lookup + extensions-core/s3-extensions + + extensions-contrib/azure-extensions + extensions-contrib/cassandra-storage + extensions-contrib/druid-rocketmq + extensions-contrib/cloudfiles-extensions + extensions-contrib/graphite-emitter + extensions-contrib/kafka-eight-simpleConsumer + extensions-contrib/rabbitmq + distribution - @@ -405,15 +404,15 @@ ${log4j.version} - + org.apache.logging.log4j log4j-jul ${log4j.version} @@ -615,7 +614,9 @@ - -Xmx1024m -Duser.language=en -Duser.country=US -Dfile.encoding=UTF-8 -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager + -Xmx1024m -Duser.language=en -Duser.country=US -Dfile.encoding=UTF-8 + -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager + true