Move Azure extension into Core (#9394)

* Move Azure extension into Core

Moving the azure extension into Core.

* * Fix build failure

* * Add The MIT License (MIT) to list of compatible licenses

* * Address review comments

* * change reference to contrib azure to core azure

* * Fix spelling mistakes.

distribution/bin/check-licenses.py

@@ -269,6 +269,7 @@ def build_compatible_license_names():
 
     compatible_licenses['The MIT License'] = 'MIT License'
     compatible_licenses['MIT License'] = 'MIT License'
+    compatible_licenses['The MIT License (MIT)'] = 'MIT License'
 
     compatible_licenses['-'] = '-'
     return compatible_licenses

distribution/pom.xml

@@ -197,6 +197,8 @@
                                         <argument>-c</argument>
                                         <argument>org.apache.druid.extensions:druid-avro-extensions</argument>
                                         <argument>-c</argument>
+                                        <argument>org.apache.druid.extensions:druid-azure-extensions</argument>
+                                        <argument>-c</argument>
                                         <argument>org.apache.druid.extensions:druid-bloom-filter</argument>
                                         <argument>-c</argument>
                                         <argument>org.apache.druid.extensions:druid-datasketches</argument>
@@ -382,8 +384,6 @@
                                         <argument>-c</argument>
                                         <argument>org.apache.druid.extensions.contrib:dropwizard-emitter</argument>
                                         <argument>-c</argument>
-                                        <argument>org.apache.druid.extensions.contrib:druid-azure-extensions</argument>
-                                        <argument>-c</argument>
                                         <argument>org.apache.druid.extensions.contrib:druid-cassandra-storage</argument>
                                         <argument>-c</argument>
                                         <argument>org.apache.druid.extensions.contrib:druid-cloudfiles-extensions</argument>

docs/development/extensions.md

@@ -37,6 +37,7 @@ Core extensions are maintained by Druid committers.
 |Name|Description|Docs|
 |----|-----------|----|
 |druid-avro-extensions|Support for data in Apache Avro data format.|[link](../development/extensions-core/avro.md)|
+|druid-azure-extensions|Microsoft Azure deep storage.|[link](../development/extensions-core/azure.md)|
 |druid-basic-security|Support for Basic HTTP authentication and role-based access control.|[link](../development/extensions-core/druid-basic-security.md)|
 |druid-bloom-filter|Support for providing Bloom filters in druid queries.|[link](../development/extensions-core/bloom-filter.md)|
 |druid-caffeine-cache|A local cache implementation backed by Caffeine.|[link](../configuration/index.md#cache-configuration)|
@@ -72,7 +73,6 @@ All of these community extensions can be downloaded using [pull-deps](../operati
 |Name|Description|Docs|
 |----|-----------|----|
 |ambari-metrics-emitter|Ambari Metrics Emitter |[link](../development/extensions-contrib/ambari-metrics-emitter.md)|
-|druid-azure-extensions|Microsoft Azure deep storage.|[link](../development/extensions-contrib/azure.md)|
 |druid-cassandra-storage|Apache Cassandra deep storage.|[link](../development/extensions-contrib/cassandra.md)|
 |druid-cloudfiles-extensions|Rackspace Cloudfiles deep storage and firehose.|[link](../development/extensions-contrib/cloudfiles.md)|
 |druid-distinctcount|DistinctCount aggregator|[link](../development/extensions-contrib/distinctcount.md)|

docs/ingestion/native-batch.md

@@ -929,7 +929,7 @@ Google Cloud Storage object:
 
 ### Azure Input Source
 
-> You need to include the [`druid-azure-extensions`](../development/extensions-contrib/azure.md) as an extension to use the Azure input source.
+> You need to include the [`druid-azure-extensions`](../development/extensions-core/azure.md) as an extension to use the Azure input source.
 
 The Azure input source is to support reading objects directly from Azure Blob store. Objects can be
 specified as list of Azure Blob store URI strings. The Azure input source is splittable and can be used

extensions-core/azure-extensions/pom.xml

@@ -22,7 +22,7 @@
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
     <modelVersion>4.0.0</modelVersion>
 
-    <groupId>org.apache.druid.extensions.contrib</groupId>
+    <groupId>org.apache.druid.extensions</groupId>
     <artifactId>druid-azure-extensions</artifactId>
     <name>druid-azure-extensions</name>
     <description>druid-azure-extensions</description>

licenses.yaml

@@ -4047,6 +4047,28 @@ libraries:
 
 ---
 
+name: Microsoft Azure SDK For Key Vault Core
+license_category: binary
+module: extensions/druid-azure-extensions
+license_name: MIT License
+copyright: Microsoft
+version: 1.0.0
+libraries:
+  - com.microsoft.azure: azure-keyvault-core
+
+---
+
+name: Microsoft Azure Storage Client SDK
+license_category: binary
+module: extensions/druid-azure-extensions
+license_name: Apache License version 2.0
+copyright: Microsoft
+version: 8.6.0
+libraries:
+  - com.microsoft.azure: azure-storage
+
+---
+
 # Web console modules start
 name: "@babel/runtime"
 license_category: binary

pom.xml

@@ -137,6 +137,7 @@
         <module>cloud/gcp-common</module>
         <!-- Core extensions -->
         <module>extensions-core/avro-extensions</module>
+        <module>extensions-core/azure-extensions</module>
         <module>extensions-core/datasketches</module>
         <module>extensions-core/druid-bloom-filter</module>
         <module>extensions-core/druid-kerberos</module>
@@ -160,7 +161,6 @@
         <module>extensions-core/google-extensions</module>
         <!-- Community extensions -->
         <module>extensions-contrib/influx-extensions</module>
-        <module>extensions-contrib/azure-extensions</module>
         <module>extensions-contrib/cassandra-storage</module>
         <module>extensions-contrib/dropwizard-emitter</module>
         <module>extensions-contrib/cloudfiles-extensions</module>

website/.spelling

@@ -442,7 +442,7 @@ Pdist
 ambari-metrics
 metricName
 trustStore
- - ../docs/development/extensions-contrib/azure.md
+ - ../docs/development/extensions-core/azure.md
 StaticAzureBlobStoreFirehose
 StaticS3Firehose
 fetchTimeout
@@ -457,6 +457,13 @@ StaticCloudFilesFirehose
 cloudfiles
 rackspace-cloudfiles-uk
 rackspace-cloudfiles-us
+StaticAzureBlobStoreFirehose
+gz
+shardSpecs
+maxCacheCapacityBytes
+maxFetchCapacityBytes
+fetchTimeout
+maxFetchRetry
  - ../docs/development/extensions-contrib/distinctcount.md
 distinctCount
 groupBy
@@ -973,6 +980,8 @@ splitHintSpec
 taskStatusCheckPeriodMs
 timeChunk
 totalNumMergeTasks
+StaticS3Firehose
+prefetchTriggerBytes
  - ../docs/ingestion/schema-design.md
 product_category
 product_id

website/i18n/en.json

@@ -92,9 +92,6 @@
       "development/extensions-contrib/ambari-metrics-emitter": {
         "title": "Ambari Metrics Emitter"
       },
-      "development/extensions-contrib/azure": {
-        "title": "Microsoft Azure"
-      },
       "development/extensions-contrib/cassandra": {
         "title": "Apache Cassandra"
       },
@@ -152,6 +149,9 @@
       "development/extensions-core/avro": {
         "title": "Apache Avro"
       },
+      "development/extensions-core/azure": {
+        "title": "Microsoft Azure"
+      },
       "development/extensions-core/bloom-filter": {
         "title": "Bloom Filter"
       },

website/redirects.json

@@ -132,7 +132,7 @@
 {"source": "design/plumber.html", "target": "../ingestion/standalone-realtime.html"}
 {"source": "design/realtime.html", "target": "../ingestion/standalone-realtime.html"}
 {"source": "development/approximate-histograms.html", "target": "extensions-core/approximate-histograms.html"}
-{"source": "development/community-extensions/azure.html", "target": "../extensions-contrib/azure.html"}
+{"source": "development/community-extensions/azure.html", "target": "../extensions-core/azure.html"}
 {"source": "development/community-extensions/cassandra.html", "target": "../extensions-contrib/cassandra.html"}
 {"source": "development/community-extensions/cloudfiles.html", "target": "../extensions-contrib/cloudfiles.html"}
 {"source": "development/community-extensions/graphite.html", "target": "../extensions-contrib/graphite.html"}

website/sidebars.json

@@ -147,6 +147,7 @@
       "design/peons",
       "development/extensions-core/approximate-histograms",
       "development/extensions-core/avro",
+      "development/extensions-core/azure",
       "development/extensions-core/bloom-filter",
       "development/extensions-core/datasketches-extension",
       "development/extensions-core/datasketches-hll",
@@ -170,7 +171,6 @@
       "development/extensions-core/stats",
       "development/extensions-core/test-stats",
       "development/extensions-contrib/ambari-metrics-emitter",
-      "development/extensions-contrib/azure",
       "development/extensions-contrib/cassandra",
       "development/extensions-contrib/cloudfiles",
       "development/extensions-contrib/distinctcount",