From d771b42ed148fc28abf6e246a8e2895b2734cf3c Mon Sep 17 00:00:00 2001 From: zachjsh Date: Tue, 25 Feb 2020 17:49:16 -0800 Subject: [PATCH] Move Azure extension into Core (#9394) * Move Azure extension into Core Moving the azure extension into Core. * * Fix build failure * * Add The MIT License (MIT) to list of compatible licenses * * Address review comments * * change reference to contrib azure to core azure * * Fix spelling mistakes. --- distribution/bin/check-licenses.py | 1 + distribution/pom.xml | 4 ++-- .../azure.md | 0 docs/development/extensions.md | 2 +- docs/ingestion/native-batch.md | 2 +- .../azure-extensions/pom.xml | 2 +- .../druid/data/input/azure/AzureEntity.java | 0 .../data/input/azure/AzureEntityFactory.java | 0 .../data/input/azure/AzureInputSource.java | 0 .../druid/firehose/azure/AzureBlob.java | 0 .../StaticAzureBlobStoreFirehoseFactory.java | 0 .../storage/azure/AzureAccountConfig.java | 0 .../druid/storage/azure/AzureByteSource.java | 0 .../storage/azure/AzureByteSourceFactory.java | 0 ...bHolderToCloudObjectLocationConverter.java | 0 .../storage/azure/AzureCloudBlobIterable.java | 0 .../azure/AzureCloudBlobIterableFactory.java | 0 .../storage/azure/AzureCloudBlobIterator.java | 0 .../azure/AzureCloudBlobIteratorFactory.java | 0 .../storage/azure/AzureDataSegmentConfig.java | 0 .../storage/azure/AzureDataSegmentKiller.java | 0 .../storage/azure/AzureDataSegmentPuller.java | 0 .../storage/azure/AzureDataSegmentPusher.java | 0 .../storage/azure/AzureInputDataConfig.java | 0 .../druid/storage/azure/AzureLoadSpec.java | 0 .../druid/storage/azure/AzureStorage.java | 0 .../azure/AzureStorageDruidModule.java | 0 .../druid/storage/azure/AzureTaskLogs.java | 0 .../storage/azure/AzureTaskLogsConfig.java | 0 .../druid/storage/azure/AzureUtils.java | 0 ...cObjectToCloudObjectLocationConverter.java | 0 .../storage/azure/blob/CloudBlobHolder.java | 0 .../azure/blob/ListBlobItemHolder.java | 0 .../azure/blob/ListBlobItemHolderFactory.java | 0 ...rg.apache.druid.initialization.DruidModule | 0 .../data/input/azure/AzureEntityTest.java | 0 .../azure/AzureInputSourceSerdeTest.java | 0 .../input/azure/AzureInputSourceTest.java | 0 ...aticAzureBlobStoreFirehoseFactoryTest.java | 0 .../storage/azure/AzureByteSourceTest.java | 0 ...derToCloudObjectLocationConverterTest.java | 0 .../azure/AzureCloudBlobIteratorTest.java | 0 .../azure/AzureDataSegmentKillerTest.java | 0 .../azure/AzureDataSegmentPullerTest.java | 0 .../azure/AzureDataSegmentPusherTest.java | 0 .../azure/AzureStorageDruidModuleTest.java | 0 .../storage/azure/AzureTaskLogsTest.java | 0 .../druid/storage/azure/AzureTestUtils.java | 0 .../druid/storage/azure/AzureUtilsTest.java | 0 licenses.yaml | 22 +++++++++++++++++++ pom.xml | 2 +- website/.spelling | 11 +++++++++- website/i18n/en.json | 6 ++--- website/redirects.json | 2 +- website/sidebars.json | 2 +- 55 files changed, 44 insertions(+), 12 deletions(-) rename docs/development/{extensions-contrib => extensions-core}/azure.md (100%) rename {extensions-contrib => extensions-core}/azure-extensions/pom.xml (99%) rename {extensions-contrib => extensions-core}/azure-extensions/src/main/java/org/apache/druid/data/input/azure/AzureEntity.java (100%) rename {extensions-contrib => extensions-core}/azure-extensions/src/main/java/org/apache/druid/data/input/azure/AzureEntityFactory.java (100%) rename {extensions-contrib => extensions-core}/azure-extensions/src/main/java/org/apache/druid/data/input/azure/AzureInputSource.java (100%) rename {extensions-contrib => extensions-core}/azure-extensions/src/main/java/org/apache/druid/firehose/azure/AzureBlob.java (100%) rename {extensions-contrib => extensions-core}/azure-extensions/src/main/java/org/apache/druid/firehose/azure/StaticAzureBlobStoreFirehoseFactory.java (100%) rename {extensions-contrib => extensions-core}/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureAccountConfig.java (100%) rename {extensions-contrib => extensions-core}/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureByteSource.java (100%) rename {extensions-contrib => extensions-core}/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureByteSourceFactory.java (100%) rename {extensions-contrib => extensions-core}/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureCloudBlobHolderToCloudObjectLocationConverter.java (100%) rename {extensions-contrib => extensions-core}/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureCloudBlobIterable.java (100%) rename {extensions-contrib => extensions-core}/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureCloudBlobIterableFactory.java (100%) rename {extensions-contrib => extensions-core}/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureCloudBlobIterator.java (100%) rename {extensions-contrib => extensions-core}/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureCloudBlobIteratorFactory.java (100%) rename {extensions-contrib => extensions-core}/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureDataSegmentConfig.java (100%) rename {extensions-contrib => extensions-core}/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureDataSegmentKiller.java (100%) rename {extensions-contrib => extensions-core}/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureDataSegmentPuller.java (100%) rename {extensions-contrib => extensions-core}/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureDataSegmentPusher.java (100%) rename {extensions-contrib => extensions-core}/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureInputDataConfig.java (100%) rename {extensions-contrib => extensions-core}/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureLoadSpec.java (100%) rename {extensions-contrib => extensions-core}/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureStorage.java (100%) rename {extensions-contrib => extensions-core}/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureStorageDruidModule.java (100%) rename {extensions-contrib => extensions-core}/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureTaskLogs.java (100%) rename {extensions-contrib => extensions-core}/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureTaskLogsConfig.java (100%) rename {extensions-contrib => extensions-core}/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureUtils.java (100%) rename {extensions-contrib => extensions-core}/azure-extensions/src/main/java/org/apache/druid/storage/azure/ICloudSpecificObjectToCloudObjectLocationConverter.java (100%) rename {extensions-contrib => extensions-core}/azure-extensions/src/main/java/org/apache/druid/storage/azure/blob/CloudBlobHolder.java (100%) rename {extensions-contrib => extensions-core}/azure-extensions/src/main/java/org/apache/druid/storage/azure/blob/ListBlobItemHolder.java (100%) rename {extensions-contrib => extensions-core}/azure-extensions/src/main/java/org/apache/druid/storage/azure/blob/ListBlobItemHolderFactory.java (100%) rename {extensions-contrib => extensions-core}/azure-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule (100%) rename {extensions-contrib => extensions-core}/azure-extensions/src/test/java/org/apache/druid/data/input/azure/AzureEntityTest.java (100%) rename {extensions-contrib => extensions-core}/azure-extensions/src/test/java/org/apache/druid/data/input/azure/AzureInputSourceSerdeTest.java (100%) rename {extensions-contrib => extensions-core}/azure-extensions/src/test/java/org/apache/druid/data/input/azure/AzureInputSourceTest.java (100%) rename {extensions-contrib => extensions-core}/azure-extensions/src/test/java/org/apache/druid/firehose/azure/StaticAzureBlobStoreFirehoseFactoryTest.java (100%) rename {extensions-contrib => extensions-core}/azure-extensions/src/test/java/org/apache/druid/storage/azure/AzureByteSourceTest.java (100%) rename {extensions-contrib => extensions-core}/azure-extensions/src/test/java/org/apache/druid/storage/azure/AzureCloudBlobHolderToCloudObjectLocationConverterTest.java (100%) rename {extensions-contrib => extensions-core}/azure-extensions/src/test/java/org/apache/druid/storage/azure/AzureCloudBlobIteratorTest.java (100%) rename {extensions-contrib => extensions-core}/azure-extensions/src/test/java/org/apache/druid/storage/azure/AzureDataSegmentKillerTest.java (100%) rename {extensions-contrib => extensions-core}/azure-extensions/src/test/java/org/apache/druid/storage/azure/AzureDataSegmentPullerTest.java (100%) rename {extensions-contrib => extensions-core}/azure-extensions/src/test/java/org/apache/druid/storage/azure/AzureDataSegmentPusherTest.java (100%) rename {extensions-contrib => extensions-core}/azure-extensions/src/test/java/org/apache/druid/storage/azure/AzureStorageDruidModuleTest.java (100%) rename {extensions-contrib => extensions-core}/azure-extensions/src/test/java/org/apache/druid/storage/azure/AzureTaskLogsTest.java (100%) rename {extensions-contrib => extensions-core}/azure-extensions/src/test/java/org/apache/druid/storage/azure/AzureTestUtils.java (100%) rename {extensions-contrib => extensions-core}/azure-extensions/src/test/java/org/apache/druid/storage/azure/AzureUtilsTest.java (100%) diff --git a/distribution/bin/check-licenses.py b/distribution/bin/check-licenses.py index aad34cb5ca3..b80bfeba840 100755 --- a/distribution/bin/check-licenses.py +++ b/distribution/bin/check-licenses.py @@ -269,6 +269,7 @@ def build_compatible_license_names(): compatible_licenses['The MIT License'] = 'MIT License' compatible_licenses['MIT License'] = 'MIT License' + compatible_licenses['The MIT License (MIT)'] = 'MIT License' compatible_licenses['-'] = '-' return compatible_licenses diff --git a/distribution/pom.xml b/distribution/pom.xml index 99b02a4910f..fc15e9343be 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -197,6 +197,8 @@ -c org.apache.druid.extensions:druid-avro-extensions -c + org.apache.druid.extensions:druid-azure-extensions + -c org.apache.druid.extensions:druid-bloom-filter -c org.apache.druid.extensions:druid-datasketches @@ -382,8 +384,6 @@ -c org.apache.druid.extensions.contrib:dropwizard-emitter -c - org.apache.druid.extensions.contrib:druid-azure-extensions - -c org.apache.druid.extensions.contrib:druid-cassandra-storage -c org.apache.druid.extensions.contrib:druid-cloudfiles-extensions diff --git a/docs/development/extensions-contrib/azure.md b/docs/development/extensions-core/azure.md similarity index 100% rename from docs/development/extensions-contrib/azure.md rename to docs/development/extensions-core/azure.md diff --git a/docs/development/extensions.md b/docs/development/extensions.md index 045fba8f8b1..96b00b4347e 100644 --- a/docs/development/extensions.md +++ b/docs/development/extensions.md @@ -37,6 +37,7 @@ Core extensions are maintained by Druid committers. |Name|Description|Docs| |----|-----------|----| |druid-avro-extensions|Support for data in Apache Avro data format.|[link](../development/extensions-core/avro.md)| +|druid-azure-extensions|Microsoft Azure deep storage.|[link](../development/extensions-core/azure.md)| |druid-basic-security|Support for Basic HTTP authentication and role-based access control.|[link](../development/extensions-core/druid-basic-security.md)| |druid-bloom-filter|Support for providing Bloom filters in druid queries.|[link](../development/extensions-core/bloom-filter.md)| |druid-caffeine-cache|A local cache implementation backed by Caffeine.|[link](../configuration/index.md#cache-configuration)| @@ -72,7 +73,6 @@ All of these community extensions can be downloaded using [pull-deps](../operati |Name|Description|Docs| |----|-----------|----| |ambari-metrics-emitter|Ambari Metrics Emitter |[link](../development/extensions-contrib/ambari-metrics-emitter.md)| -|druid-azure-extensions|Microsoft Azure deep storage.|[link](../development/extensions-contrib/azure.md)| |druid-cassandra-storage|Apache Cassandra deep storage.|[link](../development/extensions-contrib/cassandra.md)| |druid-cloudfiles-extensions|Rackspace Cloudfiles deep storage and firehose.|[link](../development/extensions-contrib/cloudfiles.md)| |druid-distinctcount|DistinctCount aggregator|[link](../development/extensions-contrib/distinctcount.md)| diff --git a/docs/ingestion/native-batch.md b/docs/ingestion/native-batch.md index 2195f83cd1f..10e57c82af0 100644 --- a/docs/ingestion/native-batch.md +++ b/docs/ingestion/native-batch.md @@ -929,7 +929,7 @@ Google Cloud Storage object: ### Azure Input Source -> You need to include the [`druid-azure-extensions`](../development/extensions-contrib/azure.md) as an extension to use the Azure input source. +> You need to include the [`druid-azure-extensions`](../development/extensions-core/azure.md) as an extension to use the Azure input source. The Azure input source is to support reading objects directly from Azure Blob store. Objects can be specified as list of Azure Blob store URI strings. The Azure input source is splittable and can be used diff --git a/extensions-contrib/azure-extensions/pom.xml b/extensions-core/azure-extensions/pom.xml similarity index 99% rename from extensions-contrib/azure-extensions/pom.xml rename to extensions-core/azure-extensions/pom.xml index 9162ad8982d..ba6e3740f18 100644 --- a/extensions-contrib/azure-extensions/pom.xml +++ b/extensions-core/azure-extensions/pom.xml @@ -22,7 +22,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 - org.apache.druid.extensions.contrib + org.apache.druid.extensions druid-azure-extensions druid-azure-extensions druid-azure-extensions diff --git a/extensions-contrib/azure-extensions/src/main/java/org/apache/druid/data/input/azure/AzureEntity.java b/extensions-core/azure-extensions/src/main/java/org/apache/druid/data/input/azure/AzureEntity.java similarity index 100% rename from extensions-contrib/azure-extensions/src/main/java/org/apache/druid/data/input/azure/AzureEntity.java rename to extensions-core/azure-extensions/src/main/java/org/apache/druid/data/input/azure/AzureEntity.java diff --git a/extensions-contrib/azure-extensions/src/main/java/org/apache/druid/data/input/azure/AzureEntityFactory.java b/extensions-core/azure-extensions/src/main/java/org/apache/druid/data/input/azure/AzureEntityFactory.java similarity index 100% rename from extensions-contrib/azure-extensions/src/main/java/org/apache/druid/data/input/azure/AzureEntityFactory.java rename to extensions-core/azure-extensions/src/main/java/org/apache/druid/data/input/azure/AzureEntityFactory.java diff --git a/extensions-contrib/azure-extensions/src/main/java/org/apache/druid/data/input/azure/AzureInputSource.java b/extensions-core/azure-extensions/src/main/java/org/apache/druid/data/input/azure/AzureInputSource.java similarity index 100% rename from extensions-contrib/azure-extensions/src/main/java/org/apache/druid/data/input/azure/AzureInputSource.java rename to extensions-core/azure-extensions/src/main/java/org/apache/druid/data/input/azure/AzureInputSource.java diff --git a/extensions-contrib/azure-extensions/src/main/java/org/apache/druid/firehose/azure/AzureBlob.java b/extensions-core/azure-extensions/src/main/java/org/apache/druid/firehose/azure/AzureBlob.java similarity index 100% rename from extensions-contrib/azure-extensions/src/main/java/org/apache/druid/firehose/azure/AzureBlob.java rename to extensions-core/azure-extensions/src/main/java/org/apache/druid/firehose/azure/AzureBlob.java diff --git a/extensions-contrib/azure-extensions/src/main/java/org/apache/druid/firehose/azure/StaticAzureBlobStoreFirehoseFactory.java b/extensions-core/azure-extensions/src/main/java/org/apache/druid/firehose/azure/StaticAzureBlobStoreFirehoseFactory.java similarity index 100% rename from extensions-contrib/azure-extensions/src/main/java/org/apache/druid/firehose/azure/StaticAzureBlobStoreFirehoseFactory.java rename to extensions-core/azure-extensions/src/main/java/org/apache/druid/firehose/azure/StaticAzureBlobStoreFirehoseFactory.java diff --git a/extensions-contrib/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureAccountConfig.java b/extensions-core/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureAccountConfig.java similarity index 100% rename from extensions-contrib/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureAccountConfig.java rename to extensions-core/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureAccountConfig.java diff --git a/extensions-contrib/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureByteSource.java b/extensions-core/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureByteSource.java similarity index 100% rename from extensions-contrib/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureByteSource.java rename to extensions-core/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureByteSource.java diff --git a/extensions-contrib/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureByteSourceFactory.java b/extensions-core/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureByteSourceFactory.java similarity index 100% rename from extensions-contrib/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureByteSourceFactory.java rename to extensions-core/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureByteSourceFactory.java diff --git a/extensions-contrib/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureCloudBlobHolderToCloudObjectLocationConverter.java b/extensions-core/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureCloudBlobHolderToCloudObjectLocationConverter.java similarity index 100% rename from extensions-contrib/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureCloudBlobHolderToCloudObjectLocationConverter.java rename to extensions-core/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureCloudBlobHolderToCloudObjectLocationConverter.java diff --git a/extensions-contrib/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureCloudBlobIterable.java b/extensions-core/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureCloudBlobIterable.java similarity index 100% rename from extensions-contrib/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureCloudBlobIterable.java rename to extensions-core/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureCloudBlobIterable.java diff --git a/extensions-contrib/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureCloudBlobIterableFactory.java b/extensions-core/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureCloudBlobIterableFactory.java similarity index 100% rename from extensions-contrib/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureCloudBlobIterableFactory.java rename to extensions-core/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureCloudBlobIterableFactory.java diff --git a/extensions-contrib/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureCloudBlobIterator.java b/extensions-core/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureCloudBlobIterator.java similarity index 100% rename from extensions-contrib/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureCloudBlobIterator.java rename to extensions-core/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureCloudBlobIterator.java diff --git a/extensions-contrib/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureCloudBlobIteratorFactory.java b/extensions-core/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureCloudBlobIteratorFactory.java similarity index 100% rename from extensions-contrib/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureCloudBlobIteratorFactory.java rename to extensions-core/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureCloudBlobIteratorFactory.java diff --git a/extensions-contrib/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureDataSegmentConfig.java b/extensions-core/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureDataSegmentConfig.java similarity index 100% rename from extensions-contrib/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureDataSegmentConfig.java rename to extensions-core/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureDataSegmentConfig.java diff --git a/extensions-contrib/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureDataSegmentKiller.java b/extensions-core/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureDataSegmentKiller.java similarity index 100% rename from extensions-contrib/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureDataSegmentKiller.java rename to extensions-core/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureDataSegmentKiller.java diff --git a/extensions-contrib/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureDataSegmentPuller.java b/extensions-core/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureDataSegmentPuller.java similarity index 100% rename from extensions-contrib/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureDataSegmentPuller.java rename to extensions-core/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureDataSegmentPuller.java diff --git a/extensions-contrib/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureDataSegmentPusher.java b/extensions-core/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureDataSegmentPusher.java similarity index 100% rename from extensions-contrib/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureDataSegmentPusher.java rename to extensions-core/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureDataSegmentPusher.java diff --git a/extensions-contrib/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureInputDataConfig.java b/extensions-core/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureInputDataConfig.java similarity index 100% rename from extensions-contrib/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureInputDataConfig.java rename to extensions-core/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureInputDataConfig.java diff --git a/extensions-contrib/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureLoadSpec.java b/extensions-core/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureLoadSpec.java similarity index 100% rename from extensions-contrib/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureLoadSpec.java rename to extensions-core/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureLoadSpec.java diff --git a/extensions-contrib/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureStorage.java b/extensions-core/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureStorage.java similarity index 100% rename from extensions-contrib/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureStorage.java rename to extensions-core/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureStorage.java diff --git a/extensions-contrib/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureStorageDruidModule.java b/extensions-core/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureStorageDruidModule.java similarity index 100% rename from extensions-contrib/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureStorageDruidModule.java rename to extensions-core/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureStorageDruidModule.java diff --git a/extensions-contrib/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureTaskLogs.java b/extensions-core/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureTaskLogs.java similarity index 100% rename from extensions-contrib/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureTaskLogs.java rename to extensions-core/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureTaskLogs.java diff --git a/extensions-contrib/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureTaskLogsConfig.java b/extensions-core/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureTaskLogsConfig.java similarity index 100% rename from extensions-contrib/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureTaskLogsConfig.java rename to extensions-core/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureTaskLogsConfig.java diff --git a/extensions-contrib/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureUtils.java b/extensions-core/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureUtils.java similarity index 100% rename from extensions-contrib/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureUtils.java rename to extensions-core/azure-extensions/src/main/java/org/apache/druid/storage/azure/AzureUtils.java diff --git a/extensions-contrib/azure-extensions/src/main/java/org/apache/druid/storage/azure/ICloudSpecificObjectToCloudObjectLocationConverter.java b/extensions-core/azure-extensions/src/main/java/org/apache/druid/storage/azure/ICloudSpecificObjectToCloudObjectLocationConverter.java similarity index 100% rename from extensions-contrib/azure-extensions/src/main/java/org/apache/druid/storage/azure/ICloudSpecificObjectToCloudObjectLocationConverter.java rename to extensions-core/azure-extensions/src/main/java/org/apache/druid/storage/azure/ICloudSpecificObjectToCloudObjectLocationConverter.java diff --git a/extensions-contrib/azure-extensions/src/main/java/org/apache/druid/storage/azure/blob/CloudBlobHolder.java b/extensions-core/azure-extensions/src/main/java/org/apache/druid/storage/azure/blob/CloudBlobHolder.java similarity index 100% rename from extensions-contrib/azure-extensions/src/main/java/org/apache/druid/storage/azure/blob/CloudBlobHolder.java rename to extensions-core/azure-extensions/src/main/java/org/apache/druid/storage/azure/blob/CloudBlobHolder.java diff --git a/extensions-contrib/azure-extensions/src/main/java/org/apache/druid/storage/azure/blob/ListBlobItemHolder.java b/extensions-core/azure-extensions/src/main/java/org/apache/druid/storage/azure/blob/ListBlobItemHolder.java similarity index 100% rename from extensions-contrib/azure-extensions/src/main/java/org/apache/druid/storage/azure/blob/ListBlobItemHolder.java rename to extensions-core/azure-extensions/src/main/java/org/apache/druid/storage/azure/blob/ListBlobItemHolder.java diff --git a/extensions-contrib/azure-extensions/src/main/java/org/apache/druid/storage/azure/blob/ListBlobItemHolderFactory.java b/extensions-core/azure-extensions/src/main/java/org/apache/druid/storage/azure/blob/ListBlobItemHolderFactory.java similarity index 100% rename from extensions-contrib/azure-extensions/src/main/java/org/apache/druid/storage/azure/blob/ListBlobItemHolderFactory.java rename to extensions-core/azure-extensions/src/main/java/org/apache/druid/storage/azure/blob/ListBlobItemHolderFactory.java diff --git a/extensions-contrib/azure-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule b/extensions-core/azure-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule similarity index 100% rename from extensions-contrib/azure-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule rename to extensions-core/azure-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule diff --git a/extensions-contrib/azure-extensions/src/test/java/org/apache/druid/data/input/azure/AzureEntityTest.java b/extensions-core/azure-extensions/src/test/java/org/apache/druid/data/input/azure/AzureEntityTest.java similarity index 100% rename from extensions-contrib/azure-extensions/src/test/java/org/apache/druid/data/input/azure/AzureEntityTest.java rename to extensions-core/azure-extensions/src/test/java/org/apache/druid/data/input/azure/AzureEntityTest.java diff --git a/extensions-contrib/azure-extensions/src/test/java/org/apache/druid/data/input/azure/AzureInputSourceSerdeTest.java b/extensions-core/azure-extensions/src/test/java/org/apache/druid/data/input/azure/AzureInputSourceSerdeTest.java similarity index 100% rename from extensions-contrib/azure-extensions/src/test/java/org/apache/druid/data/input/azure/AzureInputSourceSerdeTest.java rename to extensions-core/azure-extensions/src/test/java/org/apache/druid/data/input/azure/AzureInputSourceSerdeTest.java diff --git a/extensions-contrib/azure-extensions/src/test/java/org/apache/druid/data/input/azure/AzureInputSourceTest.java b/extensions-core/azure-extensions/src/test/java/org/apache/druid/data/input/azure/AzureInputSourceTest.java similarity index 100% rename from extensions-contrib/azure-extensions/src/test/java/org/apache/druid/data/input/azure/AzureInputSourceTest.java rename to extensions-core/azure-extensions/src/test/java/org/apache/druid/data/input/azure/AzureInputSourceTest.java diff --git a/extensions-contrib/azure-extensions/src/test/java/org/apache/druid/firehose/azure/StaticAzureBlobStoreFirehoseFactoryTest.java b/extensions-core/azure-extensions/src/test/java/org/apache/druid/firehose/azure/StaticAzureBlobStoreFirehoseFactoryTest.java similarity index 100% rename from extensions-contrib/azure-extensions/src/test/java/org/apache/druid/firehose/azure/StaticAzureBlobStoreFirehoseFactoryTest.java rename to extensions-core/azure-extensions/src/test/java/org/apache/druid/firehose/azure/StaticAzureBlobStoreFirehoseFactoryTest.java diff --git a/extensions-contrib/azure-extensions/src/test/java/org/apache/druid/storage/azure/AzureByteSourceTest.java b/extensions-core/azure-extensions/src/test/java/org/apache/druid/storage/azure/AzureByteSourceTest.java similarity index 100% rename from extensions-contrib/azure-extensions/src/test/java/org/apache/druid/storage/azure/AzureByteSourceTest.java rename to extensions-core/azure-extensions/src/test/java/org/apache/druid/storage/azure/AzureByteSourceTest.java diff --git a/extensions-contrib/azure-extensions/src/test/java/org/apache/druid/storage/azure/AzureCloudBlobHolderToCloudObjectLocationConverterTest.java b/extensions-core/azure-extensions/src/test/java/org/apache/druid/storage/azure/AzureCloudBlobHolderToCloudObjectLocationConverterTest.java similarity index 100% rename from extensions-contrib/azure-extensions/src/test/java/org/apache/druid/storage/azure/AzureCloudBlobHolderToCloudObjectLocationConverterTest.java rename to extensions-core/azure-extensions/src/test/java/org/apache/druid/storage/azure/AzureCloudBlobHolderToCloudObjectLocationConverterTest.java diff --git a/extensions-contrib/azure-extensions/src/test/java/org/apache/druid/storage/azure/AzureCloudBlobIteratorTest.java b/extensions-core/azure-extensions/src/test/java/org/apache/druid/storage/azure/AzureCloudBlobIteratorTest.java similarity index 100% rename from extensions-contrib/azure-extensions/src/test/java/org/apache/druid/storage/azure/AzureCloudBlobIteratorTest.java rename to extensions-core/azure-extensions/src/test/java/org/apache/druid/storage/azure/AzureCloudBlobIteratorTest.java diff --git a/extensions-contrib/azure-extensions/src/test/java/org/apache/druid/storage/azure/AzureDataSegmentKillerTest.java b/extensions-core/azure-extensions/src/test/java/org/apache/druid/storage/azure/AzureDataSegmentKillerTest.java similarity index 100% rename from extensions-contrib/azure-extensions/src/test/java/org/apache/druid/storage/azure/AzureDataSegmentKillerTest.java rename to extensions-core/azure-extensions/src/test/java/org/apache/druid/storage/azure/AzureDataSegmentKillerTest.java diff --git a/extensions-contrib/azure-extensions/src/test/java/org/apache/druid/storage/azure/AzureDataSegmentPullerTest.java b/extensions-core/azure-extensions/src/test/java/org/apache/druid/storage/azure/AzureDataSegmentPullerTest.java similarity index 100% rename from extensions-contrib/azure-extensions/src/test/java/org/apache/druid/storage/azure/AzureDataSegmentPullerTest.java rename to extensions-core/azure-extensions/src/test/java/org/apache/druid/storage/azure/AzureDataSegmentPullerTest.java diff --git a/extensions-contrib/azure-extensions/src/test/java/org/apache/druid/storage/azure/AzureDataSegmentPusherTest.java b/extensions-core/azure-extensions/src/test/java/org/apache/druid/storage/azure/AzureDataSegmentPusherTest.java similarity index 100% rename from extensions-contrib/azure-extensions/src/test/java/org/apache/druid/storage/azure/AzureDataSegmentPusherTest.java rename to extensions-core/azure-extensions/src/test/java/org/apache/druid/storage/azure/AzureDataSegmentPusherTest.java diff --git a/extensions-contrib/azure-extensions/src/test/java/org/apache/druid/storage/azure/AzureStorageDruidModuleTest.java b/extensions-core/azure-extensions/src/test/java/org/apache/druid/storage/azure/AzureStorageDruidModuleTest.java similarity index 100% rename from extensions-contrib/azure-extensions/src/test/java/org/apache/druid/storage/azure/AzureStorageDruidModuleTest.java rename to extensions-core/azure-extensions/src/test/java/org/apache/druid/storage/azure/AzureStorageDruidModuleTest.java diff --git a/extensions-contrib/azure-extensions/src/test/java/org/apache/druid/storage/azure/AzureTaskLogsTest.java b/extensions-core/azure-extensions/src/test/java/org/apache/druid/storage/azure/AzureTaskLogsTest.java similarity index 100% rename from extensions-contrib/azure-extensions/src/test/java/org/apache/druid/storage/azure/AzureTaskLogsTest.java rename to extensions-core/azure-extensions/src/test/java/org/apache/druid/storage/azure/AzureTaskLogsTest.java diff --git a/extensions-contrib/azure-extensions/src/test/java/org/apache/druid/storage/azure/AzureTestUtils.java b/extensions-core/azure-extensions/src/test/java/org/apache/druid/storage/azure/AzureTestUtils.java similarity index 100% rename from extensions-contrib/azure-extensions/src/test/java/org/apache/druid/storage/azure/AzureTestUtils.java rename to extensions-core/azure-extensions/src/test/java/org/apache/druid/storage/azure/AzureTestUtils.java diff --git a/extensions-contrib/azure-extensions/src/test/java/org/apache/druid/storage/azure/AzureUtilsTest.java b/extensions-core/azure-extensions/src/test/java/org/apache/druid/storage/azure/AzureUtilsTest.java similarity index 100% rename from extensions-contrib/azure-extensions/src/test/java/org/apache/druid/storage/azure/AzureUtilsTest.java rename to extensions-core/azure-extensions/src/test/java/org/apache/druid/storage/azure/AzureUtilsTest.java diff --git a/licenses.yaml b/licenses.yaml index 4c7c2c7ae3e..c352faa57dd 100644 --- a/licenses.yaml +++ b/licenses.yaml @@ -4047,6 +4047,28 @@ libraries: --- +name: Microsoft Azure SDK For Key Vault Core +license_category: binary +module: extensions/druid-azure-extensions +license_name: MIT License +copyright: Microsoft +version: 1.0.0 +libraries: + - com.microsoft.azure: azure-keyvault-core + +--- + +name: Microsoft Azure Storage Client SDK +license_category: binary +module: extensions/druid-azure-extensions +license_name: Apache License version 2.0 +copyright: Microsoft +version: 8.6.0 +libraries: + - com.microsoft.azure: azure-storage + +--- + # Web console modules start name: "@babel/runtime" license_category: binary diff --git a/pom.xml b/pom.xml index f6f14f4d7f7..f229d32dd74 100644 --- a/pom.xml +++ b/pom.xml @@ -137,6 +137,7 @@ cloud/gcp-common extensions-core/avro-extensions + extensions-core/azure-extensions extensions-core/datasketches extensions-core/druid-bloom-filter extensions-core/druid-kerberos @@ -160,7 +161,6 @@ extensions-core/google-extensions extensions-contrib/influx-extensions - extensions-contrib/azure-extensions extensions-contrib/cassandra-storage extensions-contrib/dropwizard-emitter extensions-contrib/cloudfiles-extensions diff --git a/website/.spelling b/website/.spelling index 54293ab2e25..275854f669f 100644 --- a/website/.spelling +++ b/website/.spelling @@ -442,7 +442,7 @@ Pdist ambari-metrics metricName trustStore - - ../docs/development/extensions-contrib/azure.md + - ../docs/development/extensions-core/azure.md StaticAzureBlobStoreFirehose StaticS3Firehose fetchTimeout @@ -457,6 +457,13 @@ StaticCloudFilesFirehose cloudfiles rackspace-cloudfiles-uk rackspace-cloudfiles-us +StaticAzureBlobStoreFirehose +gz +shardSpecs +maxCacheCapacityBytes +maxFetchCapacityBytes +fetchTimeout +maxFetchRetry - ../docs/development/extensions-contrib/distinctcount.md distinctCount groupBy @@ -973,6 +980,8 @@ splitHintSpec taskStatusCheckPeriodMs timeChunk totalNumMergeTasks +StaticS3Firehose +prefetchTriggerBytes - ../docs/ingestion/schema-design.md product_category product_id diff --git a/website/i18n/en.json b/website/i18n/en.json index bf8283d3415..d7824544314 100644 --- a/website/i18n/en.json +++ b/website/i18n/en.json @@ -92,9 +92,6 @@ "development/extensions-contrib/ambari-metrics-emitter": { "title": "Ambari Metrics Emitter" }, - "development/extensions-contrib/azure": { - "title": "Microsoft Azure" - }, "development/extensions-contrib/cassandra": { "title": "Apache Cassandra" }, @@ -152,6 +149,9 @@ "development/extensions-core/avro": { "title": "Apache Avro" }, + "development/extensions-core/azure": { + "title": "Microsoft Azure" + }, "development/extensions-core/bloom-filter": { "title": "Bloom Filter" }, diff --git a/website/redirects.json b/website/redirects.json index 22304dab096..7fe9577babf 100644 --- a/website/redirects.json +++ b/website/redirects.json @@ -132,7 +132,7 @@ {"source": "design/plumber.html", "target": "../ingestion/standalone-realtime.html"} {"source": "design/realtime.html", "target": "../ingestion/standalone-realtime.html"} {"source": "development/approximate-histograms.html", "target": "extensions-core/approximate-histograms.html"} -{"source": "development/community-extensions/azure.html", "target": "../extensions-contrib/azure.html"} +{"source": "development/community-extensions/azure.html", "target": "../extensions-core/azure.html"} {"source": "development/community-extensions/cassandra.html", "target": "../extensions-contrib/cassandra.html"} {"source": "development/community-extensions/cloudfiles.html", "target": "../extensions-contrib/cloudfiles.html"} {"source": "development/community-extensions/graphite.html", "target": "../extensions-contrib/graphite.html"} diff --git a/website/sidebars.json b/website/sidebars.json index e393196877d..706e937b8ae 100644 --- a/website/sidebars.json +++ b/website/sidebars.json @@ -147,6 +147,7 @@ "design/peons", "development/extensions-core/approximate-histograms", "development/extensions-core/avro", + "development/extensions-core/azure", "development/extensions-core/bloom-filter", "development/extensions-core/datasketches-extension", "development/extensions-core/datasketches-hll", @@ -170,7 +171,6 @@ "development/extensions-core/stats", "development/extensions-core/test-stats", "development/extensions-contrib/ambari-metrics-emitter", - "development/extensions-contrib/azure", "development/extensions-contrib/cassandra", "development/extensions-contrib/cloudfiles", "development/extensions-contrib/distinctcount",