mirror of https://github.com/apache/druid.git
Update `groupId` for delta-lake and iceberg extensions (#15843)
* Update the group id to org.apache.druid.extensions.contrib for contrib exts. * Note iceberg and delta lake extensions in extensions.md * properties and shell backticks * Update groupId in distribution/pom.xml * remove delta-lake from dist. * Add note on downloading extension.
This commit is contained in:
parent
26815d425b
commit
1a5b57df84
|
@ -257,8 +257,6 @@
|
||||||
<argument>org.apache.druid.extensions:druid-kubernetes-extensions</argument>
|
<argument>org.apache.druid.extensions:druid-kubernetes-extensions</argument>
|
||||||
<argument>-c</argument>
|
<argument>-c</argument>
|
||||||
<argument>org.apache.druid.extensions:druid-catalog</argument>
|
<argument>org.apache.druid.extensions:druid-catalog</argument>
|
||||||
<argument>-c</argument>
|
|
||||||
<argument>org.apache.druid.extensions:druid-deltalake-extensions</argument>
|
|
||||||
<argument>${druid.distribution.pulldeps.opts}</argument>
|
<argument>${druid.distribution.pulldeps.opts}</argument>
|
||||||
</arguments>
|
</arguments>
|
||||||
</configuration>
|
</configuration>
|
||||||
|
@ -453,9 +451,9 @@
|
||||||
<argument>-c</argument>
|
<argument>-c</argument>
|
||||||
<argument>org.apache.druid.extensions.contrib:opentelemetry-emitter</argument>
|
<argument>org.apache.druid.extensions.contrib:opentelemetry-emitter</argument>
|
||||||
<argument>-c</argument>
|
<argument>-c</argument>
|
||||||
<argument>org.apache.druid.extensions:druid-iceberg-extensions</argument>
|
<argument>org.apache.druid.extensions.contrib:druid-iceberg-extensions</argument>
|
||||||
<argument>-c</argument>
|
<argument>-c</argument>
|
||||||
<argument>org.apache.druid.extensions:druid-deltalake-extensions</argument>
|
<argument>org.apache.druid.extensions.contrib:druid-deltalake-extensions</argument>
|
||||||
<argument>-c</argument>
|
<argument>-c</argument>
|
||||||
<argument>org.apache.druid.extensions.contrib:druid-spectator-histogram</argument>
|
<argument>org.apache.druid.extensions.contrib:druid-spectator-histogram</argument>
|
||||||
</arguments>
|
</arguments>
|
||||||
|
|
|
@ -84,7 +84,9 @@ All of these community extensions can be downloaded using [pull-deps](../operati
|
||||||
|druid-cloudfiles-extensions|Rackspace Cloudfiles deep storage and firehose.|[link](../development/extensions-contrib/cloudfiles.md)|
|
|druid-cloudfiles-extensions|Rackspace Cloudfiles deep storage and firehose.|[link](../development/extensions-contrib/cloudfiles.md)|
|
||||||
|druid-compressed-bigdecimal|Compressed Big Decimal Type | [link](../development/extensions-contrib/compressed-big-decimal.md)|
|
|druid-compressed-bigdecimal|Compressed Big Decimal Type | [link](../development/extensions-contrib/compressed-big-decimal.md)|
|
||||||
|druid-ddsketch|Support for DDSketch approximate quantiles based on [DDSketch](https://github.com/datadog/sketches-java) | [link](../development/extensions-contrib/ddsketch-quantiles.md)|
|
|druid-ddsketch|Support for DDSketch approximate quantiles based on [DDSketch](https://github.com/datadog/sketches-java) | [link](../development/extensions-contrib/ddsketch-quantiles.md)|
|
||||||
|
|druid-deltalake-extensions|Support for ingesting Delta Lake tables.|[link](../development/extensions-contrib/delta-lake.md)|
|
||||||
|druid-distinctcount|DistinctCount aggregator|[link](../development/extensions-contrib/distinctcount.md)|
|
|druid-distinctcount|DistinctCount aggregator|[link](../development/extensions-contrib/distinctcount.md)|
|
||||||
|
|druid-iceberg-extensions|Support for ingesting Iceberg tables.|[link](../development/extensions-contrib/iceberg.md)|
|
||||||
|druid-redis-cache|A cache implementation for Druid based on Redis.|[link](../development/extensions-contrib/redis-cache.md)|
|
|druid-redis-cache|A cache implementation for Druid based on Redis.|[link](../development/extensions-contrib/redis-cache.md)|
|
||||||
|druid-time-min-max|Min/Max aggregator for timestamp.|[link](../development/extensions-contrib/time-min-max.md)|
|
|druid-time-min-max|Min/Max aggregator for timestamp.|[link](../development/extensions-contrib/time-min-max.md)|
|
||||||
|sqlserver-metadata-storage|Microsoft SQLServer deep storage.|[link](../development/extensions-contrib/sqlserver.md)|
|
|sqlserver-metadata-storage|Microsoft SQLServer deep storage.|[link](../development/extensions-contrib/sqlserver.md)|
|
||||||
|
@ -121,7 +123,7 @@ can load bundled extensions by adding their names to your common.runtime.propert
|
||||||
`druid.extensions.loadList` property. For example, to load the postgresql-metadata-storage and
|
`druid.extensions.loadList` property. For example, to load the postgresql-metadata-storage and
|
||||||
druid-hdfs-storage extensions, use the configuration:
|
druid-hdfs-storage extensions, use the configuration:
|
||||||
|
|
||||||
```
|
```properties
|
||||||
druid.extensions.loadList=["postgresql-metadata-storage", "druid-hdfs-storage"]
|
druid.extensions.loadList=["postgresql-metadata-storage", "druid-hdfs-storage"]
|
||||||
```
|
```
|
||||||
|
|
||||||
|
@ -146,7 +148,7 @@ if they are available from Maven, the included [pull-deps](../operations/pull-de
|
||||||
specify the full Maven coordinate of the extension in the form `groupId:artifactId:version`. For example,
|
specify the full Maven coordinate of the extension in the form `groupId:artifactId:version`. For example,
|
||||||
for the (hypothetical) extension *com.example:druid-example-extension:1.0.0*, run:
|
for the (hypothetical) extension *com.example:druid-example-extension:1.0.0*, run:
|
||||||
|
|
||||||
```
|
```shell
|
||||||
java \
|
java \
|
||||||
-cp "lib/*" \
|
-cp "lib/*" \
|
||||||
-Ddruid.extensions.directory="extensions" \
|
-Ddruid.extensions.directory="extensions" \
|
||||||
|
|
|
@ -38,6 +38,23 @@ These Delta Lake files are versioned Parquet files.
|
||||||
The Delta Lake extension uses the Delta Kernel introduced in Delta Lake 3.0.0, which is compatible with Apache Spark 3.5.x.
|
The Delta Lake extension uses the Delta Kernel introduced in Delta Lake 3.0.0, which is compatible with Apache Spark 3.5.x.
|
||||||
Older versions are unsupported, so consider upgrading to Delta Lake 3.0.x or higher to use this extension.
|
Older versions are unsupported, so consider upgrading to Delta Lake 3.0.x or higher to use this extension.
|
||||||
|
|
||||||
|
## Downloading Delta Lake extension
|
||||||
|
|
||||||
|
To download `druid-deltalake-extensions`, run the following command after replacing `<VERSION>` with the desired
|
||||||
|
Druid version:
|
||||||
|
|
||||||
|
```shell
|
||||||
|
java \
|
||||||
|
-cp "lib/*" \
|
||||||
|
-Ddruid.extensions.directory="extensions" \
|
||||||
|
-Ddruid.extensions.hadoopDependenciesDir="hadoop-dependencies" \
|
||||||
|
org.apache.druid.cli.Main tools pull-deps \
|
||||||
|
--no-default-hadoop \
|
||||||
|
-c "org.apache.druid.extensions.contrib:druid-deltalake-extensions:<VERSION>"
|
||||||
|
```
|
||||||
|
|
||||||
|
See [Loading community extensions](../../configuration/extensions.md#loading-community-extensions) for more information.
|
||||||
|
|
||||||
## Known limitations
|
## Known limitations
|
||||||
|
|
||||||
- This extension relies on the Delta Kernel API and can only read from the latest Delta table snapshot.
|
- This extension relies on the Delta Kernel API and can only read from the latest Delta table snapshot.
|
||||||
|
|
|
@ -107,6 +107,23 @@ Since the Hadoop AWS connector uses the `s3a` filesystem client, specify the war
|
||||||
The local catalog type can be used for catalogs configured on the local filesystem. Set the `icebergCatalog` type to `local`. You can use this catalog for demos or localized tests. It is not recommended for production use cases.
|
The local catalog type can be used for catalogs configured on the local filesystem. Set the `icebergCatalog` type to `local`. You can use this catalog for demos or localized tests. It is not recommended for production use cases.
|
||||||
The `warehouseSource` is set to `local` because this catalog only supports reading from a local filesystem.
|
The `warehouseSource` is set to `local` because this catalog only supports reading from a local filesystem.
|
||||||
|
|
||||||
|
## Downloading Iceberg extension
|
||||||
|
|
||||||
|
To download `druid-iceberg-extensions`, run the following command after replacing `<VERSION>` with the desired
|
||||||
|
Druid version:
|
||||||
|
|
||||||
|
```shell
|
||||||
|
java \
|
||||||
|
-cp "lib/*" \
|
||||||
|
-Ddruid.extensions.directory="extensions" \
|
||||||
|
-Ddruid.extensions.hadoopDependenciesDir="hadoop-dependencies" \
|
||||||
|
org.apache.druid.cli.Main tools pull-deps \
|
||||||
|
--no-default-hadoop \
|
||||||
|
-c "org.apache.druid.extensions.contrib:druid-iceberg-extensions:<VERSION>"
|
||||||
|
```
|
||||||
|
|
||||||
|
See [Loading community extensions](../../configuration/extensions.md#loading-community-extensions) for more information.
|
||||||
|
|
||||||
## Known limitations
|
## Known limitations
|
||||||
|
|
||||||
This section lists the known limitations that apply to the Iceberg extension.
|
This section lists the known limitations that apply to the Iceberg extension.
|
||||||
|
|
|
@ -21,7 +21,7 @@
|
||||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||||
|
|
||||||
<groupId>org.apache.druid.extensions</groupId>
|
<groupId>org.apache.druid.extensions.contrib</groupId>
|
||||||
<artifactId>druid-deltalake-extensions</artifactId>
|
<artifactId>druid-deltalake-extensions</artifactId>
|
||||||
<name>druid-deltalake-extensions</name>
|
<name>druid-deltalake-extensions</name>
|
||||||
<description>Delta Lake connector for Druid</description>
|
<description>Delta Lake connector for Druid</description>
|
||||||
|
|
|
@ -21,7 +21,7 @@
|
||||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||||
|
|
||||||
<groupId>org.apache.druid.extensions</groupId>
|
<groupId>org.apache.druid.extensions.contrib</groupId>
|
||||||
<artifactId>druid-iceberg-extensions</artifactId>
|
<artifactId>druid-iceberg-extensions</artifactId>
|
||||||
<name>druid-iceberg-extensions</name>
|
<name>druid-iceberg-extensions</name>
|
||||||
<description>druid-iceberg-extensions</description>
|
<description>druid-iceberg-extensions</description>
|
||||||
|
|
Loading…
Reference in New Issue