diff --git a/.gitignore b/.gitignore
index 0828365f2a7..adaf4d14c2b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -27,7 +27,7 @@ README
.pmdruleset.xml
.java-version
integration-tests/gen-scripts/
-/bin/
+**/bin/
*.hprof
**/.ipynb_checkpoints/
website/.yarn/
diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/InPlanningBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/InPlanningBenchmark.java
index ce01324116f..046da257423 100644
--- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/InPlanningBenchmark.java
+++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/InPlanningBenchmark.java
@@ -55,6 +55,7 @@ import org.apache.druid.sql.calcite.planner.PlannerResult;
import org.apache.druid.sql.calcite.run.SqlEngine;
import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog;
import org.apache.druid.sql.calcite.util.CalciteTests;
+import org.apache.druid.sql.hook.DruidHookDispatcher;
import org.apache.druid.timeline.DataSegment;
import org.apache.druid.timeline.partition.LinearShardSpec;
import org.openjdk.jmh.annotations.Benchmark;
@@ -202,7 +203,8 @@ public class InPlanningBenchmark
new CalciteRulesManager(ImmutableSet.of()),
CalciteTests.createJoinableFactoryWrapper(),
CatalogResolver.NULL_RESOLVER,
- new AuthConfig()
+ new AuthConfig(),
+ new DruidHookDispatcher()
);
String prefix = ("explain plan for select long1 from foo where long1 in ");
diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmark.java
index 3441e31c049..06ea2aa9e19 100644
--- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmark.java
+++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmark.java
@@ -89,6 +89,7 @@ import org.apache.druid.sql.calcite.util.CalciteTests;
import org.apache.druid.sql.calcite.util.LookylooModule;
import org.apache.druid.sql.calcite.util.QueryFrameworkUtils;
import org.apache.druid.sql.calcite.util.testoperator.CalciteTestOperatorModule;
+import org.apache.druid.sql.hook.DruidHookDispatcher;
import org.apache.druid.timeline.DataSegment;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -674,7 +675,8 @@ public class SqlBenchmark
new CalciteRulesManager(ImmutableSet.of()),
new JoinableFactoryWrapper(QueryFrameworkUtils.createDefaultJoinableFactory(injector)),
CatalogResolver.NULL_RESOLVER,
- new AuthConfig()
+ new AuthConfig(),
+ new DruidHookDispatcher()
);
return Pair.of(plannerFactory, engine);
diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java
index 8b2172182a0..d351b40f2db 100644
--- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java
+++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java
@@ -59,6 +59,7 @@ import org.apache.druid.sql.calcite.planner.PlannerResult;
import org.apache.druid.sql.calcite.run.SqlEngine;
import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog;
import org.apache.druid.sql.calcite.util.CalciteTests;
+import org.apache.druid.sql.hook.DruidHookDispatcher;
import org.apache.druid.timeline.DataSegment;
import org.apache.druid.timeline.partition.LinearShardSpec;
import org.openjdk.jmh.annotations.Benchmark;
@@ -364,7 +365,8 @@ public class SqlExpressionBenchmark
new CalciteRulesManager(ImmutableSet.of()),
CalciteTests.createJoinableFactoryWrapper(),
CatalogResolver.NULL_RESOLVER,
- new AuthConfig()
+ new AuthConfig(),
+ new DruidHookDispatcher()
);
try {
diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlGroupByBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlGroupByBenchmark.java
index bf521b83163..1f58f97b8e8 100644
--- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlGroupByBenchmark.java
+++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlGroupByBenchmark.java
@@ -57,6 +57,7 @@ import org.apache.druid.sql.calcite.planner.PlannerResult;
import org.apache.druid.sql.calcite.run.SqlEngine;
import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog;
import org.apache.druid.sql.calcite.util.CalciteTests;
+import org.apache.druid.sql.hook.DruidHookDispatcher;
import org.apache.druid.timeline.DataSegment;
import org.apache.druid.timeline.partition.LinearShardSpec;
import org.openjdk.jmh.annotations.Benchmark;
@@ -347,7 +348,8 @@ public class SqlGroupByBenchmark
new CalciteRulesManager(ImmutableSet.of()),
CalciteTests.createJoinableFactoryWrapper(),
CatalogResolver.NULL_RESOLVER,
- new AuthConfig()
+ new AuthConfig(),
+ new DruidHookDispatcher()
);
try {
diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlNestedDataBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlNestedDataBenchmark.java
index c329e9da30e..b896f8df52f 100644
--- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlNestedDataBenchmark.java
+++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlNestedDataBenchmark.java
@@ -63,6 +63,7 @@ import org.apache.druid.sql.calcite.planner.PlannerResult;
import org.apache.druid.sql.calcite.run.SqlEngine;
import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog;
import org.apache.druid.sql.calcite.util.CalciteTests;
+import org.apache.druid.sql.hook.DruidHookDispatcher;
import org.apache.druid.timeline.DataSegment;
import org.apache.druid.timeline.partition.LinearShardSpec;
import org.openjdk.jmh.annotations.Benchmark;
@@ -402,7 +403,8 @@ public class SqlNestedDataBenchmark
new CalciteRulesManager(ImmutableSet.of()),
CalciteTests.createJoinableFactoryWrapper(),
CatalogResolver.NULL_RESOLVER,
- new AuthConfig()
+ new AuthConfig(),
+ new DruidHookDispatcher()
);
try {
diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlVsNativeBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlVsNativeBenchmark.java
index bffc03469e8..2150b2100fb 100644
--- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlVsNativeBenchmark.java
+++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlVsNativeBenchmark.java
@@ -50,6 +50,7 @@ import org.apache.druid.sql.calcite.planner.PlannerResult;
import org.apache.druid.sql.calcite.run.SqlEngine;
import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog;
import org.apache.druid.sql.calcite.util.CalciteTests;
+import org.apache.druid.sql.hook.DruidHookDispatcher;
import org.apache.druid.timeline.DataSegment;
import org.apache.druid.timeline.partition.LinearShardSpec;
import org.openjdk.jmh.annotations.Benchmark;
@@ -132,7 +133,8 @@ public class SqlVsNativeBenchmark
new CalciteRulesManager(ImmutableSet.of()),
CalciteTests.createJoinableFactoryWrapper(),
CatalogResolver.NULL_RESOLVER,
- new AuthConfig()
+ new AuthConfig(),
+ new DruidHookDispatcher()
);
groupByQuery = GroupByQuery
.builder()
diff --git a/dev/quidem b/dev/quidem
index a68e08b3cfc..1a7d3f0ec96 100755
--- a/dev/quidem
+++ b/dev/quidem
@@ -28,13 +28,13 @@ OPTS+=" -Dsurefire.rerunFailingTestsCount=0"
OPTS+=" -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugin.surefire.SurefirePlugin=INFO"
[[ $@ =~ "-q" ]] && OPTS+=" -Dsurefire.trimStackTrace=true"
-OPTS+=" -pl sql -Dtest=SqlQuidemTest"
+OPTS+=" -pl quidem-ut -Dtest=QTest"
OPTS+=" org.apache.maven.plugins:maven-surefire-plugin:test"
case "$1" in
-h|--help)
cat << EOF
-Run SqlQuidemTest tests.
+Run QTest tests.
-q quiet (recommened)
-Dquidem.overwrite enables overwrite mode
-Dquidem.filter=*join* runs only tests matching path expression
diff --git a/extensions-contrib/compressed-bigdecimal/pom.xml b/extensions-contrib/compressed-bigdecimal/pom.xml
index 33b368b255e..d6c27d88812 100644
--- a/extensions-contrib/compressed-bigdecimal/pom.xml
+++ b/extensions-contrib/compressed-bigdecimal/pom.xml
@@ -166,6 +166,11 @@
com.fasterxml.jackson.core
jackson-annotations
+
+ org.reflections
+ reflections
+ test
+
diff --git a/extensions-contrib/tdigestsketch/pom.xml b/extensions-contrib/tdigestsketch/pom.xml
index 489b0b99556..4c192d62746 100644
--- a/extensions-contrib/tdigestsketch/pom.xml
+++ b/extensions-contrib/tdigestsketch/pom.xml
@@ -164,6 +164,11 @@
junit-vintage-engine
test
+
+ org.reflections
+ reflections
+ test
+
org.easymock
easymock
diff --git a/extensions-core/datasketches/pom.xml b/extensions-core/datasketches/pom.xml
index ee423909d19..75ff5a0868e 100644
--- a/extensions-core/datasketches/pom.xml
+++ b/extensions-core/datasketches/pom.xml
@@ -170,6 +170,11 @@
junit-jupiter-params
test
+
+ org.reflections
+ reflections
+ test
+
org.junit.vintage
junit-vintage-engine
diff --git a/extensions-core/druid-bloom-filter/pom.xml b/extensions-core/druid-bloom-filter/pom.xml
index 8ce419095de..b57ede7b16b 100644
--- a/extensions-core/druid-bloom-filter/pom.xml
+++ b/extensions-core/druid-bloom-filter/pom.xml
@@ -139,6 +139,11 @@
junit-vintage-engine
test
+
+ org.reflections
+ reflections
+ test
+
org.apache.druid
druid-processing
diff --git a/extensions-core/druid-catalog/pom.xml b/extensions-core/druid-catalog/pom.xml
index a48a7f1e7b8..c5bafbeb8dc 100644
--- a/extensions-core/druid-catalog/pom.xml
+++ b/extensions-core/druid-catalog/pom.xml
@@ -177,6 +177,11 @@
junit-vintage-engine
test
+
+ org.reflections
+ reflections
+ test
+
org.easymock
easymock
diff --git a/extensions-core/histogram/pom.xml b/extensions-core/histogram/pom.xml
index c252cfb87d3..a1c1f91e379 100644
--- a/extensions-core/histogram/pom.xml
+++ b/extensions-core/histogram/pom.xml
@@ -123,6 +123,11 @@
junit-vintage-engine
test
+
+ org.reflections
+ reflections
+ test
+
org.apache.druid
druid-processing
diff --git a/extensions-core/multi-stage-query/pom.xml b/extensions-core/multi-stage-query/pom.xml
index 9e637acff00..ab23a26e36f 100644
--- a/extensions-core/multi-stage-query/pom.xml
+++ b/extensions-core/multi-stage-query/pom.xml
@@ -233,6 +233,11 @@
junit-jupiter-params
test
+
+ org.reflections
+ reflections
+ test
+
org.easymock
easymock
diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/ResultsContextSerdeTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/ResultsContextSerdeTest.java
index b6023392576..72f2c817622 100644
--- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/ResultsContextSerdeTest.java
+++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/ResultsContextSerdeTest.java
@@ -43,6 +43,7 @@ import org.apache.druid.sql.calcite.schema.NamedDruidSchema;
import org.apache.druid.sql.calcite.schema.NamedViewSchema;
import org.apache.druid.sql.calcite.schema.ViewSchema;
import org.apache.druid.sql.calcite.util.CalciteTests;
+import org.apache.druid.sql.hook.DruidHookDispatcher;
import org.easymock.EasyMock;
import org.junit.Assert;
import org.junit.Before;
@@ -75,7 +76,8 @@ public class ResultsContextSerdeTest
"druid",
new CalciteRulesManager(ImmutableSet.of()),
CalciteTests.TEST_AUTHORIZER_MAPPER,
- AuthConfig.newBuilder().build()
+ AuthConfig.newBuilder().build(),
+ new DruidHookDispatcher()
);
final NativeSqlEngine engine = CalciteTests.createMockSqlEngine(
EasyMock.createMock(QuerySegmentWalker.class),
diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestBase.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestBase.java
index 080ba843204..5076e9f3d44 100644
--- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestBase.java
+++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestBase.java
@@ -186,6 +186,7 @@ import org.apache.druid.sql.calcite.util.SqlTestFramework.StandardComponentSuppl
import org.apache.druid.sql.calcite.util.TestDataBuilder;
import org.apache.druid.sql.calcite.view.InProcessViewManager;
import org.apache.druid.sql.guice.SqlBindings;
+import org.apache.druid.sql.hook.DruidHookDispatcher;
import org.apache.druid.storage.StorageConfig;
import org.apache.druid.storage.StorageConnector;
import org.apache.druid.storage.StorageConnectorModule;
@@ -562,7 +563,8 @@ public class MSQTestBase extends BaseCalciteQueryTest
new CalciteRulesManager(ImmutableSet.of()),
CalciteTests.createJoinableFactoryWrapper(),
catalogResolver,
- new AuthConfig()
+ new AuthConfig(),
+ new DruidHookDispatcher()
);
sqlStatementFactory = CalciteTests.createSqlStatementFactory(engine, plannerFactory);
diff --git a/extensions-core/stats/pom.xml b/extensions-core/stats/pom.xml
index 1b3a531dc42..048ad964bf8 100644
--- a/extensions-core/stats/pom.xml
+++ b/extensions-core/stats/pom.xml
@@ -123,7 +123,11 @@
junit-vintage-engine
test
-
+
+ org.reflections
+ reflections
+ test
+
org.apache.druid
druid-processing
diff --git a/extensions-core/testing-tools/pom.xml b/extensions-core/testing-tools/pom.xml
index 086d32559d3..fd0eed880c2 100644
--- a/extensions-core/testing-tools/pom.xml
+++ b/extensions-core/testing-tools/pom.xml
@@ -145,6 +145,11 @@
junit-jupiter-params
test
+
+ org.reflections
+ reflections
+ test
+
org.junit.vintage
junit-vintage-engine
diff --git a/pom.xml b/pom.xml
index ea9c3dcfa17..11738830512 100644
--- a/pom.xml
+++ b/pom.xml
@@ -242,6 +242,7 @@
integration-tests-ex/tools
integration-tests-ex/image
integration-tests-ex/cases
+ quidem-ut
diff --git a/quidem-ut/README.md b/quidem-ut/README.md
new file mode 100644
index 00000000000..5b594a3c8f4
--- /dev/null
+++ b/quidem-ut/README.md
@@ -0,0 +1,74 @@
+
+
+# Quidem UT
+
+Enables to write sql level tests easily.
+Can be used to write tests against existing test backends (ComponentSupplier) - by doing so the testcases can be moved closer to the excercised codes.
+
+## Usage
+
+### Install java&maven (if needed)
+
+If you don't have java&maven - one way to set that up is by using sdkman like this:
+```bash
+# install sdkman
+curl -s "https://get.sdkman.io" | bash
+# at the end of installation either open a new terminal; or follow the instructions at the end
+
+# install java&maven
+sdk install java 11.0.23-zulu
+sdk install maven
+
+# run mvn to see if it works
+mvn --version
+
+# download druid sourcces (FIXME: change this to the main repo/branch before merging)
+git clone --branch quidem-record https://github.com/kgyrtkirk/druid
+```
+
+
+
+
+### Running these tests
+
+* CI execution happens by a standard JUnit test `QTest` in this module
+* the `dev/quidem` script can be used to run these tests (after the project is built)
+
+### Launching a test generating broker
+
+* make sure to build the project first; one way to do that is:
+ ```bash
+ mvn install -pl quidem-ut/ -am -DskipTests -Pskip-static-checks
+ ```
+* launch the broker instance with:
+ ```bash
+ mvn exec:exec -pl quidem-ut -Dquidem.record.autostart=true
+ ```
+ * the broker will be running at http://localhost:12345
+ * the used test configuration backend can configured by supplying `quidem.uri`
+ ```bash
+ mvn exec:exec -pl quidem-ut -Dquidem.uri=druidtest:///?componentSupplier=ThetaSketchComponentSupplier
+ ```
+ * new record files can be started by calling http://localhost:12345/quidem/start
+ * if `quidem.record.autostart` is omitted recording will not start
+* after finished with the usage of the broker ; a new `record-123.iq` file will be in the project's worktree - it will contain all the executed statements
+ * running `dev/quidem -Dquidem.overwrite` updates the resultsets of all `iq` files around there
+ * rename the testfile to have a more descriptive name
+
diff --git a/quidem-ut/pom.xml b/quidem-ut/pom.xml
new file mode 100644
index 00000000000..ac48af25c17
--- /dev/null
+++ b/quidem-ut/pom.xml
@@ -0,0 +1,546 @@
+
+
+
+
+ 4.0.0
+
+ druid-quidem-ut
+ druid-quidem-ut
+ druid-quidem-ut
+
+
+ org.apache.druid
+ druid
+ 31.0.0-SNAPSHOT
+
+
+
+
+ confluent
+ https://packages.confluent.io/maven/
+
+
+
+
+ druidtest:///
+ false
+
+
+
+ com.amazonaws
+ amazon-kinesis-producer
+ 0.13.1
+
+
+ org.apache.avro
+ avro
+
+
+ com.opencsv
+ opencsv
+
+
+ org.apache.commons
+ commons-lang3
+
+
+ org.apache.httpcomponents
+ httpcore
+
+
+ org.apache.httpcomponents
+ httpclient
+
+
+ com.github.rvesse
+ airline
+
+
+ com.amazonaws
+ aws-java-sdk-kinesis
+ ${aws.sdk.version}
+
+
+
+ com.amazonaws
+ aws-java-sdk-core
+
+
+
+
+ com.amazonaws
+ aws-java-sdk-core
+ ${aws.sdk.version}
+
+
+ commons-codec
+ commons-codec
+
+
+ com.amazonaws
+ aws-java-sdk-s3
+ ${aws.sdk.version}
+
+
+ org.apache.druid.extensions
+ druid-orc-extensions
+ ${project.parent.version}
+
+
+ javax.servlet
+ servlet-api
+
+
+ com.squareup.okhttp
+ okhttp
+
+
+
+
+ org.apache.druid.extensions
+ druid-parquet-extensions
+ ${project.parent.version}
+
+
+ org.apache.druid.extensions
+ druid-avro-extensions
+ ${project.parent.version}
+
+
+ org.apache.druid.extensions
+ druid-protobuf-extensions
+ ${project.parent.version}
+
+
+ org.apache.druid.extensions
+ druid-s3-extensions
+ ${project.parent.version}
+
+
+ org.apache.druid.extensions
+ druid-kinesis-indexing-service
+ ${project.parent.version}
+
+
+ org.apache.druid.extensions
+ druid-azure-extensions
+ ${project.parent.version}
+
+
+ org.apache.druid.extensions
+ druid-google-extensions
+ ${project.parent.version}
+
+
+ org.apache.druid.extensions
+ druid-hdfs-storage
+ ${project.parent.version}
+
+
+ com.amazonaws
+ aws-java-sdk-bundle
+
+
+
+
+ org.apache.druid.extensions
+ druid-datasketches
+ ${project.parent.version}
+
+
+ org.apache.druid.extensions
+ druid-histogram
+ ${project.parent.version}
+
+
+ org.apache.druid
+ druid-aws-common
+ ${project.parent.version}
+
+
+ org.apache.druid
+ druid-processing
+ ${project.parent.version}
+
+
+ org.apache.druid
+ druid-sql
+ ${project.parent.version}
+
+
+ org.apache.druid
+ druid-indexing-service
+ ${project.parent.version}
+
+
+ org.apache.druid.extensions
+ mysql-metadata-storage
+ ${project.parent.version}
+
+
+ org.apache.druid.extensions
+ druid-kafka-indexing-service
+ ${project.parent.version}
+
+
+ org.apache.kafka
+ kafka-clients
+
+
+
+
+ org.apache.druid.extensions
+ druid-basic-security
+ ${project.parent.version}
+
+
+ org.apache.druid.extensions
+ druid-lookups-cached-global
+ ${project.parent.version}
+
+
+ org.apache.druid.extensions
+ druid-testing-tools
+ ${project.parent.version}
+
+
+ org.apache.druid.extensions
+ simple-client-sslcontext
+ ${project.parent.version}
+
+
+ org.apache.druid.extensions
+ druid-multi-stage-query
+ ${project.parent.version}
+
+
+ org.apache.druid
+ druid-services
+ ${project.parent.version}
+
+
+ org.apache.druid
+ druid-server
+ ${project.parent.version}
+
+
+ commons-io
+ commons-io
+
+
+ joda-time
+ joda-time
+
+
+ com.fasterxml.jackson.core
+ jackson-annotations
+
+
+ com.google.inject
+ guice
+
+
+ com.google.inject.extensions
+ guice-multibindings
+
+
+ com.fasterxml.jackson.core
+ jackson-databind
+
+
+ org.apache.kafka
+ kafka-clients
+ ${apache.kafka.version}
+
+
+ javax.ws.rs
+ jsr311-api
+
+
+ io.netty
+ netty
+
+
+ com.fasterxml.jackson.core
+ jackson-core
+
+
+ com.fasterxml.jackson.jaxrs
+ jackson-jaxrs-smile-provider
+
+
+ com.google.guava
+ guava
+
+
+ jakarta.validation
+ jakarta.validation-api
+
+
+ jakarta.inject
+ jakarta.inject-api
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
+ com.google.code.findbugs
+ jsr305
+
+
+ com.github.docker-java
+ docker-java
+
+
+ com.github.docker-java
+ docker-java-transport-jersey
+
+
+
+
+ com.github.docker-java
+ docker-java-transport-netty
+
+
+ com.github.docker-java
+ docker-java-api
+
+
+ com.github.docker-java
+ docker-java-core
+
+
+ net.hydromatic
+ quidem
+ test
+
+
+ io.netty
+ netty-transport-native-kqueue
+ osx-x86_64
+ runtime
+
+
+ io.netty
+ netty-transport-native-kqueue
+ osx-aarch_64
+ runtime
+
+
+ org.eclipse.jetty
+ jetty-servlet
+ ${jetty.version}
+
+
+ org.eclipse.jetty
+ jetty-server
+ ${jetty.version}
+
+
+ com.google.inject.extensions
+ guice-servlet
+
+
+ io.confluent
+ kafka-schema-registry-client
+ 6.2.12
+
+
+ org.slf4j
+ slf4j-log4j12
+
+
+ org.apache.avro
+ avro
+
+
+ com.fasterxml.jackson.core
+ jackson-databind
+
+
+ javax.ws.rs
+ javax.ws.rs-api
+
+
+ javax.ws.rs
+ javax.ws.rs-api
+
+
+ javax.ws.rs
+ jsr311-api
+
+
+ jakarta.ws.rs
+ jakarta.ws.rs-api
+
+
+
+
+ io.confluent
+ kafka-protobuf-provider
+ 6.2.12
+
+
+ com.google.protobuf
+ protobuf-java
+
+
+
+ org.apache.druid
+ druid-sql
+ test-jar
+ ${project.parent.version}
+
+
+
+ org.apache.druid
+ druid-processing
+ test-jar
+ ${project.parent.version}
+
+
+ org.apache.druid
+ druid-server
+ test-jar
+ ${project.parent.version}
+
+
+
+ org.apache.druid.extensions
+ druid-datasketches
+ test-jar
+ ${project.parent.version}
+ compile
+
+
+
+ org.slf4j
+ slf4j-simple
+ true
+
+
+
+ org.easymock
+ easymock
+ compile
+
+
+ org.reflections
+ reflections
+ compile
+
+
+
+
+
+ org.junit.jupiter
+ junit-jupiter
+
+
+ org.junit.vintage
+ junit-vintage-engine
+ test
+
+
+ org.junit.jupiter
+ junit-jupiter-api
+
+
+ junit
+ junit
+
+
+ org.apache.calcite.avatica
+ avatica
+
+
+ org.apache.calcite.avatica
+ avatica-core
+
+
+ org.apache.calcite
+ calcite-testkit
+ test
+
+
+ com.github.os72
+ protobuf-dynamic
+ 0.9.3
+ compile
+
+
+ com.amazonaws
+ aws-java-sdk-sts
+ ${aws.sdk.version}
+ runtime
+
+
+ org.apache.datasketches
+ datasketches-java
+ runtime
+
+
+ org.hamcrest
+ hamcrest-all
+ compile
+
+
+ org.hamcrest
+ hamcrest-core
+ compile
+
+
+
+
+
+
+ org.owasp
+ dependency-check-maven
+
+ true
+
+
+
+ org.apache.maven.plugins
+ maven-dependency-plugin
+
+ true
+
+
+
+ org.codehaus.mojo
+ exec-maven-plugin
+
+ java
+
+ -classpath
+
+ -Dquidem.uri=${quidem.uri}
+ -Dquidem.record.autostart=${quidem.record.autostart}
+ org.apache.druid.quidem.Launcher
+
+
+
+
+
+
diff --git a/quidem-ut/src/main/java/org/apache/druid/quidem/ExposedAsBrokerQueryComponentSupplierWrapper.java b/quidem-ut/src/main/java/org/apache/druid/quidem/ExposedAsBrokerQueryComponentSupplierWrapper.java
new file mode 100644
index 00000000000..f853dd317d1
--- /dev/null
+++ b/quidem-ut/src/main/java/org/apache/druid/quidem/ExposedAsBrokerQueryComponentSupplierWrapper.java
@@ -0,0 +1,370 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.quidem;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.inject.AbstractModule;
+import com.google.inject.Injector;
+import com.google.inject.Key;
+import com.google.inject.Module;
+import com.google.inject.Provides;
+import com.google.inject.name.Names;
+import org.apache.druid.cli.CliBroker;
+import org.apache.druid.cli.QueryJettyServerInitializer;
+import org.apache.druid.client.BrokerSegmentWatcherConfig;
+import org.apache.druid.client.BrokerServerView;
+import org.apache.druid.client.InternalQueryConfig;
+import org.apache.druid.client.TimelineServerView;
+import org.apache.druid.client.selector.CustomTierSelectorStrategyConfig;
+import org.apache.druid.client.selector.ServerSelectorStrategy;
+import org.apache.druid.client.selector.TierSelectorStrategy;
+import org.apache.druid.curator.CuratorModule;
+import org.apache.druid.curator.discovery.DiscoveryModule;
+import org.apache.druid.discovery.DruidNodeDiscoveryProvider;
+import org.apache.druid.guice.AnnouncerModule;
+import org.apache.druid.guice.BrokerProcessingModule;
+import org.apache.druid.guice.BrokerServiceModule;
+import org.apache.druid.guice.BuiltInTypesModule;
+import org.apache.druid.guice.CoordinatorDiscoveryModule;
+import org.apache.druid.guice.DruidInjectorBuilder;
+import org.apache.druid.guice.ExpressionModule;
+import org.apache.druid.guice.ExtensionsModule;
+import org.apache.druid.guice.JacksonConfigManagerModule;
+import org.apache.druid.guice.JavaScriptModule;
+import org.apache.druid.guice.Jerseys;
+import org.apache.druid.guice.JoinableFactoryModule;
+import org.apache.druid.guice.JsonConfigProvider;
+import org.apache.druid.guice.LazySingleton;
+import org.apache.druid.guice.LifecycleModule;
+import org.apache.druid.guice.LocalDataStorageDruidModule;
+import org.apache.druid.guice.MetadataConfigModule;
+import org.apache.druid.guice.QueryRunnerFactoryModule;
+import org.apache.druid.guice.SegmentWranglerModule;
+import org.apache.druid.guice.ServerModule;
+import org.apache.druid.guice.ServerTypeConfig;
+import org.apache.druid.guice.ServerViewModule;
+import org.apache.druid.guice.StartupLoggingModule;
+import org.apache.druid.guice.StorageNodeModule;
+import org.apache.druid.guice.annotations.Client;
+import org.apache.druid.guice.annotations.EscalatedClient;
+import org.apache.druid.guice.annotations.Json;
+import org.apache.druid.guice.http.HttpClientModule;
+import org.apache.druid.guice.security.AuthenticatorModule;
+import org.apache.druid.guice.security.AuthorizerModule;
+import org.apache.druid.guice.security.DruidAuthModule;
+import org.apache.druid.initialization.CoreInjectorBuilder;
+import org.apache.druid.initialization.Log4jShutterDownerModule;
+import org.apache.druid.initialization.ServerInjectorBuilder;
+import org.apache.druid.initialization.TombstoneDataStorageModule;
+import org.apache.druid.java.util.common.io.Closer;
+import org.apache.druid.metadata.storage.derby.DerbyMetadataStorageDruidModule;
+import org.apache.druid.query.QueryRunnerFactoryConglomerate;
+import org.apache.druid.query.QuerySegmentWalker;
+import org.apache.druid.query.RetryQueryRunnerConfig;
+import org.apache.druid.query.lookup.LookupExtractorFactoryContainerProvider;
+import org.apache.druid.rpc.guice.ServiceClientModule;
+import org.apache.druid.segment.join.JoinableFactoryWrapper;
+import org.apache.druid.segment.writeout.SegmentWriteOutMediumModule;
+import org.apache.druid.server.BrokerQueryResource;
+import org.apache.druid.server.ClientInfoResource;
+import org.apache.druid.server.DruidNode;
+import org.apache.druid.server.QueryLifecycleFactory;
+import org.apache.druid.server.QuerySchedulerProvider;
+import org.apache.druid.server.ResponseContextConfig;
+import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker;
+import org.apache.druid.server.SubqueryGuardrailHelper;
+import org.apache.druid.server.SubqueryGuardrailHelperProvider;
+import org.apache.druid.server.coordination.ServerType;
+import org.apache.druid.server.emitter.EmitterModule;
+import org.apache.druid.server.http.BrokerResource;
+import org.apache.druid.server.http.SelfDiscoveryResource;
+import org.apache.druid.server.initialization.AuthorizerMapperModule;
+import org.apache.druid.server.initialization.ExternalStorageAccessSecurityModule;
+import org.apache.druid.server.initialization.jetty.JettyServerInitializer;
+import org.apache.druid.server.initialization.jetty.JettyServerModule;
+import org.apache.druid.server.log.NoopRequestLogger;
+import org.apache.druid.server.log.RequestLogger;
+import org.apache.druid.server.metrics.QueryCountStatsProvider;
+import org.apache.druid.server.metrics.SubqueryCountStatsProvider;
+import org.apache.druid.server.router.TieredBrokerConfig;
+import org.apache.druid.server.security.AuthenticatorMapper;
+import org.apache.druid.server.security.Escalator;
+import org.apache.druid.server.security.TLSCertificateCheckerModule;
+import org.apache.druid.sql.calcite.planner.CalciteRulesManager;
+import org.apache.druid.sql.calcite.planner.CatalogResolver;
+import org.apache.druid.sql.calcite.run.NativeSqlEngine;
+import org.apache.druid.sql.calcite.run.SqlEngine;
+import org.apache.druid.sql.calcite.schema.BrokerSegmentMetadataCache;
+import org.apache.druid.sql.calcite.schema.DruidSchemaName;
+import org.apache.druid.sql.calcite.util.CalciteTests;
+import org.apache.druid.sql.calcite.util.SqlTestFramework;
+import org.apache.druid.sql.calcite.util.SqlTestFramework.Builder;
+import org.apache.druid.sql.calcite.util.SqlTestFramework.PlannerComponentSupplier;
+import org.apache.druid.sql.calcite.util.SqlTestFramework.QueryComponentSupplier;
+import org.apache.druid.sql.guice.SqlModule;
+import org.apache.druid.storage.StorageConnectorModule;
+import org.apache.druid.timeline.PruneLoadSpec;
+import org.eclipse.jetty.server.Server;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Properties;
+
+/**
+ * A wrapper class to expose a {@link QueryComponentSupplier} as a Broker service.
+ */
+public class ExposedAsBrokerQueryComponentSupplierWrapper implements QueryComponentSupplier
+{
+ private QueryComponentSupplier delegate;
+
+ public ExposedAsBrokerQueryComponentSupplierWrapper(QueryComponentSupplier delegate)
+ {
+ this.delegate = delegate;
+ }
+
+ @Override
+ public void gatherProperties(Properties properties)
+ {
+ delegate.gatherProperties(properties);
+ }
+
+ @Override
+ public void configureGuice(DruidInjectorBuilder builder)
+ {
+ }
+
+ @Override
+ public void configureGuice(CoreInjectorBuilder builder, List overrideModules)
+ {
+ delegate.configureGuice(builder);
+
+ installForServerModules(builder);
+
+ overrideModules.addAll(ExposedAsBrokerQueryComponentSupplierWrapper.brokerModules());
+ overrideModules.add(new BrokerTestModule());
+ builder.add(QuidemCaptureModule.class);
+ }
+
+ @Override
+ public QueryRunnerFactoryConglomerate createCongolmerate(Builder builder, Closer closer, ObjectMapper om)
+ {
+ return delegate.createCongolmerate(builder, closer, om);
+ }
+
+ @Override
+ public SpecificSegmentsQuerySegmentWalker createQuerySegmentWalker(QueryRunnerFactoryConglomerate conglomerate,
+ JoinableFactoryWrapper joinableFactory, Injector injector)
+ {
+ return delegate.createQuerySegmentWalker(conglomerate, joinableFactory, injector);
+ }
+
+ @Override
+ public SqlEngine createEngine(QueryLifecycleFactory qlf, ObjectMapper objectMapper, Injector injector)
+ {
+ return delegate.createEngine(qlf, objectMapper, injector);
+ }
+
+ @Override
+ public void configureJsonMapper(ObjectMapper mapper)
+ {
+ delegate.configureJsonMapper(mapper);
+ }
+
+ @Override
+ public JoinableFactoryWrapper createJoinableFactoryWrapper(LookupExtractorFactoryContainerProvider lookupProvider)
+ {
+ return delegate.createJoinableFactoryWrapper(lookupProvider);
+ }
+
+ @Override
+ public void finalizeTestFramework(SqlTestFramework sqlTestFramework)
+ {
+ delegate.finalizeTestFramework(sqlTestFramework);
+ }
+
+ @Override
+ public void close() throws IOException
+ {
+ delegate.close();
+ }
+
+ @Override
+ public PlannerComponentSupplier getPlannerComponentSupplier()
+ {
+ return delegate.getPlannerComponentSupplier();
+ }
+
+ public static class BrokerTestModule extends AbstractModule
+ {
+ @Override
+ protected void configure()
+ {
+ bind(AuthenticatorMapper.class).toInstance(CalciteTests.TEST_AUTHENTICATOR_MAPPER);
+ bind(Escalator.class).toInstance(CalciteTests.TEST_AUTHENTICATOR_ESCALATOR);
+ bind(RequestLogger.class).toInstance(new NoopRequestLogger());
+ bind(String.class)
+ .annotatedWith(DruidSchemaName.class)
+ .toInstance(CalciteTests.DRUID_SCHEMA_NAME);
+ bind(QuerySchedulerProvider.class).in(LazySingleton.class);
+ bind(CalciteRulesManager.class).toInstance(new CalciteRulesManager(ImmutableSet.of()));
+ bind(CatalogResolver.class).toInstance(CatalogResolver.NULL_RESOLVER);
+ }
+
+ @Provides
+ @LazySingleton
+ public BrokerSegmentMetadataCache provideCache()
+ {
+ return null;
+ }
+
+ @Provides
+ @LazySingleton
+ public Properties getProps()
+ {
+ Properties localProps = new Properties();
+ localProps.put("druid.enableTlsPort", "false");
+ localProps.put("druid.zk.service.enabled", "false");
+ localProps.put("druid.plaintextPort", "12345");
+ localProps.put("druid.host", "localhost");
+ localProps.put("druid.broker.segment.awaitInitializationOnStart", "false");
+ return localProps;
+ }
+
+ @Provides
+ @LazySingleton
+ public SqlEngine createMockSqlEngine(
+ final QuerySegmentWalker walker,
+ final QueryRunnerFactoryConglomerate conglomerate,
+ @Json ObjectMapper jsonMapper)
+ {
+ return new NativeSqlEngine(CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), jsonMapper);
+ }
+
+ @Provides
+ @LazySingleton
+ DruidNodeDiscoveryProvider getDruidNodeDiscoveryProvider()
+ {
+ final DruidNode coordinatorNode = CalciteTests.mockCoordinatorNode();
+ return CalciteTests.mockDruidNodeDiscoveryProvider(coordinatorNode);
+ }
+ }
+
+ /**
+ * Closely related to {@link CoreInjectorBuilder#forServer()}
+ */
+ private void installForServerModules(CoreInjectorBuilder builder)
+ {
+
+ builder.add(
+ new Log4jShutterDownerModule(),
+ new LifecycleModule(),
+ ExtensionsModule.SecondaryModule.class,
+ new DruidAuthModule(),
+ TLSCertificateCheckerModule.class,
+ EmitterModule.class,
+ HttpClientModule.global(),
+ HttpClientModule.escalatedGlobal(),
+ new HttpClientModule("druid.broker.http", Client.class, true),
+ new HttpClientModule("druid.broker.http", EscalatedClient.class, true),
+ new CuratorModule(),
+ new AnnouncerModule(),
+ new SegmentWriteOutMediumModule(),
+ new ServerModule(),
+ new StorageNodeModule(),
+ new JettyServerModule(),
+ new ExpressionModule(),
+ new BuiltInTypesModule(),
+ new DiscoveryModule(),
+ new ServerViewModule(),
+ new MetadataConfigModule(),
+ new DerbyMetadataStorageDruidModule(),
+ new JacksonConfigManagerModule(),
+ new CoordinatorDiscoveryModule(),
+ new LocalDataStorageDruidModule(),
+ new TombstoneDataStorageModule(),
+ new JavaScriptModule(),
+ new AuthenticatorModule(),
+ new AuthorizerModule(),
+ new AuthorizerMapperModule(),
+ new StartupLoggingModule(),
+ new ExternalStorageAccessSecurityModule(),
+ new ServiceClientModule(),
+ new StorageConnectorModule(),
+ new SqlModule(),
+ ServerInjectorBuilder.registerNodeRoleModule(ImmutableSet.of())
+ );
+ }
+
+ /**
+ * Closely related to {@link CliBroker#getModules}.
+ */
+ static List extends Module> brokerModules()
+ {
+ return ImmutableList.of(
+ new BrokerProcessingModule(),
+ new QueryRunnerFactoryModule(),
+ new SegmentWranglerModule(),
+ new JoinableFactoryModule(),
+ new BrokerServiceModule(),
+ binder -> {
+
+ binder.bindConstant().annotatedWith(Names.named("serviceName")).to(
+ TieredBrokerConfig.DEFAULT_BROKER_SERVICE_NAME
+ );
+ binder.bindConstant().annotatedWith(Names.named("servicePort")).to(8082);
+ binder.bindConstant().annotatedWith(Names.named("tlsServicePort")).to(8282);
+ binder.bindConstant().annotatedWith(PruneLoadSpec.class).to(true);
+ binder.bind(ResponseContextConfig.class).toInstance(ResponseContextConfig.newConfig(false));
+
+ binder.bind(TimelineServerView.class).to(BrokerServerView.class).in(LazySingleton.class);
+
+ JsonConfigProvider.bind(binder, "druid.broker.select", TierSelectorStrategy.class);
+ JsonConfigProvider.bind(binder, "druid.broker.select.tier.custom", CustomTierSelectorStrategyConfig.class);
+ JsonConfigProvider.bind(binder, "druid.broker.balancer", ServerSelectorStrategy.class);
+ JsonConfigProvider.bind(binder, "druid.broker.retryPolicy", RetryQueryRunnerConfig.class);
+ JsonConfigProvider.bind(binder, "druid.broker.segment", BrokerSegmentWatcherConfig.class);
+ JsonConfigProvider.bind(binder, "druid.broker.internal.query.config", InternalQueryConfig.class);
+ binder.bind(JettyServerInitializer.class).to(QueryJettyServerInitializer.class).in(LazySingleton.class);
+
+ binder.bind(BrokerQueryResource.class).in(LazySingleton.class);
+ Jerseys.addResource(binder, BrokerQueryResource.class);
+ binder.bind(SubqueryGuardrailHelper.class).toProvider(SubqueryGuardrailHelperProvider.class);
+ binder.bind(QueryCountStatsProvider.class).to(BrokerQueryResource.class).in(LazySingleton.class);
+ binder.bind(SubqueryCountStatsProvider.class).toInstance(new SubqueryCountStatsProvider());
+ Jerseys.addResource(binder, BrokerResource.class);
+ Jerseys.addResource(binder, ClientInfoResource.class);
+
+ LifecycleModule.register(binder, BrokerQueryResource.class);
+
+ LifecycleModule.register(binder, Server.class);
+ binder.bind(ServerTypeConfig.class).toInstance(new ServerTypeConfig(ServerType.BROKER));
+
+ binder.bind(String.class)
+ .annotatedWith(DruidSchemaName.class)
+ .toInstance(CalciteTests.DRUID_SCHEMA_NAME);
+
+ Jerseys.addResource(binder, SelfDiscoveryResource.class);
+ LifecycleModule.registerKey(binder, Key.get(SelfDiscoveryResource.class));
+ }
+ );
+ }
+}
diff --git a/quidem-ut/src/main/java/org/apache/druid/quidem/Launcher.java b/quidem-ut/src/main/java/org/apache/druid/quidem/Launcher.java
new file mode 100644
index 00000000000..b185084e0a7
--- /dev/null
+++ b/quidem-ut/src/main/java/org/apache/druid/quidem/Launcher.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.quidem;
+
+import com.google.common.base.Stopwatch;
+import org.apache.druid.cli.GuiceRunnable;
+import org.apache.druid.java.util.common.lifecycle.Lifecycle;
+import org.apache.druid.java.util.common.logger.Logger;
+import org.apache.druid.sql.calcite.SqlTestFrameworkConfig;
+import org.apache.druid.sql.calcite.SqlTestFrameworkConfig.ConfigurationInstance;
+import org.apache.druid.sql.calcite.SqlTestFrameworkConfig.SqlTestFrameworkConfigStore;
+import org.apache.druid.sql.calcite.util.SqlTestFramework;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.concurrent.TimeUnit;
+
+public class Launcher
+{
+ static final SqlTestFrameworkConfigStore CONFIG_STORE = new SqlTestFrameworkConfigStore(
+ x -> new ExposedAsBrokerQueryComponentSupplierWrapper(x)
+ );
+ private static final String QUIDEM_URI = "quidem.uri";
+ private static Logger log = new Logger(Launcher.class);
+ private final SqlTestFramework framework;
+ private final ConfigurationInstance configurationInstance;
+ private Lifecycle lifecycle;
+
+ public Launcher(String uri) throws Exception
+ {
+ SqlTestFrameworkConfig config = SqlTestFrameworkConfig.fromURL(uri);
+ configurationInstance = CONFIG_STORE.getConfigurationInstance(config);
+ framework = configurationInstance.framework;
+ }
+
+ public void start() throws Exception
+ {
+ lifecycle = GuiceRunnable.initLifecycle(framework.injector(), log);
+ if (withAutoStart()) {
+ callQuidemStart();
+ }
+ }
+
+ private void callQuidemStart() throws Exception
+ {
+ CloseableHttpClient client = HttpClients.createDefault();
+ HttpGet request = new HttpGet("http://localhost:12345/quidem/start");
+ request.addHeader("Content-Type", "application/json");
+ client.execute(request);
+ }
+
+ private boolean withAutoStart()
+ {
+ return Boolean.valueOf(System.getProperty("quidem.record.autostart", "false"));
+ }
+
+ public void shutdown()
+ {
+ lifecycle.stop();
+ }
+
+ public static void main(String[] args) throws Exception
+ {
+ String quidemUri = System.getProperty(QUIDEM_URI, "druidtest:///");
+ Properties p = System.getProperties();
+ for (Entry