Integration test cluster supports override config (#9473)

* integration test refactor

* integration test refactor

* refactor integration test

* refactor integration test

* refactor integration test

* refactor integration test

* refactor integration test

* refactor integration test

* refactor integration test

* refactor integration test

* address comments

integration-tests/common_run_cluster.sh

@@ -1,108 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Cleanup
-cleanup()
-{
-  for node in druid-historical druid-coordinator druid-overlord druid-router druid-router-permissive-tls druid-router-no-client-auth-tls druid-router-custom-check-tls druid-broker druid-middlemanager druid-zookeeper-kafka druid-metadata-storage;
-  do
-  docker stop $node
-  docker rm $node
-  done
-
-  docker network rm druid-it-net
-}
-
-# Druid environment and jars setup
-setup()
-{
-  # environment variables
-  DIR=$(cd $(dirname $0) && pwd)
-  DOCKERDIR=$DIR/docker
-  SHARED_DIR=${HOME}/shared
-  SUPERVISORDIR=/usr/lib/druid/conf
-  RESOURCEDIR=$DIR/src/test/resources
-
-  # so docker IP addr will be known during docker build
-  echo ${DOCKER_IP:=127.0.0.1} > $DOCKERDIR/docker_ip
-
-  # setup client keystore
-  ./docker/tls/generate-client-certs-and-keystores.sh
-  rm -rf docker/client_tls
-  cp -r client_tls docker/client_tls
-
-  # Make directories if they dont exist
-  mkdir -p $SHARED_DIR/logs
-  mkdir -p $SHARED_DIR/tasklogs
-
-  # install druid jars
-  rm -rf $SHARED_DIR/docker
-  cp -R docker $SHARED_DIR/docker
-  mvn -B dependency:copy-dependencies -DoutputDirectory=$SHARED_DIR/docker/lib
-
-  # install logging config
-  cp src/main/resources/log4j2.xml $SHARED_DIR/docker/lib/log4j2.xml
-
-  # copy the integration test jar, it provides test-only extension implementations
-  cp target/druid-integration-tests*.jar $SHARED_DIR/docker/lib
-
-  # one of the integration tests needs the wikiticker sample data
-  mkdir -p $SHARED_DIR/wikiticker-it
-  cp ../examples/quickstart/tutorial/wikiticker-2015-09-12-sampled.json.gz $SHARED_DIR/wikiticker-it/wikiticker-2015-09-12-sampled.json.gz
-  cp docker/wiki-simple-lookup.json $SHARED_DIR/wikiticker-it/wiki-simple-lookup.json
-}
-
-create_docker_network()
-{
-  docker network create --subnet=172.172.172.0/24 druid-it-net
-}
-
-# Start docker containers for all Druid processes and dependencies
-start_docker_containers()
-{
-  # Start zookeeper and kafka
-  docker run -d --privileged --net druid-it-net --ip 172.172.172.2 -e LANG=C.UTF-8 -e LANGUAGE=C.UTF-8 -e LC_ALL=C.UTF-8 --name druid-zookeeper-kafka -p 2181:2181 -p 9092:9092 -p 9093:9093 -v $SHARED_DIR:/shared -v $DOCKERDIR/zookeeper.conf:$SUPERVISORDIR/zookeeper.conf -v $DOCKERDIR/kafka.conf:$SUPERVISORDIR/kafka.conf druid/cluster
-
-  # Start MYSQL
-  docker run -d --privileged --net druid-it-net --ip 172.172.172.3 -e LANG=C.UTF-8 -e LANGUAGE=C.UTF-8 -e LC_ALL=C.UTF-8 --name druid-metadata-storage -v $SHARED_DIR:/shared -v $DOCKERDIR/metadata-storage.conf:$SUPERVISORDIR/metadata-storage.conf druid/cluster
-
-  # Start Overlord
-  docker run -d --privileged --net druid-it-net --ip 172.172.172.4 -e LANG=C.UTF-8 -e LANGUAGE=C.UTF-8 -e LC_ALL=C.UTF-8 --name druid-overlord -p 8090:8090 -p 8290:8290 -v $SHARED_DIR:/shared -v $DOCKERDIR/overlord.conf:$SUPERVISORDIR/overlord.conf --link druid-metadata-storage:druid-metadata-storage --link druid-zookeeper-kafka:druid-zookeeper-kafka druid/cluster
-
-  # Start Coordinator
-  docker run -d --privileged --net druid-it-net --ip 172.172.172.5 -e LANG=C.UTF-8 -e LANGUAGE=C.UTF-8 -e LC_ALL=C.UTF-8 --name druid-coordinator -p 8081:8081 -p 8281:8281 -v $SHARED_DIR:/shared -v $DOCKERDIR/coordinator.conf:$SUPERVISORDIR/coordinator.conf --link druid-overlord:druid-overlord --link druid-metadata-storage:druid-metadata-storage --link druid-zookeeper-kafka:druid-zookeeper-kafka druid/cluster
-
-  # Start Historical
-  docker run -d --privileged --net druid-it-net --ip 172.172.172.6 -e LANG=C.UTF-8 -e LANGUAGE=C.UTF-8 -e LC_ALL=C.UTF-8 --name druid-historical -p 8083:8083 -p 8283:8283 -v $SHARED_DIR:/shared -v $DOCKERDIR/historical.conf:$SUPERVISORDIR/historical.conf --link druid-zookeeper-kafka:druid-zookeeper-kafka druid/cluster
-
-  # Start Middlemanger
-  docker run -d --privileged --net druid-it-net --ip 172.172.172.7 -e LANG=C.UTF-8 -e LANGUAGE=C.UTF-8 -e LC_ALL=C.UTF-8 --name druid-middlemanager -p 8091:8091 -p 8291:8291 -p 8100:8100 -p 8101:8101 -p 8102:8102 -p 8103:8103 -p 8104:8104 -p 8105:8105 -p 8300:8300 -p 8301:8301 -p 8302:8302 -p 8303:8303 -p 8304:8304 -p 8305:8305 -v $RESOURCEDIR:/resources -v $SHARED_DIR:/shared -v $DOCKERDIR/middlemanager.conf:$SUPERVISORDIR/middlemanager.conf --link druid-zookeeper-kafka:druid-zookeeper-kafka --link druid-overlord:druid-overlord druid/cluster
-
-  # Start Broker
-  docker run -d --privileged --net druid-it-net --ip 172.172.172.8 -e LANG=C.UTF-8 -e LANGUAGE=C.UTF-8 -e LC_ALL=C.UTF-8 --name druid-broker -p 8082:8082 -p 8282:8282 -v $SHARED_DIR:/shared -v $DOCKERDIR/broker.conf:$SUPERVISORDIR/broker.conf --link druid-zookeeper-kafka:druid-zookeeper-kafka --link druid-middlemanager:druid-middlemanager --link druid-historical:druid-historical druid/cluster
-
-  # Start Router
-  docker run -d --privileged --net druid-it-net --ip 172.172.172.9 -e LANG=C.UTF-8 -e LANGUAGE=C.UTF-8 -e LC_ALL=C.UTF-8 --name druid-router -p 8888:8888 -p 9088:9088 -v $SHARED_DIR:/shared -v $DOCKERDIR/router.conf:$SUPERVISORDIR/router.conf --link druid-zookeeper-kafka:druid-zookeeper-kafka --link druid-coordinator:druid-coordinator --link druid-broker:druid-broker druid/cluster
-
-  # Start Router with permissive TLS settings (client auth enabled, no hostname verification, no revocation check)
-  docker run -d --privileged --net druid-it-net --ip 172.172.172.10 -e LANG=C.UTF-8 -e LANGUAGE=C.UTF-8 -e LC_ALL=C.UTF-8 --name druid-router-permissive-tls -p 8889:8889 -p 9089:9089 -v $SHARED_DIR:/shared -v $DOCKERDIR/router-permissive-tls.conf:$SUPERVISORDIR/router-permissive-tls.conf --link druid-zookeeper-kafka:druid-zookeeper-kafka --link druid-coordinator:druid-coordinator --link druid-broker:druid-broker druid/cluster
-
-  # Start Router with TLS but no client auth
-  docker run -d --privileged --net druid-it-net --ip 172.172.172.11 -e LANG=C.UTF-8 -e LANGUAGE=C.UTF-8 -e LC_ALL=C.UTF-8 --name druid-router-no-client-auth-tls -p 8890:8890 -p 9090:9090 -v $SHARED_DIR:/shared -v $DOCKERDIR/router-no-client-auth-tls.conf:$SUPERVISORDIR/router-no-client-auth-tls.conf --link druid-zookeeper-kafka:druid-zookeeper-kafka --link druid-coordinator:druid-coordinator --link druid-broker:druid-broker druid/cluster
-
-  # Start Router with custom TLS cert checkers
-  docker run -d --privileged --net druid-it-net --ip 172.172.172.12 -e LANG=C.UTF-8 -e LANGUAGE=C.UTF-8 -e LC_ALL=C.UTF-8 --hostname druid-router-custom-check-tls --name druid-router-custom-check-tls -p 8891:8891 -p 9091:9091 -v $SHARED_DIR:/shared -v $DOCKERDIR/router-custom-check-tls.conf:$SUPERVISORDIR/router-custom-check-tls.conf --link druid-zookeeper-kafka:druid-zookeeper-kafka --link druid-coordinator:druid-coordinator --link druid-broker:druid-broker druid/cluster
-}

integration-tests/docker/Dockerfile

@@ -55,6 +55,9 @@ RUN find /var/lib/mysql -type f -exec touch {} \; && service mysql start \
 # Setup supervisord
 ADD supervisord.conf /etc/supervisor/conf.d/supervisord.conf
 
+# Add druid configuration setup script
+ADD druid.sh /druid.sh
+
 # mysql
 ADD run-mysql.sh /run-mysql.sh
 
@@ -95,4 +98,12 @@ EXPOSE 8300 8301 8302 8303 8304 8305
 EXPOSE 9092 9093
 
 WORKDIR /var/lib/druid
-ENTRYPOINT export HOST_IP="$(resolveip -s $HOSTNAME)" && /tls/generate-server-certs-and-keystores.sh && exec /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf
+ENTRYPOINT /tls/generate-server-certs-and-keystores.sh \
+            # Create druid service config files with all the config variables
+            && . /druid.sh; setupConfig \
+            # Export the service config file path to use in supervisord conf file
+            && export DRUID_COMMON_CONF_DIR="$(. /druid.sh; getConfPath ${DRUID_SERVICE})" \
+            # Export the common config file path to use in supervisord conf file
+            && export DRUID_SERVICE_CONF_DIR="$(. /druid.sh; getConfPath _common)" \
+            # Run Druid service using supervisord
+            && exec /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf

integration-tests/docker/broker.conf

@@ -1,66 +0,0 @@
-[program:druid-broker]
-command=java
-  -server
-  -Xmx512m
-  -Xms512m
-  -XX:NewSize=256m
-  -XX:MaxNewSize=256m
-  -XX:+UseG1GC
-  -Duser.timezone=UTC
-  -Dfile.encoding=UTF-8
-  -Ddruid.host=%(ENV_HOST_IP)s
-  -Ddruid.zk.service.host=druid-zookeeper-kafka
-  -Ddruid.processing.buffer.sizeBytes=25000000
-  -Ddruid.query.groupBy.maxOnDiskStorage=300000000
-  -Ddruid.server.http.numThreads=40
-  -Ddruid.processing.numThreads=1
-  -Ddruid.broker.http.numConnections=20
-  -Ddruid.broker.http.readTimeout=PT5M
-  -Ddruid.broker.cache.useCache=true
-  -Ddruid.broker.cache.populateCache=true
-  -Ddruid.lookup.namespace.cache.type=onHeap
-  -Ddruid.cache.sizeInBytes=40000000
-  -Ddruid.lookup.numLookupLoadingThreads=1
-  -Ddruid.auth.authenticatorChain="[\"basic\"]"
-  -Ddruid.auth.authenticator.basic.type=basic
-  -Ddruid.auth.authenticator.basic.initialAdminPassword=priest
-  -Ddruid.auth.authenticator.basic.initialInternalClientPassword=warlock
-  -Ddruid.auth.authenticator.basic.authorizerName=basic
-  -Ddruid.auth.basic.common.cacheDirectory=/tmp/authCache/broker
-  -Ddruid.auth.basic.common.maxSyncRetries=20
-  -Ddruid.escalator.type=basic
-  -Ddruid.escalator.internalClientUsername=druid_system
-  -Ddruid.escalator.internalClientPassword=warlock
-  -Ddruid.escalator.authorizerName=basic
-  -Ddruid.auth.authorizers="[\"basic\"]"
-  -Ddruid.auth.authorizer.basic.type=basic
-  -Ddruid.sql.enable=true
-  -Ddruid.sql.avatica.enable=true
-  -Ddruid.enableTlsPort=true
-  -Ddruid.server.https.keyStorePath=/tls/server.jks
-  -Ddruid.server.https.keyStoreType=jks
-  -Ddruid.server.https.certAlias=druid
-  -Ddruid.server.https.keyManagerPassword=druid123
-  -Ddruid.server.https.keyStorePassword=druid123
-  -Ddruid.server.https.requireClientCertificate=true
-  -Ddruid.server.https.trustStorePath=/tls/truststore.jks
-  -Ddruid.server.https.trustStorePassword=druid123
-  -Ddruid.server.https.trustStoreAlgorithm=PKIX
-  -Ddruid.server.https.validateHostnames=true
-  -Ddruid.server.https.crlPath=/tls/revocations.crl
-  -Ddruid.client.https.trustStoreAlgorithm=PKIX
-  -Ddruid.client.https.protocol=TLSv1.2
-  -Ddruid.client.https.trustStorePath=/tls/truststore.jks
-  -Ddruid.client.https.trustStorePassword=druid123
-  -Ddruid.client.https.keyStorePath=/tls/server.jks
-  -Ddruid.client.https.certAlias=druid
-  -Ddruid.client.https.keyManagerPassword=druid123
-  -Ddruid.client.https.keyStorePassword=druid123
-  -Dlog4j.configurationFile=/shared/docker/lib/log4j2.xml
-  -cp /shared/docker/lib/*:/usr/local/druid/lib/mysql-connector-java.jar
-  org.apache.druid.cli.Main server broker
-redirect_stderr=true
-autorestart=false
-priority=100
-stdout_logfile=/shared/logs/broker.log
-environment=AWS_REGION=us-east-1

integration-tests/docker/coordinator.conf

@@ -1,61 +0,0 @@
-[program:druid-coordinator]
-command=java
-  -server
-  -Xmx128m
-  -Xms128m
-  -XX:+UseG1GC
-  -Duser.timezone=UTC
-  -Dfile.encoding=UTF-8
-  -Ddruid.host=%(ENV_HOST_IP)s
-  -Ddruid.server.http.numThreads=20
-  -Ddruid.metadata.storage.type=mysql
-  -Ddruid.metadata.storage.connector.connectURI=jdbc:mysql://druid-metadata-storage/druid
-  -Ddruid.metadata.storage.connector.user=druid
-  -Ddruid.metadata.storage.connector.password=diurd
-  -Ddruid.zk.service.host=druid-zookeeper-kafka
-  -Ddruid.coordinator.startDelay=PT5S
-  -Ddruid.lookup.numLookupLoadingThreads=1
-  -Ddruid.manager.lookups.hostUpdateTimeout=PT30S
-  -Ddruid.manager.lookups.period=10000
-  -Ddruid.manager.lookups.threadPoolSize=2
-  -Ddruid.auth.authenticatorChain="[\"basic\"]"
-  -Ddruid.auth.authenticator.basic.type=basic
-  -Ddruid.auth.authenticator.basic.initialAdminPassword=priest
-  -Ddruid.auth.authenticator.basic.initialInternalClientPassword=warlock
-  -Ddruid.auth.authenticator.basic.authorizerName=basic
-  -Ddruid.auth.basic.common.cacheDirectory=/tmp/authCache/coordinator
-  -Ddruid.escalator.type=basic
-  -Ddruid.escalator.internalClientUsername=druid_system
-  -Ddruid.escalator.internalClientPassword=warlock
-  -Ddruid.escalator.authorizerName=basic
-  -Ddruid.auth.authorizers="[\"basic\"]"
-  -Ddruid.auth.authorizer.basic.type=basic
-  -Ddruid.auth.unsecuredPaths="[\"/druid/coordinator/v1/loadqueue\"]"
-  -Ddruid.enableTlsPort=true
-  -Ddruid.server.https.keyStorePath=/tls/server.jks
-  -Ddruid.server.https.keyStoreType=jks
-  -Ddruid.server.https.certAlias=druid
-  -Ddruid.server.https.keyManagerPassword=druid123
-  -Ddruid.server.https.keyStorePassword=druid123
-  -Ddruid.server.https.requireClientCertificate=true
-  -Ddruid.server.https.trustStorePath=/tls/truststore.jks
-  -Ddruid.server.https.trustStorePassword=druid123
-  -Ddruid.server.https.trustStoreAlgorithm=PKIX
-  -Ddruid.server.https.validateHostnames=true
-  -Ddruid.server.https.crlPath=/tls/revocations.crl
-  -Ddruid.client.https.trustStoreAlgorithm=PKIX
-  -Ddruid.client.https.protocol=TLSv1.2
-  -Ddruid.client.https.trustStorePath=/tls/truststore.jks
-  -Ddruid.client.https.trustStorePassword=druid123
-  -Ddruid.client.https.keyStorePath=/tls/server.jks
-  -Ddruid.client.https.certAlias=druid
-  -Ddruid.client.https.keyManagerPassword=druid123
-  -Ddruid.client.https.keyStorePassword=druid123
-  -Dlog4j.configurationFile=/shared/docker/lib/log4j2.xml
-  -cp /shared/docker/lib/*:/usr/local/druid/lib/mysql-connector-java.jar
-  org.apache.druid.cli.Main server coordinator
-redirect_stderr=true
-priority=100
-autorestart=false
-stdout_logfile=/shared/logs/coordinator.log
-environment=AWS_REGION=us-east-1

integration-tests/docker/druid.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+getConfPath() {
+    cluster_conf_base=/tmp/conf/druid/cluster
+    case "$1" in
+    _common) echo $cluster_conf_base/_common ;;
+    historical) echo $cluster_conf_base/data/historical ;;
+    middleManager) echo $cluster_conf_base/data/middleManager ;;
+    coordinator) echo $cluster_conf_base/master/coordinator ;;
+    broker) echo $cluster_conf_base/query/broker ;;
+    router) echo $cluster_conf_base/query/router ;;
+    overlord) echo $cluster_conf_base/master/overlord ;;
+    esac
+}
+
+# Delete the old key (if existing) and append new key=value
+setKey() {
+    service="$1"
+    key="$2"
+    value="$3"
+    service_conf=$(getConfPath $service)/runtime.properties
+    # Delete from all
+    sed -ri "/$key=/d" $COMMON_CONF_DIR/common.runtime.properties
+    [ -f $service_conf ] && sed -ri "/$key=/d" $service_conf
+    [ -f $service_conf ] && echo "$key=$value" >>$service_conf
+    [ -f $service_conf ] || echo "$key=$value" >>$COMMON_CONF_DIR/common.runtime.properties
+
+    echo "Setting $key=$value in $service_conf"
+}
+
+setupConfig() {
+  echo "$(date -Is) configuring service $DRUID_SERVICE"
+
+  # We put all the config in /tmp/conf to allow for a
+  # read-only root filesystem
+  mkdir -p /tmp/conf/druid
+
+  COMMON_CONF_DIR=$(getConfPath _common)
+  SERVICE_CONF_DIR=$(getConfPath ${DRUID_SERVICE})
+
+  mkdir -p $COMMON_CONF_DIR
+  mkdir -p $SERVICE_CONF_DIR
+  touch $COMMON_CONF_DIR/common.runtime.properties
+  touch $SERVICE_CONF_DIR/runtime.properties
+
+  setKey $DRUID_SERVICE druid.host $(resolveip -s $HOSTNAME)
+  setKey $DRUID_SERVICE druid.worker.ip $(resolveip -s $HOSTNAME)
+
+
+  # Write out all the environment variables starting with druid_ to druid service config file
+  # This will replace _ with . in the key
+  env | grep ^druid_ | while read evar;
+  do
+      # Can't use IFS='=' to parse since var might have = in it (e.g. password)
+      val=$(echo "$evar" | sed -e 's?[^=]*=??')
+      var=$(echo "$evar" | sed -e 's?^\([^=]*\)=.*?\1?g' -e 's?_?.?g')
+      setKey $DRUID_SERVICE "$var" "$val"
+  done
+}

integration-tests/docker/environment-configs/broker

@@ -0,0 +1,39 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+DRUID_SERVICE=broker
+DRUID_LOG_PATH=/shared/logs/broker.log
+
+# JAVA OPTS
+SERVICE_DRUID_JAVA_OPTS=-server -Xmx512m -Xms512m -XX:NewSize=256m -XX:MaxNewSize=256m -XX:+UseG1GC
+
+# Druid configs
+druid_processing_buffer_sizeBytes=25000000
+druid_query_groupBy_maxOnDiskStorage=300000000
+druid_server_http_numThreads=40
+druid_processing_numThreads=1
+druid_broker_http_numConnections=20
+druid_broker_http_readTimeout=PT5M
+druid_broker_cache_useCache=true
+druid_broker_cache_populateCache=true
+druid_lookup_namespace_cache_type=onHeap
+druid_cache_sizeInBytes=40000000
+druid_auth_basic_common_cacheDirectory=/tmp/authCache/broker
+druid_sql_avatica_enable=true
+druid_server_https_crlPath=/tls/revocations.crl

integration-tests/docker/environment-configs/common

@@ -0,0 +1,64 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+LANG=C.UTF-8
+LANGUAGE=C.UTF-8
+LC_ALL=C.UTF-8
+
+# JAVA OPTS
+COMMON_DRUID_JAVA_OPTS=-Duser.timezone=UTC -Dfile.encoding=UTF-8 -Dlog4j.configurationFile=/shared/docker/lib/log4j2.xml
+DRUID_DEP_LIB_DIR=/shared/docker/lib/*:/usr/local/druid/lib/mysql-connector-java.jar
+
+# Druid configs
+druid_auth_authenticator_basic_authorizerName=basic
+druid_auth_authenticator_basic_initialAdminPassword=priest
+druid_auth_authenticator_basic_initialInternalClientPassword=warlock
+druid_auth_authenticator_basic_type=basic
+druid_auth_authenticatorChain=["basic"]
+druid_auth_authorizer_basic_type=basic
+druid_auth_authorizers=["basic"]
+druid_client_https_certAlias=druid
+druid_client_https_keyManagerPassword=druid123
+druid_client_https_keyStorePassword=druid123
+druid_client_https_keyStorePath=/tls/server.jks
+druid_client_https_protocol=TLSv1.2
+druid_client_https_trustStoreAlgorithm=PKIX
+druid_client_https_trustStorePassword=druid123
+druid_client_https_trustStorePath=/tls/truststore.jks
+druid_enableTlsPort=true
+druid_escalator_authorizerName=basic
+druid_escalator_internalClientPassword=warlock
+druid_escalator_internalClientUsername=druid_system
+druid_escalator_type=basic
+druid_lookup_numLookupLoadingThreads=1
+druid_server_http_numThreads=20
+druid_server_https_certAlias=druid
+druid_server_https_keyManagerPassword=druid123
+druid_server_https_keyStorePassword=druid123
+druid_server_https_keyStorePath=/tls/server.jks
+druid_server_https_keyStoreType=jks
+druid_server_https_requireClientCertificate=true
+druid_server_https_trustStoreAlgorithm=PKIX
+druid_server_https_trustStorePassword=druid123
+druid_server_https_trustStorePath=/tls/truststore.jks
+druid_server_https_validateHostnames=true
+druid_zk_service_host=druid-zookeeper-kafka
+druid_auth_basic_common_maxSyncRetries=20
+druid_indexer_logs_directory=/shared/tasklogs
+druid_sql_enable=true

integration-tests/docker/environment-configs/coordinator

@@ -0,0 +1,37 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+DRUID_SERVICE=coordinator
+DRUID_LOG_PATH=/shared/logs/coordinator.log
+
+# JAVA OPTS
+SERVICE_DRUID_JAVA_OPTS=-server -Xmx128m -Xms128m -XX:+UseG1GC
+
+# Druid configs
+druid_metadata_storage_type=mysql
+druid_metadata_storage_connector_connectURI=jdbc:mysql://druid-metadata-storage/druid
+druid_metadata_storage_connector_user=druid
+druid_metadata_storage_connector_password=diurd
+druid_coordinator_startDelay=PT5S
+druid_manager_lookups_hostUpdateTimeout=PT30S
+druid_manager_lookups_period=10000
+druid_manager_lookups_threadPoolSize=2
+druid_auth_basic_common_cacheDirectory=/tmp/authCache/coordinator
+druid_auth_unsecuredPaths=["/druid/coordinator/v1/loadqueue"]
+druid_server_https_crlPath=/tls/revocations.crl

integration-tests/docker/environment-configs/historical

@@ -0,0 +1,35 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+DRUID_SERVICE=historical
+DRUID_LOG_PATH=/shared/logs/historical.log
+
+# JAVA OPTS
+SERVICE_DRUID_JAVA_OPTS=-server -Xmx512m -Xms512m -XX:NewSize=256m -XX:MaxNewSize=256m -XX:+UseG1GC
+
+# Druid configs
+druid_s3_accessKey=AKIAJI7DG7CDECGBQ6NA
+druid_s3_secretKey=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
+druid_processing_buffer_sizeBytes=25000000
+druid_processing_numThreads=2
+druid_query_groupBy_maxOnDiskStorage=300000000
+druid_segmentCache_locations=[{"path":"/shared/druid/indexCache","maxSize":5000000000}]
+druid_server_maxSize=5000000000
+druid_auth_basic_common_cacheDirectory=/tmp/authCache/historical
+druid_server_https_crlPath=/tls/revocations.crl

integration-tests/docker/environment-configs/middlemanager

@@ -0,0 +1,41 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+DRUID_SERVICE=middleManager
+DRUID_LOG_PATH=/shared/logs/middlemanager.log
+
+# JAVA OPTS
+SERVICE_DRUID_JAVA_OPTS=-server -Xmx64m -Xms64m -XX:+UseG1GC
+
+# Druid configs
+druid_server_http_numThreads=100
+druid_worker_capacity=3
+druid_storage_storageDirectory=/shared/storage
+druid_indexer_runner_javaOptsArray=["-server", "-Xmx256m", "-Xms256m", "-XX:NewSize=128m", "-XX:MaxNewSize=128m", "-XX:+UseG1GC", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-Dlog4j.configurationFile=/shared/docker/lib/log4j2.xml"]
+
+druid_indexer_fork_property_druid_processing_buffer_sizeBytes=25000000
+druid_indexer_fork_property_druid_processing_numThreads=1
+druid_indexer_fork_server_http_numThreads=20
+druid_s3_accessKey=AKIAJI7DG7CDECGBQ6NA
+druid_s3_secretKey=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
+druid_selectors_indexing_serviceName=druid/overlord
+druid_indexer_task_chathandler_type=announce
+druid_auth_basic_common_cacheDirectory=/tmp/authCache/middleManager
+druid_startup_logging_logProperties=true
+druid_server_https_crlPath=/tls/revocations.crl

integration-tests/docker/environment-configs/overlord

@@ -0,0 +1,35 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+DRUID_SERVICE=overlord
+DRUID_LOG_PATH=/shared/logs/overlord.log
+
+# JAVA OPTS
+SERVICE_DRUID_JAVA_OPTS=-server -Xmx128m -Xms128m -XX:+UseG1GC
+
+# Druid configs
+druid_metadata_storage_type=mysql
+druid_metadata_storage_connector_connectURI=jdbc:mysql://druid-metadata-storage/druid
+druid_metadata_storage_connector_user=druid
+druid_metadata_storage_connector_password=diurd
+druid_indexer_storage_type=metadata
+druid_indexer_runner_type=remote
+druid_auth_authenticatorChain=["basic"]
+druid_auth_basic_common_cacheDirectory=/tmp/authCache/overlord
+druid_server_https_crlPath=/tls/revocations.crl

integration-tests/docker/environment-configs/router

@@ -0,0 +1,29 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+DRUID_SERVICE=router
+DRUID_LOG_PATH=/shared/logs/router.log
+
+# JAVA OPTS
+SERVICE_DRUID_JAVA_OPTS=-server -Xmx128m -XX:+UseG1GC
+
+# Druid configs
+druid_auth_basic_common_cacheDirectory=/tmp/authCache/router
+druid_sql_avatica_enable=true
+druid_server_https_crlPath=/tls/revocations.crl

integration-tests/docker/environment-configs/router-custom-check-tls

@@ -0,0 +1,34 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+DRUID_SERVICE=router
+DRUID_LOG_PATH=/shared/logs/router-custom-check-tls.log
+
+# JAVA OPTS
+SERVICE_DRUID_JAVA_OPTS=-server -Xmx128m -XX:+UseConcMarkSweepGC -XX:+PrintGCDetails
+
+# Druid configs
+druid_plaintextPort=8891
+druid_tlsPort=9091
+druid_server_http_numThreads=100
+druid_router_managementProxy_enabled=true
+druid_auth_basic_common_cacheDirectory=/tmp/authCache/router-custom-check-tls
+druid_sql_avatica_enable=true
+druid_client_https_validateHostnames=false
+druid_tls_certificateChecker=integration-test

integration-tests/docker/environment-configs/router-no-client-auth-tls

@@ -0,0 +1,33 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+DRUID_SERVICE=router
+DRUID_LOG_PATH=/shared/logs/router-no-client-auth-tls.log
+
+# JAVA OPTS
+SERVICE_DRUID_JAVA_OPTS=-server -Xmx128m -XX:+UseConcMarkSweepGC -XX:+PrintGCDetails
+
+# Druid configs
+druid_plaintextPort=8890
+druid_tlsPort=9090
+druid_server_http_numThreads=100
+druid_auth_basic_common_cacheDirectory=/tmp/authCache/router-no-client-auth-tls
+druid_sql_avatica_enable=true
+druid_server_https_requireClientCertificate=false
+druid_server_https_validateHostnames=false

integration-tests/docker/environment-configs/router-permissive-tls

@@ -0,0 +1,32 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+DRUID_SERVICE=router
+DRUID_LOG_PATH=/shared/logs/router-permissive-tls.log
+
+# JAVA OPTS
+SERVICE_DRUID_JAVA_OPTS=-server -Xmx128m -XX:+UseConcMarkSweepGC -XX:+PrintGCDetails
+
+# Druid configs
+druid_plaintextPort=8889
+druid_tlsPort=9089
+druid_server_http_numThreads=100
+druid_auth_basic_common_cacheDirectory=/tmp/authCache/router-permissive-tls
+druid_sql_avatica_enable=true
+druid_server_https_validateHostnames=false

integration-tests/docker/historical.conf

@@ -1,62 +0,0 @@
-[program:druid-historical]
-command=java
-  -server
-  -Xmx512m
-  -Xms512m
-  -XX:NewSize=256m
-  -XX:MaxNewSize=256m
-  -XX:+UseG1GC
-  -Duser.timezone=UTC
-  -Dfile.encoding=UTF-8
-  -Ddruid.host=%(ENV_HOST_IP)s
-  -Ddruid.zk.service.host=druid-zookeeper-kafka
-  -Ddruid.s3.accessKey=AKIAJI7DG7CDECGBQ6NA
-  -Ddruid.s3.secretKey=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
-  -Ddruid.processing.buffer.sizeBytes=25000000
-  -Ddruid.processing.numThreads=2
-  -Ddruid.query.groupBy.maxOnDiskStorage=300000000
-  -Ddruid.server.http.numThreads=20
-  -Ddruid.segmentCache.locations="[{\"path\":\"/shared/druid/indexCache\",\"maxSize\":5000000000}]"
-  -Ddruid.server.maxSize=5000000000
-  -Ddruid.lookup.numLookupLoadingThreads=1
-  -Ddruid.auth.authenticatorChain="[\"basic\"]"
-  -Ddruid.auth.authenticator.basic.type=basic
-  -Ddruid.auth.authenticator.basic.initialAdminPassword=priest
-  -Ddruid.auth.authenticator.basic.initialInternalClientPassword=warlock
-  -Ddruid.auth.authenticator.basic.authorizerName=basic
-  -Ddruid.auth.basic.common.cacheDirectory=/tmp/authCache/historical
-  -Ddruid.auth.basic.common.maxSyncRetries=20
-  -Ddruid.escalator.type=basic
-  -Ddruid.escalator.internalClientUsername=druid_system
-  -Ddruid.escalator.internalClientPassword=warlock
-  -Ddruid.escalator.authorizerName=basic
-  -Ddruid.auth.authorizers="[\"basic\"]"
-  -Ddruid.auth.authorizer.basic.type=basic
-  -Ddruid.enableTlsPort=true
-  -Ddruid.server.https.keyStorePath=/tls/server.jks
-  -Ddruid.server.https.keyStoreType=jks
-  -Ddruid.server.https.certAlias=druid
-  -Ddruid.server.https.keyManagerPassword=druid123
-  -Ddruid.server.https.keyStorePassword=druid123
-  -Ddruid.server.https.requireClientCertificate=true
-  -Ddruid.server.https.trustStorePath=/tls/truststore.jks
-  -Ddruid.server.https.trustStorePassword=druid123
-  -Ddruid.server.https.trustStoreAlgorithm=PKIX
-  -Ddruid.server.https.validateHostnames=true
-  -Ddruid.server.https.crlPath=/tls/revocations.crl
-  -Ddruid.client.https.trustStoreAlgorithm=PKIX
-  -Ddruid.client.https.protocol=TLSv1.2
-  -Ddruid.client.https.trustStorePath=/tls/truststore.jks
-  -Ddruid.client.https.trustStorePassword=druid123
-  -Ddruid.client.https.keyStorePath=/tls/server.jks
-  -Ddruid.client.https.certAlias=druid
-  -Ddruid.client.https.keyManagerPassword=druid123
-  -Ddruid.client.https.keyStorePassword=druid123
-  -Dlog4j.configurationFile=/shared/docker/lib/log4j2.xml
-  -cp /shared/docker/lib/*:/usr/local/druid/lib/mysql-connector-java.jar
-  org.apache.druid.cli.Main server historical
-redirect_stderr=true
-priority=100
-autorestart=false
-stdout_logfile=/shared/logs/historical.log
-environment=AWS_REGION=us-east-1

integration-tests/docker/middlemanager.conf

@@ -1,66 +0,0 @@
-[program:druid-middlemanager]
-command=java
-  -server
-  -Xmx64m
-  -Xms64m
-  -XX:+UseG1GC
-  -Duser.timezone=UTC
-  -Dfile.encoding=UTF-8
-  -Ddruid.host=%(ENV_HOST_IP)s
-  -Ddruid.server.http.numThreads=100
-  -Ddruid.zk.service.host=druid-zookeeper-kafka
-  -Ddruid.worker.capacity=3
-  -Ddruid.indexer.logs.directory=/shared/tasklogs
-  -Ddruid.storage.storageDirectory=/shared/storage
-  -Ddruid.indexer.runner.javaOpts="-server -Xmx256m -Xms256m -XX:NewSize=128m -XX:MaxNewSize=128m -XX:+UseG1GC -XX:+PrintGCDetails -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Dlog4j.configurationFile=/shared/docker/lib/log4j2.xml"
-  -Ddruid.indexer.fork.property.druid.processing.buffer.sizeBytes=25000000
-  -Ddruid.indexer.fork.property.druid.processing.numThreads=1
-  -Ddruid.indexer.fork.server.http.numThreads=20
-  -Ddruid.s3.accessKey=AKIAJI7DG7CDECGBQ6NA
-  -Ddruid.s3.secretKey=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
-  -Ddruid.worker.ip=%(ENV_HOST_IP)s
-  -Ddruid.selectors.indexing.serviceName=druid/overlord
-  -Ddruid.indexer.task.chathandler.type=announce
-  -Ddruid.lookup.numLookupLoadingThreads=1
-  -Ddruid.auth.authenticatorChain="[\"basic\"]"
-  -Ddruid.auth.authenticator.basic.type=basic
-  -Ddruid.auth.authenticator.basic.initialAdminPassword=priest
-  -Ddruid.auth.authenticator.basic.initialInternalClientPassword=warlock
-  -Ddruid.auth.authenticator.basic.authorizerName=basic
-  -Ddruid.auth.basic.common.cacheDirectory=/tmp/authCache/middleManager
-  -Ddruid.auth.basic.common.maxSyncRetries=20
-  -Ddruid.escalator.type=basic
-  -Ddruid.escalator.internalClientUsername=druid_system
-  -Ddruid.escalator.internalClientPassword=warlock
-  -Ddruid.escalator.authorizerName=basic
-  -Ddruid.auth.authorizers="[\"basic\"]"
-  -Ddruid.auth.authorizer.basic.type=basic
-  -Ddruid.enableTlsPort=true
-  -Ddruid.server.https.keyStorePath=/tls/server.jks
-  -Ddruid.server.https.keyStoreType=jks
-  -Ddruid.server.https.certAlias=druid
-  -Ddruid.server.https.keyManagerPassword=druid123
-  -Ddruid.server.https.keyStorePassword=druid123
-  -Ddruid.server.https.requireClientCertificate=true
-  -Ddruid.server.https.trustStorePath=/tls/truststore.jks
-  -Ddruid.server.https.trustStorePassword=druid123
-  -Ddruid.server.https.trustStoreAlgorithm=PKIX
-  -Ddruid.server.https.validateHostnames=true
-  -Ddruid.server.https.crlPath=/tls/revocations.crl
-  -Ddruid.client.https.trustStoreAlgorithm=PKIX
-  -Ddruid.client.https.protocol=TLSv1.2
-  -Ddruid.client.https.trustStorePath=/tls/truststore.jks
-  -Ddruid.client.https.trustStorePassword=druid123
-  -Ddruid.client.https.keyStorePath=/tls/server.jks
-  -Ddruid.client.https.certAlias=druid
-  -Ddruid.client.https.keyManagerPassword=druid123
-  -Ddruid.client.https.keyStorePassword=druid123
-  -Ddruid.startup.logging.logProperties=true
-  -Dlog4j.configurationFile=/shared/docker/lib/log4j2.xml
-  -cp /shared/docker/lib/*:/usr/local/druid/lib/mysql-connector-java.jar
-  org.apache.druid.cli.Main server middleManager
-redirect_stderr=true
-priority=100
-autorestart=false
-stdout_logfile=/shared/logs/middlemanager.log
-environment=AWS_REGION=us-east-1

integration-tests/docker/overlord.conf

@@ -1,60 +0,0 @@
-[program:druid-overlord]
-command=java
-  -server
-  -Xmx128m
-  -Xms128m
-  -XX:+UseG1GC
-  -Duser.timezone=UTC
-  -Dfile.encoding=UTF-8
-  -Ddruid.host=%(ENV_HOST_IP)s
-  -Ddruid.server.http.numThreads=20
-  -Ddruid.metadata.storage.type=mysql
-  -Ddruid.metadata.storage.connector.connectURI=jdbc:mysql://druid-metadata-storage/druid
-  -Ddruid.metadata.storage.connector.user=druid
-  -Ddruid.metadata.storage.connector.password=diurd
-  -Ddruid.zk.service.host=druid-zookeeper-kafka
-  -Ddruid.indexer.storage.type=metadata
-  -Ddruid.indexer.logs.directory=/shared/tasklogs
-  -Ddruid.indexer.runner.type=remote
-  -Ddruid.lookup.numLookupLoadingThreads=1
-  -Ddruid.auth.authenticatorChain="[\"basic\"]"
-  -Ddruid.auth.authenticator.basic.type=basic
-  -Ddruid.auth.authenticator.basic.initialAdminPassword=priest
-  -Ddruid.auth.authenticator.basic.initialInternalClientPassword=warlock
-  -Ddruid.auth.authenticator.basic.authorizerName=basic
-  -Ddruid.auth.basic.common.cacheDirectory=/tmp/authCache/overlord
-  -Ddruid.auth.basic.common.maxSyncRetries=20
-  -Ddruid.escalator.type=basic
-  -Ddruid.escalator.internalClientUsername=druid_system
-  -Ddruid.escalator.internalClientPassword=warlock
-  -Ddruid.escalator.authorizerName=basic
-  -Ddruid.auth.authorizers="[\"basic\"]"
-  -Ddruid.auth.authorizer.basic.type=basic
-  -Ddruid.enableTlsPort=true
-  -Ddruid.server.https.keyStorePath=/tls/server.jks
-  -Ddruid.server.https.keyStoreType=jks
-  -Ddruid.server.https.certAlias=druid
-  -Ddruid.server.https.keyManagerPassword=druid123
-  -Ddruid.server.https.keyStorePassword=druid123
-  -Ddruid.server.https.requireClientCertificate=true
-  -Ddruid.server.https.trustStorePath=/tls/truststore.jks
-  -Ddruid.server.https.trustStorePassword=druid123
-  -Ddruid.server.https.trustStoreAlgorithm=PKIX
-  -Ddruid.server.https.validateHostnames=true
-  -Ddruid.server.https.crlPath=/tls/revocations.crl
-  -Ddruid.client.https.trustStoreAlgorithm=PKIX
-  -Ddruid.client.https.protocol=TLSv1.2
-  -Ddruid.client.https.trustStorePath=/tls/truststore.jks
-  -Ddruid.client.https.trustStorePassword=druid123
-  -Ddruid.client.https.keyStorePath=/tls/server.jks
-  -Ddruid.client.https.certAlias=druid
-  -Ddruid.client.https.keyManagerPassword=druid123
-  -Ddruid.client.https.keyStorePassword=druid123
-  -Dlog4j.configurationFile=/shared/docker/lib/log4j2.xml
-  -cp /shared/docker/lib/*:/usr/local/druid/lib/mysql-connector-java.jar
-  org.apache.druid.cli.Main server overlord
-redirect_stderr=true
-priority=100
-autorestart=false
-stdout_logfile=/shared/logs/overlord.log
-environment=AWS_REGION=us-east-1

integration-tests/docker/router-custom-check-tls.conf

@@ -1,57 +0,0 @@
-[program:druid-router-custom-check-tls]
-command=java
-  -server
-  -Xmx128m
-  -XX:+UseConcMarkSweepGC
-  -XX:+PrintGCDetails
-  -Duser.timezone=UTC
-  -Dfile.encoding=UTF-8
-  -Ddruid.host=%(ENV_HOST_IP)s
-  -Ddruid.plaintextPort=8891
-  -Ddruid.tlsPort=9091
-  -Ddruid.zk.service.host=druid-zookeeper-kafka
-  -Ddruid.server.http.numThreads=100
-  -Ddruid.lookup.numLookupLoadingThreads=1
-  -Ddruid.router.managementProxy.enabled=true
-  -Ddruid.auth.authenticatorChain="[\"basic\"]"
-  -Ddruid.auth.authenticator.basic.type=basic
-  -Ddruid.auth.authenticator.basic.initialAdminPassword=priest
-  -Ddruid.auth.authenticator.basic.initialInternalClientPassword=warlock
-  -Ddruid.auth.authenticator.basic.authorizerName=basic
-  -Ddruid.auth.basic.common.cacheDirectory=/tmp/authCache/router-custom-check-tls
-  -Ddruid.auth.basic.common.maxSyncRetries=20
-  -Ddruid.escalator.type=basic
-  -Ddruid.escalator.internalClientUsername=druid_system
-  -Ddruid.escalator.internalClientPassword=warlock
-  -Ddruid.escalator.authorizerName=basic
-  -Ddruid.auth.authorizers="[\"basic\"]"
-  -Ddruid.auth.authorizer.basic.type=basic
-  -Ddruid.sql.enable=true
-  -Ddruid.sql.avatica.enable=true
-  -Ddruid.enableTlsPort=true
-  -Ddruid.server.https.keyStorePath=/tls/server.jks
-  -Ddruid.server.https.keyStoreType=jks
-  -Ddruid.server.https.certAlias=druid
-  -Ddruid.server.https.keyManagerPassword=druid123
-  -Ddruid.server.https.keyStorePassword=druid123
-  -Ddruid.server.https.requireClientCertificate=true
-  -Ddruid.server.https.trustStorePath=/tls/truststore.jks
-  -Ddruid.server.https.trustStorePassword=druid123
-  -Ddruid.server.https.trustStoreAlgorithm=PKIX
-  -Ddruid.server.https.validateHostnames=true
-  -Ddruid.client.https.trustStoreAlgorithm=PKIX
-  -Ddruid.client.https.protocol=TLSv1.2
-  -Ddruid.client.https.trustStorePath=/tls/truststore.jks
-  -Ddruid.client.https.trustStorePassword=druid123
-  -Ddruid.client.https.keyStorePath=/tls/server.jks
-  -Ddruid.client.https.certAlias=druid
-  -Ddruid.client.https.keyManagerPassword=druid123
-  -Ddruid.client.https.keyStorePassword=druid123
-  -Ddruid.client.https.validateHostnames=false
-  -Ddruid.tls.certificateChecker=integration-test
-  -cp /shared/docker/lib/*
-  org.apache.druid.cli.Main server router
-redirect_stderr=true
-priority=100
-autorestart=false
-stdout_logfile=/shared/logs/router-custom-check-tls.log

integration-tests/docker/router-no-client-auth-tls.conf

@@ -1,55 +0,0 @@
-[program:druid-router-no-client-auth-tls]
-command=java
-  -server
-  -Xmx128m
-  -XX:+UseConcMarkSweepGC
-  -XX:+PrintGCDetails
-  -Duser.timezone=UTC
-  -Dfile.encoding=UTF-8
-  -Ddruid.host=%(ENV_HOST_IP)s
-  -Ddruid.plaintextPort=8890
-  -Ddruid.tlsPort=9090
-  -Ddruid.zk.service.host=druid-zookeeper-kafka
-  -Ddruid.server.http.numThreads=100
-  -Ddruid.lookup.numLookupLoadingThreads=1
-  -Ddruid.auth.authenticatorChain="[\"basic\"]"
-  -Ddruid.auth.authenticator.basic.type=basic
-  -Ddruid.auth.authenticator.basic.initialAdminPassword=priest
-  -Ddruid.auth.authenticator.basic.initialInternalClientPassword=warlock
-  -Ddruid.auth.authenticator.basic.authorizerName=basic
-  -Ddruid.auth.basic.common.cacheDirectory=/tmp/authCache/router-no-client-auth-tls
-  -Ddruid.auth.basic.common.maxSyncRetries=20
-  -Ddruid.escalator.type=basic
-  -Ddruid.escalator.internalClientUsername=druid_system
-  -Ddruid.escalator.internalClientPassword=warlock
-  -Ddruid.escalator.authorizerName=basic
-  -Ddruid.auth.authorizers="[\"basic\"]"
-  -Ddruid.auth.authorizer.basic.type=basic
-  -Ddruid.sql.enable=true
-  -Ddruid.sql.avatica.enable=true
-  -Ddruid.enableTlsPort=true
-  -Ddruid.server.https.keyStorePath=/tls/server.jks
-  -Ddruid.server.https.keyStoreType=jks
-  -Ddruid.server.https.certAlias=druid
-  -Ddruid.server.https.keyManagerPassword=druid123
-  -Ddruid.server.https.keyStorePassword=druid123
-  -Ddruid.server.https.requireClientCertificate=false
-  -Ddruid.server.https.trustStorePath=/tls/truststore.jks
-  -Ddruid.server.https.trustStorePassword=druid123
-  -Ddruid.server.https.trustStoreAlgorithm=PKIX
-  -Ddruid.server.https.validateHostnames=false
-  -Ddruid.client.https.trustStoreAlgorithm=PKIX
-  -Ddruid.client.https.protocol=TLSv1.2
-  -Ddruid.client.https.trustStorePath=/tls/truststore.jks
-  -Ddruid.client.https.trustStorePassword=druid123
-  -Ddruid.client.https.keyStorePath=/tls/server.jks
-  -Ddruid.client.https.certAlias=druid
-  -Ddruid.client.https.keyManagerPassword=druid123
-  -Ddruid.client.https.keyStorePassword=druid123
-  -Dlog4j.configurationFile=/shared/docker/lib/log4j2.xml
-  -cp /shared/docker/lib/*:/usr/local/druid/lib/mysql-connector-java.jar
-  org.apache.druid.cli.Main server router
-redirect_stderr=true
-priority=100
-autorestart=false
-stdout_logfile=/shared/logs/router-no-client-auth-tls.log

integration-tests/docker/router-permissive-tls.conf

@@ -1,55 +0,0 @@
-[program:druid-router-permissive-tls]
-command=java
-  -server
-  -Xmx128m
-  -XX:+UseConcMarkSweepGC
-  -XX:+PrintGCDetails
-  -Duser.timezone=UTC
-  -Dfile.encoding=UTF-8
-  -Ddruid.host=%(ENV_HOST_IP)s
-  -Ddruid.plaintextPort=8889
-  -Ddruid.tlsPort=9089
-  -Ddruid.zk.service.host=druid-zookeeper-kafka
-  -Ddruid.server.http.numThreads=100
-  -Ddruid.lookup.numLookupLoadingThreads=1
-  -Ddruid.auth.authenticatorChain="[\"basic\"]"
-  -Ddruid.auth.authenticator.basic.type=basic
-  -Ddruid.auth.authenticator.basic.initialAdminPassword=priest
-  -Ddruid.auth.authenticator.basic.initialInternalClientPassword=warlock
-  -Ddruid.auth.authenticator.basic.authorizerName=basic
-  -Ddruid.auth.basic.common.cacheDirectory=/tmp/authCache/router-permissive-tls
-  -Ddruid.auth.basic.common.maxSyncRetries=20
-  -Ddruid.escalator.type=basic
-  -Ddruid.escalator.internalClientUsername=druid_system
-  -Ddruid.escalator.internalClientPassword=warlock
-  -Ddruid.escalator.authorizerName=basic
-  -Ddruid.auth.authorizers="[\"basic\"]"
-  -Ddruid.auth.authorizer.basic.type=basic
-  -Ddruid.sql.enable=true
-  -Ddruid.sql.avatica.enable=true
-  -Ddruid.enableTlsPort=true
-  -Ddruid.server.https.keyStorePath=/tls/server.jks
-  -Ddruid.server.https.keyStoreType=jks
-  -Ddruid.server.https.certAlias=druid
-  -Ddruid.server.https.keyManagerPassword=druid123
-  -Ddruid.server.https.keyStorePassword=druid123
-  -Ddruid.server.https.requireClientCertificate=true
-  -Ddruid.server.https.trustStorePath=/tls/truststore.jks
-  -Ddruid.server.https.trustStorePassword=druid123
-  -Ddruid.server.https.trustStoreAlgorithm=PKIX
-  -Ddruid.server.https.validateHostnames=false
-  -Ddruid.client.https.trustStoreAlgorithm=PKIX
-  -Ddruid.client.https.protocol=TLSv1.2
-  -Ddruid.client.https.trustStorePath=/tls/truststore.jks
-  -Ddruid.client.https.trustStorePassword=druid123
-  -Ddruid.client.https.keyStorePath=/tls/server.jks
-  -Ddruid.client.https.certAlias=druid
-  -Ddruid.client.https.keyManagerPassword=druid123
-  -Ddruid.client.https.keyStorePassword=druid123
-  -Dlog4j.configurationFile=/shared/docker/lib/log4j2.xml
-  -cp /shared/docker/lib/*:/usr/local/druid/lib/mysql-connector-java.jar
-  org.apache.druid.cli.Main server router
-redirect_stderr=true
-priority=100
-autorestart=false
-stdout_logfile=/shared/logs/router-permissive-tls.log

integration-tests/docker/router.conf

@@ -1,54 +0,0 @@
-[program:druid-router]
-command=java
-  -server
-  -Xmx128m
-  -XX:+UseG1GC
-  -Duser.timezone=UTC
-  -Dfile.encoding=UTF-8
-  -Ddruid.host=%(ENV_HOST_IP)s
-  -Ddruid.zk.service.host=druid-zookeeper-kafka
-  -Ddruid.server.http.numThreads=20
-  -Ddruid.lookup.numLookupLoadingThreads=1
-  -Ddruid.auth.authenticatorChain="[\"basic\"]"
-  -Ddruid.auth.authenticator.basic.type=basic
-  -Ddruid.auth.authenticator.basic.initialAdminPassword=priest
-  -Ddruid.auth.authenticator.basic.initialInternalClientPassword=warlock
-  -Ddruid.auth.authenticator.basic.authorizerName=basic
-  -Ddruid.auth.basic.common.cacheDirectory=/tmp/authCache/router
-  -Ddruid.auth.basic.common.maxSyncRetries=20
-  -Ddruid.escalator.type=basic
-  -Ddruid.escalator.internalClientUsername=druid_system
-  -Ddruid.escalator.internalClientPassword=warlock
-  -Ddruid.escalator.authorizerName=basic
-  -Ddruid.auth.authorizers="[\"basic\"]"
-  -Ddruid.auth.authorizer.basic.type=basic
-  -Ddruid.sql.enable=true
-  -Ddruid.sql.avatica.enable=true
-  -Ddruid.enableTlsPort=true
-  -Ddruid.server.https.keyStorePath=/tls/server.jks
-  -Ddruid.server.https.keyStoreType=jks
-  -Ddruid.server.https.certAlias=druid
-  -Ddruid.server.https.keyManagerPassword=druid123
-  -Ddruid.server.https.keyStorePassword=druid123
-  -Ddruid.server.https.requireClientCertificate=true
-  -Ddruid.server.https.trustStorePath=/tls/truststore.jks
-  -Ddruid.server.https.trustStorePassword=druid123
-  -Ddruid.server.https.trustStoreAlgorithm=PKIX
-  -Ddruid.server.https.validateHostnames=true
-  -Ddruid.server.https.crlPath=/tls/revocations.crl
-  -Ddruid.client.https.trustStoreAlgorithm=PKIX
-  -Ddruid.client.https.protocol=TLSv1.2
-  -Ddruid.client.https.trustStorePath=/tls/truststore.jks
-  -Ddruid.client.https.trustStorePassword=druid123
-  -Ddruid.client.https.keyStorePath=/tls/server.jks
-  -Ddruid.client.https.certAlias=druid
-  -Ddruid.client.https.keyManagerPassword=druid123
-  -Ddruid.client.https.keyStorePassword=druid123
-  -Dlog4j.configurationFile=/shared/docker/lib/log4j2.xml
-  -cp /shared/docker/lib/*:/usr/local/druid/lib/mysql-connector-java.jar
-  org.apache.druid.cli.Main server router
-redirect_stderr=true
-priority=100
-autorestart=false
-stdout_logfile=/shared/logs/router.log
-environment=AWS_REGION=us-east-1

integration-tests/docker/service-supervisords/druid.conf

@@ -0,0 +1,7 @@
+[program:druid-service]
+command=java %(ENV_SERVICE_DRUID_JAVA_OPTS)s %(ENV_COMMON_DRUID_JAVA_OPTS)s -cp %(ENV_DRUID_COMMON_CONF_DIR)s:%(ENV_DRUID_SERVICE_CONF_DIR)s:%(ENV_DRUID_DEP_LIB_DIR)s org.apache.druid.cli.Main server %(ENV_DRUID_SERVICE)s
+redirect_stderr=true
+priority=100
+autorestart=false
+stdout_logfile=%(ENV_DRUID_LOG_PATH)s
+environment=AWS_REGION=us-east-1

integration-tests/pom.xml

@@ -266,7 +266,7 @@
         <profile>
             <id>integration-tests</id>
             <properties>
-                <jvm.runtime>8</jvm.runtime>
+                <override.config.path></override.config.path>
             </properties>
             <build>
                 <plugins>
@@ -281,7 +281,12 @@
                                 </goals>
                                 <phase>pre-integration-test</phase>
                                 <configuration>
-                                    <executable>${project.basedir}/run_cluster_using_java_runtime_${jvm.runtime}.sh</executable>
+                                    <environmentVariables>
+                                    <DRUID_INTEGRATION_TEST_JVM_RUNTIME>${jvm.runtime}</DRUID_INTEGRATION_TEST_JVM_RUNTIME>
+                                    <DRUID_INTEGRATION_TEST_GROUP>${groups}</DRUID_INTEGRATION_TEST_GROUP>
+                                    <DRUID_INTEGRATION_TEST_OVERRIDE_CONFIG_PATH>${override.config.path}</DRUID_INTEGRATION_TEST_OVERRIDE_CONFIG_PATH>
+                                    </environmentVariables>
+                                    <executable>${project.basedir}/run_cluster.sh</executable>
                                 </configuration>
                             </execution>
                             <execution>

integration-tests/run_cluster.sh

@@ -0,0 +1,153 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Cleanup old images/containers
+{
+  for node in druid-historical druid-coordinator druid-overlord druid-router druid-router-permissive-tls druid-router-no-client-auth-tls druid-router-custom-check-tls druid-broker druid-middlemanager druid-zookeeper-kafka druid-metadata-storage;
+  do
+  docker stop $node
+  docker rm $node
+  done
+
+  docker network rm druid-it-net
+}
+
+# Druid environment and jars setup
+{
+  # environment variables
+  DIR=$(cd $(dirname $0) && pwd)
+  DOCKERDIR=$DIR/docker
+  SERVICE_SUPERVISORDS_DIR=$DOCKERDIR/service-supervisords
+  ENVIRONMENT_CONFIGS_DIR=$DOCKERDIR/environment-configs
+  SHARED_DIR=${HOME}/shared
+  SUPERVISORDIR=/usr/lib/druid/conf
+  RESOURCEDIR=$DIR/src/test/resources
+
+  # so docker IP addr will be known during docker build
+  echo ${DOCKER_IP:=127.0.0.1} > $DOCKERDIR/docker_ip
+
+  # setup client keystore
+  ./docker/tls/generate-client-certs-and-keystores.sh
+  rm -rf docker/client_tls
+  cp -r client_tls docker/client_tls
+
+  # Make directories if they dont exist
+  mkdir -p $SHARED_DIR/logs
+  mkdir -p $SHARED_DIR/tasklogs
+
+  # install druid jars
+  rm -rf $SHARED_DIR/docker
+  cp -R docker $SHARED_DIR/docker
+  mvn -B dependency:copy-dependencies -DoutputDirectory=$SHARED_DIR/docker/lib
+
+  # install logging config
+  cp src/main/resources/log4j2.xml $SHARED_DIR/docker/lib/log4j2.xml
+
+  # copy the integration test jar, it provides test-only extension implementations
+  cp target/druid-integration-tests*.jar $SHARED_DIR/docker/lib
+
+  # one of the integration tests needs the wikiticker sample data
+  mkdir -p $SHARED_DIR/wikiticker-it
+  cp ../examples/quickstart/tutorial/wikiticker-2015-09-12-sampled.json.gz $SHARED_DIR/wikiticker-it/wikiticker-2015-09-12-sampled.json.gz
+  cp docker/wiki-simple-lookup.json $SHARED_DIR/wikiticker-it/wiki-simple-lookup.json
+
+  # setup all enviornment variables to be pass to the containers
+  COMMON_ENV="--env-file=$ENVIRONMENT_CONFIGS_DIR/common"
+  BROKER_ENV="--env-file=$ENVIRONMENT_CONFIGS_DIR/broker"
+  COORDINATOR_ENV="--env-file=$ENVIRONMENT_CONFIGS_DIR/coordinator"
+  HISTORICAL_ENV="--env-file=$ENVIRONMENT_CONFIGS_DIR/historical"
+  MIDDLEMANAGER_ENV="--env-file=$ENVIRONMENT_CONFIGS_DIR/middlemanager"
+  OVERLORD_ENV="--env-file=$ENVIRONMENT_CONFIGS_DIR/overlord"
+  ROUTER_ENV="--env-file=$ENVIRONMENT_CONFIGS_DIR/router"
+  ROUTER_CUSTOM_CHECK_TLS_ENV="--env-file=$ENVIRONMENT_CONFIGS_DIR/router-custom-check-tls"
+  ROUTER_NO_CLIENT_AUTH_TLS_ENV="--env-file=$ENVIRONMENT_CONFIGS_DIR/router-no-client-auth-tls"
+  ROUTER_PERMISSIVE_TLS_ENV="--env-file=$ENVIRONMENT_CONFIGS_DIR/router-permissive-tls"
+
+  OVERRIDE_ENV=""
+  if [ -z "$DRUID_INTEGRATION_TEST_OVERRIDE_CONFIG_PATH" ]
+  then
+      echo "\$DRUID_INTEGRATION_TEST_OVERRIDE_CONFIG_PATH is not set. No override config file provided"
+  else
+      echo "\$DRUID_INTEGRATION_TEST_OVERRIDE_CONFIG_PATH is set with value ${DRUID_INTEGRATION_TEST_OVERRIDE_CONFIG_PATH}"
+      OVERRIDE_ENV="--env-file=$DRUID_INTEGRATION_TEST_OVERRIDE_CONFIG_PATH"
+  fi
+}
+
+# Create docker network
+{
+  docker network create --subnet=172.172.172.0/24 druid-it-net
+}
+
+# Build Druid Cluster Image
+if [ -z "$DRUID_INTEGRATION_TEST_JVM_RUNTIME" ]
+then
+      echo "\$DRUID_INTEGRATION_TEST_JVM_RUNTIME is not set. Running integration test with image running Java 8"
+      docker build -t druid/cluster --build-arg DOCKER_IMAGE=imply/druiditbase:openjdk-1.8.0_191-1 $SHARED_DIR/docker
+else
+      echo "\$DRUID_INTEGRATION_TEST_JVM_RUNTIME is set with value ${DRUID_INTEGRATION_TEST_JVM_RUNTIME}"
+      case "${DRUID_INTEGRATION_TEST_JVM_RUNTIME}" in
+      8)
+        echo "Running integration test with image running Java 8"
+        docker build -t druid/cluster --build-arg DOCKER_IMAGE=imply/druiditbase:openjdk-1.8.0_191-1 $SHARED_DIR/docker
+        ;;
+      11)
+        echo "Running integration test with image running Java 11"
+        docker build -t druid/cluster --build-arg DOCKER_IMAGE=imply/druiditbase:openjdk-11.0.5-1 $SHARED_DIR/docker
+        ;;
+      *)
+        echo "Invalid JVM Runtime given. Stopping"
+        exit 1
+        ;;
+      esac
+fi
+
+
+# Start docker containers for all Druid processes and dependencies
+{
+  # Start zookeeper and kafka
+  docker run -d --privileged --net druid-it-net --ip 172.172.172.2 ${COMMON_ENV} --name druid-zookeeper-kafka -p 2181:2181 -p 9092:9092 -p 9093:9093 -v $SHARED_DIR:/shared -v $SERVICE_SUPERVISORDS_DIR/zookeeper.conf:$SUPERVISORDIR/zookeeper.conf -v $SERVICE_SUPERVISORDS_DIR/kafka.conf:$SUPERVISORDIR/kafka.conf druid/cluster
+
+  # Start MYSQL
+  docker run -d --privileged --net druid-it-net --ip 172.172.172.3 ${COMMON_ENV} --name druid-metadata-storage -v $SHARED_DIR:/shared -v $SERVICE_SUPERVISORDS_DIR/metadata-storage.conf:$SUPERVISORDIR/metadata-storage.conf druid/cluster
+
+  # Start Overlord
+  docker run -d --privileged --net druid-it-net --ip 172.172.172.4 ${COMMON_ENV} ${OVERLORD_ENV} ${OVERRIDE_ENV} --name druid-overlord -p 8090:8090 -p 8290:8290 -v $SHARED_DIR:/shared -v $SERVICE_SUPERVISORDS_DIR/druid.conf:$SUPERVISORDIR/druid.conf --link druid-metadata-storage:druid-metadata-storage --link druid-zookeeper-kafka:druid-zookeeper-kafka druid/cluster
+
+  # Start Coordinator
+  docker run -d --privileged --net druid-it-net --ip 172.172.172.5 ${COMMON_ENV} ${COORDINATOR_ENV} ${OVERRIDE_ENV} --name druid-coordinator -p 8081:8081 -p 8281:8281 -v $SHARED_DIR:/shared -v $SERVICE_SUPERVISORDS_DIR/druid.conf:$SUPERVISORDIR/druid.conf --link druid-overlord:druid-overlord --link druid-metadata-storage:druid-metadata-storage --link druid-zookeeper-kafka:druid-zookeeper-kafka druid/cluster
+
+  # Start Historical
+  docker run -d --privileged --net druid-it-net --ip 172.172.172.6 ${COMMON_ENV} ${HISTORICAL_ENV} ${OVERRIDE_ENV} --name druid-historical -p 8083:8083 -p 8283:8283 -v $SHARED_DIR:/shared -v $SERVICE_SUPERVISORDS_DIR/druid.conf:$SUPERVISORDIR/druid.conf --link druid-zookeeper-kafka:druid-zookeeper-kafka druid/cluster
+
+  # Start Middlemanger
+  docker run -d --privileged --net druid-it-net --ip 172.172.172.7 ${COMMON_ENV} ${MIDDLEMANAGER_ENV} ${OVERRIDE_ENV} --name druid-middlemanager -p 8091:8091 -p 8291:8291 -p 8100:8100 -p 8101:8101 -p 8102:8102 -p 8103:8103 -p 8104:8104 -p 8105:8105 -p 8300:8300 -p 8301:8301 -p 8302:8302 -p 8303:8303 -p 8304:8304 -p 8305:8305 -v $RESOURCEDIR:/resources -v $SHARED_DIR:/shared -v $SERVICE_SUPERVISORDS_DIR/druid.conf:$SUPERVISORDIR/druid.conf --link druid-zookeeper-kafka:druid-zookeeper-kafka --link druid-overlord:druid-overlord druid/cluster
+
+  # Start Broker
+  docker run -d --privileged --net druid-it-net --ip 172.172.172.8 ${COMMON_ENV} ${BROKER_ENV} ${OVERRIDE_ENV} --name druid-broker -p 8082:8082 -p 8282:8282 -v $SHARED_DIR:/shared -v $SERVICE_SUPERVISORDS_DIR/druid.conf:$SUPERVISORDIR/druid.conf --link druid-zookeeper-kafka:druid-zookeeper-kafka --link druid-middlemanager:druid-middlemanager --link druid-historical:druid-historical druid/cluster
+
+  # Start Router
+  docker run -d --privileged --net druid-it-net --ip 172.172.172.9 ${COMMON_ENV} ${ROUTER_ENV} ${OVERRIDE_ENV} --name druid-router -p 8888:8888 -p 9088:9088 -v $SHARED_DIR:/shared -v $SERVICE_SUPERVISORDS_DIR/druid.conf:$SUPERVISORDIR/druid.conf --link druid-zookeeper-kafka:druid-zookeeper-kafka --link druid-coordinator:druid-coordinator --link druid-broker:druid-broker druid/cluster
+
+  # Start Router with permissive TLS settings (client auth enabled, no hostname verification, no revocation check)
+  docker run -d --privileged --net druid-it-net --ip 172.172.172.10 ${COMMON_ENV} ${ROUTER_PERMISSIVE_TLS_ENV} ${OVERRIDE_ENV} --name druid-router-permissive-tls -p 8889:8889 -p 9089:9089 -v $SHARED_DIR:/shared -v $SERVICE_SUPERVISORDS_DIR/druid.conf:$SUPERVISORDIR/druid.conf --link druid-zookeeper-kafka:druid-zookeeper-kafka --link druid-coordinator:druid-coordinator --link druid-broker:druid-broker druid/cluster
+
+  # Start Router with TLS but no client auth
+  docker run -d --privileged --net druid-it-net --ip 172.172.172.11 ${COMMON_ENV} ${ROUTER_NO_CLIENT_AUTH_TLS_ENV} ${OVERRIDE_ENV} --name druid-router-no-client-auth-tls -p 8890:8890 -p 9090:9090 -v $SHARED_DIR:/shared -v $SERVICE_SUPERVISORDS_DIR/druid.conf:$SUPERVISORDIR/druid.conf --link druid-zookeeper-kafka:druid-zookeeper-kafka --link druid-coordinator:druid-coordinator --link druid-broker:druid-broker druid/cluster
+
+  # Start Router with custom TLS cert checkers
+  docker run -d --privileged --net druid-it-net --ip 172.172.172.12 ${COMMON_ENV} ${ROUTER_CUSTOM_CHECK_TLS_ENV} ${OVERRIDE_ENV} --hostname druid-router-custom-check-tls --name druid-router-custom-check-tls -p 8891:8891 -p 9091:9091 -v $SHARED_DIR:/shared -v $SERVICE_SUPERVISORDS_DIR/druid.conf:$SUPERVISORDIR/druid.conf --link druid-zookeeper-kafka:druid-zookeeper-kafka --link druid-coordinator:druid-coordinator --link druid-broker:druid-broker druid/cluster
+}
+

integration-tests/run_cluster_using_java_runtime_11.sh

@@ -1,31 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Source the common script
-DIR="${BASH_SOURCE%/*}"
-if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
-. "$DIR/common_run_cluster.sh"
-
-cleanup
-
-setup
-
-create_docker_network
-
-# Build Druid Cluster Image (Image running Java 11)
-docker build -t druid/cluster --build-arg DOCKER_IMAGE=imply/druiditbase:openjdk-11.0.5-1 $SHARED_DIR/docker
-
-start_docker_containers

integration-tests/run_cluster_using_java_runtime_8.sh

@@ -1,31 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Source the common script
-DIR="${BASH_SOURCE%/*}"
-if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
-. "$DIR/common_run_cluster.sh"
-
-cleanup
-
-setup
-
-create_docker_network
-
-# Build Druid Cluster Image (Image running Java 8)
-docker build -t druid/cluster --build-arg DOCKER_IMAGE=imply/druiditbase:openjdk-1.8.0_191-1 $SHARED_DIR/docker
-
-start_docker_containers

pom.xml

@@ -1870,6 +1870,7 @@
                                 <exclude>codestyle/*-forbidden-apis.txt</exclude>
                                 <exclude>conf/**</exclude>
                                 <exclude>docker/*.conf</exclude>
+                                <exclude>docker/service-supervisords/*.conf</exclude>
                                 <exclude>target/**</exclude>
                                 <exclude>licenses/**</exclude>
                                 <exclude>**/test/resources/**</exclude>