Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Generate the IT docker-compose.yaml files (#13669) 

Generate IT docker-compose.sh files

Generates test-specific docker-compose.sh files using a simple
Python template script.
This commit is contained in:
Paul Rogers 2023-02-21 15:03:02 -08:00 committed by GitHub
parent c6b1576fc1
commit 5dadbdf4d0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
27 changed files with 1057 additions and 1015 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -6,6 +6,7 @@ target
*.tar.gz
*.swp
*.swo
*.pyc
.classpath
.idea
.project

5
docs/development/build.md
View File

@ -38,7 +38,8 @@ make sure it has `/master/` in the URL.
##### Other dependencies
- Distribution builds require Python 3.x and the `pyyaml` module
- Distribution builds require Python 3.x and the `pyyaml` module.
- Integration tests require `pyyaml` version 5.1 or later.
##### Downloading the source
@ -86,7 +87,7 @@ mvn clean install -Phadoop3
To generate distribution with hadoop3 dependencies, run :
```bash
mvn clean install -Papache-release,dist-hadoop3,rat,hadoop3 -DskipTests
mvn clean install -Papache-release,dist-hadoop3,rat,hadoop3 -DskipTests
```
#### Potential issues

330
integration-tests-ex/cases/cluster.sh
View File

@ -21,16 +21,21 @@
# Maps category names to cluster names. The mapping here must match
# that in the test category classes when @Cluster is used.
# Fail if any command fails
set -e
# Enable for debugging
#set -x
export MODULE_DIR=$(cd $(dirname $0) && pwd)
function usage {
cat <<EOF
cat <<EOF
Usage: $0 cmd [category]
-h, help
Display this message
prepare category
Generate the docker-compose.yaml file for the category for debugging.
up category
Start the cluster
down category
@ -38,130 +43,179 @@ Usage: $0 cmd [category]
status category
Status of the cluster (for debugging within build scripts)
compose-cmd category
Pass the command to Docker compose.
Pass the command to Docker compose. Cluster should already be up.
gen category
Generate docker-compose.yaml files (only.) Done automatically as
part of up. Use only for debugging.
EOF
}
# Command name is required
if [ $# -eq 0 ]; then
usage 1>&2
exit 1
usage 1>&2
exit 1
fi
CMD=$1
shift
# All commands need env vars
ENV_FILE=$MODULE_DIR/../image/target/env.sh
if [ ! -f $ENV_FILE ]; then
echo "Please build the Docker test image before testing" 1>&2
exit 1
fi
function check_env_file {
export ENV_FILE=$MODULE_DIR/../image/target/env.sh
if [ ! -f $ENV_FILE ]; then
echo "Please build the Docker test image before testing" 1>&2
exit 1
fi
source $ENV_FILE
source $ENV_FILE
}
function category {
if [ $# -eq 0 ]; then
usage 1>&2
exit 1
fi
export CATEGORY=$1
# The untranslated category is used for the local name of the
# shared folder.
if [ $# -eq 0 ]; then
usage 1>&2
exit 1
fi
export CATEGORY=$1
# The untranslated category is used for the local name of the
# shared folder.
# DRUID_INTEGRATION_TEST_GROUP is used in
# docker-compose files and here. Despite the name, it is the
# name of the cluster configuration we want to run, not the
# test category. Multiple categories can map to the same cluster
# definition.
# DRUID_INTEGRATION_TEST_GROUP is used in
# docker-compose files and here. Despite the name, it is the
# name of the cluster configuration we want to run, not the
# test category. Multiple categories can map to the same cluster
# definition.
# Map from category name to shared cluster definition name.
# Add an entry here if you create a new category that shares
# a definition.
case $CATEGORY in
"InputSource")
export DRUID_INTEGRATION_TEST_GROUP=BatchIndex
;;
"InputFormat")
export DRUID_INTEGRATION_TEST_GROUP=BatchIndex
;;
"Catalog")
export DRUID_INTEGRATION_TEST_GROUP=BatchIndex
;;
*)
export DRUID_INTEGRATION_TEST_GROUP=$CATEGORY
;;
esac
# Map from category name to shared cluster definition name.
# Add an entry here if you create a new category that shares
# a definition.
case $CATEGORY in
"InputSource")
export DRUID_INTEGRATION_TEST_GROUP=BatchIndex
;;
"InputFormat")
export DRUID_INTEGRATION_TEST_GROUP=BatchIndex
;;
"Catalog")
export DRUID_INTEGRATION_TEST_GROUP=BatchIndex
;;
*)
export DRUID_INTEGRATION_TEST_GROUP=$CATEGORY
;;
esac
export CLUSTER_DIR=$MODULE_DIR/cluster/$DRUID_INTEGRATION_TEST_GROUP
if [ ! -d $CLUSTER_DIR ]; then
echo "Cluster directory $CLUSTER_DIR does not exist." 1>&2
echo "$USAGE" 1>&2
exit 1
fi
export TARGET_DIR=$MODULE_DIR/target
export SHARED_DIR=$TARGET_DIR/$CATEGORY
export ENV_FILE="$TARGET_DIR/${CATEGORY}.env"
export CLUSTER_DIR=$MODULE_DIR/cluster/$DRUID_INTEGRATION_TEST_GROUP
export TARGET_DIR=$MODULE_DIR/target
export SHARED_DIR=$TARGET_DIR/$CATEGORY
export ENV_FILE="$TARGET_DIR/${CATEGORY}.env"
}
# Dump lots of information to debug Docker failures when run inside
# of a build environment where we can't inspect Docker directly.
function show_status {
echo "===================================="
ls -l target/shared
echo "docker ps -a"
docker ps -a
# Was: --filter status=exited
for id in $(docker ps -a --format "{{.ID}}"); do
echo "===================================="
echo "Logs for Container ID $id"
docker logs $id | tail -n 20
done
echo "===================================="
echo "===================================="
ls -l target/shared
echo "docker ps -a"
docker ps -a
# Was: --filter status=exited
for id in $(docker ps -a --format "{{.ID}}"); do
echo "===================================="
echo "Logs for Container ID $id"
docker logs $id | tail -n 20
done
echo "===================================="
}
function build_shared_dir {
mkdir -p $SHARED_DIR
# Must start with an empty DB to keep MySQL happy
rm -rf $SHARED_DIR/db
mkdir -p $SHARED_DIR/logs
mkdir -p $SHARED_DIR/tasklogs
mkdir -p $SHARED_DIR/db
mkdir -p $SHARED_DIR/kafka
mkdir -p $SHARED_DIR/resources
cp $MODULE_DIR/assets/log4j2.xml $SHARED_DIR/resources
# Permissions in some build setups are screwed up. See above. The user
# which runs Docker does not have permission to write into the /shared
# directory. Force ownership to allow writing.
chmod -R a+rwx $SHARED_DIR
mkdir -p $SHARED_DIR
# Must start with an empty DB to keep MySQL happy
rm -rf $SHARED_DIR/db
mkdir -p $SHARED_DIR/logs
mkdir -p $SHARED_DIR/tasklogs
mkdir -p $SHARED_DIR/db
mkdir -p $SHARED_DIR/kafka
mkdir -p $SHARED_DIR/resources
cp $MODULE_DIR/assets/log4j2.xml $SHARED_DIR/resources
# Permissions in some build setups are screwed up. See above. The user
# which runs Docker does not have permission to write into the /shared
# directory. Force ownership to allow writing.
chmod -R a+rwx $SHARED_DIR
}
# Each test must have a default docker-compose.yaml file which corresponds to using
# Either generate the docker-compose file, or use "static" versions.
function docker_file {
# If a template exists, generate the docker-compose.yaml file. Copy over the Common
# folder.
TEMPLATE_DIR=$MODULE_DIR/templates
TEMPLATE_SCRIPT=${DRUID_INTEGRATION_TEST_GROUP}.py
if [ -f "$TEMPLATE_DIR/$TEMPLATE_SCRIPT" ]; then
export COMPOSE_DIR=$TARGET_DIR/cluster/$DRUID_INTEGRATION_TEST_GROUP
mkdir -p $COMPOSE_DIR
pushd $TEMPLATE_DIR > /dev/null
python3 $TEMPLATE_SCRIPT
popd > /dev/null
cp -r $MODULE_DIR/cluster/Common $TARGET_DIR/cluster
else
# Else, use the existing non-template file in place.
if [ ! -d $CLUSTER_DIR ]; then
echo "Cluster directory $CLUSTER_DIR does not exist." 1>&2
echo "$USAGE" 1>&2
exit 1
fi
export COMPOSE_DIR=$CLUSTER_DIR
choose_static_file
fi
}
# Each test that uses static (non-generated) docker compose files
# must have a default docker-compose.yaml file which corresponds to using
# the MiddleManager (or no indexer). A test can optionally include a second file called
# docker-compose-indexer.yaml which uses the Indexer in place of Middle Manager.
function docker_file {
compose_args=""
if [ -n "$USE_INDEXER" ]; then
# Sanity check: USE_INDEXER must be "indexer" or "middleManager"
# if it is set at all.
if [ "$USE_INDEXER" != "indexer" ] && [ "$USE_INDEXER" != "middleManager" ]
then
echo "USE_INDEXER must be 'indexer' or 'middleManager' (is '$USE_INDEXER')" 1>&2
exit 1
fi
if [ "$USE_INDEXER" == "indexer" ]; then
compose_file=docker-compose-indexer.yaml
if [ ! -f "$CLUSTER_DIR/$compose_file" ]; then
echo "USE_INDEXER=$USE_INDEXER, but $CLUSTER_DIR/$compose_file is missing" 1>&2
exit 1
fi
compose_args="-f $compose_file"
fi
fi
echo $compose_args
function choose_static_file {
export DOCKER_ARGS=""
if [ -n "$USE_INDEXER" ]; then
# Sanity check: USE_INDEXER must be "indexer" or "middleManager"
# if it is set at all.
if [ "$USE_INDEXER" != "indexer" ] && [ "$USE_INDEXER" != "middleManager" ]
then
echo "USE_INDEXER must be 'indexer' or 'middleManager' (it is '$USE_INDEXER')" 1>&2
exit 1
fi
if [ "$USE_INDEXER" == "indexer" ]; then
compose_file=docker-compose-indexer.yaml
if [ ! -f "$CLUSTER_DIR/$compose_file" ]; then
echo "USE_INDEXER=$USE_INDEXER, but $CLUSTER_DIR/$compose_file is missing" 1>&2
exit 1
fi
export DOCKER_ARGS="-f $compose_file"
fi
fi
}
function verify_docker_file {
if [ -f "$CLUSTER_DIR/docker-compose.yaml" ]; then
# Use the existing non-template file in place.
export COMPOSE_DIR=$CLUSTER_DIR
return 0
fi
# The docker compose file must have been generated via up
export COMPOSE_DIR=$TARGET_DIR/cluster/$DRUID_INTEGRATION_TEST_GROUP
if [ ! -f "$COMPOSE_DIR/docker-compose.yaml" ]; then
echo "$COMPOSE_DIR/docker-compose.yaml is missing. Is cluster up? Did you do a 'clean' after 'up'?" 1>&2
fi
}
# Determine if docker-compose is available. If not, assume Docker supports
# the compose subcommand
set +e
if which docker-compose > /dev/null
then
DOCKER_COMPOSE='docker-compose'
else
DOCKER_COMPOSE='docker compose'
fi
set -e
# Print environment for debugging
#env
@ -177,37 +231,57 @@ fi
set -e
case $CMD in
"-h" )
usage
;;
"help" )
usage
$DOCKER_COMPOSE help
;;
"up" )
category $*
echo "Starting cluster $DRUID_INTEGRATION_TEST_GROUP"
build_shared_dir
cd $CLUSTER_DIR
$DOCKER_COMPOSE `docker_file` up -d
# Enable the following for debugging
#show_status
;;
"status" )
category $*
cd $CLUSTER_DIR
show_status
;;
"down" )
category $*
# Enable the following for debugging
#show_status
cd $CLUSTER_DIR
$DOCKER_COMPOSE `docker_file` $CMD
;;
"*" )
category $*
cd $CLUSTER_DIR
$DOCKER_COMPOSE `docker_file` $CMD
;;
"-h" )
usage
;;
"help" )
usage
$DOCKER_COMPOSE help
;;
"prepare" )
check_env_file
category $*
build_shared_dir
docker_file
;;
"gen" )
category $*
build_shared_dir
docker_file
echo "Generated file is in $COMPOSE_DIR"
;;
"up" )
check_env_file
category $*
echo "Starting cluster $DRUID_INTEGRATION_TEST_GROUP"
build_shared_dir
docker_file
cd $COMPOSE_DIR
$DOCKER_COMPOSE $DOCKER_ARGS up -d
# Enable the following for debugging
#show_status
;;
"status" )
check_env_file
category $*
docker_file
cd $COMPOSE_DIR
show_status
;;
"down" )
check_env_file
category $*
# Enable the following for debugging
#show_status
verify_docker_file
cd $COMPOSE_DIR
$DOCKER_COMPOSE $DOCKER_ARGS $CMD
;;
"*" )
check_env_file
category $*
verify_docker_file
cd $COMPOSE_DIR
$DOCKER_COMPOSE $DOCKER_ARGS $CMD
;;
esac

132
integration-tests-ex/cases/cluster/AzureDeepStorage/docker-compose.yaml
View File

@ -1,132 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# -------------------------------------------------------------------------
# Cluster for the Azure deep storage test.
#
# Required env vars:
#
# AZURE_ACCOUNT
# AZURE_KEY
# AZURE_CONTAINER
networks:
druid-it-net:
name: druid-it-net
ipam:
config:
- subnet: 172.172.172.0/24
services:
zookeeper:
extends:
file: ../Common/dependencies.yaml
service: zookeeper
metadata:
extends:
file: ../Common/dependencies.yaml
service: metadata
coordinator:
extends:
file: ../Common/druid.yaml
service: coordinator
container_name: coordinator
environment:
- DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP}
- druid_test_loadList=druid-azure-extensions
- druid_azure_account=${AZURE_ACCOUNT}
- druid_azure_key=${AZURE_KEY}
- druid_azure_container=${AZURE_CONTAINER}
# The frequency with which the coordinator polls the database
# for changes. The DB population code has to wait at least this
# long for the coordinator to notice changes.
- druid_manager_segments_pollDuration=PT5S
- druid_coordinator_period=PT10S
depends_on:
- zookeeper
- metadata
overlord:
extends:
file: ../Common/druid.yaml
service: overlord
container_name: overlord
environment:
- DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP}
- druid_test_loadList=druid-azure-extensions
- druid_azure_account=${AZURE_ACCOUNT}
- druid_azure_key=${AZURE_KEY}
- druid_azure_container=${AZURE_CONTAINER}
depends_on:
- zookeeper
- metadata
broker:
extends:
file: ../Common/druid.yaml
service: broker
environment:
- DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP}
- druid_test_loadList=druid-azure-extensions
- druid_azure_account=${AZURE_ACCOUNT}
- druid_azure_key=${AZURE_KEY}
- druid_azure_container=${AZURE_CONTAINER}
depends_on:
- zookeeper
router:
extends:
file: ../Common/druid.yaml
service: router
environment:
- DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP}
- druid_test_loadList=druid-azure-extensions
- druid_azure_account=${AZURE_ACCOUNT}
- druid_azure_key=${AZURE_KEY}
- druid_azure_container=${AZURE_CONTAINER}
depends_on:
- zookeeper
historical:
extends:
file: ../Common/druid.yaml
service: historical
environment:
- DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP}
- druid_test_loadList=druid-azure-extensions
- druid_azure_account=${AZURE_ACCOUNT}
- druid_azure_key=${AZURE_KEY}
- druid_azure_container=${AZURE_CONTAINER}
depends_on:
- zookeeper
indexer:
extends:
file: ../Common/druid.yaml
service: indexer
environment:
- DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP}
- druid_test_loadList=druid-azure-extensions
- druid_storage_type=azure
- druid_azure_account=${AZURE_ACCOUNT}
- druid_azure_key=${AZURE_KEY}
- druid_azure_container=${AZURE_CONTAINER}
volumes:
# Test data
- ../data:/resources
depends_on:
- zookeeper

98
integration-tests-ex/cases/cluster/BatchIndex/docker-compose-indexer.yaml
View File

@ -1,98 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
networks:
druid-it-net:
name: druid-it-net
ipam:
config:
- subnet: 172.172.172.0/24
services:
zookeeper:
extends:
file: ../Common/dependencies.yaml
service: zookeeper
metadata:
extends:
file: ../Common/dependencies.yaml
service: metadata
coordinator:
extends:
file: ../Common/druid.yaml
service: coordinator
container_name: coordinator
environment:
- DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP}
# The frequency with which the coordinator polls the database
# for changes. The DB population code has to wait at least this
# long for the coordinator to notice changes.
- druid_manager_segments_pollDuration=PT5S
- druid_coordinator_period=PT10S
depends_on:
- zookeeper
- metadata
overlord:
extends:
file: ../Common/druid.yaml
service: overlord
container_name: overlord
environment:
- DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP}
depends_on:
- zookeeper
- metadata
broker:
extends:
file: ../Common/druid.yaml
service: broker
environment:
- DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP}
depends_on:
- zookeeper
router:
extends:
file: ../Common/druid.yaml
service: router
environment:
- DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP}
depends_on:
- zookeeper
historical:
extends:
file: ../Common/druid.yaml
service: historical
environment:
- DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP}
depends_on:
- zookeeper
indexer:
extends:
file: ../Common/druid.yaml
service: indexer
environment:
- DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP}
volumes:
# Test data
- ../../resources:/resources
depends_on:
- zookeeper

98
integration-tests-ex/cases/cluster/BatchIndex/docker-compose.yaml
View File

@ -1,98 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
networks:
druid-it-net:
name: druid-it-net
ipam:
config:
- subnet: 172.172.172.0/24
services:
zookeeper:
extends:
file: ../Common/dependencies.yaml
service: zookeeper
metadata:
extends:
file: ../Common/dependencies.yaml
service: metadata
coordinator:
extends:
file: ../Common/druid.yaml
service: coordinator
container_name: coordinator
environment:
- DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP}
# The frequency with which the coordinator polls the database
# for changes. The DB population code has to wait at least this
# long for the coordinator to notice changes.
- druid_manager_segments_pollDuration=PT5S
- druid_coordinator_period=PT10S
depends_on:
- zookeeper
- metadata
overlord:
extends:
file: ../Common/druid.yaml
service: overlord
container_name: overlord
environment:
- DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP}
depends_on:
- zookeeper
- metadata
broker:
extends:
file: ../Common/druid.yaml
service: broker
environment:
- DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP}
depends_on:
- zookeeper
router:
extends:
file: ../Common/druid.yaml
service: router
environment:
- DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP}
depends_on:
- zookeeper
historical:
extends:
file: ../Common/druid.yaml
service: historical
environment:
- DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP}
depends_on:
- zookeeper
middlemanager:
extends:
file: ../Common/druid.yaml
service: middlemanager
environment:
- DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP}
volumes:
# Test data
- ../../resources:/resources
depends_on:
- zookeeper

14
integration-tests-ex/cases/cluster/Common/druid.yaml
View File

@ -61,6 +61,8 @@ services:
- environment-configs/common.env
- environment-configs/overlord.env
- ${OVERRIDE_ENV}
environment:
- DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP}
coordinator:
image: ${DRUID_IT_IMAGE_NAME}
@ -78,6 +80,8 @@ services:
- environment-configs/common.env
- environment-configs/coordinator.env
- ${OVERRIDE_ENV}
environment:
- DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP}
historical:
image: ${DRUID_IT_IMAGE_NAME}
@ -95,6 +99,8 @@ services:
- environment-configs/common.env
- environment-configs/historical.env
- ${OVERRIDE_ENV}
environment:
- DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP}
middlemanager:
image: ${DRUID_IT_IMAGE_NAME}
@ -124,6 +130,8 @@ services:
- environment-configs/common.env
- environment-configs/middlemanager.env
- ${OVERRIDE_ENV}
environment:
- DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP}
indexer:
image: ${DRUID_IT_IMAGE_NAME}
@ -141,6 +149,8 @@ services:
- environment-configs/common.env
- environment-configs/indexer.env
- ${OVERRIDE_ENV}
environment:
- DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP}
broker:
image: ${DRUID_IT_IMAGE_NAME}
@ -158,6 +168,8 @@ services:
- environment-configs/common.env
- environment-configs/broker.env
- ${OVERRIDE_ENV}
environment:
- DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP}
router:
image: ${DRUID_IT_IMAGE_NAME}
@ -175,3 +187,5 @@ services:
- environment-configs/common.env
- environment-configs/router.env
- ${OVERRIDE_ENV}
environment:
- DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP}

6
integration-tests-ex/cases/cluster/Common/environment-configs/coordinator.env
View File

@ -37,3 +37,9 @@ druid_coordinator_period_indexingPeriod=PT180000S
# 2x indexing period so that kill period is valid
druid_coordinator_kill_period=PT360000S
druid_coordinator_period=PT1S
# The frequency with which the coordinator polls the database
# for changes. The DB population code has to wait at least this
# long for the coordinator to notice changes.
druid_manager_segments_pollDuration=PT5S
druid_coordinator_period=PT10S

155
integration-tests-ex/cases/cluster/GcsDeepStorage/docker-compose.yaml
View File

@ -1,155 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# -------------------------------------------------------------------------
# Cluster for the Google Cluster Storage (GCS) deep storage test.
#
# Required env vars:
#
# GOOGLE_BUCKET
# GOOGLE_PREFIX
# GOOGLE_APPLICATION_CREDENTIALS - must point to a file that holds the Google
# credentials. Mounted into each Druid container.
networks:
druid-it-net:
name: druid-it-net
ipam:
config:
- subnet: 172.172.172.0/24
services:
zookeeper:
extends:
file: ../Common/dependencies.yaml
service: zookeeper
metadata:
extends:
file: ../Common/dependencies.yaml
service: metadata
coordinator:
extends:
file: ../Common/druid.yaml
service: coordinator
container_name: coordinator
environment:
- DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP}
- druid_test_loadList=druid-google-extensions
- druid_storage_type=google
- druid_google_bucket=${GOOGLE_BUCKET}
- druid_google_prefix=${GOOGLE_PREFIX}
- GOOGLE_APPLICATION_CREDENTIALS=/resources/credentials.json
# The frequency with which the coordinator polls the database
# for changes. The DB population code has to wait at least this
# long for the coordinator to notice changes.
- druid_manager_segments_pollDuration=PT5S
- druid_coordinator_period=PT10S
volumes:
# Mount credentials file
- ${GOOGLE_APPLICATION_CREDENTIALS}:/resources/credentials.json
depends_on:
- zookeeper
- metadata
overlord:
extends:
file: ../Common/druid.yaml
service: overlord
container_name: overlord
environment:
- DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP}
- druid_test_loadList=druid-google-extensions
- druid_storage_type=google
- druid_google_bucket=${GOOGLE_BUCKET}
- druid_google_prefix=${GOOGLE_PREFIX}
- GOOGLE_APPLICATION_CREDENTIALS=/resources/credentials.json
volumes:
# Mount credentials file
- ${GOOGLE_APPLICATION_CREDENTIALS}:/resources/credentials.json
depends_on:
- zookeeper
- metadata
broker:
extends:
file: ../Common/druid.yaml
service: broker
environment:
- DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP}
- druid_test_loadList=druid-google-extensions
- druid_storage_type=google
- druid_google_bucket=${GOOGLE_BUCKET}
- druid_google_prefix=${GOOGLE_PREFIX}
- GOOGLE_APPLICATION_CREDENTIALS=/resources/credentials.json
volumes:
# Mount credentials file
- ${GOOGLE_APPLICATION_CREDENTIALS}:/resources/credentials.json
depends_on:
- zookeeper
router:
extends:
file: ../Common/druid.yaml
service: router
environment:
- DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP}
- druid_test_loadList=druid-google-extensions
- druid_storage_type=google
- druid_google_bucket=${GOOGLE_BUCKET}
- druid_google_prefix=${GOOGLE_PREFIX}
- GOOGLE_APPLICATION_CREDENTIALS=/resources/credentials.json
volumes:
# Mount credentials file
- ${GOOGLE_APPLICATION_CREDENTIALS}:/resources/credentials.json
depends_on:
- zookeeper
historical:
extends:
file: ../Common/druid.yaml
service: historical
environment:
- DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP}
- druid_test_loadList=druid-google-extensions
- druid_storage_type=google
- druid_google_bucket=${GOOGLE_BUCKET}
- druid_google_prefix=${GOOGLE_PREFIX}
- GOOGLE_APPLICATION_CREDENTIALS=/resources/credentials.json
volumes:
# Mount credentials file
- ${GOOGLE_APPLICATION_CREDENTIALS}:/resources/credentials.json
depends_on:
- zookeeper
indexer:
extends:
file: ../Common/druid.yaml
service: indexer
environment:
- DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP}
- druid_test_loadList=druid-google-extensions
- druid_storage_type=google
- druid_google_bucket=${GOOGLE_BUCKET}
- druid_google_prefix=${GOOGLE_PREFIX}
- GOOGLE_APPLICATION_CREDENTIALS=/resources/credentials.json
volumes:
# Mount credentials file
- ${GOOGLE_APPLICATION_CREDENTIALS}:/resources/credentials.json
# Test data
- ../data:/resources
depends_on:
- zookeeper

157
integration-tests-ex/cases/cluster/HighAvailability/docker-compose.yaml
View File

@ -1,157 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
networks:
druid-it-net:
name: druid-it-net
ipam:
config:
- subnet: 172.172.172.0/24
services:
zookeeper:
extends:
file: ../Common/dependencies.yaml
service: zookeeper
metadata:
extends:
file: ../Common/dependencies.yaml
service: metadata
coordinator-one:
extends:
file: ../Common/druid.yaml
service: coordinator
container_name: coordinator-one
environment:
- DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP}
- DRUID_INSTANCE=one
# The frequency with which the coordinator polls the database
# for changes. The DB population code has to wait at least this
# long for the coordinator to notice changes.
- druid_manager_segments_pollDuration=PT5S
- druid_coordinator_period=PT10S
- druid_host=coordinator-one
depends_on:
- zookeeper
- metadata
# The second Coordinator (and Overlord) cannot extend
# The base service: they need distinct ports.
coordinator-two:
image: ${DRUID_IT_IMAGE_NAME}
container_name: coordinator-two
networks:
druid-it-net:
ipv4_address: 172.172.172.120
ports:
- 18081:8081
- 18281:8281
- 15006:8000
volumes:
- ${SHARED_DIR}:/shared
env_file:
- ../Common/environment-configs/common.env
- ../Common/environment-configs/coordinator.env
environment:
- DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP}
- DRUID_INSTANCE=two
- druid_manager_segments_pollDuration=PT5S
- druid_coordinator_period=PT10S
- druid_host=coordinator-two
depends_on:
- zookeeper
- metadata
overlord-one:
extends:
file: ../Common/druid.yaml
service: overlord
container_name: overlord-one
environment:
- DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP}
- DRUID_INSTANCE=one
- druid_host=overlord-one
depends_on:
- zookeeper
- metadata
overlord-two:
image: ${DRUID_IT_IMAGE_NAME}
container_name: overlord-two
networks:
druid-it-net:
ipv4_address: 172.172.172.110
ports:
- 18090:8090
- 18290:8290
- 15009:8000
volumes:
- ${SHARED_DIR}:/shared
env_file:
- ../Common/environment-configs/common.env
- ../Common/environment-configs/overlord.env
environment:
- DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP}
- DRUID_INSTANCE=two
- druid_host=overlord-two
depends_on:
- zookeeper
- metadata
broker:
extends:
file: ../Common/druid.yaml
service: broker
environment:
- DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP}
depends_on:
- zookeeper
router:
extends:
file: ../Common/druid.yaml
service: router
environment:
- DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP}
depends_on:
- zookeeper
# The custom node role has no base definition. Also, there is
# no environment file: the needed environment settings are
# given here.
custom-node-role:
image: ${DRUID_IT_IMAGE_NAME}
container_name: custom-node-role
networks:
druid-it-net:
ipv4_address: 172.172.172.90
ports:
- 50011:50011
- 9301:9301
- 9501:9501
- 5010:8000
volumes:
- ${SHARED_DIR}:/shared
env_file:
- ../Common/environment-configs/common.env
environment:
- DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP}
- DRUID_SERVICE=custom-node-role
- SERVICE_DRUID_JAVA_OPTS=-Xmx64m -Xms64m
- druid_host=custom-node-role
- druid_auth_basic_common_cacheDirectory=/tmp/authCache/custom_node_role
- druid_server_https_crlPath=/tls/revocations.crl

98
integration-tests-ex/cases/cluster/MultiStageQuery/docker-compose.yaml
View File

@ -1,98 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
networks:
druid-it-net:
name: druid-it-net
ipam:
config:
- subnet: 172.172.172.0/24
services:
zookeeper:
extends:
file: ../Common/dependencies.yaml
service: zookeeper
metadata:
extends:
file: ../Common/dependencies.yaml
service: metadata
coordinator:
extends:
file: ../Common/druid.yaml
service: coordinator
container_name: coordinator
environment:
- DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP}
- druid_manager_segments_pollDuration=PT5S
- druid_coordinator_period=PT10S
depends_on:
- zookeeper
- metadata
overlord:
extends:
file: ../Common/druid.yaml
service: overlord
container_name: overlord
environment:
- DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP}
depends_on:
- zookeeper
- metadata
broker:
extends:
file: ../Common/druid.yaml
service: broker
environment:
- DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP}
depends_on:
- zookeeper
router:
extends:
file: ../Common/druid.yaml
service: router
environment:
- DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP}
depends_on:
- zookeeper
historical:
extends:
file: ../Common/druid.yaml
service: historical
environment:
- DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP}
depends_on:
- zookeeper
indexer:
extends:
file: ../Common/druid.yaml
service: indexer
environment:
- DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP}
- druid_msq_intermediate_storage_enable=true
- druid_msq_intermediate_storage_type=local
- druid_msq_intermediate_storage_basePath=/shared/durablestorage/
volumes:
# Test data
- ../../resources:/resources
depends_on:
- zookeeper

129
integration-tests-ex/cases/cluster/S3DeepStorage/docker-compose.yaml
View File

@ -1,129 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# -------------------------------------------------------------------------
# Cluster for the S3 deep storage test.
#
# Required env vars:
#
# AWS_REGION
# AWS_ACCESS_KEY_ID
# AWS_SECRET_ACCESS_KEY
networks:
druid-it-net:
name: druid-it-net
ipam:
config:
- subnet: 172.172.172.0/24
services:
zookeeper:
extends:
file: ../Common/dependencies.yaml
service: zookeeper
metadata:
extends:
file: ../Common/dependencies.yaml
service: metadata
coordinator:
extends:
file: ../Common/druid.yaml
service: coordinator
container_name: coordinator
environment:
- DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP}
# The frequency with which the coordinator polls the database
# for changes. The DB population code has to wait at least this
# long for the coordinator to notice changes.
- druid_manager_segments_pollDuration=PT5S
- druid_coordinator_period=PT10S
- AWS_REGION=${AWS_REGION}
- druid_s3_accessKey=${AWS_ACCESS_KEY_ID}
- druid_s3_secretKey=${AWS_SECRET_ACCESS_KEY}
depends_on:
- zookeeper
- metadata
overlord:
extends:
file: ../Common/druid.yaml
service: overlord
container_name: overlord
environment:
- DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP}
- AWS_REGION=${AWS_REGION}
- druid_s3_accessKey=${AWS_ACCESS_KEY_ID}
- druid_s3_secretKey=${AWS_SECRET_ACCESS_KEY}
depends_on:
- zookeeper
- metadata
broker:
extends:
file: ../Common/druid.yaml
service: broker
environment:
- DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP}
- AWS_REGION=${AWS_REGION}
- druid_s3_accessKey=${AWS_ACCESS_KEY_ID}
- druid_s3_secretKey=${AWS_SECRET_ACCESS_KEY}
depends_on:
- zookeeper
router:
extends:
file: ../Common/druid.yaml
service: router
environment:
- DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP}
- AWS_REGION=${AWS_REGION}
- druid_s3_accessKey=${AWS_ACCESS_KEY_ID}
- druid_s3_secretKey=${AWS_SECRET_ACCESS_KEY}
depends_on:
- zookeeper
historical:
extends:
file: ../Common/druid.yaml
service: historical
environment:
- DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP}
- AWS_REGION=${AWS_REGION}
- druid_s3_accessKey=${AWS_ACCESS_KEY_ID}
- druid_s3_secretKey=${AWS_SECRET_ACCESS_KEY}
depends_on:
- zookeeper
indexer:
extends:
file: ../Common/druid.yaml
service: indexer
environment:
- DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP}
- druid_storage_type=s3
- druid_storage_bucket=${DRUID_CLOUD_BUCKET}
# Using DRUID_CLOUD_PATH env as baseKey as well.
- druid_storage_baseKey=${DRUID_CLOUD_PATH}
- druid_s3_accessKey=${AWS_ACCESS_KEY_ID}
- druid_s3_secretKey=${AWS_SECRET_ACCESS_KEY}
- AWS_REGION=${AWS_REGION}
volumes:
# Test data
- ../data:/resources
depends_on:
- zookeeper

5
integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/cluster/DruidClusterClient.java
View File

@ -32,6 +32,7 @@ import org.apache.druid.java.util.http.client.response.StatusResponseHandler;
import org.apache.druid.java.util.http.client.response.StatusResponseHolder;
import org.apache.druid.server.DruidNode;
import org.apache.druid.testing.guice.TestClient;
import org.apache.druid.testsEx.config.ClusterConfig;
import org.apache.druid.testsEx.config.ResolvedConfig;
import org.apache.druid.testsEx.config.ResolvedDruidService;
import org.apache.druid.testsEx.config.ResolvedService.ResolvedInstance;
@ -335,8 +336,8 @@ public class DruidClusterClient
*/
public void validate()
{
RE exception = new RE("Just building for the stack trace");
log.info(exception, "Starting cluster validation");
log.info("Starting cluster validation");
log.info("This cluster uses " + (ClusterConfig.isIndexer() ? "Indexer" : "Middle Manager"));
for (ResolvedDruidService service : config.requireDruid().values()) {
for (ResolvedInstance instance : service.requireInstances()) {
validateInstance(service, instance);

5
integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/ClusterConfig.java
View File

@ -170,6 +170,11 @@ public class ClusterConfig
return new ResolvedConfig(clusterName, resolveIncludes(), configTags);
}
public static boolean isIndexer()
{
return "indexer".equals(System.getenv("USE_INDEXER"));
}
/**
* Create the set of configuration tags for this run. At present, the only options
* are "middleManager" or "indexer" corresponding to the value of the

43
integration-tests-ex/cases/templates/AzureDeepStorage.py Normal file
View File

@ -0,0 +1,43 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from template import BaseTemplate, generate
class Template(BaseTemplate):
def gen_header_comment(self):
self.emit('''
# Cluster for the Azure deep storage test.
#
# Required env vars:
#
# AZURE_ACCOUNT
# AZURE_KEY
# AZURE_CONTAINER
''')
def extend_druid_service(self, service):
self.add_env(service, 'druid_test_loadList', 'druid-azure-extensions')
self.add_property(service, 'druid.storage.type', 'azure')
self.add_property(service, 'druid.azure.account', '${AZURE_ACCOUNT}')
self.add_property(service, 'druid.azure.key', '${AZURE_KEY}')
self.add_property(service, 'druid.azure.container', '${AZURE_CONTAINER}')
# This test uses different data than the default.
def define_data_dir(self, service):
self.add_volume(service, '../data', '/resources')
generate(__file__, Template())

18
integration-tests-ex/cases/templates/BatchIndex.py Normal file
View File

@ -0,0 +1,18 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from template import BaseTemplate, generate
generate(__file__, BaseTemplate())

45
integration-tests-ex/cases/templates/GcsDeepStorage.py Normal file
View File

@ -0,0 +1,45 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from template import BaseTemplate, generate
class Template(BaseTemplate):
def gen_header_comment(self):
self.emit('''
# Cluster for the Google Cluster Storage (GCS) deep storage test.
#
# Required env vars:
#
# GOOGLE_BUCKET
# GOOGLE_PREFIX
# GOOGLE_APPLICATION_CREDENTIALS - must point to a file that holds the Google
# credentials. Mounted into each Druid container.
''')
def extend_druid_service(self, service):
self.add_env(service, 'druid_test_loadList', 'druid-google-extensions')
self.add_property(service, 'druid.storage.type', 'google')
self.add_property(service, 'druid.google.bucket', '${GOOGLE_BUCKET}')
self.add_property(service, 'druid.google.prefix', '${GOOGLE_PREFIX}')
self.add_env(service, 'GOOGLE_APPLICATION_CREDENTIALS', '/resources/credentials.json')
self.add_volume(service, '${GOOGLE_APPLICATION_CREDENTIALS}', '/resources/credentials.json')
# This test uses different data than the default.
def define_data_dir(self, service):
self.add_volume(service, '../data', '/resources')
generate(__file__, Template())

85
integration-tests-ex/cases/templates/HighAvailability.py Normal file
View File

@ -0,0 +1,85 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from template import BaseTemplate, generate
from template import COORDINATOR, ZOO_KEEPER, METADATA, OVERLORD
# The second Coordinator (and Overlord) cannot extend
# The base service: they need distinct ports.
class Template(BaseTemplate):
def define_coordinator(self):
self.define_coordinator_one()
self.define_coordinator_two()
def define_coordinator_one(self):
service_name = COORDINATOR + '-one'
service = self.define_master_service(service_name, COORDINATOR)
service['container_name'] = service_name
self.add_env(service, 'DRUID_INSTANCE', 'one')
self.add_env(service, 'druid_host', service_name)
service['container_name'] = service_name
def define_coordinator_two(self):
service_name = COORDINATOR + '-two'
service = self.define_full_service(service_name, COORDINATOR, 120)
service['container_name'] = service_name
self.add_env(service, 'DRUID_INSTANCE', 'two')
self.add_env(service, 'druid_host', service_name)
service['ports'] = [ '18081:8081', '18281:8281', '15006:8000' ]
self.add_depends(service, [ ZOO_KEEPER, METADATA ] )
def define_overlord(self):
self.define_overlord_one()
self.define_overlord_two()
def define_overlord_one(self):
service_name = OVERLORD + '-one'
service = self.define_master_service(service_name, OVERLORD)
service['container_name'] = service_name
self.add_env(service, 'DRUID_INSTANCE', 'one')
self.add_env(service, 'druid_host', service_name)
def define_overlord_two(self):
service_name = OVERLORD + '-two'
service = self.define_full_service(service_name, OVERLORD, 110)
service['container_name'] = service_name
self.add_env(service, 'DRUID_INSTANCE', 'two')
self.add_env(service, 'druid_host', service_name)
service['ports'] = [ '18090:8090', '18290:8290', '15009:8000' ]
self.add_depends(service, [ ZOO_KEEPER, METADATA ] )
# No indexer in this cluster
def define_indexer(self):
pass
# No historical in this cluster
def define_historical(self):
pass
# The custom node role has no base definition. Also, there is
# no environment file: the needed environment settings are
# given here.
def define_custom_services(self):
service_name = 'custom-node-role'
service = self.define_full_service(service_name, None, 90)
service['container_name'] = service_name
self.add_env(service, 'DRUID_SERVICE', service_name)
self.add_env(service, 'SERVICE_DRUID_JAVA_OPTS', '-Xmx64m -Xms64m')
self.add_env(service, 'druid_host', service_name)
service['ports'] = [ '50011:50011', '9301:9301', '9501:9501', '5010:8000' ]
self.add_depends(service, [ ZOO_KEEPER ] )
generate(__file__, Template())

26
integration-tests-ex/cases/templates/MultiStageQuery.py Normal file
View File

@ -0,0 +1,26 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from template import BaseTemplate, generate
class Template(BaseTemplate):
def define_indexer(self):
service = super().define_indexer()
self.add_property(service, 'druid.msq.intermediate.storage.enable', 'true')
self.add_property(service, 'druid.msq.intermediate.storage.type', 'local')
self.add_property(service, 'druid.msq.intermediate.storage.basePath', '/shared/durablestorage/')
generate(__file__, Template())

46
integration-tests-ex/cases/templates/S3DeepStorage.py Normal file
View File

@ -0,0 +1,46 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from template import BaseTemplate, generate
class Template(BaseTemplate):
def gen_header_comment(self):
self.emit('''
# Cluster for the S3 deep storage test.
#
# Required env vars:
#
# DRUID_CLOUD_BUCKET
# DRUID_CLOUD_PATH
# AWS_REGION
# AWS_ACCESS_KEY_ID
# AWS_SECRET_ACCESS_KEY
''')
def extend_druid_service(self, service):
self.add_property(service, 'druid.storage.type', 's3')
self.add_property(service, 'druid.s3.accessKey', '${AWS_ACCESS_KEY_ID}')
self.add_property(service, 'druid.s3.secretKey', '${AWS_SECRET_ACCESS_KEY}')
self.add_property(service, 'druid.storage.bucket', '${DRUID_CLOUD_BUCKET}')
self.add_property(service, 'druid.storage.baseKey', '${DRUID_CLOUD_PATH}')
self.add_env(service, 'AWS_REGION', '${AWS_REGION}')
# This test uses different data than the default.
def define_data_dir(self, service):
self.add_volume(service, '../data', '/resources')
generate(__file__, Template())

430
integration-tests-ex/cases/templates/template.py Normal file
View File

@ -0,0 +1,430 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
'''
Generates a docker-compose.yaml file from a test-specific template. Each
test template either uses the base template directly, or extends the template
to customize bits of the cluster.
Since the cluster is defined as YAML, the cluster definition is build up
internally as a Python data structure made up of maps, arrays and scalars.
PyYaml does the grunt work of converting the data structure to the YAML file.
'''
import yaml, os, os.path
from pathlib import Path
# Constants used frequently in the template.
DRUID_NETWORK = 'druid-it-net'
DRUID_SUBNET = '172.172.172'
ZOO_KEEPER = 'zookeeper'
METADATA = 'metadata'
COORDINATOR = 'coordinator'
OVERLORD = 'overlord'
ROUTER = 'router'
BROKER = 'broker'
HISTORICAL = 'historical'
INDEXER = 'indexer'
MIDDLE_MANAGER = 'middlemanager'
def generate(template_path, template):
'''
Main routine to generate a docker-compose file from a script with the
given template_path, using the template class given. The template path is
a convenient way to locate directories in the file system using information
that Python itself provides.
'''
# Compute the cluster (test category) name from the template path which
# we assume to be module/<something>/<template>/<something>.py
template_path = Path(template_path)
cluster = template_path.stem
# Move up to the module (that is, the cases folder) relative to the template file.
module_dir = Path(__file__).parent.parent
# The target location for the output file is <module>/target/cluster/<cluster>/docker-compose.yaml
target_dir = module_dir.joinpath("target")
target_file = target_dir.joinpath('cluster', cluster, 'docker-compose.yaml')
# Defer back to the template class to create the output into the docker-compose.yaml file.
with target_file.open("w") as f:
template.generate_file(f, cluster)
f.close()
class BaseTemplate:
def __init__(self):
# Cluster is the object tree for the docker-compose.yaml file for our test cluster.
# The tree is a map of objects, each of which is a map of values. The values are
# typicaly scalars, maps or arrays. These are represented generically in Python.
self.cluster = {}
def generate_file(self, out_file, cluster):
'''
Generates the docker-compose.yaml file contents as a header plus a YAML-serialized
form of the cluster object tree. The file is meant to be generated, used and discarded.
As a result, we don't worry about generating line-by-line comments: those should appear
in the template.
'''
self.cluster_name = cluster
self.define_cluster()
self.out_file = out_file
self.generate()
def define_cluster(self):
'''
Overall method to define the test cluster.
'''
self.define_network()
self.define_support_services()
self.define_druid_services()
self.define_custom_services()
def define_support_services(self):
'''
Define support services which run as containers, but are not provided by
Druid.
'''
self.define_zk()
self.define_metadata()
def define_druid_services(self):
'''
Define the set of Druid services. Override this method to provide ad-hoc
services unique to a test. If the test creates multiple versions of a
service, provide that by overriding the individual service method.
'''
self.define_coordinator()
self.define_overlord()
self.define_broker()
self.define_router()
self.define_historical()
self.define_indexer()
def define_custom_services(self):
'''
Override to define additional services for the cluster.
'''
pass
def generate(self):
'''
Emit output to the target file.
'''
self.gen_header()
self.gen_header_comment()
self.gen_body()
def emit(self, text):
'''
Emits text to the target file. Used for header comments.
'''
# Chop off the newline that occurs when ''' is on a separate line
if len(text) > 0 and text[0] == '\n':
text = text[1:]
self.out_file.write(text)
def gen_header(self):
'''
Emit the standard file header.
'''
self.emit('''
# THIS FILE IS GENERATED -- DO NOT EDIT!
#
# Instead, edit the template from which this file was generated.
# Template: templates/{}.py
'''.format(self.cluster_name))
def gen_header_comment(self):
'''
Override to generate a custom header comment after the standard header.
'''
pass
def gen_body(self):
'''
Convert the cluster tree into YAML using the pyaml library.
'''
try:
# Version 5.1 or later: sort the keys in the order we created them.
# This makes doing diffs easier when making changes.
yaml.dump(self.cluster, self.out_file, sort_keys=False)
except TypeError:
# For builds that use pyyaml older than 5.1. Keys will be emitted
# in random order.
yaml.dump(self.cluster, self.out_file)
def define_network(self):
self.cluster['networks'] = {
'druid-it-net': {
'name': DRUID_NETWORK,
'ipam': {
'config': [
{'subnet': DRUID_SUBNET + '.0/24'}
]
}
}
}
def add_service(self, name, service):
'''
Add a service to the 'services' key in the cluster tree.
'''
services = self.cluster.setdefault('services', {})
services[name] = service
def add_volume(self, service, local, container):
'''
Adds a volume to a service.
'''
volumes = service.setdefault('volumes', [])
volumes.append(local + ':' + container)
def add_env(self, service, var, value):
'''
Adds an environment variable to a service.
'''
vars = service.setdefault('environment', [])
vars.append(var + '=' + value)
def add_property(self, service, prop, value):
'''
Sets a property for a service. The property is of the same form as the
.properties file: druid.some.property.
This method converts the property to the env var form so you don't have to.
'''
var = prop.replace('.', '_')
self.add_env(service, var, value)
def add_env_file(self, service, env_file):
'''
Add an environment file to a service.
'''
env_files = service.setdefault('env_file', [])
env_files.append(env_file)
def add_env_config(self, service, base_name):
'''
Add to a service one of the standard environment config files in
the Common/environment-configs directory
'''
self.add_env_file(service, '../Common/environment-configs/' + base_name + '.env')
def add_port(self, service, local, container):
'''
Add a port mapping to the service
'''
ports = service.setdefault('ports', [])
ports.append(local + ':' + container)
def define_external_service(self, name) -> dict:
'''
Defines a support service external to Druid as a reference to a service
defined in dependencies.yaml.
'''
service = {'extends': {
'file': '../Common/dependencies.yaml',
'service': name
}}
self.add_service(name, service)
return service
def define_zk(self) -> dict:
'''
Define the ZooKeeper service. Returns the service.
'''
return self.define_external_service(ZOO_KEEPER)
def define_metadata(self) -> dict:
'''
Defines the metadata (MySQL) service. Returns the service
'''
return self.define_external_service(METADATA)
def define_druid_service(self, name, base) -> dict:
'''
Defines a Druid service as a reference to the base definition in
the druid.yaml file. Used when referencing, and extending, a standard
service definition. Cannot be used for a second instance of a Druid
service: such services have to be defined from scratch since they
need unique port mappings and container names.
'''
service = {}
if base is not None:
service['extends'] = {
'file': '../Common/druid.yaml',
'service': base
}
self.extend_druid_service(service)
self.add_service(name, service)
return service
def extend_druid_service(self, service):
'''
Override this to add options to all Druid services.
'''
pass
def add_depends(self, service, items):
'''
Adds a service dependency to a service.
'''
if items is not None and len(items) > 0:
depends = service.setdefault('depends_on', [])
depends += items
def define_master_service(self, name, base) -> dict:
'''
Defines a "master" service: one which depends on the metadata service.
'''
service = self.define_druid_service(name, base)
self.add_depends(service, [ZOO_KEEPER, METADATA])
return service
def define_std_master_service(self, name) -> dict:
'''
Defines a "standard" master service in which the service name is
the same as the service defined in druid.yaml.
'''
return self.define_master_service(name, name)
def define_coordinator(self) -> dict:
'''
Defines a coordinator based on the standard definition. Override to
customize environment variables, mounts, etc.
'''
return self.define_std_master_service(COORDINATOR)
def define_overlord(self) -> dict:
'''
Defines an overlord based on the standard definition. Override to
customize environment variables, mounts, etc.
'''
return self.define_std_master_service(OVERLORD)
def define_worker_service(self, name, base) -> dict:
'''
Defines a Druid "worker" service: one that depends only on ZooKeeper.
'''
service = self.define_druid_service(name, base)
self.add_depends(service, [ZOO_KEEPER])
return service
def define_std_worker_service(self, name) -> dict:
'''
Define a worker service in which the service name for this cluster is the
same as the service name in druid.yaml.
'''
return self.define_worker_service(name, name)
def define_broker(self) -> dict:
'''
Defines a broker based on the standard definition. Override to
customize environment variables, mounts, etc.
'''
return self.define_std_worker_service(BROKER)
def define_router(self) -> dict:
'''
Defines a router based on the standard definition. Override to
customize environment variables, mounts, etc.
'''
return self.define_std_worker_service(ROUTER)
def define_historical(self) -> dict:
'''
Defines a historical based on the standard definition. Override to
customize environment variables, mounts, etc.
'''
return self.define_std_worker_service(HISTORICAL)
def define_std_indexer(self, base) -> dict:
'''
Defines a standard indexer service in which the service name in this
cluster is the same as the definition in druid.yaml. The service mounts
the standard data directory.
'''
service = self.define_worker_service(INDEXER, base)
self.define_data_dir(service)
return service
def define_data_dir(self, service):
'''
Define the input data directory mounted into the selected indexer service.
'''
self.add_volume(service, '${MODULE_DIR}/resources', '/resources')
def define_indexer_service(self) -> dict:
'''
Defines an indexer based on the standard definition. Override to
customize environment variables, mounts, etc.
'''
return self.define_std_indexer(INDEXER)
def define_middle_manager_service(self) -> dict:
'''
Defines a middle manager based on the standard definition. Override to
customize environment variables, mounts, etc.
'''
return self.define_std_indexer(MIDDLE_MANAGER)
def get_indexer_option(self) -> str:
'''
Choose which "indexer" to use: middle manager or indexer (the specific service)
based on the USE_INDEXER environment variable. Defaults to middle manager.
'''
value = os.environ.get('USE_INDEXER')
if value is None:
value = MIDDLE_MANAGER
return value
def define_indexer(self):
'''
Defines the cluster's indexer (generic term) service as either indexer (specific
service) or middle manager, depending on the USE_INDEXER environment variable.
'''
value = self.get_indexer_option()
key = value.lower()
if key == INDEXER:
return self.define_indexer_service()
if key == MIDDLE_MANAGER:
return self.define_middle_manager_service()
raise Exception("Invalid USE_INDEXER value: [" + value + ']')
def define_full_service(self, name, base, host_node):
'''
Create a clone of a service as defined in druid.yaml. Use this when
creating a second instance of a service, since the second must use
distinct host IP and ports.
'''
service = {
'image': '${DRUID_IT_IMAGE_NAME}',
'networks': {
DRUID_NETWORK : {
'ipv4_address': DRUID_SUBNET + '.' + str(host_node)
}
},
'volumes' : [ '${SHARED_DIR}:/shared' ]
}
self.add_env_config(service, 'common')
if base is not None:
self.add_env_config(service, base)
self.add_env_file(service, '${OVERRIDE_ENV}')
self.add_env(service, 'DRUID_INTEGRATION_TEST_GROUP', '${DRUID_INTEGRATION_TEST_GROUP}')
self.add_service(name, service)
return service

58
integration-tests-ex/docs/compose.md
View File

@ -82,6 +82,13 @@ files that define properties as environment variables. All are located in
* `<service>.env` - Properties unique to one service. This is the test equivalent to
the `service/runtime.properties` files.
### MySQL Driver
Unit tests can use any MySQL driver, typically MySQL or MariaDB. The tests use MySQL
by default. Choose a different driver by setting the `MYSQL_DRIVER_CLASSNAME` environment
variable when running tests. The variable chooses the selected driver both in the Druid
server running in a container, and in the test "clients".
### Special Environment Variables
Druid properties can be a bit awkward and verbose in a test environment. A number of
@ -236,3 +243,54 @@ To define a test cluster, do the following:
* If you need multiple instances of the same service, extend that service
twice, and define distinct names and port numbers.
* Add any test-specific environment configuration required.
## Generating `docker-compose.yaml` Files
Each test has somewhat different needs for its test cluster. Yet, there is a
great amount of consistency across test clusters and across services. The result,
if we create files by hand, is a great amount of copy/paste redundancy, with all
the problems that copy/paste implies.
As an alternative, the framework provides a simple-minded way to generate the
`docker-compose.yaml` file using a simple Python-based template mechanism. To use
this:
* Omit the test cluster directory: `cluster/<category>`.
* Instead, create a template file: `templates/<category>.py`.
* The minimal file appears below:
```python
from template import BaseTemplate, generate
generate(__file__, BaseTemplate())
```
The above will generate a "generic" cluster: one of each kind of service, with
either a Middle Manager or Indexer depending on the `USE_INDEXER`
env var.
You customize your specific cluster by creating a test-specific template class
which overrides the various methods that build up the cluster. By using Python,
we first build the cluster as a set of Python dictionaries and arrays, then
we let [PyYAML](https://pyyaml.org/wiki/PyYAMLDocumentation) convert the objects
to a YAML file. Many methods exist to help you populate the configuration tree.
See any of the existing files for examples.
For example, you can:
* Add test-specific environment config to one, some or all services.
* Add or remove services.
* Create multiples of selected services.
The advantage is that, as Druid evolves and we change the basics, those changes
are automatically propagated to all test clusters.
Once you've created your file, the test framework will re-generate the
`docker-compose.yaml` file on each run to reflect any per-run customization.
The generated file is found in `target/cluster/<category>/docker-compose.yaml`.
As with all generated files: resist the temptation to change the generated file:
change the template instead.
The generated `docker-compose.yaml` file goes into a temporary folder:
`target/cluster/<category>`. The script copies over the `Common` directory
as well.

1
integration-tests-ex/docs/docker.md
View File

@ -299,4 +299,3 @@ simple-client-sslcontext
```
If more are needed, they should be added during the image build.

2
integration-tests-ex/docs/druid-config.md
View File

@ -42,7 +42,7 @@ variables. Thus there are two kinds:
## Configuration Flow
We use `docker-compose` to gather up the variables. From most specific
We use `docker compose` to gather up the variables. From most specific
(highest priority) to most general, configuration comes from:
* An environment variable set by the script which launches Docker Compose.

42
integration-tests-ex/docs/guide.md
View File

@ -246,9 +246,31 @@ Indexer. To run on Indexer:
* In the environment, `export USE_INDEXER=indexer`. (Use `middleManager`
otherwise. If the variable is not set, `middleManager` is the default.)
Then, there are two ways to handle indexer-specific configuration: the crude-but-effective
way and the subtle way.
#### Using Two Docker-Compose Files
The crude way, which involves much copy/paste and results in two files which must be maintained
in sync:
* The `cluster/<category>/docker-compose.yaml` file should be for the Middle manager. Create
a separate file called `cluster/<category>/docker-compose-indexer.yaml` to define the
Indexer-based cluster.
#### Generated Docker-Compose File
The fancy way is to use the `docker-compose.yaml` generation template described elsewhere.
In that case, the script will automatically generate either the Middle Manager, or the Indexer,
depending on the environment variable mentioned above.
#### Client Configuration
The client will choose Middle Manager or Indexer automatially if you set the
`USE_INDEXER` environment variable in your IDE. (When run via the build
process, the environment variable is already set.)
* The test `src/test/resources/cluster/<category>/docker.yaml` file should contain a conditional
entry to select define either the Middle Manager or Indexer. Example:
@ -265,3 +287,23 @@ Indexer. To run on Indexer:
Now, the test will run on Indexer if the above environment variable is set, Middle Manager
otherwise.
#### Disable Individual Tests
You may have a test that can run only on Middle Manager or Indexer. The crude-but-effective
way to handle this is:
```
@Test
public void myMMOnlyTest()
{
if (ClusterConfig.isIndexer()) {
return; // Runs only on MM
}
// The MM-only test code here
}
```
It would be possible to define an annotation, managed by the `DruidTestRunner`, if this
becomes something we need to do often.

1
integration-tests-ex/image/build-image.sh
View File

@ -42,7 +42,6 @@ export MYSQL_IMAGE_VERSION=$MYSQL_IMAGE_VERSION
export CONFLUENT_VERSION=$CONFLUENT_VERSION
export MARIADB_VERSION=$MARIADB_VERSION
export HADOOP_VERSION=$HADOOP_VERSION
export MYSQL_DRIVER_CLASSNAME=$MYSQL_DRIVER_CLASSNAME
export DRUID_IT_IMAGE_NAME=$DRUID_IT_IMAGE_NAME
EOF

42
it.sh
View File

@ -34,23 +34,26 @@ Usage: $0 cmd [category]
ci
build Druid and the distribution for CI pipelines
build
build Druid and the distribution
Build Druid and the distribution
dist
build the Druid distribution (only)
Build the Druid distribution (only)
tools
build druid-it-tools
Build druid-it-tools
image
build the test image
Build the test image
up <category>
start the cluster for category
Start the cluster for category
down <category>
stop the cluster for category
Stop the cluster for category
test <category>
start the cluster, run the test for category, and stop the cluster
Start the cluster, run the test for category, and stop the cluster
tail <category>
show the last 20 lines of each container log
Show the last 20 lines of each container log
gen
Generate docker-compose.yaml files (done automatically on up)
run one IT in Travis (build dist, image, run test, tail logs)
github <category>
run one IT in Github Workflows (run test, tail logs)
Run one IT in Github Workflows (run test, tail logs)
prune
prune Docker volumes
@ -100,7 +103,7 @@ function tail_logs
function build_override {
mkdir -p target
OVERRIDE_FILE="override.env"
OVERRIDE_FILE="$(pwd)/target/override.env"
rm -f "$OVERRIDE_FILE"
touch "$OVERRIDE_FILE"
@ -128,8 +131,7 @@ function build_override {
# environment into the container.
# Reuse the OVERRIDE_ENV variable to pass the full list to Docker compose
target_dir=`pwd`
export OVERRIDE_ENV="$target_dir/$OVERRIDE_FILE"
export OVERRIDE_ENV="$OVERRIDE_FILE"
}
function prepare_category {
@ -143,7 +145,6 @@ function prepare_category {
function prepare_docker {
cd $DRUID_DEV/integration-tests-ex/cases
build_override
verify_env_vars
}
function require_env_var {
@ -176,6 +177,8 @@ function verify_env_vars {
fi
;;
"S3DeepStorage")
require_env_var DRUID_CLOUD_BUCKET
require_env_var DRUID_CLOUD_PATH
require_env_var AWS_REGION
require_env_var AWS_ACCESS_KEY_ID
require_env_var AWS_SECRET_ACCESS_KEY
@ -183,6 +186,11 @@ function verify_env_vars {
esac
}
if [ $# = 0 ]; then
usage
exit 1
fi
CMD=$1
shift
MAVEN_IGNORE="-P skip-static-checks,skip-tests -Dmaven.javadoc.skip=true"
@ -207,9 +215,17 @@ case $CMD in
cd $DRUID_DEV/integration-tests-ex/image
mvn install -P test-image $MAVEN_IGNORE
;;
"gen")
# Generate the docker-compose.yaml files. Mostly for debugging
# since the up command does generation implicitly.
prepare_category $1
prepare_docker
./cluster.sh gen $CATEGORY
;;
"up" )
prepare_category $1
prepare_docker
verify_env_vars
./cluster.sh up $CATEGORY
;;
"down" )