diff --git a/integration-tests/README.md b/integration-tests/README.md index d5476cda8a3..43061f56d26 100644 --- a/integration-tests/README.md +++ b/integration-tests/README.md @@ -66,7 +66,9 @@ Integration tests can also be run with either Java 8 or Java 11 by adding -Djvm. can either be 8 or 11. Druid's configuration (using Docker) can be overrided by providing -Doverride.config.path=. -The file must contain one property per line, the key must start with `druid_` and the format should be snake case. +The file must contain one property per line, the key must start with `druid_` and the format should be snake case. +Note that when bringing up docker containers through mvn and -Doverride.config.path is provided, additional +Druid routers for security group integration test (permissive tls, no client auth tls, custom check tls) will not be started. ## Docker compose @@ -74,17 +76,22 @@ Docker compose yamls located in "docker" folder docker-compose.base.yml - Base file that defines all containers for integration test -docker-compose.yml - Defines minimal Druid cluster that can be used for non cluster tests +docker-compose.yml - Defines Druid cluster with default configuration that is used for running integration tests in Travis CI. docker-compose -f docker-compose.yml up // DRUID_INTEGRATION_TEST_GROUP - this variable is used in Druid docker container for "security" and "query" test group. Use next docker-compose if you want to run security/query tests. DRUID_INTEGRATION_TEST_GROUP=security docker-compose -f docker-compose.yml up -docker-compose.override-env.yml - the same configuration as docker-compose.yml + override-env variable that needed to run cloud tests +docker-compose.override-env.yml - Defines Druid cluster with default configuration plus any additional and/or overriden configurations from override-env file. // OVERRIDE_ENV - variable that must contains path to Druid configuration file OVERRIDE_ENV=./environment-configs/override-examples/s3 docker-compose -f docker-compose.override-env.yml up +docker-compose.security.yml - Defines three additional Druid router services with permissive tls, no client auth tls, and custom check tls respectively. +This is meant to be use together with docker-compose.yml or docker-compose.override-env.yml and is only needed for the "security" group integration test. + + docker-compose -f docker-compose.yml -f docker-compose.security.yml up + docker-compose.druid-hadoop.yml - for starting Apache Hadoop 2.8.5 cluster with the same setup as the Druid tutorial docker-compose -f docker-compose.druid-hadoop.yml up @@ -99,12 +106,14 @@ mvn clean install -pl integration-tests -P integration-tests -Ddocker.run.skip=t 2. Run druid cluster by docker-compose: ``` -- Basic Druid cluster: +- Basic Druid cluster (skip this if running Druid cluster with override configs): docker-compose -f integration-tests/docker/docker-compose.yml up -- Druid cluster with override env for cloud integration tests: +- Druid cluster with override configs (skip this if running Basic Druid cluster): OVERRIDE_ENV= docker-compose -f ${DOCKERDIR}/docker-compose.override-env.yml up -- Druid hadoop: +- Druid hadoop (if needed): docker-compose -f ${DOCKERDIR}/docker-compose.druid-hadoop.yml up +- Druid routers for security group integration test (if needed): + docker-compose -f ${DOCKERDIR}/docker-compose.security.yml up ``` 3. Run maven command to execute tests with -Ddocker.build.skip=true -Ddocker.run.skip=true diff --git a/integration-tests/docker/docker-compose.override-env.yml b/integration-tests/docker/docker-compose.override-env.yml index 344ee0385eb..47f6579b22e 100644 --- a/integration-tests/docker/docker-compose.override-env.yml +++ b/integration-tests/docker/docker-compose.override-env.yml @@ -121,72 +121,6 @@ services: - druid-coordinator - druid-broker - druid-router-permissive-tls: - extends: - file: docker-compose.base.yml - service: druid-router-permissive-tls - env_file: - - ${OVERRIDE_ENV} - environment: - - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP} - links: - - druid-zookeeper-kafka:druid-zookeeper-kafka - - druid-coordinator:druid-coordinator - - druid-broker:druid-broker - depends_on: - - druid-zookeeper-kafka - - druid-metadata-storage - - druid-overlord - - druid-coordinator - - druid-historical - - druid-middlemanager - - druid-broker - - druid-router - - druid-router-no-client-auth-tls: - extends: - file: docker-compose.base.yml - service: druid-router-no-client-auth-tls - env_file: - - ${OVERRIDE_ENV} - environment: - - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP} - links: - - druid-zookeeper-kafka:druid-zookeeper-kafka - - druid-coordinator:druid-coordinator - - druid-broker:druid-broker - depends_on: - - druid-zookeeper-kafka - - druid-metadata-storage - - druid-overlord - - druid-coordinator - - druid-historical - - druid-middlemanager - - druid-broker - - druid-router - - druid-router-custom-check-tls: - extends: - file: docker-compose.base.yml - service: druid-router-custom-check-tls - env_file: - - ${OVERRIDE_ENV} - environment: - - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP} - links: - - druid-zookeeper-kafka:druid-zookeeper-kafka - - druid-coordinator:druid-coordinator - - druid-broker:druid-broker - depends_on: - - druid-zookeeper-kafka - - druid-metadata-storage - - druid-overlord - - druid-coordinator - - druid-historical - - druid-middlemanager - - druid-broker - - druid-router - networks: druid-it-net: name: druid-it-net diff --git a/integration-tests/docker/docker-compose.security.yml b/integration-tests/docker/docker-compose.security.yml new file mode 100644 index 00000000000..a14f9c76902 --- /dev/null +++ b/integration-tests/docker/docker-compose.security.yml @@ -0,0 +1,70 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: "2.2" +services: + druid-router-permissive-tls: + extends: + file: docker-compose.base.yml + service: druid-router-permissive-tls + links: + - druid-zookeeper-kafka:druid-zookeeper-kafka + - druid-coordinator:druid-coordinator + - druid-broker:druid-broker + depends_on: + - druid-zookeeper-kafka + - druid-metadata-storage + - druid-overlord + - druid-coordinator + - druid-historical + - druid-middlemanager + - druid-broker + - druid-router + + druid-router-no-client-auth-tls: + extends: + file: docker-compose.base.yml + service: druid-router-no-client-auth-tls + links: + - druid-zookeeper-kafka:druid-zookeeper-kafka + - druid-coordinator:druid-coordinator + - druid-broker:druid-broker + depends_on: + - druid-zookeeper-kafka + - druid-metadata-storage + - druid-overlord + - druid-coordinator + - druid-historical + - druid-middlemanager + - druid-broker + - druid-router + + druid-router-custom-check-tls: + extends: + file: docker-compose.base.yml + service: druid-router-custom-check-tls + links: + - druid-zookeeper-kafka:druid-zookeeper-kafka + - druid-coordinator:druid-coordinator + - druid-broker:druid-broker + depends_on: + - druid-zookeeper-kafka + - druid-metadata-storage + - druid-overlord + - druid-coordinator + - druid-historical + - druid-middlemanager + - druid-broker + - druid-router diff --git a/integration-tests/docker/docker-compose.yml b/integration-tests/docker/docker-compose.yml index 43fe11545d3..29dde5e5ef7 100644 --- a/integration-tests/docker/docker-compose.yml +++ b/integration-tests/docker/docker-compose.yml @@ -111,60 +111,6 @@ services: - druid-coordinator - druid-broker - druid-router-permissive-tls: - extends: - file: docker-compose.base.yml - service: druid-router-permissive-tls - links: - - druid-zookeeper-kafka:druid-zookeeper-kafka - - druid-coordinator:druid-coordinator - - druid-broker:druid-broker - depends_on: - - druid-zookeeper-kafka - - druid-metadata-storage - - druid-overlord - - druid-coordinator - - druid-historical - - druid-middlemanager - - druid-broker - - druid-router - - druid-router-no-client-auth-tls: - extends: - file: docker-compose.base.yml - service: druid-router-no-client-auth-tls - links: - - druid-zookeeper-kafka:druid-zookeeper-kafka - - druid-coordinator:druid-coordinator - - druid-broker:druid-broker - depends_on: - - druid-zookeeper-kafka - - druid-metadata-storage - - druid-overlord - - druid-coordinator - - druid-historical - - druid-middlemanager - - druid-broker - - druid-router - - druid-router-custom-check-tls: - extends: - file: docker-compose.base.yml - service: druid-router-custom-check-tls - links: - - druid-zookeeper-kafka:druid-zookeeper-kafka - - druid-coordinator:druid-coordinator - - druid-broker:druid-broker - depends_on: - - druid-zookeeper-kafka - - druid-metadata-storage - - druid-overlord - - druid-coordinator - - druid-historical - - druid-middlemanager - - druid-broker - - druid-router - networks: druid-it-net: name: druid-it-net diff --git a/integration-tests/script/docker_run_cluster.sh b/integration-tests/script/docker_run_cluster.sh index 48fcd02c923..8f3bd92c793 100755 --- a/integration-tests/script/docker_run_cluster.sh +++ b/integration-tests/script/docker_run_cluster.sh @@ -47,7 +47,14 @@ fi if [ -z "$DRUID_INTEGRATION_TEST_OVERRIDE_CONFIG_PATH" ] then - docker-compose -f ${DOCKERDIR}/docker-compose.yml up -d + if [ "$DRUID_INTEGRATION_TEST_GROUP" = "security" ] + then + # Start default Druid services and additional druid router (custom-check-tls, permissive-tls, no-client-auth-tls) + docker-compose -f ${DOCKERDIR}/docker-compose.yml -f ${DOCKERDIR}/docker-compose.security.yml up -d + else + # Start default Druid services + docker-compose -f ${DOCKERDIR}/docker-compose.yml up -d + fi else # run druid cluster with override config OVERRIDE_ENV=$DRUID_INTEGRATION_TEST_OVERRIDE_CONFIG_PATH docker-compose -f ${DOCKERDIR}/docker-compose.override-env.yml up -d