Reduce memory footprint of integration test by not starting unneeded containers (#10150)

* Reduce memory footprint of integration test

* fix README

* fix README

* fix error in script

* fix security IT

integration-tests/README.md

@@ -67,6 +67,8 @@ can either be 8 or 11.
 
 Druid's configuration (using Docker) can be overrided by providing -Doverride.config.path=<PATH_TO_FILE>. 
 The file must contain one property per line, the key must start with `druid_` and the format should be snake case.
+Note that when bringing up docker containers through mvn and -Doverride.config.path is provided, additional
+Druid routers for security group integration test (permissive tls, no client auth tls, custom check tls) will not be started.   
 
 ## Docker compose
 
@@ -74,17 +76,22 @@ Docker compose yamls located in "docker" folder
 
 docker-compose.base.yml - Base file that defines all containers for integration test
 
-docker-compose.yml - Defines minimal Druid cluster that can be used for non cluster tests
+docker-compose.yml - Defines Druid cluster with default configuration that is used for running integration tests in Travis CI.
     
     docker-compose -f docker-compose.yml up
     // DRUID_INTEGRATION_TEST_GROUP - this variable is used in Druid docker container for "security" and "query" test group. Use next docker-compose if you want to run security/query tests.
     DRUID_INTEGRATION_TEST_GROUP=security docker-compose -f docker-compose.yml up
 
-docker-compose.override-env.yml - the same configuration as docker-compose.yml + override-env variable that needed to run cloud tests 
+docker-compose.override-env.yml - Defines Druid cluster with default configuration plus any additional and/or overriden configurations from override-env file.
 
     // OVERRIDE_ENV - variable that must contains path to Druid configuration file 
     OVERRIDE_ENV=./environment-configs/override-examples/s3 docker-compose -f docker-compose.override-env.yml up
     
+docker-compose.security.yml - Defines three additional Druid router services with permissive tls, no client auth tls, and custom check tls respectively. 
+This is meant to be use together with docker-compose.yml or docker-compose.override-env.yml and is only needed for the "security" group integration test. 
+
+    docker-compose -f docker-compose.yml -f docker-compose.security.yml up 
+    
 docker-compose.druid-hadoop.yml - for starting Apache Hadoop 2.8.5 cluster with the same setup as the Druid tutorial
 
     docker-compose -f docker-compose.druid-hadoop.yml up
@@ -99,12 +106,14 @@ mvn clean install -pl integration-tests -P integration-tests -Ddocker.run.skip=t
 2. Run druid cluster by docker-compose:
 
 ```
-- Basic Druid cluster:
+- Basic Druid cluster (skip this if running Druid cluster with override configs):
 docker-compose -f integration-tests/docker/docker-compose.yml up
-- Druid cluster with override env for cloud integration tests:
+- Druid cluster with override configs (skip this if running Basic Druid cluster):
 OVERRIDE_ENV=<PATH_TO_ENV> docker-compose -f ${DOCKERDIR}/docker-compose.override-env.yml up
-- Druid hadoop:
+- Druid hadoop (if needed):
 docker-compose -f ${DOCKERDIR}/docker-compose.druid-hadoop.yml up
+- Druid routers for security group integration test (if needed):
+ docker-compose -f ${DOCKERDIR}/docker-compose.security.yml up
 ```
 
 3. Run maven command to execute tests with -Ddocker.build.skip=true -Ddocker.run.skip=true

integration-tests/docker/docker-compose.override-env.yml

@@ -121,72 +121,6 @@ services:
       - druid-coordinator
       - druid-broker
 
-  druid-router-permissive-tls:
-    extends:
-      file: docker-compose.base.yml
-      service: druid-router-permissive-tls
-    env_file:
-      - ${OVERRIDE_ENV}
-    environment:
-      - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP}
-    links:
-      - druid-zookeeper-kafka:druid-zookeeper-kafka
-      - druid-coordinator:druid-coordinator
-      - druid-broker:druid-broker
-    depends_on:
-      - druid-zookeeper-kafka
-      - druid-metadata-storage
-      - druid-overlord
-      - druid-coordinator
-      - druid-historical
-      - druid-middlemanager
-      - druid-broker
-      - druid-router
-
-  druid-router-no-client-auth-tls:
-    extends:
-      file: docker-compose.base.yml
-      service: druid-router-no-client-auth-tls
-    env_file:
-      - ${OVERRIDE_ENV}
-    environment:
-      - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP}
-    links:
-      - druid-zookeeper-kafka:druid-zookeeper-kafka
-      - druid-coordinator:druid-coordinator
-      - druid-broker:druid-broker
-    depends_on:
-      - druid-zookeeper-kafka
-      - druid-metadata-storage
-      - druid-overlord
-      - druid-coordinator
-      - druid-historical
-      - druid-middlemanager
-      - druid-broker
-      - druid-router
-
-  druid-router-custom-check-tls:
-    extends:
-      file: docker-compose.base.yml
-      service: druid-router-custom-check-tls
-    env_file:
-      - ${OVERRIDE_ENV}
-    environment:
-      - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP}
-    links:
-      - druid-zookeeper-kafka:druid-zookeeper-kafka
-      - druid-coordinator:druid-coordinator
-      - druid-broker:druid-broker
-    depends_on:
-      - druid-zookeeper-kafka
-      - druid-metadata-storage
-      - druid-overlord
-      - druid-coordinator
-      - druid-historical
-      - druid-middlemanager
-      - druid-broker
-      - druid-router
-
 networks:
   druid-it-net:
     name: druid-it-net

integration-tests/docker/docker-compose.security.yml

@@ -0,0 +1,70 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "2.2"
+services:
+  druid-router-permissive-tls:
+    extends:
+      file: docker-compose.base.yml
+      service: druid-router-permissive-tls
+    links:
+      - druid-zookeeper-kafka:druid-zookeeper-kafka
+      - druid-coordinator:druid-coordinator
+      - druid-broker:druid-broker
+    depends_on:
+      - druid-zookeeper-kafka
+      - druid-metadata-storage
+      - druid-overlord
+      - druid-coordinator
+      - druid-historical
+      - druid-middlemanager
+      - druid-broker
+      - druid-router
+
+  druid-router-no-client-auth-tls:
+    extends:
+      file: docker-compose.base.yml
+      service: druid-router-no-client-auth-tls
+    links:
+      - druid-zookeeper-kafka:druid-zookeeper-kafka
+      - druid-coordinator:druid-coordinator
+      - druid-broker:druid-broker
+    depends_on:
+      - druid-zookeeper-kafka
+      - druid-metadata-storage
+      - druid-overlord
+      - druid-coordinator
+      - druid-historical
+      - druid-middlemanager
+      - druid-broker
+      - druid-router
+
+  druid-router-custom-check-tls:
+    extends:
+      file: docker-compose.base.yml
+      service: druid-router-custom-check-tls
+    links:
+      - druid-zookeeper-kafka:druid-zookeeper-kafka
+      - druid-coordinator:druid-coordinator
+      - druid-broker:druid-broker
+    depends_on:
+      - druid-zookeeper-kafka
+      - druid-metadata-storage
+      - druid-overlord
+      - druid-coordinator
+      - druid-historical
+      - druid-middlemanager
+      - druid-broker
+      - druid-router

integration-tests/docker/docker-compose.yml

@@ -111,60 +111,6 @@ services:
       - druid-coordinator
       - druid-broker
 
-  druid-router-permissive-tls:
-    extends:
-      file: docker-compose.base.yml
-      service: druid-router-permissive-tls
-    links:
-      - druid-zookeeper-kafka:druid-zookeeper-kafka
-      - druid-coordinator:druid-coordinator
-      - druid-broker:druid-broker
-    depends_on:
-      - druid-zookeeper-kafka
-      - druid-metadata-storage
-      - druid-overlord
-      - druid-coordinator
-      - druid-historical
-      - druid-middlemanager
-      - druid-broker
-      - druid-router
-
-  druid-router-no-client-auth-tls:
-    extends:
-      file: docker-compose.base.yml
-      service: druid-router-no-client-auth-tls
-    links:
-      - druid-zookeeper-kafka:druid-zookeeper-kafka
-      - druid-coordinator:druid-coordinator
-      - druid-broker:druid-broker
-    depends_on:
-      - druid-zookeeper-kafka
-      - druid-metadata-storage
-      - druid-overlord
-      - druid-coordinator
-      - druid-historical
-      - druid-middlemanager
-      - druid-broker
-      - druid-router
-
-  druid-router-custom-check-tls:
-    extends:
-      file: docker-compose.base.yml
-      service: druid-router-custom-check-tls
-    links:
-      - druid-zookeeper-kafka:druid-zookeeper-kafka
-      - druid-coordinator:druid-coordinator
-      - druid-broker:druid-broker
-    depends_on:
-      - druid-zookeeper-kafka
-      - druid-metadata-storage
-      - druid-overlord
-      - druid-coordinator
-      - druid-historical
-      - druid-middlemanager
-      - druid-broker
-      - druid-router
-
 networks:
   druid-it-net:
     name: druid-it-net

integration-tests/script/docker_run_cluster.sh

@@ -47,7 +47,14 @@ fi
 
   if [ -z "$DRUID_INTEGRATION_TEST_OVERRIDE_CONFIG_PATH" ]
   then
+     if [ "$DRUID_INTEGRATION_TEST_GROUP" = "security" ]
+     then
+      # Start default Druid services and additional druid router (custom-check-tls, permissive-tls, no-client-auth-tls)
+      docker-compose -f ${DOCKERDIR}/docker-compose.yml -f ${DOCKERDIR}/docker-compose.security.yml up -d
+     else
+      # Start default Druid services
       docker-compose -f ${DOCKERDIR}/docker-compose.yml up -d
+     fi
   else
     # run druid cluster with override config
     OVERRIDE_ENV=$DRUID_INTEGRATION_TEST_OVERRIDE_CONFIG_PATH docker-compose -f ${DOCKERDIR}/docker-compose.override-env.yml up -d