From ed623d626f5e821d9056e745bc3b8f344ea91784 Mon Sep 17 00:00:00 2001 From: Paul Rogers Date: Sat, 14 Jan 2023 14:34:06 -0800 Subject: [PATCH] Support both Indexer and MiddleManager in ITs (#13660) Support both indexer and MM in ITs Support for the DRUID_INTEGRATION_TEST_INDEXER variable Conditional client cluster configuration Cleanup of OVERRIDE_ENV file handling Enforce setting of test-specific env vars Cleanup of unused bits --- integration-tests-ex/cases/cluster.sh | 82 ++++----- .../AzureDeepStorage/docker-compose.yaml | 9 + .../BatchIndex/docker-compose-indexer.yaml | 98 ++++++++++ .../cluster/BatchIndex/docker-compose.yaml | 4 +- .../GcsDeepStorage/docker-compose.yaml | 22 ++- .../cluster/S3DeepStorage/docker-compose.yaml | 9 + .../druid/testsEx/config/ClusterConfig.java | 21 ++- .../druid/testsEx/config/Initializer.java | 1 - .../druid/testsEx/config/KafkaConfig.java | 3 +- .../druid/testsEx/config/MetastoreConfig.java | 3 +- .../druid/testsEx/config/ResolvedConfig.java | 11 +- .../druid/testsEx/config/ResolvedService.java | 2 +- .../druid/testsEx/config/ServiceConfig.java | 43 +++-- .../src/test/resources/config-test/test.yaml | 5 + integration-tests-ex/docs/guide.md | 26 +++ integration-tests-ex/docs/test-config.md | 23 ++- it.sh | 174 ++++++++++++++---- 17 files changed, 421 insertions(+), 115 deletions(-) create mode 100644 integration-tests-ex/cases/cluster/BatchIndex/docker-compose-indexer.yaml diff --git a/integration-tests-ex/cases/cluster.sh b/integration-tests-ex/cases/cluster.sh index 142b4fd3a42..605a228294b 100755 --- a/integration-tests-ex/cases/cluster.sh +++ b/integration-tests-ex/cases/cluster.sh @@ -51,28 +51,28 @@ fi CMD=$1 shift +# All commands need env vars +ENV_FILE=$MODULE_DIR/../image/target/env.sh +if [ ! -f $ENV_FILE ]; then + echo "Please build the Docker test image before testing" 1>&2 + exit 1 +fi + +source $ENV_FILE + function category { if [ $# -eq 0 ]; then usage 1>&2 exit 1 fi export CATEGORY=$1 - - # All commands need env vars - ENV_FILE=$MODULE_DIR/../image/target/env.sh - if [ ! -f $ENV_FILE ]; then - echo "Please build the Docker test image before testing" 1>&2 - exit 1 - fi - - source $ENV_FILE # The untranslated category is used for the local name of the # shared folder. # DRUID_INTEGRATION_TEST_GROUP is used in # docker-compose files and here. Despite the name, it is the # name of the cluster configuration we want to run, not the - # test category. Multiple categories an map to the same cluster + # test category. Multiple categories can map to the same cluster # definition. # Map from category name to shared cluster definition name. @@ -105,34 +105,6 @@ function category { export ENV_FILE="$TARGET_DIR/${CATEGORY}.env" } -function build_override { - - mkdir -p target - rm -f "$ENV_FILE" - touch "$ENV_FILE" - - # User-local settings? - LOCAL_ENV="$HOME/druid-it/${CATEGORY}.env" - if [ -f "$LOCAL_ENV" ]; then - cat "$LOCAL_ENV" >> "$ENV_FILE" - fi - - # Provided override file - if [ -n "$OVERRIDE_ENV" ]; then - if [ ! -f "$OVERRIDE_ENV" ]; then - echo "Environment override file (OVERRIDE_ENV) not found: $OVERRIDE_ENV" 1>&2 - exit 1 - fi - cat "$OVERRIDE_ENV" >> "$ENV_FILE" - fi - - # Add all environment variables of the form druid_* - env | grep "^druid_" >> "$ENV_FILE" - - # Reuse the OVERRIDE_ENV variable to pass the full list to Docker compose - export OVERRIDE_ENV="$ENV_FILE" -} - # Dump lots of information to debug Docker failures when run inside # of a build environment where we can't inspect Docker directly. function show_status { @@ -165,6 +137,31 @@ function build_shared_dir { chmod -R a+rwx $SHARED_DIR } +# Each test must have a default docker-compose.yaml file which corresponds to using +# the MiddleManager (or no indexer). A test can optionally include a second file called +# docker-compose-indexer.yaml which uses the Indexer in place of Middle Manager. +function docker_file { + compose_args="" + if [ -n "$DRUID_INTEGRATION_TEST_INDEXER" ]; then + # Sanity check: DRUID_INTEGRATION_TEST_INDEXER must be "indexer" or "middleManager" + # if it is set at all. + if [ "$DRUID_INTEGRATION_TEST_INDEXER" != "indexer" ] && [ "$DRUID_INTEGRATION_TEST_INDEXER" != "middleManager" ] + then + echo "DRUID_INTEGRATION_TEST_INDEXER must be 'indexer' or 'middleManager' (is '$DRUID_INTEGRATION_TEST_INDEXER')" 1>&2 + exit 1 + fi + if [ "$DRUID_INTEGRATION_TEST_INDEXER" == "indexer" ]; then + compose_file=docker-compose-indexer.yaml + if [ ! -f "$CLUSTER_DIR/$compose_file" ]; then + echo "DRUID_INTEGRATION_TEST_INDEXER=$DRUID_INTEGRATION_TEST_INDEXER, but $CLUSTER_DIR/$compose_file is missing" 1>&2 + exit 1 + fi + compose_args="-f $compose_file" + fi + fi + echo $compose_args +} + # Print environment for debugging #env @@ -179,10 +176,9 @@ case $CMD in "up" ) category $* echo "Starting cluster $DRUID_INTEGRATION_TEST_GROUP" - build_override build_shared_dir cd $CLUSTER_DIR - docker-compose up -d + docker-compose `docker_file` up -d # Enable the following for debugging #show_status ;; @@ -196,12 +192,12 @@ case $CMD in # Enable the following for debugging #show_status cd $CLUSTER_DIR - echo OVERRIDE_ENV="$ENV_FILE" docker-compose $CMD - OVERRIDE_ENV="$ENV_FILE" docker-compose $CMD + echo docker-compose `docker_file` $CMD + docker-compose `docker_file` $CMD ;; "*" ) category $* cd $CLUSTER_DIR - OVERRIDE_ENV="$ENV_FILE" docker-compose $CMD + docker-compose `docker_file` $CMD ;; esac diff --git a/integration-tests-ex/cases/cluster/AzureDeepStorage/docker-compose.yaml b/integration-tests-ex/cases/cluster/AzureDeepStorage/docker-compose.yaml index 427188db2d0..75d9b046297 100644 --- a/integration-tests-ex/cases/cluster/AzureDeepStorage/docker-compose.yaml +++ b/integration-tests-ex/cases/cluster/AzureDeepStorage/docker-compose.yaml @@ -12,6 +12,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# ------------------------------------------------------------------------- + +# Cluster for the Azure deep storage test. +# +# Required env vars: +# +# AZURE_ACCOUNT +# AZURE_KEY +# AZURE_CONTAINER networks: druid-it-net: diff --git a/integration-tests-ex/cases/cluster/BatchIndex/docker-compose-indexer.yaml b/integration-tests-ex/cases/cluster/BatchIndex/docker-compose-indexer.yaml new file mode 100644 index 00000000000..f8235db6255 --- /dev/null +++ b/integration-tests-ex/cases/cluster/BatchIndex/docker-compose-indexer.yaml @@ -0,0 +1,98 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +networks: + druid-it-net: + name: druid-it-net + ipam: + config: + - subnet: 172.172.172.0/24 + +services: + zookeeper: + extends: + file: ../Common/dependencies.yaml + service: zookeeper + + metadata: + extends: + file: ../Common/dependencies.yaml + service: metadata + + coordinator: + extends: + file: ../Common/druid.yaml + service: coordinator + container_name: coordinator + environment: + - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP} + # The frequency with which the coordinator polls the database + # for changes. The DB population code has to wait at least this + # long for the coordinator to notice changes. + - druid_manager_segments_pollDuration=PT5S + - druid_coordinator_period=PT10S + depends_on: + - zookeeper + - metadata + + overlord: + extends: + file: ../Common/druid.yaml + service: overlord + container_name: overlord + environment: + - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP} + depends_on: + - zookeeper + - metadata + + broker: + extends: + file: ../Common/druid.yaml + service: broker + environment: + - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP} + depends_on: + - zookeeper + + router: + extends: + file: ../Common/druid.yaml + service: router + environment: + - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP} + depends_on: + - zookeeper + + historical: + extends: + file: ../Common/druid.yaml + service: historical + environment: + - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP} + depends_on: + - zookeeper + + indexer: + extends: + file: ../Common/druid.yaml + service: indexer + environment: + - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP} + volumes: + # Test data + - ../../resources:/resources + depends_on: + - zookeeper diff --git a/integration-tests-ex/cases/cluster/BatchIndex/docker-compose.yaml b/integration-tests-ex/cases/cluster/BatchIndex/docker-compose.yaml index f8235db6255..7778c1d3d05 100644 --- a/integration-tests-ex/cases/cluster/BatchIndex/docker-compose.yaml +++ b/integration-tests-ex/cases/cluster/BatchIndex/docker-compose.yaml @@ -85,10 +85,10 @@ services: depends_on: - zookeeper - indexer: + middlemanager: extends: file: ../Common/druid.yaml - service: indexer + service: middlemanager environment: - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP} volumes: diff --git a/integration-tests-ex/cases/cluster/GcsDeepStorage/docker-compose.yaml b/integration-tests-ex/cases/cluster/GcsDeepStorage/docker-compose.yaml index 6a59daabed4..3b024104b72 100644 --- a/integration-tests-ex/cases/cluster/GcsDeepStorage/docker-compose.yaml +++ b/integration-tests-ex/cases/cluster/GcsDeepStorage/docker-compose.yaml @@ -12,6 +12,16 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# ------------------------------------------------------------------------- + +# Cluster for the Google Cluster Storage (GCS) deep storage test. +# +# Required env vars: +# +# GOOGLE_BUCKET +# GOOGLE_PREFIX +# GOOGLE_APPLICATION_CREDENTIALS - must point to a file that holds the Google +# credentials. Mounted into each Druid container. networks: druid-it-net: @@ -49,7 +59,7 @@ services: - druid_manager_segments_pollDuration=PT5S - druid_coordinator_period=PT10S volumes: - # Copy credentials file + # Mount credentials file - ${GOOGLE_APPLICATION_CREDENTIALS}:/resources/credentials.json depends_on: - zookeeper @@ -68,7 +78,7 @@ services: - druid_google_prefix=${GOOGLE_PREFIX} - GOOGLE_APPLICATION_CREDENTIALS=/resources/credentials.json volumes: - # Copy credentials file + # Mount credentials file - ${GOOGLE_APPLICATION_CREDENTIALS}:/resources/credentials.json depends_on: - zookeeper @@ -86,7 +96,7 @@ services: - druid_google_prefix=${GOOGLE_PREFIX} - GOOGLE_APPLICATION_CREDENTIALS=/resources/credentials.json volumes: - # Copy credentials file + # Mount credentials file - ${GOOGLE_APPLICATION_CREDENTIALS}:/resources/credentials.json depends_on: - zookeeper @@ -103,7 +113,7 @@ services: - druid_google_prefix=${GOOGLE_PREFIX} - GOOGLE_APPLICATION_CREDENTIALS=/resources/credentials.json volumes: - # Copy credentials file + # Mount credentials file - ${GOOGLE_APPLICATION_CREDENTIALS}:/resources/credentials.json depends_on: - zookeeper @@ -120,7 +130,7 @@ services: - druid_google_prefix=${GOOGLE_PREFIX} - GOOGLE_APPLICATION_CREDENTIALS=/resources/credentials.json volumes: - # Copy credentials file + # Mount credentials file - ${GOOGLE_APPLICATION_CREDENTIALS}:/resources/credentials.json depends_on: - zookeeper @@ -137,7 +147,7 @@ services: - druid_google_prefix=${GOOGLE_PREFIX} - GOOGLE_APPLICATION_CREDENTIALS=/resources/credentials.json volumes: - # Copy credentials file + # Mount credentials file - ${GOOGLE_APPLICATION_CREDENTIALS}:/resources/credentials.json # Test data - ../data:/resources diff --git a/integration-tests-ex/cases/cluster/S3DeepStorage/docker-compose.yaml b/integration-tests-ex/cases/cluster/S3DeepStorage/docker-compose.yaml index b809d9dc258..b7a3d745594 100644 --- a/integration-tests-ex/cases/cluster/S3DeepStorage/docker-compose.yaml +++ b/integration-tests-ex/cases/cluster/S3DeepStorage/docker-compose.yaml @@ -12,6 +12,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# ------------------------------------------------------------------------- + +# Cluster for the S3 deep storage test. +# +# Required env vars: +# +# AWS_REGION +# AWS_ACCESS_KEY_ID +# AWS_SECRET_ACCESS_KEY networks: druid-it-net: diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/ClusterConfig.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/ClusterConfig.java index 363f7648b07..7c5d509fa8c 100644 --- a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/ClusterConfig.java +++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/ClusterConfig.java @@ -24,6 +24,7 @@ import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.google.common.collect.ImmutableSet; import org.apache.druid.java.util.common.ISE; import org.apache.druid.testsEx.config.ServiceConfig.DruidConfig; import org.apache.druid.testsEx.config.ServiceConfig.ZKConfig; @@ -35,6 +36,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; /** * Java representation of the test configuration YAML. @@ -164,7 +166,24 @@ public class ClusterConfig public ResolvedConfig resolve(String clusterName) { - return new ResolvedConfig(clusterName, resolveIncludes()); + Set configTags = createConfigTags(); + return new ResolvedConfig(clusterName, resolveIncludes(), configTags); + } + + /** + * Create the set of configuration tags for this run. At present, the only options + * are "middleManager" or "indexer" corresponding to the value of the + * {@code DRUID_INTEGRATION_TEST_INDEXER} env var which says whether this cluster has + * an indexer or middle manager. + */ + private Set createConfigTags() + { + String indexer = "middleManager"; + String indexerValue = System.getenv("DRUID_INTEGRATION_TEST_INDEXER"); + if (indexerValue != null) { + indexer = indexerValue; + } + return ImmutableSet.of(indexer); } public ClusterConfig resolveIncludes() diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/Initializer.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/Initializer.java index e9e92625ba5..ebd4e003ce0 100644 --- a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/Initializer.java +++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/Initializer.java @@ -400,7 +400,6 @@ public class Initializer private final ResolvedConfig clusterConfig; private final Injector injector; private final Lifecycle lifecycle; - private MetastoreClient metastoreClient; private DruidClusterClient clusterClient; private Initializer(Builder builder) diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/KafkaConfig.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/KafkaConfig.java index 00e785d940d..4aa67079465 100644 --- a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/KafkaConfig.java +++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/KafkaConfig.java @@ -28,10 +28,9 @@ public class KafkaConfig extends ServiceConfig { @JsonCreator public KafkaConfig( - @JsonProperty("service") String service, @JsonProperty("instances") List instances ) { - super(service, instances); + super(instances); } } diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/MetastoreConfig.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/MetastoreConfig.java index cad2fd5293e..1c4a99015d9 100644 --- a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/MetastoreConfig.java +++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/MetastoreConfig.java @@ -58,7 +58,6 @@ public class MetastoreConfig extends ServiceConfig @JsonCreator public MetastoreConfig( - @JsonProperty("service") String service, @JsonProperty("driver") String driver, @JsonProperty("connectURI") String connectURI, @JsonProperty("user") String user, @@ -67,7 +66,7 @@ public class MetastoreConfig extends ServiceConfig @JsonProperty("instances") List instances ) { - super(service, instances); + super(instances); this.driver = driver; this.connectURI = connectURI; this.user = user; diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/ResolvedConfig.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/ResolvedConfig.java index bdfc17dac11..46b0aa58b4f 100644 --- a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/ResolvedConfig.java +++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/ResolvedConfig.java @@ -39,6 +39,7 @@ import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; +import java.util.Set; public class ResolvedConfig { @@ -67,7 +68,7 @@ public class ResolvedConfig private final ResolvedMetastore metastore; private final Map druidServices = new HashMap<>(); - public ResolvedConfig(String category, ClusterConfig config) + public ResolvedConfig(String category, ClusterConfig config, Set configTags) { this.category = category; type = config.type() == null ? ClusterType.docker : config.type(); @@ -115,8 +116,14 @@ public class ResolvedConfig if (config.druid() != null) { for (Entry entry : config.druid().entrySet()) { + DruidConfig service = entry.getValue(); + if (service.ifTag() != null) { + if (!configTags.contains(service.ifTag())) { + continue; + } + } druidServices.put(entry.getKey(), - new ResolvedDruidService(this, entry.getValue(), entry.getKey())); + new ResolvedDruidService(this, service, entry.getKey())); } } } diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/ResolvedService.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/ResolvedService.java index 5bd377f7079..20b769dea13 100644 --- a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/ResolvedService.java +++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/ResolvedService.java @@ -36,7 +36,7 @@ public class ResolvedService public ResolvedService(ResolvedConfig root, ServiceConfig config, String name) { - this.service = config.service() == null ? name : config.service(); + this.service = name; for (ServiceInstance instanceConfig : config.instances()) { this.instances.add(new ResolvedInstance(root, instanceConfig, this)); } diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/ServiceConfig.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/ServiceConfig.java index 08b07bc4ed8..6d8197c475a 100644 --- a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/ServiceConfig.java +++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/ServiceConfig.java @@ -24,29 +24,21 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.annotation.JsonProperty; +import javax.annotation.Nullable; + import java.util.List; public class ServiceConfig { - protected final String service; protected List instances; public ServiceConfig( - String service, List instances ) { - this.service = service; this.instances = instances; } - @JsonProperty("service") - @JsonInclude(Include.NON_NULL) - public String service() - { - return service; - } - @JsonProperty("instances") @JsonInclude(Include.NON_NULL) public List instances() @@ -74,12 +66,11 @@ public class ServiceConfig @JsonCreator public ZKConfig( - @JsonProperty("service") String service, @JsonProperty("startTimeoutSecs") int startTimeoutSecs, @JsonProperty("instances") List instances ) { - super(service, instances); + super(instances); this.startTimeoutSecs = startTimeoutSecs; } @@ -96,21 +87,41 @@ public class ServiceConfig * in the {@code druid} map: 
    * druid:
    *   broker:  # <-- key (service name)
+   *     if: config-tag
    *     instances:
    *       ...
    *
+ * + * Where {@code config-tag} is a string that indicates a config + * option. At present there are two: + *
    + *
  • {@code middleManager}: cluster users the Middle Manager.
    • + *
  • {@code indexer}: cluster uses the Indexer.
    • + *
  • + * + * A service is included in the resolved config only if the corresponding + * config tag is set. */ public static class DruidConfig extends ServiceConfig { + private final String ifTag; + @JsonCreator public DruidConfig( - // Note: service is not actually used. - @JsonProperty("service") String service, + @JsonProperty("if") String ifTag, @JsonProperty("instances") List instances ) { - super(service, instances); + super(instances); + this.ifTag = ifTag; + } + + @Nullable + @JsonProperty("if") + @JsonInclude(Include.NON_NULL) + public String ifTag() + { + return ifTag; } } - } diff --git a/integration-tests-ex/cases/src/test/resources/config-test/test.yaml b/integration-tests-ex/cases/src/test/resources/config-test/test.yaml index 92745caea11..29942db64f7 100644 --- a/integration-tests-ex/cases/src/test/resources/config-test/test.yaml +++ b/integration-tests-ex/cases/src/test/resources/config-test/test.yaml @@ -31,6 +31,11 @@ druid: instances: - port: 8083 middlemanager: + if: middleManager + instances: + - port: 8091 + indexer: + if: indexer instances: - port: 8091 broker: diff --git a/integration-tests-ex/docs/guide.md b/integration-tests-ex/docs/guide.md index 795a725e690..7ec77d4d535 100644 --- a/integration-tests-ex/docs/guide.md +++ b/integration-tests-ex/docs/guide.md @@ -239,3 +239,29 @@ test. Your test will run in parallel with all other IT categories, which is why we offered the advice above: the test has to have a good reason to fire up yet another build task. +### Choosing the Middle Manager or Indexer + +Tests should run on the Middle Manager by default. Tests can optionally run on the +Indexer. To run on Indexer: + +* In the environment, `export DRUID_INTEGRATION_TEST_INDEXER=indexer`. (Use `middleManager` + otherwise. If the variable is not set, `middleManager` is the default.) +* The `cluster//docker-compose.yaml` file should be for the Middle manager. Create + a separate file called `cluster//docker-compose-indexer.yaml` to define the + Indexer-based cluster. +* The test `src/test/resources/cluster//docker.yaml` file should contain a conditional + entry to select define either the Middle Manager or Indexer. Example: + +```yaml + middlemanager: + if: middleManager + instances: + - port: 8091 + indexer: + if: indexer + instances: + - port: 8091 +``` + +Now, the test will run on Indexer if the above environment variable is set, Middle Manager +otherwise. diff --git a/integration-tests-ex/docs/test-config.md b/integration-tests-ex/docs/test-config.md index 4a5c7a583f7..d2ccdb2e682 100644 --- a/integration-tests-ex/docs/test-config.md +++ b/integration-tests-ex/docs/test-config.md @@ -414,15 +414,28 @@ changes itself somehow. Generic object to describe Docker Compose services. -#### `service` +#### `if` + +Conditionally defines a service. The system defines a set of configutation tags. +At present there are only two: + +* `middleManager`: the cluster runs a MiddleManager +* `indexer`: the cluster runs an Indexer + +The `if` tag conditionally enables a service only if the corresponding tag is set. +Thus, for a cluster that can use either a middle manager or an indexer: ```yaml -service: + middlemanager: + if: middleManager + instances: + - port: 8091 + indexer: + if: indexer + instances: + - port: 8091 ``` -Name of the service as known to Docker Compose. Defaults to be -the same as the service name used in this configuration file. - #### `instances` ```yaml diff --git a/it.sh b/it.sh index 97ee1ca05be..710ac33c960 100755 --- a/it.sh +++ b/it.sh @@ -17,9 +17,14 @@ #-------------------------------------------------------------------- # Utility script for running the new integration tests, since the Maven -# commands are unwieldy. +# commands are unwieldy. Allows straightforward usage of ITs on the desktop +# and in various build scripts. Handles configuration of various kinds. + set -e +# Enable for debugging +#set -x + export DRUID_DEV=$(cd $(dirname $0) && pwd) function usage @@ -46,6 +51,15 @@ Usage: $0 cmd [category] run one IT in Travis (build dist, image, run test, tail logs) prune prune Docker volumes + +Environment: + OVERRIDE_ENV: optional, name of env file to pass to Docker + DRUID_INTEGRATION_TEST_INDEXER: Set to middleManager (default if not set) + or "indexer". If "indexer", requires docker-compose-indexer.yaml exist. + druid_*: passed to the container. + Other, test-specific variables. + +See docs for additional details. EOF } @@ -60,6 +74,113 @@ function tail_logs done } +# Many tests require us to pass information into containers using environment variables. +# The Docker environment is distinct from the environment running this script. We bridge +# the two by passing into Docker compose a file that contains all env vars we want to +# "export" from our local environment into the container environment. +# There are three ways to provide these options: +# +# 1. Directly in the environment. (Simplest and best.) We support a fixed set of variables: +# +# 2. For ad-hoc use, as var=value pairs in a file with the same name as the +# test catagory, in the home folder under ~/druid-it. Example: +# BatchIndex.env. Use this to hold credentials and other info which you must +# pass into tests when running locally. +# 3. A file given by the OVERRIDE_ENV environment variable. That is, OVERRIDE_ENV holds +# the path to a file of var=value pairs. Historically, this file was created by a +# build environment such as Travis. However, it is actually simpler just to use +# option 1: just set the values in the environment and let Linux pass them through to +# this script. +# 4. Environment variables of the form "druid_" used to create the Druid config file. +# +# All of the above are combined into a temporary environment file which is then passed +# into Docker compose. +function build_override { + + mkdir -p target + OVERRIDE_FILE="override.env" + rm -f "$OVERRIDE_FILE" + touch "$OVERRIDE_FILE" + + # Provided override file + if [ -n "$OVERRIDE_ENV" ]; then + if [ ! -f "$OVERRIDE_ENV" ]; then + echo "Environment override file OVERRIDE_ENV not found: $OVERRIDE_ENV" 1>&2 + exit 1 + fi + cat "$OVERRIDE_ENV" >> "$OVERRIDE_FILE" + fi + + # User-local settings? + LOCAL_ENV="$HOME/druid-it/${CATEGORY}.env" + if [ -f "$LOCAL_ENV" ]; then + cat "$LOCAL_ENV" >> "$OVERRIDE_FILE" + fi + + # Add all environment variables of the form druid_* + set +e # Grep gives exit status 1 if no lines match. Let's not fail. + env | grep "^druid_" >> "$OVERRIDE_FILE" + set -e + + # TODO: Add individual env vars that we want to pass from the local + # environment into the container. + + # Reuse the OVERRIDE_ENV variable to pass the full list to Docker compose + target_dir=`pwd` + export OVERRIDE_ENV="$target_dir/$OVERRIDE_FILE" +} + +function prepare_category { + if [ $# -eq 0 ]; then + usage 1>&2 + exit 1 + fi + export CATEGORY=$1 +} + +function prepare_docker { + cd $DRUID_DEV/integration-tests-ex/cases + build_override + verify_env_vars +} + +function require_env_var { + if [ -n "$1" ]; then + echo "$1 must be set for test category $CATEGORY" 1>&2 + exit 1 + fi +} + +# Verfiy any test-specific environment variables that must be set in this local +# environment (and generally passed into the Docker container via docker-compose.yaml). +# +# Add entries here as you add env var references in docker-compose.yaml. Doing so +# ensures we get useful error messages when we forget to set something, rather than +# some cryptic use-specific error. +function verify_env_vars { + case $CATEGORY in + "AzureDeepStorage") + require_env_var AZURE_ACCOUNT + require_env_var AZURE_KEY + require_env_var AZURE_CONTAINER + ;; + "GcsDeepStorage") + require_env_var GOOGLE_BUCKET + require_env_var GOOGLE_PREFIX + require_env_var GOOGLE_APPLICATION_CREDENTIALS + if [ ! -f "$GOOGLE_APPLICATION_CREDENTIALS" ]; then + echo "Required file GOOGLE_APPLICATION_CREDENTIALS=$GOOGLE_APPLICATION_CREDENTIALS is missing" 1>&2 + exit 1 + fi + ;; + "S3DeepStorage") + require_env_var AWS_REGION + require_env_var AWS_ACCESS_KEY_ID + require_env_var AWS_SECRET_ACCESS_KEY + ;; + esac +} + CMD=$1 shift MAVEN_IGNORE="-P skip-static-checks,skip-tests -Dmaven.javadoc.skip=true" @@ -82,48 +203,33 @@ case $CMD in mvn install -P test-image $MAVEN_IGNORE ;; "up" ) - if [ -z "$1" ]; then - usage - exit 1 - fi - cd $DRUID_DEV/integration-tests-ex/cases - ./cluster.sh up $1 + prepare_category $1 + prepare_docker + ./cluster.sh up $CATEGORY ;; "down" ) - if [ -z "$1" ]; then - usage - exit 1 - fi - cd $DRUID_DEV/integration-tests-ex/cases - ./cluster.sh down $1 + prepare_category $1 + prepare_docker + ./cluster.sh down $CATEGORY ;; "test" ) - if [ -z "$1" ]; then - usage - exit 1 - fi - cd $DRUID_DEV/integration-tests-ex/cases - mvn verify -P skip-static-checks,docker-tests,IT-$1 \ + prepare_category $1 + prepare_docker + mvn verify -P skip-static-checks,docker-tests,IT-$CATEGORY \ -Dmaven.javadoc.skip=true -DskipUTs=true \ -pl :druid-it-cases ;; "tail" ) - if [ -z "$1" ]; then - usage - exit 1 - fi - tail_logs $1 + prepare_category $1 + tail_logs $CATEGORY + ;; + "travis" ) + prepare_category $1 + $0 dist + $0 image + $0 test $CATEGORY + $0 tail $CATEGORY ;; - "travis" ) - if [ -z "$1" ]; then - usage - exit 1 - fi - $0 dist - $0 image - $0 test $1 - $0 tail $1 - ;; "prune" ) # Caution: this removes all volumes, which is generally what you # want when testing.