move integration tests from ZooKeeper 3.4.x to 3.5.x (#10786)

* move integration tests from ZooKeeper 3.4.x to 3.5.x
* run a subset of our integration tests with ZK 3.4 for backwards compatibility testing.
* remove need to build separate docker-base image
- use multi-stage build for the base image
- use openjdk base image instead of building our own JDK base
- workaround Debian not including MySQL by using MariaDB
- download mysql connector directly instead of using distro version
* fix incorrect openssl command failing on Debian
* keep mysql connector version in sync with pom version

.travis.yml

@@ -26,6 +26,7 @@ cache:
 
 env:
   global:
+    - ZK_VERSION=3.5 # default ZK version for integration tests
     - DOCKER_IP=127.0.0.1  # for integration tests
     - MVN="mvn -B"
     - > # Various options to make execution of maven goals faster (e.g., mvn install)
@@ -513,6 +514,16 @@ jobs:
       jdk: openjdk8
       env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager'
 
+    # Subset of integration tests to run with ZooKeeper 3.4.x for backwards compatibility
+    - <<: *integration_tests
+      name: "(Compile=openjdk8, Run=openjdk8, ZK=3.4) leadership and high availability integration tests"
+      jdk: openjdk8
+      env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' ZK_VERSION=3.4
+
+    - <<: *integration_kafka_format_tests
+      name: "(Compile=openjdk8, Run=openjdk8, ZK=3.4) Kafka index integration test with various formats"
+      env: TESTNG_GROUPS='-Dgroups=kafka-data-format' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' ZK_VERSION=3.4
+
     # END - Integration tests for Compile with Java 8 and Run with Java 8
 
     # START - Integration tests for Compile with Java 8 and Run with Java 11
@@ -575,6 +586,13 @@ jobs:
       name: "(Compile=openjdk8, Run=openjdk11) leadership and high availability integration tests"
       jdk: openjdk8
       env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=11' USE_INDEXER='middleManager'
+
+    # Subset of integration tests to run with ZooKeeper 3.4.x for backwards compatibility
+    - <<: *integration_tests
+      name: "(Compile=openjdk8, Run=openjdk11, ZK=3.4) leadership and high availability integration tests"
+      jdk: openjdk8
+      env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=11' USE_INDEXER='middleManager' ZK_VERSION=3.4
+
     # END - Integration tests for Compile with Java 8 and Run with Java 11
 
     - &integration_batch_index_k8s

integration-tests/docker-base/README.md

@@ -1,31 +0,0 @@
-<!--
-  ~ Licensed to the Apache Software Foundation (ASF) under one
-  ~ or more contributor license agreements.  See the NOTICE file
-  ~ distributed with this work for additional information
-  ~ regarding copyright ownership.  The ASF licenses this file
-  ~ to you under the Apache License, Version 2.0 (the
-  ~ "License"); you may not use this file except in compliance
-  ~ with the License.  You may obtain a copy of the License at
-  ~
-  ~   http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing,
-  ~ software distributed under the License is distributed on an
-  ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-  ~ KIND, either express or implied.  See the License for the
-  ~ specific language governing permissions and limitations
-  ~ under the License.
-  -->
-
-## Base Docker Image for Integration Tests
-
-### Building Docker Image for Integration Tests
-
-Run the following commands from <DRUIDS_REPO_DIR>/integration-tests/docker-base
-
-To build docker image for Java JDK8:
-- docker build -t druidbase:<NEW_TAG> -f jdk8/Dockerfile .
-
-To build docker image for Java JDK11:
-- docker build -t druidbase:<NEW_TAG> -f jdk11/Dockerfile .
-

integration-tests/docker-base/jdk11/Dockerfile

@@ -1,62 +0,0 @@
-# Based on the following projects/files:
-# - SequenceIQ hadoop-docker project hosted at https://github.com/sequenceiq/hadoop-docker
-# - AdoptOpenJDK openjdk-docker project hosted at https://github.com/AdoptOpenJDK/openjdk-docker
-# and modified at the Apache Software Foundation (ASF).
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-FROM ubuntu:16.04
-
-# Install Java JDK 11 (OpenJDK 11.0.5)
-# Sourced from AdoptOpenJDK openjdk-docker project (https://github.com/AdoptOpenJDK/openjdk-docker)
-RUN apt-get update \
-    && apt-get install -y --no-install-recommends curl ca-certificates fontconfig locales \
-    && echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen \
-    && locale-gen en_US.UTF-8 \
-    && rm -rf /var/lib/apt/lists/*
-
-RUN set -eux; \
-    ARCH="$(dpkg --print-architecture)"; \
-    case "${ARCH}" in \
-       armhf) \
-         ESUM='c6b1fda3f8807028cbfcc34a4ded2e8a5a6b6239d2bcc1f06673ea6b1530df94'; \
-         BINARY_URL='https://github.com/AdoptOpenJDK/openjdk11-binaries/releases/download/jdk-11.0.5%2B10/OpenJDK11U-jdk_arm_linux_hotspot_11.0.5_10.tar.gz'; \
-         ;; \
-       ppc64el|ppc64le) \
-         ESUM='d763481ddc29ac0bdefb24216b3a0bf9afbb058552682567a075f9c0f7da5814'; \
-         BINARY_URL='https://github.com/AdoptOpenJDK/openjdk11-binaries/releases/download/jdk-11.0.5%2B10/OpenJDK11U-jdk_ppc64le_linux_hotspot_11.0.5_10.tar.gz'; \
-         ;; \
-       amd64|x86_64) \
-         ESUM='6dd0c9c8a740e6c19149e98034fba8e368fd9aa16ab417aa636854d40db1a161'; \
-         BINARY_URL='https://github.com/AdoptOpenJDK/openjdk11-binaries/releases/download/jdk-11.0.5%2B10/OpenJDK11U-jdk_x64_linux_hotspot_11.0.5_10.tar.gz'; \
-         ;; \
-       *) \
-         echo "Unsupported arch: ${ARCH}"; \
-         exit 1; \
-         ;; \
-    esac; \
-    curl -LfsSo /tmp/openjdk.tar.gz ${BINARY_URL}; \
-    echo "${ESUM} */tmp/openjdk.tar.gz" | sha256sum -c -; \
-    mkdir -p /opt/java/openjdk; \
-    cd /opt/java/openjdk; \
-    tar -xf /tmp/openjdk.tar.gz --strip-components=1; \
-    rm -rf /tmp/openjdk.tar.gz;
-
-ENV JAVA_HOME=/opt/java/openjdk \
-    PATH="/opt/java/openjdk/bin:$PATH"
-
-# Bundle everything into one script so cleanup can reduce image size.
-# Otherwise docker's layered images mean that things are not actually deleted.
-
-COPY setup.sh /root/setup.sh
-RUN chmod 0755 /root/setup.sh && /root/setup.sh

integration-tests/docker-base/jdk8/Dockerfile

@@ -1,27 +0,0 @@
-# Based on the SequenceIQ hadoop-docker project hosted at
-# https://github.com/sequenceiq/hadoop-docker, and modified at
-# the Apache Software Foundation (ASF).
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-FROM ubuntu:16.04
-
-# Install Java JDK 8
-RUN apt-get update \
-    && apt-get install -y openjdk-8-jdk
-
-# Bundle everything into one script so cleanup can reduce image size.
-# Otherwise docker's layered images mean that things are not actually deleted.
-
-COPY setup.sh /root/setup.sh
-RUN chmod 0755 /root/setup.sh && /root/setup.sh

integration-tests/docker/Dockerfile

@@ -13,16 +13,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+ARG JDK_VERSION=8-slim
+FROM openjdk:$JDK_VERSION as druidbase
 
-# This is default value for base image in case DOCKER_IMAGE is not given when building
-ARG DOCKER_IMAGE=imply/druiditbase:openjdk-1.8.0_191-1
-# Base image is built from integration-tests/docker-base in the Druid repo
-FROM $DOCKER_IMAGE
+# Bundle everything into one script so cleanup can reduce image size.
+# Otherwise docker's layered images mean that things are not actually deleted.
+
+COPY base-setup.sh /root/base-setup.sh
+RUN /root/base-setup.sh && rm -f /root/base-setup.sh
+
+FROM druidbase
+ARG MYSQL_VERSION
 
 # Verify Java version
-ARG DOCKER_IMAGE
-ENV DOCKER_IMAGE_USED=$DOCKER_IMAGE
-RUN echo "Built using base docker image DOCKER_IMAGE_USED=$DOCKER_IMAGE_USED"
 RUN java -version
 
 RUN echo "[mysqld]\ncharacter-set-server=utf8\ncollation-server=utf8_bin\n" >> /etc/mysql/my.cnf
@@ -37,9 +40,9 @@ RUN find /var/lib/mysql -type f -exec touch {} \; && /etc/init.d/mysql start \
 ADD lib/* /usr/local/druid/lib/
 
 # Download the MySQL Java connector
-ENV DEBIAN_FRONTEND=noninteractive
-RUN apt-get update && apt-get install -y --no-install-recommends apt-utils libmysql-java
-RUN ln -sf /usr/share/java/mysql-connector-java.jar /usr/local/druid/lib/mysql-connector-java.jar
+# target path must match the exact path referenced in environment-configs/common
+RUN wget -q "https://repo1.maven.org/maven2/mysql/mysql-connector-java/$MYSQL_VERSION/mysql-connector-java-$MYSQL_VERSION.jar" \
+    -O /usr/local/druid/lib/mysql-connector-java.jar
 
 # Add sample data
 # touch is needed because OverlayFS's copy-up operation breaks POSIX standards. See https://github.com/docker/for-linux/issues/72.

integration-tests/docker/base-setup.sh

@@ -25,24 +25,29 @@ apt-get update
 apt-get install -y wget
 
 # MySQL (Metadata store)
-apt-get install -y mysql-server
+apt-get install -y default-mysql-server
 
 # Supervisor
 apt-get install -y supervisor
 
 # Zookeeper
 
-#ZK_VERSION=3.5.8
-#ZK_TAR=apache-zookeeper-$ZK_VERSION-bin
+install_zk() {
+  wget -q -O /tmp/$ZK_TAR.tar.gz "https://archive.apache.org/dist/zookeeper/zookeeper-$ZK_VERSION/$ZK_TAR.tar.gz"
+  tar -xzf /tmp/$ZK_TAR.tar.gz -C /usr/local
+  cp /usr/local/$ZK_TAR/conf/zoo_sample.cfg /usr/local/$ZK_TAR/conf/zoo.cfg
+  rm /tmp/$ZK_TAR.tar.gz
+}
 
-ZK_VERISON=3.4.14
+ZK_VERSION=3.4.14
 ZK_TAR=zookeeper-$ZK_VERSION
+install_zk
+ln -s /usr/local/$ZK_TAR /usr/local/zookeeper-3.4
 
-wget -q -O /tmp/$ZK_TAR.tar.gz "https://apache.org/dist/zookeeper/zookeeper-$ZK_VERSION/$ZK_TAR.tar.gz"
-tar -xzf /tmp/$ZK_TAR.tar.gz -C /usr/local
-cp /usr/local/$ZK_TAR/conf/zoo_sample.cfg /usr/local/$ZK_TAR/conf/zoo.cfg
-ln -s /usr/local/$ZK_TAR /usr/local/zookeeper
-rm /tmp/$ZK_TAR.tar.gz
+ZK_VERSION=3.5.9
+ZK_TAR=apache-zookeeper-$ZK_VERSION-bin
+install_zk
+ln -s /usr/local/$ZK_TAR /usr/local/zookeeper-3.5
 
 # Kafka
 # Match the version to the Kafka client used by KafkaSupervisor

integration-tests/docker/docker-compose.base.yml

@@ -53,6 +53,8 @@ services:
       - ./service-supervisords/kafka.conf:/usr/lib/druid/conf/kafka.conf
     env_file:
       - ./environment-configs/common
+    environment:
+      - ZK_VERSION
 
   druid-metadata-storage:
     image: druid/cluster

integration-tests/docker/service-supervisords/zookeeper.conf

@@ -1,5 +1,5 @@
 [program:zookeeper]
-command=/usr/local/zookeeper/bin/zkServer.sh start-foreground
+command=/usr/local/zookeeper-%(ENV_ZK_VERSION)s/bin/zkServer.sh start-foreground
 user=daemon
 priority=0
 stdout_logfile=/shared/logs/zookeeper.log

integration-tests/docker/tls/generate-server-certs-and-keystores.sh

@@ -61,7 +61,7 @@ DNS.2 = localhost
 EOT
 
 # Generate a server certificate for this machine
-openssl genrsa -out server.key 1024 -sha256
+openssl genrsa -out server.key 1024
 openssl req -new -out server.csr -key server.key -reqexts req_ext -config csr.conf
 openssl x509 -req -days 3650 -in server.csr -CA root.pem -CAkey root.key -set_serial 0x22222222 -out server.pem -sha256 -extfile csr.conf -extensions req_ext
 

integration-tests/pom.xml

@@ -416,6 +416,7 @@
                                         <DRUID_INTEGRATION_TEST_SKIP_BUILD_DOCKER>${docker.build.skip}</DRUID_INTEGRATION_TEST_SKIP_BUILD_DOCKER>
                                         <DRUID_INTEGRATION_TEST_SKIP_RUN_DOCKER>${docker.run.skip}</DRUID_INTEGRATION_TEST_SKIP_RUN_DOCKER>
                                         <DRUID_INTEGRATION_TEST_INDEXER>${it.indexer}</DRUID_INTEGRATION_TEST_INDEXER>
+                                        <MYSQL_VERSION>${mysql.version}</MYSQL_VERSION>
                                     </environmentVariables>
                                     <executable>${project.basedir}/build_run_cluster.sh</executable>
                                 </configuration>

integration-tests/script/docker_build_containers.sh

@@ -21,18 +21,18 @@ set -e
 
 if [ -z "$DRUID_INTEGRATION_TEST_JVM_RUNTIME" ]
 then
-  echo "\$DRUID_INTEGRATION_TEST_JVM_RUNTIME is not set. Build druid-cluster with Java 8"
-  docker build -t druid/cluster --build-arg DOCKER_IMAGE=imply/druiditbase:openjdk-1.8.0_191-1 $SHARED_DIR/docker
+  echo "\$DRUID_INTEGRATION_TEST_JVM_RUNTIME is not set. Building druid-cluster with default Java version"
+  docker build -t druid/cluster --build-arg MYSQL_VERSION $SHARED_DIR/docker
 else
   echo "\$DRUID_INTEGRATION_TEST_JVM_RUNTIME is set with value ${DRUID_INTEGRATION_TEST_JVM_RUNTIME}"
   case "${DRUID_INTEGRATION_TEST_JVM_RUNTIME}" in
   8)
     echo "Build druid-cluster with Java 8"
-    docker build -t druid/cluster --build-arg DOCKER_IMAGE=imply/druiditbase:openjdk-1.8.0_191-1 $SHARED_DIR/docker
+    docker build -t druid/cluster --build-arg JDK_VERSION=8-slim --build-arg MYSQL_VERSION $SHARED_DIR/docker
     ;;
   11)
     echo "Build druid-cluster with Java 11"
-    docker build -t druid/cluster --build-arg DOCKER_IMAGE=imply/druiditbase:openjdk-11.0.5-1 $SHARED_DIR/docker
+    docker build -t druid/cluster --build-arg JDK_VERSION=11-slim --build-arg MYSQL_VERSION $SHARED_DIR/docker
     ;;
   *)
     echo "Invalid JVM Runtime given. Stopping"