From c346ce64b1497f8bbd547bee16fa98cc0d3c74f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Sun, 31 Jan 2021 08:35:39 -0800 Subject: [PATCH] move integration tests from ZooKeeper 3.4.x to 3.5.x (#10786) * move integration tests from ZooKeeper 3.4.x to 3.5.x * run a subset of our integration tests with ZK 3.4 for backwards compatibility testing. * remove need to build separate docker-base image - use multi-stage build for the base image - use openjdk base image instead of building our own JDK base - workaround Debian not including MySQL by using MariaDB - download mysql connector directly instead of using distro version * fix incorrect openssl command failing on Debian * keep mysql connector version in sync with pom version --- .travis.yml | 18 ++++++ integration-tests/docker-base/README.md | 31 ---------- .../docker-base/jdk11/Dockerfile | 62 ------------------- integration-tests/docker-base/jdk8/Dockerfile | 27 -------- integration-tests/docker/Dockerfile | 23 ++++--- .../setup.sh => docker/base-setup.sh} | 23 ++++--- .../docker/docker-compose.base.yml | 2 + .../service-supervisords/zookeeper.conf | 2 +- .../generate-server-certs-and-keystores.sh | 2 +- integration-tests/pom.xml | 1 + .../script/docker_build_containers.sh | 8 +-- 11 files changed, 54 insertions(+), 145 deletions(-) delete mode 100644 integration-tests/docker-base/README.md delete mode 100644 integration-tests/docker-base/jdk11/Dockerfile delete mode 100644 integration-tests/docker-base/jdk8/Dockerfile rename integration-tests/{docker-base/setup.sh => docker/base-setup.sh} (76%) mode change 100644 => 100755 diff --git a/.travis.yml b/.travis.yml index bfdd4958281..d5c9391d556 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,6 +26,7 @@ cache: env: global: + - ZK_VERSION=3.5 # default ZK version for integration tests - DOCKER_IP=127.0.0.1 # for integration tests - MVN="mvn -B" - > # Various options to make execution of maven goals faster (e.g., mvn install) @@ -513,6 +514,16 @@ jobs: jdk: openjdk8 env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' + # Subset of integration tests to run with ZooKeeper 3.4.x for backwards compatibility + - <<: *integration_tests + name: "(Compile=openjdk8, Run=openjdk8, ZK=3.4) leadership and high availability integration tests" + jdk: openjdk8 + env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' ZK_VERSION=3.4 + + - <<: *integration_kafka_format_tests + name: "(Compile=openjdk8, Run=openjdk8, ZK=3.4) Kafka index integration test with various formats" + env: TESTNG_GROUPS='-Dgroups=kafka-data-format' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' ZK_VERSION=3.4 + # END - Integration tests for Compile with Java 8 and Run with Java 8 # START - Integration tests for Compile with Java 8 and Run with Java 11 @@ -575,6 +586,13 @@ jobs: name: "(Compile=openjdk8, Run=openjdk11) leadership and high availability integration tests" jdk: openjdk8 env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=11' USE_INDEXER='middleManager' + + # Subset of integration tests to run with ZooKeeper 3.4.x for backwards compatibility + - <<: *integration_tests + name: "(Compile=openjdk8, Run=openjdk11, ZK=3.4) leadership and high availability integration tests" + jdk: openjdk8 + env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=11' USE_INDEXER='middleManager' ZK_VERSION=3.4 + # END - Integration tests for Compile with Java 8 and Run with Java 11 - &integration_batch_index_k8s diff --git a/integration-tests/docker-base/README.md b/integration-tests/docker-base/README.md deleted file mode 100644 index 00923f99cf4..00000000000 --- a/integration-tests/docker-base/README.md +++ /dev/null @@ -1,31 +0,0 @@ - - -## Base Docker Image for Integration Tests - -### Building Docker Image for Integration Tests - -Run the following commands from /integration-tests/docker-base - -To build docker image for Java JDK8: -- docker build -t druidbase: -f jdk8/Dockerfile . - -To build docker image for Java JDK11: -- docker build -t druidbase: -f jdk11/Dockerfile . - diff --git a/integration-tests/docker-base/jdk11/Dockerfile b/integration-tests/docker-base/jdk11/Dockerfile deleted file mode 100644 index c3ea66d5396..00000000000 --- a/integration-tests/docker-base/jdk11/Dockerfile +++ /dev/null @@ -1,62 +0,0 @@ -# Based on the following projects/files: -# - SequenceIQ hadoop-docker project hosted at https://github.com/sequenceiq/hadoop-docker -# - AdoptOpenJDK openjdk-docker project hosted at https://github.com/AdoptOpenJDK/openjdk-docker -# and modified at the Apache Software Foundation (ASF). -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -FROM ubuntu:16.04 - -# Install Java JDK 11 (OpenJDK 11.0.5) -# Sourced from AdoptOpenJDK openjdk-docker project (https://github.com/AdoptOpenJDK/openjdk-docker) -RUN apt-get update \ - && apt-get install -y --no-install-recommends curl ca-certificates fontconfig locales \ - && echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen \ - && locale-gen en_US.UTF-8 \ - && rm -rf /var/lib/apt/lists/* - -RUN set -eux; \ - ARCH="$(dpkg --print-architecture)"; \ - case "${ARCH}" in \ - armhf) \ - ESUM='c6b1fda3f8807028cbfcc34a4ded2e8a5a6b6239d2bcc1f06673ea6b1530df94'; \ - BINARY_URL='https://github.com/AdoptOpenJDK/openjdk11-binaries/releases/download/jdk-11.0.5%2B10/OpenJDK11U-jdk_arm_linux_hotspot_11.0.5_10.tar.gz'; \ - ;; \ - ppc64el|ppc64le) \ - ESUM='d763481ddc29ac0bdefb24216b3a0bf9afbb058552682567a075f9c0f7da5814'; \ - BINARY_URL='https://github.com/AdoptOpenJDK/openjdk11-binaries/releases/download/jdk-11.0.5%2B10/OpenJDK11U-jdk_ppc64le_linux_hotspot_11.0.5_10.tar.gz'; \ - ;; \ - amd64|x86_64) \ - ESUM='6dd0c9c8a740e6c19149e98034fba8e368fd9aa16ab417aa636854d40db1a161'; \ - BINARY_URL='https://github.com/AdoptOpenJDK/openjdk11-binaries/releases/download/jdk-11.0.5%2B10/OpenJDK11U-jdk_x64_linux_hotspot_11.0.5_10.tar.gz'; \ - ;; \ - *) \ - echo "Unsupported arch: ${ARCH}"; \ - exit 1; \ - ;; \ - esac; \ - curl -LfsSo /tmp/openjdk.tar.gz ${BINARY_URL}; \ - echo "${ESUM} */tmp/openjdk.tar.gz" | sha256sum -c -; \ - mkdir -p /opt/java/openjdk; \ - cd /opt/java/openjdk; \ - tar -xf /tmp/openjdk.tar.gz --strip-components=1; \ - rm -rf /tmp/openjdk.tar.gz; - -ENV JAVA_HOME=/opt/java/openjdk \ - PATH="/opt/java/openjdk/bin:$PATH" - -# Bundle everything into one script so cleanup can reduce image size. -# Otherwise docker's layered images mean that things are not actually deleted. - -COPY setup.sh /root/setup.sh -RUN chmod 0755 /root/setup.sh && /root/setup.sh diff --git a/integration-tests/docker-base/jdk8/Dockerfile b/integration-tests/docker-base/jdk8/Dockerfile deleted file mode 100644 index 7784bf5924b..00000000000 --- a/integration-tests/docker-base/jdk8/Dockerfile +++ /dev/null @@ -1,27 +0,0 @@ -# Based on the SequenceIQ hadoop-docker project hosted at -# https://github.com/sequenceiq/hadoop-docker, and modified at -# the Apache Software Foundation (ASF). -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -FROM ubuntu:16.04 - -# Install Java JDK 8 -RUN apt-get update \ - && apt-get install -y openjdk-8-jdk - -# Bundle everything into one script so cleanup can reduce image size. -# Otherwise docker's layered images mean that things are not actually deleted. - -COPY setup.sh /root/setup.sh -RUN chmod 0755 /root/setup.sh && /root/setup.sh \ No newline at end of file diff --git a/integration-tests/docker/Dockerfile b/integration-tests/docker/Dockerfile index 54fab63b4fd..10e82b98644 100644 --- a/integration-tests/docker/Dockerfile +++ b/integration-tests/docker/Dockerfile @@ -13,16 +13,19 @@ # See the License for the specific language governing permissions and # limitations under the License. +ARG JDK_VERSION=8-slim +FROM openjdk:$JDK_VERSION as druidbase -# This is default value for base image in case DOCKER_IMAGE is not given when building -ARG DOCKER_IMAGE=imply/druiditbase:openjdk-1.8.0_191-1 -# Base image is built from integration-tests/docker-base in the Druid repo -FROM $DOCKER_IMAGE +# Bundle everything into one script so cleanup can reduce image size. +# Otherwise docker's layered images mean that things are not actually deleted. + +COPY base-setup.sh /root/base-setup.sh +RUN /root/base-setup.sh && rm -f /root/base-setup.sh + +FROM druidbase +ARG MYSQL_VERSION # Verify Java version -ARG DOCKER_IMAGE -ENV DOCKER_IMAGE_USED=$DOCKER_IMAGE -RUN echo "Built using base docker image DOCKER_IMAGE_USED=$DOCKER_IMAGE_USED" RUN java -version RUN echo "[mysqld]\ncharacter-set-server=utf8\ncollation-server=utf8_bin\n" >> /etc/mysql/my.cnf @@ -37,9 +40,9 @@ RUN find /var/lib/mysql -type f -exec touch {} \; && /etc/init.d/mysql start \ ADD lib/* /usr/local/druid/lib/ # Download the MySQL Java connector -ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get install -y --no-install-recommends apt-utils libmysql-java -RUN ln -sf /usr/share/java/mysql-connector-java.jar /usr/local/druid/lib/mysql-connector-java.jar +# target path must match the exact path referenced in environment-configs/common +RUN wget -q "https://repo1.maven.org/maven2/mysql/mysql-connector-java/$MYSQL_VERSION/mysql-connector-java-$MYSQL_VERSION.jar" \ + -O /usr/local/druid/lib/mysql-connector-java.jar # Add sample data # touch is needed because OverlayFS's copy-up operation breaks POSIX standards. See https://github.com/docker/for-linux/issues/72. diff --git a/integration-tests/docker-base/setup.sh b/integration-tests/docker/base-setup.sh old mode 100644 new mode 100755 similarity index 76% rename from integration-tests/docker-base/setup.sh rename to integration-tests/docker/base-setup.sh index 3b9d27d6d4f..186874a16d3 --- a/integration-tests/docker-base/setup.sh +++ b/integration-tests/docker/base-setup.sh @@ -25,24 +25,29 @@ apt-get update apt-get install -y wget # MySQL (Metadata store) -apt-get install -y mysql-server +apt-get install -y default-mysql-server # Supervisor apt-get install -y supervisor # Zookeeper -#ZK_VERSION=3.5.8 -#ZK_TAR=apache-zookeeper-$ZK_VERSION-bin +install_zk() { + wget -q -O /tmp/$ZK_TAR.tar.gz "https://archive.apache.org/dist/zookeeper/zookeeper-$ZK_VERSION/$ZK_TAR.tar.gz" + tar -xzf /tmp/$ZK_TAR.tar.gz -C /usr/local + cp /usr/local/$ZK_TAR/conf/zoo_sample.cfg /usr/local/$ZK_TAR/conf/zoo.cfg + rm /tmp/$ZK_TAR.tar.gz +} -ZK_VERISON=3.4.14 +ZK_VERSION=3.4.14 ZK_TAR=zookeeper-$ZK_VERSION +install_zk +ln -s /usr/local/$ZK_TAR /usr/local/zookeeper-3.4 -wget -q -O /tmp/$ZK_TAR.tar.gz "https://apache.org/dist/zookeeper/zookeeper-$ZK_VERSION/$ZK_TAR.tar.gz" -tar -xzf /tmp/$ZK_TAR.tar.gz -C /usr/local -cp /usr/local/$ZK_TAR/conf/zoo_sample.cfg /usr/local/$ZK_TAR/conf/zoo.cfg -ln -s /usr/local/$ZK_TAR /usr/local/zookeeper -rm /tmp/$ZK_TAR.tar.gz +ZK_VERSION=3.5.9 +ZK_TAR=apache-zookeeper-$ZK_VERSION-bin +install_zk +ln -s /usr/local/$ZK_TAR /usr/local/zookeeper-3.5 # Kafka # Match the version to the Kafka client used by KafkaSupervisor diff --git a/integration-tests/docker/docker-compose.base.yml b/integration-tests/docker/docker-compose.base.yml index a8f63be514f..dfe6b898dfc 100644 --- a/integration-tests/docker/docker-compose.base.yml +++ b/integration-tests/docker/docker-compose.base.yml @@ -53,6 +53,8 @@ services: - ./service-supervisords/kafka.conf:/usr/lib/druid/conf/kafka.conf env_file: - ./environment-configs/common + environment: + - ZK_VERSION druid-metadata-storage: image: druid/cluster diff --git a/integration-tests/docker/service-supervisords/zookeeper.conf b/integration-tests/docker/service-supervisords/zookeeper.conf index 1ee5247817f..6039e263048 100644 --- a/integration-tests/docker/service-supervisords/zookeeper.conf +++ b/integration-tests/docker/service-supervisords/zookeeper.conf @@ -1,5 +1,5 @@ [program:zookeeper] -command=/usr/local/zookeeper/bin/zkServer.sh start-foreground +command=/usr/local/zookeeper-%(ENV_ZK_VERSION)s/bin/zkServer.sh start-foreground user=daemon priority=0 stdout_logfile=/shared/logs/zookeeper.log diff --git a/integration-tests/docker/tls/generate-server-certs-and-keystores.sh b/integration-tests/docker/tls/generate-server-certs-and-keystores.sh index 28bf09a5461..931b6e4752c 100755 --- a/integration-tests/docker/tls/generate-server-certs-and-keystores.sh +++ b/integration-tests/docker/tls/generate-server-certs-and-keystores.sh @@ -61,7 +61,7 @@ DNS.2 = localhost EOT # Generate a server certificate for this machine -openssl genrsa -out server.key 1024 -sha256 +openssl genrsa -out server.key 1024 openssl req -new -out server.csr -key server.key -reqexts req_ext -config csr.conf openssl x509 -req -days 3650 -in server.csr -CA root.pem -CAkey root.key -set_serial 0x22222222 -out server.pem -sha256 -extfile csr.conf -extensions req_ext diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index 53c3b373c30..ddcdaaa3c04 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -416,6 +416,7 @@ ${docker.build.skip} ${docker.run.skip} ${it.indexer} + ${mysql.version} ${project.basedir}/build_run_cluster.sh diff --git a/integration-tests/script/docker_build_containers.sh b/integration-tests/script/docker_build_containers.sh index b578ca7ab2c..6bff5020acc 100755 --- a/integration-tests/script/docker_build_containers.sh +++ b/integration-tests/script/docker_build_containers.sh @@ -21,18 +21,18 @@ set -e if [ -z "$DRUID_INTEGRATION_TEST_JVM_RUNTIME" ] then - echo "\$DRUID_INTEGRATION_TEST_JVM_RUNTIME is not set. Build druid-cluster with Java 8" - docker build -t druid/cluster --build-arg DOCKER_IMAGE=imply/druiditbase:openjdk-1.8.0_191-1 $SHARED_DIR/docker + echo "\$DRUID_INTEGRATION_TEST_JVM_RUNTIME is not set. Building druid-cluster with default Java version" + docker build -t druid/cluster --build-arg MYSQL_VERSION $SHARED_DIR/docker else echo "\$DRUID_INTEGRATION_TEST_JVM_RUNTIME is set with value ${DRUID_INTEGRATION_TEST_JVM_RUNTIME}" case "${DRUID_INTEGRATION_TEST_JVM_RUNTIME}" in 8) echo "Build druid-cluster with Java 8" - docker build -t druid/cluster --build-arg DOCKER_IMAGE=imply/druiditbase:openjdk-1.8.0_191-1 $SHARED_DIR/docker + docker build -t druid/cluster --build-arg JDK_VERSION=8-slim --build-arg MYSQL_VERSION $SHARED_DIR/docker ;; 11) echo "Build druid-cluster with Java 11" - docker build -t druid/cluster --build-arg DOCKER_IMAGE=imply/druiditbase:openjdk-11.0.5-1 $SHARED_DIR/docker + docker build -t druid/cluster --build-arg JDK_VERSION=11-slim --build-arg MYSQL_VERSION $SHARED_DIR/docker ;; *) echo "Invalid JVM Runtime given. Stopping"