From e6ec4310b1e05530725775ad76cb5abb3f649dad Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Wed, 8 Nov 2017 19:56:31 -0800 Subject: [PATCH] IT: Switch to OpenJDK8 base image. (#5060) * IT: Switch to OpenJDK8 base image. Also split the Docker image into a base image and a child image, and build the base image ahead of time for efficiency's sake. Also upgrade ZK to 3.4.10. * Additional comments about ZK upgrades. --- Dockerfile | 0 docs/content/tutorials/cluster.md | 6 ++-- docs/content/tutorials/quickstart.md | 6 ++-- integration-tests/docker-base/Dockerfile | 7 ++++ integration-tests/docker-base/setup.sh | 42 ++++++++++++++++++++++++ integration-tests/docker/Dockerfile | 36 ++------------------ pom.xml | 5 ++- 7 files changed, 61 insertions(+), 41 deletions(-) delete mode 100644 Dockerfile create mode 100644 integration-tests/docker-base/Dockerfile create mode 100644 integration-tests/docker-base/setup.sh diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/docs/content/tutorials/cluster.md b/docs/content/tutorials/cluster.md index 5f838c5799a..ac13b594dae 100644 --- a/docs/content/tutorials/cluster.md +++ b/docs/content/tutorials/cluster.md @@ -291,9 +291,9 @@ rsync -az druid-#{DRUIDVERSION}/ COORDINATION_SERVER:druid-#{DRUIDVERSION}/ Log on to your coordination server and install Zookeeper: ```bash -curl http://www.gtlib.gatech.edu/pub/apache/zookeeper/zookeeper-3.4.9/zookeeper-3.4.9.tar.gz -o zookeeper-3.4.9.tar.gz -tar -xzf zookeeper-3.4.9.tar.gz -cd zookeeper-3.4.9 +curl http://www.gtlib.gatech.edu/pub/apache/zookeeper/zookeeper-3.4.10/zookeeper-3.4.10.tar.gz -o zookeeper-3.4.10.tar.gz +tar -xzf zookeeper-3.4.10.tar.gz +cd zookeeper-3.4.10 cp conf/zoo_sample.cfg conf/zoo.cfg ./bin/zkServer.sh start ``` diff --git a/docs/content/tutorials/quickstart.md b/docs/content/tutorials/quickstart.md index 0c38194fcaa..4d35b392cc1 100644 --- a/docs/content/tutorials/quickstart.md +++ b/docs/content/tutorials/quickstart.md @@ -50,9 +50,9 @@ Druid currently has a dependency on [Apache ZooKeeper](http://zookeeper.apache.o need to download and run Zookeeper. ```bash -curl http://www.gtlib.gatech.edu/pub/apache/zookeeper/zookeeper-3.4.6/zookeeper-3.4.6.tar.gz -o zookeeper-3.4.6.tar.gz -tar -xzf zookeeper-3.4.6.tar.gz -cd zookeeper-3.4.6 +curl http://www.gtlib.gatech.edu/pub/apache/zookeeper/zookeeper-3.4.10/zookeeper-3.4.10.tar.gz -o zookeeper-3.4.10.tar.gz +tar -xzf zookeeper-3.4.10.tar.gz +cd zookeeper-3.4.10 cp conf/zoo_sample.cfg conf/zoo.cfg ./bin/zkServer.sh start ``` diff --git a/integration-tests/docker-base/Dockerfile b/integration-tests/docker-base/Dockerfile new file mode 100644 index 00000000000..6309f06eb07 --- /dev/null +++ b/integration-tests/docker-base/Dockerfile @@ -0,0 +1,7 @@ +FROM ubuntu:16.04 + +# Bundle everything into one script so cleanup can reduce image size. +# Otherwise docker's layered images mean that things are not actually deleted. + +COPY setup.sh /root/setup.sh +RUN chmod 0755 /root/setup.sh && /root/setup.sh diff --git a/integration-tests/docker-base/setup.sh b/integration-tests/docker-base/setup.sh new file mode 100644 index 00000000000..24fec230ef5 --- /dev/null +++ b/integration-tests/docker-base/setup.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +set -e +set -u + +export DEBIAN_FRONTEND=noninteractive + +apt-get update + +# wget +apt-get install -y wget + +# Java +apt-get install -y openjdk-8-jdk + +# MySQL (Metadata store) +apt-get install -y mysql-server + +# Supervisor +apt-get install -y supervisor + +# Zookeeper +wget -q -O - http://www.us.apache.org/dist/zookeeper/zookeeper-3.4.10/zookeeper-3.4.10.tar.gz | tar -xzf - -C /usr/local \ + && cp /usr/local/zookeeper-3.4.10/conf/zoo_sample.cfg /usr/local/zookeeper-3.4.10/conf/zoo.cfg \ + && ln -s /usr/local/zookeeper-3.4.10 /usr/local/zookeeper + +# Kafka +# Match the version to the Kafka client used by KafkaSupervisor +wget -q -O - http://www.us.apache.org/dist/kafka/0.10.2.0/kafka_2.11-0.10.2.0.tgz | tar -xzf - -C /usr/local \ + && ln -s /usr/local/kafka_2.11-0.10.2.0 /usr/local/kafka + +# Druid system user +adduser --system --group --no-create-home druid \ + && mkdir -p /var/lib/druid \ + && chown druid:druid /var/lib/druid + +# clean up time +apt-get clean \ + && rm -rf /tmp/* \ + /var/tmp/* \ + /var/lib/apt/lists \ + /root/.m2 diff --git a/integration-tests/docker/Dockerfile b/integration-tests/docker/Dockerfile index 6174ce07f1e..88cadeb440a 100644 --- a/integration-tests/docker/Dockerfile +++ b/integration-tests/docker/Dockerfile @@ -1,37 +1,5 @@ -# This is intended to be a temporary unblocker for Travis CI -# We should revert this when ppa:webupd8team/java repo maintainers fix the issue shown here: https://github.com/druid-io/druid/pull/4970 -# Or if we stick to using a non-base Ubuntu image, the custom image should reside in an org repo and not an individual repo -# https://hub.docker.com/r/jonweiimply/ubuntu-j8/ -FROM jonweiimply/ubuntu-j8 - -# MySQL (Metadata store) -RUN apt-get install -y mysql-server - -# Supervisor -RUN apt-get install -y supervisor - -# Zookeeper -RUN wget -q -O - http://www.us.apache.org/dist/zookeeper/zookeeper-3.4.6/zookeeper-3.4.6.tar.gz | tar -xzf - -C /usr/local \ - && cp /usr/local/zookeeper-3.4.6/conf/zoo_sample.cfg /usr/local/zookeeper-3.4.6/conf/zoo.cfg \ - && ln -s /usr/local/zookeeper-3.4.6 /usr/local/zookeeper - -# Kafka -# Match the version to the Kafka client used by KafkaSupervisor -RUN wget -q -O - http://www.us.apache.org/dist/kafka/0.10.2.0/kafka_2.11-0.10.2.0.tgz | tar -xzf - -C /usr/local \ - && ln -s /usr/local/kafka_2.11-0.10.2.0 /usr/local/kafka - -# Druid system user -RUN adduser --system --group --no-create-home druid \ - && mkdir -p /var/lib/druid \ - && chown druid:druid /var/lib/druid - -# clean up time -RUN apt-get clean \ - && rm -rf /tmp/* \ - /var/tmp/* \ - /var/lib/apt/lists \ - /root/.m2 - +# Base image is built from integration-tests/docker-base in the Druid repo +FROM imply/druiditbase # Setup metadata store RUN /etc/init.d/mysql start \ diff --git a/pom.xml b/pom.xml index 4fb02b7fbed..d74a3c7e5cf 100644 --- a/pom.xml +++ b/pom.xml @@ -80,6 +80,9 @@ Need to update Druid to use Jackson 2.6+ --> 1.10.77 2.5.5 + + + 3.4.10 @@ -250,7 +253,7 @@ org.apache.zookeeper zookeeper - 3.4.10 + ${zookeeper.version} org.slf4j