Add docker container for druid (#6896)

* Add docker container for druid

This container is an 'omnibus' (since there is such a high
overlap with the various services). It includes all contrib
extension as well as the msql connector.

It is intended to be run as `docker run NAME SERVICE`
(e.g. docker run druid:latest broker)

* Add Apache license header

* Resolve issues from Pull Request review

* Add comments at top of script per PR comments

* Revert BUILDKIT. Not available everywhere.

* Don't set hostname, allow default (IP)

Some environments (e.g. Kubernetes Deployments) don't resolve
hostname to IP.

* Switch to amd64 glibc-based busybox from 32-bit uclibc

* Override service-specific configuration

* Replace MAINTAINER w/ LABEL

* Add mysql connector to application classpath

This works around issue #3770
https://github.com/apache/incubator-druid/issues/3770

* Add docker-compose and sample environment

Signed-off-by: Don Bowman <don@agilicus.com>

.dockerignore

@@ -0,0 +1,37 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+.git
+**/*.jar
+**/*.class
+dist
+target
+*.iml
+*.ipr
+*.iws
+*.tar.gz
+*.swp
+*.swo
+.classpath
+.idea
+.project
+.settings/
+*.log
+*.DS_Store
+_site
+dependency-reduced-pom.xml

distribution/docker/Dockerfile

@@ -0,0 +1,56 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+FROM maven:3-jdk-8 as builder
+
+COPY . /src
+WORKDIR /src
+RUN mvn install -ff -DskipTests -Dforbiddenapis.skip=true -Pdist -Pbundle-contrib-exts
+
+RUN \
+ VER=$(mvn -B org.apache.maven.plugins:maven-help-plugin:3.1.1:evaluate -Dexpression=project.version -q -DforceStdout=true -f pom.xml 2>/dev/null) \
+ && tar -zxf ./distribution/target/apache-druid-${VER}-bin.tar.gz -C /opt \
+ && ln -s /opt/apache-druid-${VER} /opt/druid
+
+RUN wget -O /opt/druid/extensions/mysql-metadata-storage/mysql-connector-java-5.1.38.jar http://central.maven.org/maven2/mysql/mysql-connector-java/5.1.38/mysql-connector-java-5.1.38.jar \
+ && sha256sum --ignore-missing -c /src/distribution/docker/sha256sums.txt \
+ && ln -s /opt/druid/extensions/mysql-metadata-storage/mysql-connector-java-5.1.38.jar /opt/druid/lib
+
+RUN addgroup --gid 1000 druid \
+ && adduser --home /opt/druid --shell /bin/sh --no-create-home --uid 1000 --gecos '' --gid 1000 --disabled-password druid \
+ && mkdir -p /opt/druid/var \
+ && chown -R druid:druid /opt/druid \
+ && chmod 775 /opt/druid/var
+
+FROM amd64/busybox:1.30.0-glibc as busybox
+FROM gcr.io/distroless/java
+LABEL maintainer="Don Bowman <don@agilicus.com>"
+
+COPY --from=busybox /bin/busybox /busybox/busybox
+RUN ["/busybox/busybox", "--install", "/bin"]
+COPY --from=builder /etc/passwd /etc/passwd
+COPY --from=builder /etc/group /etc/group
+COPY --from=builder --chown=druid /opt /opt
+COPY distribution/docker/druid.sh /druid.sh
+RUN chown -R druid:druid /opt/druid
+USER druid
+VOLUME /opt/druid/var
+WORKDIR /opt/druid
+
+ENTRYPOINT ["/druid.sh"]

distribution/docker/README.md

@@ -0,0 +1,26 @@
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~   http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing,
+  ~ software distributed under the License is distributed on an
+  ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  ~ KIND, either express or implied.  See the License for the
+  ~ specific language governing permissions and limitations
+  ~ under the License.
+  -->
+
+## Build
+
+From the root of the repo, run `docker build -t druid:tag -f distribution/docker/Dockerfile .`
+
+## Run
+
+Edit `environment` to suite. Run 'docker-compose -f distribution/docker/docker-compose.yml up`

distribution/docker/docker-compose.yml

@@ -0,0 +1,124 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+version: "2.2"
+
+volumes:
+  metadata_data: {}
+  middle_var: {}
+  historical_var: {}
+  broker_var: {}
+  coordinator_var: {}
+  overlord_var: {}
+
+services:
+  postgres:
+    container_name: postgres
+    image: postgres:latest
+    volumes:
+      - metadata_data:/var/lib/postgresql/data
+    environment:
+      - POSTGRES_PASSWORD=FoolishPassword
+      - POSTGRES_USER=druid
+      - POSTGRES_DB=druid
+
+  # Need 3.5 or later for container nodes
+  zookeeper:
+    container_name: zookeeper
+    image: zookeeper:3.5
+    environment:
+      - ZOO_MY_ID=1
+
+  coordinator:
+    image: druid
+    container_name: coordinator
+    volumes:
+      - coordinator_var:/opt/druid/var
+    depends_on: 
+      - zookeeper
+      - postgres
+    ports:
+      - "3001:8081"
+    command:
+      - coordinator
+    env_file:
+      - environment
+
+  broker:
+    image: druid
+    container_name: broker
+    volumes:
+      - broker_var:/opt/druid/var
+    depends_on: 
+      - zookeeper
+      - postgres
+      - coordinator
+    ports:
+      - "3002:8082"
+    command:
+      - broker
+    env_file:
+      - environment
+
+  historical:
+    image: druid
+    container_name: historical
+    volumes:
+      - historical_var:/opt/druid/var
+    depends_on: 
+      - zookeeper
+      - postgres
+      - coordinator
+    ports:
+      - "3003:8083"
+    command:
+      - historical
+    env_file:
+      - environment
+
+  overlord:
+    image: druid
+    container_name: overlord
+    volumes:
+      - overlord_var:/opt/druid/var
+    depends_on: 
+      - zookeeper
+      - postgres
+    ports:
+      - "4000:8090"
+    command:
+      - overlord
+    env_file:
+      - environment
+
+  middlemanager:
+    image: druid
+    container_name: middlemanager
+    volumes:
+      - middle_var:/opt/druid/var
+    depends_on: 
+      - zookeeper
+      - postgres
+      - coordinator
+    ports:
+      - "4001:8091"
+    command:
+      - middleManager
+    env_file:
+      - environment
+

distribution/docker/druid.sh

@@ -0,0 +1,137 @@
+#!/bin/sh
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# NOTE: this is a 'run' script for the stock tarball
+# It takes 1 required argument (the name of the service,
+# e.g. 'broker', 'historical' etc). Any additional arguments
+# are passed to that service.
+#
+# It accepts 'JAVA_OPTS' as an environment variable
+#
+# Additional env vars:
+# - DRUID_LOG4J -- set the entire log4j.xml verbatim
+# - DRUID_LOG_LEVEL -- override the default log level in default log4j
+# - DRUID_XMX -- set Java Xmx
+# - DRUID_XMS -- set Java Xms
+# - DRUID_MAXNEWSIZE -- set Java max new size
+# - DRUID_NEWSIZE -- set Java new size
+# - DRUID_MAXDIRECTMEMORYSIZE -- set Java max direct memory size
+#
+# - DRUID_CONFIG -- full path to a file for druid 'common' properties
+# - DRUID_CONFIG_${service} -- full path to a file for druid 'service' properties
+
+set -e
+SERVICE="$1"
+
+echo "$(date -Is) startup service $SERVICE"
+
+# We put all the config in /tmp/conf to allow for a
+# read-only root filesystem
+cp -r /opt/druid/conf /tmp/conf
+
+# Delete the old key (if existing) and append new key=value
+setKey() {
+    service="$1"
+    key="$2"
+    value="$3"
+    case "$service" in
+        _common)
+          fname=common.runtime.properties ;;
+        *)
+          fname=runtime.properties ;;
+    esac
+    # Delete from all
+    sed -ri "/$key=/d" /tmp/conf/druid/_common/common.runtime.properties
+    [ -f /tmp/conf/druid/$service/$fname ] && sed -ri "/$key=/d" /tmp/conf/druid/$service/$fname
+    [ -f /tmp/conf/druid/$service/$fname ] && echo "$key=$value" >> /tmp/conf/druid/$service/$fname
+    [ -f /tmp/conf/druid/$service/$fname ] || echo "$key=$value" >> /tmp/conf/druid/_common/$fname
+}
+
+setJavaKey() {
+    service="$1"
+    key=$2
+    value=$3
+    file=/tmp/conf/druid/$service/jvm.config
+    sed -ri "/$key/d" $file
+    echo $value >> $file
+}
+
+## Setup host names
+if [ -n "${ZOOKEEPER}" ]
+then
+    setKey _common druid.zk.service.host "${ZOOKEEPER}"
+fi
+
+setKey $SERVICE druid.host $(ip r get 1 | awk '{print $7;exit}')
+
+
+env |grep ^druid_ | while read evar
+do
+    # Can't use IFS='=' to parse since var might have = in it (e.g. password)
+    val=$(echo "$evar" | sed -e 's?[^=]*=??')
+    var=$(echo "$evar" | sed -e 's?^\([^=]*\)=.*?\1?g' -e 's?_?.?g')
+    setKey $SERVICE "$var" "$val"
+done
+
+env |grep ^s3service | while read evar
+do
+    val=$(echo "$evar" | sed -e 's?[^=]*=??')
+    var=$(echo "$evar" | sed -e 's?^\([^=]*\)=.*?\1?g' -e 's?_?.?' -e 's?_?-?g')
+    echo "$var=$val" >> /tmp/conf/druid/_common/jets3t.properties
+done
+
+# This is to allow configuration via a Kubernetes configMap without
+# e.g. using subPath (you can also mount the configMap on /tmp/conf/druid)
+if [ -n "$DRUID_CONFIG_COMMON" ]
+then
+    cp -f "$DRUID_CONFIG_COMMON" /tmp/conf/druid/_common/common.runtime.properties
+fi
+
+SCONFIG=$(printf "%s_%s" DRUID_CONFIG ${SERVICE})
+SCONFIG=$(eval echo \$$(echo $SCONFIG))
+
+if [ -n "${SCONFIG}" ]
+then
+    cp -f "${SCONFIG}" /tmp/conf/druid/${SERVICE}/runtime.properties
+fi
+
+# Now do the java options
+
+if [ -n "$DRUID_XMX" ]; then setJavaKey ${SERVICE} -Xmx -Xmx${DRUID_XMX}; fi
+if [ -n "$DRUID_XMS" ]; then setJavaKey ${SERVICE} -Xms -Xms${DRUID_XMS}; fi
+if [ -n "$DRUID_MAXNEWSIZE" ]; then setJavaKey ${SERVICE} -XX:MaxNewSize -XX:MaxNewSize=${DRUID_MAXNEWSIZE}; fi
+if [ -n "$DRUID_NEWSIZE" ]; then setJavaKey ${SERVICE} -XX:NewSize -XX:MaxNewSize=${DRUID_NEWSIZE}; fi
+if [ -n "$DRUID_MAXDIRECTMEMORYSIZE" ]; then setJavaKey ${SERVICE} -XX:MaxDirectMemorySize -XX:MaxDirectMemorySize=${DRUID_MAXDIRECTMEMORYSIZE}; fi
+
+JAVA_OPTS="$JAVA_OPTS $(cat /tmp/conf/druid/${SERVICE}/jvm.config | xargs)"
+
+if [ -n "$DRUID_LOG_LEVEL" ]
+then
+    sed -ri 's/"info"/"'$DRUID_LOG_LEVEL'"/g' /tmp/conf/druid/_common/log4j2.xml
+fi
+
+if [ -n "$DRUID_LOG4J" ]
+then
+    echo "$DRUID_LOG4J" > /tmp/conf/druid/_common/log4j2.xml
+fi
+
+mkdir -p var/tmp var/druid/segments var/druid/indexing-logs var/druid/task var/druid/hadoop-tmp var/druid/segment-cache
+exec java ${JAVA_OPTS} -cp /tmp/conf/druid/_common:/tmp/conf/druid/${SERVICE}:lib/*: org.apache.druid.cli.Main server $@

distribution/docker/environment

@@ -0,0 +1,51 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Java tuning
+DRUID_XMX=1g
+DRUID_XMS=1g
+DRUID_MAXNEWSIZE=250m
+DRUID_NEWSIZE=250m
+DRUID_MAXDIRECTMEMORYSIZE=6172m
+
+druid_emitter_logging_logLevel=debug
+
+druid_extensions_loadList=["druid-histogram", "druid-datasketches", "druid-lookups-cached-global", "druid-azure-extensions", "postgresql-metadata-storage"]
+
+druid_zk_service_host=zookeeper
+
+druid_metadata_storage_host=
+druid_metadata_storage_type=postgresql
+druid_metadata_storage_connector_connectURI=jdbc:postgresql://postgres:5432/druid
+druid_metadata_storage_connector_user=druid
+druid_metadata_storage_connector_password=FoolishPassword
+
+druid_coordinator_balancer_strategy=cachingCost
+
+druid_indexer_runner_javaOptsArray=["-server", "-Xmx1g", "-Xms1g", "-XX:MaxDirectMemorySize=3g", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"]
+druid_indexer_fork_property_druid_processing_buffer_sizeBytes=268435456
+
+druid_storage_type=azure
+druid_azure_account=YOURACCOUNT
+druid_azure_key=YOURKEY
+druid_azure_container=druid
+druid_azure_protocol=https
+druid_azure_maxTries=3
+
+DRUID_LOG4J=<?xml version="1.0" encoding="UTF-8" ?><Configuration status="WARN"><Appenders><Console name="Console" target="SYSTEM_OUT"><PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/></Console></Appenders><Loggers><Root level="info"><AppenderRef ref="Console"/></Root><Logger name="org.apache.druid.jetty.RequestLog" additivity="false" level="DEBUG"><AppenderRef ref="Console"/></Logger></Loggers></Configuration>

distribution/docker/sha256sums.txt

@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+b95bf9fe25cb5428f378a62fc842e177ca004b4ae1f9054968b2a396dcc1ec22  /opt/druid/extensions/mysql-metadata-storage/mysql-connector-java-5.1.38.jar

distribution/src/assembly/source-assembly.xml

@@ -45,6 +45,7 @@
                 <exclude>%regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?release\.properties]</exclude>
 
                 <exclude>.gitignore</exclude>
+                <exclude>.dockerignore</exclude>
                 <exclude>.travis.yml</exclude>
                 <exclude>publications/**</exclude>
                 <exclude>upload.sh</exclude>