Consolidate the number of Dockerfiles (#11187)

* Consolidate the number of Dockerfiles

* add build-arguments to choose which Java base image to use at runtime
* default to building image with Java 11
* base k8s integration test image off of the default image: this ensures
  our docker image now gets tested as part of integration tests.

* upgrade maven help plugin to 3.2.0

distribution/docker/Dockerfile

@@ -17,7 +17,11 @@
 # under the License.
 #
 
-FROM maven:3-jdk-8-slim as builder
+ARG JDK_VERSION=8
+FROM maven:3-jdk-11-slim as builder
+# Rebuild from source in this stage
+# This can be unset if the tarball was already built outside of Docker
+ARG BUILD_FROM_SOURCE="true"
 
 RUN export DEBIAN_FRONTEND=noninteractive \
     && apt-get -qq update \
@@ -25,13 +29,15 @@ RUN export DEBIAN_FRONTEND=noninteractive \
 
 COPY . /src
 WORKDIR /src
-RUN mvn -B -ff -q dependency:go-offline \
+RUN if [ "$BUILD_FROM_SOURCE" = "true" ]; then \
+      mvn -B -ff -q dependency:go-offline \
       install \
       -Pdist,bundle-contrib-exts \
       -Pskip-static-checks,skip-tests \
-      -Dmaven.javadoc.skip=true
+      -Dmaven.javadoc.skip=true \
+      ; fi
 
-RUN VERSION=$(mvn -B -q org.apache.maven.plugins:maven-help-plugin:3.1.1:evaluate \
+RUN VERSION=$(mvn -B -q org.apache.maven.plugins:maven-help-plugin:3.2.0:evaluate \
       -Dexpression=project.version -DforceStdout=true \
     ) \
  && tar -zxf ./distribution/target/apache-druid-${VERSION}-bin.tar.gz -C /opt \
@@ -39,7 +45,7 @@ RUN VERSION=$(mvn -B -q org.apache.maven.plugins:maven-help-plugin:3.1.1:evaluat
 
 FROM amd64/busybox:1.30.0-glibc as busybox
 
-FROM gcr.io/distroless/java:8
+FROM gcr.io/distroless/java:$JDK_VERSION
 LABEL maintainer="Apache Druid Developers <dev@druid.apache.org>"
 
 COPY --from=busybox /bin/busybox /busybox/busybox

distribution/docker/Dockerfile.java11

@@ -1,61 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-FROM maven:3-jdk-11-slim as builder
-
-RUN export DEBIAN_FRONTEND=noninteractive \
-    && apt-get -qq update \
-    && apt-get -qq -y install --no-install-recommends python3 python3-yaml
-
-COPY . /src
-WORKDIR /src
-RUN mvn -B -ff -q dependency:go-offline \
-      install \
-      -Pdist,bundle-contrib-exts \
-      -Pskip-static-checks,skip-tests \
-      -Dmaven.javadoc.skip=true
-
-RUN VERSION=$(mvn -B -q org.apache.maven.plugins:maven-help-plugin:3.1.1:evaluate \
-      -Dexpression=project.version -DforceStdout=true \
-    ) \
- && tar -zxf ./distribution/target/apache-druid-${VERSION}-bin.tar.gz -C /opt \
- && ln -s /opt/apache-druid-${VERSION} /opt/druid
-
-FROM amd64/busybox:1.30.0-glibc as busybox
-
-FROM gcr.io/distroless/java:11
-LABEL maintainer="Apache Druid Developers <dev@druid.apache.org>"
-
-COPY --from=busybox /bin/busybox /busybox/busybox
-RUN ["/busybox/busybox", "--install", "/bin"]
-
-RUN addgroup -S -g 1000 druid \
- && adduser -S -u 1000 -D -H -h /opt/druid -s /bin/sh -g '' -G druid druid
-
-COPY --chown=druid:druid --from=builder /opt /opt
-COPY distribution/docker/druid.sh /druid.sh
-RUN mkdir /opt/druid/var \
- && chown druid:druid /opt/druid/var \
- && chmod 775 /opt/druid/var
-
-USER druid
-VOLUME /opt/druid/var
-WORKDIR /opt/druid
-
-ENTRYPOINT ["/druid.sh"]

distribution/docker/DockerfileBuildTarAdvanced

@@ -18,49 +18,12 @@
 #
 
 # Using For IT on K8s
-FROM maven:3-jdk-8-slim as builder
+ARG BASE_IMAGE=apache/druid
-
+FROM $BASE_IMAGE
-RUN export DEBIAN_FRONTEND=noninteractive \
-    && apt-get -qq update \
-    && apt-get -qq -y install --no-install-recommends python3 python3-yaml
-
-COPY . /src
-WORKDIR /src
-
-RUN VERSION=$(mvn -B -q org.apache.maven.plugins:maven-help-plugin:3.1.1:evaluate \
-      -Dexpression=project.version -DforceStdout=true \
-    ) \
- && tar -zxf ./distribution/target/apache-druid-${VERSION}-bin.tar.gz -C /opt \
- && ln -s /opt/apache-druid-${VERSION} /opt/druid
-
-FROM amd64/busybox:1.30.0-glibc as busybox
-
-FROM gcr.io/distroless/java:8
-LABEL maintainer="Apache Druid Developers <dev@druid.apache.org>"
-
-COPY --from=busybox /bin/busybox /busybox/busybox
-RUN ["/busybox/busybox", "--install", "/bin"]
 
+USER 0
 RUN mkdir -p /shared/wikiticker-it
-
 COPY examples/quickstart/tutorial/wikiticker-2015-09-12-sampled.json.gz /shared/wikiticker-it/wikiticker-2015-09-12-sampled.json.gz
 COPY integration-tests/docker/wiki-simple-lookup.json /shared/wikiticker-it/wiki-simple-lookup.json
 
-RUN addgroup -S -g 1000 druid \
- && adduser -S -u 1000 -D -H -h /opt/druid -s /bin/sh -g '' -G druid druid \
- && mkdir -p /opt/druid/var \
- && chown -R druid:druid /opt \
- && chmod 775 /opt/druid/var
-
-COPY --chown=druid:druid --from=builder /opt /opt
-COPY distribution/docker/druid.sh /druid.sh
-
-RUN mkdir -p /opt/druid/var \
- && chown -R druid:druid /opt \
- && chmod 775 /opt/druid/var
-
 USER druid
-VOLUME /opt/druid/var
-WORKDIR /opt/druid
-
-ENTRYPOINT ["/druid.sh"]

integration-tests/script/setup_druid_on_k8s.sh

@@ -32,7 +32,8 @@ mvn -B -ff -q dependency:go-offline \
       -Pskip-static-checks,skip-tests \
       -Dmaven.javadoc.skip=true
 
-docker build -t druid/cluster:v1 -f distribution/docker/DockerfileBuildTarAdvanced .
+docker build --build-arg BUILD_FROM_SOURCE=0 -t druid/base:v1 -f distribution/docker/Dockerfile .
+docker build --build-arg BASE_IMAGE=druid/base:v1 -t druid/cluster:v1 -f distribution/docker/DockerfileBuildTarAdvanced .
 
 # This tmp dir is used for MiddleManager pod and Historical Pod to cache segments.
 sudo rm -rf tmp
@@ -50,4 +51,3 @@ sleep 60
 
 $KUBECTL get pod
 $KUBECTL get svc
-