diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000000..d7ae5a9cf59 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,37 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +.git +**/*.jar +**/*.class +dist +target +*.iml +*.ipr +*.iws +*.tar.gz +*.swp +*.swo +.classpath +.idea +.project +.settings/ +*.log +*.DS_Store +_site +dependency-reduced-pom.xml diff --git a/distribution/docker/Dockerfile b/distribution/docker/Dockerfile new file mode 100644 index 00000000000..13c53e76077 --- /dev/null +++ b/distribution/docker/Dockerfile @@ -0,0 +1,56 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +FROM maven:3-jdk-8 as builder + +COPY . /src +WORKDIR /src +RUN mvn install -ff -DskipTests -Dforbiddenapis.skip=true -Pdist -Pbundle-contrib-exts + +RUN \ + VER=$(mvn -B org.apache.maven.plugins:maven-help-plugin:3.1.1:evaluate -Dexpression=project.version -q -DforceStdout=true -f pom.xml 2>/dev/null) \ + && tar -zxf ./distribution/target/apache-druid-${VER}-bin.tar.gz -C /opt \ + && ln -s /opt/apache-druid-${VER} /opt/druid + +RUN wget -O /opt/druid/extensions/mysql-metadata-storage/mysql-connector-java-5.1.38.jar http://central.maven.org/maven2/mysql/mysql-connector-java/5.1.38/mysql-connector-java-5.1.38.jar \ + && sha256sum --ignore-missing -c /src/distribution/docker/sha256sums.txt \ + && ln -s /opt/druid/extensions/mysql-metadata-storage/mysql-connector-java-5.1.38.jar /opt/druid/lib + +RUN addgroup --gid 1000 druid \ + && adduser --home /opt/druid --shell /bin/sh --no-create-home --uid 1000 --gecos '' --gid 1000 --disabled-password druid \ + && mkdir -p /opt/druid/var \ + && chown -R druid:druid /opt/druid \ + && chmod 775 /opt/druid/var + +FROM amd64/busybox:1.30.0-glibc as busybox +FROM gcr.io/distroless/java +LABEL maintainer="Don Bowman " + +COPY --from=busybox /bin/busybox /busybox/busybox +RUN ["/busybox/busybox", "--install", "/bin"] +COPY --from=builder /etc/passwd /etc/passwd +COPY --from=builder /etc/group /etc/group +COPY --from=builder --chown=druid /opt /opt +COPY distribution/docker/druid.sh /druid.sh +RUN chown -R druid:druid /opt/druid +USER druid +VOLUME /opt/druid/var +WORKDIR /opt/druid + +ENTRYPOINT ["/druid.sh"] diff --git a/distribution/docker/README.md b/distribution/docker/README.md new file mode 100644 index 00000000000..fdc580c79e7 --- /dev/null +++ b/distribution/docker/README.md @@ -0,0 +1,26 @@ + + +## Build + +From the root of the repo, run `docker build -t druid:tag -f distribution/docker/Dockerfile .` + +## Run + +Edit `environment` to suite. Run 'docker-compose -f distribution/docker/docker-compose.yml up` diff --git a/distribution/docker/docker-compose.yml b/distribution/docker/docker-compose.yml new file mode 100644 index 00000000000..0b962414345 --- /dev/null +++ b/distribution/docker/docker-compose.yml @@ -0,0 +1,124 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +version: "2.2" + +volumes: + metadata_data: {} + middle_var: {} + historical_var: {} + broker_var: {} + coordinator_var: {} + overlord_var: {} + +services: + postgres: + container_name: postgres + image: postgres:latest + volumes: + - metadata_data:/var/lib/postgresql/data + environment: + - POSTGRES_PASSWORD=FoolishPassword + - POSTGRES_USER=druid + - POSTGRES_DB=druid + + # Need 3.5 or later for container nodes + zookeeper: + container_name: zookeeper + image: zookeeper:3.5 + environment: + - ZOO_MY_ID=1 + + coordinator: + image: druid + container_name: coordinator + volumes: + - coordinator_var:/opt/druid/var + depends_on: + - zookeeper + - postgres + ports: + - "3001:8081" + command: + - coordinator + env_file: + - environment + + broker: + image: druid + container_name: broker + volumes: + - broker_var:/opt/druid/var + depends_on: + - zookeeper + - postgres + - coordinator + ports: + - "3002:8082" + command: + - broker + env_file: + - environment + + historical: + image: druid + container_name: historical + volumes: + - historical_var:/opt/druid/var + depends_on: + - zookeeper + - postgres + - coordinator + ports: + - "3003:8083" + command: + - historical + env_file: + - environment + + overlord: + image: druid + container_name: overlord + volumes: + - overlord_var:/opt/druid/var + depends_on: + - zookeeper + - postgres + ports: + - "4000:8090" + command: + - overlord + env_file: + - environment + + middlemanager: + image: druid + container_name: middlemanager + volumes: + - middle_var:/opt/druid/var + depends_on: + - zookeeper + - postgres + - coordinator + ports: + - "4001:8091" + command: + - middleManager + env_file: + - environment + diff --git a/distribution/docker/druid.sh b/distribution/docker/druid.sh new file mode 100755 index 00000000000..b142ec6962d --- /dev/null +++ b/distribution/docker/druid.sh @@ -0,0 +1,137 @@ +#!/bin/sh + +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# NOTE: this is a 'run' script for the stock tarball +# It takes 1 required argument (the name of the service, +# e.g. 'broker', 'historical' etc). Any additional arguments +# are passed to that service. +# +# It accepts 'JAVA_OPTS' as an environment variable +# +# Additional env vars: +# - DRUID_LOG4J -- set the entire log4j.xml verbatim +# - DRUID_LOG_LEVEL -- override the default log level in default log4j +# - DRUID_XMX -- set Java Xmx +# - DRUID_XMS -- set Java Xms +# - DRUID_MAXNEWSIZE -- set Java max new size +# - DRUID_NEWSIZE -- set Java new size +# - DRUID_MAXDIRECTMEMORYSIZE -- set Java max direct memory size +# +# - DRUID_CONFIG -- full path to a file for druid 'common' properties +# - DRUID_CONFIG_${service} -- full path to a file for druid 'service' properties + +set -e +SERVICE="$1" + +echo "$(date -Is) startup service $SERVICE" + +# We put all the config in /tmp/conf to allow for a +# read-only root filesystem +cp -r /opt/druid/conf /tmp/conf + +# Delete the old key (if existing) and append new key=value +setKey() { + service="$1" + key="$2" + value="$3" + case "$service" in + _common) + fname=common.runtime.properties ;; + *) + fname=runtime.properties ;; + esac + # Delete from all + sed -ri "/$key=/d" /tmp/conf/druid/_common/common.runtime.properties + [ -f /tmp/conf/druid/$service/$fname ] && sed -ri "/$key=/d" /tmp/conf/druid/$service/$fname + [ -f /tmp/conf/druid/$service/$fname ] && echo "$key=$value" >> /tmp/conf/druid/$service/$fname + [ -f /tmp/conf/druid/$service/$fname ] || echo "$key=$value" >> /tmp/conf/druid/_common/$fname +} + +setJavaKey() { + service="$1" + key=$2 + value=$3 + file=/tmp/conf/druid/$service/jvm.config + sed -ri "/$key/d" $file + echo $value >> $file +} + +## Setup host names +if [ -n "${ZOOKEEPER}" ] +then + setKey _common druid.zk.service.host "${ZOOKEEPER}" +fi + +setKey $SERVICE druid.host $(ip r get 1 | awk '{print $7;exit}') + + +env |grep ^druid_ | while read evar +do + # Can't use IFS='=' to parse since var might have = in it (e.g. password) + val=$(echo "$evar" | sed -e 's?[^=]*=??') + var=$(echo "$evar" | sed -e 's?^\([^=]*\)=.*?\1?g' -e 's?_?.?g') + setKey $SERVICE "$var" "$val" +done + +env |grep ^s3service | while read evar +do + val=$(echo "$evar" | sed -e 's?[^=]*=??') + var=$(echo "$evar" | sed -e 's?^\([^=]*\)=.*?\1?g' -e 's?_?.?' -e 's?_?-?g') + echo "$var=$val" >> /tmp/conf/druid/_common/jets3t.properties +done + +# This is to allow configuration via a Kubernetes configMap without +# e.g. using subPath (you can also mount the configMap on /tmp/conf/druid) +if [ -n "$DRUID_CONFIG_COMMON" ] +then + cp -f "$DRUID_CONFIG_COMMON" /tmp/conf/druid/_common/common.runtime.properties +fi + +SCONFIG=$(printf "%s_%s" DRUID_CONFIG ${SERVICE}) +SCONFIG=$(eval echo \$$(echo $SCONFIG)) + +if [ -n "${SCONFIG}" ] +then + cp -f "${SCONFIG}" /tmp/conf/druid/${SERVICE}/runtime.properties +fi + +# Now do the java options + +if [ -n "$DRUID_XMX" ]; then setJavaKey ${SERVICE} -Xmx -Xmx${DRUID_XMX}; fi +if [ -n "$DRUID_XMS" ]; then setJavaKey ${SERVICE} -Xms -Xms${DRUID_XMS}; fi +if [ -n "$DRUID_MAXNEWSIZE" ]; then setJavaKey ${SERVICE} -XX:MaxNewSize -XX:MaxNewSize=${DRUID_MAXNEWSIZE}; fi +if [ -n "$DRUID_NEWSIZE" ]; then setJavaKey ${SERVICE} -XX:NewSize -XX:MaxNewSize=${DRUID_NEWSIZE}; fi +if [ -n "$DRUID_MAXDIRECTMEMORYSIZE" ]; then setJavaKey ${SERVICE} -XX:MaxDirectMemorySize -XX:MaxDirectMemorySize=${DRUID_MAXDIRECTMEMORYSIZE}; fi + +JAVA_OPTS="$JAVA_OPTS $(cat /tmp/conf/druid/${SERVICE}/jvm.config | xargs)" + +if [ -n "$DRUID_LOG_LEVEL" ] +then + sed -ri 's/"info"/"'$DRUID_LOG_LEVEL'"/g' /tmp/conf/druid/_common/log4j2.xml +fi + +if [ -n "$DRUID_LOG4J" ] +then + echo "$DRUID_LOG4J" > /tmp/conf/druid/_common/log4j2.xml +fi + +mkdir -p var/tmp var/druid/segments var/druid/indexing-logs var/druid/task var/druid/hadoop-tmp var/druid/segment-cache +exec java ${JAVA_OPTS} -cp /tmp/conf/druid/_common:/tmp/conf/druid/${SERVICE}:lib/*: org.apache.druid.cli.Main server $@ diff --git a/distribution/docker/environment b/distribution/docker/environment new file mode 100644 index 00000000000..e61a9f307dc --- /dev/null +++ b/distribution/docker/environment @@ -0,0 +1,51 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Java tuning +DRUID_XMX=1g +DRUID_XMS=1g +DRUID_MAXNEWSIZE=250m +DRUID_NEWSIZE=250m +DRUID_MAXDIRECTMEMORYSIZE=6172m + +druid_emitter_logging_logLevel=debug + +druid_extensions_loadList=["druid-histogram", "druid-datasketches", "druid-lookups-cached-global", "druid-azure-extensions", "postgresql-metadata-storage"] + +druid_zk_service_host=zookeeper + +druid_metadata_storage_host= +druid_metadata_storage_type=postgresql +druid_metadata_storage_connector_connectURI=jdbc:postgresql://postgres:5432/druid +druid_metadata_storage_connector_user=druid +druid_metadata_storage_connector_password=FoolishPassword + +druid_coordinator_balancer_strategy=cachingCost + +druid_indexer_runner_javaOptsArray=["-server", "-Xmx1g", "-Xms1g", "-XX:MaxDirectMemorySize=3g", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] +druid_indexer_fork_property_druid_processing_buffer_sizeBytes=268435456 + +druid_storage_type=azure +druid_azure_account=YOURACCOUNT +druid_azure_key=YOURKEY +druid_azure_container=druid +druid_azure_protocol=https +druid_azure_maxTries=3 + +DRUID_LOG4J= diff --git a/distribution/docker/sha256sums.txt b/distribution/docker/sha256sums.txt new file mode 100644 index 00000000000..6f858cd512e --- /dev/null +++ b/distribution/docker/sha256sums.txt @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +b95bf9fe25cb5428f378a62fc842e177ca004b4ae1f9054968b2a396dcc1ec22 /opt/druid/extensions/mysql-metadata-storage/mysql-connector-java-5.1.38.jar diff --git a/distribution/src/assembly/source-assembly.xml b/distribution/src/assembly/source-assembly.xml index 59903d4fe98..0d7e0199331 100644 --- a/distribution/src/assembly/source-assembly.xml +++ b/distribution/src/assembly/source-assembly.xml @@ -45,6 +45,7 @@ %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?release\.properties] .gitignore + .dockerignore .travis.yml publications/** upload.sh