2019-02-08 07:12:28 -05:00
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
2023-11-01 00:25:54 -04:00
ARG JDK_VERSION = 17
2022-09-15 21:25:52 -04:00
# The platform is explicitly specified as x64 to build the Druid distribution.
# This is because it's not able to build the distribution on arm64 due to dependency problem of web-console. See: https://github.com/apache/druid/issues/13012
# Since only java jars are shipped in the final image, it's OK to build the distribution on x64.
# Once the web-console dependency problem is resolved, we can remove the --platform directive.
FROM --platform=linux/amd64 maven:3.8.6-jdk-11-slim as builder
2021-05-07 13:41:34 -04:00
# Rebuild from source in this stage
# This can be unset if the tarball was already built outside of Docker
ARG BUILD_FROM_SOURCE = "true"
2019-02-08 07:12:28 -05:00
2019-08-13 11:33:06 -04:00
RUN export DEBIAN_FRONTEND = noninteractive \
&& apt-get -qq update \
&& apt-get -qq -y install --no-install-recommends python3 python3-yaml
2019-08-06 11:32:59 -04:00
2019-02-08 07:12:28 -05:00
COPY . /src
WORKDIR /src
2022-09-15 21:25:52 -04:00
RUN --mount= type = cache,target= /root/.m2 if [ " $BUILD_FROM_SOURCE " = "true" ] ; then \
2024-02-21 10:53:35 -05:00
mvn -B -ff -q \
2019-08-13 11:33:06 -04:00
install \
-Pdist,bundle-contrib-exts \
Web console basic end-to-end-test (#9595)
Load data and query (i.e., automate
https://druid.apache.org/docs/latest/tutorials/tutorial-batch.html) to
have some basic checks ensuring the web console is wired up to druid
correctly.
The new end-to-end tests (tutorial-batch.spec.ts) are added to
`web-console/e2e-tests`. Within that directory:
- `components` represent the various tabs of the web console. Currently,
abstractions for `load data`, `ingestion`, `datasources`, and `query`
are implemented.
- `components/load-data/data-connector` contains abstractions for the
different data source options available to the data loader's `Connect`
step. Currently, only the `Local file` data source connector is
implemented.
- `components/load-data/config` contains abstractions for the different
configuration options available for each step of the data loader flow.
Currently, the `Configure Schema`, `Partition`, and `Publish` steps
have initial implementation of their configuration options.
- `util` contains various helper methods for the tests and does not
contain abstractions of the web console.
Changes to add the new tests to CI:
- `.travis.yml`: New "web console end-to-end tests" job
- `web-console/jest.*.js`: Refactor jest configurations to have
different flavors for unit tests and for end-to-end tests. In
particular, the latter adds a jest setup configuration to wait for the
web console to be ready (`web-console/e2e-tests/util/setup.ts`).
- `web-console/package.json`: Refactor run scripts to add new script for
running end-to-end tests.
- `web-console/script/druid`: Utility scripts for building, starting,
and stopping druid.
Other changes:
- `pom.xml`: Refactor various settings disable java static checks and to
disable java tests into two new maven profiles. Since the same
settings are used in several places (e.g., .travis.yml, Dockerfiles,
etc.), having them in maven profiles makes it more maintainable.
- `web-console/src/console-application.tsx`: Fix typo ("the the").
2020-04-09 15:38:09 -04:00
-Pskip-static-checks,skip-tests \
2024-02-26 10:48:55 -05:00
-Dmaven.javadoc.skip= true -T1C \
2021-05-07 13:41:34 -04:00
; fi
2019-02-08 07:12:28 -05:00
2022-09-15 21:25:52 -04:00
RUN --mount= type = cache,target= /root/.m2 VERSION = $( mvn -B -q org.apache.maven.plugins:maven-help-plugin:3.2.0:evaluate \
2019-08-13 11:33:06 -04:00
-Dexpression= project.version -DforceStdout= true \
) \
&& tar -zxf ./distribution/target/apache-druid-${ VERSION } -bin.tar.gz -C /opt \
2021-06-01 20:33:27 -04:00
&& mv /opt/apache-druid-${ VERSION } /opt/druid
2019-02-08 07:12:28 -05:00
2023-11-01 00:25:54 -04:00
FROM alpine:3 as bash-static
2022-09-15 21:25:52 -04:00
ARG TARGETARCH
#
# Download bash-static binary to execute scripts that require bash.
# Although bash-static supports multiple platforms, but there's no need for us to support all those platform, amd64 and arm64 are enough.
#
ARG BASH_URL_BASE = "https://github.com/robxu9/bash-static/releases/download/5.1.016-1.2.3"
RUN if [ " $TARGETARCH " = "arm64" ] ; then \
BASH_URL = " ${ BASH_URL_BASE } /bash-linux-aarch64 " ; \
elif [ " $TARGETARCH " = "amd64" ] ; then \
BASH_URL = " ${ BASH_URL_BASE } /bash-linux-x86_64 " ; \
else \
echo " Unsupported architecture ( $TARGETARCH ) " && exit 1; \
fi ; \
echo " Downloading bash-static from ${ BASH_URL } " \
2023-11-01 00:25:54 -04:00
&& wget ${ BASH_URL } -O /bin/bash
FROM busybox:1.35.0-glibc as busybox
FROM gcr.io/distroless/java$JDK_VERSION-debian12
LABEL maintainer = "Apache Druid Developers <dev@druid.apache.org>"
COPY --from= busybox /bin/busybox /busybox/busybox
RUN [ "/busybox/busybox" , "--install" , "/bin" ]
2022-09-15 21:25:52 -04:00
2019-08-13 11:33:06 -04:00
RUN addgroup -S -g 1000 druid \
2021-04-30 11:51:53 -04:00
&& adduser -S -u 1000 -D -H -h /opt/druid -s /bin/sh -g '' -G druid druid
2019-08-13 11:33:06 -04:00
2023-11-01 00:25:54 -04:00
COPY --from= bash-static /bin/bash /bin/bash
RUN chmod 755 /bin/bash
2020-10-19 13:29:20 -04:00
COPY distribution/docker/druid.sh /druid.sh
Support for middle manager less druid, tasks launch as k8s jobs (#13156)
* Support for middle manager less druid, tasks launch as k8s jobs
* Fixing forking task runner test
* Test cleanup, dependency cleanup, intellij inspections cleanup
* Changes per PR review
Add configuration option to disable http/https proxy for the k8s client
Update the docs to provide more detail about sidecar support
* Removing un-needed log lines
* Small changes per PR review
* Upon task completion we callback to the overlord to update the status / locaiton, for slower k8s clusters, this reduces locking time significantly
* Merge conflict fix
* Fixing tests and docs
* update tiny-cluster.yaml
changed `enableTaskLevelLogPush` to `encapsulatedTask`
* Apply suggestions from code review
Co-authored-by: Abhishek Agarwal <1477457+abhishekagarwal87@users.noreply.github.com>
* Minor changes per PR request
* Cleanup, adding test to AbstractTask
* Add comment in peon.sh
* Bumping code coverage
* More tests to make code coverage happy
* Doh a duplicate dependnecy
* Integration test setup is weird for k8s, will do this in a different PR
* Reverting back all integration test changes, will do in anotbher PR
* use StringUtils.base64 instead of Base64
* Jdk is nasty, if i compress in jdk 11 in jdk 17 the decompressed result is different
Co-authored-by: Rahul Gidwani <r_gidwani@apple.com>
Co-authored-by: Abhishek Agarwal <1477457+abhishekagarwal87@users.noreply.github.com>
2022-11-02 22:44:47 -04:00
COPY distribution/docker/peon.sh /peon.sh
2024-02-26 10:48:55 -05:00
COPY distribution/docker/deduplicate_jars.sh /deduplicate_jars.sh
2021-06-01 20:33:27 -04:00
# create necessary directories which could be mounted as volume
2024-02-26 10:48:55 -05:00
# copy and de-duplicate jars from builder in same layer to reduce image size
2021-06-01 20:33:27 -04:00
# /opt/druid/var is used to keep individual files(e.g. log) of each Druid service
# /opt/shared is used to keep segments and task logs shared among Druid services
2024-02-26 10:48:55 -05:00
RUN --mount= type = bind,from= builder,source= /opt,target= /builder/opt \
mkdir -p /opt/druid/var /opt/shared \
&& cp -r /builder/opt/druid /opt/ \
&& /deduplicate_jars.sh /opt/druid \
2024-04-26 11:46:00 -04:00
&& chown -R druid:druid /opt/druid /opt/shared \
2021-06-01 20:33:27 -04:00
&& chmod 775 /opt/druid/var /opt/shared
2020-10-19 13:29:20 -04:00
2019-02-08 07:12:28 -05:00
USER druid
VOLUME /opt/druid/var
WORKDIR /opt/druid
ENTRYPOINT [ "/druid.sh" ]