diff --git a/examples/quickstart/tutorial/hadoop/docker/Dockerfile b/examples/quickstart/tutorial/hadoop/docker/Dockerfile index c9c30ac2043..0237d61cbb9 100644 --- a/examples/quickstart/tutorial/hadoop/docker/Dockerfile +++ b/examples/quickstart/tutorial/hadoop/docker/Dockerfile @@ -45,7 +45,8 @@ ENV JAVA_HOME /usr/lib/jvm/zulu-8 ENV PATH $PATH:$JAVA_HOME/bin # hadoop -RUN curl -s https://archive.apache.org/dist/hadoop/core/hadoop-2.8.5/hadoop-2.8.5.tar.gz | tar -xz -C /usr/local/ +ARG APACHE_ARCHIVE_MIRROR_HOST=https://archive.apache.org +RUN curl -s ${APACHE_ARCHIVE_MIRROR_HOST}/dist/hadoop/core/hadoop-2.8.5/hadoop-2.8.5.tar.gz | tar -xz -C /usr/local/ RUN cd /usr/local && ln -s ./hadoop-2.8.5 hadoop ENV HADOOP_PREFIX /usr/local/hadoop diff --git a/integration-tests/README.md b/integration-tests/README.md index 3bba04ca476..1d00e56612e 100644 --- a/integration-tests/README.md +++ b/integration-tests/README.md @@ -46,6 +46,12 @@ environment variable to localhost on your system, as follows: export DOCKER_IP=127.0.0.1 ``` +Optionally, you can also set `APACHE_ARCHIVE_MIRROR_HOST` to override `https://archive.apache.org` host. This host is used to download archives such as hadoop and kafka during building docker images: + +``` +export APACHE_ARCHIVE_MIRROR_HOST=https://example.com/remote-generic-repo +``` + ## Running tests To run all tests from a test group using docker and mvn run the following command: diff --git a/integration-tests/docker/Dockerfile b/integration-tests/docker/Dockerfile index 10e82b98644..421d3474055 100644 --- a/integration-tests/docker/Dockerfile +++ b/integration-tests/docker/Dockerfile @@ -20,7 +20,8 @@ FROM openjdk:$JDK_VERSION as druidbase # Otherwise docker's layered images mean that things are not actually deleted. COPY base-setup.sh /root/base-setup.sh -RUN /root/base-setup.sh && rm -f /root/base-setup.sh +ARG APACHE_ARCHIVE_MIRROR_HOST=https://archive.apache.org +RUN APACHE_ARCHIVE_MIRROR_HOST=${APACHE_ARCHIVE_MIRROR_HOST} /root/base-setup.sh && rm -f /root/base-setup.sh FROM druidbase ARG MYSQL_VERSION diff --git a/integration-tests/docker/base-setup.sh b/integration-tests/docker/base-setup.sh index 186874a16d3..e68f6bcccdc 100755 --- a/integration-tests/docker/base-setup.sh +++ b/integration-tests/docker/base-setup.sh @@ -18,6 +18,7 @@ set -e set -u export DEBIAN_FRONTEND=noninteractive +APACHE_ARCHIVE_MIRROR_HOST=${APACHE_ARCHIVE_MIRROR_HOST:-https://archive.apache.org} apt-get update @@ -33,7 +34,7 @@ apt-get install -y supervisor # Zookeeper install_zk() { - wget -q -O /tmp/$ZK_TAR.tar.gz "https://archive.apache.org/dist/zookeeper/zookeeper-$ZK_VERSION/$ZK_TAR.tar.gz" + wget -q -O /tmp/$ZK_TAR.tar.gz "$APACHE_ARCHIVE_MIRROR_HOST/dist/zookeeper/zookeeper-$ZK_VERSION/$ZK_TAR.tar.gz" tar -xzf /tmp/$ZK_TAR.tar.gz -C /usr/local cp /usr/local/$ZK_TAR/conf/zoo_sample.cfg /usr/local/$ZK_TAR/conf/zoo.cfg rm /tmp/$ZK_TAR.tar.gz @@ -52,7 +53,7 @@ ln -s /usr/local/$ZK_TAR /usr/local/zookeeper-3.5 # Kafka # Match the version to the Kafka client used by KafkaSupervisor KAFKA_VERSION=2.7.0 -wget -q -O /tmp/kafka_2.13-$KAFKA_VERSION.tgz "https://apache.org/dist/kafka/$KAFKA_VERSION/kafka_2.13-$KAFKA_VERSION.tgz" +wget -q -O /tmp/kafka_2.13-$KAFKA_VERSION.tgz "$APACHE_ARCHIVE_MIRROR_HOST/dist/kafka/$KAFKA_VERSION/kafka_2.13-$KAFKA_VERSION.tgz" tar -xzf /tmp/kafka_2.13-$KAFKA_VERSION.tgz -C /usr/local ln -s /usr/local/kafka_2.13-$KAFKA_VERSION /usr/local/kafka rm /tmp/kafka_2.13-$KAFKA_VERSION.tgz diff --git a/integration-tests/script/docker_build_containers.sh b/integration-tests/script/docker_build_containers.sh index 6bff5020acc..ef3df477dec 100755 --- a/integration-tests/script/docker_build_containers.sh +++ b/integration-tests/script/docker_build_containers.sh @@ -28,11 +28,11 @@ else case "${DRUID_INTEGRATION_TEST_JVM_RUNTIME}" in 8) echo "Build druid-cluster with Java 8" - docker build -t druid/cluster --build-arg JDK_VERSION=8-slim --build-arg MYSQL_VERSION $SHARED_DIR/docker + docker build -t druid/cluster --build-arg JDK_VERSION=8-slim --build-arg MYSQL_VERSION --build-arg APACHE_ARCHIVE_MIRROR_HOST $SHARED_DIR/docker ;; 11) echo "Build druid-cluster with Java 11" - docker build -t druid/cluster --build-arg JDK_VERSION=11-slim --build-arg MYSQL_VERSION $SHARED_DIR/docker + docker build -t druid/cluster --build-arg JDK_VERSION=11-slim --build-arg MYSQL_VERSION --build-arg APACHE_ARCHIVE_MIRROR_HOST $SHARED_DIR/docker ;; *) echo "Invalid JVM Runtime given. Stopping" @@ -44,5 +44,5 @@ fi # Build Hadoop docker if needed if [ -n "$DRUID_INTEGRATION_TEST_BUILD_HADOOP_DOCKER" ] && [ "$DRUID_INTEGRATION_TEST_BUILD_HADOOP_DOCKER" == true ] then - docker build -t druid-it/hadoop:2.8.5 $HADOOP_DOCKER_DIR + docker build -t druid-it/hadoop:2.8.5 --build-arg APACHE_ARCHIVE_MIRROR_HOST $HADOOP_DOCKER_DIR fi